Corrects column reference error in mapping service, variable names in… #167
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to Google Cloud Platform | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - dev | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BUCKET_REGION: ${{ vars.GCP_PROJECT_REGION }} | |
| BUCKET_URL: gs://${{ vars.PROJECT_NAME }}-${{ vars.ENV == 'prod' && 'p' || 't' }}-${{ vars.GCP_PULUMI_BUCKET_NAME }} | |
| EXTRACT_DIR: ./services/extract | |
| INFRA_DIR: ./infra/gcp/persistent | |
| PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| TRANSFORM_DIR: ./services/transform | |
| jobs: | |
| lint: | |
| name: Lint and Format | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Scan Extract Directory | |
| uses: astral-sh/ruff-action@v3 | |
| with: | |
| args: check | |
| src: ${{ env.EXTRACT_DIR }} | |
| - name: Scan Transform Directory | |
| uses: astral-sh/ruff-action@v3 | |
| with: | |
| args: check | |
| src: ${{ env.TRANSFORM_DIR }} | |
| scan: | |
| name: Scan for Secrets | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Scan with Gitleaks | |
| uses: gitleaks/gitleaks-action@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE}} | |
| deploy: | |
| name: Deploy Infrastructure | |
| runs-on: ubuntu-latest | |
| needs: [lint, scan] | |
| environment: ${{ github.ref_name == 'main' && 'prod' || 'test' }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_SERVICE_ACCOUNT_EMAIL }} | |
| - name: Set up Docker | |
| run: gcloud auth configure-docker ${{ vars.GCP_PROJECT_REGION }}-docker.pkg.dev --quiet | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| - name: Create bucket for Pulumi infrastructure | |
| run: | | |
| if gsutil ls -b $BUCKET_URL 2>/dev/null; then | |
| echo "Infrastructure bucket already exists. Skipping." | |
| else | |
| echo "Creating infrastructure bucket at '$BUCKET_URL'." | |
| gsutil mb -p $PROJECT_ID -c STANDARD -l $BUCKET_REGION $BUCKET_URL | |
| echo "Bucket created successfully." | |
| fi | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: ${{ env.INFRA_DIR }}/.python-version | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| version: "0.8.13" | |
| - name: Install the project | |
| working-directory: ${{ env.INFRA_DIR }} | |
| run: uv sync --locked | |
| - name: Refresh infrastructure | |
| uses: pulumi/actions@v6 | |
| with: | |
| command: refresh | |
| stack-name: ${{ github.ref_name == 'main' && 'prod' || 'test' }} | |
| work-dir: ${{ env.INFRA_DIR }} | |
| cloud-url: ${{ env.BUCKET_URL}} | |
| env: | |
| CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} | |
| CLOUDFLARE_R2_BUCKET_URL: ${{ vars.CLOUDFLARE_R2_BUCKET_URL }} | |
| CLOUDFLARE_R2_ENDPOINT_URL: ${{ vars.CLOUDFLARE_R2_ENDPOINT_URL }} | |
| CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} | |
| DJANGO_PORT: ${{ vars.DJANGO_PORT }} | |
| DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }} | |
| ENV: ${{ vars.ENV }} | |
| EXTRACTION_PIPELINE_MAX_RETRIES: ${{ vars.EXTRACTION_PIPELINE_MAX_RETRIES }} | |
| EXTRACTION_PIPELINE_MAX_WAIT: ${{ vars.EXTRACTION_PIPELINE_MAX_WAIT }} | |
| EXTRACTION_PIPELINE_POLLING_INTERVAL: ${{ vars.EXTRACTION_PIPELINE_POLLING_INTERVAL }} | |
| EXTRACTION_PIPELINE_SCHEDULE: ${{ vars.EXTRACTION_PIPELINE_SCHEDULE }} | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_PROJECT_REGION: ${{ vars.GCP_PROJECT_REGION }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| OUTPUT_FILE_MAX_AGE: ${{ vars.OUTPUT_FILE_MAX_AGE }} | |
| OUTPUT_FILE_NAME: ${{ vars.OUTPUT_FILE_NAME }} | |
| OUTPUT_FILE_TOTAL_MAX_ATTEMPTS: ${{ vars.OUTPUT_FILE_TOTAL_MAX_ATTEMPTS }} | |
| POSTGRES_DB: ${{ vars.POSTGRES_DB }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| POSTGRES_USER: ${{ vars.POSTGRES_USER }} | |
| PULUMI_CONFIG_PASSPHRASE: "" | |
| - name: Deploy infrastructure | |
| uses: pulumi/actions@v6 | |
| with: | |
| command: up | |
| stack-name: ${{ github.ref_name == 'main' && 'prod' || 'test' }} | |
| work-dir: ${{ env.INFRA_DIR }} | |
| cloud-url: ${{ env.BUCKET_URL}} | |
| upsert: true | |
| env: | |
| CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} | |
| CLOUDFLARE_R2_BUCKET_URL: ${{ vars.CLOUDFLARE_R2_BUCKET_URL }} | |
| CLOUDFLARE_R2_ENDPOINT_URL: ${{ vars.CLOUDFLARE_R2_ENDPOINT_URL }} | |
| CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} | |
| DJANGO_PORT: ${{ vars.DJANGO_PORT }} | |
| DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }} | |
| ENV: ${{ vars.ENV }} | |
| EXTRACTION_PIPELINE_MAX_RETRIES: ${{ vars.EXTRACTION_PIPELINE_MAX_RETRIES }} | |
| EXTRACTION_PIPELINE_MAX_WAIT: ${{ vars.EXTRACTION_PIPELINE_MAX_WAIT }} | |
| EXTRACTION_PIPELINE_POLLING_INTERVAL: ${{ vars.EXTRACTION_PIPELINE_POLLING_INTERVAL }} | |
| EXTRACTION_PIPELINE_SCHEDULE: ${{ vars.EXTRACTION_PIPELINE_SCHEDULE }} | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_PROJECT_REGION: ${{ vars.GCP_PROJECT_REGION }} | |
| GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| OUTPUT_FILE_MAX_AGE: ${{ vars.OUTPUT_FILE_MAX_AGE }} | |
| OUTPUT_FILE_NAME: ${{ vars.OUTPUT_FILE_NAME }} | |
| OUTPUT_FILE_TOTAL_MAX_ATTEMPTS: ${{ vars.OUTPUT_FILE_TOTAL_MAX_ATTEMPTS }} | |
| POSTGRES_DB: ${{ vars.POSTGRES_DB }} | |
| POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} | |
| POSTGRES_USER: ${{ vars.POSTGRES_USER }} | |
| PULUMI_CONFIG_PASSPHRASE: "" |