great_expectations/.github/workflows/data_source_cleanup.yml at eec491e4487df4fedf3500e29172a30568ce5317 · fivetran/great_expectations · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# Script(s) to clean up any loose data left behind from test runs.
# These can primarily happen if CI is stopped while tests are running
# for big query, etc.

name: Data Source Cleanup

on:
  workflow_dispatch:
  schedule:
    - cron: "0 * * * *" # Run at the start of each hour

jobs:
  cleanup-big-query:
    runs-on: ubuntu-latest
    env:
      # google
      GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
      GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
      GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
      GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Create JSON file GCP
        run: |
          echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m bigquery
          pip install -e .
      - name: Run BigQuery cleanup script
        run: |
          python ./scripts/cleanup/cleanup_big_query.py

  cleanup-redshift:
    runs-on: ubuntu-latest
    env:
      # aws-redshift
      REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}}
      REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}}
      REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}}
      REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}}
      REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}}
      REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py

      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m gx-redshift
          pip install -e .
      - name: Run Redshift cleanup script
        run: |
          python ./scripts/cleanup/cleanup_redshift.py

  cleanup-databricks:
    runs-on: ubuntu-latest
    env:
      # databricks
      DATABRICKS_TOKEN: ${{secrets.DATABRICKS_SQL_TOKEN}}
      DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
      DATABRICKS_HTTP_PATH: ${{secrets.DATABRICKS_HTTP_PATH}}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m databricks
          pip install -e .
      - name: Run Databricks cleanup script
        run: |
          python ./scripts/cleanup/cleanup_databricks.py

  cleanup-snowflake:
    runs-on: ubuntu-latest
    env:
      # snowflake
      SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}}
      SNOWFLAKE_CI_USER_PASSWORD: ${{secrets.SNOWFLAKE_CI_USER_PASSWORD}}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            reqs/requirements-dev-test.txt
            setup.py
      - name: Install dependencies
        run: |
          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
          invoke deps --gx-install -m snowflake
          pip install -e .
      - name: Run Snowflake cleanup script
        run: |
          python ./scripts/cleanup/cleanup_snowflake.py