ci.yaml

name: CI
run-name: "${{ github.event_name == 'workflow_dispatch' && format('Dispatch : Run CI with JDK {0} ({1})', inputs.INT_TEST_JAVA_RUNTIME_VERSION, inputs.INT_TEST_JAVA_RUNTIME_VENDOR) || '' }}"

on:
  # When triggered by a pull request, basic integration tests will run in addition to the tests related
  # to the file changes. JDK 8 (Temurin) will be used at runtime.
  pull_request:

  # When triggered by a dispatch, selected tests will run with the given integration tests JDK runtime.
  workflow_dispatch:
    inputs:
      ALL:
        description: Run all integration tests
        type: boolean
        default: true
      BASIC:
        description: Run basic integration tests covering DynamoDB, PostgreSQL 17, and multi-storage
        type: boolean
        default: false
      BLOB_STORAGE:
        description: Run Blob Storage integration tests
        type: boolean
        default: false
      CASSANDRA:
        description: Run Cassandra integration tests
        type: boolean
        default: false
      COSMOS:
        description: Run Cosmos DB integration tests
        type: boolean
        default: false
      DYNAMO:
        description: Run DynamoDB integration tests
        type: boolean
        default: false
      JDBC:
        description: Run JDBC storages integration tests
        type: boolean
        default: false
      MULTI_STORAGE:
        description: Run multi-storage integration tests
        type: boolean
        default: false
      INTEGRATION_TEST_ONLY:
        description: Run only integration tests. Unit tests and static analysis will be skipped.
        type: boolean
        default: false
      INT_TEST_JAVA_RUNTIME_VERSION:
        description: JDK version used to run the integration test
        type: choice
        default: '8'
        options:
          - '8'
          - '11'
          - '17'
          - '21'
      INT_TEST_JAVA_RUNTIME_VENDOR:
        description: Vendor of the JDK used to run the integration test
        type: choice
        default: 'temurin'
        options:
          - 'corretto'
          - 'microsoft'
          - 'oracle'
          - 'temurin'

  # When triggered by another action, all tests will run with the given integration test JDK runtime.
  workflow_call:
    inputs:
      ALL:
        description: Run all integration tests
        type: boolean
        default: true
      INT_TEST_JAVA_RUNTIME_VERSION:
        description: JDK version used to run the integration test
        type: string
        required: true
      INT_TEST_JAVA_RUNTIME_VENDOR:
        description: Vendor of the JDK used to run the integration test
        type: string
        required: true
      INTEGRATION_TEST_ONLY:
        description: Run only integration tests
        type: boolean
        default: false
env:
  TERM: dumb
  ARTIFACT_SUFFIX: ${{ inputs.INT_TEST_JAVA_RUNTIME_VERSION || '8' }}_${{ inputs.INT_TEST_JAVA_RUNTIME_VENDOR || 'temurin' }}

jobs:
  check:
    name: Gradle check
    runs-on: ubuntu-latest
    if: ${{ !inputs.INTEGRATION_TEST_ONLY }}

    steps:
      - uses: actions/checkout@v6

      - name: Set up JDK 8
        uses: actions/setup-java@v5
        with:
          java-version: 8
          distribution: temurin

      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v6

      - name: Execute Gradle 'check' task
        id: run-gradle-check
        run: ./gradlew check buildSrc:check

      - name: Save Gradle test reports
        if: always()
        run: |
          mkdir -p /tmp/gradle_test_reports/core
          mkdir -p /tmp/gradle_test_reports/schema-loader
          mkdir -p /tmp/gradle_test_reports/data-loader/core
          mkdir -p /tmp/gradle_test_reports/data-loader/cli
          cp -a core/build/reports/tests/test /tmp/gradle_test_reports/core/
          cp -a schema-loader/build/reports/tests/test /tmp/gradle_test_reports/schema-loader/
          cp -a data-loader/core/build/reports/tests/test /tmp/gradle_test_reports/data-loader/core/
          cp -a data-loader/cli/build/reports/tests/test /tmp/gradle_test_reports/data-loader/cli/

      - name: Upload Gradle test reports
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{ steps.run-gradle-check.outcome == 'success' && 'success' || 'failure' }}_gradle_check_reports_${{ env.ARTIFACT_SUFFIX }}
          path: /tmp/gradle_test_reports

      - name: Save SpotBugs reports
        if: always()
        run: |
          mkdir -p /tmp/gradle_spotbugs_reports/core
          mkdir -p /tmp/gradle_spotbugs_reports/schema-loader
          mkdir -p /tmp/gradle_spotbugs_reports/integration-test
          mkdir -p /tmp/gradle_spotbugs_reports/data-loader/core
          mkdir -p /tmp/gradle_spotbugs_reports/data-loader/cli
          cp -a core/build/reports/spotbugs /tmp/gradle_spotbugs_reports/core/
          cp -a schema-loader/build/reports/spotbugs /tmp/gradle_spotbugs_reports/schema-loader/
          cp -a integration-test/build/reports/spotbugs /tmp/gradle_spotbugs_reports/integration-test/
          cp -a data-loader/core/build/reports/spotbugs /tmp/gradle_spotbugs_reports/data-loader/core/
          cp -a data-loader/cli/build/reports/spotbugs /tmp/gradle_spotbugs_reports/data-loader/cli/

      - name: Upload Spotbugs reports
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{ steps.run-gradle-check.outcome == 'success' && 'success' || 'failure' }}_gradle_spotbugs_reports_${{ env.ARTIFACT_SUFFIX }}
          path: /tmp/gradle_spotbugs_reports

  dockerfile-lint:
    name: Lint dockerfiles
    runs-on: ubuntu-latest
    if: ${{ !inputs.INTEGRATION_TEST_ONLY }}

    steps:
      - uses: actions/checkout@v6

      - name: Set up JDK 8
        uses: actions/setup-java@v5
        with:
          java-version: 8
          distribution: temurin

      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v6

      - name: Dockerfile Lint for ScalarDB Schema Loader
        run: ./gradlew schema-loader:dockerfileLint

      - name: Dockerfile Lint for ScalarDB Data Loader CLI
        run: ./gradlew data-loader:cli:dockerfileLint

  # Identify which integration tests to run, then extract the `ci/tests-config.yaml` into a matrix of jobs to be ingested by the `integration-test` job
  integration-test-jobs-planner:
    name: Plan integration test jobs
    runs-on: ubuntu-latest
    env:
      ALL: ${{ inputs.ALL }}
      BASIC: ${{ inputs.BASIC }}
      BLOB_STORAGE: ${{ inputs.BLOB_STORAGE }}
      CASSANDRA: ${{ inputs.CASSANDRA }}
      COSMOS: ${{ inputs.COSMOS }}
      DYNAMO: ${{ inputs.DYNAMO }}
      JDBC: ${{ inputs.JDBC }}
      MULTI_STORAGE: ${{ inputs.MULTI_STORAGE }}
      TRIGGERED_BY_WF_DISPATCH: ${{ github.event_name == 'workflow_dispatch' }}
      EXPANDED_TESTS_CONFIG_FILE: expanded-tests-config.json
    outputs:
      test_matrix: ${{ steps.generate-matrix-for-wf-dispatch.outputs.matrix }}
    steps:
      - uses: actions/checkout@v6

      - name: Install Groovy
        run: |
          curl -s "https://get.sdkman.io?ci=true" | bash
          source "$HOME/.sdkman/bin/sdkman-init.sh"
          sdk install groovy
          echo "$HOME/.sdkman/candidates/groovy/current/bin" >> $GITHUB_PATH

      - name: Check file changes and init environment variables indicating which tests to run
        id: generate-matrix-by-changes
        if: ${{ env.TRIGGERED_BY_WF_DISPATCH == 'false' }}
        env:
          PR_BASE_BRANCH: ${{ github.event.pull_request.base.ref }}
        run: groovy ci/plan-tests-based-on-file-changes.groovy

      - name: Validate workflow dispatch inputs
        if: env.TRIGGERED_BY_WF_DISPATCH
        run: |
          if [ "$ALL" != "true" ]; then
            # Check if at least one specific test category is enabled
            if [ "$BASIC" != "true" ] && \
               [ "$BLOB_STORAGE" != "true" ] && \
               [ "$CASSANDRA" != "true" ] && \
               [ "$COSMOS" != "true" ] && \
               [ "$DYNAMO" != "true" ] && \
               [ "$JDBC" != "true" ] && \
               [ "$MULTI_STORAGE" != "true" ]; then
              echo "Error: When the ALL workflow input is false, at least one specific test category must be enabled."
              echo "Please enable at least one of: BASIC, BLOB_STORAGE, CASSANDRA, COSMOS, DYNAMO, JDBC, MULTI_STORAGE."
              exit 1
            fi
          fi

      - name: Generate the integration tests matrix
        id: generate-matrix-for-wf-dispatch
        run: |
          # Generate minimal JSON matrix (stdout) and full JSON config ($EXPANDED_TESTS_CONFIG_FILE)
          MATRIX_JSON=$(groovy ci/generate-test-matrix.groovy)
          echo "matrix=${MATRIX_JSON}" >> $GITHUB_OUTPUT

#       Github does not allow outputting passwords, contained in setup and run scripts of the tests configuration,
#       directly as matrix strategy parameters because they are flagged as sensitive data.
#       So, we upload the full test configurations as an artifact for the integration test jobs to consume
      - name: Upload expanded tests configuration
        uses: actions/upload-artifact@v7
        with:
          name: ci_internal_tests_config-${{ env.ARTIFACT_SUFFIX }}
          path: ${{ env.EXPANDED_TESTS_CONFIG_FILE }}
          overwrite: true

      - name: Print expanded tests configuration
        run: cat ${{ env.EXPANDED_TESTS_CONFIG_FILE }}

  integration-test:
    name: ${{ matrix.display_name }} integration test (${{ (matrix.group_commit_enabled == 'true') && 'group commit' || 'default' }})
    needs:
      - integration-test-jobs-planner
    runs-on: ${{ matrix.runner }}
    env:
      INT_TEST_JAVA_RUNTIME_VERSION: ${{ inputs.INT_TEST_JAVA_RUNTIME_VERSION || '8' }}
      INT_TEST_JAVA_RUNTIME_VENDOR: ${{ inputs.INT_TEST_JAVA_RUNTIME_VENDOR || 'temurin' }}
      GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }}
      CR_PAT: ${{ secrets.CR_PAT }}

    strategy:
      fail-fast: false
      # The matrix parameters are:
      # - label
      # - display_name
      # - group_commit_enabled
      # - runner
      matrix: ${{ fromJSON(needs.integration-test-jobs-planner.outputs.test_matrix) }}

    steps:
      - uses: actions/checkout@v6

      - name: Download test configuration
        uses: actions/download-artifact@v8
        with:
          name: ci_internal_tests_config-${{ env.ARTIFACT_SUFFIX }}

      - name: Load test configuration
        shell: bash
        run: |
          # Find the test entry matching label and group_commit_enabled
          LABEL="${{ matrix.label }}"
          GROUP_COMMIT="${{ matrix.group_commit_enabled }}"

          # Extract the test configuration using jq
          CONFIG=$(jq -r ".[] | select(.label == \"$LABEL\" and .group_commit_enabled == \"$GROUP_COMMIT\")" expanded-tests-config.json)
          
          # Set the SETUP_CMD, DB_HEALTHCHECK_CMD, CREATE_TEST_USER_CMD, and RUN_CMD environment variables
          SETUP_CMD=$(echo "$CONFIG" | jq -r '.setup // ""')
          echo "SETUP_CMD<<EOF" >> $GITHUB_ENV
          echo "$SETUP_CMD" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          
          DB_HEALTHCHECK_CMD=$(echo "$CONFIG" | jq -r '.db_healthcheck // ""')
          echo "DB_HEALTHCHECK_CMD<<EOF" >> $GITHUB_ENV
          echo "$DB_HEALTHCHECK_CMD" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          
          CREATE_TEST_USER_CMD=$(echo "$CONFIG" | jq -r '.create_test_user // ""')
          echo "CREATE_TEST_USER_CMD<<EOF" >> $GITHUB_ENV
          echo "$CREATE_TEST_USER_CMD" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

          RUN_CMD=$(echo "$CONFIG" | jq -r '.run // ""')
          echo "RUN_CMD<<EOF" >> $GITHUB_ENV
          echo "$RUN_CMD" >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Set up the database
        run: ${{ env.SETUP_CMD }}

      - name: Set up JDK 8
        uses: actions/setup-java@v5
        with:
          java-version: 8
          distribution: temurin

      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
        id: setup-int-test-jdk
        uses: actions/setup-java@v5
        # On Linux runner, skip if Oracle JDK is specified
        # On Windows runner, skip if Oracle JDK 8 or 11 is specified
        if: ${{ (runner.os == 'Linux' && env.INT_TEST_JAVA_RUNTIME_VENDOR != 'oracle') || (runner.os == 'Windows' && !(env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' && (env.INT_TEST_JAVA_RUNTIME_VERSION == '8' || env.INT_TEST_JAVA_RUNTIME_VERSION == '11')))  }}
        with:
          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}

      - name: For Linux runner, set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
        if: ${{ runner.os == 'Linux' && steps.setup-int-test-jdk.outcome == 'skipped' }}
        # For Linux runner, install Oracle JDK for any versions from Oracle Container Registry
        run: |
          # For Linux runner, install Oracle JDK for any versions from Oracle Container Registry
          echo ${{ secrets.OCR_TOKEN }} | docker login container-registry.oracle.com -u ${{ secrets.OCR_USERNAME }} --password-stdin
          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"

      - name: For Windows runner, set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
        if: ${{ runner.os == 'Windows' && steps.setup-int-test-jdk.outcome == 'skipped' }}
        # For Windows runner, install Oracle JDK 8 or 11 from Scalar Container Registry
        run: |
          echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
          $container_id=$(docker create "ghcr.io/scalar-labs/oracle/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}-windows")
          docker cp "${container_id}:oracle-jdk.exe" .
          docker rm "$container_id"
          Write-Host "Install Oracle JDK"
          Start-Process "oracle-jdk.exe" -NoNewWindow -Wait -ArgumentList "/s"
          Write-Host "Oracle JDK installation successful"
          if ( ${env:INT_TEST_JAVA_RUNTIME_VERSION} -eq '8' ) {
            $jdk_root_dir = "jdk-1.8"
          } else {
            $jdk_root_dir = "jdk-11"
          }
          echo "JAVA_HOME=C:\Program Files\Java\${jdk_root_dir}" >> ${env:GITHUB_ENV}

      - name: Setup Gradle
        uses: gradle/actions/setup-gradle@v6

      - name: Wait for the database to be ready
        if: ${{ env.DB_HEALTHCHECK_CMD != '' }}
        shell: bash
        run: |
          # Waiting for the database to be ready (10 minutes timeout)
          SECONDS=0
          until bash -c "${DB_HEALTHCHECK_CMD}";
            do
              if (( SECONDS > 600 )); then
                echo "Timeout: Database failed to start within 10 minutes"
                exit 1
              fi
              echo "The database is not yet ready"
              sleep 2s
            done
          echo "The database is ready"

      - name: Create test user
        if: ${{ env.CREATE_TEST_USER_CMD != '' }}
        run: ${{ env.CREATE_TEST_USER_CMD }}

      - name: Run integration test
        id: run-integration-test
        env:
          ORG_GRADLE_PROJECT_javaVersion: '8'
          ORG_GRADLE_PROJECT_javaVendor: 'temurin'
          ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVersion: "${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}"
          ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVendor: "${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}"
          GROUP_COMMIT_OPTIONS: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"'
        run: ${{ env.RUN_CMD }} ${{ matrix.group_commit_enabled == 'true' && env.GROUP_COMMIT_OPTIONS || '' }}

      - name: Detect test report folder
        id: detect-report-folder
        if: always()
        shell: bash
        run: |
          report_dir="core/build/reports/tests"
          folder_name=$(ls -1 "$report_dir" | head -n 1)
          report_path=$report_dir/$folder_name
          if [ -d "$report_path" ]; then
            echo "report_path=$report_path" >> "$GITHUB_OUTPUT"
            echo "Detected test report folder: $report_path"
          else
            echo "Test report folder could not be found"
          fi

      - name: Upload Gradle test reports
        if: ${{ always() && steps.detect-report-folder.outputs.report_path != '' }}
        uses: actions/upload-artifact@v7
        with:
          name: ${{ steps.run-integration-test.outcome == 'success' && 'success' || 'failure' }}_${{ matrix.label }}_${{ matrix.group_commit_enabled == 'true' && 'group_commit' || 'default' }}_test_${{ env.ARTIFACT_SUFFIX }}
          path: ${{ steps.detect-report-folder.outputs.report_path }}

  merge-artifacts:
    if: always()
    name: Merge all artifacts
    runs-on: ubuntu-latest
    needs:
      - check
      - dockerfile-lint
      - integration-test
    steps:
      - name: Delete test configuration
        uses: geekyeggo/delete-artifact@v6
        if: needs.integration-test.result == 'success'
        with:
          name: ci_internal_tests_config-${{ env.ARTIFACT_SUFFIX }}
          failOnError: false

      - name: Merge all artifacts
        uses: actions/upload-artifact/merge@v7
        with:
          name: scalardb_core_ci_report_${{ env.ARTIFACT_SUFFIX }}_run_#${{ github.run_attempt }}
          pattern: "*_${{ env.ARTIFACT_SUFFIX }}"
          separate-directories: true
          delete-merged: true