feat: add more e2e tests to frontend #315
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Tests | |
| on: | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| use_local_images: | |
| description: 'Build images locally instead of pulling from DockerHub' | |
| required: false | |
| type: boolean | |
| default: true | |
| env: | |
| NODE_VERSION: "22" | |
| PYTHON_VERSION: "3.13" | |
| PLAYWRIGHT_BROWSERS_PATH: "ms-playwright" | |
| PLAYWRIGHT_VERSION: "1.57.0" | |
| # Total number of parallel E2E test shards | |
| TOTAL_SHARDS: 4 | |
| OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGFLOW_AUTO_LOGIN: "True" | |
| LANGFLOW_NEW_USER_IS_ACTIVE: "True" | |
| LANGFLOW_ENABLE_SUPERUSER_CLI: "True" | |
| LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }} | |
| LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }} | |
| NUDGES_FLOW_ID: ${{ vars.NUDGES_FLOW_ID || 'ebc01d31-1976-46ce-a385-b0240327226c' }} | |
| CI_IMAGE_TAG: ci-${{ github.sha }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} | |
| WATSONX_ENDPOINT: ${{ secrets.WATSONX_ENDPOINT }} | |
| WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} | |
| OLLAMA_ENDPOINT: ${{ secrets.OLLAMA_ENDPOINT }} | |
| GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.GOOGLE_OAUTH_CLIENT_ID }} | |
| GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.GOOGLE_OAUTH_CLIENT_SECRET }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }} | |
| # Single-node test cluster never reaches the multi-node readiness counts | |
| OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false" | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| e2e: ${{ steps.filter.outputs.e2e }} | |
| integration: ${{ steps.filter.outputs.integration }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Detect changes | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| e2e: | |
| - '**' | |
| - '!docs/**' | |
| - '!**/*.md' | |
| integration: | |
| - '**' | |
| - '!docs/**' | |
| - '!**/*.md' | |
| build-images: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| steps: | |
| - run: df -h | |
| - name: Cleanup Docker cache | |
| run: | | |
| docker compose -f docker-compose.yml down -v --remove-orphans || true | |
| docker system prune -f || true | |
| - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents) | |
| run: | | |
| for i in 1 2 3; do | |
| docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break | |
| echo "Attempt $i failed, retrying in 5s..." | |
| sleep 5 | |
| done || true | |
| - run: df -h | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Verify workspace | |
| run: | | |
| echo "Current directory: $(pwd)" | |
| echo "Workspace: ${GITHUB_WORKSPACE}" | |
| ls -la | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate Image Cache Keys | |
| id: cache-keys | |
| run: | | |
| OPENSEARCH_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh) | |
| BACKEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock) | |
| FRONTEND_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend) | |
| LANGFLOW_HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py) | |
| echo "opensearch=$OPENSEARCH_HASH" >> $GITHUB_OUTPUT | |
| echo "backend=$BACKEND_HASH" >> $GITHUB_OUTPUT | |
| echo "frontend=$FRONTEND_HASH" >> $GITHUB_OUTPUT | |
| echo "langflow=$LANGFLOW_HASH" >> $GITHUB_OUTPUT | |
| - name: Cache OpenSearch Image | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/image-cache/opensearch.tar | |
| key: ${{ runner.os }}-image-opensearch-${{ steps.cache-keys.outputs.opensearch }} | |
| - name: Cache Backend Image | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/image-cache/backend.tar | |
| key: ${{ runner.os }}-image-backend-${{ steps.cache-keys.outputs.backend }} | |
| - name: Cache Frontend Image | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/image-cache/frontend.tar | |
| key: ${{ runner.os }}-image-frontend-${{ steps.cache-keys.outputs.frontend }} | |
| - name: Cache Langflow Image | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/image-cache/langflow.tar | |
| key: ${{ runner.os }}-image-langflow-${{ steps.cache-keys.outputs.langflow }} | |
| - name: Build OpenRAG images | |
| run: | | |
| USE_LOCAL="${{ inputs.use_local_images }}" | |
| mkdir -p /tmp/image-cache | |
| build_or_load_image() { | |
| local name=$1 | |
| local dockerfile=$2 | |
| local tar_path="/tmp/image-cache/${name}.tar" | |
| if [ -f "$tar_path" ]; then | |
| echo "Cache hit for ${name}. Loading image from ${tar_path}..." | |
| docker load -i "$tar_path" | |
| echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..." | |
| docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}" | |
| else | |
| echo "Cache miss for ${name}. Building..." | |
| if docker buildx build \ | |
| --load \ | |
| -t "langflowai/openrag-${name}:cached" \ | |
| -t "langflowai/openrag-${name}:${CI_IMAGE_TAG}" \ | |
| -f "$dockerfile" .; then | |
| echo "Saving built image to ${tar_path}..." | |
| docker save "langflowai/openrag-${name}:cached" -o "$tar_path" | |
| else | |
| echo "Build failed for ${name}." | |
| return 1 | |
| fi | |
| fi | |
| } | |
| for i in 1 2 3; do | |
| echo "Build attempt $i..." | |
| if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then | |
| echo "Building PR images locally..." | |
| if build_or_load_image "opensearch" "Dockerfile" && \ | |
| build_or_load_image "backend" "Dockerfile.backend" && \ | |
| build_or_load_image "frontend" "Dockerfile.frontend" && \ | |
| build_or_load_image "langflow" "Dockerfile.langflow"; then | |
| break | |
| fi | |
| else | |
| echo "Preparing DockerHub images..." | |
| if docker pull langflowai/openrag-backend:latest && \ | |
| docker pull langflowai/openrag-frontend:latest && \ | |
| docker pull langflowai/openrag-langflow:latest && \ | |
| docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}" && \ | |
| docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}" && \ | |
| docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}" && \ | |
| build_or_load_image "opensearch" "Dockerfile"; then | |
| break | |
| fi | |
| fi | |
| echo "Build attempt $i failed. Retrying in 10 seconds..." | |
| sleep 10 | |
| if [ $i -eq 3 ]; then | |
| echo "Build failed after 3 attempts." | |
| exit 1 | |
| fi | |
| done | |
| - name: Save OpenRAG images | |
| run: OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-save-images | |
| - name: Upload OpenRAG image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: openrag-ci-images-${{ github.sha }} | |
| path: .ci-artifacts/openrag-ci-images.tar | |
| retention-days: 1 | |
| compression-level: 0 | |
| test-suite: | |
| needs: [changes, build-images] | |
| if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suite: [core, sdk-python, sdk-typescript] | |
| name: tests (${{ matrix.suite }}) | |
| steps: | |
| - run: df -h | |
| - name: Cleanup Docker cache | |
| run: | | |
| docker compose -f docker-compose.yml down -v --remove-orphans || true | |
| docker system prune -f || true | |
| - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents) | |
| run: | | |
| for i in 1 2 3; do | |
| docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break | |
| echo "Attempt $i failed, retrying in 5s..." | |
| sleep 5 | |
| done || true | |
| - run: df -h | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Verify workspace | |
| run: | | |
| echo "Current directory: $(pwd)" | |
| echo "Workspace: ${GITHUB_WORKSPACE}" | |
| ls -la | |
| - name: Set up UV | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: latest | |
| - name: Set up Node.js | |
| if: matrix.suite == 'sdk-typescript' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Python version | |
| run: uv python install 3.13 | |
| - name: Download OpenRAG image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openrag-ci-images-${{ github.sha }} | |
| path: .ci-artifacts | |
| - name: Load OpenRAG images | |
| run: docker load -i .ci-artifacts/openrag-ci-images.tar | |
| - name: Run ${{ matrix.suite }} integration suite | |
| env: | |
| OPENSEARCH_HOST: localhost | |
| OPENSEARCH_PORT: 9200 | |
| OPENSEARCH_USERNAME: admin | |
| OPENSEARCH_PASSWORD: ${{ env.OPENSEARCH_PASSWORD }} | |
| LOG_LEVEL: DEBUG | |
| # Force no-auth mode so tests bypass OAuth | |
| GOOGLE_OAUTH_CLIENT_ID: "" | |
| GOOGLE_OAUTH_CLIENT_SECRET: "" | |
| # Disable startup ingest noise unless a test enables it | |
| DISABLE_STARTUP_INGEST: "true" | |
| run: | | |
| chmod +x scripts/ci/run_integration_suite.sh | |
| OPENRAG_VERSION="${CI_IMAGE_TAG}" TEST_SUITE="${{ matrix.suite }}" make test-ci-suite | |
| echo "Keys directory after tests:" | |
| ls -la keys/ || echo "No keys directory" | |
| - name: Upload service logs | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: service-logs-${{ matrix.suite }} | |
| path: service-logs/ | |
| retention-days: 7 | |
| e2e-run: | |
| needs: [changes, build-images] | |
| if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: [1, 2, 3, 4] | |
| total_shards: [4] | |
| name: e2e (shard ${{ matrix.shard }}/${{ matrix.total_shards }}) | |
| steps: | |
| - name: Cleanup Docker cache | |
| run: | | |
| docker compose -f docker-compose.yml down -v --remove-orphans || true | |
| docker system prune -f || true | |
| - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents) | |
| run: | | |
| for i in 1 2 3; do | |
| docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break | |
| echo "Attempt $i failed, retrying in 5s..." | |
| sleep 5 | |
| done || true | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up UV | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: latest | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install Python | |
| run: uv python install 3.13 | |
| - name: Install frontend dependencies | |
| working-directory: frontend | |
| run: npm ci | |
| - name: Install Playwright Browsers | |
| working-directory: frontend | |
| run: | | |
| for i in 1 2 3; do | |
| echo "Installing Playwright browsers (attempt $i)..." | |
| if npx playwright install --with-deps chromium; then | |
| break | |
| fi | |
| echo "Attempt $i failed. Retrying in 10 seconds..." | |
| sleep 10 | |
| if [ $i -eq 3 ]; then | |
| echo "Failed to install Playwright browsers after 3 attempts." | |
| exit 1 | |
| fi | |
| done | |
| - name: Download OpenRAG image artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: openrag-ci-images-${{ github.sha }} | |
| path: .ci-artifacts | |
| - name: Load OpenRAG images | |
| run: docker load -i .ci-artifacts/openrag-ci-images.tar | |
| - name: Setup E2E infrastructure | |
| run: | | |
| chmod +x scripts/setup-e2e.sh | |
| OPENRAG_VERSION="${CI_IMAGE_TAG}" ./scripts/setup-e2e.sh | |
| - name: Run Playwright tests (shard ${{ matrix.shard }}/${{ matrix.total_shards }}) | |
| working-directory: frontend | |
| env: | |
| CI: "true" | |
| OPENSEARCH_HOST: localhost | |
| OPENSEARCH_PORT: "9200" | |
| OPENSEARCH_USERNAME: admin | |
| OPENSEARCH_PASSWORD: ${{ env.OPENSEARCH_PASSWORD }} | |
| GOOGLE_OAUTH_CLIENT_ID: "" | |
| GOOGLE_OAUTH_CLIENT_SECRET: "" | |
| run: npx playwright test --shard=${{ matrix.shard }}/${{ matrix.total_shards }} | |
| - name: Collect service logs on failure | |
| if: failure() | |
| run: | | |
| mkdir -p service-logs | |
| docker logs openrag-opensearch > service-logs/opensearch.log 2>&1 || true | |
| docker logs openrag-backend > service-logs/backend.log 2>&1 || true | |
| docker logs openrag-frontend > service-logs/frontend.log 2>&1 || true | |
| docker logs openrag-langflow > service-logs/langflow.log 2>&1 || true | |
| cp ~/.openrag/tui/docling-serve.log service-logs/docling.log 2>&1 || true | |
| - name: Upload service logs | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: service-logs-shard-${{ matrix.shard }} | |
| path: service-logs/ | |
| retention-days: 7 | |
| - name: Upload blob report (for merge) | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: blob-report-shard-${{ matrix.shard }} | |
| path: frontend/blob-report/ | |
| retention-days: 1 | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| name: playwright-test-results-shard-${{ matrix.shard }} | |
| path: frontend/test-results/ | |
| retention-days: 7 | |
| - name: Teardown infrastructure | |
| if: always() | |
| run: | | |
| make docling-stop || true | |
| docker rm -f openrag-backend-proxy 2>/dev/null || true | |
| make clean || true | |
| docker system prune -f || true | |
| e2e-merge-reports: | |
| if: ${{ !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }} | |
| needs: [changes, e2e-run] | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install frontend dependencies | |
| working-directory: frontend | |
| run: npm ci | |
| - name: Download all shard blob reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: frontend/all-blob-reports | |
| pattern: blob-report-shard-* | |
| merge-multiple: true | |
| - name: Merge into HTML report | |
| working-directory: frontend | |
| run: npx playwright merge-reports --reporter html ./all-blob-reports | |
| - name: Upload merged HTML report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report | |
| path: frontend/playwright-report/ | |
| retention-days: 14 | |
| tests: | |
| needs: [changes, build-images, test-suite] | |
| if: always() | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| steps: | |
| - name: Check split integration result | |
| run: | | |
| if [ "${{ needs.changes.outputs.integration }}" = "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| if [ "${{ needs.build-images.result }}" != "success" ]; then | |
| echo "build-images failed" | |
| exit 1 | |
| fi | |
| if [ "${{ needs.test-suite.result }}" != "success" ]; then | |
| echo "one or more integration suites failed" | |
| exit 1 | |
| fi | |
| fi | |
| echo "all integration suites passed or skipped" | |
| e2e: | |
| needs: [changes, build-images, e2e-run] | |
| if: always() | |
| runs-on: | |
| labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"] | |
| steps: | |
| - name: Check E2E result | |
| run: | | |
| if [ "${{ needs.changes.outputs.e2e }}" = "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| if [ "${{ needs.build-images.result }}" != "success" ]; then | |
| echo "build-images failed" | |
| exit 1 | |
| fi | |
| if [ "${{ needs.e2e-run.result }}" != "success" ]; then | |
| echo "one or more E2E shards failed" | |
| exit 1 | |
| fi | |
| fi | |
| echo "E2E tests passed or skipped" |