Merge branch 'main' into fix_upgradeSaaSLibraryPy

Author: lucaseduoli

.env.example

@@ -163,6 +163,35 @@ MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
+# --- Azure Blob Storage connector (Enterprise/SaaS) ------------------------
+# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the
+# AWS S3 / IBM COS connectors). Credentials are normally entered per-connection
+# in the UI; these env vars are OPTIONAL fallbacks / defaults for the config
+# dialog. Two auth modes are supported: a connection string, OR account name +
+# key (+ optional custom blob endpoint).
+#
+# Local dev / Azurite testing: set OPENRAG_DEV_AZURE_BLOB=true to enable the
+# connector without IBM_AUTH_ENABLED. NEVER use in production.
+# OPENRAG_DEV_AZURE_BLOB=false
+#
+# Connection-string mode (covers real Azure accounts and the Azurite emulator):
+#   - Real account: copy from Azure Portal -> Storage account -> Access keys
+#   - Azurite (local dev): AZURE_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true"
+# AZURE_STORAGE_CONNECTION_STRING=
+#
+# Account-key mode (set account name + key; endpoint optional):
+#   - Azurite (host-side backend):
+#     - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1
+#     - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+#     - AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1
+#   - Azurite (in-compose backend):
+#     - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1
+#     - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+#     - AZURE_STORAGE_ENDPOINT=http://azurite:10000/devstoreaccount1
+# AZURE_STORAGE_ACCOUNT_NAME=
+# AZURE_STORAGE_ACCOUNT_KEY=
+# AZURE_STORAGE_ENDPOINT=
+
 # OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestion
 WEBHOOK_BASE_URL=
 

.github/workflows/codeql.yml

@@ -30,7 +30,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/lint-frontend.yml

@@ -48,3 +48,33 @@ jobs:
       - name: TypeScript type check
         working-directory: frontend
         run: npm run typecheck
+
+  react-doctor:
+    name: React Doctor
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm ci
+
+      - name: React Doctor score check
+        working-directory: frontend
+        run: |
+          SCORE=$(npx -y react-doctor@0.5.8 --score --yes)
+          echo "React Doctor score: $SCORE"
+          if [ "$SCORE" -lt 40 ]; then
+            echo "::error::React Doctor score $SCORE is below the minimum threshold of 40"
+            npx -y react-doctor@0.5.8 --verbose --yes
+            exit 1
+          fi

.github/workflows/react-doctor.yml

@@ -0,0 +1,52 @@
+# React Doctor — finds security, performance, correctness, accessibility,
+# bundle-size, and architecture issues in React codebases.
+#
+# Docs: https://www.react.doctor/ci
+# Source: https://github.com/millionco/react-doctor
+
+name: React Doctor
+
+on:
+  # Scans the PR's changed files and posts a sticky summary comment listing only the new issues introduced relative to the merge base of the target branch.
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  # Scans `main` on every push to track the health-score trend and catch regressions that slipped past PR review.
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+  statuses: write
+
+# Cancels any in-flight scan for the same PR (or branch, on push) the moment a new commit arrives, so reviewers only ever see the latest run.
+concurrency:
+  group: react-doctor-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  react-doctor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: millionco/react-doctor@v2
+        with:
+          directory: frontend
+          version: "0.5.8"
+        # Advisory by default: React Doctor reports findings on every PR — a
+        # sticky summary comment, inline review comments, and a commit status
+        # with the health score — but never fails the check, so it won't red-X
+        # a teammate's PR on day one. When your team trusts the signal, graduate
+        # the gate: uncomment the block below and set blocking to "error" (fail
+        # on new error-severity findings) or "warning" (fail on any finding).
+        # Full reference: https://www.react.doctor/ci
+        #   blocking: error          # Gate level: "none" (advisory, the default) | "warning" | "error"
+        #   scope: full              # On PRs, scan the whole project instead of just changed files
+        #   comment: false           # Disable the sticky PR summary comment
+        #   review-comments: false   # Disable inline review comments on changed lines
+        #   commit-status: false     # Disable the commit status (score + counts, links to the run)
+        #   project: "web,admin"     # In a monorepo, scan specific workspace project(s)

.github/workflows/test-ci.yml

@@ -35,7 +35,7 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   LANGFLOW_URL_INGEST_FLOW_ID: ${{ vars.LANGFLOW_URL_INGEST_FLOW_ID || '72c3d17c-2dac-4a73-b48a-6518473d7830' }}
   # Single-node test cluster never reaches the multi-node readiness counts
-  OPENSEARCH_NODE_COUNT_CHECK: "false"
+  OPENSEARCH_NODE_COUNT_CHECK_ENABLED: "false"
 
 jobs:
   changes:
@@ -53,25 +53,13 @@ jobs:
         with:
           filters: |
             e2e:
-              - 'src/**'
-              - 'frontend/**'
-              - 'tests/**'
-              - 'scripts/**'
-              - 'flows/**'
-              - 'docker-compose.yml'
-              - 'Dockerfile*'
-              - 'Makefile'
-              - '.github/workflows/test-ci.yml'
+              - '**'
+              - '!docs/**'
+              - '!**/*.md'
             integration:
-              - 'src/**.py'
-              - 'tests/**.py'
-              - 'pyproject.toml'
-              - 'uv.lock'
-              - 'sdks/**'
-              - 'flows/**'
-              - 'Makefile'
-              - 'scripts/**'
-              - '.github/workflows/test-ci.yml'
+              - '**'
+              - '!docs/**'
+              - '!**/*.md'
 
   build-images:
     needs: changes
@@ -83,8 +71,6 @@ jobs:
 
       - name: Cleanup Docker cache
         run: |
-          docker system prune -af || true
-          docker builder prune -af || true
           docker compose -f docker-compose.yml down -v --remove-orphans || true
 
       - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
@@ -162,8 +148,6 @@ jobs:
 
       - name: Cleanup Docker cache
         run: |
-          docker system prune -af || true
-          docker builder prune -af || true
           docker compose -f docker-compose.yml down -v --remove-orphans || true
 
       - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
@@ -221,10 +205,19 @@ jobs:
           # Disable startup ingest noise unless a test enables it
           DISABLE_STARTUP_INGEST: "true"
         run: |
+          chmod +x scripts/ci/run_integration_suite.sh
           OPENRAG_VERSION="${CI_IMAGE_TAG}" TEST_SUITE="${{ matrix.suite }}" make test-ci-suite
           echo "Keys directory after tests:"
           ls -la keys/ || echo "No keys directory"
 
+      - name: Upload service logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: service-logs-${{ matrix.suite }}
+          path: service-logs/
+          retention-days: 7
+
   e2e-run:
     needs: [changes, build-images]
     if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}
@@ -233,8 +226,6 @@ jobs:
     steps:
       - name: Cleanup Docker cache
         run: |
-          docker system prune -af || true
-          docker builder prune -af || true
           docker compose -f docker-compose.yml down -v --remove-orphans || true
 
       - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
@@ -268,7 +259,18 @@ jobs:
       - name: Install Playwright Browsers
         working-directory: frontend
         run: |
-          npx playwright install --with-deps chromium
+          for i in 1 2 3; do
+            echo "Installing Playwright browsers (attempt $i)..."
+            if npx playwright install --with-deps chromium; then
+              break
+            fi
+            echo "Attempt $i failed. Retrying in 10 seconds..."
+            sleep 10
+            if [ $i -eq 3 ]; then
+              echo "Failed to install Playwright browsers after 3 attempts."
+              exit 1
+            fi
+          done
 
       - name: Download OpenRAG image artifact
         uses: actions/download-artifact@v4

CONTRIBUTING.md

@@ -262,6 +262,8 @@ make clear-os-data  # Clear OpenSearch data directory completely
 make factory-reset  # Complete reset: containers, volumes, and data
 ```
 
+The reset also removes any legacy `opensearch-data` directory, so old OpenSearch index files do not linger after cleanup.
+
 ---
 
 ## Makefile Help System

Dockerfile.langflow

@@ -12,11 +12,14 @@ RUN microdnf install -y \
     && microdnf clean all
 
 # Install Langflow with targeted extras and CPU-only wheels for PyTorch/Docling
+# ibm-watsonx-ai and langchain-ibm are needed because they are not included in the pip installation of langflow when python version is 3.14
 RUN pip install --no-cache-dir \
     --extra-index-url https://download.pytorch.org/whl/cpu \
     torch==2.10.0 \
     torchvision==0.25.0 \
     "langflow-base[docling,openai,anthropic,google,ollama,opensearch,langfuse]==0.9.6" \
+    "ibm-watsonx-ai>=1.3.1,<2.0.0" \
+    "langchain-ibm~=1.1.0" \
     uv \
     && pip uninstall -y clickhouse-connect \
     && pip uninstall -y ragstack-ai-knowledge-store