Merge branch 'main' into restrict-ingestion-section-clear-fix

Author: Vchen7629

.env.example

@@ -173,6 +173,26 @@ MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
+# --- IBM Cloud Object Storage connector (Enterprise/SaaS) ------------------
+# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the
+# AWS S3 / Azure Blob connectors). Credentials are normally entered per-connection
+# in the UI; these env vars are OPTIONAL fallbacks / defaults. Two auth modes are
+# supported: IAM (api_key + service_instance_id, default) or HMAC (access key +
+# secret, S3-compatible -- works against MinIO for local testing).
+IBM_COS_ENDPOINT=
+IBM_COS_API_KEY=
+IBM_COS_SERVICE_INSTANCE_ID=
+IBM_COS_HMAC_ACCESS_KEY_ID=
+IBM_COS_HMAC_SECRET_ACCESS_KEY=
+#
+# Local dev / MinIO testing: set OPENRAG_DEV_IBM_COS=true to enable the
+# connector without IBM_AUTH_ENABLED. NEVER use in production.
+# OPENRAG_DEV_IBM_COS=false
+# Frontend counterpart: also set NEXT_PUBLIC_DEV_IBM_COS=true so the Connectors
+# tab and admin Connectors Permission page show IBM COS locally (requires a
+# frontend restart to take effect -- Next.js reads NEXT_PUBLIC_* at startup).
+# NEXT_PUBLIC_DEV_IBM_COS=false
+
 # --- Azure Blob Storage connector (Enterprise/SaaS) ------------------------
 # Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the
 # AWS S3 / IBM COS connectors). Credentials are normally entered per-connection

.github/workflows/test-ci.yml

@@ -67,8 +67,25 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v10
+        continue-on-error: true
+        with:
+          root-reserve-mb: 40960
+          temp-reserve-mb: 30720
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          build-mount-path: '/var/lib/docker'
+
+      - name: Restart Docker
+        continue-on-error: true
+        run: sudo service docker restart || sudo systemctl restart docker || true
+
       - run: df -h
 
       - name: Cleanup Docker cache
@@ -115,25 +132,25 @@ jobs:
         uses: actions/cache@v4
         with:
           path: /tmp/image-cache/opensearch.tar
-          key: ${{ runner.os }}-image-opensearch-${{ steps.cache-keys.outputs.opensearch }}
+          key: ${{ runner.os }}-${{ runner.arch }}-opensearch-${{ steps.cache-keys.outputs.opensearch }}
 
       - name: Cache Backend Image
         uses: actions/cache@v4
         with:
           path: /tmp/image-cache/backend.tar
-          key: ${{ runner.os }}-image-backend-${{ steps.cache-keys.outputs.backend }}
+          key: ${{ runner.os }}-${{ runner.arch }}-backend-${{ steps.cache-keys.outputs.backend }}
 
       - name: Cache Frontend Image
         uses: actions/cache@v4
         with:
           path: /tmp/image-cache/frontend.tar
-          key: ${{ runner.os }}-image-frontend-${{ steps.cache-keys.outputs.frontend }}
+          key: ${{ runner.os }}-${{ runner.arch }}-frontend-${{ steps.cache-keys.outputs.frontend }}
 
       - name: Cache Langflow Image
         uses: actions/cache@v4
         with:
           path: /tmp/image-cache/langflow.tar
-          key: ${{ runner.os }}-image-langflow-${{ steps.cache-keys.outputs.langflow }}
+          key: ${{ runner.os }}-${{ runner.arch }}-langflow-${{ steps.cache-keys.outputs.langflow }}
 
       - name: Build OpenRAG images
         run: |
@@ -145,14 +162,40 @@ jobs:
             local name=$1
             local dockerfile=$2
             local tar_path="/tmp/image-cache/${name}.tar"
+            local use_cache=false
             
             if [ -f "$tar_path" ]; then
-              echo "Cache hit for ${name}. Loading image from ${tar_path}..."
+              echo "Found cache file for ${name}. Inspecting architecture..."
               docker load -i "$tar_path"
-              echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."
-              docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"
-            else
-              echo "Cache miss for ${name}. Building..."
+              
+              # Get host normalized architecture
+              local host_arch="$(uname -m)"
+              local host_arch_norm=""
+              if [[ "$host_arch" == "x86_64" ]]; then
+                host_arch_norm="amd64"
+              elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
+                host_arch_norm="arm64"
+              else
+                host_arch_norm="$host_arch"
+              fi
+              
+              local loaded_arch=""
+              loaded_arch=$(docker inspect --format '{{.Architecture}}' "langflowai/openrag-${name}:cached" 2>/dev/null || echo "unknown")
+              
+              if [ "$loaded_arch" = "$host_arch_norm" ]; then
+                echo "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."
+                use_cache=true
+                echo "Tagging loaded image as langflowai/openrag-${name}:${CI_IMAGE_TAG}..."
+                docker tag "langflowai/openrag-${name}:cached" "langflowai/openrag-${name}:${CI_IMAGE_TAG}"
+              else
+                echo "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."
+                docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true
+                rm -f "$tar_path"
+              fi
+            fi
+            
+            if [ "$use_cache" = "false" ]; then
+              echo "Cache miss/incompatible for ${name}. Building..."
               if docker buildx build \
                 --load \
                 -t "langflowai/openrag-${name}:cached" \
@@ -226,13 +269,30 @@ jobs:
     needs: [changes, build-images]
     if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
         suite: [core, sdk-python, sdk-typescript]
     name: tests (${{ matrix.suite }})
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v10
+        continue-on-error: true
+        with:
+          root-reserve-mb: 40960
+          temp-reserve-mb: 30720
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          build-mount-path: '/var/lib/docker'
+
+      - name: Restart Docker
+        continue-on-error: true
+        run: sudo service docker restart || sudo systemctl restart docker || true
+
       - run: df -h
 
       - name: Cleanup Docker cache
@@ -327,14 +387,31 @@ jobs:
     needs: [changes, build-images]
     if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     strategy:
       fail-fast: false
       matrix:
         shard: [1, 2, 3, 4, 5, 6, 7, 8]
         total_shards: [8]
     name: e2e (shard ${{ matrix.shard }}/${{ matrix.total_shards }})
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@v10
+        continue-on-error: true
+        with:
+          root-reserve-mb: 40960
+          temp-reserve-mb: 30720
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          build-mount-path: '/var/lib/docker'
+
+      - name: Restart Docker
+        continue-on-error: true
+        run: sudo service docker restart || sudo systemctl restart docker || true
+
       - name: Cleanup Docker cache
         run: |
           docker compose -f docker-compose.yml down -v --remove-orphans || true
@@ -473,7 +550,7 @@ jobs:
     if: ${{ !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}
     needs: [changes, e2e-run]
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     permissions:
       contents: read
     steps:
@@ -513,7 +590,7 @@ jobs:
     needs: [changes, build-images, test-suite]
     if: always()
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     steps:
       - name: Check split integration result
         run: |
@@ -533,7 +610,7 @@ jobs:
     needs: [changes, build-images, e2e-run]
     if: always()
     runs-on:
-      labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+      labels: "ubuntu-latest"
     steps:
       - name: Check E2E result
         run: |

enhancements/connectors/ibm_cos/connector.py

@@ -1,12 +1,16 @@
-"""IBM Cloud Object Storage connector for OpenRAG."""
+"""IBM Cloud Object Storage connector for OpenRAG.
+
+Enterprise/SaaS-only ``bucket``-kind connector, gated by ``IBM_AUTH_ENABLED``
+(or ``OPENRAG_DEV_IBM_COS=true`` for local dev/MinIO testing).
+"""
 
 import mimetypes
 import os
 from datetime import UTC, datetime
 from posixpath import basename
 from typing import Any
 
-from config.settings import IBM_AUTH_ENABLED
+from config.settings import IBM_AUTH_ENABLED, is_dev_ibm_cos_enabled
 from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
 from utils.logging_config import get_logger
 
@@ -64,7 +68,10 @@ class IBMCOSConnector(BaseConnector):
 
     @classmethod
     def is_available(cls, manager, user_id=None) -> bool:
-        return IBM_AUTH_ENABLED
+        # Enterprise/SaaS gate is IBM_AUTH_ENABLED, like the other bucket
+        # connectors (aws_s3, azure_blob). OPENRAG_DEV_IBM_COS=true bypasses it
+        # for local dev (e.g. against MinIO in HMAC mode; never in production).
+        return IBM_AUTH_ENABLED or is_dev_ibm_cos_enabled()
 
     @classmethod
     def register_routes(cls, app) -> None:

frontend/app/api/queries/useGetSettingsQuery.ts

@@ -79,6 +79,7 @@ export interface Settings {
   localhost_url?: string;
   ingest_via_chat?: boolean;
   show_provider_ingest_settings?: boolean;
+  show_shared_upload_toggle?: boolean;
   segment_write_key?: string;
   environment?: string;
   langflow_port?: string | number | null;