feat: group acls

.env.example

@@ -34,6 +34,10 @@ FETCH_OPENRAG_DOCS_AT_STARTUP=false
 # Default: 3600 seconds (60 minutes) total timeout
 # INGESTION_TIMEOUT=3600
 
+# OPTIONAL: OpenSearch JWT TTL for per-user document security tokens (in seconds)
+# Default: INGESTION_TIMEOUT + 300 seconds
+# OPENRAG_OPENSEARCH_JWT_TTL=3900
+
 # OPTIONAL: Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
 # Default: 25
 # UPLOAD_BATCH_SIZE=25

AGENTS.md

@@ -44,12 +44,12 @@ Read the `SKILL.md` files directly. The frontmatter `description` tells you when
 
 **RBAC is opt-in.** `OPENRAG_RBAC_ENFORCE` defaults to `false`, which makes OpenRAG behave like the pre-RBAC release: every authenticated user has full access; API-key role overrides are also bypassed. To turn the permissions system on (admin/developer/user/viewer roles, `require_permission` gates, audit denials), set `OPENRAG_RBAC_ENFORCE=true`. Available in all `OPENRAG_RUN_MODE` values — operators own the trade-off. The startup event logs the enforcement state on every boot.
 
-**Dev-local with backend on host.** When you run `make dev-local-cpu` (or `dev-local`) and then `make backend` on the host, OpenSearch needs to resolve `openrag-backend` to the host machine so it can fetch JWKS for OIDC validation. The base `docker-compose.yml` does NOT add this alias — it would break CI, where the backend is a docker-compose service. To enable the host-backend mode, layer the override file:
+**Dev-local with backend on host.** When you run `make dev-local-cpu` (or `dev-local`) and then `make backend` on the host, OpenSearch needs to resolve `openrag-backend` to the host machine so it can fetch JWKS for OIDC validation. Langflow also needs that name for backend ingest callbacks. The base `docker-compose.yml` does NOT add this alias — it would break CI, where the backend is a docker-compose service. To enable the host-backend mode, layer the override file:
 
 ```bash
 docker compose -f docker-compose.yml -f docker-compose.host-backend.yml up -d opensearch dashboards langflow
 make backend     # in another terminal
 make frontend    # in another terminal
 ```
 
-The override only adds `extra_hosts: openrag-backend:host-gateway` to the OpenSearch service. Without it, OIDC works because docker DNS routes `openrag-backend` to the in-compose backend container.
+The override only adds `extra_hosts: openrag-backend:host-gateway` to the OpenSearch and Langflow services. Without it, OIDC and ingest callbacks work because docker DNS routes `openrag-backend` to the in-compose backend container.

Makefile

@@ -871,6 +871,7 @@ test-ci: ensure-langflow-data ensure-backend-volumes ## Start infra, run integra
 	GOOGLE_OAUTH_CLIENT_ID="" \
 	GOOGLE_OAUTH_CLIENT_SECRET="" \
 	OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
+	LANGFLOW_OPENSEARCH_HOST=opensearch LANGFLOW_OPENSEARCH_PORT=9200 \
 	OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
 	DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
 	uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \
@@ -995,6 +996,7 @@ test-ci-local: ensure-langflow-data ensure-backend-volumes ## Same as test-ci bu
 	GOOGLE_OAUTH_CLIENT_ID="" \
 	GOOGLE_OAUTH_CLIENT_SECRET="" \
 	OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
+	LANGFLOW_OPENSEARCH_HOST=opensearch LANGFLOW_OPENSEARCH_PORT=9200 \
 	OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
 	DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
 	uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \

cloud_securityconfig/roles.yml

@@ -3,55 +3,44 @@ _meta:
   config_version: 2
 
 openrag_user_role:
-  description: "DLS: user can read/write docs they own or are allowed on"
+  description: "DLS: user can read docs they own or are allowed on"
   cluster_permissions:
-    - "indices:data/write/bulk"
-    - "indices:data/write/index"
     - "indices:data/read/scroll"
     - "indices:data/read/scroll/clear"
     - "cluster:monitor/*"
-    - "cluster:admin/opensearch/notifications/configs/create"
-    - "cluster:admin/opensearch/notifications/configs/list"
-    - "cluster:admin/opensearch/notifications/configs/get"
-    - "cluster:admin/opensearch/notifications/configs/update"
-    - "cluster:admin/opensearch/notifications/configs/delete"
-    - "cluster:admin/opensearch/alerting/*"
 
   index_permissions:
-    - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*",orag*]
+    - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]
       allowed_actions:
-        - crud
-        - create_index
+        - read
         - indices:admin/mappings/get
-        - indices:admin/mappings/put
         - indices:admin/exists
         - indices:admin/get
-        - indices:admin/refresh
-        - indices:data/write/delete/byquery
-        - indices:data/write/update/byquery
       dls: >
         {"bool":{"should":[
           {"term":{"owner":"${user.name}"}},
+          {"term":{"owner":"${attr.jwt.email}"}},
           {"term":{"allowed_users":"${user.name}"}},
+          {"term":{"allowed_users":"${attr.jwt.email}"}},
+          {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},
           {"bool":{"must_not":{"exists":{"field":"owner"}}}}
         ],"minimum_should_match":1}}
+    - index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]
+      allowed_actions:
+        - read
+        - indices:admin/exists
+        - indices:data/read/get
+        - indices:data/read/search
+      dls: >
+        {"term":{"user_name":"${user.name}"}}
     - index_patterns: ["api_keys", "api_keys*"]
       allowed_actions:
-        - crud
-        - create_index
+        - read
         - indices:admin/mappings/get
-        - indices:admin/mappings/put
         - indices:admin/exists
         - indices:admin/get
-        - indices:admin/refresh
-        - indices:data/write/delete/byquery
-        - indices:data/write/update/byquery
       dls: >
         {"bool":{"should":[
           {"term":{"user_id":"${user.name}"}},
           {"bool":{"must_not":{"exists":{"field":"user_id"}}}}
         ],"minimum_should_match":1}}
-    - index_patterns: [".opendistro-alerting-config"]
-      allowed_actions:
-        - crud
-        - indices:admin/get

docker-compose.host-backend.yml

@@ -28,3 +28,6 @@ services:
   opensearch:
     extra_hosts:
       - "openrag-backend:host-gateway"
+  langflow:
+    extra_hosts:
+      - "openrag-backend:host-gateway"

docker-compose.yml

@@ -56,6 +56,7 @@ services:
     environment:
       - OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
       - LANGFLOW_URL=http://langflow:7860
+      - OPENRAG_BACKEND_INTERNAL_URL=${OPENRAG_BACKEND_INTERNAL_URL:-http://openrag-backend:8000}
       - LANGFLOW_PUBLIC_URL=${LANGFLOW_PUBLIC_URL}
       - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
       - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
@@ -101,7 +102,7 @@ services:
       - LANGFLOW_VERSION=${LANGFLOW_VERSION}
       - LOG_FORMAT=${LOG_FORMAT}
       - SERVICE_NAME=${SERVICE_NAME}
-      - SESSION_SECRET=${SESSION_SECRET}
+      - SESSION_SECRET=${SESSION_SECRET:-your-secret-key-change-in-production}
       - JWT_SIGNING_KEY=${JWT_SIGNING_KEY}
       - OPENRAG_ENCRYPTION_KEY=${OPENRAG_ENCRYPTION_KEY}
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
@@ -180,7 +181,12 @@ services:
       - SOURCE_URL=None
       - ALLOWED_USERS=[]
       - ALLOWED_GROUPS=[]
+      - ALLOWED_PRINCIPALS=[]
       - OPENRAG-QUERY-FILTER="{}"
+      - OPENRAG_INGEST_URL=OPENRAG_INGEST_URL
+      - OPENRAG_INGEST_TOKEN=OPENRAG_INGEST_TOKEN
+      - OPENRAG_INGEST_RUN_ID=OPENRAG_INGEST_RUN_ID
+      - OPENRAG_INGEST_BATCH_SIZE=100
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
       - OPENSEARCH_HOST=${LANGFLOW_OPENSEARCH_HOST:-${OPENSEARCH_HOST:-opensearch}}
       - OPENSEARCH_PORT=${LANGFLOW_OPENSEARCH_PORT:-${OPENSEARCH_PORT:-9200}}
@@ -192,7 +198,7 @@ services:
       - MIMETYPE=None
       - FILESIZE=0
       - SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-text-embedding-3-small}
-      - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME
+      - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE
       - LANGFLOW_LOG_LEVEL=DEBUG
       - LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}
       - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}