Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 84 additions & 119 deletions .github/workflows/test-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ name: CI Tests

on:
pull_request:
# Push to main only warms the shared layer cache (build-images); the test/e2e
# jobs are gated off `push` below so a merge doesn't run the full suite.
push:
branches: [main]
workflow_dispatch:
inputs:
use_local_images:
Expand Down Expand Up @@ -73,7 +77,7 @@ jobs:

build-images:
needs: changes
if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
if: ${{ needs.changes.outputs.e2e == 'true' || needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'push' }}
runs-on:
labels: "ubuntu-latest"
permissions:
Expand Down Expand Up @@ -104,132 +108,93 @@ jobs:
run: |
echo "name=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT

- name: Generate Image Cache Key
id: cache-key
- name: Determine build parameters
id: params
run: |
if [ "${{ matrix.image }}" = "opensearch" ]; then
HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile patch-netty.sh securityconfig cloud_securityconfig opensearch-entrypoint-wrapper.sh)
elif [ "${{ matrix.image }}" = "backend" ]; then
HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.backend src enhancements flows securityconfig cloud_securityconfig alembic.ini alembic openrag-documents scripts/backend-entrypoint.sh pyproject.toml uv.lock)
elif [ "${{ matrix.image }}" = "frontend" ]; then
HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.frontend frontend)
elif [ "${{ matrix.image }}" = "langflow" ]; then
HASH=$(python3 scripts/ci/get_image_hash.py Dockerfile.langflow flows scripts/langflow-entrypoint.py)
echo "dockerfile=Dockerfile" >> "$GITHUB_OUTPUT"
else
echo "dockerfile=Dockerfile.${{ matrix.image }}" >> "$GITHUB_OUTPUT"
fi
echo "key=$HASH" >> $GITHUB_OUTPUT

- name: Cache Image
uses: actions/cache@v4
with:
path: /tmp/image-cache/${{ matrix.image }}.tar
key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.image }}-${{ steps.cache-key.outputs.key }}

- name: Build and Push OpenRAG image
run: |
USE_LOCAL="${{ inputs.use_local_images }}"
IS_FORK="${{ env.IS_FORK }}"

if [ "$IS_FORK" = "true" ]; then
REGISTRY_IMAGE="langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}"
if [ "${{ env.IS_FORK }}" = "true" ]; then
echo "image=langflowai/openrag-${{ matrix.image }}:${CI_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
else
REGISTRY_IMAGE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}"
echo "image=ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi

mkdir -p /tmp/image-cache

build_or_load_image() {
local name=$1
local dockerfile=$2
local tar_path="/tmp/image-cache/${name}.tar"
local use_cache=false

if [ -f "$tar_path" ]; then
echo "Found cache file for ${name}. Inspecting architecture..."
docker load -i "$tar_path"

# Get host normalized architecture
local host_arch="$(uname -m)"
local host_arch_norm=""
if [[ "$host_arch" == "x86_64" ]]; then
host_arch_norm="amd64"
elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
host_arch_norm="arm64"
else
host_arch_norm="$host_arch"
fi

local loaded_arch=""
loaded_arch=$(docker inspect --format '{{.Architecture}}' "langflowai/openrag-${name}:cached" 2>/dev/null || echo "unknown")

if [ "$loaded_arch" = "$host_arch_norm" ]; then
echo "Cache hit for ${name}. Loaded image matches host architecture ($host_arch_norm)."
use_cache=true
echo "Tagging loaded image as ${REGISTRY_IMAGE}..."
docker tag "langflowai/openrag-${name}:cached" "${REGISTRY_IMAGE}"
else
echo "Cache image architecture ($loaded_arch) does not match host architecture ($host_arch_norm). Discarding cache."
docker rmi "langflowai/openrag-${name}:cached" 2>/dev/null || true
rm -f "$tar_path"
fi
fi

if [ "$use_cache" = "false" ]; then
echo "Cache miss/incompatible for ${name}. Building..."
if docker buildx build \
--load \
-t "langflowai/openrag-${name}:cached" \
-t "${REGISTRY_IMAGE}" \
-f "$dockerfile" .; then
echo "Saving built image to ${tar_path}..."
docker save "langflowai/openrag-${name}:cached" -o "$tar_path"
else
echo "Build failed for ${name}."
return 1
fi
fi
}

pull_image() {
local name=$1
echo "Pulling latest image for ${name} from DockerHub..."
if docker pull "langflowai/openrag-${name}:latest"; then
docker tag "langflowai/openrag-${name}:latest" "${REGISTRY_IMAGE}"
else
echo "Pull failed for ${name}."
return 1
fi
}

if [ "${{ matrix.image }}" = "opensearch" ]; then
DOCKERFILE="Dockerfile"
# Build locally for PRs, local dispatch, and the OpenSearch image
# (which has no DockerHub publish); otherwise reuse the prebuilt
# DockerHub image.
if [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "push" ] || [ "${{ inputs.use_local_images }}" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then
echo "mode=build" >> "$GITHUB_OUTPUT"
else
DOCKERFILE="Dockerfile.${{ matrix.image }}"
echo "mode=pull" >> "$GITHUB_OUTPUT"
fi
# Shared, non-branch-scoped registry cache. `main` (push) writes it;
# every non-fork PR reads it, so a PR that doesn't touch this image
# gets its layers for free on the first run. PRs also keep a per-PR
# gha cache so a second commit in the same PR reuses its own layers.
REG_CACHE="ghcr.io/${{ steps.repo.outputs.name }}/openrag-${{ matrix.image }}:buildcache"
{
echo "cache_from<<EOF"
echo "type=registry,ref=${REG_CACHE}"
echo "type=gha,scope=${{ matrix.image }}"
echo "EOF"
} >> "$GITHUB_OUTPUT"
if [ "${{ github.event_name }}" = "push" ]; then
echo "cache_to=type=registry,ref=${REG_CACHE},mode=max" >> "$GITHUB_OUTPUT"
else
echo "cache_to=type=gha,mode=max,scope=${{ matrix.image }}" >> "$GITHUB_OUTPUT"
fi

# Build with BuildKit layer caching (cache_from/cache_to computed above)
# so unchanged layers -- e.g. Langflow's torch/pip-install layer or the
# backend uv-sync layer -- are reused instead of rebuilt. Reads the shared
# registry cache seeded by main plus this PR's own gha cache; a flows-only
# change no longer forces a full re-install because that layer's cache key
# doesn't depend on flows/. Non-fork builds push straight to GHCR (no local
# load / docker save / tar shuttle).
- name: Build and push image (GHCR)
if: steps.params.outputs.mode == 'build' && env.IS_FORK != 'true'
uses: docker/build-push-action@v6
with:
context: .
file: ${{ steps.params.outputs.dockerfile }}
push: true
tags: ${{ steps.params.outputs.image }}
cache-from: ${{ steps.params.outputs.cache_from }}
cache-to: ${{ steps.params.outputs.cache_to }}
provenance: false

# Fork PRs cannot push to GHCR or write the Actions cache, so build and
# load locally (still reading any available layer cache). The fork
# fallback steps below save and upload the image as an artifact.
- name: Build image (fork fallback)
if: steps.params.outputs.mode == 'build' && env.IS_FORK == 'true'
uses: docker/build-push-action@v6
with:
context: .
file: ${{ steps.params.outputs.dockerfile }}
load: true
tags: ${{ steps.params.outputs.image }}
cache-from: type=gha,scope=${{ matrix.image }}
provenance: false

- name: Pull prebuilt image from DockerHub
if: steps.params.outputs.mode == 'pull'
run: |
for i in 1 2 3; do
echo "Build/pull attempt $i..."
if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ] || [ "${{ matrix.image }}" = "opensearch" ]; then
if build_or_load_image "${{ matrix.image }}" "$DOCKERFILE"; then
if [ "$IS_FORK" != "true" ]; then
docker push "${REGISTRY_IMAGE}"
fi
break
fi
else
if pull_image "${{ matrix.image }}"; then
if [ "$IS_FORK" != "true" ]; then
docker push "${REGISTRY_IMAGE}"
fi
break
if docker pull "langflowai/openrag-${{ matrix.image }}:latest"; then
docker tag "langflowai/openrag-${{ matrix.image }}:latest" "${{ steps.params.outputs.image }}"
if [ "${{ env.IS_FORK }}" != "true" ]; then
docker push "${{ steps.params.outputs.image }}"
fi
exit 0
fi
echo "Attempt $i failed. Retrying in 10 seconds..."
echo "Pull attempt $i failed. Retrying in 10s..."
sleep 10
if [ $i -eq 3 ]; then
echo "Build/pull failed after 3 attempts."
exit 1
fi
done
echo "Pull failed after 3 attempts."
exit 1

- name: Save OpenRAG image (Fork fallback)
if: env.IS_FORK == 'true'
Expand Down Expand Up @@ -260,7 +225,7 @@ jobs:

test-suite:
needs: [changes, build-images]
if: ${{ needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch' }}
if: ${{ github.event_name != 'push' && (needs.changes.outputs.integration == 'true' || github.event_name == 'workflow_dispatch') }}
runs-on:
labels: "ubuntu-latest"
permissions:
Expand Down Expand Up @@ -370,7 +335,7 @@ jobs:

e2e-run:
needs: [changes, build-images]
if: ${{ needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch' }}
if: ${{ github.event_name != 'push' && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') }}
runs-on:
labels: "ubuntu-latest"
permissions:
Expand Down Expand Up @@ -540,7 +505,7 @@ jobs:
done || true

e2e-merge-reports:
if: ${{ !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}
if: ${{ github.event_name != 'push' && !cancelled() && (needs.changes.outputs.e2e == 'true' || github.event_name == 'workflow_dispatch') && needs.e2e-run.result != 'skipped' }}
needs: [changes, e2e-run]
runs-on:
labels: "ubuntu-latest"
Expand Down Expand Up @@ -581,7 +546,7 @@ jobs:

tests:
needs: [changes, build-images, test-suite]
if: always()
if: always() && github.event_name != 'push'
runs-on:
labels: "ubuntu-latest"
permissions:
Expand All @@ -603,7 +568,7 @@ jobs:

e2e:
needs: [changes, build-images, e2e-run]
if: always()
if: always() && github.event_name != 'push'
runs-on:
labels: "ubuntu-latest"
permissions:
Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ ENV_FILE ?= .env

# Load variables from $(ENV_FILE) if present so `make` commands pick them up
# Strip quotes from values to avoid issues with tools that don't handle them like python-dotenv does

ifneq (,$(wildcard $(ENV_FILE)))
include $(ENV_FILE)
export $(shell sed -n 's/^\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' $(ENV_FILE))
Expand Down
2 changes: 2 additions & 0 deletions frontend/components/chat-renderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ export function ChatRenderer({
setOnboardingComplete,
} = useChat();

//TODO: comment to test CI

// Initialize onboarding state from backend settings
const [currentStep, setCurrentStep] = useState<number>(
settings?.onboarding?.current_step ?? 0,
Expand Down
Loading