feat(perf): add system performance test runner

.github/workflows/perf-test.yml

@@ -0,0 +1,93 @@
+name: Performance Tests
+
+on:
+  schedule:
+    # Run daily at 06:00 UTC
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+    inputs:
+      scale:
+        description: "Test scale"
+        type: choice
+        options:
+          - tiny
+          - small
+          - medium
+          - large
+        default: small
+      suite:
+        description: "Suite to run (blank = all)"
+        type: choice
+        options:
+          - ""
+          - retain
+          - recall
+        default: ""
+      ref:
+        description: "Git ref to test (branch, tag, or SHA). Defaults to main."
+        type: string
+        default: ""
+
+concurrency:
+  group: perf-test
+  cancel-in-progress: true
+
+jobs:
+  perf-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ inputs.ref || github.ref }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+        prune-cache: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version-file: ".python-version"
+
+    - name: Cache HuggingFace models
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/huggingface
+        key: ${{ runner.os }}-huggingface-${{ hashFiles('hindsight-api-slim/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-huggingface-
+
+    - name: Pre-download models
+      working-directory: ./hindsight-api-slim
+      run: |
+        uv run --frozen --all-extras --index-strategy unsafe-best-match python -c "
+        from sentence_transformers import SentenceTransformer
+        print('Downloading embedding model...')
+        SentenceTransformer('BAAI/bge-small-en-v1.5')
+        print('Model downloaded successfully')
+        "
+
+    - name: Install hindsight-dev dependencies
+      run: |
+        cd hindsight-dev && uv sync --frozen --index-strategy unsafe-best-match
+
+    - name: Run perf tests
+      run: |
+        SUITE_ARG=""
+        if [ -n "${{ inputs.suite }}" ]; then
+          SUITE_ARG="--suite ${{ inputs.suite }}"
+        fi
+        ./scripts/benchmarks/run-perf-test.sh \
+          --scale ${{ inputs.scale || 'small' }} \
+          $SUITE_ARG \
+          --output perf-results.json
+
+    - name: Upload perf results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: perf-results-${{ github.sha }}
+        path: hindsight-dev/perf-results.json
+        retention-days: 90

CLAUDE.md

@@ -68,8 +68,9 @@ cd hindsight-control-plane && npm run dev
 ./scripts/benchmarks/run-locomo.sh
 
 # Performance benchmarks
+./scripts/benchmarks/run-perf-test.sh                      # System perf (mock LLM + pg0)
+./scripts/benchmarks/run-perf-test.sh --scale tiny          # Quick smoke test
 ./scripts/benchmarks/run-consolidation.sh
-./scripts/benchmarks/run-retain-perf.sh --document <path>  # Requires API server running
 
 # Results viewer
 ./scripts/benchmarks/start-visualizer.sh  # View results at localhost:8001

hindsight-dev/benchmarks/README.md

@@ -79,31 +79,29 @@ Tests consolidation throughput and identifies bottlenecks.
 NUM_MEMORIES=200 ./scripts/benchmarks/run-consolidation.sh
 ```
 
-### Retain Performance
+### System Performance Test
 
-Measures retain operation performance (throughput and token usage).
-
-**Prerequisites:** API server must be running (`./scripts/dev/start-api.sh`)
+Runs retain throughput and recall latency benchmarks using mock LLM + pg0.
+No external dependencies needed.
 
 ```bash
-# Basic usage
-./scripts/benchmarks/run-retain-perf.sh \
-    --document hindsight-dev/benchmarks/perf/test_data/sample_document.txt
-
-# Save results to JSON
-./scripts/benchmarks/run-retain-perf.sh \
-    --document ./my_document.txt \
-    --bank-id my-test-bank \
-    --output results/retain_perf.json
+# Run all suites at default (small) scale
+./scripts/benchmarks/run-perf-test.sh
+
+# Quick smoke test
+./scripts/benchmarks/run-perf-test.sh --scale tiny
+
+# Single suite
+./scripts/benchmarks/run-perf-test.sh --suite retain
+
+# Save results
+./scripts/benchmarks/run-perf-test.sh --output results.json
 ```
 
 **Options:**
-- `--document PATH` - Document file to retain (required)
-- `--bank-id ID` - Bank ID to use (default: perf-test)
-- `--context TEXT` - Optional context
-- `--api-url URL` - API URL (default: http://localhost:8000)
-- `--timeout SECONDS` - Request timeout (default: 300)
-- `--output PATH` - Save results to JSON file
+- `--scale {tiny,small,medium,large}` - Test scale (default: small)
+- `--suite {retain,recall}` - Run specific suite (default: all)
+- `--output PATH` - Save JSON results to file
 
 See [perf/README.md](perf/README.md) for detailed documentation.
 

hindsight-dev/benchmarks/perf/README.md

@@ -0,0 +1,70 @@
+# Performance Benchmarks
+
+## System Performance Test (`perf-test`)
+
+Orchestrates independent benchmark suites using mock LLM + pg0 to produce
+deterministic, LLM-independent performance baselines.
+
+```bash
+uv run perf-test                        # all suites, small scale
+uv run perf-test --suite retain         # single suite
+uv run perf-test --scale medium         # larger scale
+uv run perf-test --output results.json  # save JSON results
+```
+
+### Suites
+
+| Suite | What it measures |
+|-------|-----------------|
+| `retain` | Full retain pipeline with mock LLM: fact extraction callback, embedding generation, DB writes, entity linking |
+| `recall` | Pre-populated bank recall: 4-way parallel retrieval (semantic, BM25, graph, temporal), RRF fusion, percentile latency |
+
+### Scale Configurations
+
+| Scale | Retain items | Recall bank size | Recall iterations | Recall concurrency |
+|-------|-------------|-----------------|-------------------|-------------------|
+| `tiny` | 20 | 20 | 5 | 1 |
+| `small` | 200 | 200 | 20 | 4 |
+| `medium` | 1,000 | 1,000 | 50 | 8 |
+| `large` | 5,000 | 5,000 | 100 | 16 |
+
+### CI
+
+The `perf-test.yml` workflow runs daily and on manual dispatch. Scale is
+configurable via workflow input (defaults to `small`). Results are uploaded as
+artifacts with 90-day retention.
+
+## Standalone Benchmarks
+
+### `retain_perf.py`
+
+Retain operation benchmark with two modes:
+
+- **In-memory**: Direct `retain_batch_async` call (single file or directory)
+- **Async**: Full async workflow with mock LLM, worker poller, and deadlock stress testing
+
+```bash
+uv run python hindsight-dev/benchmarks/perf/retain_perf.py \
+    --document <file_path> --in-memory
+```
+
+### `recall_perf.py`
+
+Large-bank recall load test. Generates synthetic banks with Zipf-distributed
+entities, then benchmarks recall latency at scale with per-step breakdown.
+
+```bash
+# Generate bank
+uv run python hindsight-dev/benchmarks/perf/recall_perf.py generate \
+    --bank-id my-bank --scale small
+
+# Benchmark
+uv run python hindsight-dev/benchmarks/perf/recall_perf.py benchmark \
+    --bank-id my-bank --query "database migration" --iterations 20
+
+# Clean up
+uv run python hindsight-dev/benchmarks/perf/recall_perf.py clean \
+    --bank-id my-bank
+```
+
+Scales: `tiny` (1), `mini` (50), `small` (2K), `medium` (10K), `large` (33K), `very-large` (100K).

hindsight-dev/benchmarks/perf/recall_perf.py

@@ -778,6 +778,12 @@ async def cmd_generate(bank_id: str, scale: str, workers: int = 16, with_observa
 # ---------------------------------------------------------------------------
 
 
+class _RRFCrossEncoder:
+    """Stub cross encoder that reports itself as the RRF passthrough provider."""
+
+    provider_name = "rrf"
+
+
 class _RRFReranker:
     """
     Drop-in replacement for CrossEncoderReranker that uses RRF scores only.
@@ -786,6 +792,8 @@ class _RRFReranker:
     reflect pure DB interaction costs.
     """
 
+    cross_encoder = _RRFCrossEncoder()
+
     async def ensure_initialized(self) -> None:
         pass