Auto Update LLM Models #70
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto Update LLM Models | |
| # Daily reconciliation against llm-stats.com (via the zeroeval Stats API): | |
| # scripts/reconcile_models.py runs once and produces three signals: | |
| # | |
| # - new_models candidates not yet registered in OCS | |
| # - price_changes existing seed entries whose upstream rate has moved | |
| # - missing_pricing OCS-managed (default_models) models with no seed entry | |
| # | |
| # Job 1 (reconcile) runs the script. The price-change side effects (rewrite | |
| # seed + emit migration) are deterministic, so the same job opens the | |
| # "Pricing update" PR and the "missing pricing" issue. Job 2 (new-models-pr) | |
| # only runs when there are candidates and invokes Claude Code for the | |
| # judgement calls (which providers, which params class, deprecations) per | |
| # docs/developer_guides/managing_models.md. | |
| # | |
| # Model metadata is sourced from https://llm-stats.com (via api.zeroeval.com). | |
| on: | |
| schedule: | |
| - cron: '0 4 * * *' # Daily at 04:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| days: | |
| description: 'Lookback window in days (max 30)' | |
| required: false | |
| default: '1' | |
| type: string | |
| concurrency: | |
| group: auto-update-models | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| reconcile: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| issues: write | |
| outputs: | |
| has_new_models: ${{ steps.reconcile.outputs.has_new_models }} | |
| new_model_count: ${{ steps.reconcile.outputs.new_model_count }} | |
| new_model_ids: ${{ steps.reconcile.outputs.new_model_ids }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| - name: Reconcile upstream model catalogue and pricing seed | |
| id: reconcile | |
| env: | |
| LLM_STATS_BEARER_TOKEN: ${{ secrets.LLM_STATS_BEARER_TOKEN }} | |
| run: | | |
| # Stdlib-only - no dependency install needed. | |
| python3 scripts/reconcile_models.py \ | |
| --bearer-token "$LLM_STATS_BEARER_TOKEN" \ | |
| --days "${{ inputs.days || '1' }}" \ | |
| --output reconciliation.json | |
| - name: Upload reconciliation payload | |
| if: steps.reconcile.outputs.has_new_models == 'true' | |
| uses: actions/upload-artifact@v7.0.1 | |
| with: | |
| name: reconciliation-payload | |
| path: | | |
| reconciliation.json | |
| retention-days: 7 | |
| - name: Configure git author | |
| if: steps.reconcile.outputs.has_price_changes == 'true' | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| - name: Commit, push, and open Pricing update PR | |
| if: steps.reconcile.outputs.has_price_changes == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_TITLE: ${{ steps.reconcile.outputs.pricing_pr_title }} | |
| PR_BODY_PATH: ${{ steps.reconcile.outputs.pricing_pr_body_path }} | |
| run: | | |
| BRANCH="auto-pricing/$(date +%Y%m%d-%H%M%S)" | |
| git checkout -b "$BRANCH" | |
| git add apps/cost_tracking/seed_data/llm_pricing.json apps/cost_tracking/migrations/ | |
| git commit -m "$PR_TITLE" | |
| git push -u origin "$BRANCH" | |
| gh pr create \ | |
| --base main \ | |
| --head "$BRANCH" \ | |
| --title "$PR_TITLE" \ | |
| --body-file "$PR_BODY_PATH" | |
| - name: Open or update missing-pricing issue | |
| if: steps.reconcile.outputs.has_missing_pricing == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_BODY_PATH: ${{ steps.reconcile.outputs.missing_pricing_issue_body_path }} | |
| MISSING_COUNT: ${{ steps.reconcile.outputs.missing_pricing_count }} | |
| run: | | |
| # One open issue at a time. Edit it if it already exists so the body | |
| # reflects the latest gap list; otherwise open a new one. Labelled | |
| # `cost-tracking,auto-models` so it filters cleanly. Body is | |
| # prefixed with a team @-mention so the platform team is notified. | |
| TITLE="Missing pricing for ${MISSING_COUNT} OCS-managed model(s)" | |
| { | |
| echo "cc @dimagi/open-chat-studio" | |
| echo | |
| cat "$ISSUE_BODY_PATH" | |
| } > issue-body.md | |
| EXISTING=$(gh issue list \ | |
| --state open \ | |
| --label cost-tracking \ | |
| --search "Missing pricing for in:title" \ | |
| --json number --jq '.[0].number // empty') | |
| if [ -n "$EXISTING" ]; then | |
| gh issue edit "$EXISTING" --title "$TITLE" --body-file issue-body.md | |
| else | |
| gh issue create \ | |
| --title "$TITLE" \ | |
| --body-file issue-body.md \ | |
| --label cost-tracking \ | |
| --label auto-models | |
| fi | |
| new-models-pr: | |
| needs: reconcile | |
| if: needs.reconcile.outputs.has_new_models == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| id-token: write | |
| services: | |
| postgres: | |
| image: pgvector/pgvector:pg16 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres_password | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 6379:6379 | |
| env: | |
| DJANGO_DATABASE_USER: postgres | |
| DJANGO_DATABASE_PASSWORD: postgres_password | |
| SECRET_KEY: secret-test-key | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download reconciliation payload | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: reconciliation-payload | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.13' | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - name: Install Python dependencies | |
| run: | | |
| uv venv | |
| uv sync --locked --dev | |
| echo "$PWD/.venv/bin" >> "$GITHUB_PATH" | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24.x | |
| - name: Install Node dependencies | |
| run: npm ci | |
| - name: Configure git author | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| - name: Create branch | |
| id: branch | |
| run: | | |
| BRANCH_NAME="auto-models/$(date +%Y%m%d-%H%M%S)" | |
| git checkout -b "$BRANCH_NAME" | |
| echo "branch_name=$BRANCH_NAME" >> "$GITHUB_OUTPUT" | |
| - name: Run Claude Code | |
| id: claude | |
| uses: anthropics/claude-code-action@v1 | |
| with: | |
| anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} | |
| prompt: | | |
| You are adding newly released LLM models to Open Chat Studio. | |
| ## Input | |
| `scripts/reconcile_models.py` has already fetched candidates from | |
| llm-stats.com, removed models already registered in OCS, and resolved | |
| per-1K-token pricing. Its output is at `./reconciliation.json`. Do | |
| NOT re-fetch, re-dedup, or re-derive pricing - that work is done and | |
| unit-tested. You only care about the `new_models[]` section of the | |
| payload here; the `price_changes` and `missing_pricing` sections are | |
| handled in a separate job and are not your concern. | |
| The payload covers the last ${{ inputs.days || '1' }} day(s). Key fields: | |
| - `new_models[]` - candidates to consider. Each has: `id`, `org`, | |
| `context_window`, `ocs_providers` (the default OCS provider mapping), | |
| `already_registered_providers`, `already_priced_providers`, | |
| `source_url`, `sources`, `details_error`, and `pricing`. | |
| - `pricing.has_pricing` / `pricing.rates` (per-1K-token) / | |
| `pricing.source` (`llm_stats` or `litellm`) / | |
| `pricing.llm_pricing_entries` - ready-to-paste `llm_pricing.json` | |
| entries, one per provider that still needs pricing. | |
| - `pricing_entries` - flat list of all generated pricing entries. | |
| - `unpriced_models[]` - models the script could not price. | |
| ${{ needs.reconcile.outputs.new_model_count }} new model(s): | |
| ${{ needs.reconcile.outputs.new_model_ids }} | |
| ## Your task (judgement + applying the generated output) | |
| 1. Read `reconciliation.json` and | |
| `apps/service_providers/llm_service/default_models.py`. | |
| 2. For each model in `new_models`, decide whether to add it: | |
| - Skip models that don't make sense for OCS (embedding-only, | |
| image-only, or preview models already superseded). | |
| - Use `ocs_providers` as the default provider mapping. Apply | |
| judgement: for an `openai` model keep `azure` only if it is | |
| generally available on Azure OpenAI - otherwise drop `azure` and | |
| add to `openai` only. Skip any provider already in | |
| `already_registered_providers`. | |
| 3. Follow `docs/developer_guides/managing_models.md` exactly when adding: | |
| - Register it in `DEFAULT_LLM_PROVIDER_MODELS` using `context_window` | |
| as `token_limit`. | |
| - Pick an existing parameters class that matches the model family; | |
| only create a new one if none fits. | |
| - Create a new Django migration that calls `llm_model_migration()`. | |
| - Strip `llm_model_migration()` (and any `notify_deprecated_models` / | |
| `remove_deprecated_models` data migrations) from earlier migrations | |
| in `apps/service_providers/migrations/` so they only run once. | |
| 4. Apply pricing: for each model you add, append its | |
| `pricing.llm_pricing_entries` to | |
| `apps/cost_tracking/seed_data/llm_pricing.json`, keeping only the | |
| providers you actually registered. Trust the unit conversion but | |
| sanity-check the magnitudes. Do NOT invent prices - any model in | |
| `unpriced_models` (or a provider with no generated entry) goes under | |
| `## Needs follow-up` instead. After editing, confirm the file is | |
| still valid JSON. | |
| 5. If `context_window` or other essential info is missing, consult | |
| `sources` / `source_url` with WebFetch. If a model has a | |
| `details_error`, the detail fetch failed - use the top-level fields | |
| and what you know about the family. If you still can't determine | |
| `token_limit`, add the model with a conservative placeholder | |
| (e.g. 128000) and a `# TODO: verify token_limit` comment, and flag | |
| it under `## Needs follow-up`. | |
| 6. Lint and test the changed files: | |
| - `uv run ruff check apps/ --fix` | |
| - `uv run ruff format apps/service_providers/ apps/cost_tracking/` | |
| - `uv run pytest apps/service_providers/tests/test_default_models.py -v` | |
| Lint/test any other files your migration touched. | |
| 7. Commit the changes with a descriptive message. | |
| 8. Push the branch and open a PR against `main` using the repo's PR | |
| template (`.github/pull_request_template.md`). | |
| - PR title: `Auto: add new LLM models from llm-stats.com (YYYY-MM-DD)` | |
| (today's UTC date). | |
| - The PR description MUST: | |
| * List each added model with its `source_url`. | |
| * Include the line: `Model metadata sourced from | |
| [llm-stats.com](https://llm-stats.com) via the zeroeval Stats | |
| API.` | |
| * Include a `## Needs follow-up` section for any unpriced models or | |
| missing info. | |
| * Check the "The migrations are backwards compatible" box - | |
| `llm_model_migration()` is additive (only seeds new rows) so | |
| these migrations are inherently backwards-compatible. | |
| 9. Apply the `claude` and `auto-models` labels to the PR if they exist. | |
| 10. If after review there is nothing to add, do NOT open an empty PR - | |
| print a short summary to stdout and exit. | |
| 11. ALWAYS write a decision report to `./decision_report.md` before | |
| exiting, regardless of outcome (PR opened, nothing to add, or | |
| partial). Use this exact structure: | |
| ```markdown | |
| # Auto Update Models - Decision Report | |
| **Run date (UTC):** <YYYY-MM-DD HH:MM> | |
| **Candidates considered:** <count> | |
| **Outcome:** <PR opened | Nothing to add | Error> | |
| **PR:** <PR URL if opened, else "n/a"> | |
| ## Per-model decisions | |
| ### `<model-id>` (org: `<org>`) | |
| - **Verdict:** Added | Skipped | Needs follow-up | |
| - **Mapped providers:** <e.g. openai, azure> (if added) | |
| - **Pricing:** <source: llm_stats | litellm | unpriced> | |
| - **Reason:** <one-sentence rationale - why added/skipped, or | |
| what info was missing> | |
| - **Source:** <source_url> | |
| <repeat for every model in new_models> | |
| ## Notes | |
| <any other context worth surfacing - sources consulted, edge | |
| cases, decisions deferred to follow-up, etc.> | |
| ``` | |
| Every model in `new_models` MUST appear in the "Per-model | |
| decisions" section with an explicit verdict and reason. This | |
| file is the audit trail for the run. | |
| Branch: ${{ steps.branch.outputs.branch_name }} (already created and | |
| checked out). | |
| claude_args: | | |
| --allowedTools "Bash(uv run:*),Bash(uv sync:*),Bash(python manage.py:*),Bash(npm run lint:*),Bash(ls:*),Bash(cat:*),Bash(git add:*),Bash(git commit:*),Bash(git push:*),Bash(git checkout:*),Bash(git status:*),Bash(git diff:*),Bash(git log:*),Bash(gh pr create:*),Bash(gh pr edit:*),Bash(gh label:*),Read,Write,Edit,Glob,Grep,WebFetch" | |
| - name: Append decision report to step summary | |
| if: always() | |
| run: | | |
| if [ -f decision_report.md ]; then | |
| cat decision_report.md >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| { | |
| echo "# Auto Update Models - Decision Report" | |
| echo "" | |
| echo "**No decision report was produced.** Claude Code exited without" | |
| echo "writing \`decision_report.md\`. Check the Claude step logs above" | |
| echo "for what happened." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi |