fix(ollama): make native num_ctx opt-in (#2589)

* fix(ollama): make native num_ctx opt-in

* docs(ollama): add HINDSIGHT_API_LLM_OLLAMA_NUM_CTX to .env.example

* fix(config): keep Ollama num_ctx optional for direct config construction

---------

Co-authored-by: r266-tech <r266-tech@users.noreply.github.com>
Co-authored-by: Ben <ben.bartholomew@vectorize.io>

Author: r266-tech

.env.example

@@ -59,6 +59,15 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1
 # HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1
 # HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct
 
+# Example: Ollama local configuration (native provider)
+# HINDSIGHT_API_LLM_PROVIDER=ollama
+# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1
+# HINDSIGHT_API_LLM_MODEL=gemma3:12b
+# Native Ollama context-window override (num_ctx). Leave unset to let Ollama use
+# the model Modelfile / server default; set a positive integer only to force a
+# specific context size (e.g. 16384 to keep the previous request behavior).
+# HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384
+
 # Multi-LLM strategies: configure extra LLMs by index alongside the primary above,
 # then pick a routing strategy. Unset = single primary LLM (default). Members are
 # numbered from 1; indices must be contiguous. Each operation can override with a

hindsight-api-slim/hindsight_api/config.py

@@ -147,6 +147,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]:
 ENV_LLM_DEFAULT_HEADERS = "HINDSIGHT_API_LLM_DEFAULT_HEADERS"
 ENV_LLM_STRICT_SCHEMA = "HINDSIGHT_API_LLM_STRICT_SCHEMA"
 ENV_LLM_SEND_BANK_AS_USER = "HINDSIGHT_API_LLM_SEND_BANK_AS_USER"
+ENV_LLM_OLLAMA_NUM_CTX = "HINDSIGHT_API_LLM_OLLAMA_NUM_CTX"
 
 # Per-operation sampling temperature. Each internal LLM call uses a temperature
 # tuned for its task (deterministic extraction vs. creative reflection). These
@@ -1594,6 +1595,9 @@ class HindsightConfig:
     # LiteLLM, Helicone) key attribution on the OpenAI `user` field. Opt-in; never
     # overrides a `user` the caller already set.
     llm_send_bank_as_user: bool
+    # Optional native Ollama context window override. Unset lets Ollama use the
+    # model/server default instead of forcing a Hindsight-wide value.
+    llm_ollama_num_ctx: int | None = field(default=None, kw_only=True)
 
     # Per-operation sampling temperature. None means the temperature parameter is
     # omitted from the call (for models that reject explicit temperatures). See
@@ -2340,6 +2344,10 @@ def from_env(cls) -> "HindsightConfig":
             llm_strict_schema=os.getenv(ENV_LLM_STRICT_SCHEMA, str(DEFAULT_LLM_STRICT_SCHEMA)).lower() in ("true", "1"),
             llm_send_bank_as_user=os.getenv(ENV_LLM_SEND_BANK_AS_USER, str(DEFAULT_LLM_SEND_BANK_AS_USER)).lower()
             in ("true", "1"),
+            llm_ollama_num_ctx=_parse_optional_positive_int(
+                ENV_LLM_OLLAMA_NUM_CTX,
+                os.getenv(ENV_LLM_OLLAMA_NUM_CTX),
+            ),
             llm_temperature_verification=_resolve_operation_temperature(
                 ENV_LLM_TEMPERATURE_VERIFICATION, DEFAULT_LLM_TEMPERATURE_VERIFICATION
             ),

hindsight-api-slim/hindsight_api/engine/llm_wrapper.py

@@ -235,6 +235,17 @@ def requires_api_key(provider: str) -> bool:
     return provider.lower() not in _PROVIDERS_WITHOUT_API_KEY
 
 
+def _validate_ollama_num_ctx(value: Any) -> int | None:
+    """Validate a native Ollama context-window override."""
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}")
+    if value < 1:
+        raise ValueError(f"ollama_num_ctx must be >= 1, got {value}")
+    return value
+
+
 def create_llm_provider(
     provider: str,
     api_key: str,
@@ -254,6 +265,7 @@ def create_llm_provider(
     litellmrouter_config: dict[str, Any] | None = None,
     gemini_service_tier: str | None = None,
     timeout: float | None = None,
+    ollama_num_ctx: int | None = None,
 ) -> Any:  # Returns LLMInterface
     """
     Factory function to create the appropriate LLM provider implementation.
@@ -268,6 +280,8 @@ def create_llm_provider(
         openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper).
         bedrock_service_tier: Bedrock service tier (for Bedrock provider) - None (default), "flex", "priority", or "reserved".
         gemini_service_tier: Gemini service tier (for Gemini provider) - None (default) or "flex" (50% cheaper).
+        ollama_num_ctx: Native Ollama context window override. None lets Ollama use the
+            model/server default.
         extra_body: Extra request-body params merged into the provider's native
             call. Threaded into OpenAI-compatible, Fireworks, Anthropic, Gemini/
             VertexAI and LiteLLM providers (each merges them in its own parameter
@@ -291,6 +305,8 @@ def create_llm_provider(
     Returns:
         LLMInterface implementation for the specified provider.
     """
+    ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
+
     from .providers import (
         AnthropicLLM,
         ClaudeCodeLLM,
@@ -494,6 +510,7 @@ def create_llm_provider(
             groq_service_tier=groq_service_tier,
             openai_service_tier=openai_service_tier,
             extra_body=extra_body,
+            ollama_num_ctx=ollama_num_ctx,
             timeout=timeout,
         )
 
@@ -531,6 +548,7 @@ def __init__(
         max_retries: int | None = None,
         initial_backoff: float | None = None,
         max_backoff: float | None = None,
+        ollama_num_ctx: int | None = None,
     ):
         """
         Initialize LLM provider.
@@ -545,6 +563,8 @@ def __init__(
             openai_service_tier: OpenAI service tier (None or "flex") - from config.
             bedrock_service_tier: Bedrock service tier (None, "flex", "priority", "reserved") - from config.
             gemini_service_tier: Gemini service tier (None or "flex") - from config.
+            ollama_num_ctx: Native Ollama context window override. ``None`` lets Ollama
+                use the model/server default.
             gemini_safety_settings: Safety settings for Gemini/VertexAI providers.
             extra_body: Extra request-body params merged into the provider's native call
                 (OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI, LiteLLM).
@@ -598,6 +618,7 @@ def __init__(
         self.openai_service_tier = openai_service_tier
         self.bedrock_service_tier = bedrock_service_tier
         self.gemini_service_tier = gemini_service_tier
+        self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
         # Gemini safety settings (instance default; can be overridden per-request via context var)
         self.gemini_safety_settings = gemini_safety_settings
         # Gemini prompt caching: when True, retain extraction (and any future
@@ -742,6 +763,7 @@ def __init__(
             gemini_safety_settings=self.gemini_safety_settings,
             prompt_cache_enabled=self.prompt_cache_enabled,
             litellmrouter_config=router_config,
+            ollama_num_ctx=self.ollama_num_ctx,
             timeout=self.timeout,
         )
 
@@ -1260,6 +1282,7 @@ def from_env(cls) -> "LLMProvider":
             ENV_LLM_GROQ_SERVICE_TIER,
             ENV_LLM_LITELLMROUTER_CONFIG,
             ENV_LLM_MODEL,
+            ENV_LLM_OLLAMA_NUM_CTX,
             ENV_LLM_OPENAI_SERVICE_TIER,
             ENV_LLM_PROMPT_CACHE_ENABLED,
             ENV_LLM_PROVIDER,
@@ -1270,6 +1293,7 @@ def from_env(cls) -> "LLMProvider":
             ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY,
             _get_default_model_for_provider,
             _parse_llm_router_config,
+            _parse_optional_positive_int,
             parse_gemini_service_tier,
         )
 
@@ -1314,6 +1338,7 @@ def from_env(cls) -> "LLMProvider":
             ),
             gemini_safety_settings=json.loads(os.getenv(ENV_LLM_GEMINI_SAFETY_SETTINGS, "null")),
             prompt_cache_enabled=prompt_cache_enabled,
+            ollama_num_ctx=_parse_optional_positive_int(ENV_LLM_OLLAMA_NUM_CTX, os.getenv(ENV_LLM_OLLAMA_NUM_CTX)),
             litellmrouter_config=_parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG),
             vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or None,
             vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION) or None,

hindsight-api-slim/hindsight_api/engine/memory_engine.py

@@ -438,6 +438,7 @@ def _member_to_llm(member: "LLMMemberConfig", config: HindsightConfig, defaults:
         reasoning_effort=member.reasoning_effort or config.llm_reasoning_effort,
         extra_body=member.extra_body,
         default_headers=member.default_headers or config.llm_default_headers,
+        ollama_num_ctx=config.llm_ollama_num_ctx,
         bedrock_service_tier=member.bedrock_service_tier,
         gemini_service_tier=member.gemini_service_tier or config.llm_gemini_service_tier,
         gemini_safety_settings=_get_raw_config().llm_gemini_safety_settings,
@@ -1085,6 +1086,7 @@ def pick(field: str) -> Any:
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1129,6 +1131,7 @@ def pick(field: str) -> Any:
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.retain_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1167,6 +1170,7 @@ def pick(field: str) -> Any:
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.reflect_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,
@@ -1205,6 +1209,7 @@ def pick(field: str) -> Any:
             reasoning_effort=config.llm_reasoning_effort,
             extra_body=config.llm_extra_body,
             default_headers=config.llm_default_headers,
+            ollama_num_ctx=config.llm_ollama_num_ctx,
             litellmrouter_config=config.consolidation_llm_litellmrouter_config or config.llm_litellmrouter_config,
             bedrock_service_tier=config.llm_bedrock_service_tier,
             gemini_service_tier=config.llm_gemini_service_tier,

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -48,6 +48,18 @@
 DEFAULT_LLM_SEED = 4242
 JSON_MODE_USER_HINT = "Return valid json only."
 
+
+def _validate_ollama_num_ctx(value: Any) -> int | None:
+    """Validate a native Ollama context-window override."""
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}")
+    if value < 1:
+        raise ValueError(f"ollama_num_ctx must be >= 1, got {value}")
+    return value
+
+
 # Self-hosted OpenAI-compatible servers that advertise tool_choice="required"
 # but silently ignore it: instead of forcing a tool call they return
 # finish_reason "stop"/"tool_calls" with an EMPTY tool_calls array and no error.
@@ -480,6 +492,8 @@ def __init__(
         timeout: float | None = None,
         groq_service_tier: str | None = None,
         extra_body: dict[str, Any] | None = None,
+        *,
+        ollama_num_ctx: int | None = None,
         **kwargs: Any,
     ):
         """
@@ -494,6 +508,8 @@ def __init__(
             timeout: Request timeout in seconds (uses env var or 120s default).
             groq_service_tier: Groq service tier ("on_demand", "flex", "auto").
             extra_body: Extra body params merged into every API call.
+            ollama_num_ctx: Native Ollama context window override. None lets Ollama use
+                the model/server default.
             **kwargs: Additional provider-specific parameters.
         """
         super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs)
@@ -574,6 +590,7 @@ def __init__(
         # Service tier configuration (from config, not env vars)
         self.groq_service_tier = groq_service_tier
         self.openai_service_tier = kwargs.get("openai_service_tier")
+        self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx)
         # User-configured extra body params (merged into every API call)
         self._config_extra_body = extra_body or {}
 
@@ -1389,9 +1406,10 @@ async def _call_ollama_native(
 
         # Add optional parameters with optimized defaults for Ollama
         options: dict[str, Any] = {
-            "num_ctx": 16384,  # 16k context window for larger prompts
             "num_batch": 512,  # Optimal batch size for prompt processing
         }
+        if self.ollama_num_ctx is not None:
+            options["num_ctx"] = self.ollama_num_ctx
         if max_completion_tokens:
             options["num_predict"] = max_completion_tokens
         if temperature is not None:

hindsight-api-slim/tests/test_config_validation.py

@@ -122,6 +122,53 @@ def test_retain_structured_chunk_size_reads_from_env():
     assert config.retain_structured_chunk_size == 9000
 
 
+def test_llm_ollama_num_ctx_defaults_to_none(monkeypatch):
+    """Unset Ollama num_ctx override lets Ollama use its model/server default."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.delenv(ENV_LLM_OLLAMA_NUM_CTX, raising=False)
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.llm_ollama_num_ctx is None
+
+
+def test_llm_ollama_num_ctx_keeps_direct_construction_default():
+    """Direct HindsightConfig construction should not require the new field."""
+    from dataclasses import fields
+
+    from hindsight_api.config import HindsightConfig
+
+    config_field = next(item for item in fields(HindsightConfig) if item.name == "llm_ollama_num_ctx")
+
+    assert config_field.default is None
+    assert config_field.kw_only
+
+
+def test_llm_ollama_num_ctx_reads_positive_int(monkeypatch):
+    """The native Ollama context override is parsed as a positive integer."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "65536")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    config = HindsightConfig.from_env()
+
+    assert config.llm_ollama_num_ctx == 65536
+
+
+def test_llm_ollama_num_ctx_rejects_non_positive_values(monkeypatch):
+    """Zero would be accepted by neither Ollama nor downstream range logic."""
+    from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig
+
+    monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "0")
+    monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock")
+
+    with pytest.raises(ValueError, match=ENV_LLM_OLLAMA_NUM_CTX):
+        HindsightConfig.from_env()
+
+
 def test_retain_structured_chunk_size_can_be_less_than_chunk_size():
     """Structured-chunk cap can be smaller than the retain chunk target."""
     from hindsight_api.config import HindsightConfig