diff --git a/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py b/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py index d42559e2b9..ab6ebe894f 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/orchestrator.py @@ -549,7 +549,11 @@ async def _extract_and_embed( embeddings = await embedding_processing.generate_embeddings_batch(embeddings_model, augmented_texts) log_buffer.append(f" Generate embeddings: {len(embeddings)} embeddings in {time.time() - step_start:.3f}s") - processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)] + processed_facts = [ + pf + for ef, emb in zip(extracted_facts, embeddings) + if (pf := ProcessedFact.from_extracted_fact(ef, emb)) is not None + ] return extracted_facts, processed_facts, chunks, usage diff --git a/hindsight-api-slim/hindsight_api/engine/retain/types.py b/hindsight-api-slim/hindsight_api/engine/retain/types.py index 85982de7dd..03ef684c10 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/types.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/types.py @@ -5,11 +5,14 @@ from content input to fact storage. """ +import logging from dataclasses import dataclass, field from datetime import datetime from typing import Literal, TypedDict from uuid import UUID +logger = logging.getLogger(__name__) + class RetainContentDict(TypedDict, total=False): """Type definition for content items in retain_batch_async. @@ -187,10 +190,47 @@ def is_duplicate(self) -> bool: """Check if this fact was marked as a duplicate.""" return self.unit_id is None + @staticmethod + def _is_degenerate_text(text: str) -> bool: + """Check if fact text has zero information content. + + Rejects empty strings, whitespace-only, single punctuation marks, + and common LLM hallucination patterns that carry no semantic meaning. + """ + stripped = (text or "").strip() + if not stripped: + return True + # Single or repeated punctuation patterns with no semantic content + degenerate_patterns = { + "...", + "…", + "-", + "--", + "---", + ".", + "..", + "•", + "·", + "*", + "**", + "***", + "_,_", + "_, _, _", + } + if stripped in degenerate_patterns: + return True + # Strings composed entirely of punctuation and whitespace + if all(c in ".,;:!?-–—…\"'`´ \t\n\r" for c in stripped): + return True + # Very short text (<= 2 chars) that is only punctuation + if len(stripped) <= 2 and all(not c.isalnum() for c in stripped): + return True + return False + @staticmethod def from_extracted_fact( extracted_fact: "ExtractedFact", embedding: list[float], chunk_id: str | None = None - ) -> "ProcessedFact": + ) -> "ProcessedFact | None": """ Create ProcessedFact from ExtractedFact. @@ -200,8 +240,17 @@ def from_extracted_fact( chunk_id: Optional chunk ID Returns: - ProcessedFact ready for storage + ProcessedFact ready for storage, or None if the fact text is degenerate + (zero information content — punctuation-only, empty, etc.) """ + fact_text = extracted_fact.fact_text or "" + if ProcessedFact._is_degenerate_text(fact_text): + logger.warning( + f"Rejected degenerate fact text: type={extracted_fact.fact_type}, " + f"text={fact_text[:80]!r}, entities={extracted_fact.entities}" + ) + return None + # Use occurred dates only if explicitly provided by LLM occurred_start = extracted_fact.occurred_start occurred_end = extracted_fact.occurred_end @@ -211,7 +260,7 @@ def from_extracted_fact( entities = [EntityRef(name=name) for name in extracted_fact.entities] return ProcessedFact( - fact_text=extracted_fact.fact_text, + fact_text=fact_text, fact_type=extracted_fact.fact_type, embedding=embedding, occurred_start=occurred_start, diff --git a/hindsight-api-slim/hindsight_api/engine/transfer/importer.py b/hindsight-api-slim/hindsight_api/engine/transfer/importer.py index 92f69ac1bc..cd00a64e83 100644 --- a/hindsight-api-slim/hindsight_api/engine/transfer/importer.py +++ b/hindsight-api-slim/hindsight_api/engine/transfer/importer.py @@ -481,7 +481,11 @@ async def _import_one_document( if extracted_facts: augmented = embedding_processing.augment_texts_with_dates(extracted_facts, format_date_fn) embeddings = await embedding_processing.generate_embeddings_batch(embeddings_model, augmented) - processed_facts = [ProcessedFact.from_extracted_fact(ef, emb) for ef, emb in zip(extracted_facts, embeddings)] + processed_facts = [ + pf + for ef, emb in zip(extracted_facts, embeddings) + if (pf := ProcessedFact.from_extracted_fact(ef, emb)) is not None + ] contents = [RetainContent(content=document.original_text or "")] chunk_meta = [