Skip to content

Commit d3860b2

Browse files
edwinjosechittilappillyedwinjosechittilappilly
committed
Add custom metadata ingestion and filtering
1 parent 0f6c01f commit d3860b2

50 files changed

Lines changed: 1967 additions & 149 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎alembic/versions/0008_metadata_fields.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
"""custom metadata field catalog
2+
3+
Revision ID: 0008_metadata_fields
4+
Revises: 0007_add_knowledge_delete_anonymous
5+
Create Date: 2026-07-16 00:00:00.000000
6+
"""
7+
8+
from collections.abc import Sequence
9+
10+
import sqlalchemy as sa
11+
12+
from alembic import op
13+
14+
revision: str = "0008_metadata_fields"
15+
down_revision: str | Sequence[str] | None = "0007_add_knowledge_delete_anonymous"
16+
branch_labels: str | Sequence[str] | None = None
17+
depends_on: str | Sequence[str] | None = None
18+
19+
20+
def upgrade() -> None:
21+
op.create_table(
22+
"metadata_fields",
23+
sa.Column("key", sa.String(length=64), nullable=False),
24+
sa.Column("metadata_type", sa.String(length=16), nullable=False),
25+
sa.Column("created_at", sa.DateTime(), nullable=False),
26+
sa.Column("updated_at", sa.DateTime(), nullable=False),
27+
sa.PrimaryKeyConstraint("key", name="pk_metadata_fields"),
28+
)
29+
op.create_index(
30+
"ix_metadata_fields_metadata_type",
31+
"metadata_fields",
32+
["metadata_type"],
33+
unique=False,
34+
)
35+
36+
37+
def downgrade() -> None:
38+
op.drop_index("ix_metadata_fields_metadata_type", table_name="metadata_fields")
39+
op.drop_table("metadata_fields")

‎frontend/app/api/queries/useGetSearchQuery.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
useQuery,
44
useQueryClient,
55
} from "@tanstack/react-query";
6+
import type { MetadataGroup } from "@/components/metadata-filter-builder";
67
import type { ParsedQueryData } from "@/contexts/knowledge-filter-context";
78
import { SEARCH_CONSTANTS } from "@/lib/constants";
89
import { buildSearchPayloadFilters } from "@/lib/filter-normalization";
@@ -16,6 +17,7 @@ export interface SearchPayload {
1617
document_types?: string[];
1718
owners?: string[];
1819
connector_types?: string[];
20+
metadata?: MetadataGroup;
1921
};
2022
}
2123

@@ -39,6 +41,7 @@ export interface ChunkResult {
3941
index?: number;
4042
allowed_users?: string[];
4143
allowed_groups?: string[];
44+
metadata?: Record<string, { type: string; value: unknown }>;
4245
}
4346

4447
export interface File {

‎frontend/app/chat/_types/types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ export interface SelectedFilters {
8888
document_types: string[];
8989
owners: string[];
9090
connector_types: string[];
91+
metadata?: import("@/components/metadata-filter-builder").MetadataGroup;
9192
}
9293

9394
export interface KnowledgeFilterData {
@@ -109,6 +110,7 @@ export interface RequestBody {
109110
document_types?: string[];
110111
owners?: string[];
111112
connector_types?: string[];
113+
metadata?: import("@/components/metadata-filter-builder").MetadataGroup;
112114
};
113115
filter_id?: string;
114116
limit?: number;

‎frontend/components/cloud-picker/custom-metadata-editor.tsx‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
"use client";
2+
3+
import { Plus, Trash2 } from "lucide-react";
4+
import { useEffect, useMemo, useState } from "react";
5+
import { Button } from "@/components/ui/button";
6+
import { Input } from "@/components/ui/input";
7+
import {
8+
Select,
9+
SelectContent,
10+
SelectItem,
11+
SelectTrigger,
12+
SelectValue,
13+
} from "@/components/ui/select";
14+
import type { CustomMetadataEntry, CustomMetadataType } from "./types";
15+
16+
interface MetadataFieldSuggestion {
17+
key: string;
18+
type: CustomMetadataType;
19+
}
20+
21+
interface Props {
22+
value: CustomMetadataEntry[];
23+
onChange: (value: CustomMetadataEntry[]) => void;
24+
}
25+
26+
const defaultValue = (type: CustomMetadataType): string | number | boolean => {
27+
if (type === "number") return 0;
28+
if (type === "boolean") return false;
29+
return "";
30+
};
31+
32+
export function CustomMetadataEditor({ value, onChange }: Props) {
33+
const [suggestions, setSuggestions] = useState<MetadataFieldSuggestion[]>([]);
34+
35+
useEffect(() => {
36+
let active = true;
37+
fetch("/api/metadata/fields")
38+
.then((response) => (response.ok ? response.json() : { fields: [] }))
39+
.then((result) => {
40+
if (active) setSuggestions(result.fields ?? []);
41+
})
42+
.catch(() => undefined);
43+
return () => {
44+
active = false;
45+
};
46+
}, []);
47+
48+
const fieldTypes = useMemo(
49+
() => new Map(suggestions.map((field) => [field.key, field.type])),
50+
[suggestions],
51+
);
52+
53+
const update = (index: number, patch: Partial<CustomMetadataEntry>) => {
54+
const next = [...value];
55+
next[index] = { ...next[index], ...patch };
56+
onChange(next);
57+
};
58+
59+
return (
60+
<div className="mt-6 space-y-3 border-t pt-4">
61+
<div>
62+
<div className="text-sm font-semibold">Custom metadata</div>
63+
<div className="text-sm text-muted-foreground">
64+
Applied to every document selected for this ingest.
65+
</div>
66+
</div>
67+
<datalist id="custom-metadata-keys">
68+
{suggestions.map((field) => (
69+
<option key={field.key} value={field.key} />
70+
))}
71+
</datalist>
72+
{value.map((entry, index) => {
73+
const knownType = fieldTypes.get(entry.key.trim().toLowerCase());
74+
return (
75+
<div
76+
className="grid grid-cols-[minmax(0,1fr)_130px_minmax(0,1fr)_36px] gap-2"
77+
key={`${index}-${entry.key}`}
78+
>
79+
<Input
80+
aria-label="Metadata key"
81+
list="custom-metadata-keys"
82+
placeholder="supplier"
83+
value={entry.key}
84+
onChange={(event) => {
85+
const key = event.target.value.toLowerCase();
86+
const suggestedType = fieldTypes.get(key);
87+
update(index, {
88+
key,
89+
...(suggestedType && suggestedType !== entry.type
90+
? {
91+
type: suggestedType,
92+
value: defaultValue(suggestedType),
93+
}
94+
: {}),
95+
});
96+
}}
97+
/>
98+
<Select
99+
disabled={!!knownType}
100+
value={knownType ?? entry.type}
101+
onValueChange={(type: CustomMetadataType) =>
102+
update(index, { type, value: defaultValue(type) })
103+
}
104+
>
105+
<SelectTrigger aria-label="Metadata type">
106+
<SelectValue />
107+
</SelectTrigger>
108+
<SelectContent>
109+
<SelectItem value="string">Text</SelectItem>
110+
<SelectItem value="number">Number</SelectItem>
111+
<SelectItem value="date">Date</SelectItem>
112+
<SelectItem value="boolean">Boolean</SelectItem>
113+
</SelectContent>
114+
</Select>
115+
{entry.type === "boolean" ? (
116+
<Select
117+
value={String(entry.value)}
118+
onValueChange={(next) =>
119+
update(index, { value: next === "true" })
120+
}
121+
>
122+
<SelectTrigger aria-label="Metadata value">
123+
<SelectValue />
124+
</SelectTrigger>
125+
<SelectContent>
126+
<SelectItem value="true">True</SelectItem>
127+
<SelectItem value="false">False</SelectItem>
128+
</SelectContent>
129+
</Select>
130+
) : (
131+
<Input
132+
aria-label="Metadata value"
133+
type={
134+
entry.type === "date"
135+
? "date"
136+
: entry.type === "number"
137+
? "number"
138+
: "text"
139+
}
140+
value={String(entry.value)}
141+
onChange={(event) =>
142+
update(index, {
143+
value:
144+
entry.type === "number"
145+
? Number(event.target.value)
146+
: event.target.value,
147+
})
148+
}
149+
/>
150+
)}
151+
<Button
152+
aria-label="Remove metadata"
153+
onClick={() =>
154+
onChange(value.filter((_, itemIndex) => itemIndex !== index))
155+
}
156+
size="icon"
157+
type="button"
158+
variant="ghost"
159+
>
160+
<Trash2 className="h-4 w-4" />
161+
</Button>
162+
</div>
163+
);
164+
})}
165+
<Button
166+
onClick={() =>
167+
onChange([...value, { key: "", type: "string", value: "" }])
168+
}
169+
size="sm"
170+
type="button"
171+
variant="outline"
172+
>
173+
<Plus className="h-4 w-4" /> Add metadata
174+
</Button>
175+
</div>
176+
);
177+
}

‎frontend/components/cloud-picker/ingest-settings.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import {
3535
} from "@/components/ui/tooltip";
3636
import { useAuth } from "@/contexts/auth-context";
3737
import { knowledgeToIngestSettings } from "@/lib/ingest-settings-knowledge";
38+
import { CustomMetadataEditor } from "./custom-metadata-editor";
3839
import type { IngestSettings as IngestSettingsType } from "./types";
3940

4041
interface IngestSettingsProps {
@@ -285,6 +286,12 @@ export const IngestSettings = ({
285286
</div>
286287
</div>
287288
)}
289+
{showAdvancedSettings && (
290+
<CustomMetadataEditor
291+
value={currentSettings.metadata ?? []}
292+
onChange={(metadata) => handleSettingsChange({ metadata })}
293+
/>
294+
)}
288295

289296
<div className={showAdvancedSettings ? "" : "mt-6"}>
290297
{showShared && (

‎frontend/components/cloud-picker/types.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,17 @@ export interface IngestSettings {
115115
embeddingModel: string;
116116
/** When true, index without an owner so all users in the instance can retrieve the document. COS only. */
117117
shared?: boolean;
118+
/** Typed custom metadata applied to every document in the ingest batch. */
119+
metadata?: CustomMetadataEntry[];
120+
}
121+
122+
export type CustomMetadataType = "string" | "number" | "date" | "boolean";
123+
export type CustomMetadataValue = string | number | boolean;
124+
125+
export interface CustomMetadataEntry {
126+
key: string;
127+
type: CustomMetadataType;
128+
value: CustomMetadataValue;
118129
}
119130

120131
/** Inline error message if chunk settings are invalid; otherwise null. */

0 commit comments

Comments
 (0)