feat: Visual parser spike

frontend/app/api/mutations/useSyncConnector.ts

@@ -77,6 +77,9 @@ const syncConnector = async ({
     bucket_filter?: string[];
     /** When true, replace any indexed document with the same filename. */
     replace_duplicates?: boolean;
+    /** When true (OSS only), ingest in preview mode so the layout/visual
+     * parser preview is cached per file. */
+    preview?: boolean;
     /** When true (COS only), index without an owner so all users in the instance can retrieve the document. */
     shared?: boolean;
   };

frontend/app/api/queries/useIngestPreviewQuery.ts

@@ -0,0 +1,137 @@
+import {
+  type UseQueryOptions,
+  useQuery,
+  useQueryClient,
+} from "@tanstack/react-query";
+
+interface DoclingPreviewStats {
+  page_count: number;
+  text_count: number;
+  table_count: number;
+  picture_count: number;
+}
+
+interface DoclingPreviewResponse {
+  task_id: string;
+  document: Record<string, unknown>;
+  stats: DoclingPreviewStats;
+  expires_at: number;
+  document_id?: string;
+  file_path?: string;
+  filename?: string;
+}
+
+interface IndexProofChunk {
+  chunk_id: string;
+  page?: number;
+  text_preview: string;
+  char_count: number;
+}
+
+interface IndexProofResponse {
+  task_id: string;
+  ready: boolean;
+  phase?: string;
+  chunk_count: number;
+  embedding_model?: string;
+  embedding_dimensions?: number;
+  chunks: IndexProofChunk[];
+  document_id?: string;
+}
+
+// Stop polling after this many attempts so a file that never produces a preview
+// (un-cacheable format, expired/failed task, cap reached) can't poll forever.
+// At 1.5s/poll this is ~90s, comfortably longer than a single-file parse.
+const MAX_PREVIEW_POLLS = 60;
+const PREVIEW_POLL_INTERVAL_MS = 1500;
+
+function withFileParam(path: string, filePath?: string | null): string {
+  return filePath ? `${path}?file=${encodeURIComponent(filePath)}` : path;
+}
+
+async function fetchDoclingPreview(
+  taskId: string,
+  filePath?: string | null,
+): Promise<DoclingPreviewResponse | null> {
+  const response = await fetch(
+    withFileParam(`/api/ingest/preview/${taskId}/docling`, filePath),
+  );
+  if (response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`Parse preview unavailable (${response.status})`);
+  }
+  return response.json();
+}
+
+async function fetchIndexProof(
+  taskId: string,
+  filePath?: string | null,
+): Promise<IndexProofResponse> {
+  const response = await fetch(
+    withFileParam(`/api/ingest/preview/${taskId}/index-proof`, filePath),
+  );
+  if (!response.ok) {
+    throw new Error(`Index proof unavailable (${response.status})`);
+  }
+  return response.json();
+}
+
+export function useDoclingPreviewQuery(
+  taskId: string | null,
+  enabled: boolean,
+  filePath?: string | null,
+  options?: Omit<
+    UseQueryOptions<DoclingPreviewResponse | null>,
+    "queryKey" | "queryFn" | "enabled"
+  >,
+) {
+  const queryClient = useQueryClient();
+
+  return useQuery(
+    {
+      queryKey: ["ingest-preview", "docling", taskId, filePath ?? null],
+      queryFn: () => fetchDoclingPreview(taskId as string, filePath),
+      enabled: enabled && !!taskId,
+      refetchInterval: (query) => {
+        if (query.state.data?.document) return false;
+        if (!(enabled && taskId)) return false;
+        if (query.state.dataUpdateCount >= MAX_PREVIEW_POLLS) return false;
+        return PREVIEW_POLL_INTERVAL_MS;
+      },
+      retry: 1,
+      refetchOnWindowFocus: false,
+      ...options,
+    },
+    queryClient,
+  );
+}
+
+export function useIndexProofQuery(
+  taskId: string | null,
+  enabled: boolean,
+  filePath?: string | null,
+  options?: Omit<
+    UseQueryOptions<IndexProofResponse>,
+    "queryKey" | "queryFn" | "enabled"
+  >,
+) {
+  const queryClient = useQueryClient();
+
+  return useQuery(
+    {
+      queryKey: ["ingest-preview", "index-proof", taskId, filePath ?? null],
+      queryFn: () => fetchIndexProof(taskId as string, filePath),
+      enabled: enabled && !!taskId,
+      refetchInterval: (query) => {
+        if (query.state.data?.ready) return false;
+        if (query.state.dataUpdateCount >= MAX_PREVIEW_POLLS) return false;
+        return PREVIEW_POLL_INTERVAL_MS;
+      },
+      refetchOnWindowFocus: false,
+      ...options,
+    },
+    queryClient,
+  );
+}

frontend/app/onboarding/_components/onboarding-upload.tsx

@@ -7,22 +7,28 @@ import { useUpdateOnboardingStateMutation } from "@/app/api/mutations/useUpdateO
 import { useGetNudgesQuery } from "@/app/api/queries/useGetNudgesQuery";
 import { useGetTasksQuery } from "@/app/api/queries/useGetTasksQuery";
 import { AnimatedProviderSteps } from "@/app/onboarding/_components/animated-provider-steps";
+import { IngestPreviewPanel } from "@/components/ingest-review";
 import { SUPPORTED_EXTENSIONS } from "@/components/knowledge-dropdown";
 import { Button } from "@/components/ui/button";
+import { useAuth } from "@/contexts/auth-context";
 import { trackButton } from "@/lib/analytics";
+import { isIngestPreviewEnabled } from "@/lib/ingest-preview";
 import { uploadFile } from "@/lib/upload-utils";
 
 interface OnboardingUploadProps {
   onComplete: () => void;
 }
 
 const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
+  const { runMode } = useAuth();
+  const ingestPreviewEnabled = isIngestPreviewEnabled(runMode);
   const fileInputRef = useRef<HTMLInputElement>(null);
   const completeTimeoutRef = useRef<ReturnType<typeof setTimeout>>(undefined);
   const [isUploading, setIsUploading] = useState(false);
   const [currentStep, setCurrentStep] = useState<number | null>(null);
   const [uploadedFilename, setUploadedFilename] = useState<string | null>(null);
   const [uploadedTaskId, setUploadedTaskId] = useState<string | null>(null);
+  const [previewFile, setPreviewFile] = useState<File | null>(null);
   const [shouldCreateFilter, setShouldCreateFilter] = useState(false);
   const [isCreatingFilter, setIsCreatingFilter] = useState(false);
   const [error, setError] = useState<string | null>(null);
@@ -32,9 +38,9 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
 
   const STEP_LIST = [
     "Uploading your document",
-    "Generating embeddings",
-    "Ingesting document",
-    "Processing your document",
+    "Reading layout & structure",
+    "Creating searchable chunks",
+    "Indexing for search",
   ];
 
   // Query tasks to track completion
@@ -62,6 +68,23 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
       return;
     }
 
+    const filesArray = matchingTask.files
+      ? (Object.values(matchingTask.files) as {
+          status: string;
+          error?: string;
+          phase?: string;
+        }[])
+      : [];
+    const filePhase = filesArray[0]?.phase;
+
+    if (filePhase === "docling" && currentStep < 1) {
+      setCurrentStep(1);
+    } else if (filePhase === "langflow" && currentStep < 2) {
+      setCurrentStep(2);
+    } else if (filePhase === "complete" && currentStep < 3) {
+      setCurrentStep(3);
+    }
+
     // Check if the matching task is still active (pending, running, or processing)
     const isTaskActive =
       matchingTask.status === "pending" ||
@@ -73,12 +96,6 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
       matchingTask.status === "failed" || matchingTask.status === "error";
 
     // Check if any file inside the task failed
-    const filesArray = matchingTask.files
-      ? (Object.values(matchingTask.files) as {
-          status: string;
-          error?: string;
-        }[])
-      : [];
     const hasFailedFile = filesArray.some(
       (file) => file.status === "failed" || file.status === "error",
     );
@@ -214,7 +231,16 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
     setError(null);
     try {
       setCurrentStep(0);
-      const result = await uploadFile(file, true, true); // Pass createFilter=true
+      if (ingestPreviewEnabled) {
+        setPreviewFile(file);
+      }
+      const result = await uploadFile(
+        file,
+        true,
+        true,
+        undefined,
+        ingestPreviewEnabled,
+      );
       console.log("Document upload task started successfully");
 
       // Store task ID to track the specific upload task
@@ -257,6 +283,7 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
       // Reset on error
       setCurrentStep(null);
       setUploadedTaskId(null);
+      setPreviewFile(null);
     } finally {
       setIsUploading(false);
     }
@@ -339,6 +366,14 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
             isCompleted={false}
             steps={STEP_LIST}
           />
+          {ingestPreviewEnabled &&
+            currentStep !== null &&
+            (previewFile || uploadedTaskId) && (
+              <IngestPreviewPanel
+                taskId={uploadedTaskId}
+                previewFile={previewFile}
+              />
+            )}
         </motion.div>
       )}
     </AnimatePresence>

frontend/app/upload/[provider]/page.tsx

@@ -10,16 +10,19 @@ import { useGetConnectorTokenQuery } from "@/app/api/queries/useGetConnectorToke
 import { type CloudFile, UnifiedCloudPicker } from "@/components/cloud-picker";
 import { getIngestChunkSettingsError } from "@/components/cloud-picker/types";
 import { DuplicateHandlingDialog } from "@/components/duplicate-handling-dialog";
+import { IngestReviewDialog } from "@/components/ingest-review";
 import { Button } from "@/components/ui/button";
 import {
   Tooltip,
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import { useAuth } from "@/contexts/auth-context";
 import { useTask } from "@/contexts/task-context";
 import { useSessionIngestSettings } from "@/hooks/useSessionIngestSettings";
 import { trackProcessFailure, trackStartProcess } from "@/lib/analytics";
 import { getConnectorDescriptor } from "@/lib/connectors/registry";
+import { isIngestPreviewEnabled } from "@/lib/ingest-preview";
 
 interface ConnectorDuplicateCheckResponse {
   duplicate_names?: string[];
@@ -34,6 +37,8 @@ export default function UploadProviderPage() {
   const router = useRouter();
   const provider = params.provider as string;
   const { addTask } = useTask();
+  const { runMode } = useAuth();
+  const ingestPreviewEnabled = isIngestPreviewEnabled(runMode);
 
   const {
     data: connectors = [],
@@ -77,6 +82,9 @@ export default function UploadProviderPage() {
     duplicateCount: number;
   } | null>(null);
   const isOverwriteConfirmedRef = useRef(false);
+  const [previewOpen, setPreviewOpen] = useState(false);
+  const [previewTaskId, setPreviewTaskId] = useState<string | null>(null);
+  const [previewFilename, setPreviewFilename] = useState<string>("");
 
   const accessToken = tokenData?.access_token || null;
   const isLoading =
@@ -122,6 +130,7 @@ export default function UploadProviderPage() {
           })),
           settings: ingestSettings,
           replace_duplicates: replaceDuplicates,
+          preview: ingestPreviewEnabled,
         },
       },
       {
@@ -132,7 +141,17 @@ export default function UploadProviderPage() {
               connectorType: connector.type,
               source: "connector",
             });
-            router.push("/knowledge");
+            if (ingestPreviewEnabled) {
+              // Show the live parse/index preview for the connector files.
+              // Navigation to /knowledge happens when the dialog is closed.
+              setPreviewTaskId(taskIds[0]);
+              setPreviewFilename(
+                files.length === 1 ? files[0].name : `${files.length} files`,
+              );
+              setPreviewOpen(true);
+            } else {
+              router.push("/knowledge");
+            }
           }
         },
         onError: (err) => {
@@ -455,6 +474,19 @@ export default function UploadProviderPage() {
         duplicateNames={pendingSync?.duplicateNames}
         duplicateCount={pendingSync?.duplicateCount}
       />
+
+      <IngestReviewDialog
+        open={ingestPreviewEnabled && previewOpen}
+        onOpenChange={(open) => {
+          setPreviewOpen(open);
+          if (!open) {
+            setPreviewTaskId(null);
+            router.push("/knowledge");
+          }
+        }}
+        taskId={previewTaskId}
+        filename={previewFilename}
+      />
     </>
   );
 }