Merge remote-tracking branch 'origin/main' into feat/langflow_ingestion_disable

Author: lucaseduoli

.gitignore

@@ -24,7 +24,6 @@ wheels/
 !src/tui/_assets/flows/*.json
 !src/tui/_assets/flows/components/*.json
 !frontend/*.json
-!**/.claude-plugin/*.json
 .DS_Store
 
 /config/
@@ -39,20 +38,16 @@ langflow-data/
 node_modules
 
 # AI Tools
-# Per-contributor Claude Code state is ignored, but shared skills and
-# plugin manifests (.claude/skills, .claude-plugin) are tracked.
-/.claude/*
-!/.claude/skills/
+/.claude
+/CLAUDE.md
 /.gemini
 /GEMINI.md
 /.bob
+/AGENTS.md
 documents/warmup_ocr.pdf
 documents/openrag-documentation.pdf
 documents/ibm_anthropic.pdf
 documents/docling.pdf
 /opensearch-data-new-lf
 /opensearch-data2
-data/openrag.db
-data/openrag.db
 *.db
-/data

Makefile

@@ -534,6 +534,7 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove image
 	echo "  - Remove all volumes"; \
 	echo "  - Delete langflow-data directory"; \
 	echo "  - Delete config directory"; \
+	echo "  - Delete data directory (database and session configs)"; \
 	echo "  - Delete JWT keys (private_key.pem, public_key.pem)"; \
 	echo "  - Remove OpenRAG images"; \
 	echo ""; \
@@ -559,6 +560,16 @@ factory-reset: ## Complete reset (stop, remove volumes, clear data, remove image
 		rm -rf config; \
 		echo "$(PURPLE)config removed$(NC)"; \
 	fi; \
+	if [ -d "data" ]; then \
+		echo "Removing data..."; \
+		rm -rf data; \
+		echo "$(PURPLE)data removed$(NC)"; \
+	fi; \
+	if [ -n "$$OPENRAG_DATA_PATH" ] && [ -d "$$OPENRAG_DATA_PATH" ]; then \
+		echo "Removing $$OPENRAG_DATA_PATH..."; \
+		rm -rf "$$OPENRAG_DATA_PATH"; \
+		echo "$(PURPLE)$$OPENRAG_DATA_PATH removed$(NC)"; \
+	fi; \
 	if [ -f "keys/private_key.pem" ] || [ -f "keys/public_key.pem" ]; then \
 		echo "Removing JWT keys..."; \
 		rm -f keys/private_key.pem keys/public_key.pem 2>/dev/null || \

frontend/app/api/mutations/useSyncConnector.ts

@@ -13,6 +13,27 @@ interface SyncResponse {
   errors?: Array<{ connector_type: string; error: string }> | null;
 }
 
+export interface OrphanFile {
+  document_id: string;
+  filename: string;
+}
+
+export interface SyncPreviewResponse {
+  connector_type: string;
+  synced_count: number;
+  orphans: OrphanFile[];
+  /** False when strict gating aborted orphan detection (e.g. an active
+   * connection was unauthenticated). UI should reflect that deletions
+   * cannot be predicted in that case. */
+  orphans_available: boolean;
+}
+
+export interface SyncAllPreviewResponse {
+  orphans_by_type: Record<string, OrphanFile[]>;
+  synced_count_by_type: Record<string, number>;
+  orphans_available_by_type: Record<string, boolean>;
+}
+
 // Sync all cloud connectors
 const syncAllConnectors = async (): Promise<SyncResponse> => {
   const response = await fetch("/api/connectors/sync-all", {
@@ -92,3 +113,44 @@ export const useSyncConnector = () => {
     },
   });
 };
+
+const syncConnectorPreview = async (
+  connectorType: string,
+): Promise<SyncPreviewResponse> => {
+  const response = await fetch(
+    `/api/connectors/${connectorType}/sync-preview`,
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+    },
+  );
+
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({}));
+    throw new Error(
+      error.error || `Failed to preview sync for ${connectorType}`,
+    );
+  }
+
+  return response.json();
+};
+
+const syncAllConnectorsPreview = async (): Promise<SyncAllPreviewResponse> => {
+  const response = await fetch("/api/connectors/sync-all-preview", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+  });
+
+  if (!response.ok) {
+    const error = await response.json().catch(() => ({}));
+    throw new Error(error.error || "Failed to preview sync");
+  }
+
+  return response.json();
+};
+
+export const useSyncConnectorPreview = () =>
+  useMutation({ mutationFn: syncConnectorPreview });
+
+export const useSyncAllConnectorsPreview = () =>
+  useMutation({ mutationFn: syncAllConnectorsPreview });

frontend/app/api/queries/useGetSearchQuery.ts

@@ -86,7 +86,7 @@ export const useGetSearchQuery = (
   query: string,
   queryData?: ParsedQueryData | null,
   options?: Omit<
-    UseQueryOptions<SearchResult, Error, SearchResult>,
+    UseQueryOptions<SearchResult, Error, SearchResult, any[]>,
     "queryKey" | "queryFn"
   >,
 ) => {

frontend/app/knowledge/page.tsx

@@ -55,9 +55,14 @@ import GoogleDriveIcon from "../../components/icons/google-drive-logo";
 import IBMCOSIcon from "../../components/icons/ibm-cos-icon";
 import OneDriveIcon from "../../components/icons/one-drive-logo";
 import SharePointIcon from "../../components/icons/share-point-logo";
+import { SyncConfirmDialog } from "../../components/sync-confirm-dialog";
 import { useDeleteDocument } from "../api/mutations/useDeleteDocument";
 import { useRefreshOpenragDocs } from "../api/mutations/useRefreshOpenragDocs";
-import { useSyncAllConnectors } from "../api/mutations/useSyncConnector";
+import {
+  type SyncAllPreviewResponse,
+  useSyncAllConnectors,
+  useSyncAllConnectorsPreview,
+} from "../api/mutations/useSyncConnector";
 
 /** List-files uses term filters; "*" means "any" in the UI — do not send it literally. */
 function listFilesFilterParam(values?: string[]): string | undefined {
@@ -119,7 +124,51 @@ function SearchPage() {
 
   const deleteDocumentMutation = useDeleteDocument();
   const syncAllConnectorsMutation = useSyncAllConnectors();
+  const syncAllPreviewMutation = useSyncAllConnectorsPreview();
   const refreshOpenragDocsMutation = useRefreshOpenragDocs();
+  const [syncDialogOpen, setSyncDialogOpen] = useState(false);
+  const [syncPreview, setSyncPreview] = useState<SyncAllPreviewResponse | null>(
+    null,
+  );
+
+  const handleOpenSyncDialog = useCallback(async () => {
+    setSyncPreview(null);
+    setSyncDialogOpen(true);
+    try {
+      const preview = await syncAllPreviewMutation.mutateAsync();
+      setSyncPreview(preview);
+    } catch (error) {
+      setSyncDialogOpen(false);
+      toast.error(
+        error instanceof Error ? error.message : "Failed to preview sync",
+      );
+    }
+  }, [syncAllPreviewMutation]);
+
+  const handleConfirmSync = useCallback(async () => {
+    try {
+      const result = await syncAllConnectorsMutation.mutateAsync();
+      if (result.status === "no_files") {
+        toast.info(
+          result.message ||
+            "No cloud files to sync. Add files from cloud connectors first.",
+        );
+      } else if (
+        result.synced_connectors &&
+        result.synced_connectors.length > 0
+      ) {
+        toast.success(
+          `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
+        );
+      } else if (result.errors && result.errors.length > 0) {
+        toast.error("Some connectors failed to sync");
+      }
+    } catch (error) {
+      toast.error(
+        error instanceof Error ? error.message : "Failed to sync connectors",
+      );
+    }
+  }, [syncAllConnectorsMutation]);
 
   useEffect(() => {
     refreshTasks();
@@ -761,36 +810,14 @@ function SearchPage() {
               type="button"
               variant="outline"
               className="rounded-lg flex-shrink-0"
-              disabled={syncAllConnectorsMutation.isPending}
-              onClick={async () => {
-                try {
-                  toast.info("Syncing all cloud connectors...");
-                  const result = await syncAllConnectorsMutation.mutateAsync();
-                  if (result.status === "no_files") {
-                    toast.info(
-                      result.message ||
-                        "No cloud files to sync. Add files from cloud connectors first.",
-                    );
-                  } else if (
-                    result.synced_connectors &&
-                    result.synced_connectors.length > 0
-                  ) {
-                    toast.success(
-                      `Sync started for ${result.synced_connectors.join(", ")}. Check task notifications for progress.`,
-                    );
-                  } else if (result.errors && result.errors.length > 0) {
-                    toast.error("Some connectors failed to sync");
-                  }
-                } catch (error) {
-                  toast.error(
-                    error instanceof Error
-                      ? error.message
-                      : "Failed to sync connectors",
-                  );
-                }
-              }}
+              disabled={
+                syncAllConnectorsMutation.isPending ||
+                syncAllPreviewMutation.isPending
+              }
+              onClick={handleOpenSyncDialog}
             >
-              {syncAllConnectorsMutation.isPending ? (
+              {syncAllConnectorsMutation.isPending ||
+              syncAllPreviewMutation.isPending ? (
                 <>
                   <RefreshCw className="h-4 w-4 mr-2 animate-spin" />
                   Syncing...
@@ -961,6 +988,18 @@ function SearchPage() {
         <p className="my-2">Documents to be deleted:</p>
         {formatFilesToDelete(selectedRows)}
       </DeleteConfirmationDialog>
+
+      <SyncConfirmDialog
+        open={syncDialogOpen}
+        onOpenChange={setSyncDialogOpen}
+        onConfirm={handleConfirmSync}
+        isLoading={syncAllPreviewMutation.isPending || syncPreview === null}
+        isSyncing={syncAllConnectorsMutation.isPending}
+        isSyncAll
+        orphansByType={syncPreview?.orphans_by_type}
+        orphansAvailableByType={syncPreview?.orphans_available_by_type}
+        syncedCountByType={syncPreview?.synced_count_by_type}
+      />
     </>
   );
 }

frontend/app/onboarding/_components/onboarding-card.tsx

@@ -510,7 +510,10 @@ const OnboardingCard = ({
                 >
                   <div className="pb-6 flex items-center gap-4">
                     <X className="w-4 h-4 text-destructive shrink-0" />
-                    <span className="text-mmd text-muted-foreground">
+                    <span
+                      data-testid="onboarding-error"
+                      className="text-mmd text-muted-foreground"
+                    >
                       {error}
                     </span>
                   </div>

frontend/app/onboarding/_components/onboarding-upload.tsx

@@ -24,6 +24,7 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
   const [uploadedTaskId, setUploadedTaskId] = useState<string | null>(null);
   const [shouldCreateFilter, setShouldCreateFilter] = useState(false);
   const [isCreatingFilter, setIsCreatingFilter] = useState(false);
+  const [error, setError] = useState<string | null>(null);
 
   const createFilterMutation = useCreateFilter();
   const updateOnboardingMutation = useUpdateOnboardingStateMutation();
@@ -63,6 +64,41 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
       matchingTask.status === "running" ||
       matchingTask.status === "processing";
 
+    // Check if matching task failed or has error status
+    const failedTask =
+      matchingTask.status === "failed" || matchingTask.status === "error";
+
+    // Check if any file inside the task failed
+    const filesArray = matchingTask.files
+      ? (Object.values(matchingTask.files) as {
+          status: string;
+          error?: string;
+        }[])
+      : [];
+    const hasFailedFile = filesArray.some(
+      (file) => file.status === "failed" || file.status === "error",
+    );
+
+    if (failedTask || hasFailedFile) {
+      let errorMessage = "Document ingestion failed. Please try again.";
+      if (matchingTask.error) {
+        errorMessage = matchingTask.error;
+      } else {
+        const failedFile = filesArray.find(
+          (file) =>
+            (file.status === "failed" || file.status === "error") && file.error,
+        );
+        if (failedFile?.error) {
+          errorMessage = failedFile.error;
+        }
+      }
+
+      setError(errorMessage);
+      setCurrentStep(null);
+      setUploadedTaskId(null);
+      return;
+    }
+
     // If task is completed or has processed files, complete the onboarding step
     if (!isTaskActive || (matchingTask.processed_files ?? 0) > 0) {
       // Set to final step to show "Done"
@@ -169,6 +205,7 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
 
   const performUpload = async (file: File) => {
     setIsUploading(true);
+    setError(null);
     try {
       setCurrentStep(0);
       const result = await uploadFile(file, true, true); // Pass createFilter=true
@@ -193,6 +230,7 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
       const errorMessage =
         error instanceof Error ? error.message : "Upload failed";
       console.error("Upload failed", errorMessage);
+      setError(errorMessage);
 
       // Dispatch event that chat context can listen to
       // This avoids circular dependency issues
@@ -246,6 +284,25 @@ const OnboardingUpload = ({ onComplete }: OnboardingUploadProps) => {
           exit={{ opacity: 0, y: -24 }}
           transition={{ duration: 0.4, ease: "easeInOut" }}
         >
+          <AnimatePresence mode="wait">
+            {error && (
+              <motion.div
+                key="error"
+                initial={{ opacity: 1, y: 0, height: "auto" }}
+                exit={{ opacity: 0, y: -10, height: 0 }}
+              >
+                <div className="pb-6 flex items-center gap-4">
+                  <X className="w-4 h-4 text-destructive shrink-0" />
+                  <span
+                    data-testid="onboarding-upload-error"
+                    className="text-mmd text-muted-foreground"
+                  >
+                    {error}
+                  </span>
+                </div>
+              </motion.div>
+            )}
+          </AnimatePresence>
           <Button
             size="sm"
             variant="outline"

frontend/components/cloud-picker/file-item.tsx

@@ -31,10 +31,24 @@ const getMimeTypeLabel = (mimeType: string) => {
     "application/vnd.google-apps.folder": "Folder",
     "application/pdf": "PDF",
     "text/plain": "Text",
+    "text/csv": "CSV",
+    "text/html": "HTML",
+    "application/xml": "XML",
+    "application/json": "JSON",
+    "application/msword": "Word Doc",
+    "application/vnd.ms-excel": "Excel",
+    "application/vnd.ms-powerpoint": "PowerPoint",
     "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
       "Word Doc",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+      "Excel",
     "application/vnd.openxmlformats-officedocument.presentationml.presentation":
       "PowerPoint",
+    "application/octet-stream": "File",
+    "image/jpeg": "JPEG",
+    "image/png": "PNG",
+    "image/gif": "GIF",
+    "image/svg+xml": "SVG",
   };
 
   return typeMap[mimeType] || mimeType?.split("/").pop() || "Document";