fix bugs for exist files and fix private files

DataSource/Aws/setAwsConnection.ts

@@ -33,7 +33,6 @@ export const setAWSConnection = async (formData: FormData) => {
     } : undefined,
     metadata: config.data.metadata,
     isConfigSet: true,
-    isSyncing: true,
     limitPages:config.data.pageLimit,
     limitFiles: config.data.fileLimit,
   }).where(eq(connections.id, config.data.connectionId))

DataSource/DirectUpload/UpdateConfigDirect/UpdateConfigDirect.tsx

@@ -19,7 +19,7 @@ export const UpdateConfigDirect = ({ connection, status }: { connection: Connect
 
   return (<Dialog open={open} onOpenChange={setOpen} >
     <DialogTrigger asChild>
-      <Button data-test="btn-config" size='sm' variant={connection.isConfigSet ? 'ghost' : 'default'} disabled={status === 'PROCESSING' || (!status && connection.isSyncing)}>
+      <Button data-test="btn-config" size='sm' variant={connection.isConfigSet ? 'ghost' : 'default'} disabled={status === 'PROCESSING' || (!status && !!connection.jobId)}>
         <Settings2 />
         Configure
       </Button>

DataSource/DirectUpload/setDirectUploadConnection.ts

@@ -1,6 +1,7 @@
 import { databaseDrizzle } from "@/db";
 import { connections, processedFiles } from "@/db/schemas/connections";
-import { qdrant_collection_name, qdrantCLient } from "@/qdrant";
+import { tryAndCatch } from "@/lib/try-catch";
+import { qdrant_collection_name, qdrantClient } from "@/qdrant";
 import { and, eq } from "drizzle-orm";
 import { z } from "zod";
 
@@ -52,6 +53,7 @@ const directUploadConfig = z.object({
 })
 
 const updateDirectUploadConfig = z.object({
+  userId: z.string().min(5),
   connectionId: z.string().min(5),
   pageLimit: z.string().nullable().transform((str, ctx): number | null => {
     try {
@@ -91,6 +93,7 @@ const updateDirectUploadConfig = z.object({
 
 export const updateDirectUploadConnection = async (formData: FormData) => {
   const config = updateDirectUploadConfig.safeParse({
+    userId: formData.get("userId"),
     connectionId: formData.get("connectionId"),
     identifier: formData.get("identifier"),
     metadata: formData.get("metadata") || "{}",
@@ -110,7 +113,7 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
     throw new Error(`Validation errors - ${errors}`)
   }
 
-  const connectionChunksIds: { chunksIds: string[] }[] = [];
+  const connectionChunksIds: { chunksIds: string[], name: string }[] = [];
 
   await databaseDrizzle
     .update(connections)
@@ -125,14 +128,18 @@ export const updateDirectUploadConnection = async (formData: FormData) => {
       .where(and(
         eq(processedFiles.connectionId, config.data.connectionId),
         eq(processedFiles.name, fileName)
-      )).returning({ chunksIds: processedFiles.chunksIds })
+      )).returning({ chunksIds: processedFiles.chunksIds, name: processedFiles.name })
     connectionChunksIds.push(...files)
   }
 
-  for (const { chunksIds } of connectionChunksIds) {
-    await qdrantCLient.delete(qdrant_collection_name, {
+  for (const { chunksIds, name } of connectionChunksIds) {
+    await tryAndCatch(qdrantClient.delete(qdrant_collection_name, {
       points: chunksIds,
-    })
+      filter: {
+        must: [{ "key": "_document_id", "match": { value: name } },
+        { key: "_userId", match: { value: config.data.userId } }]
+      }
+    }))
   }
 
   const files = config.data.files.map(async (file) => ({
@@ -188,7 +195,6 @@ export const setDirectUploadConnection = async (formData: FormData) => {
     service: 'DIRECT_UPLOAD',
     metadata: metadata,
     isConfigSet: true,
-    isSyncing: true,
     limitPages: pageLimit,
     limitFiles: fileLimit,
   }).returning({ id: connections.id })

DataSource/Dropbox/setDropboxConnection.ts

@@ -32,7 +32,6 @@ export const setDropboxConnection = async (formData: FormData) => {
     } : undefined,
     metadata: config.data.metadata,
     isConfigSet: true,
-    isSyncing: true,
     limitFiles: config.data.fileLimit,
     limitPages: config.data.pageLimit,
   }).where(eq(connections.id, config.data.connectionId))

DataSource/GoogleDrive/setGoogleDriveConnection.ts

@@ -34,7 +34,6 @@ export const setGoogleDriveConnection = async (formData: FormData) => {
     limitPages: config.data.pageLimit,
     limitFiles: config.data.fileLimit,
     isConfigSet: true,
-    isSyncing: true,
   }).where(eq(connections.id, config.data.connectionId))
 
   return {

actions/connctions/delete/index.ts

@@ -2,8 +2,9 @@
 import { authOptions } from "@/auth";
 import { databaseDrizzle } from "@/db";
 import { connections, processedFiles } from "@/db/schemas/connections";
+import { tryAndCatch } from "@/lib/try-catch";
 import { fromErrorToFormState, toFormState } from "@/lib/zodErrorHandle";
-import { qdrant_collection_name, qdrantCLient } from "@/qdrant";
+import { qdrant_collection_name, qdrantClient } from "@/qdrant";
 import { eq } from "drizzle-orm";
 import { getServerSession } from "next-auth";
 import { revalidatePath } from "next/cache";
@@ -26,14 +27,19 @@ export async function deleteConnectionConfig(_: FormState, formData: FormData) {
     })
 
     const connectionChunksIds = await databaseDrizzle
-      .select({ chunksIds: processedFiles.chunksIds })
+      .select({ chunksIds: processedFiles.chunksIds, name: processedFiles.name })
       .from(processedFiles)
       .where(eq(processedFiles.connectionId, id))
 
-    for (const { chunksIds } of connectionChunksIds) {
-      await qdrantCLient.delete(qdrant_collection_name, {
+    for (const { chunksIds, name } of connectionChunksIds) {
+      await tryAndCatch(qdrantClient.delete(qdrant_collection_name, {
         points: chunksIds,
-      })
+        filter: {
+          must: [
+            { key: "_document_id", "match": { value: name } },
+            { key: "_userId", match: { value: session.user.id } }]
+        }
+      }))
     }
 
     await databaseDrizzle

actions/connctions/sync/index.ts

@@ -77,7 +77,6 @@ export const syncConnectionConfig = async (_: FormState, formData: FormData) =>
       links: []
     })
     await databaseDrizzle.update(connections).set({
-      isSyncing: true,
       jobId: jobId
     }).where(eq(connections.id, connectionId));
 

app/api/chunks/[...ids]/route.ts

@@ -4,7 +4,7 @@ import { NextRequest, NextResponse } from "next/server";
 import { APIError } from "@/lib/APIError";
 import { databaseDrizzle } from "@/db";
 import { arrayContains } from "drizzle-orm";
-import { qdrant_collection_name, qdrantCLient } from "@/qdrant";
+import { qdrant_collection_name, qdrantClient } from "@/qdrant";
 
 type Params = {
   params: Promise<{
@@ -62,7 +62,7 @@ export async function GET(request: NextRequest, { params }: Params) {
       )
     }
 
-    const { data: chunks, error: retrieveError } = await tryAndCatch(qdrantCLient.retrieve(qdrant_collection_name, {
+    const { data: chunks, error: retrieveError } = await tryAndCatch(qdrantClient.retrieve(qdrant_collection_name, {
       ids: chunksIds,
       with_payload: true,
     }))

app/api/connections/[id]/files/route.ts

@@ -2,7 +2,7 @@ import { databaseDrizzle } from "@/db"
 import { processedFiles } from "@/db/schemas/connections"
 import { checkAuth } from "@/lib/api_key"
 import { tryAndCatch } from "@/lib/try-catch"
-import { qdrant_collection_name, qdrantCLient } from "@/qdrant"
+import { qdrant_collection_name, qdrantClient } from "@/qdrant"
 import { and, eq } from "drizzle-orm"
 import { NextRequest, NextResponse } from "next/server"
 import { APIError } from "openai"
@@ -80,12 +80,11 @@ export async function DELETE(request: NextRequest, { params }: Params) {
     }
 
     const { id } = await params
-    const connectionChunksIds: { chunksIds: string[] }[] = [];
+    const connectionChunksIds: { chunksIds: string[], name: string }[] = [];
     const connection = await databaseDrizzle.query.connections.findFirst({
       where: (c, ops) => ops.and(ops.eq(c.userId, userId), ops.eq(c.id, id)),
       columns: {
         id: true,
-        isSyncing: true,
         jobId: true,
       },
     })
@@ -97,7 +96,7 @@ export async function DELETE(request: NextRequest, { params }: Params) {
       }, { status: 403 })
     }
 
-    if (connection.isSyncing || connection.jobId) {
+    if (connection.jobId) {
       return NextResponse.json(
         {
           code: 'processing_in_progress',
@@ -117,7 +116,7 @@ export async function DELETE(request: NextRequest, { params }: Params) {
         .where(and(
           eq(processedFiles.connectionId, id),
           eq(processedFiles.name, fileName)
-        )).returning({ chunksIds: processedFiles.chunksIds }))
+        )).returning({ chunksIds: processedFiles.chunksIds, name: processedFiles.name }))
 
       if (error) {
         return NextResponse.json({
@@ -128,10 +127,16 @@ export async function DELETE(request: NextRequest, { params }: Params) {
       connectionChunksIds.push(...data)
     }
 
-    for (const { chunksIds } of connectionChunksIds) {
-      const { error } = await tryAndCatch(qdrantCLient.delete(qdrant_collection_name, {
+    for (const { chunksIds, name } of connectionChunksIds) {
+      const { error } = await tryAndCatch(qdrantClient.delete(qdrant_collection_name, {
         points: chunksIds,
-        wait: isWait
+        wait: isWait,
+        filter: {
+          must: [
+            { key: "_document_id", match: { value: name } },
+            { key: "_userId", match: { value: userId } }
+          ]
+        }
       }))
       if (error) {
         return NextResponse.json({

app/api/connections/[id]/route.ts

@@ -5,7 +5,7 @@ import { APIError } from "@/lib/APIError";
 import { databaseDrizzle } from "@/db";
 import { eq } from "drizzle-orm";
 import { connections } from "@/db/schemas/connections";
-import { qdrant_collection_name, qdrantCLient } from "@/qdrant";
+import { qdrant_collection_name, qdrantClient } from "@/qdrant";
 import { setConnectionToProcess } from "@/fileProcessors/connectors";
 import { connectionProcessFiles, directProcessFiles } from "@/fileProcessors";
 import { addToProcessFilesQueue } from "@/workers/queues/jobs/processFiles.job";
@@ -109,12 +109,12 @@ export async function DELETE(request: NextRequest, { params }: Params) {
       where: (conn, ops) => ops.and(ops.eq(conn.id, id), ops.eq(conn.userId, userId)),
       columns: {
         jobId: true,
-        isSyncing: true,
       },
       with: {
         files: {
           columns: {
-            chunksIds: true
+            chunksIds: true,
+            name: true,
           }
         }
       }
@@ -139,7 +139,7 @@ export async function DELETE(request: NextRequest, { params }: Params) {
       )
     }
 
-    if (conn.isSyncing || conn.jobId) {
+    if (conn.jobId) {
       return NextResponse.json(
         {
           code: 'processing_in_progress',
@@ -148,10 +148,16 @@ export async function DELETE(request: NextRequest, { params }: Params) {
         { status: 409 }
       )
     }
-    for (const { chunksIds } of conn.files) {
-      await tryAndCatch(qdrantCLient.delete(qdrant_collection_name, {
+    for (const { chunksIds, name } of conn.files) {
+      await tryAndCatch(qdrantClient.delete(qdrant_collection_name, {
         points: chunksIds,
         wait: wait === "true",
+        filter: {
+          must: [
+            { key: "_document_id", match: { value: name } },
+            { key: "_userId", match: { value: userId } },
+          ]
+        }
       }))
     }
 
@@ -190,9 +196,9 @@ export async function PUT(request: NextRequest, { params }: Params) {
     const { data: conn, error: queryError } = await tryAndCatch(databaseDrizzle.query.connections.findFirst({
       where: (conn, ops) => ops.and(ops.eq(conn.id, id), ops.eq(conn.userId, userId)),
       columns: {
-        isSyncing: true,
         service: true,
         folderName: true,
+        jobId: true,
       }
     }))
     if (queryError) {
@@ -215,7 +221,7 @@ export async function PUT(request: NextRequest, { params }: Params) {
       )
     }
 
-    if (conn.isSyncing) {
+    if (conn.jobId) {
       return NextResponse.json(
         {
           code: 'already_syncing',

app/api/connections/[id]/status/route.ts

@@ -29,7 +29,7 @@ export async function GET(request: NextRequest, { params }: Params) {
       where: (conn, ops) => ops.and(ops.eq(conn.userId, userId!), ops.eq(conn.id, id)),
       columns: {
         isConfigSet: true,
-        isSyncing: true,
+        jobId:true,
       },
     }))
     if (queryError) {
@@ -53,7 +53,7 @@ export async function GET(request: NextRequest, { params }: Params) {
 
     const response = {
       status: conn.isConfigSet ? 'ready' : 'config_missing',
-      isSyncing: conn.isSyncing
+      isSyncing: !!conn.jobId
     }
 
     return NextResponse.json(response, { status: 200 });

app/api/connections/[id]/sync/route.ts

@@ -35,8 +35,8 @@ export async function POST(request: NextRequest, { params }: Params) {
         id: true,
         service: true,
         isConfigSet: true,
-        isSyncing: true,
-        metadata: true
+        metadata: true,
+        jobId:true,
       },
       with: {
         files: {
@@ -75,7 +75,7 @@ export async function POST(request: NextRequest, { params }: Params) {
         { status: 400 },
       )
     }
-    if (conn.isSyncing) {
+    if (conn.jobId) {
       return NextResponse.json(
         {
           code: 'already_syncing',

app/api/retrievals/route.ts

@@ -4,7 +4,7 @@ import { apiKeys, users } from '@/db/schemas/users';
 import { hashApiKey } from '@/lib/api_key';
 import { Plans } from '@/lib/Plans';
 import { expandQuery, generateHypotheticalAnswer, vectorizeText } from '@/openAi';
-import { qdrant_collection_name, qdrantCLient } from '@/qdrant';
+import { qdrant_collection_name, qdrantClient } from '@/qdrant';
 import { RetrievalFilter } from '@/validations/retrievalsFilteringSchema'
 import { and, eq, sql } from 'drizzle-orm';
 import { getServerSession } from 'next-auth';
@@ -105,17 +105,22 @@ export async function POST(request: NextRequest) {
           `Please upgrade to increase your limit.`
       }, {
         status: 429,
-        headers: { "Retry-After": "3600" } 
+        headers: { "Retry-After": "3600" }
       });
     }
 
     const { query, filter, top_chunk, rerank, min_score_threshold } = validation.data
     const queries = await expandQuery(query)
     const vectors = await vectorizeText(queries)
-
-    const queryPoints = await qdrantCLient.search(qdrant_collection_name, {
+    const queryPoints = await qdrantClient.search(qdrant_collection_name, {
       vector: vectors,
-      filter: filter ? { must: [{ nested: { key: "_metadata", filter: filter } }] } : undefined,
+      filter: filter ? {
+        must: [
+          { nested: { key: "_metadata", filter: filter } },
+          { key: "_userId", match: { value: userId } }
+        ]
+      } :
+        { must: [{ key: "_userId", match: { value: userId } }] },
       limit: rerank ? top_chunk * 2 : top_chunk,
       with_payload: true,
       with_vector: true

app/api/upload/route.ts

@@ -25,7 +25,6 @@ export async function POST(request: NextRequest) {
       code: "bad_request",
       message: errorForm.message,
     }, { status: 400 })
-
     formData.set("userId", userId)
     formData.set("service", "DIRECT_UPLOAD")
     const {