add page data to vector db

2 files changed, 66 insertions, 17 deletions

diff --git a/apps/web/src/actions/db.ts b/apps/web/src/actions/db.ts
index cd54c1bd..23f9a7ad 100644
--- a/apps/web/src/actions/db.ts
+++ b/apps/web/src/actions/db.ts
@@ -13,6 +13,7 @@ import {
 import { SearchResult } from "@/contexts/MemoryContext";
 import { like, eq, and, sql, exists, asc, notExists } from "drizzle-orm";
 import { union } from "drizzle-orm/sqlite-core";
+import { env } from "@/env";
 
 // @todo: (future) pagination not yet needed
 export async function searchMemoriesAndSpaces(
@@ -237,6 +238,44 @@ export async function addMemory(
     return null;
   }
 
+  if (!content.content || content.content == "") {
+    const resp = await fetch(
+      `https://cf-ai-backend.dhravya.workers.dev/getPageContent?url=${content.url}`,
+      {
+        headers: {
+          "X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
+        },
+      },
+    );
+
+    const data = await resp.text();
+
+    content.content = data;
+  }
+
+  if (!content.content || content.content == "") {
+    return null;
+  }
+
+  // Add to vectorDB
+  const res = (await Promise.race([
+    fetch("https://cf-ai-backend.dhravya.workers.dev/add", {
+      method: "POST",
+      headers: {
+        "X-Custom-Auth-Key": env.BACKEND_SECURITY_KEY,
+      },
+      body: JSON.stringify({ ...content, user: user.email }),
+    }),
+    new Promise((_, reject) =>
+      setTimeout(() => reject(new Error("Request timed out")), 40000),
+    ),
+  ])) as Response;
+
+  if (res.status !== 200) {
+    console.log(res.status, res.statusText);
+    return null;
+  }
+
   const [addedMemory] = await db
     .insert(storedContent)
     .values({
diff --git a/apps/web/src/app/api/store/route.ts b/apps/web/src/app/api/store/route.ts
index d592bc53..7fa92b16 100644
--- a/apps/web/src/app/api/store/route.ts
+++ b/apps/web/src/app/api/store/route.ts
@@ -71,7 +71,6 @@ export async function POST(req: NextRequest) {
   };
 
   const metadata = await getMetaData(data.url);
-
   let storeToSpaces = data.spaces;
 
   if (!storeToSpaces) {
@@ -87,16 +86,21 @@ export async function POST(req: NextRequest) {
 
   console.log("count", count[0].count);
 
-  const { id } = (await db.insert(storedContent).values({
-    content: data.pageContent,
-    title: metadata.title,
-    description: metadata.description,
-    url: data.url,
-    baseUrl: metadata.baseUrl,
-    image: metadata.image,
-    savedAt: new Date(),
-    user: session.user.id,
-  }).returning({ id: storedContent.id }))[0];
+  const { id } = (
+    await db
+      .insert(storedContent)
+      .values({
+        content: data.pageContent,
+        title: metadata.title,
+        description: metadata.description,
+        url: data.url,
+        baseUrl: metadata.baseUrl,
+        image: metadata.image,
+        savedAt: new Date(),
+        user: session.user.id,
+      })
+      .returning({ id: storedContent.id })
+  )[0];
 
   if (!id) {
     return NextResponse.json(
@@ -108,12 +112,18 @@ export async function POST(req: NextRequest) {
   const spaceData = await db
     .select()
     .from(space)
-    .where(and(inArray(space.name, storeToSpaces), eq(space.user, session.user.id)))
-		.all()
-
-	await Promise.all([spaceData.forEach(async space => {
-		await db.insert(contentToSpace).values({ contentId: id, spaceId: space.id })
-	})])
+    .where(
+      and(inArray(space.name, storeToSpaces), eq(space.user, session.user.id)),
+    )
+    .all();
+
+  await Promise.all([
+    spaceData.forEach(async (space) => {
+      await db
+        .insert(contentToSpace)
+        .values({ contentId: id, spaceId: space.id });
+    }),
+  ]);
 
   const res = (await Promise.race([
     fetch("https://cf-ai-backend.dhravya.workers.dev/add", {