From 6e1d53e28a056e429c54e1e6af45eaa7939daa41 Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Wed, 31 Jul 2024 10:56:40 +0530 Subject: queues so far Co-authored-by: Dhravya Shah --- apps/cf-ai-backend/src/db/index.ts | 7 + apps/cf-ai-backend/src/db/schema.ts | 93 + apps/cf-ai-backend/src/errors/baseError.ts | 46 + apps/cf-ai-backend/src/errors/results.ts | 28 + apps/cf-ai-backend/src/helper.ts | 21 +- apps/cf-ai-backend/src/index.ts | 75 +- .../src/queueConsumer/chunkers/chonker.ts | 47 + .../src/queueConsumer/chunkers/chunkPageOrNotes.ts | 13 + .../src/queueConsumer/chunkers/chunkTweet.ts | 65 + .../src/queueConsumer/helpers/initQuery.ts | 58 + .../src/queueConsumer/helpers/processNotes.ts | 36 + .../src/queueConsumer/helpers/processPage.ts | 42 + .../src/queueConsumer/helpers/processTweet.ts | 81 + apps/cf-ai-backend/src/queueConsumer/index.ts | 204 +++ .../src/queueConsumer/utils/get-metadata.ts | 57 + .../src/queueConsumer/utils/typeDecider.ts | 34 + apps/cf-ai-backend/src/types.ts | 17 +- apps/cf-ai-backend/src/utils/chonker.ts | 47 - apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts | 13 - apps/cf-ai-backend/src/utils/chunkTweet.ts | 65 - apps/web/app/actions/doers.ts | 481 ++--- apps/web/lib/get-metadata.ts | 40 - apps/web/migrations/0000_omniscient_stick.sql | 152 ++ .../web/migrations/0000_steep_moira_mactaggert.sql | 135 -- apps/web/migrations/meta/0000_snapshot.json | 1847 +++++++++++--------- apps/web/migrations/meta/_journal.json | 24 +- apps/web/server/db/schema.ts | 26 + 27 files changed, 2347 insertions(+), 1407 deletions(-) create mode 100644 apps/cf-ai-backend/src/db/index.ts create mode 100644 apps/cf-ai-backend/src/db/schema.ts create mode 100644 apps/cf-ai-backend/src/errors/baseError.ts create mode 100644 apps/cf-ai-backend/src/errors/results.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/chunkers/chonker.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/chunkers/chunkPageOrNotes.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/chunkers/chunkTweet.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/helpers/processNotes.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/index.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/utils/get-metadata.ts create mode 100644 apps/cf-ai-backend/src/queueConsumer/utils/typeDecider.ts delete mode 100644 apps/cf-ai-backend/src/utils/chonker.ts delete mode 100644 apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts delete mode 100644 apps/cf-ai-backend/src/utils/chunkTweet.ts delete mode 100644 apps/web/lib/get-metadata.ts create mode 100644 apps/web/migrations/0000_omniscient_stick.sql delete mode 100644 apps/web/migrations/0000_steep_moira_mactaggert.sql (limited to 'apps') diff --git a/apps/cf-ai-backend/src/db/index.ts b/apps/cf-ai-backend/src/db/index.ts new file mode 100644 index 00000000..e16cec25 --- /dev/null +++ b/apps/cf-ai-backend/src/db/index.ts @@ -0,0 +1,7 @@ +import { drizzle } from "drizzle-orm/d1"; +import { Env } from "../types"; + +import * as schema from "./schema"; + +export const database = (env: Env) => + drizzle(env.DATABASE, { schema, logger: true }); diff --git a/apps/cf-ai-backend/src/db/schema.ts b/apps/cf-ai-backend/src/db/schema.ts new file mode 100644 index 00000000..71e2c884 --- /dev/null +++ b/apps/cf-ai-backend/src/db/schema.ts @@ -0,0 +1,93 @@ +import { + index, + int, + primaryKey, + sqliteTableCreator, + text, + integer, +} from "drizzle-orm/sqlite-core"; + +export const createTable = sqliteTableCreator((name) => `${name}`); + +export const users = createTable( + "user", + { + id: text("id") + .primaryKey() + .$defaultFn(() => crypto.randomUUID()), + name: text("name"), + email: text("email").notNull(), + emailVerified: integer("emailVerified", { mode: "timestamp_ms" }), + image: text("image"), + telegramId: text("telegramId"), + hasOnboarded: integer("hasOnboarded", { mode: "boolean" }).default(false), + }, + (user) => ({ + emailIdx: index("users_email_idx").on(user.email), + telegramIdx: index("users_telegram_idx").on(user.telegramId), + idIdx: index("users_id_idx").on(user.id), + }), +); + +export type User = typeof users.$inferSelect; + +export const storedContent = createTable( + "storedContent", + { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + content: text("content").notNull(), + title: text("title", { length: 255 }), + description: text("description", { length: 255 }), + url: text("url").notNull(), + savedAt: int("savedAt", { mode: "timestamp" }).notNull(), + baseUrl: text("baseUrl", { length: 255 }).unique(), + ogImage: text("ogImage", { length: 255 }), + type: text("type").default("page"), + image: text("image", { length: 255 }), + userId: text("user").references(() => users.id, { + onDelete: "cascade", + }), + noteId: integer("noteId"), + }, + (sc) => ({ + urlIdx: index("storedContent_url_idx").on(sc.url), + savedAtIdx: index("storedContent_savedAt_idx").on(sc.savedAt), + titleInx: index("storedContent_title_idx").on(sc.title), + userIdx: index("storedContent_user_idx").on(sc.userId), + }), +); + +export type Content = typeof storedContent.$inferSelect; + + +export const contentToSpace = createTable( + "contentToSpace", + { + contentId: integer("contentId") + .notNull() + .references(() => storedContent.id, { onDelete: "cascade" }), + spaceId: integer("spaceId") + .notNull() + .references(() => space.id, { onDelete: "cascade" }), + }, + (cts) => ({ + compoundKey: primaryKey({ columns: [cts.contentId, cts.spaceId] }), + }), +); + +export const space = createTable( + "space", + { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + name: text("name").notNull().unique().default("none"), + user: text("user", { length: 255 }).references(() => users.id, { + onDelete: "cascade", + }), + createdAt: int("createdAt", { mode: "timestamp" }).notNull(), + numItems: integer("numItems").notNull().default(0), + }, + (space) => ({ + nameIdx: index("spaces_name_idx").on(space.name), + userIdx: index("spaces_user_idx").on(space.user), + }), +); \ No newline at end of file diff --git a/apps/cf-ai-backend/src/errors/baseError.ts b/apps/cf-ai-backend/src/errors/baseError.ts new file mode 100644 index 00000000..2723d45b --- /dev/null +++ b/apps/cf-ai-backend/src/errors/baseError.ts @@ -0,0 +1,46 @@ +export class BaseHttpError extends Error { + public status: number; + public message: string; + + constructor(status: number, message: string) { + super(message); + this.status = status; + this.message = message; + Object.setPrototypeOf(this, new.target.prototype); // Restore prototype chain + } + } + + + export class BaseError extends Error { + type: string; + message: string; + source: string; + ignoreLog: boolean; + + constructor( + type: string, + message?: string, + source?: string, + ignoreLog = false + ) { + super(); + + Object.setPrototypeOf(this, new.target.prototype); + + this.type = type; + this.message = + message ?? + "An unknown error occurred. If this persists, please contact us."; + this.source = source ?? "unspecified"; + this.ignoreLog = ignoreLog; + } + + toJSON(): Record { + return { + type: this.type, + message: this.message, + source: this.source, + }; + } + } + \ No newline at end of file diff --git a/apps/cf-ai-backend/src/errors/results.ts b/apps/cf-ai-backend/src/errors/results.ts new file mode 100644 index 00000000..87ea0c63 --- /dev/null +++ b/apps/cf-ai-backend/src/errors/results.ts @@ -0,0 +1,28 @@ +import { BaseError } from "./baseError"; + +export type Result = + | { ok: true; value: T } + | { ok: false; error: E }; + +export const Ok = (data: T): Result => { + return { ok: true, value: data }; +}; + +export const Err = (error: E): Result => { + return { ok: false, error }; +}; + +export async function wrap( + p: Promise, + errorFactory: (err: Error) => E, +): Promise> { + try { + return Ok(await p); + } catch (e) { + return Err(errorFactory(e as Error)); + } +} + +export function isErr(result: Result): result is { ok: false; error: E } { + return !result.ok; + } \ No newline at end of file diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 2a68879a..1568996a 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -132,12 +132,12 @@ export async function batchCreateChunksAndEmbeddings({ store, body, chunks, - context, + env: env, }: { store: CloudflareVectorizeStore; body: z.infer; chunks: Chunks; - context: Context<{ Bindings: Env }>; + env: Env; }) { //! NOTE that we use #supermemory-web to ensure that //! If a user saves it through the extension, we don't want other users to be able to see it. @@ -149,7 +149,7 @@ export async function batchCreateChunksAndEmbeddings({ random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - const allIds = await context.env.KV.list({ prefix: uuid }); + const allIds = await env.KV.list({ prefix: uuid }); // If some chunks for that content already exist, we'll just update the metadata to include // the user. @@ -159,7 +159,7 @@ export async function batchCreateChunksAndEmbeddings({ //Search in a batch of 20 for (let i = 0; i < savedVectorIds.length; i += 20) { const batch = savedVectorIds.slice(i, i + 20); - const batchVectors = await context.env.VECTORIZE_INDEX.getByIds(batch); + const batchVectors = await env.VECTORIZE_INDEX.getByIds(batch); vectors.push(...batchVectors); } console.log( @@ -192,7 +192,7 @@ export async function batchCreateChunksAndEmbeddings({ await Promise.all( results.map((result) => { - return context.env.VECTORIZE_INDEX.upsert(result); + return env.VECTORIZE_INDEX.upsert(result); }), ); return; @@ -243,8 +243,7 @@ export async function batchCreateChunksAndEmbeddings({ }); console.log("these are the doucment ids", ids); console.log("Docs added:", docs); - const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = - context.env; + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, { chunkIds: ids, urlid: ourID }, @@ -281,8 +280,7 @@ export async function batchCreateChunksAndEmbeddings({ const docs = await store.addDocuments(preparedDocuments, { ids: ids }); console.log("Docs added:", docs); - const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = - context.env; + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, { chunkIds: ids, urlid: ourID }, @@ -319,8 +317,7 @@ export async function batchCreateChunksAndEmbeddings({ const docs = await store.addDocuments(preparedDocuments, { ids: ids }); console.log("Docs added:", docs); - const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = - context.env; + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, { chunkIds: ids, urlid: ourID }, @@ -355,7 +352,7 @@ export async function batchCreateChunksAndEmbeddings({ const docs = await store.addDocuments(preparedDocuments, { ids: ids }); console.log("Docs added:", docs); - const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = context.env; + const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, { chunkIds: ids, urlid: ourID }, diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 629ff172..b072f100 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { boolean, z } from "zod"; import { Hono } from "hono"; import { CoreMessage, generateText, streamText, tool } from "ai"; import { @@ -9,6 +9,7 @@ import { PageOrNoteChunks, TweetChunks, vectorObj, + vectorBody, } from "./types"; import { batchCreateChunksAndEmbeddings, @@ -20,11 +21,15 @@ import { logger } from "hono/logger"; import { poweredBy } from "hono/powered-by"; import { bearerAuth } from "hono/bearer-auth"; import { zValidator } from "@hono/zod-validator"; -import chunkText from "./utils/chonker"; +import chunkText from "./queueConsumer/chunkers/chonker"; import { systemPrompt, template } from "./prompts/prompt1"; import { swaggerUI } from "@hono/swagger-ui"; -import { chunkThread } from "./utils/chunkTweet"; -import { chunkNote, chunkPage } from "./utils/chunkPageOrNotes"; +// import { chunkThread } from "./utils/chunkTweet"; +import { + chunkNote, + chunkPage, +} from "./queueConsumer/chunkers/chunkPageOrNotes"; +import { queue } from "./queueConsumer"; const app = new Hono<{ Bindings: Env }>(); @@ -68,37 +73,44 @@ app.get("/api/health", (c) => { return c.json({ status: "ok" }); }); -app.post("/api/add", zValidator("json", vectorObj), async (c) => { +app.post("/api/add", zValidator("json", vectorBody), async (c) => { try { + // console.log("api/add hit!!!!"); const body = c.req.valid("json"); + const spaceNumbers = body.spaces.map((s: string) => Number(s)); + await c.env.EMBEDCHUNKS_QUEUE.send({ + content: body.url, + user: body.user, + space: spaceNumbers, + }); - const { store } = await initQuery(c); + // const { store } = await initQuery(c); - console.log(body.spaces); - let chunks: TweetChunks | PageOrNoteChunks; - // remove everything in tags - // const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); + // console.log(body.spaces); + // let chunks: TweetChunks | PageOrNoteChunks; + // // remove everything in tags + // // const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); - switch (body.type) { - case "tweet": - chunks = chunkThread(body.pageContent); - break; + // switch (body.type) { + // case "tweet": + // chunks = chunkThread(body.pageContent); + // break; - case "page": - chunks = chunkPage(body.pageContent); - break; + // case "page": + // chunks = chunkPage(body.pageContent); + // break; - case "note": - chunks = chunkNote(body.pageContent); - break; - } + // case "note": + // chunks = chunkNote(body.pageContent); + // break; + // } - await batchCreateChunksAndEmbeddings({ - store, - body, - chunks: chunks, - context: c, - }); + // await batchCreateChunksAndEmbeddings({ + // store, + // body, + // chunks: chunks, + // env: c, + // }); return c.json({ status: "ok" }); } catch (error) { @@ -180,7 +192,7 @@ app.post( title: "Image content from the web", }, chunks: chunks, - context: c, + env: c.env, }); return c.json({ status: "ok" }); @@ -330,7 +342,7 @@ app.post( title: `${addString.slice(0, 30)}... (Added from chatbot)`, }, chunks: vectorContent, - context: c, + env: c.env, }); return c.json({ @@ -664,4 +676,7 @@ app.get( }, ); -export default app; +export default { + fetch: app.fetch, + queue, +}; diff --git a/apps/cf-ai-backend/src/queueConsumer/chunkers/chonker.ts b/apps/cf-ai-backend/src/queueConsumer/chunkers/chonker.ts new file mode 100644 index 00000000..18788dab --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/chunkers/chonker.ts @@ -0,0 +1,47 @@ +import nlp from "compromise"; + +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ +export default function chunkText( + text: string, + maxChunkSize: number, + overlap: number = 0.2, +): string[] { + const sentences = nlp(text).sentences().out("array"); + const chunks = []; + let currentChunk: string[] = []; + let currentSize = 0; + + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + currentChunk.push(sentence); + currentSize += sentence.length; + + if (currentSize >= maxChunkSize) { + // Calculate overlap + const overlapSize = Math.floor(currentChunk.length * overlap); + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: i - currentChunk.length + 1, + end: i, + }); + + // Prepare the next chunk with overlap + currentChunk = currentChunk.slice(-overlapSize); + currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0); + } + } + + if (currentChunk.length > 0) { + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: sentences.length - currentChunk.length, + end: sentences.length, + }); + } + + return chunks.map((chunk) => chunk.text); +} diff --git a/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkPageOrNotes.ts b/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkPageOrNotes.ts new file mode 100644 index 00000000..0da01c3f --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkPageOrNotes.ts @@ -0,0 +1,13 @@ +import chunkText from "./chonker"; +import { PageOrNoteChunks } from "../../types"; +export function chunkPage(pageContent: string): PageOrNoteChunks { + const chunks = chunkText(pageContent, 1536); + + return { type: "page", chunks: chunks }; +} + +export function chunkNote(noteContent: string): PageOrNoteChunks { + const chunks = chunkText(noteContent, 1536); + + return { type: "note", chunks: chunks }; +} diff --git a/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkTweet.ts b/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkTweet.ts new file mode 100644 index 00000000..f4dd2e16 --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/chunkers/chunkTweet.ts @@ -0,0 +1,65 @@ +import { TweetChunks } from "../../types"; +import chunkText from "./chonker"; +import { getRawTweet } from "@repo/shared-types/utils"; + +interface Tweet { + id: string; + text: string; + links: Array; + images: Array; + videos: Array; +} +interface Metadata { + tweetId: string; + tweetLinks: any[]; + tweetVids: any[]; + tweetImages: any[]; +} + +export interface ThreadTweetData { + chunkedTweet: string[]; + metadata: Metadata; +} + +export function chunkThread(threadText: string): TweetChunks { + const thread = JSON.parse(threadText); + if (typeof thread == "string") { + console.log("DA WORKER FAILED DO SOMEHTING FIX DA WROKER"); + const rawTweet = getRawTweet(thread); + const parsedTweet: any = JSON.parse(rawTweet); + + const chunkedTweet = chunkText(parsedTweet.text, 1536); + const metadata: Metadata = { + tweetId: parsedTweet.id_str, + tweetLinks: parsedTweet.entities.urls.map((url: any) => url.expanded_url), + tweetVids: + parsedTweet.extended_entities?.media + .filter((media: any) => media.type === "video") + .map((media: any) => media.video_info!.variants[0].url) || [], + tweetImages: + parsedTweet.extended_entities?.media + .filter((media: any) => media.type === "photo") + .map((media: any) => media.media_url_https!) || [], + }; + + const chunks = [{ chunkedTweet: chunkedTweet, metadata }]; + + return { type: "tweet", chunks }; + } else { + console.log(JSON.stringify(thread)); + const chunkedTweets = thread.map((tweet: Tweet) => { + const chunkedTweet = chunkText(tweet.text, 1536); + + const metadata = { + tweetId: tweet.id, + tweetLinks: tweet.links, + tweetVids: tweet.videos, + tweetImages: tweet.images, + }; + + return { chunkedTweet, metadata }; + }); + + return { type: "tweet", chunks: chunkedTweets }; + } +} diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts new file mode 100644 index 00000000..a7d85c23 --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts @@ -0,0 +1,58 @@ +import { Env } from "../../types"; +import { OpenAIEmbeddings } from "../../utils/OpenAIEmbedder"; +import { CloudflareVectorizeStore } from "@langchain/cloudflare"; +import { createOpenAI } from "@ai-sdk/openai"; +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createAnthropic } from "@ai-sdk/anthropic"; + +export async function initQQuery( + env: Env, + model: string = "gpt-4o", +) { + const embeddings = new OpenAIEmbeddings({ + apiKey: env.OPENAI_API_KEY, + modelName: "text-embedding-3-small", + }); + + const store = new CloudflareVectorizeStore(embeddings, { + index: env.VECTORIZE_INDEX, + }); + + let selectedModel: + | ReturnType> + | ReturnType> + | ReturnType>; + + switch (model) { + case "claude-3-opus": + const anthropic = createAnthropic({ + apiKey: env.ANTHROPIC_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic", + }); + selectedModel = anthropic.chat("claude-3-opus-20240229"); + console.log("Selected model: ", selectedModel); + break; + case "gemini-1.5-pro": + const googleai = createGoogleGenerativeAI({ + apiKey: env.GOOGLE_AI_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/google-vertex-ai", + }); + selectedModel = googleai.chat("models/gemini-1.5-pro-latest"); + console.log("Selected model: ", selectedModel); + break; + case "gpt-4o": + default: + const openai = createOpenAI({ + apiKey: env.OPENAI_API_KEY, + baseURL: + "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai", + compatibility: "strict", + }); + selectedModel = openai.chat("gpt-4o-mini"); + break; + } + + return { store, model: selectedModel }; +} \ No newline at end of file diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processNotes.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processNotes.ts new file mode 100644 index 00000000..466690cc --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processNotes.ts @@ -0,0 +1,36 @@ +import { Result, Ok, Err } from "../../errors/results"; +import { BaseError } from "../../errors/baseError"; +import { Metadata } from "../utils/get-metadata"; + +class ProcessNotesError extends BaseError { + constructor(message?: string, source?: string) { + super("[Note Processing Error]", message, source); + } +} + +type ProcessNoteResult = { + noteContent: { noteId: number; noteContent: string }; + metadata: Metadata; +}; + +export function processNote( + content: string, +): Result { + try { + const pageContent = content; + const noteId = new Date().getTime(); + + const metadata = { + baseUrl: `https://supermemory.ai/note/${noteId}`, + description: `Note created at ${new Date().toLocaleString()}`, + image: "https://supermemory.ai/logo.png", + title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, + }; + + const noteContent = { noteId: noteId, noteContent: pageContent }; + return Ok({ noteContent, metadata }); + } catch (e) { + console.error("[Note Processing Error]", e); + return Err(new ProcessNotesError((e as Error).message, "processNote")); + } +} diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts new file mode 100644 index 00000000..6b28c975 --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts @@ -0,0 +1,42 @@ +import { Result, Ok, Err, isErr } from "../../errors/results"; +import { BaseError } from "../../errors/baseError"; +import { getMetaData, Metadata } from "../utils/get-metadata"; + +class ProcessPageError extends BaseError { + constructor(message?: string, source?: string) { + super("[Page Proceessing Error]", message, source); + } +} + +type PageProcessResult = { pageContent: string; metadata: Metadata }; + +export async function processPage( + url: string, +): Promise> { + try { + const response = await fetch("https://md.dhr.wtf/?url=" + url, { + headers: { + Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + }, + }); + const pageContent = await response.text(); + if (!pageContent) { + return Err( + new ProcessPageError( + "Failed to get response form markdowner", + "processPage", + ), + ); + } + console.log("[This is the page content]", pageContent); + const metadataResult = await getMetaData(url); + if (isErr(metadataResult)) { + throw metadataResult.error; + } + const metadata = metadataResult.value; + return Ok({ pageContent, metadata }); + } catch (e) { + console.error("[Page Processing Error]", e); + return Err(new ProcessPageError((e as Error).message, "processPage")); + } +} diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts new file mode 100644 index 00000000..ef5d9f5b --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts @@ -0,0 +1,81 @@ +import { Tweet } from "react-tweet/api"; +import { Result, Ok, Err, isErr } from "../../errors/results"; +import { BaseError } from "../../errors/baseError"; +import { getMetaData, Metadata } from "../utils/get-metadata"; +import { tweetToMd } from "@repo/shared-types/utils"; // can I do this? + +class ProcessTweetError extends BaseError { + constructor(message?: string, source?: string) { + super("[Tweet Proceessing Error]", message, source); + } +} + +type GetTweetResult = Tweet; + +export const getTweetData = async ( + tweetID: string, +): Promise> => { + try { + console.log("is fetch defined here?"); + const url = `https://cdn.syndication.twimg.com/tweet-result?id=${tweetID}&lang=en&features=tfw_timeline_list%3A%3Btfw_follower_count_sunset%3Atrue%3Btfw_tweet_edit_backend%3Aon%3Btfw_refsrc_session%3Aon%3Btfw_fosnr_soft_interventions_enabled%3Aon%3Btfw_show_birdwatch_pivots_enabled%3Aon%3Btfw_show_business_verified_badge%3Aon%3Btfw_duplicate_scribes_to_settings%3Aon%3Btfw_use_profile_image_shape_enabled%3Aon%3Btfw_show_blue_verified_badge%3Aon%3Btfw_legacy_timeline_sunset%3Atrue%3Btfw_show_gov_verified_badge%3Aon%3Btfw_show_business_affiliate_badge%3Aon%3Btfw_tweet_edit_frontend%3Aon&token=4c2mmul6mnh`; + + const resp = await fetch(url, { + headers: { + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", + Accept: "application/json", + "Accept-Language": "en-US,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br", + Connection: "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Cache-Control": "max-age=0", + TE: "Trailers", + }, + }); + console.log(resp.status); + + const data = (await resp.json()) as Tweet; + + return Ok(data); + } catch (e) { + console.error("[Tweet Proceessing Error]", e); + return Err(new ProcessTweetError(e, "getTweetData")); + } +}; + +export const getThreadData = async ( + tweetUrl: string, + cf_thread_endpoint: string, + authKey: string, +): Promise> => { + const threadRequest = await fetch(cf_thread_endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: authKey, + }, + body: JSON.stringify({ url: tweetUrl }), + }); + if (threadRequest.status !== 200) { + return Err( + new ProcessTweetError( + `Failed to fetch the thread: ${tweetUrl}, Reason: ${threadRequest.statusText}`, + "getThreadData", + ), + ); + } + + const thread = await threadRequest.text(); + console.log("[thread response]"); + + if (thread.trim().length === 2) { + console.log("Thread is an empty array"); + return Err( + new ProcessTweetError( + "[THREAD FETCHING SERVICE] Got no content form thread worker", + "getThreadData", + ), + ); + } + return Ok(thread); +}; diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts new file mode 100644 index 00000000..8657603d --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -0,0 +1,204 @@ +import { Env, PageOrNoteChunks, TweetChunks, vectorObj } from "../types"; +import { typeDecider } from "./utils/typeDecider"; +import { isErr, wrap } from "../errors/results"; +import { processNote } from "./helpers/processNotes"; +import { processPage } from "./helpers/processPage"; +import { getThreadData, getTweetData } from "./helpers/processTweet"; +import { tweetToMd } from "@repo/shared-types/utils"; +import { initQQuery } from "./helpers/initQuery"; +import { chunkNote, chunkPage } from "./chunkers/chunkPageOrNotes"; +import { chunkThread } from "./chunkers/chunkTweet"; +import { batchCreateChunksAndEmbeddings } from "../helper"; +import { z } from "zod"; +import { Metadata } from "./utils/get-metadata"; +import { BaseError } from "../errors/baseError"; +import { database } from "../db"; +import { storedContent, space, contentToSpace } from "../db/schema"; +import { and, eq, inArray, sql } from "drizzle-orm"; + +class VectorInsertError extends BaseError { + constructor(message?: string, source?: string) { + super("[Vector Insert Error]", message, source); + } +} +const vectorErrorFactory = (err: Error) => new VectorInsertError(err.message); + +class D1InsertError extends BaseError { + constructor(message?: string, source?: string) { + super("[D1 Insert Error]", message, source); + } +} + +export async function queue( + batch: MessageBatch<{ content: string; space: Array; user: string }>, + env: Env, +): Promise { + console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); + for (let message of batch.messages) { + console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); + console.log("is thie even running?", message.body); + const body = message.body; + console.log("v got shit in the queue", body); + + const typeResult = typeDecider(body.content); + + if (isErr(typeResult)) { + throw typeResult.error; + } + console.log(typeResult.value); + const type = typeResult.value; + + let pageContent: string; + let vectorData: string; + let metadata: Metadata; + let storeToSpaces = body.space; + let chunks: TweetChunks | PageOrNoteChunks; + let noteId = 0; + switch (type) { + case "note": { + console.log("note hit"); + const note = processNote(body.content); + if (isErr(note)) { + throw note.error; + } + pageContent = note.value.noteContent.noteContent; + noteId = note.value.noteContent.noteId; + metadata = note.value.metadata; + vectorData = pageContent; + chunks = chunkNote(pageContent); + break; + } + case "page": { + console.log("page hit"); + const page = await processPage(body.content); + if (isErr(page)) { + throw page.error; + } + pageContent = page.value.pageContent; + metadata = page.value.metadata; + vectorData = pageContent; + chunks = chunkPage(pageContent); + break; + } + + case "tweet": { + console.log("tweet hit"); + console.log(body.content.split("/").pop()); + const tweet = await getTweetData(body.content.split("/").pop()); + console.log(tweet); + const thread = await getThreadData( + body.content, + env.THREAD_CF_WORKER, + env.THREAD_CF_AUTH, + ); + + if (isErr(tweet)) { + throw tweet.error; + } + pageContent = tweetToMd(tweet.value); + console.log(pageContent); + metadata = { + baseUrl: body.content, + description: tweet.value.text.slice(0, 200), + image: tweet.value.user.profile_image_url_https, + title: `Tweet by ${tweet.value.user.name}`, + }; + if (isErr(thread)) { + console.log("Thread worker is down!"); + vectorData = JSON.stringify(pageContent); + console.error(thread.error); + } else { + vectorData = thread.value; + } + chunks = chunkThread(vectorData); + break; + } + } + + // see what's up with the storedToSpaces in this block + const { store } = await initQQuery(env); + + type body = z.infer; + + const Chunkbody: body = { + pageContent: pageContent, + spaces: storeToSpaces.map((spaceId) => spaceId.toString()), + user: body.user, + type: type, + url: metadata.baseUrl, + description: metadata.description, + title: metadata.description, + }; + const vectorResult = await wrap( + batchCreateChunksAndEmbeddings({ + store: store, + body: Chunkbody, + chunks: chunks, + env: env, + }), + vectorErrorFactory, + ); + + if (isErr(vectorResult)) { + throw vectorResult.error; + } + const saveToDbUrl = + (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + + "#supermemory-user-" + + body.user; + let contentId: number; + const db = database(env); + const insertResponse = await db + .insert(storedContent) + .values({ + content: pageContent as string, + title: metadata.title, + description: metadata.description, + url: saveToDbUrl, + baseUrl: saveToDbUrl, + image: metadata.image, + savedAt: new Date(), + userId: body.user, + type: type, + noteId: noteId, + }) + .returning({ id: storedContent.id }); + + if (!insertResponse[0]?.id) { + throw new D1InsertError( + "something went worng when inserting to database", + "inresertResponse", + ); + } + contentId = insertResponse[0]?.id; + if (storeToSpaces.length > 0) { + // Adding the many-to-many relationship between content and spaces + const spaceData = await db + .select() + .from(space) + .where(and(inArray(space.id, storeToSpaces), eq(space.user, body.user))) + .all(); + + await Promise.all( + spaceData.map(async (s) => { + await db + .insert(contentToSpace) + .values({ contentId: contentId, spaceId: s.id }); + + await db.update(space).set({ numItems: s.numItems + 1 }); + }), + ); + } + } +} + +/* +To do: +1. Abstract and shitft the entrie creatememory function to the queue consumer --> Hopefully done +2. Make the front end use that instead of whatever khichidi is going on right now +3. remove getMetada form the lib file as it's not being used anywhere else +4. Figure out the limit stuff ( server action for that seems fine because no use in limiting after they already in the queue rigth? ) +5. Figure out the initQuery stuff ( ;( ) --> This is a bad way of doing stuff :0 +6. How do I hande the content already exists wala use case? +7. Figure out retry and not add shit to the vectirze over and over again on failure +*/ diff --git a/apps/cf-ai-backend/src/queueConsumer/utils/get-metadata.ts b/apps/cf-ai-backend/src/queueConsumer/utils/get-metadata.ts new file mode 100644 index 00000000..95916506 --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/utils/get-metadata.ts @@ -0,0 +1,57 @@ +import * as cheerio from "cheerio"; +import { Result, Ok, Err } from "../../errors/results"; +import { BaseError } from "../../errors/baseError"; + +class GetMetadataError extends BaseError { + constructor(message?: string, source?: string) { + super("[Fetch Metadata Error]", message, source); + } +} +export type Metadata = { + title: string; + description: string; + image: string; + baseUrl: string; +}; +// TODO: THIS SHOULD PROBABLY ALSO FETCH THE OG-IMAGE +export async function getMetaData( + url: string, +): Promise> { + try { + const response = await fetch(url); + const html = await response.text(); + + const $ = cheerio.load(html); + + // Extract the base URL + const baseUrl = url; + + // Extract title + const title = $("title").text().trim(); + + const description = $("meta[name=description]").attr("content") ?? ""; + + const _favicon = + $("link[rel=icon]").attr("href") ?? "https://supermemory.dhr.wtf/web.svg"; + + let favicon = + _favicon.trim().length > 0 + ? _favicon.trim() + : "https://supermemory.dhr.wtf/web.svg"; + if (favicon.startsWith("/")) { + favicon = baseUrl + favicon; + } else if (favicon.startsWith("./")) { + favicon = baseUrl + favicon.slice(1); + } + + return Ok({ + title, + description, + image: favicon, + baseUrl, + }); + } catch (e) { + console.error("[Metadata Fetch Error]", e); + return Err(new GetMetadataError((e as Error).message, "getMetaData")); + } +} diff --git a/apps/cf-ai-backend/src/queueConsumer/utils/typeDecider.ts b/apps/cf-ai-backend/src/queueConsumer/utils/typeDecider.ts new file mode 100644 index 00000000..037ab40c --- /dev/null +++ b/apps/cf-ai-backend/src/queueConsumer/utils/typeDecider.ts @@ -0,0 +1,34 @@ +import { Result, Ok, Err } from "../../errors/results"; +import { BaseError } from "../../errors/baseError"; + +export type contentType = "page" | "tweet" | "note"; + +class GetTypeError extends BaseError { + constructor(message?: string, source?: string) { + super("[Decide Type Error]", message, source); + } +} +export const typeDecider = ( + content: string, +): Result => { + try { + // if the content is a URL, then it's a page. if its a URL with https://x.com/user/status/123, then it's a tweet. else, it's a note. + // do strict checking with regex + if ( + content.match(/https?:\/\/(x\.com|twitter\.com)\/[\w]+\/[\w]+\/[\d]+/) + ) { + return Ok("tweet"); + } else if ( + content.match( + /^(https?:\/\/)?(www\.)?[a-z0-9]+([-.]{1}[a-z0-9]+)*\.[a-z]{2,5}(\/.*)?$/i, + ) + ) { + return Ok("page"); + } else { + return Ok("note"); + } + } catch (e) { + console.error("[Decide Type Error]", e); + return Err(new GetTypeError((e as Error).message, "typeDecider")); + } +}; diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 5ef81f20..5294e0a1 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -1,6 +1,6 @@ import { sourcesZod } from "@repo/shared-types"; import { z } from "zod"; -import { ThreadTweetData } from "./utils/chunkTweet"; +import { ThreadTweetData } from "./queueConsumer/chunkers/chunkTweet"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; @@ -11,13 +11,23 @@ export type Env = { CF_KV_AUTH_TOKEN: string; KV_NAMESPACE_ID: string; CF_ACCOUNT_ID: string; + DATABASE: D1Database; MY_QUEUE: Queue; KV: KVNamespace; + EMBEDCHUNKS_QUEUE: Queue; MYBROWSER: unknown; ANTHROPIC_API_KEY: string; + THREAD_CF_AUTH: string; + THREAD_CF_WORKER: string; NODE_ENV: string; }; +export interface JobData { + content: string; + space: Array; + user: string; +} + export interface TweetData { tweetText: string; postUrl: string; @@ -80,3 +90,8 @@ export const vectorObj = z.object({ user: z.string(), type: z.string().optional().default("page"), }); +export const vectorBody = z.object({ + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), +}); diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts deleted file mode 100644 index 18788dab..00000000 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ /dev/null @@ -1,47 +0,0 @@ -import nlp from "compromise"; - -/** - * Split text into chunks of specified max size with some overlap for continuity. - */ -export default function chunkText( - text: string, - maxChunkSize: number, - overlap: number = 0.2, -): string[] { - const sentences = nlp(text).sentences().out("array"); - const chunks = []; - let currentChunk: string[] = []; - let currentSize = 0; - - for (let i = 0; i < sentences.length; i++) { - const sentence = sentences[i]; - currentChunk.push(sentence); - currentSize += sentence.length; - - if (currentSize >= maxChunkSize) { - // Calculate overlap - const overlapSize = Math.floor(currentChunk.length * overlap); - const chunkText = currentChunk.join(" "); - chunks.push({ - text: chunkText, - start: i - currentChunk.length + 1, - end: i, - }); - - // Prepare the next chunk with overlap - currentChunk = currentChunk.slice(-overlapSize); - currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0); - } - } - - if (currentChunk.length > 0) { - const chunkText = currentChunk.join(" "); - chunks.push({ - text: chunkText, - start: sentences.length - currentChunk.length, - end: sentences.length, - }); - } - - return chunks.map((chunk) => chunk.text); -} diff --git a/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts b/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts deleted file mode 100644 index f04ed0c5..00000000 --- a/apps/cf-ai-backend/src/utils/chunkPageOrNotes.ts +++ /dev/null @@ -1,13 +0,0 @@ -import chunkText from "./chonker"; -import { PageOrNoteChunks } from "../types"; -export function chunkPage(pageContent: string): PageOrNoteChunks { - const chunks = chunkText(pageContent, 1536); - - return { type: "page", chunks: chunks }; -} - -export function chunkNote(noteContent: string): PageOrNoteChunks { - const chunks = chunkText(noteContent, 1536); - - return { type: "note", chunks: chunks }; -} diff --git a/apps/cf-ai-backend/src/utils/chunkTweet.ts b/apps/cf-ai-backend/src/utils/chunkTweet.ts deleted file mode 100644 index 78f0f261..00000000 --- a/apps/cf-ai-backend/src/utils/chunkTweet.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { TweetChunks } from "../types"; -import chunkText from "./chonker"; -import { getRawTweet } from "@repo/shared-types/utils"; - -interface Tweet { - id: string; - text: string; - links: Array; - images: Array; - videos: Array; -} -interface Metadata { - tweetId: string; - tweetLinks: any[]; - tweetVids: any[]; - tweetImages: any[]; -} - -export interface ThreadTweetData { - chunkedTweet: string[]; - metadata: Metadata; -} - -export function chunkThread(threadText: string): TweetChunks { - const thread = JSON.parse(threadText); - if (typeof thread == "string") { - console.log("DA WORKER FAILED DO SOMEHTING FIX DA WROKER"); - const rawTweet = getRawTweet(thread); - const parsedTweet: any = JSON.parse(rawTweet); - - const chunkedTweet = chunkText(parsedTweet.text, 1536); - const metadata: Metadata = { - tweetId: parsedTweet.id_str, - tweetLinks: parsedTweet.entities.urls.map((url: any) => url.expanded_url), - tweetVids: - parsedTweet.extended_entities?.media - .filter((media: any) => media.type === "video") - .map((media: any) => media.video_info!.variants[0].url) || [], - tweetImages: - parsedTweet.extended_entities?.media - .filter((media: any) => media.type === "photo") - .map((media: any) => media.media_url_https!) || [], - }; - - const chunks = [{ chunkedTweet: chunkedTweet, metadata }]; - - return { type: "tweet", chunks }; - } else { - console.log(JSON.stringify(thread)); - const chunkedTweets = thread.map((tweet: Tweet) => { - const chunkedTweet = chunkText(tweet.text, 1536); - - const metadata = { - tweetId: tweet.id, - tweetLinks: tweet.links, - tweetVids: tweet.videos, - tweetImages: tweet.images, - }; - - return { chunkedTweet, metadata }; - }); - - return { type: "tweet", chunks: chunkedTweets }; - } -} diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index 9a831921..910226a5 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -15,7 +15,7 @@ import { import { ServerActionReturnType } from "./types"; import { auth } from "../../server/auth"; import { Tweet } from "react-tweet/api"; -import { getMetaData } from "@/lib/get-metadata"; +// import { getMetaData } from "@/lib/get-metadata"; import { and, eq, inArray, sql } from "drizzle-orm"; import { LIMITS } from "@/lib/constants"; import { ChatHistory } from "@repo/shared-types"; @@ -197,122 +197,16 @@ export const createMemory = async (input: { return { error: "Not authenticated", success: false }; } - const type = typeDecider(input.content); - - let pageContent = input.content; - let metadata: Awaited>; - let vectorData: string; - - if (!(await limit(data.user.id, type))) { - return { - success: false, - data: 0, - error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, - }; - } - - let noteId = 0; - - if (type === "page") { - const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { - headers: { - Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, - }, - }); - pageContent = await response.text(); - vectorData = pageContent; - try { - metadata = await getMetaData(input.content); - } catch (e) { - return { - success: false, - error: "Failed to fetch metadata for the page. Please try again later.", - }; - } - } else if (type === "tweet") { - //Request the worker for the entire thread - - let thread: string; - let errorOccurred: boolean = false; - - try { - const cf_thread_endpoint = process.env.THREAD_CF_WORKER; - const authKey = process.env.THREAD_CF_AUTH; - const threadRequest = await fetch(cf_thread_endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: authKey, - }, - body: JSON.stringify({ url: input.content }), - }); - - if (threadRequest.status !== 200) { - throw new Error( - `Failed to fetch the thread: ${input.content}, Reason: ${threadRequest.statusText}`, - ); - } - - thread = await threadRequest.text(); - if (thread.trim().length === 2) { - console.log("Thread is an empty array"); - throw new Error( - "[THREAD FETCHING SERVICE] Got no content form thread worker", - ); - } - } catch (e) { - console.log("[THREAD FETCHING SERVICE] Failed to fetch the thread", e); - errorOccurred = true; - } - - const tweet = await getTweetData(input.content.split("/").pop() as string); - - pageContent = tweetToMd(tweet); - console.log("THis ishte page content!!", pageContent); - //@ts-ignore - vectorData = errorOccurred ? JSON.stringify(pageContent) : thread; - metadata = { - baseUrl: input.content, - description: tweet.text.slice(0, 200), - image: tweet.user.profile_image_url_https, - title: `Tweet by ${tweet.user.name}`, - }; - } else if (type === "note") { - pageContent = input.content; - vectorData = pageContent; - noteId = new Date().getTime(); - metadata = { - baseUrl: `https://supermemory.ai/note/${noteId}`, - description: `Note created at ${new Date().toLocaleString()}`, - image: "https://supermemory.ai/logo.png", - title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, - }; - } else { - return { - success: false, - data: 0, - error: "Invalid type", - }; - } - - let storeToSpaces = input.spaces; - - if (!storeToSpaces) { - storeToSpaces = []; - } - - const vectorSaveResponse = await fetch( + + // make the backend reqeust for the queue here + const vectorSaveResponses = await fetch( `${process.env.BACKEND_BASE_URL}/api/add`, { method: "POST", body: JSON.stringify({ - pageContent: vectorData, - title: metadata.title, - description: metadata.description, - url: metadata.baseUrl, - spaces: storeToSpaces.map((spaceId) => spaceId.toString()), + url: input.content, + spaces: input.spaces, user: data.user.id, - type, }), headers: { "Content-Type": "application/json", @@ -321,125 +215,249 @@ export const createMemory = async (input: { }, ); - if (!vectorSaveResponse.ok) { - const errorData = await vectorSaveResponse.text(); - console.error(errorData); - return { - success: false, - data: 0, - error: `Failed to save to vector store. Backend returned error: ${errorData}`, - }; - } - - let contentId: number; - - const response = (await vectorSaveResponse.json()) as { - status: string; - chunkedInput: string; - message?: string; - }; - - try { - if (response.status !== "ok") { - if (response.status === "error") { - return { - success: false, - data: 0, - error: response.message, - }; - } else { - return { - success: false, - data: 0, - error: `Failed to save to vector store. Backend returned error: ${response.message}`, - }; - } - } - } catch (e) { - return { - success: false, - data: 0, - error: `Failed to save to vector store. Backend returned error: ${e}`, - }; - } - - const saveToDbUrl = - (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + - "#supermemory-user-" + - data.user.id; - - // Insert into database - try { - const insertResponse = await db - .insert(storedContent) - .values({ - content: pageContent, - title: metadata.title, - description: metadata.description, - url: saveToDbUrl, - baseUrl: saveToDbUrl, - image: metadata.image, - savedAt: new Date(), - userId: data.user.id, - type, - noteId, - }) - .returning({ id: storedContent.id }); - revalidatePath("/memories"); - revalidatePath("/home"); - - if (!insertResponse[0]?.id) { - return { - success: false, - data: 0, - error: "Something went wrong while saving the document to the database", - }; - } - - contentId = insertResponse[0]?.id; - } catch (e) { - const error = e as Error; - console.log("Error: ", error.message); - - if ( - error.message.includes( - "D1_ERROR: UNIQUE constraint failed: storedContent.baseUrl", - ) - ) { - return { - success: false, - data: 0, - error: "Content already exists", - }; - } - - return { - success: false, - data: 0, - error: "Failed to save to database with error: " + error.message, - }; - } - - if (storeToSpaces.length > 0) { - // Adding the many-to-many relationship between content and spaces - const spaceData = await db - .select() - .from(space) - .where( - and(inArray(space.id, storeToSpaces), eq(space.user, data.user.id)), - ) - .all(); - - await Promise.all( - spaceData.map(async (s) => { - await db - .insert(contentToSpace) - .values({ contentId: contentId, spaceId: s.id }); - - await db.update(space).set({ numItems: s.numItems + 1 }); - }), - ); - } +// const type = typeDecider(input.content); + +// let pageContent = input.content; +// let metadata: Awaited>; +// let vectorData: string; + +// if (!(await limit(data.user.id, type))) { +// return { +// success: false, +// data: 0, +// error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, +// }; +// } + +// let noteId = 0; + +// if (type === "page") { +// const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { +// headers: { +// Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, +// }, +// }); +// pageContent = await response.text(); +// vectorData = pageContent; +// try { +// metadata = await getMetaData(input.content); +// } catch (e) { +// return { +// success: false, +// error: "Failed to fetch metadata for the page. Please try again later.", +// }; +// } +// } else if (type === "tweet") { +// //Request the worker for the entire thread + +// let thread: string; +// let errorOccurred: boolean = false; + +// try { +// const cf_thread_endpoint = process.env.THREAD_CF_WORKER; +// const authKey = process.env.THREAD_CF_AUTH; +// const threadRequest = await fetch(cf_thread_endpoint, { +// method: "POST", +// headers: { +// "Content-Type": "application/json", +// Authorization: authKey, +// }, +// body: JSON.stringify({ url: input.content }), +// }); + +// if (threadRequest.status !== 200) { +// throw new Error( +// `Failed to fetch the thread: ${input.content}, Reason: ${threadRequest.statusText}`, +// ); +// } + +// thread = await threadRequest.text(); +// if (thread.trim().length === 2) { +// console.log("Thread is an empty array"); +// throw new Error( +// "[THREAD FETCHING SERVICE] Got no content form thread worker", +// ); +// } +// } catch (e) { +// console.log("[THREAD FETCHING SERVICE] Failed to fetch the thread", e); +// errorOccurred = true; +// } + +// const tweet = await getTweetData(input.content.split("/").pop() as string); + +// pageContent = tweetToMd(tweet); +// console.log("THis ishte page content!!", pageContent); +// //@ts-ignore +// vectorData = errorOccurred ? JSON.stringify(pageContent) : thread; +// metadata = { +// baseUrl: input.content, +// description: tweet.text.slice(0, 200), +// image: tweet.user.profile_image_url_https, +// title: `Tweet by ${tweet.user.name}`, +// }; +// } else if (type === "note") { +// pageContent = input.content; +// vectorData = pageContent; +// noteId = new Date().getTime(); +// metadata = { +// baseUrl: `https://supermemory.ai/note/${noteId}`, +// description: `Note created at ${new Date().toLocaleString()}`, +// image: "https://supermemory.ai/logo.png", +// title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, +// }; +// } else { +// return { +// success: false, +// data: 0, +// error: "Invalid type", +// }; +// } + +// let storeToSpaces = input.spaces; + +// if (!storeToSpaces) { +// storeToSpaces = []; +// } + +// const vectorSaveResponse = await fetch( +// `${process.env.BACKEND_BASE_URL}/api/add`, +// { +// method: "POST", +// body: JSON.stringify({ +// pageContent: vectorData, +// title: metadata.title, +// description: metadata.description, +// url: metadata.baseUrl, +// spaces: storeToSpaces.map((spaceId) => spaceId.toString()), +// user: data.user.id, +// type, +// }), +// headers: { +// "Content-Type": "application/json", +// Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, +// }, +// }, +// ); + +// if (!vectorSaveResponse.ok) { +// const errorData = await vectorSaveResponse.text(); +// console.error(errorData); +// return { +// success: false, +// data: 0, +// error: `Failed to save to vector store. Backend returned error: ${errorData}`, +// }; +// } + +// let contentId: number; + +// const response = (await vectorSaveResponse.json()) as { +// status: string; +// chunkedInput: string; +// message?: string; +// }; + +// try { +// if (response.status !== "ok") { +// if (response.status === "error") { +// return { +// success: false, +// data: 0, +// error: response.message, +// }; +// } else { +// return { +// success: false, +// data: 0, +// error: `Failed to save to vector store. Backend returned error: ${response.message}`, +// }; +// } +// } +// } catch (e) { +// return { +// success: false, +// data: 0, +// error: `Failed to save to vector store. Backend returned error: ${e}`, +// }; +// } + +// const saveToDbUrl = +// (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + +// "#supermemory-user-" + +// data.user.id; + +// // Insert into database +// try { +// const insertResponse = await db +// .insert(storedContent) +// .values({ +// content: pageContent, +// title: metadata.title, +// description: metadata.description, +// url: saveToDbUrl, +// baseUrl: saveToDbUrl, +// image: metadata.image, +// savedAt: new Date(), +// userId: data.user.id, +// type, +// noteId, +// }) +// .returning({ id: storedContent.id }); +// revalidatePath("/memories"); +// revalidatePath("/home"); + +// if (!insertResponse[0]?.id) { +// return { +// success: false, +// data: 0, +// error: "Something went wrong while saving the document to the database", +// }; +// } + +// contentId = insertResponse[0]?.id; +// } catch (e) { +// const error = e as Error; +// console.log("Error: ", error.message); + +// if ( +// error.message.includes( +// "D1_ERROR: UNIQUE constraint failed: storedContent.baseUrl", +// ) +// ) { +// return { +// success: false, +// data: 0, +// error: "Content already exists", +// }; +// } + +// return { +// success: false, +// data: 0, +// error: "Failed to save to database with error: " + error.message, +// }; +// } + +// if (storeToSpaces.length > 0) { +// // Adding the many-to-many relationship between content and spaces +// const spaceData = await db +// .select() +// .from(space) +// .where( +// and(inArray(space.id, storeToSpaces), eq(space.user, data.user.id)), +// ) +// .all(); + +// await Promise.all( +// spaceData.map(async (s) => { +// await db +// .insert(contentToSpace) +// .values({ contentId: contentId, spaceId: s.id }); + +// await db.update(space).set({ numItems: s.numItems + 1 }); +// }), +// ); +// } return { success: true, @@ -457,6 +475,7 @@ export const createChatThread = async ( return { error: "Not authenticated", success: false }; } + const thread = await db .insert(chatThreads) .values({ diff --git a/apps/web/lib/get-metadata.ts b/apps/web/lib/get-metadata.ts deleted file mode 100644 index c81397ff..00000000 --- a/apps/web/lib/get-metadata.ts +++ /dev/null @@ -1,40 +0,0 @@ -"use server"; -import * as cheerio from "cheerio"; - -// TODO: THIS SHOULD PROBABLY ALSO FETCH THE OG-IMAGE -export async function getMetaData(url: string) { - const response = await fetch(url); - const html = await response.text(); - - const $ = cheerio.load(html); - - // Extract the base URL - const baseUrl = url; - - // Extract title - const title = $("title").text().trim(); - - const description = $("meta[name=description]").attr("content") ?? ""; - - const _favicon = - $("link[rel=icon]").attr("href") ?? "https://supermemory.dhr.wtf/web.svg"; - - let favicon = - _favicon.trim().length > 0 - ? _favicon.trim() - : "https://supermemory.dhr.wtf/web.svg"; - if (favicon.startsWith("/")) { - favicon = baseUrl + favicon; - } else if (favicon.startsWith("./")) { - favicon = baseUrl + favicon.slice(1); - } - - // Prepare the metadata object - const metadata = { - title, - description, - image: favicon, - baseUrl, - }; - return metadata; -} diff --git a/apps/web/migrations/0000_omniscient_stick.sql b/apps/web/migrations/0000_omniscient_stick.sql new file mode 100644 index 00000000..542d6d0e --- /dev/null +++ b/apps/web/migrations/0000_omniscient_stick.sql @@ -0,0 +1,152 @@ +CREATE TABLE `account` ( + `userId` text NOT NULL, + `type` text NOT NULL, + `provider` text NOT NULL, + `providerAccountId` text NOT NULL, + `refresh_token` text, + `access_token` text, + `expires_at` integer, + `token_type` text, + `scope` text, + `id_token` text, + `session_state` text, + PRIMARY KEY(`provider`, `providerAccountId`), + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `authenticator` ( + `credentialID` text NOT NULL, + `userId` text NOT NULL, + `providerAccountId` text NOT NULL, + `credentialPublicKey` text NOT NULL, + `counter` integer NOT NULL, + `credentialDeviceType` text NOT NULL, + `credentialBackedUp` integer NOT NULL, + `transports` text, + PRIMARY KEY(`credentialID`, `userId`), + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `canvas` ( + `id` text PRIMARY KEY NOT NULL, + `title` text DEFAULT 'Untitled' NOT NULL, + `description` text DEFAULT 'Untitled' NOT NULL, + `url` text DEFAULT '' NOT NULL, + `userId` text NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `chatHistory` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `threadId` text NOT NULL, + `question` text NOT NULL, + `answerParts` text, + `answerSources` text, + `answerJustification` text, + `createdAt` integer DEFAULT '"2024-07-29T17:06:56.122Z"' NOT NULL, + FOREIGN KEY (`threadId`) REFERENCES `chatThread`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `chatThread` ( + `id` text PRIMARY KEY NOT NULL, + `firstMessage` text NOT NULL, + `userId` text NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `contentToSpace` ( + `contentId` integer NOT NULL, + `spaceId` integer NOT NULL, + PRIMARY KEY(`contentId`, `spaceId`), + FOREIGN KEY (`contentId`) REFERENCES `storedContent`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `jobs` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `userId` text NOT NULL, + `url` text NOT NULL, + `status` text NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `lastAttemptAt` integer, + `error` blob, + `createdAt` integer NOT NULL, + `updatedAt` integer NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `session` ( + `sessionToken` text PRIMARY KEY NOT NULL, + `userId` text NOT NULL, + `expires` integer NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `space` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `name` text DEFAULT 'none' NOT NULL, + `user` text(255), + `createdAt` integer NOT NULL, + `numItems` integer DEFAULT 0 NOT NULL, + FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `spacesAccess` ( + `spaceId` integer NOT NULL, + `userEmail` text NOT NULL, + PRIMARY KEY(`spaceId`, `userEmail`), + FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `storedContent` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `content` text NOT NULL, + `title` text(255), + `description` text(255), + `url` text NOT NULL, + `savedAt` integer NOT NULL, + `baseUrl` text(255), + `ogImage` text(255), + `type` text DEFAULT 'page', + `image` text(255), + `user` text, + `noteId` integer, + FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `user` ( + `id` text PRIMARY KEY NOT NULL, + `name` text, + `email` text NOT NULL, + `emailVerified` integer, + `image` text, + `telegramId` text, + `hasOnboarded` integer DEFAULT false +); +--> statement-breakpoint +CREATE TABLE `verificationToken` ( + `identifier` text NOT NULL, + `token` text NOT NULL, + `expires` integer NOT NULL, + PRIMARY KEY(`identifier`, `token`) +); +--> statement-breakpoint +CREATE UNIQUE INDEX `authenticator_credentialID_unique` ON `authenticator` (`credentialID`);--> statement-breakpoint +CREATE INDEX `canvas_user_userId` ON `canvas` (`userId`);--> statement-breakpoint +CREATE INDEX `chatHistory_thread_idx` ON `chatHistory` (`threadId`);--> statement-breakpoint +CREATE INDEX `chatThread_user_idx` ON `chatThread` (`userId`);--> statement-breakpoint +CREATE INDEX `jobs_userId_idx` ON `jobs` (`userId`);--> statement-breakpoint +CREATE INDEX `jobs_status_idx` ON `jobs` (`status`);--> statement-breakpoint +CREATE INDEX `jobs_createdAt_idx` ON `jobs` (`createdAt`);--> statement-breakpoint +CREATE INDEX `jobs_url_idx` ON `jobs` (`url`);--> statement-breakpoint +CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint +CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint +CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint +CREATE UNIQUE INDEX `storedContent_baseUrl_unique` ON `storedContent` (`baseUrl`);--> statement-breakpoint +CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint +CREATE INDEX `storedContent_savedAt_idx` ON `storedContent` (`savedAt`);--> statement-breakpoint +CREATE INDEX `storedContent_title_idx` ON `storedContent` (`title`);--> statement-breakpoint +CREATE INDEX `storedContent_user_idx` ON `storedContent` (`user`);--> statement-breakpoint +CREATE INDEX `users_email_idx` ON `user` (`email`);--> statement-breakpoint +CREATE INDEX `users_telegram_idx` ON `user` (`telegramId`);--> statement-breakpoint +CREATE INDEX `users_id_idx` ON `user` (`id`); \ No newline at end of file diff --git a/apps/web/migrations/0000_steep_moira_mactaggert.sql b/apps/web/migrations/0000_steep_moira_mactaggert.sql deleted file mode 100644 index 5813639d..00000000 --- a/apps/web/migrations/0000_steep_moira_mactaggert.sql +++ /dev/null @@ -1,135 +0,0 @@ -CREATE TABLE `account` ( - `userId` text NOT NULL, - `type` text NOT NULL, - `provider` text NOT NULL, - `providerAccountId` text NOT NULL, - `refresh_token` text, - `access_token` text, - `expires_at` integer, - `token_type` text, - `scope` text, - `id_token` text, - `session_state` text, - PRIMARY KEY(`provider`, `providerAccountId`), - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `authenticator` ( - `credentialID` text NOT NULL, - `userId` text NOT NULL, - `providerAccountId` text NOT NULL, - `credentialPublicKey` text NOT NULL, - `counter` integer NOT NULL, - `credentialDeviceType` text NOT NULL, - `credentialBackedUp` integer NOT NULL, - `transports` text, - PRIMARY KEY(`credentialID`, `userId`), - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `canvas` ( - `id` text PRIMARY KEY NOT NULL, - `title` text DEFAULT 'Untitled' NOT NULL, - `description` text DEFAULT 'Untitled' NOT NULL, - `url` text DEFAULT '' NOT NULL, - `userId` text NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `chatHistory` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `threadId` text NOT NULL, - `question` text NOT NULL, - `answerParts` text, - `answerSources` text, - `answerJustification` text, - `createdAt` integer DEFAULT '"2024-07-25T22:31:50.848Z"' NOT NULL, - FOREIGN KEY (`threadId`) REFERENCES `chatThread`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `chatThread` ( - `id` text PRIMARY KEY NOT NULL, - `firstMessage` text NOT NULL, - `userId` text NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `contentToSpace` ( - `contentId` integer NOT NULL, - `spaceId` integer NOT NULL, - PRIMARY KEY(`contentId`, `spaceId`), - FOREIGN KEY (`contentId`) REFERENCES `storedContent`(`id`) ON UPDATE no action ON DELETE cascade, - FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `session` ( - `sessionToken` text PRIMARY KEY NOT NULL, - `userId` text NOT NULL, - `expires` integer NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `space` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text DEFAULT 'none' NOT NULL, - `user` text(255), - `createdAt` integer NOT NULL, - `numItems` integer DEFAULT 0 NOT NULL, - FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `spacesAccess` ( - `spaceId` integer NOT NULL, - `userEmail` text NOT NULL, - PRIMARY KEY(`spaceId`, `userEmail`), - FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `storedContent` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `content` text NOT NULL, - `title` text(255), - `description` text(255), - `url` text NOT NULL, - `savedAt` integer NOT NULL, - `baseUrl` text(255), - `ogImage` text(255), - `type` text DEFAULT 'page', - `image` text(255), - `user` text, - `noteId` integer, - FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `user` ( - `id` text PRIMARY KEY NOT NULL, - `name` text, - `email` text NOT NULL, - `emailVerified` integer, - `image` text, - `telegramId` text, - `hasOnboarded` integer DEFAULT false -); ---> statement-breakpoint -CREATE TABLE `verificationToken` ( - `identifier` text NOT NULL, - `token` text NOT NULL, - `expires` integer NOT NULL, - PRIMARY KEY(`identifier`, `token`) -); ---> statement-breakpoint -CREATE UNIQUE INDEX `authenticator_credentialID_unique` ON `authenticator` (`credentialID`);--> statement-breakpoint -CREATE INDEX `canvas_user_userId` ON `canvas` (`userId`);--> statement-breakpoint -CREATE INDEX `chatHistory_thread_idx` ON `chatHistory` (`threadId`);--> statement-breakpoint -CREATE INDEX `chatThread_user_idx` ON `chatThread` (`userId`);--> statement-breakpoint -CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint -CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint -CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint -CREATE UNIQUE INDEX `storedContent_baseUrl_unique` ON `storedContent` (`baseUrl`);--> statement-breakpoint -CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint -CREATE INDEX `storedContent_savedAt_idx` ON `storedContent` (`savedAt`);--> statement-breakpoint -CREATE INDEX `storedContent_title_idx` ON `storedContent` (`title`);--> statement-breakpoint -CREATE INDEX `storedContent_user_idx` ON `storedContent` (`user`);--> statement-breakpoint -CREATE INDEX `users_email_idx` ON `user` (`email`);--> statement-breakpoint -CREATE INDEX `users_telegram_idx` ON `user` (`telegramId`);--> statement-breakpoint -CREATE INDEX `users_id_idx` ON `user` (`id`); \ No newline at end of file diff --git a/apps/web/migrations/meta/0000_snapshot.json b/apps/web/migrations/meta/0000_snapshot.json index a7689010..8141d674 100644 --- a/apps/web/migrations/meta/0000_snapshot.json +++ b/apps/web/migrations/meta/0000_snapshot.json @@ -1,822 +1,1027 @@ { - "version": "6", - "dialect": "sqlite", - "id": "8705302a-eae7-4fbf-9ce8-8ae23df228a2", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "account": { - "name": "account", - "columns": { - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "refresh_token": { - "name": "refresh_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "access_token": { - "name": "access_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expires_at": { - "name": "expires_at", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "token_type": { - "name": "token_type", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "scope": { - "name": "scope", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "id_token": { - "name": "id_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "session_state": { - "name": "session_state", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "account_userId_user_id_fk": { - "name": "account_userId_user_id_fk", - "tableFrom": "account", - "tableTo": "user", - "columnsFrom": ["userId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "account_provider_providerAccountId_pk": { - "columns": ["provider", "providerAccountId"], - "name": "account_provider_providerAccountId_pk" - } - }, - "uniqueConstraints": {} - }, - "authenticator": { - "name": "authenticator", - "columns": { - "credentialID": { - "name": "credentialID", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialPublicKey": { - "name": "credentialPublicKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "counter": { - "name": "counter", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialDeviceType": { - "name": "credentialDeviceType", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialBackedUp": { - "name": "credentialBackedUp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "transports": { - "name": "transports", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "authenticator_credentialID_unique": { - "name": "authenticator_credentialID_unique", - "columns": ["credentialID"], - "isUnique": true - } - }, - "foreignKeys": { - "authenticator_userId_user_id_fk": { - "name": "authenticator_userId_user_id_fk", - "tableFrom": "authenticator", - "tableTo": "user", - "columnsFrom": ["userId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "authenticator_userId_credentialID_pk": { - "columns": ["credentialID", "userId"], - "name": "authenticator_userId_credentialID_pk" - } - }, - "uniqueConstraints": {} - }, - "canvas": { - "name": "canvas", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "title": { - "name": "title", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "''" - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "canvas_user_userId": { - "name": "canvas_user_userId", - "columns": ["userId"], - "isUnique": false - } - }, - "foreignKeys": { - "canvas_userId_user_id_fk": { - "name": "canvas_userId_user_id_fk", - "tableFrom": "canvas", - "tableTo": "user", - "columnsFrom": ["userId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "chatHistory": { - "name": "chatHistory", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "threadId": { - "name": "threadId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "question": { - "name": "question", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "answerParts": { - "name": "answerParts", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "answerSources": { - "name": "answerSources", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "answerJustification": { - "name": "answerJustification", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'\"2024-07-25T22:31:50.848Z\"'" - } - }, - "indexes": { - "chatHistory_thread_idx": { - "name": "chatHistory_thread_idx", - "columns": ["threadId"], - "isUnique": false - } - }, - "foreignKeys": { - "chatHistory_threadId_chatThread_id_fk": { - "name": "chatHistory_threadId_chatThread_id_fk", - "tableFrom": "chatHistory", - "tableTo": "chatThread", - "columnsFrom": ["threadId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "chatThread": { - "name": "chatThread", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "firstMessage": { - "name": "firstMessage", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "chatThread_user_idx": { - "name": "chatThread_user_idx", - "columns": ["userId"], - "isUnique": false - } - }, - "foreignKeys": { - "chatThread_userId_user_id_fk": { - "name": "chatThread_userId_user_id_fk", - "tableFrom": "chatThread", - "tableTo": "user", - "columnsFrom": ["userId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "contentToSpace": { - "name": "contentToSpace", - "columns": { - "contentId": { - "name": "contentId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "contentToSpace_contentId_storedContent_id_fk": { - "name": "contentToSpace_contentId_storedContent_id_fk", - "tableFrom": "contentToSpace", - "tableTo": "storedContent", - "columnsFrom": ["contentId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "contentToSpace_spaceId_space_id_fk": { - "name": "contentToSpace_spaceId_space_id_fk", - "tableFrom": "contentToSpace", - "tableTo": "space", - "columnsFrom": ["spaceId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "contentToSpace_contentId_spaceId_pk": { - "columns": ["contentId", "spaceId"], - "name": "contentToSpace_contentId_spaceId_pk" - } - }, - "uniqueConstraints": {} - }, - "session": { - "name": "session", - "columns": { - "sessionToken": { - "name": "sessionToken", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "session_userId_user_id_fk": { - "name": "session_userId_user_id_fk", - "tableFrom": "session", - "tableTo": "user", - "columnsFrom": ["userId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "space": { - "name": "space", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'none'" - }, - "user": { - "name": "user", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "numItems": { - "name": "numItems", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - } - }, - "indexes": { - "space_name_unique": { - "name": "space_name_unique", - "columns": ["name"], - "isUnique": true - }, - "spaces_name_idx": { - "name": "spaces_name_idx", - "columns": ["name"], - "isUnique": false - }, - "spaces_user_idx": { - "name": "spaces_user_idx", - "columns": ["user"], - "isUnique": false - } - }, - "foreignKeys": { - "space_user_user_id_fk": { - "name": "space_user_user_id_fk", - "tableFrom": "space", - "tableTo": "user", - "columnsFrom": ["user"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "spacesAccess": { - "name": "spacesAccess", - "columns": { - "spaceId": { - "name": "spaceId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userEmail": { - "name": "userEmail", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "spacesAccess_spaceId_space_id_fk": { - "name": "spacesAccess_spaceId_space_id_fk", - "tableFrom": "spacesAccess", - "tableTo": "space", - "columnsFrom": ["spaceId"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "spacesAccess_spaceId_userEmail_pk": { - "columns": ["spaceId", "userEmail"], - "name": "spacesAccess_spaceId_userEmail_pk" - } - }, - "uniqueConstraints": {} - }, - "storedContent": { - "name": "storedContent", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "title": { - "name": "title", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "description": { - "name": "description", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "savedAt": { - "name": "savedAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "baseUrl": { - "name": "baseUrl", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "ogImage": { - "name": "ogImage", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'page'" - }, - "image": { - "name": "image", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "user": { - "name": "user", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "noteId": { - "name": "noteId", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "storedContent_baseUrl_unique": { - "name": "storedContent_baseUrl_unique", - "columns": ["baseUrl"], - "isUnique": true - }, - "storedContent_url_idx": { - "name": "storedContent_url_idx", - "columns": ["url"], - "isUnique": false - }, - "storedContent_savedAt_idx": { - "name": "storedContent_savedAt_idx", - "columns": ["savedAt"], - "isUnique": false - }, - "storedContent_title_idx": { - "name": "storedContent_title_idx", - "columns": ["title"], - "isUnique": false - }, - "storedContent_user_idx": { - "name": "storedContent_user_idx", - "columns": ["user"], - "isUnique": false - } - }, - "foreignKeys": { - "storedContent_user_user_id_fk": { - "name": "storedContent_user_user_id_fk", - "tableFrom": "storedContent", - "tableTo": "user", - "columnsFrom": ["user"], - "columnsTo": ["id"], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "user": { - "name": "user", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "emailVerified": { - "name": "emailVerified", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "image": { - "name": "image", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "telegramId": { - "name": "telegramId", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "hasOnboarded": { - "name": "hasOnboarded", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "users_email_idx": { - "name": "users_email_idx", - "columns": ["email"], - "isUnique": false - }, - "users_telegram_idx": { - "name": "users_telegram_idx", - "columns": ["telegramId"], - "isUnique": false - }, - "users_id_idx": { - "name": "users_id_idx", - "columns": ["id"], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "verificationToken": { - "name": "verificationToken", - "columns": { - "identifier": { - "name": "identifier", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "token": { - "name": "token", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "verificationToken_identifier_token_pk": { - "columns": ["identifier", "token"], - "name": "verificationToken_identifier_token_pk" - } - }, - "uniqueConstraints": {} - } - }, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - } -} + "version": "6", + "dialect": "sqlite", + "id": "1197a463-b72a-47c8-b018-ddce31ef9c31", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": [ + "provider", + "providerAccountId" + ], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {} + }, + "authenticator": { + "name": "authenticator", + "columns": { + "credentialID": { + "name": "credentialID", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialPublicKey": { + "name": "credentialPublicKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialDeviceType": { + "name": "credentialDeviceType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialBackedUp": { + "name": "credentialBackedUp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "authenticator_credentialID_unique": { + "name": "authenticator_credentialID_unique", + "columns": [ + "credentialID" + ], + "isUnique": true + } + }, + "foreignKeys": { + "authenticator_userId_user_id_fk": { + "name": "authenticator_userId_user_id_fk", + "tableFrom": "authenticator", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "authenticator_userId_credentialID_pk": { + "columns": [ + "credentialID", + "userId" + ], + "name": "authenticator_userId_credentialID_pk" + } + }, + "uniqueConstraints": {} + }, + "canvas": { + "name": "canvas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "canvas_user_userId": { + "name": "canvas_user_userId", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canvas_userId_user_id_fk": { + "name": "canvas_userId_user_id_fk", + "tableFrom": "canvas", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "chatHistory": { + "name": "chatHistory", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "threadId": { + "name": "threadId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "question": { + "name": "question", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answerParts": { + "name": "answerParts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "answerSources": { + "name": "answerSources", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "answerJustification": { + "name": "answerJustification", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'\"2024-07-29T17:06:56.122Z\"'" + } + }, + "indexes": { + "chatHistory_thread_idx": { + "name": "chatHistory_thread_idx", + "columns": [ + "threadId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chatHistory_threadId_chatThread_id_fk": { + "name": "chatHistory_threadId_chatThread_id_fk", + "tableFrom": "chatHistory", + "tableTo": "chatThread", + "columnsFrom": [ + "threadId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "chatThread": { + "name": "chatThread", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "firstMessage": { + "name": "firstMessage", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "chatThread_user_idx": { + "name": "chatThread_user_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "chatThread_userId_user_id_fk": { + "name": "chatThread_userId_user_id_fk", + "tableFrom": "chatThread", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "contentToSpace": { + "name": "contentToSpace", + "columns": { + "contentId": { + "name": "contentId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "contentToSpace_contentId_storedContent_id_fk": { + "name": "contentToSpace_contentId_storedContent_id_fk", + "tableFrom": "contentToSpace", + "tableTo": "storedContent", + "columnsFrom": [ + "contentId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "contentToSpace_spaceId_space_id_fk": { + "name": "contentToSpace_spaceId_space_id_fk", + "tableFrom": "contentToSpace", + "tableTo": "space", + "columnsFrom": [ + "spaceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "contentToSpace_contentId_spaceId_pk": { + "columns": [ + "contentId", + "spaceId" + ], + "name": "contentToSpace_contentId_spaceId_pk" + } + }, + "uniqueConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastAttemptAt": { + "name": "lastAttemptAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "jobs_userId_idx": { + "name": "jobs_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "jobs_status_idx": { + "name": "jobs_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "jobs_createdAt_idx": { + "name": "jobs_createdAt_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + }, + "jobs_url_idx": { + "name": "jobs_url_idx", + "columns": [ + "url" + ], + "isUnique": false + } + }, + "foreignKeys": { + "jobs_userId_user_id_fk": { + "name": "jobs_userId_user_id_fk", + "tableFrom": "jobs", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "space": { + "name": "space", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "user": { + "name": "user", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "numItems": { + "name": "numItems", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "space_name_unique": { + "name": "space_name_unique", + "columns": [ + "name" + ], + "isUnique": true + }, + "spaces_name_idx": { + "name": "spaces_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "spaces_user_idx": { + "name": "spaces_user_idx", + "columns": [ + "user" + ], + "isUnique": false + } + }, + "foreignKeys": { + "space_user_user_id_fk": { + "name": "space_user_user_id_fk", + "tableFrom": "space", + "tableTo": "user", + "columnsFrom": [ + "user" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "spacesAccess": { + "name": "spacesAccess", + "columns": { + "spaceId": { + "name": "spaceId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "spacesAccess_spaceId_space_id_fk": { + "name": "spacesAccess_spaceId_space_id_fk", + "tableFrom": "spacesAccess", + "tableTo": "space", + "columnsFrom": [ + "spaceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spacesAccess_spaceId_userEmail_pk": { + "columns": [ + "spaceId", + "userEmail" + ], + "name": "spacesAccess_spaceId_userEmail_pk" + } + }, + "uniqueConstraints": {} + }, + "storedContent": { + "name": "storedContent", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "savedAt": { + "name": "savedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "baseUrl": { + "name": "baseUrl", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ogImage": { + "name": "ogImage", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'page'" + }, + "image": { + "name": "image", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user": { + "name": "user", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "noteId": { + "name": "noteId", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "storedContent_baseUrl_unique": { + "name": "storedContent_baseUrl_unique", + "columns": [ + "baseUrl" + ], + "isUnique": true + }, + "storedContent_url_idx": { + "name": "storedContent_url_idx", + "columns": [ + "url" + ], + "isUnique": false + }, + "storedContent_savedAt_idx": { + "name": "storedContent_savedAt_idx", + "columns": [ + "savedAt" + ], + "isUnique": false + }, + "storedContent_title_idx": { + "name": "storedContent_title_idx", + "columns": [ + "title" + ], + "isUnique": false + }, + "storedContent_user_idx": { + "name": "storedContent_user_idx", + "columns": [ + "user" + ], + "isUnique": false + } + }, + "foreignKeys": { + "storedContent_user_user_id_fk": { + "name": "storedContent_user_user_id_fk", + "tableFrom": "storedContent", + "tableTo": "user", + "columnsFrom": [ + "user" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "telegramId": { + "name": "telegramId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hasOnboarded": { + "name": "hasOnboarded", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "users_email_idx": { + "name": "users_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "users_telegram_idx": { + "name": "users_telegram_idx", + "columns": [ + "telegramId" + ], + "isUnique": false + }, + "users_id_idx": { + "name": "users_id_idx", + "columns": [ + "id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": [ + "identifier", + "token" + ], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} \ No newline at end of file diff --git a/apps/web/migrations/meta/_journal.json b/apps/web/migrations/meta/_journal.json index d79e2607..254b90c6 100644 --- a/apps/web/migrations/meta/_journal.json +++ b/apps/web/migrations/meta/_journal.json @@ -1,13 +1,13 @@ { - "version": "6", - "dialect": "sqlite", - "entries": [ - { - "idx": 0, - "version": "6", - "when": 1721946710900, - "tag": "0000_steep_moira_mactaggert", - "breakpoints": true - } - ] -} + "version": "6", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1722272816127, + "tag": "0000_omniscient_stick", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/apps/web/server/db/schema.ts b/apps/web/server/db/schema.ts index 32b80719..11711997 100644 --- a/apps/web/server/db/schema.ts +++ b/apps/web/server/db/schema.ts @@ -7,6 +7,7 @@ import { sqliteTableCreator, text, integer, + blob, } from "drizzle-orm/sqlite-core"; import type { AdapterAccountType } from "next-auth/adapters"; @@ -242,3 +243,28 @@ export const canvas = createTable( export type ChatThread = typeof chatThreads.$inferSelect; export type ChatHistory = typeof chatHistory.$inferSelect; + +export const jobs = createTable( + "jobs", + { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + userId: text("userId") + .notNull() + .references(() => users.id, { onDelete: "cascade" }), + url: text("url").notNull(), + status: text("status").notNull(), + attempts: integer("attempts").notNull().default(0), + lastAttemptAt: integer("lastAttemptAt"), + error: blob("error"), + createdAt: integer("createdAt").notNull(), + updatedAt: integer("updatedAt").notNull(), + }, + (job) => ({ + userIdx: index("jobs_userId_idx").on(job.userId), + statusIdx: index("jobs_status_idx").on(job.status), + createdAtIdx: index("jobs_createdAt_idx").on(job.createdAt), + urlIdx: index("jobs_url_idx").on(job.url), + }), +); + +export type Job = typeof jobs.$inferSelect; -- cgit v1.2.3 From 241276be588312aec4f9e09a23c7951379238da8 Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Wed, 31 Jul 2024 11:37:54 +0530 Subject: db schema in packages --- apps/cf-ai-backend/src/db/index.ts | 2 +- apps/cf-ai-backend/src/db/schema.ts | 93 -- apps/cf-ai-backend/src/index.ts | 7 - apps/cf-ai-backend/src/queueConsumer/index.ts | 2 +- apps/cf-ai-backend/wrangler.toml | 23 +- apps/web/app/(dash)/(memories)/content.tsx | 2 +- .../app/(dash)/(memories)/space/[spaceid]/page.tsx | 2 +- apps/web/app/(dash)/dialogContentContainer.tsx | 2 +- apps/web/app/(dash)/menu.tsx | 2 +- apps/web/app/actions/doers.ts | 5 +- apps/web/app/actions/fetchers.ts | 2 +- apps/web/app/api/chat/history/route.ts | 2 +- apps/web/app/api/chat/route.ts | 2 +- apps/web/app/api/ensureAuth.ts | 2 +- apps/web/app/api/getCount/route.ts | 2 +- apps/web/app/api/me/route.ts | 2 +- apps/web/app/api/memories/route.ts | 2 +- apps/web/app/api/spaces/route.ts | 2 +- apps/web/app/api/store/friend/route.ts | 44 - apps/web/app/api/store/helper.ts | 2 +- apps/web/app/api/telegram/route.ts | 2 +- apps/web/app/ref/page.tsx | 2 +- apps/web/drizzle.config.ts | 12 - apps/web/migrations/0000_omniscient_stick.sql | 152 --- apps/web/migrations/meta/0000_snapshot.json | 1027 -------------------- apps/web/migrations/meta/_journal.json | 13 - apps/web/next.config.mjs | 3 + apps/web/package.json | 1 - apps/web/server/auth.ts | 2 +- apps/web/server/db/index.ts | 2 +- apps/web/server/db/schema.ts | 270 ----- apps/web/wrangler.toml | 5 +- 32 files changed, 48 insertions(+), 1645 deletions(-) delete mode 100644 apps/cf-ai-backend/src/db/schema.ts delete mode 100644 apps/web/app/api/store/friend/route.ts delete mode 100644 apps/web/drizzle.config.ts delete mode 100644 apps/web/migrations/0000_omniscient_stick.sql delete mode 100644 apps/web/migrations/meta/0000_snapshot.json delete mode 100644 apps/web/migrations/meta/_journal.json delete mode 100644 apps/web/server/db/schema.ts (limited to 'apps') diff --git a/apps/cf-ai-backend/src/db/index.ts b/apps/cf-ai-backend/src/db/index.ts index e16cec25..b7a05fa5 100644 --- a/apps/cf-ai-backend/src/db/index.ts +++ b/apps/cf-ai-backend/src/db/index.ts @@ -1,7 +1,7 @@ import { drizzle } from "drizzle-orm/d1"; import { Env } from "../types"; -import * as schema from "./schema"; +import * as schema from "@repo/db/schema"; export const database = (env: Env) => drizzle(env.DATABASE, { schema, logger: true }); diff --git a/apps/cf-ai-backend/src/db/schema.ts b/apps/cf-ai-backend/src/db/schema.ts deleted file mode 100644 index 71e2c884..00000000 --- a/apps/cf-ai-backend/src/db/schema.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { - index, - int, - primaryKey, - sqliteTableCreator, - text, - integer, -} from "drizzle-orm/sqlite-core"; - -export const createTable = sqliteTableCreator((name) => `${name}`); - -export const users = createTable( - "user", - { - id: text("id") - .primaryKey() - .$defaultFn(() => crypto.randomUUID()), - name: text("name"), - email: text("email").notNull(), - emailVerified: integer("emailVerified", { mode: "timestamp_ms" }), - image: text("image"), - telegramId: text("telegramId"), - hasOnboarded: integer("hasOnboarded", { mode: "boolean" }).default(false), - }, - (user) => ({ - emailIdx: index("users_email_idx").on(user.email), - telegramIdx: index("users_telegram_idx").on(user.telegramId), - idIdx: index("users_id_idx").on(user.id), - }), -); - -export type User = typeof users.$inferSelect; - -export const storedContent = createTable( - "storedContent", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - content: text("content").notNull(), - title: text("title", { length: 255 }), - description: text("description", { length: 255 }), - url: text("url").notNull(), - savedAt: int("savedAt", { mode: "timestamp" }).notNull(), - baseUrl: text("baseUrl", { length: 255 }).unique(), - ogImage: text("ogImage", { length: 255 }), - type: text("type").default("page"), - image: text("image", { length: 255 }), - userId: text("user").references(() => users.id, { - onDelete: "cascade", - }), - noteId: integer("noteId"), - }, - (sc) => ({ - urlIdx: index("storedContent_url_idx").on(sc.url), - savedAtIdx: index("storedContent_savedAt_idx").on(sc.savedAt), - titleInx: index("storedContent_title_idx").on(sc.title), - userIdx: index("storedContent_user_idx").on(sc.userId), - }), -); - -export type Content = typeof storedContent.$inferSelect; - - -export const contentToSpace = createTable( - "contentToSpace", - { - contentId: integer("contentId") - .notNull() - .references(() => storedContent.id, { onDelete: "cascade" }), - spaceId: integer("spaceId") - .notNull() - .references(() => space.id, { onDelete: "cascade" }), - }, - (cts) => ({ - compoundKey: primaryKey({ columns: [cts.contentId, cts.spaceId] }), - }), -); - -export const space = createTable( - "space", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - name: text("name").notNull().unique().default("none"), - user: text("user", { length: 255 }).references(() => users.id, { - onDelete: "cascade", - }), - createdAt: int("createdAt", { mode: "timestamp" }).notNull(), - numItems: integer("numItems").notNull().default(0), - }, - (space) => ({ - nameIdx: index("spaces_name_idx").on(space.name), - userIdx: index("spaces_user_idx").on(space.user), - }), -); \ No newline at end of file diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index b072f100..4949fab3 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -33,13 +33,6 @@ import { queue } from "./queueConsumer"; const app = new Hono<{ Bindings: Env }>(); -app.get( - "/ui", - swaggerUI({ - url: "/doc", - }), -); - // ------- MIDDLEWARES ------- app.use("*", poweredBy()); app.use("*", timing()); diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts index 8657603d..8b9064cb 100644 --- a/apps/cf-ai-backend/src/queueConsumer/index.ts +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -13,7 +13,7 @@ import { z } from "zod"; import { Metadata } from "./utils/get-metadata"; import { BaseError } from "../errors/baseError"; import { database } from "../db"; -import { storedContent, space, contentToSpace } from "../db/schema"; +import { storedContent, space, contentToSpace } from "@repo/db/schema"; import { and, eq, inArray, sql } from "drizzle-orm"; class VectorInsertError extends BaseError { diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index ea93fd63..ca8e2b1d 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -3,6 +3,9 @@ main = "src/index.ts" compatibility_date = "2024-02-23" node_compat = true +tail_consumers = [{service = "new-cf-ai-backend-tail"}] + + # [env.preview] [[vectorize]] binding = "VECTORIZE_INDEX" @@ -24,8 +27,24 @@ binding = "AI" [[kv_namespaces]] binding = "KV" -id = "37a90353da63401e84e20e71165531d0" -preview_id = "c58b6202814f4224acea97627d0c18aa" +id = "569c3f2a510943729019f5d1f798e7d9" +preview_id = "569c3f2a510943729019f5d1f798e7d9" [placement] mode = "smart" + +[[queues.producers]] + queue = "embedchunks-queue" + binding ="EMBEDCHUNKS_QUEUE" + +[[queues.consumers]] + queue = "embedchunks-queue" + max_batch_size = 100 + max_retries = 10 + dead_letter_queue = "embedchunks-dlq" + + +[[d1_databases]] +binding = "DATABASE" +database_name = "supermemlocal" +database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" diff --git a/apps/web/app/(dash)/(memories)/content.tsx b/apps/web/app/(dash)/(memories)/content.tsx index 4514d851..431109be 100644 --- a/apps/web/app/(dash)/(memories)/content.tsx +++ b/apps/web/app/(dash)/(memories)/content.tsx @@ -1,6 +1,6 @@ "use client"; -import { Content, StoredSpace } from "@/server/db/schema"; +import { Content, StoredSpace } from "@repo/db/schema"; import { MemoriesIcon, NextIcon, SearchIcon, UrlIcon } from "@repo/ui/icons"; import { ArrowLeftIcon, diff --git a/apps/web/app/(dash)/(memories)/space/[spaceid]/page.tsx b/apps/web/app/(dash)/(memories)/space/[spaceid]/page.tsx index ed1ea1cc..8ad9d9cc 100644 --- a/apps/web/app/(dash)/(memories)/space/[spaceid]/page.tsx +++ b/apps/web/app/(dash)/(memories)/space/[spaceid]/page.tsx @@ -3,7 +3,7 @@ import { redirect } from "next/navigation"; import MemoriesPage from "../../content"; import { db } from "@/server/db"; import { and, eq } from "drizzle-orm"; -import { spacesAccess } from "@/server/db/schema"; +import { spacesAccess } from "@repo/db/schema"; import { auth } from "@/server/auth"; async function Page({ params: { spaceid } }: { params: { spaceid: number } }) { diff --git a/apps/web/app/(dash)/dialogContentContainer.tsx b/apps/web/app/(dash)/dialogContentContainer.tsx index 4e8d81ef..c6cacd35 100644 --- a/apps/web/app/(dash)/dialogContentContainer.tsx +++ b/apps/web/app/(dash)/dialogContentContainer.tsx @@ -1,4 +1,4 @@ -import { StoredSpace } from "@/server/db/schema"; +import { StoredSpace } from "@repo/db/schema"; import { useEffect, useMemo, useState } from "react"; import { createMemory, createSpace } from "../actions/doers"; import ComboboxWithCreate from "@repo/ui/shadcn/combobox"; diff --git a/apps/web/app/(dash)/menu.tsx b/apps/web/app/(dash)/menu.tsx index 70439c7d..1acab2c8 100644 --- a/apps/web/app/(dash)/menu.tsx +++ b/apps/web/app/(dash)/menu.tsx @@ -28,7 +28,7 @@ import { getSpaces } from "../actions/fetchers"; import { HomeIcon } from "@heroicons/react/24/solid"; import { createMemory, createSpace } from "../actions/doers"; import ComboboxWithCreate from "@repo/ui/shadcn/combobox"; -import { StoredSpace } from "@/server/db/schema"; +import { StoredSpace } from "@repo/db/schema"; import useMeasure from "react-use-measure"; function Menu() { diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index 910226a5..f17032b9 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -11,7 +11,7 @@ import { spacesAccess, storedContent, users, -} from "../../server/db/schema"; +} from "@repo/db/schema"; import { ServerActionReturnType } from "./types"; import { auth } from "../../server/auth"; import { Tweet } from "react-tweet/api"; @@ -836,8 +836,7 @@ export async function getQuerySuggestions() { }; } - const fullQuery = content - .map((c) => `${c.title} \n\n${c.content}`) + const fullQuery = (content?.map((c) => `${c.title} \n\n${c.content}`) ?? []) .join(" "); const suggestionsCall = (await env.AI.run( diff --git a/apps/web/app/actions/fetchers.ts b/apps/web/app/actions/fetchers.ts index 5f72089a..f00feb3c 100644 --- a/apps/web/app/actions/fetchers.ts +++ b/apps/web/app/actions/fetchers.ts @@ -15,7 +15,7 @@ import { StoredSpace, User, users, -} from "../../server/db/schema"; +} from "@repo/db/schema"; import { ServerActionReturnType } from "./types"; import { auth } from "../../server/auth"; import { ChatHistory, SourceZod } from "@repo/shared-types"; diff --git a/apps/web/app/api/chat/history/route.ts b/apps/web/app/api/chat/history/route.ts index 98b66064..197b8ee6 100644 --- a/apps/web/app/api/chat/history/route.ts +++ b/apps/web/app/api/chat/history/route.ts @@ -2,7 +2,7 @@ import { NextRequest } from "next/server"; import { ensureAuth } from "../../ensureAuth"; import { db } from "@/server/db"; import { eq } from "drizzle-orm"; -import { chatThreads } from "@/server/db/schema"; +import { chatThreads } from "@repo/db/schema"; export const runtime = "edge"; diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index a14c96df..78878e40 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -8,7 +8,7 @@ import { import { ensureAuth } from "../ensureAuth"; import { z } from "zod"; import { db } from "@/server/db"; -import { chatHistory as chatHistoryDb, chatThreads } from "@/server/db/schema"; +import { chatHistory as chatHistoryDb, chatThreads } from "@repo/db/schema"; import { and, eq, gt, sql } from "drizzle-orm"; import { join } from "path"; diff --git a/apps/web/app/api/ensureAuth.ts b/apps/web/app/api/ensureAuth.ts index 1fcd2914..92a5e3e8 100644 --- a/apps/web/app/api/ensureAuth.ts +++ b/apps/web/app/api/ensureAuth.ts @@ -1,6 +1,6 @@ import { NextRequest } from "next/server"; import { db } from "../../server/db"; -import { accounts, sessions, users } from "../../server/db/schema"; +import { accounts, sessions, users } from "@repo/db/schema"; import { eq } from "drizzle-orm"; export async function ensureAuth(req: NextRequest) { diff --git a/apps/web/app/api/getCount/route.ts b/apps/web/app/api/getCount/route.ts index f91b7b94..4fd77efd 100644 --- a/apps/web/app/api/getCount/route.ts +++ b/apps/web/app/api/getCount/route.ts @@ -1,6 +1,6 @@ import { db } from "@/server/db"; import { and, eq, ne, sql } from "drizzle-orm"; -import { sessions, storedContent, users } from "@/server/db/schema"; +import { sessions, storedContent, users } from "@repo/db/schema"; import { type NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; diff --git a/apps/web/app/api/me/route.ts b/apps/web/app/api/me/route.ts index ab408f3e..25aa27bc 100644 --- a/apps/web/app/api/me/route.ts +++ b/apps/web/app/api/me/route.ts @@ -1,6 +1,6 @@ import { db } from "@/server/db"; import { eq } from "drizzle-orm"; -import { sessions, users } from "@/server/db/schema"; +import { sessions, users } from "@repo/db/schema"; import { type NextRequest, NextResponse } from "next/server"; export const runtime = "edge"; diff --git a/apps/web/app/api/memories/route.ts b/apps/web/app/api/memories/route.ts index acb43b5d..0084524e 100644 --- a/apps/web/app/api/memories/route.ts +++ b/apps/web/app/api/memories/route.ts @@ -6,7 +6,7 @@ import { contentToSpace, storedContent, users, -} from "@/server/db/schema"; +} from "@repo/db/schema"; import { ensureAuth } from "../ensureAuth"; export const runtime = "edge"; diff --git a/apps/web/app/api/spaces/route.ts b/apps/web/app/api/spaces/route.ts index e85e07ed..27ff0dfb 100644 --- a/apps/web/app/api/spaces/route.ts +++ b/apps/web/app/api/spaces/route.ts @@ -1,5 +1,5 @@ import { db } from "@/server/db"; -import { space } from "@/server/db/schema"; +import { space } from "@repo/db/schema"; import { eq } from "drizzle-orm"; import { NextRequest, NextResponse } from "next/server"; import { ensureAuth } from "../ensureAuth"; diff --git a/apps/web/app/api/store/friend/route.ts b/apps/web/app/api/store/friend/route.ts deleted file mode 100644 index 554b1cee..00000000 --- a/apps/web/app/api/store/friend/route.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { type NextRequest } from "next/server"; -import { createMemoryFromAPI } from "../helper"; - -type FriendData = { - id: string; - created_at: string; - transcript: string; - structured: { - title: string; - overview: string; - action_items: [ - { - description: string; - }, - ]; - }; -}; - -export async function POST(req: NextRequest) { - const body: FriendData = await req.json(); - - const userId = new URL(req.url).searchParams.get("uid"); - - if (!userId) { - return new Response( - JSON.stringify({ status: 400, body: "Missing user ID" }), - ); - } - - await createMemoryFromAPI({ - data: { - title: "Friend: " + body.structured.title, - description: body.structured.overview, - pageContent: - body.transcript + "\n\n" + JSON.stringify(body.structured.action_items), - spaces: [], - type: "note", - url: "https://basedhardware.com", - }, - userId: userId, - }); - - return new Response(JSON.stringify({ status: 200, body: "success" })); -} diff --git a/apps/web/app/api/store/helper.ts b/apps/web/app/api/store/helper.ts index f8833970..2dc42125 100644 --- a/apps/web/app/api/store/helper.ts +++ b/apps/web/app/api/store/helper.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { db } from "@/server/db"; -import { contentToSpace, space, storedContent } from "@/server/db/schema"; +import { contentToSpace, space, storedContent } from "@repo/db/schema"; import { and, eq, inArray } from "drizzle-orm"; import { LIMITS } from "@/lib/constants"; import { limit } from "@/app/actions/doers"; diff --git a/apps/web/app/api/telegram/route.ts b/apps/web/app/api/telegram/route.ts index 78837e5f..dddfa2f4 100644 --- a/apps/web/app/api/telegram/route.ts +++ b/apps/web/app/api/telegram/route.ts @@ -1,5 +1,5 @@ import { db } from "@/server/db"; -import { storedContent, users } from "@/server/db/schema"; +import { storedContent, users } from "@repo/db/schema"; import { cipher } from "@/server/encrypt"; import { eq } from "drizzle-orm"; import { Bot, webhookCallback } from "grammy"; diff --git a/apps/web/app/ref/page.tsx b/apps/web/app/ref/page.tsx index f61e9616..c582fe5c 100644 --- a/apps/web/app/ref/page.tsx +++ b/apps/web/app/ref/page.tsx @@ -2,7 +2,7 @@ import { Button } from "@repo/ui/shadcn/button"; import { auth, signIn, signOut } from "../../server/auth"; import { db } from "../../server/db"; import { sql } from "drizzle-orm"; -import { users } from "../../server/db/schema"; +import { users } from "@repo/db/schema"; import { getThemeToggler } from "../../lib/get-theme-button"; export const runtime = "edge"; diff --git a/apps/web/drizzle.config.ts b/apps/web/drizzle.config.ts deleted file mode 100644 index 58116123..00000000 --- a/apps/web/drizzle.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { type Config } from "drizzle-kit"; - -export default { - schema: "./server/db/schema.ts", - dialect: "sqlite", - driver: "d1", - dbCredentials: { - wranglerConfigPath: "./wrangler.toml", - dbName: "", - }, - out: "migrations", -} satisfies Config; diff --git a/apps/web/migrations/0000_omniscient_stick.sql b/apps/web/migrations/0000_omniscient_stick.sql deleted file mode 100644 index 542d6d0e..00000000 --- a/apps/web/migrations/0000_omniscient_stick.sql +++ /dev/null @@ -1,152 +0,0 @@ -CREATE TABLE `account` ( - `userId` text NOT NULL, - `type` text NOT NULL, - `provider` text NOT NULL, - `providerAccountId` text NOT NULL, - `refresh_token` text, - `access_token` text, - `expires_at` integer, - `token_type` text, - `scope` text, - `id_token` text, - `session_state` text, - PRIMARY KEY(`provider`, `providerAccountId`), - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `authenticator` ( - `credentialID` text NOT NULL, - `userId` text NOT NULL, - `providerAccountId` text NOT NULL, - `credentialPublicKey` text NOT NULL, - `counter` integer NOT NULL, - `credentialDeviceType` text NOT NULL, - `credentialBackedUp` integer NOT NULL, - `transports` text, - PRIMARY KEY(`credentialID`, `userId`), - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `canvas` ( - `id` text PRIMARY KEY NOT NULL, - `title` text DEFAULT 'Untitled' NOT NULL, - `description` text DEFAULT 'Untitled' NOT NULL, - `url` text DEFAULT '' NOT NULL, - `userId` text NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `chatHistory` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `threadId` text NOT NULL, - `question` text NOT NULL, - `answerParts` text, - `answerSources` text, - `answerJustification` text, - `createdAt` integer DEFAULT '"2024-07-29T17:06:56.122Z"' NOT NULL, - FOREIGN KEY (`threadId`) REFERENCES `chatThread`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `chatThread` ( - `id` text PRIMARY KEY NOT NULL, - `firstMessage` text NOT NULL, - `userId` text NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `contentToSpace` ( - `contentId` integer NOT NULL, - `spaceId` integer NOT NULL, - PRIMARY KEY(`contentId`, `spaceId`), - FOREIGN KEY (`contentId`) REFERENCES `storedContent`(`id`) ON UPDATE no action ON DELETE cascade, - FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `jobs` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `userId` text NOT NULL, - `url` text NOT NULL, - `status` text NOT NULL, - `attempts` integer DEFAULT 0 NOT NULL, - `lastAttemptAt` integer, - `error` blob, - `createdAt` integer NOT NULL, - `updatedAt` integer NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `session` ( - `sessionToken` text PRIMARY KEY NOT NULL, - `userId` text NOT NULL, - `expires` integer NOT NULL, - FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `space` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text DEFAULT 'none' NOT NULL, - `user` text(255), - `createdAt` integer NOT NULL, - `numItems` integer DEFAULT 0 NOT NULL, - FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `spacesAccess` ( - `spaceId` integer NOT NULL, - `userEmail` text NOT NULL, - PRIMARY KEY(`spaceId`, `userEmail`), - FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `storedContent` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `content` text NOT NULL, - `title` text(255), - `description` text(255), - `url` text NOT NULL, - `savedAt` integer NOT NULL, - `baseUrl` text(255), - `ogImage` text(255), - `type` text DEFAULT 'page', - `image` text(255), - `user` text, - `noteId` integer, - FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade -); ---> statement-breakpoint -CREATE TABLE `user` ( - `id` text PRIMARY KEY NOT NULL, - `name` text, - `email` text NOT NULL, - `emailVerified` integer, - `image` text, - `telegramId` text, - `hasOnboarded` integer DEFAULT false -); ---> statement-breakpoint -CREATE TABLE `verificationToken` ( - `identifier` text NOT NULL, - `token` text NOT NULL, - `expires` integer NOT NULL, - PRIMARY KEY(`identifier`, `token`) -); ---> statement-breakpoint -CREATE UNIQUE INDEX `authenticator_credentialID_unique` ON `authenticator` (`credentialID`);--> statement-breakpoint -CREATE INDEX `canvas_user_userId` ON `canvas` (`userId`);--> statement-breakpoint -CREATE INDEX `chatHistory_thread_idx` ON `chatHistory` (`threadId`);--> statement-breakpoint -CREATE INDEX `chatThread_user_idx` ON `chatThread` (`userId`);--> statement-breakpoint -CREATE INDEX `jobs_userId_idx` ON `jobs` (`userId`);--> statement-breakpoint -CREATE INDEX `jobs_status_idx` ON `jobs` (`status`);--> statement-breakpoint -CREATE INDEX `jobs_createdAt_idx` ON `jobs` (`createdAt`);--> statement-breakpoint -CREATE INDEX `jobs_url_idx` ON `jobs` (`url`);--> statement-breakpoint -CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint -CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint -CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint -CREATE UNIQUE INDEX `storedContent_baseUrl_unique` ON `storedContent` (`baseUrl`);--> statement-breakpoint -CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint -CREATE INDEX `storedContent_savedAt_idx` ON `storedContent` (`savedAt`);--> statement-breakpoint -CREATE INDEX `storedContent_title_idx` ON `storedContent` (`title`);--> statement-breakpoint -CREATE INDEX `storedContent_user_idx` ON `storedContent` (`user`);--> statement-breakpoint -CREATE INDEX `users_email_idx` ON `user` (`email`);--> statement-breakpoint -CREATE INDEX `users_telegram_idx` ON `user` (`telegramId`);--> statement-breakpoint -CREATE INDEX `users_id_idx` ON `user` (`id`); \ No newline at end of file diff --git a/apps/web/migrations/meta/0000_snapshot.json b/apps/web/migrations/meta/0000_snapshot.json deleted file mode 100644 index 8141d674..00000000 --- a/apps/web/migrations/meta/0000_snapshot.json +++ /dev/null @@ -1,1027 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "1197a463-b72a-47c8-b018-ddce31ef9c31", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "account": { - "name": "account", - "columns": { - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "refresh_token": { - "name": "refresh_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "access_token": { - "name": "access_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expires_at": { - "name": "expires_at", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "token_type": { - "name": "token_type", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "scope": { - "name": "scope", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "id_token": { - "name": "id_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "session_state": { - "name": "session_state", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "account_userId_user_id_fk": { - "name": "account_userId_user_id_fk", - "tableFrom": "account", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "account_provider_providerAccountId_pk": { - "columns": [ - "provider", - "providerAccountId" - ], - "name": "account_provider_providerAccountId_pk" - } - }, - "uniqueConstraints": {} - }, - "authenticator": { - "name": "authenticator", - "columns": { - "credentialID": { - "name": "credentialID", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialPublicKey": { - "name": "credentialPublicKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "counter": { - "name": "counter", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialDeviceType": { - "name": "credentialDeviceType", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "credentialBackedUp": { - "name": "credentialBackedUp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "transports": { - "name": "transports", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "authenticator_credentialID_unique": { - "name": "authenticator_credentialID_unique", - "columns": [ - "credentialID" - ], - "isUnique": true - } - }, - "foreignKeys": { - "authenticator_userId_user_id_fk": { - "name": "authenticator_userId_user_id_fk", - "tableFrom": "authenticator", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "authenticator_userId_credentialID_pk": { - "columns": [ - "credentialID", - "userId" - ], - "name": "authenticator_userId_credentialID_pk" - } - }, - "uniqueConstraints": {} - }, - "canvas": { - "name": "canvas", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "title": { - "name": "title", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "description": { - "name": "description", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "''" - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "canvas_user_userId": { - "name": "canvas_user_userId", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": { - "canvas_userId_user_id_fk": { - "name": "canvas_userId_user_id_fk", - "tableFrom": "canvas", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "chatHistory": { - "name": "chatHistory", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "threadId": { - "name": "threadId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "question": { - "name": "question", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "answerParts": { - "name": "answerParts", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "answerSources": { - "name": "answerSources", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "answerJustification": { - "name": "answerJustification", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'\"2024-07-29T17:06:56.122Z\"'" - } - }, - "indexes": { - "chatHistory_thread_idx": { - "name": "chatHistory_thread_idx", - "columns": [ - "threadId" - ], - "isUnique": false - } - }, - "foreignKeys": { - "chatHistory_threadId_chatThread_id_fk": { - "name": "chatHistory_threadId_chatThread_id_fk", - "tableFrom": "chatHistory", - "tableTo": "chatThread", - "columnsFrom": [ - "threadId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "chatThread": { - "name": "chatThread", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "firstMessage": { - "name": "firstMessage", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "chatThread_user_idx": { - "name": "chatThread_user_idx", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": { - "chatThread_userId_user_id_fk": { - "name": "chatThread_userId_user_id_fk", - "tableFrom": "chatThread", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "contentToSpace": { - "name": "contentToSpace", - "columns": { - "contentId": { - "name": "contentId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "contentToSpace_contentId_storedContent_id_fk": { - "name": "contentToSpace_contentId_storedContent_id_fk", - "tableFrom": "contentToSpace", - "tableTo": "storedContent", - "columnsFrom": [ - "contentId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - }, - "contentToSpace_spaceId_space_id_fk": { - "name": "contentToSpace_spaceId_space_id_fk", - "tableFrom": "contentToSpace", - "tableTo": "space", - "columnsFrom": [ - "spaceId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "contentToSpace_contentId_spaceId_pk": { - "columns": [ - "contentId", - "spaceId" - ], - "name": "contentToSpace_contentId_spaceId_pk" - } - }, - "uniqueConstraints": {} - }, - "jobs": { - "name": "jobs", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "attempts": { - "name": "attempts", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "lastAttemptAt": { - "name": "lastAttemptAt", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "error": { - "name": "error", - "type": "blob", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updatedAt": { - "name": "updatedAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "jobs_userId_idx": { - "name": "jobs_userId_idx", - "columns": [ - "userId" - ], - "isUnique": false - }, - "jobs_status_idx": { - "name": "jobs_status_idx", - "columns": [ - "status" - ], - "isUnique": false - }, - "jobs_createdAt_idx": { - "name": "jobs_createdAt_idx", - "columns": [ - "createdAt" - ], - "isUnique": false - }, - "jobs_url_idx": { - "name": "jobs_url_idx", - "columns": [ - "url" - ], - "isUnique": false - } - }, - "foreignKeys": { - "jobs_userId_user_id_fk": { - "name": "jobs_userId_user_id_fk", - "tableFrom": "jobs", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "session": { - "name": "session", - "columns": { - "sessionToken": { - "name": "sessionToken", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "session_userId_user_id_fk": { - "name": "session_userId_user_id_fk", - "tableFrom": "session", - "tableTo": "user", - "columnsFrom": [ - "userId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "space": { - "name": "space", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'none'" - }, - "user": { - "name": "user", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "numItems": { - "name": "numItems", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - } - }, - "indexes": { - "space_name_unique": { - "name": "space_name_unique", - "columns": [ - "name" - ], - "isUnique": true - }, - "spaces_name_idx": { - "name": "spaces_name_idx", - "columns": [ - "name" - ], - "isUnique": false - }, - "spaces_user_idx": { - "name": "spaces_user_idx", - "columns": [ - "user" - ], - "isUnique": false - } - }, - "foreignKeys": { - "space_user_user_id_fk": { - "name": "space_user_user_id_fk", - "tableFrom": "space", - "tableTo": "user", - "columnsFrom": [ - "user" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "spacesAccess": { - "name": "spacesAccess", - "columns": { - "spaceId": { - "name": "spaceId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userEmail": { - "name": "userEmail", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "spacesAccess_spaceId_space_id_fk": { - "name": "spacesAccess_spaceId_space_id_fk", - "tableFrom": "spacesAccess", - "tableTo": "space", - "columnsFrom": [ - "spaceId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "spacesAccess_spaceId_userEmail_pk": { - "columns": [ - "spaceId", - "userEmail" - ], - "name": "spacesAccess_spaceId_userEmail_pk" - } - }, - "uniqueConstraints": {} - }, - "storedContent": { - "name": "storedContent", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "title": { - "name": "title", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "description": { - "name": "description", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "savedAt": { - "name": "savedAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "baseUrl": { - "name": "baseUrl", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "ogImage": { - "name": "ogImage", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'page'" - }, - "image": { - "name": "image", - "type": "text(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "user": { - "name": "user", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "noteId": { - "name": "noteId", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "storedContent_baseUrl_unique": { - "name": "storedContent_baseUrl_unique", - "columns": [ - "baseUrl" - ], - "isUnique": true - }, - "storedContent_url_idx": { - "name": "storedContent_url_idx", - "columns": [ - "url" - ], - "isUnique": false - }, - "storedContent_savedAt_idx": { - "name": "storedContent_savedAt_idx", - "columns": [ - "savedAt" - ], - "isUnique": false - }, - "storedContent_title_idx": { - "name": "storedContent_title_idx", - "columns": [ - "title" - ], - "isUnique": false - }, - "storedContent_user_idx": { - "name": "storedContent_user_idx", - "columns": [ - "user" - ], - "isUnique": false - } - }, - "foreignKeys": { - "storedContent_user_user_id_fk": { - "name": "storedContent_user_user_id_fk", - "tableFrom": "storedContent", - "tableTo": "user", - "columnsFrom": [ - "user" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "user": { - "name": "user", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "email": { - "name": "email", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "emailVerified": { - "name": "emailVerified", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "image": { - "name": "image", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "telegramId": { - "name": "telegramId", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "hasOnboarded": { - "name": "hasOnboarded", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "users_email_idx": { - "name": "users_email_idx", - "columns": [ - "email" - ], - "isUnique": false - }, - "users_telegram_idx": { - "name": "users_telegram_idx", - "columns": [ - "telegramId" - ], - "isUnique": false - }, - "users_id_idx": { - "name": "users_id_idx", - "columns": [ - "id" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "verificationToken": { - "name": "verificationToken", - "columns": { - "identifier": { - "name": "identifier", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "token": { - "name": "token", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "verificationToken_identifier_token_pk": { - "columns": [ - "identifier", - "token" - ], - "name": "verificationToken_identifier_token_pk" - } - }, - "uniqueConstraints": {} - } - }, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - } -} \ No newline at end of file diff --git a/apps/web/migrations/meta/_journal.json b/apps/web/migrations/meta/_journal.json deleted file mode 100644 index 254b90c6..00000000 --- a/apps/web/migrations/meta/_journal.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "entries": [ - { - "idx": 0, - "version": "6", - "when": 1722272816127, - "tag": "0000_omniscient_stick", - "breakpoints": true - } - ] -} \ No newline at end of file diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index c0001fa5..934f39ae 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -9,6 +9,9 @@ const baseNextConfig = { env: { TELEGRAM_BOT_TOKEN: process.env.TELEGRAM_BOT_TOKEN, }, + eslint: { + disableDuringBuilds: true + } }; let selectedCofig = baseNextConfig; diff --git a/apps/web/package.json b/apps/web/package.json index 98016d8c..a5332157 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -45,7 +45,6 @@ "@types/node": "^20.11.24", "@types/react": "^18.2.61", "@types/react-dom": "^18.2.19", - "drizzle-kit": "0.21.2", "eslint": "^8.57.0", "postcss": "^8.4.38", "typescript": "^5.3.3", diff --git a/apps/web/server/auth.ts b/apps/web/server/auth.ts index 78671551..5d8b15d1 100644 --- a/apps/web/server/auth.ts +++ b/apps/web/server/auth.ts @@ -2,7 +2,7 @@ import NextAuth, { NextAuthResult } from "next-auth"; import Google from "next-auth/providers/google"; import { DrizzleAdapter } from "@auth/drizzle-adapter"; import { db } from "./db"; -import { accounts, sessions, users, verificationTokens } from "./db/schema"; +import { accounts, sessions, users, verificationTokens } from "@repo/db/schema"; export const { handlers: { GET, POST }, diff --git a/apps/web/server/db/index.ts b/apps/web/server/db/index.ts index a9ec9106..52f3e350 100644 --- a/apps/web/server/db/index.ts +++ b/apps/web/server/db/index.ts @@ -1,6 +1,6 @@ import { drizzle } from "drizzle-orm/d1"; -import * as schema from "./schema"; +import * as schema from "@repo/db/schema"; export const db = drizzle(process.env.DATABASE, { schema, diff --git a/apps/web/server/db/schema.ts b/apps/web/server/db/schema.ts deleted file mode 100644 index 11711997..00000000 --- a/apps/web/server/db/schema.ts +++ /dev/null @@ -1,270 +0,0 @@ -import { create } from "domain"; -import { relations, sql } from "drizzle-orm"; -import { - index, - int, - primaryKey, - sqliteTableCreator, - text, - integer, - blob, -} from "drizzle-orm/sqlite-core"; -import type { AdapterAccountType } from "next-auth/adapters"; - -export const createTable = sqliteTableCreator((name) => `${name}`); - -export const users = createTable( - "user", - { - id: text("id") - .primaryKey() - .$defaultFn(() => crypto.randomUUID()), - name: text("name"), - email: text("email").notNull(), - emailVerified: integer("emailVerified", { mode: "timestamp_ms" }), - image: text("image"), - telegramId: text("telegramId"), - hasOnboarded: integer("hasOnboarded", { mode: "boolean" }).default(false), - }, - (user) => ({ - emailIdx: index("users_email_idx").on(user.email), - telegramIdx: index("users_telegram_idx").on(user.telegramId), - idIdx: index("users_id_idx").on(user.id), - }), -); - -export type User = typeof users.$inferSelect; - -export const accounts = createTable( - "account", - { - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - type: text("type").$type().notNull(), - provider: text("provider").notNull(), - providerAccountId: text("providerAccountId").notNull(), - refresh_token: text("refresh_token"), - access_token: text("access_token"), - expires_at: integer("expires_at"), - token_type: text("token_type"), - scope: text("scope"), - id_token: text("id_token"), - session_state: text("session_state"), - }, - (account) => ({ - compoundKey: primaryKey({ - columns: [account.provider, account.providerAccountId], - }), - }), -); - -export const sessions = createTable("session", { - sessionToken: text("sessionToken").primaryKey(), - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - expires: integer("expires", { mode: "timestamp_ms" }).notNull(), -}); - -export const verificationTokens = createTable( - "verificationToken", - { - identifier: text("identifier").notNull(), - token: text("token").notNull(), - expires: integer("expires", { mode: "timestamp_ms" }).notNull(), - }, - (verificationToken) => ({ - compositePk: primaryKey({ - columns: [verificationToken.identifier, verificationToken.token], - }), - }), -); - -export const authenticators = createTable( - "authenticator", - { - credentialID: text("credentialID").notNull().unique(), - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - providerAccountId: text("providerAccountId").notNull(), - credentialPublicKey: text("credentialPublicKey").notNull(), - counter: integer("counter").notNull(), - credentialDeviceType: text("credentialDeviceType").notNull(), - credentialBackedUp: integer("credentialBackedUp", { - mode: "boolean", - }).notNull(), - transports: text("transports"), - }, - (authenticator) => ({ - compositePK: primaryKey({ - columns: [authenticator.userId, authenticator.credentialID], - }), - }), -); - -export const storedContent = createTable( - "storedContent", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - content: text("content").notNull(), - title: text("title", { length: 255 }), - description: text("description", { length: 255 }), - url: text("url").notNull(), - savedAt: int("savedAt", { mode: "timestamp" }).notNull(), - baseUrl: text("baseUrl", { length: 255 }).unique(), - ogImage: text("ogImage", { length: 255 }), - type: text("type").default("page"), - image: text("image", { length: 255 }), - userId: text("user").references(() => users.id, { - onDelete: "cascade", - }), - noteId: integer("noteId"), - }, - (sc) => ({ - urlIdx: index("storedContent_url_idx").on(sc.url), - savedAtIdx: index("storedContent_savedAt_idx").on(sc.savedAt), - titleInx: index("storedContent_title_idx").on(sc.title), - userIdx: index("storedContent_user_idx").on(sc.userId), - }), -); - -export type Content = typeof storedContent.$inferSelect; - -export const contentToSpace = createTable( - "contentToSpace", - { - contentId: integer("contentId") - .notNull() - .references(() => storedContent.id, { onDelete: "cascade" }), - spaceId: integer("spaceId") - .notNull() - .references(() => space.id, { onDelete: "cascade" }), - }, - (cts) => ({ - compoundKey: primaryKey({ columns: [cts.contentId, cts.spaceId] }), - }), -); - -export const space = createTable( - "space", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - name: text("name").notNull().unique().default("none"), - user: text("user", { length: 255 }).references(() => users.id, { - onDelete: "cascade", - }), - createdAt: int("createdAt", { mode: "timestamp" }).notNull(), - numItems: integer("numItems").notNull().default(0), - }, - (space) => ({ - nameIdx: index("spaces_name_idx").on(space.name), - userIdx: index("spaces_user_idx").on(space.user), - }), -); - -export const spacesAccess = createTable( - "spacesAccess", - { - spaceId: integer("spaceId") - .notNull() - .references(() => space.id, { onDelete: "cascade" }), - userEmail: text("userEmail").notNull(), - }, - (spaceAccess) => ({ - compoundKey: primaryKey({ - columns: [spaceAccess.spaceId, spaceAccess.userEmail], - }), - }), -); - -export type StoredContent = Omit; -export type StoredSpace = typeof space.$inferSelect; -export type ChachedSpaceContent = StoredContent & { - space: number; -}; - -export const chatThreads = createTable( - "chatThread", - { - id: text("id") - .notNull() - .primaryKey() - .$defaultFn(() => crypto.randomUUID()), - firstMessage: text("firstMessage").notNull(), - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - }, - (thread) => ({ - userIdx: index("chatThread_user_idx").on(thread.userId), - }), -); - -export const chatHistory = createTable( - "chatHistory", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - threadId: text("threadId") - .notNull() - .references(() => chatThreads.id, { onDelete: "cascade" }), - question: text("question").notNull(), - answer: text("answerParts"), // Single answer part as string - answerSources: text("answerSources"), // JSON stringified array of objects - answerJustification: text("answerJustification"), - createdAt: int("createdAt", { mode: "timestamp" }) - .notNull() - .default(new Date()), - }, - (history) => ({ - threadIdx: index("chatHistory_thread_idx").on(history.threadId), - }), -); - -export const canvas = createTable( - "canvas", - { - id: text("id") - .notNull() - .primaryKey() - .$defaultFn(() => crypto.randomUUID()), - title: text("title").default("Untitled").notNull(), - description: text("description").default("Untitled").notNull(), - imageUrl: text("url").default("").notNull(), - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - }, - (canvas) => ({ - userIdx: index("canvas_user_userId").on(canvas.userId), - }), -); - -export type ChatThread = typeof chatThreads.$inferSelect; -export type ChatHistory = typeof chatHistory.$inferSelect; - -export const jobs = createTable( - "jobs", - { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), - userId: text("userId") - .notNull() - .references(() => users.id, { onDelete: "cascade" }), - url: text("url").notNull(), - status: text("status").notNull(), - attempts: integer("attempts").notNull().default(0), - lastAttemptAt: integer("lastAttemptAt"), - error: blob("error"), - createdAt: integer("createdAt").notNull(), - updatedAt: integer("updatedAt").notNull(), - }, - (job) => ({ - userIdx: index("jobs_userId_idx").on(job.userId), - statusIdx: index("jobs_status_idx").on(job.status), - createdAtIdx: index("jobs_createdAt_idx").on(job.createdAt), - urlIdx: index("jobs_url_idx").on(job.url), - }), -); - -export type Job = typeof jobs.$inferSelect; diff --git a/apps/web/wrangler.toml b/apps/web/wrangler.toml index 7f3fa047..7c4f63c8 100644 --- a/apps/web/wrangler.toml +++ b/apps/web/wrangler.toml @@ -26,8 +26,9 @@ bucket_name = "dev-r2-anycontext" [[d1_databases]] binding = "DATABASE" -database_name = "dev-d1-anycontext" -database_id = "fc562605-157a-4f60-b439-2a24ffed5b4c" +database_name = "supermemlocal" +database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" + [[env.production.d1_databases]] -- cgit v1.2.3 From 4e018fb3465d5e418efda627401fa0d21c03290e Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Wed, 31 Jul 2024 00:48:03 -0700 Subject: drizzle logic back in apps/web other stuff in the same place. --- apps/web/drizzle.config.ts | 12 + apps/web/migrations/0000_fixed_pandemic.sql | 152 +++++ apps/web/migrations/meta/0000_snapshot.json | 926 ++++++++++++++++++++++++++++ apps/web/migrations/meta/_journal.json | 13 + apps/web/next.config.mjs | 40 +- apps/web/wrangler.toml | 6 +- 6 files changed, 1107 insertions(+), 42 deletions(-) create mode 100644 apps/web/drizzle.config.ts create mode 100644 apps/web/migrations/0000_fixed_pandemic.sql create mode 100644 apps/web/migrations/meta/0000_snapshot.json create mode 100644 apps/web/migrations/meta/_journal.json (limited to 'apps') diff --git a/apps/web/drizzle.config.ts b/apps/web/drizzle.config.ts new file mode 100644 index 00000000..5df2ca29 --- /dev/null +++ b/apps/web/drizzle.config.ts @@ -0,0 +1,12 @@ +import { type Config } from "drizzle-kit"; + +export default { + schema: "../../packages/db/schema.ts", + dialect: "sqlite", + driver: "d1", + dbCredentials: { + wranglerConfigPath: "./wrangler.toml", + dbName: "", + }, + out: "migrations", +} satisfies Config; diff --git a/apps/web/migrations/0000_fixed_pandemic.sql b/apps/web/migrations/0000_fixed_pandemic.sql new file mode 100644 index 00000000..09b5431a --- /dev/null +++ b/apps/web/migrations/0000_fixed_pandemic.sql @@ -0,0 +1,152 @@ +CREATE TABLE `account` ( + `userId` text NOT NULL, + `type` text NOT NULL, + `provider` text NOT NULL, + `providerAccountId` text NOT NULL, + `refresh_token` text, + `access_token` text, + `expires_at` integer, + `token_type` text, + `scope` text, + `id_token` text, + `session_state` text, + PRIMARY KEY(`provider`, `providerAccountId`), + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `authenticator` ( + `credentialID` text NOT NULL, + `userId` text NOT NULL, + `providerAccountId` text NOT NULL, + `credentialPublicKey` text NOT NULL, + `counter` integer NOT NULL, + `credentialDeviceType` text NOT NULL, + `credentialBackedUp` integer NOT NULL, + `transports` text, + PRIMARY KEY(`credentialID`, `userId`), + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `canvas` ( + `id` text PRIMARY KEY NOT NULL, + `title` text DEFAULT 'Untitled' NOT NULL, + `description` text DEFAULT 'Untitled' NOT NULL, + `url` text DEFAULT '' NOT NULL, + `userId` text NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `chatHistory` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `threadId` text NOT NULL, + `question` text NOT NULL, + `answerParts` text, + `answerSources` text, + `answerJustification` text, + `createdAt` integer DEFAULT '"2024-07-31T07:35:53.819Z"' NOT NULL, + FOREIGN KEY (`threadId`) REFERENCES `chatThread`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `chatThread` ( + `id` text PRIMARY KEY NOT NULL, + `firstMessage` text NOT NULL, + `userId` text NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `contentToSpace` ( + `contentId` integer NOT NULL, + `spaceId` integer NOT NULL, + PRIMARY KEY(`contentId`, `spaceId`), + FOREIGN KEY (`contentId`) REFERENCES `storedContent`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `jobs` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `userId` text NOT NULL, + `url` text NOT NULL, + `status` text NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `lastAttemptAt` integer, + `error` blob, + `createdAt` integer NOT NULL, + `updatedAt` integer NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `session` ( + `sessionToken` text PRIMARY KEY NOT NULL, + `userId` text NOT NULL, + `expires` integer NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `space` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `name` text DEFAULT 'none' NOT NULL, + `user` text(255), + `createdAt` integer NOT NULL, + `numItems` integer DEFAULT 0 NOT NULL, + FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `spacesAccess` ( + `spaceId` integer NOT NULL, + `userEmail` text NOT NULL, + PRIMARY KEY(`spaceId`, `userEmail`), + FOREIGN KEY (`spaceId`) REFERENCES `space`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `storedContent` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `content` text NOT NULL, + `title` text(255), + `description` text(255), + `url` text NOT NULL, + `savedAt` integer NOT NULL, + `baseUrl` text(255), + `ogImage` text(255), + `type` text DEFAULT 'page', + `image` text(255), + `user` text, + `noteId` integer, + FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE TABLE `user` ( + `id` text PRIMARY KEY NOT NULL, + `name` text, + `email` text NOT NULL, + `emailVerified` integer, + `image` text, + `telegramId` text, + `hasOnboarded` integer DEFAULT false +); +--> statement-breakpoint +CREATE TABLE `verificationToken` ( + `identifier` text NOT NULL, + `token` text NOT NULL, + `expires` integer NOT NULL, + PRIMARY KEY(`identifier`, `token`) +); +--> statement-breakpoint +CREATE UNIQUE INDEX `authenticator_credentialID_unique` ON `authenticator` (`credentialID`);--> statement-breakpoint +CREATE INDEX `canvas_user_userId` ON `canvas` (`userId`);--> statement-breakpoint +CREATE INDEX `chatHistory_thread_idx` ON `chatHistory` (`threadId`);--> statement-breakpoint +CREATE INDEX `chatThread_user_idx` ON `chatThread` (`userId`);--> statement-breakpoint +CREATE INDEX `jobs_userId_idx` ON `jobs` (`userId`);--> statement-breakpoint +CREATE INDEX `jobs_status_idx` ON `jobs` (`status`);--> statement-breakpoint +CREATE INDEX `jobs_createdAt_idx` ON `jobs` (`createdAt`);--> statement-breakpoint +CREATE INDEX `jobs_url_idx` ON `jobs` (`url`);--> statement-breakpoint +CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint +CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint +CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint +CREATE UNIQUE INDEX `storedContent_baseUrl_unique` ON `storedContent` (`baseUrl`);--> statement-breakpoint +CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint +CREATE INDEX `storedContent_savedAt_idx` ON `storedContent` (`savedAt`);--> statement-breakpoint +CREATE INDEX `storedContent_title_idx` ON `storedContent` (`title`);--> statement-breakpoint +CREATE INDEX `storedContent_user_idx` ON `storedContent` (`user`);--> statement-breakpoint +CREATE INDEX `users_email_idx` ON `user` (`email`);--> statement-breakpoint +CREATE INDEX `users_telegram_idx` ON `user` (`telegramId`);--> statement-breakpoint +CREATE INDEX `users_id_idx` ON `user` (`id`); \ No newline at end of file diff --git a/apps/web/migrations/meta/0000_snapshot.json b/apps/web/migrations/meta/0000_snapshot.json new file mode 100644 index 00000000..3bb8617f --- /dev/null +++ b/apps/web/migrations/meta/0000_snapshot.json @@ -0,0 +1,926 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "3fbdb153-2764-4b09-ac22-05c3a131ec35", + "prevId": "00000000-0000-0000-0000-000000000000", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": ["provider", "providerAccountId"], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {} + }, + "authenticator": { + "name": "authenticator", + "columns": { + "credentialID": { + "name": "credentialID", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialPublicKey": { + "name": "credentialPublicKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "counter": { + "name": "counter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialDeviceType": { + "name": "credentialDeviceType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credentialBackedUp": { + "name": "credentialBackedUp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transports": { + "name": "transports", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "authenticator_credentialID_unique": { + "name": "authenticator_credentialID_unique", + "columns": ["credentialID"], + "isUnique": true + } + }, + "foreignKeys": { + "authenticator_userId_user_id_fk": { + "name": "authenticator_userId_user_id_fk", + "tableFrom": "authenticator", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "authenticator_userId_credentialID_pk": { + "columns": ["credentialID", "userId"], + "name": "authenticator_userId_credentialID_pk" + } + }, + "uniqueConstraints": {} + }, + "canvas": { + "name": "canvas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "canvas_user_userId": { + "name": "canvas_user_userId", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": { + "canvas_userId_user_id_fk": { + "name": "canvas_userId_user_id_fk", + "tableFrom": "canvas", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "chatHistory": { + "name": "chatHistory", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "threadId": { + "name": "threadId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "question": { + "name": "question", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answerParts": { + "name": "answerParts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "answerSources": { + "name": "answerSources", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "answerJustification": { + "name": "answerJustification", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'\"2024-07-31T07:35:53.819Z\"'" + } + }, + "indexes": { + "chatHistory_thread_idx": { + "name": "chatHistory_thread_idx", + "columns": ["threadId"], + "isUnique": false + } + }, + "foreignKeys": { + "chatHistory_threadId_chatThread_id_fk": { + "name": "chatHistory_threadId_chatThread_id_fk", + "tableFrom": "chatHistory", + "tableTo": "chatThread", + "columnsFrom": ["threadId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "chatThread": { + "name": "chatThread", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "firstMessage": { + "name": "firstMessage", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "chatThread_user_idx": { + "name": "chatThread_user_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": { + "chatThread_userId_user_id_fk": { + "name": "chatThread_userId_user_id_fk", + "tableFrom": "chatThread", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "contentToSpace": { + "name": "contentToSpace", + "columns": { + "contentId": { + "name": "contentId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "contentToSpace_contentId_storedContent_id_fk": { + "name": "contentToSpace_contentId_storedContent_id_fk", + "tableFrom": "contentToSpace", + "tableTo": "storedContent", + "columnsFrom": ["contentId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "contentToSpace_spaceId_space_id_fk": { + "name": "contentToSpace_spaceId_space_id_fk", + "tableFrom": "contentToSpace", + "tableTo": "space", + "columnsFrom": ["spaceId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "contentToSpace_contentId_spaceId_pk": { + "columns": ["contentId", "spaceId"], + "name": "contentToSpace_contentId_spaceId_pk" + } + }, + "uniqueConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastAttemptAt": { + "name": "lastAttemptAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "jobs_userId_idx": { + "name": "jobs_userId_idx", + "columns": ["userId"], + "isUnique": false + }, + "jobs_status_idx": { + "name": "jobs_status_idx", + "columns": ["status"], + "isUnique": false + }, + "jobs_createdAt_idx": { + "name": "jobs_createdAt_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "jobs_url_idx": { + "name": "jobs_url_idx", + "columns": ["url"], + "isUnique": false + } + }, + "foreignKeys": { + "jobs_userId_user_id_fk": { + "name": "jobs_userId_user_id_fk", + "tableFrom": "jobs", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["userId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "space": { + "name": "space", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "user": { + "name": "user", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "numItems": { + "name": "numItems", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "space_name_unique": { + "name": "space_name_unique", + "columns": ["name"], + "isUnique": true + }, + "spaces_name_idx": { + "name": "spaces_name_idx", + "columns": ["name"], + "isUnique": false + }, + "spaces_user_idx": { + "name": "spaces_user_idx", + "columns": ["user"], + "isUnique": false + } + }, + "foreignKeys": { + "space_user_user_id_fk": { + "name": "space_user_user_id_fk", + "tableFrom": "space", + "tableTo": "user", + "columnsFrom": ["user"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "spacesAccess": { + "name": "spacesAccess", + "columns": { + "spaceId": { + "name": "spaceId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "spacesAccess_spaceId_space_id_fk": { + "name": "spacesAccess_spaceId_space_id_fk", + "tableFrom": "spacesAccess", + "tableTo": "space", + "columnsFrom": ["spaceId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spacesAccess_spaceId_userEmail_pk": { + "columns": ["spaceId", "userEmail"], + "name": "spacesAccess_spaceId_userEmail_pk" + } + }, + "uniqueConstraints": {} + }, + "storedContent": { + "name": "storedContent", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "savedAt": { + "name": "savedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "baseUrl": { + "name": "baseUrl", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ogImage": { + "name": "ogImage", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'page'" + }, + "image": { + "name": "image", + "type": "text(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user": { + "name": "user", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "noteId": { + "name": "noteId", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "storedContent_baseUrl_unique": { + "name": "storedContent_baseUrl_unique", + "columns": ["baseUrl"], + "isUnique": true + }, + "storedContent_url_idx": { + "name": "storedContent_url_idx", + "columns": ["url"], + "isUnique": false + }, + "storedContent_savedAt_idx": { + "name": "storedContent_savedAt_idx", + "columns": ["savedAt"], + "isUnique": false + }, + "storedContent_title_idx": { + "name": "storedContent_title_idx", + "columns": ["title"], + "isUnique": false + }, + "storedContent_user_idx": { + "name": "storedContent_user_idx", + "columns": ["user"], + "isUnique": false + } + }, + "foreignKeys": { + "storedContent_user_user_id_fk": { + "name": "storedContent_user_user_id_fk", + "tableFrom": "storedContent", + "tableTo": "user", + "columnsFrom": ["user"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "telegramId": { + "name": "telegramId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hasOnboarded": { + "name": "hasOnboarded", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "users_email_idx": { + "name": "users_email_idx", + "columns": ["email"], + "isUnique": false + }, + "users_telegram_idx": { + "name": "users_telegram_idx", + "columns": ["telegramId"], + "isUnique": false + }, + "users_id_idx": { + "name": "users_id_idx", + "columns": ["id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": ["identifier", "token"], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + } +} diff --git a/apps/web/migrations/meta/_journal.json b/apps/web/migrations/meta/_journal.json new file mode 100644 index 00000000..59ab4ea6 --- /dev/null +++ b/apps/web/migrations/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "6", + "dialect": "sqlite", + "entries": [ + { + "idx": 0, + "version": "6", + "when": 1722411353835, + "tag": "0000_fixed_pandemic", + "breakpoints": true + } + ] +} diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 934f39ae..307d5bdc 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -1,6 +1,5 @@ import MillionLint from "@million/lint"; import { setupDevPlatform } from "@cloudflare/next-on-pages/next-dev"; -import { withSentryConfig } from "@sentry/nextjs"; /** @type {import('next').NextConfig} */ const baseNextConfig = { @@ -10,8 +9,8 @@ const baseNextConfig = { TELEGRAM_BOT_TOKEN: process.env.TELEGRAM_BOT_TOKEN, }, eslint: { - disableDuringBuilds: true - } + disableDuringBuilds: true, + }, }; let selectedCofig = baseNextConfig; @@ -24,41 +23,6 @@ if (process.env.NODE_ENV === "development") { export default selectedCofig; -//! Disabled sentry for now because of unreasonably large bundle size -// export default withSentryConfig(selectedCofig, { -// // For all available options, see: -// // https://github.com/getsentry/sentry-webpack-plugin#options - -// org: "none-h00", -// project: "javascript-nextjs", -// // Only print logs for uploading source maps in CI -// silent: !process.env.CI, - -// // For all available options, see: -// // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ - -// // Upload a larger set of source maps for prettier stack traces (increases build time) -// widenClientFileUpload: true, - -// // Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers. -// // This can increase your server load as well as your hosting bill. -// // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client- -// // side errors will fail. -// tunnelRoute: "/monitoring", - -// // Hides source maps from generated client bundles -// hideSourceMaps: true, - -// // Automatically tree-shake Sentry logger statements to reduce bundle size -// disableLogger: true, - -// // Enables automatic instrumentation of Vercel Cron Monitors. (Does not yet work with App Router route handlers.) -// // See the following for more information: -// // https://docs.sentry.io/product/crons/ -// // https://vercel.com/docs/cron-jobs -// automaticVercelMonitors: true, -// }); - // we only need to use the utility during development so we can check NODE_ENV // (note: this check is recommended but completely optional) if (process.env.NODE_ENV === "development") { diff --git a/apps/web/wrangler.toml b/apps/web/wrangler.toml index 7c4f63c8..a6232450 100644 --- a/apps/web/wrangler.toml +++ b/apps/web/wrangler.toml @@ -26,10 +26,8 @@ bucket_name = "dev-r2-anycontext" [[d1_databases]] binding = "DATABASE" -database_name = "supermemlocal" -database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" - - +database_name = "dev-d1-anycontext" +database_id = "fc562605-157a-4f60-b439-2a24ffed5b4c" [[env.production.d1_databases]] binding = "DATABASE" -- cgit v1.2.3 From e4fd7f5aacc3c9f7f000e1858248d49aa4d3410e Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Mon, 5 Aug 2024 21:25:11 +0530 Subject: move limit to backend and thread service binding --- apps/cf-ai-backend/src/errors/baseError.ts | 2 +- apps/cf-ai-backend/src/errors/results.ts | 19 +- apps/cf-ai-backend/src/helper.ts | 23 +- apps/cf-ai-backend/src/index.ts | 117 +++-- .../src/queueConsumer/helpers/initQuery.ts | 58 --- .../src/queueConsumer/helpers/processPage.ts | 13 +- .../src/queueConsumer/helpers/processTweet.ts | 73 +-- apps/cf-ai-backend/src/queueConsumer/index.ts | 336 ++++++++++--- apps/cf-ai-backend/src/types.ts | 3 + apps/cf-ai-backend/wrangler.toml | 6 +- apps/web/app/actions/doers.ts | 526 ++++++++++----------- apps/web/app/api/store/helper.ts | 18 +- apps/web/lib/constants.ts | 6 - 13 files changed, 684 insertions(+), 516 deletions(-) delete mode 100644 apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts (limited to 'apps') diff --git a/apps/cf-ai-backend/src/errors/baseError.ts b/apps/cf-ai-backend/src/errors/baseError.ts index 2723d45b..0dcc2203 100644 --- a/apps/cf-ai-backend/src/errors/baseError.ts +++ b/apps/cf-ai-backend/src/errors/baseError.ts @@ -7,7 +7,7 @@ export class BaseHttpError extends Error { this.status = status; this.message = message; Object.setPrototypeOf(this, new.target.prototype); // Restore prototype chain - } + } } diff --git a/apps/cf-ai-backend/src/errors/results.ts b/apps/cf-ai-backend/src/errors/results.ts index 87ea0c63..ccce1396 100644 --- a/apps/cf-ai-backend/src/errors/results.ts +++ b/apps/cf-ai-backend/src/errors/results.ts @@ -14,15 +14,18 @@ export const Err = (error: E): Result => { export async function wrap( p: Promise, - errorFactory: (err: Error) => E, -): Promise> { + errorFactory: (err: Error, source: string) => E, + source: string = "unspecified" + ): Promise> { try { - return Ok(await p); + return Ok(await p); } catch (e) { - return Err(errorFactory(e as Error)); + return Err(errorFactory(e as Error, source)); } -} + } -export function isErr(result: Result): result is { ok: false; error: E } { - return !result.ok; - } \ No newline at end of file +export function isErr( + result: Result, +): result is { ok: false; error: E } { + return !result.ok; +} diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 1568996a..eadd9c21 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -9,17 +9,14 @@ import { z } from "zod"; import { seededRandom } from "./utils/seededRandom"; import { bulkInsertKv } from "./utils/kvBulkInsert"; -export async function initQuery( - c: Context<{ Bindings: Env }>, - model: string = "gpt-4o", -) { +export async function initQuery(env: Env, model: string = "gpt-4o") { const embeddings = new OpenAIEmbeddings({ - apiKey: c.env.OPENAI_API_KEY, + apiKey: env.OPENAI_API_KEY, modelName: "text-embedding-3-small", }); const store = new CloudflareVectorizeStore(embeddings, { - index: c.env.VECTORIZE_INDEX, + index: env.VECTORIZE_INDEX, }); let selectedModel: @@ -30,7 +27,7 @@ export async function initQuery( switch (model) { case "claude-3-opus": const anthropic = createAnthropic({ - apiKey: c.env.ANTHROPIC_API_KEY, + apiKey: env.ANTHROPIC_API_KEY, baseURL: "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic", }); @@ -39,7 +36,7 @@ export async function initQuery( break; case "gemini-1.5-pro": const googleai = createGoogleGenerativeAI({ - apiKey: c.env.GOOGLE_AI_API_KEY, + apiKey: env.GOOGLE_AI_API_KEY, baseURL: "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/google-vertex-ai", }); @@ -49,7 +46,7 @@ export async function initQuery( case "gpt-4o": default: const openai = createOpenAI({ - apiKey: c.env.OPENAI_API_KEY, + apiKey: env.OPENAI_API_KEY, baseURL: "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai", compatibility: "strict", @@ -204,7 +201,7 @@ export async function batchCreateChunksAndEmbeddings({ const commonMetaData = { type: body.type ?? "tweet", title: body.title?.slice(0, 50) ?? "", - description: body.description ?? "", + description: body.description?.slice(0, 50) ?? "", url: body.url, [sanitizeKey(`user-${body.user}`)]: 1, }; @@ -255,7 +252,7 @@ export async function batchCreateChunksAndEmbeddings({ const commonMetaData = { type: body.type ?? "page", title: body.title?.slice(0, 50) ?? "", - description: body.description ?? "", + description: body.description?.slice(0, 50) ?? "", url: body.url, [sanitizeKey(`user-${body.user}`)]: 1, }; @@ -292,7 +289,7 @@ export async function batchCreateChunksAndEmbeddings({ const commonMetaData = { title: body.title?.slice(0, 50) ?? "", type: body.type ?? "page", - description: body.description ?? "", + description: body.description?.slice(0, 50) ?? "", url: body.url, [sanitizeKey(`user-${body.user}`)]: 1, }; @@ -328,7 +325,7 @@ export async function batchCreateChunksAndEmbeddings({ const commonMetaData = { type: body.type ?? "image", title: body.title, - description: body.description ?? "", + description: body.description?.slice(0, 50) ?? "", url: body.url, [sanitizeKey(`user-${body.user}`)]: 1, }; diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 4949fab3..a46d0080 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -24,12 +24,18 @@ import { zValidator } from "@hono/zod-validator"; import chunkText from "./queueConsumer/chunkers/chonker"; import { systemPrompt, template } from "./prompts/prompt1"; import { swaggerUI } from "@hono/swagger-ui"; +import { database } from "./db"; +import { storedContent } from "@repo/db/schema"; +import { sql, and, eq } from "drizzle-orm"; +import { LIMITS } from "@repo/shared-types"; +import { typeDecider } from "./queueConsumer/utils/typeDecider"; // import { chunkThread } from "./utils/chunkTweet"; import { chunkNote, chunkPage, } from "./queueConsumer/chunkers/chunkPageOrNotes"; import { queue } from "./queueConsumer"; +import { isErr } from "./errors/results"; const app = new Hono<{ Bindings: Env }>(); @@ -68,42 +74,75 @@ app.get("/api/health", (c) => { app.post("/api/add", zValidator("json", vectorBody), async (c) => { try { - // console.log("api/add hit!!!!"); const body = c.req.valid("json"); - const spaceNumbers = body.spaces.map((s: string) => Number(s)); - await c.env.EMBEDCHUNKS_QUEUE.send({ - content: body.url, - user: body.user, - space: spaceNumbers, - }); - - // const { store } = await initQuery(c); + //This is something I don't like + // console.log("api/add hit!!!!"); + //Have to do limit on this also duplicate check here + const db = database(c.env); + const typeResult = typeDecider(body.url); + + const saveToDbUrl = + (body.url.split("#supermemory-user-")[0] ?? body.url) + // Why does this have to be a split from #supermemory-user? + "#supermemory-user-" + + body.user; + + console.log( + "---------------------------------------------------------------------------------------------------------------------------------------------", + saveToDbUrl, + ); + const alreadyExist = await db + .select() + .from(storedContent) + .where(eq(storedContent.baseUrl, saveToDbUrl)); + console.log( + "------------------------------------------------", + JSON.stringify(alreadyExist), + ); - // console.log(body.spaces); - // let chunks: TweetChunks | PageOrNoteChunks; - // // remove everything in tags - // // const newPageContent = body.pageContent?.replace(/.*?<\/raw>/g, ""); + if (alreadyExist.length > 0) { + console.log( + "------------------------------------------------------------------------------------------------I exist------------------------", + ); + return c.json({ status: "error", message: "the content already exists" }); + } - // switch (body.type) { - // case "tweet": - // chunks = chunkThread(body.pageContent); - // break; + if (isErr(typeResult)) { + throw typeResult.error; + } + // limiting in the backend + const type = typeResult.value; + const countResult = await db + .select({ + count: sql`count(*)`.mapWith(Number), + }) + .from(storedContent) + .where( + and(eq(storedContent.userId, body.user), eq(storedContent.type, type)), + ); - // case "page": - // chunks = chunkPage(body.pageContent); - // break; + const currentCount = countResult[0]?.count || 0; + const totalLimit = LIMITS[type as keyof typeof LIMITS]; + const remainingLimit = totalLimit - currentCount; + const items = 1; + const isWithinLimit = items <= remainingLimit; - // case "note": - // chunks = chunkNote(body.pageContent); - // break; - // } + // unique contraint check - // await batchCreateChunksAndEmbeddings({ - // store, - // body, - // chunks: chunks, - // env: c, - // }); + if (isWithinLimit) { + const spaceNumbers = body.spaces.map((s: string) => Number(s)); + await c.env.EMBEDCHUNKS_QUEUE.send({ + content: body.url, + user: body.user, + space: spaceNumbers, + type: type, + }); + } else { + return c.json({ + status: "error", + message: + "You have exceed the current limit for this type of document, please try removing something form memories ", + }); + } return c.json({ status: "ok" }); } catch (error) { @@ -137,7 +176,7 @@ app.post( async (c) => { const body = c.req.valid("form"); - const { store } = await initQuery(c); + const { store } = await initQuery(c.env); if (!(body.images || body["images[]"])) { return c.json({ status: "error", message: "No images found" }, 400); @@ -203,7 +242,7 @@ app.get( async (c) => { const query = c.req.valid("query"); - const { model } = await initQuery(c); + const { model } = await initQuery(c.env); const response = await streamText({ model, prompt: query.query }); const r = response.toTextStreamResponse(); @@ -221,7 +260,7 @@ app.get( [`user-${user}`]: 1, }; - const { store } = await initQuery(c); + const { store } = await initQuery(c.env); const queryAsVector = await store.embeddings.embedQuery(query); const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { @@ -273,7 +312,7 @@ app.post( const { query, user } = c.req.valid("query"); const { chatHistory } = c.req.valid("json"); - const { store, model } = await initQuery(c); + const { store, model } = await initQuery(c.env); let task: "add" | "chat" = "chat"; let thingToAdd: "page" | "image" | "text" | undefined = undefined; @@ -448,7 +487,7 @@ app.post( const spaces = query.spaces?.split(",") ?? [undefined]; // Get the AI model maker and vector store - const { model, store } = await initQuery(c, query.model); + const { model, store } = await initQuery(c.env, query.model); if (!body.sources) { const filter: VectorizeVectorMetadataFilter = { @@ -591,6 +630,8 @@ app.post( } } + //Serach mem0 + const preparedContext = body.sources.normalizedData.map( ({ metadata, score, normalizedScore }) => ({ context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, @@ -601,7 +642,7 @@ app.post( const initialMessages: CoreMessage[] = [ { role: "user", content: systemPrompt }, - { role: "assistant", content: "Hello, how can I help?" }, + { role: "assistant", content: "Hello, how can I help?" }, // prase and add memory json here ]; const prompt = template({ @@ -637,7 +678,7 @@ app.delete( async (c) => { const { websiteUrl, user } = c.req.valid("query"); - const { store } = await initQuery(c); + const { store } = await initQuery(c.env); await deleteDocument({ url: websiteUrl, user, c, store }); @@ -657,7 +698,7 @@ app.get( ), async (c) => { const { context, request } = c.req.valid("query"); - const { model } = await initQuery(c); + const { model } = await initQuery(c.env); const response = await streamText({ model, diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts deleted file mode 100644 index a7d85c23..00000000 --- a/apps/cf-ai-backend/src/queueConsumer/helpers/initQuery.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Env } from "../../types"; -import { OpenAIEmbeddings } from "../../utils/OpenAIEmbedder"; -import { CloudflareVectorizeStore } from "@langchain/cloudflare"; -import { createOpenAI } from "@ai-sdk/openai"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; -import { createAnthropic } from "@ai-sdk/anthropic"; - -export async function initQQuery( - env: Env, - model: string = "gpt-4o", -) { - const embeddings = new OpenAIEmbeddings({ - apiKey: env.OPENAI_API_KEY, - modelName: "text-embedding-3-small", - }); - - const store = new CloudflareVectorizeStore(embeddings, { - index: env.VECTORIZE_INDEX, - }); - - let selectedModel: - | ReturnType> - | ReturnType> - | ReturnType>; - - switch (model) { - case "claude-3-opus": - const anthropic = createAnthropic({ - apiKey: env.ANTHROPIC_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/anthropic", - }); - selectedModel = anthropic.chat("claude-3-opus-20240229"); - console.log("Selected model: ", selectedModel); - break; - case "gemini-1.5-pro": - const googleai = createGoogleGenerativeAI({ - apiKey: env.GOOGLE_AI_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/google-vertex-ai", - }); - selectedModel = googleai.chat("models/gemini-1.5-pro-latest"); - console.log("Selected model: ", selectedModel); - break; - case "gpt-4o": - default: - const openai = createOpenAI({ - apiKey: env.OPENAI_API_KEY, - baseURL: - "https://gateway.ai.cloudflare.com/v1/47c2b4d598af9d423c06fc9f936226d5/supermemory/openai", - compatibility: "strict", - }); - selectedModel = openai.chat("gpt-4o-mini"); - break; - } - - return { store, model: selectedModel }; -} \ No newline at end of file diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts index 6b28c975..f967736e 100644 --- a/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts @@ -10,13 +10,14 @@ class ProcessPageError extends BaseError { type PageProcessResult = { pageContent: string; metadata: Metadata }; -export async function processPage( - url: string, -): Promise> { +export async function processPage(input: { + url: string; + securityKey: string; +}): Promise> { try { - const response = await fetch("https://md.dhr.wtf/?url=" + url, { + const response = await fetch("https://md.dhr.wtf/?url=" + input.url, { headers: { - Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + Authorization: "Bearer " + input.securityKey, }, }); const pageContent = await response.text(); @@ -29,7 +30,7 @@ export async function processPage( ); } console.log("[This is the page content]", pageContent); - const metadataResult = await getMetaData(url); + const metadataResult = await getMetaData(input.url); if (isErr(metadataResult)) { throw metadataResult.error; } diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts index ef5d9f5b..8d83f2dc 100644 --- a/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processTweet.ts @@ -3,6 +3,7 @@ import { Result, Ok, Err, isErr } from "../../errors/results"; import { BaseError } from "../../errors/baseError"; import { getMetaData, Metadata } from "../utils/get-metadata"; import { tweetToMd } from "@repo/shared-types/utils"; // can I do this? +import { Env } from "../../types"; class ProcessTweetError extends BaseError { constructor(message?: string, source?: string) { @@ -43,39 +44,45 @@ export const getTweetData = async ( } }; -export const getThreadData = async ( - tweetUrl: string, - cf_thread_endpoint: string, - authKey: string, -): Promise> => { - const threadRequest = await fetch(cf_thread_endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: authKey, - }, - body: JSON.stringify({ url: tweetUrl }), - }); - if (threadRequest.status !== 200) { - return Err( - new ProcessTweetError( - `Failed to fetch the thread: ${tweetUrl}, Reason: ${threadRequest.statusText}`, - "getThreadData", - ), - ); - } - - const thread = await threadRequest.text(); - console.log("[thread response]"); +export const getThreadData = async (input: { + tweetUrl: string; + env: Env; +}): Promise> => { + try { + // const threadRequest = await fetch(input.cf_thread_endpoint, { + // method: "POST", + // headers: { + // "Content-Type": "application/json", + // Authorization: input.authKey, + // }, + // body: JSON.stringify({ url: input.tweetUrl }), + // }); + // if (threadRequest.status !== 200) { + // console.log(await threadRequest.text()); + // console.log(input.tweetUrl); + // return Err( + // new ProcessTweetError( + // `Failed to fetch the thread: ${input.tweetUrl}, Reason: ${threadRequest.statusText}`, + // "getThreadData", + // ), + // ); + // } + //@ts-ignore + const thread = await input.env.THREAD.processTweets(input.tweetUrl); + console.log("[thread response]", thread); - if (thread.trim().length === 2) { - console.log("Thread is an empty array"); - return Err( - new ProcessTweetError( - "[THREAD FETCHING SERVICE] Got no content form thread worker", - "getThreadData", - ), - ); + if (!thread.length) { + console.log("Thread is an empty array"); + return Err( + new ProcessTweetError( + "[THREAD FETCHING SERVICE] Got no content form thread worker", + "getThreadData", + ), + ); + } + return Ok(thread); + } catch (e) { + console.error("[Thread Processing Error]", e); + return Err(new ProcessTweetError((e as Error).message, "getThreadData")); } - return Ok(thread); }; diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts index 8b9064cb..8ca23739 100644 --- a/apps/cf-ai-backend/src/queueConsumer/index.ts +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -5,15 +5,21 @@ import { processNote } from "./helpers/processNotes"; import { processPage } from "./helpers/processPage"; import { getThreadData, getTweetData } from "./helpers/processTweet"; import { tweetToMd } from "@repo/shared-types/utils"; -import { initQQuery } from "./helpers/initQuery"; import { chunkNote, chunkPage } from "./chunkers/chunkPageOrNotes"; import { chunkThread } from "./chunkers/chunkTweet"; -import { batchCreateChunksAndEmbeddings } from "../helper"; +import { batchCreateChunksAndEmbeddings, initQuery } from "../helper"; import { z } from "zod"; import { Metadata } from "./utils/get-metadata"; import { BaseError } from "../errors/baseError"; import { database } from "../db"; -import { storedContent, space, contentToSpace } from "@repo/db/schema"; +import { + storedContent, + space, + contentToSpace, + users, + jobs, + Job, +} from "@repo/db/schema"; import { and, eq, inArray, sql } from "drizzle-orm"; class VectorInsertError extends BaseError { @@ -29,24 +35,99 @@ class D1InsertError extends BaseError { } } +const d1ErrorFactory = (err: Error, source: string) => + new D1InsertError(err.message, source); + +const calculateExponentialBackoff = ( + attempts: number, + baseDelaySeconds: number, +) => { + return baseDelaySeconds ** attempts; +}; + +const BASE_DELAY_SECONDS = 1.5; export async function queue( - batch: MessageBatch<{ content: string; space: Array; user: string }>, + batch: MessageBatch<{ + content: string; + space: Array; + user: string; + type: string; + }>, env: Env, ): Promise { + const db = database(env); console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); for (let message of batch.messages) { console.log(env.CF_ACCOUNT_ID, env.CF_KV_AUTH_TOKEN); console.log("is thie even running?", message.body); const body = message.body; - console.log("v got shit in the queue", body); - const typeResult = typeDecider(body.content); + const type = body.type; + const userExists = await wrap( + db.select().from(users).where(eq(users.id, body.user)).limit(1), + d1ErrorFactory, + "Error when trying to verify user", + ); + + if (isErr(userExists)) { + throw userExists.error; + } + + //check if this is a retry job.. by checking if the combination of the userId and the url already exists on the queue + let jobId; + const existingJob = await wrap( + db + .select() + .from(jobs) + .where( + and( + eq(jobs.userId, userExists.value[0].id), + eq(jobs.url, body.content), + ), + ) + .limit(1), + d1ErrorFactory, + "Error when checking for existing job", + ); + + if (isErr(existingJob)) { + throw existingJob.error; + } - if (isErr(typeResult)) { - throw typeResult.error; + if (existingJob.value.length > 0) { + jobId = existingJob.value[0].id; + await wrap( + db + .update(jobs) + .set({ + attempts: existingJob.value[0].attempts + 1, + updatedAt: new Date(), + }) + .where(eq(jobs.id, jobId)), + d1ErrorFactory, + "Error when updating job attempts", + ); + } else { + const job = await wrap( + db + .insert(jobs) + .values({ + userId: userExists.value[0].id as string, + url: body.content, + status: "Processing", + attempts: 1, + createdAt: new Date(), + updatedAt: new Date(), + }) + .returning({ jobId: jobs.id }), + d1ErrorFactory, + "Error When inserting into jobs table", + ); + if (isErr(job)) { + throw job.error; + } + jobId = job.value[0].jobId; } - console.log(typeResult.value); - const type = typeResult.value; let pageContent: string; let vectorData: string; @@ -70,8 +151,12 @@ export async function queue( } case "page": { console.log("page hit"); - const page = await processPage(body.content); + const page = await processPage({ + url: body.content, + securityKey: env.MD_SEC_KEY, + }); if (isErr(page)) { + console.log("there is a page error here"); throw page.error; } pageContent = page.value.pageContent; @@ -83,15 +168,13 @@ export async function queue( case "tweet": { console.log("tweet hit"); - console.log(body.content.split("/").pop()); const tweet = await getTweetData(body.content.split("/").pop()); - console.log(tweet); - const thread = await getThreadData( - body.content, - env.THREAD_CF_WORKER, - env.THREAD_CF_AUTH, - ); - + console.log(env.THREAD_CF_WORKER, env.THREAD_CF_AUTH); + const thread = await getThreadData({ + tweetUrl: body.content, + env: env, + }); + console.log("[This is the thread]", thread); if (isErr(tweet)) { throw tweet.error; } @@ -108,6 +191,7 @@ export async function queue( vectorData = JSON.stringify(pageContent); console.error(thread.error); } else { + console.log("thread worker is fine"); vectorData = thread.value; } chunks = chunkThread(vectorData); @@ -115,8 +199,27 @@ export async function queue( } } + //add to mem0, abstract + + // const mem0Response = fetch('https://api.mem0.ai/v1/memories/', { + // method: 'POST', + // headers: { + // 'Content-Type': 'application/json', + // Authorization: `Token ${process.env.MEM0_API_KEY}`, + // }, + // body: JSON.stringify({ + // messages: [ + // { + // role: 'user', + // content: query, + // }, + // ], + // user_id: user?.user?.email, + // }), + // }); + // see what's up with the storedToSpaces in this block - const { store } = await initQQuery(env); + const { store } = await initQuery(env); type body = z.infer; @@ -129,66 +232,135 @@ export async function queue( description: metadata.description, title: metadata.description, }; - const vectorResult = await wrap( - batchCreateChunksAndEmbeddings({ - store: store, - body: Chunkbody, - chunks: chunks, - env: env, - }), - vectorErrorFactory, - ); - if (isErr(vectorResult)) { - throw vectorResult.error; - } - const saveToDbUrl = - (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + - "#supermemory-user-" + - body.user; - let contentId: number; - const db = database(env); - const insertResponse = await db - .insert(storedContent) - .values({ - content: pageContent as string, - title: metadata.title, - description: metadata.description, - url: saveToDbUrl, - baseUrl: saveToDbUrl, - image: metadata.image, - savedAt: new Date(), - userId: body.user, - type: type, - noteId: noteId, - }) - .returning({ id: storedContent.id }); - - if (!insertResponse[0]?.id) { - throw new D1InsertError( - "something went worng when inserting to database", - "inresertResponse", + try { + const vectorResult = await wrap( + batchCreateChunksAndEmbeddings({ + store: store, + body: Chunkbody, + chunks: chunks, + env: env, + }), + vectorErrorFactory, + "Error when Inserting into vector database", ); - } - contentId = insertResponse[0]?.id; - if (storeToSpaces.length > 0) { - // Adding the many-to-many relationship between content and spaces - const spaceData = await db - .select() - .from(space) - .where(and(inArray(space.id, storeToSpaces), eq(space.user, body.user))) - .all(); - await Promise.all( - spaceData.map(async (s) => { - await db - .insert(contentToSpace) - .values({ contentId: contentId, spaceId: s.id }); + if (isErr(vectorResult)) { + await db + .update(jobs) + .set({ error: vectorResult.error }) + .where(eq(jobs.id, jobId)); + message.retry({ + delaySeconds: calculateExponentialBackoff( + message.attempts, + BASE_DELAY_SECONDS, + ), + }); + throw vectorResult.error; + } - await db.update(space).set({ numItems: s.numItems + 1 }); - }), + const saveToDbUrl = + (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + + "#supermemory-user-" + + body.user; + let contentId: number; + + const insertResponse = await wrap( + db + .insert(storedContent) + .values({ + content: pageContent as string, + title: metadata.title, + description: metadata.description, + url: saveToDbUrl, + baseUrl: saveToDbUrl, + image: metadata.image, + savedAt: new Date(), + userId: body.user, + type: type, + noteId: noteId, + }) + .returning({ id: storedContent.id }), + d1ErrorFactory, + "Error when inserting into storedContent", + ); + + if (isErr(insertResponse)) { + await db + .update(jobs) + .set({ error: insertResponse.error }) + .where(eq(jobs.id, jobId)); + message.retry({ + delaySeconds: calculateExponentialBackoff( + message.attempts, + BASE_DELAY_SECONDS, + ), + }); + throw insertResponse.error; + } + console.log(JSON.stringify(insertResponse)); + contentId = insertResponse[0]?.id; + console.log("this is the content Id", contentId); + if (storeToSpaces.length > 0) { + // Adding the many-to-many relationship between content and spaces + const spaceData = await wrap( + db + .select() + .from(space) + .where( + and(inArray(space.id, storeToSpaces), eq(space.user, body.user)), + ) + .all(), + d1ErrorFactory, + "Error when getting data from spaces", + ); + + if (isErr(spaceData)) { + throw spaceData.error; + } + try { + await Promise.all( + spaceData.value.map(async (s) => { + try { + await db + .insert(contentToSpace) + .values({ contentId: contentId, spaceId: s.id }); + + await db.update(space).set({ numItems: s.numItems + 1 }); + } catch (e) { + console.error(`Error updating space ${s.id}:`, e); + throw e; + } + }), + ); + } catch (e) { + console.error("Error in updateSpacesWithContent:", e); + throw new Error(`Failed to update spaces: ${e.message}`); + } + } + } catch (e) { + console.error("Error in simulated transaction", e.message); + console.log("Rooling back changes"); + message.retry({ + delaySeconds: calculateExponentialBackoff( + message.attempts, + BASE_DELAY_SECONDS, + ), + }); + throw new D1InsertError( + "Error when inserting into d1", + "D1 stuff after the vectorize", ); } + + // After the d1 and vectories suceeds then finally update the jobs table to indicate that the job has completed + + await db + .update(jobs) + .set({ status: "Processed" }) + .where(eq(jobs.id, jobId)); + + return; } } @@ -199,6 +371,18 @@ To do: 3. remove getMetada form the lib file as it's not being used anywhere else 4. Figure out the limit stuff ( server action for that seems fine because no use in limiting after they already in the queue rigth? ) 5. Figure out the initQuery stuff ( ;( ) --> This is a bad way of doing stuff :0 -6. How do I hande the content already exists wala use case? -7. Figure out retry and not add shit to the vectirze over and over again on failure +6. How do I hande the content already exists wala use case? --> Also how do I figure out limits? + + + +8. Wrap the d1 thing in a transaction and then write to vectorize if d1 is sucessful if it's not then just error out ( if d1 fails dlq, recoverable failure --> retry ) + +Firt write to d1 in a transaction ( sotredContent + sapces ) --> write to vectorize --> vectorize failes --> reset d1 alternatively first we can also do the vectorise stuff if that suceeds then do the d1 stuff in a batch right? + + +DEBUG: +What's hapenning: +1. The stuff in the d1 is updating but nothing is hapenning in the vectorize for some reason + + */ diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 5294e0a1..e4f13f1b 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -18,14 +18,17 @@ export type Env = { MYBROWSER: unknown; ANTHROPIC_API_KEY: string; THREAD_CF_AUTH: string; + THREAD: { processTweets: () => Promise> }; THREAD_CF_WORKER: string; NODE_ENV: string; + MD_SEC_KEY: string; }; export interface JobData { content: string; space: Array; user: string; + type: string } export interface TweetData { diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index ca8e2b1d..665ca593 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -3,8 +3,12 @@ main = "src/index.ts" compatibility_date = "2024-02-23" node_compat = true -tail_consumers = [{service = "new-cf-ai-backend-tail"}] +# tail_consumers = [{service = "new-cf-ai-backend-tail"}] +[[services]] +binding = "THREAD" +service = "tweet-thread" +entrypoint = "ThreadWorker" # [env.preview] [[vectorize]] diff --git a/apps/web/app/actions/doers.ts b/apps/web/app/actions/doers.ts index f17032b9..500a8608 100644 --- a/apps/web/app/actions/doers.ts +++ b/apps/web/app/actions/doers.ts @@ -17,7 +17,7 @@ import { auth } from "../../server/auth"; import { Tweet } from "react-tweet/api"; // import { getMetaData } from "@/lib/get-metadata"; import { and, eq, inArray, sql } from "drizzle-orm"; -import { LIMITS } from "@/lib/constants"; +import { LIMITS } from "@repo/shared-types"; import { ChatHistory } from "@repo/shared-types"; import { decipher } from "@/server/encrypt"; import { redirect } from "next/navigation"; @@ -104,25 +104,6 @@ const typeDecider = (content: string): "page" | "tweet" | "note" => { } }; -export const limit = async ( - userId: string, - type = "page", - items: number = 1, -) => { - const countResult = await db - .select({ - count: sql`count(*)`.mapWith(Number), - }) - .from(storedContent) - .where(and(eq(storedContent.userId, userId), eq(storedContent.type, type))); - - const currentCount = countResult[0]?.count || 0; - const totalLimit = LIMITS[type as keyof typeof LIMITS]; - const remainingLimit = totalLimit - currentCount; - - return items <= remainingLimit; -}; - export const addUserToSpace = async (userEmail: string, spaceId: number) => { const data = await auth(); @@ -197,7 +178,6 @@ export const createMemory = async (input: { return { error: "Not authenticated", success: false }; } - // make the backend reqeust for the queue here const vectorSaveResponses = await fetch( `${process.env.BACKEND_BASE_URL}/api/add`, @@ -214,250 +194,262 @@ export const createMemory = async (input: { }, }, ); + const response = (await vectorSaveResponses.json()) as { + status: string; + message?: string; + }; + + if (response.status !== "ok") { + return { + success: false, + data: 0, + error: response.message, + }; + } -// const type = typeDecider(input.content); - -// let pageContent = input.content; -// let metadata: Awaited>; -// let vectorData: string; - -// if (!(await limit(data.user.id, type))) { -// return { -// success: false, -// data: 0, -// error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, -// }; -// } - -// let noteId = 0; - -// if (type === "page") { -// const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { -// headers: { -// Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, -// }, -// }); -// pageContent = await response.text(); -// vectorData = pageContent; -// try { -// metadata = await getMetaData(input.content); -// } catch (e) { -// return { -// success: false, -// error: "Failed to fetch metadata for the page. Please try again later.", -// }; -// } -// } else if (type === "tweet") { -// //Request the worker for the entire thread - -// let thread: string; -// let errorOccurred: boolean = false; - -// try { -// const cf_thread_endpoint = process.env.THREAD_CF_WORKER; -// const authKey = process.env.THREAD_CF_AUTH; -// const threadRequest = await fetch(cf_thread_endpoint, { -// method: "POST", -// headers: { -// "Content-Type": "application/json", -// Authorization: authKey, -// }, -// body: JSON.stringify({ url: input.content }), -// }); - -// if (threadRequest.status !== 200) { -// throw new Error( -// `Failed to fetch the thread: ${input.content}, Reason: ${threadRequest.statusText}`, -// ); -// } - -// thread = await threadRequest.text(); -// if (thread.trim().length === 2) { -// console.log("Thread is an empty array"); -// throw new Error( -// "[THREAD FETCHING SERVICE] Got no content form thread worker", -// ); -// } -// } catch (e) { -// console.log("[THREAD FETCHING SERVICE] Failed to fetch the thread", e); -// errorOccurred = true; -// } - -// const tweet = await getTweetData(input.content.split("/").pop() as string); - -// pageContent = tweetToMd(tweet); -// console.log("THis ishte page content!!", pageContent); -// //@ts-ignore -// vectorData = errorOccurred ? JSON.stringify(pageContent) : thread; -// metadata = { -// baseUrl: input.content, -// description: tweet.text.slice(0, 200), -// image: tweet.user.profile_image_url_https, -// title: `Tweet by ${tweet.user.name}`, -// }; -// } else if (type === "note") { -// pageContent = input.content; -// vectorData = pageContent; -// noteId = new Date().getTime(); -// metadata = { -// baseUrl: `https://supermemory.ai/note/${noteId}`, -// description: `Note created at ${new Date().toLocaleString()}`, -// image: "https://supermemory.ai/logo.png", -// title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, -// }; -// } else { -// return { -// success: false, -// data: 0, -// error: "Invalid type", -// }; -// } - -// let storeToSpaces = input.spaces; - -// if (!storeToSpaces) { -// storeToSpaces = []; -// } - -// const vectorSaveResponse = await fetch( -// `${process.env.BACKEND_BASE_URL}/api/add`, -// { -// method: "POST", -// body: JSON.stringify({ -// pageContent: vectorData, -// title: metadata.title, -// description: metadata.description, -// url: metadata.baseUrl, -// spaces: storeToSpaces.map((spaceId) => spaceId.toString()), -// user: data.user.id, -// type, -// }), -// headers: { -// "Content-Type": "application/json", -// Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, -// }, -// }, -// ); - -// if (!vectorSaveResponse.ok) { -// const errorData = await vectorSaveResponse.text(); -// console.error(errorData); -// return { -// success: false, -// data: 0, -// error: `Failed to save to vector store. Backend returned error: ${errorData}`, -// }; -// } - -// let contentId: number; - -// const response = (await vectorSaveResponse.json()) as { -// status: string; -// chunkedInput: string; -// message?: string; -// }; - -// try { -// if (response.status !== "ok") { -// if (response.status === "error") { -// return { -// success: false, -// data: 0, -// error: response.message, -// }; -// } else { -// return { -// success: false, -// data: 0, -// error: `Failed to save to vector store. Backend returned error: ${response.message}`, -// }; -// } -// } -// } catch (e) { -// return { -// success: false, -// data: 0, -// error: `Failed to save to vector store. Backend returned error: ${e}`, -// }; -// } - -// const saveToDbUrl = -// (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + -// "#supermemory-user-" + -// data.user.id; - -// // Insert into database -// try { -// const insertResponse = await db -// .insert(storedContent) -// .values({ -// content: pageContent, -// title: metadata.title, -// description: metadata.description, -// url: saveToDbUrl, -// baseUrl: saveToDbUrl, -// image: metadata.image, -// savedAt: new Date(), -// userId: data.user.id, -// type, -// noteId, -// }) -// .returning({ id: storedContent.id }); -// revalidatePath("/memories"); -// revalidatePath("/home"); - -// if (!insertResponse[0]?.id) { -// return { -// success: false, -// data: 0, -// error: "Something went wrong while saving the document to the database", -// }; -// } - -// contentId = insertResponse[0]?.id; -// } catch (e) { -// const error = e as Error; -// console.log("Error: ", error.message); - -// if ( -// error.message.includes( -// "D1_ERROR: UNIQUE constraint failed: storedContent.baseUrl", -// ) -// ) { -// return { -// success: false, -// data: 0, -// error: "Content already exists", -// }; -// } - -// return { -// success: false, -// data: 0, -// error: "Failed to save to database with error: " + error.message, -// }; -// } - -// if (storeToSpaces.length > 0) { -// // Adding the many-to-many relationship between content and spaces -// const spaceData = await db -// .select() -// .from(space) -// .where( -// and(inArray(space.id, storeToSpaces), eq(space.user, data.user.id)), -// ) -// .all(); - -// await Promise.all( -// spaceData.map(async (s) => { -// await db -// .insert(contentToSpace) -// .values({ contentId: contentId, spaceId: s.id }); - -// await db.update(space).set({ numItems: s.numItems + 1 }); -// }), -// ); -// } + // const type = typeDecider(input.content); + + // let pageContent = input.content; + // let metadata: Awaited>; + // let vectorData: string; + + // if (!(await limit(data.user.id, type))) { + // return { + // success: false, + // data: 0, + // error: `You have exceeded the limit of ${LIMITS[type as keyof typeof LIMITS]} ${type}s.`, + // }; + // } --> How would this fit in the backend??? + + // let noteId = 0; + + // if (type === "page") { + // const response = await fetch("https://md.dhr.wtf/?url=" + input.content, { + // headers: { + // Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + // }, + // }); + // pageContent = await response.text(); + // vectorData = pageContent; + // try { + // metadata = await getMetaData(input.content); + // } catch (e) { + // return { + // success: false, + // error: "Failed to fetch metadata for the page. Please try again later.", + // }; + // } + // } else if (type === "tweet") { + // //Request the worker for the entire thread + + // let thread: string; + // let errorOccurred: boolean = false; + + // try { + // const cf_thread_endpoint = process.env.THREAD_CF_WORKER; + // const authKey = process.env.THREAD_CF_AUTH; + // const threadRequest = await fetch(cf_thread_endpoint, { + // method: "POST", + // headers: { + // "Content-Type": "application/json", + // Authorization: authKey, + // }, + // body: JSON.stringify({ url: input.content }), + // }); + + // if (threadRequest.status !== 200) { + // throw new Error( + // `Failed to fetch the thread: ${input.content}, Reason: ${threadRequest.statusText}`, + // ); + // } + + // thread = await threadRequest.text(); + // if (thread.trim().length === 2) { + // console.log("Thread is an empty array"); + // throw new Error( + // "[THREAD FETCHING SERVICE] Got no content form thread worker", + // ); + // } + // } catch (e) { + // console.log("[THREAD FETCHING SERVICE] Failed to fetch the thread", e); + // errorOccurred = true; + // } + + // const tweet = await getTweetData(input.content.split("/").pop() as string); + + // pageContent = tweetToMd(tweet); + // console.log("THis ishte page content!!", pageContent); + // //@ts-ignore + // vectorData = errorOccurred ? JSON.stringify(pageContent) : thread; + // metadata = { + // baseUrl: input.content, + // description: tweet.text.slice(0, 200), + // image: tweet.user.profile_image_url_https, + // title: `Tweet by ${tweet.user.name}`, + // }; + // } else if (type === "note") { + // pageContent = input.content; + // vectorData = pageContent; + // noteId = new Date().getTime(); + // metadata = { + // baseUrl: `https://supermemory.ai/note/${noteId}`, + // description: `Note created at ${new Date().toLocaleString()}`, + // image: "https://supermemory.ai/logo.png", + // title: `${pageContent.slice(0, 20)} ${pageContent.length > 20 ? "..." : ""}`, + // }; + // } else { + // return { + // success: false, + // data: 0, + // error: "Invalid type", + // }; + // } + + // let storeToSpaces = input.spaces; + + // if (!storeToSpaces) { + // storeToSpaces = []; + // } + + // const vectorSaveResponse = await fetch( + // `${process.env.BACKEND_BASE_URL}/api/add`, + // { + // method: "POST", + // body: JSON.stringify({ + // pageContent: vectorData, + // title: metadata.title, + // description: metadata.description, + // url: metadata.baseUrl, + // spaces: storeToSpaces.map((spaceId) => spaceId.toString()), + // user: data.user.id, + // type, + // }), + // headers: { + // "Content-Type": "application/json", + // Authorization: "Bearer " + process.env.BACKEND_SECURITY_KEY, + // }, + // }, + // ); + + // if (!vectorSaveResponse.ok) { + // const errorData = await vectorSaveResponse.text(); + // console.error(errorData); + // return { + // success: false, + // data: 0, + // error: `Failed to save to vector store. Backend returned error: ${errorData}`, + // }; + // } + + // let contentId: number; + + // const response = (await vectorSaveResponse.json()) as { + // status: string; + // chunkedInput: string; + // message?: string; + // }; + + // try { + // if (response.status !== "ok") { + // if (response.status === "error") { + // return { + // success: false, + // data: 0, + // error: response.message, + // }; + // } else { + // return { + // success: false, + // data: 0, + // error: `Failed to save to vector store. Backend returned error: ${response.message}`, + // }; + // } + // } + // } catch (e) { + // return { + // success: false, + // data: 0, + // error: `Failed to save to vector store. Backend returned error: ${e}`, + // }; + // } + + // const saveToDbUrl = + // (metadata.baseUrl.split("#supermemory-user-")[0] ?? metadata.baseUrl) + + // "#supermemory-user-" + + // data.user.id; + + // // Insert into database + // try { + // const insertResponse = await db + // .insert(storedContent) + // .values({ + // content: pageContent, + // title: metadata.title, + // description: metadata.description, + // url: saveToDbUrl, + // baseUrl: saveToDbUrl, + // image: metadata.image, + // savedAt: new Date(), + // userId: data.user.id, + // type, + // noteId, + // }) + // .returning({ id: storedContent.id }); + // revalidatePath("/memories"); + // revalidatePath("/home"); + + // if (!insertResponse[0]?.id) { + // return { + // success: false, + // data: 0, + // error: "Something went wrong while saving the document to the database", + // }; + // } + + // contentId = insertResponse[0]?.id; + // } catch (e) { + // const error = e as Error; + // console.log("Error: ", error.message); + + // if ( + // error.message.includes( + // "D1_ERROR: UNIQUE constraint failed: storedContent.baseUrl", + // ) + // ) { + // return { + // success: false, + // data: 0, + // error: "Content already exists", + // }; + // } + + // return { + // success: false, + // data: 0, + // error: "Failed to save to database with error: " + error.message, + // }; + // } + + // if (storeToSpaces.length > 0) { + // // Adding the many-to-many relationship between content and spaces + // const spaceData = await db + // .select() + // .from(space) + // .where( + // and(inArray(space.id, storeToSpaces), eq(space.user, data.user.id)), + // ) + // .all(); + + // await Promise.all( + // spaceData.map(async (s) => { + // await db + // .insert(contentToSpace) + // .values({ contentId: contentId, spaceId: s.id }); + + // await db.update(space).set({ numItems: s.numItems + 1 }); + // }), + // ); + // } return { success: true, @@ -475,7 +467,6 @@ export const createChatThread = async ( return { error: "Not authenticated", success: false }; } - const thread = await db .insert(chatThreads) .values({ @@ -836,8 +827,9 @@ export async function getQuerySuggestions() { }; } - const fullQuery = (content?.map((c) => `${c.title} \n\n${c.content}`) ?? []) - .join(" "); + const fullQuery = ( + content?.map((c) => `${c.title} \n\n${c.content}`) ?? [] + ).join(" "); const suggestionsCall = (await env.AI.run( // @ts-ignore diff --git a/apps/web/app/api/store/helper.ts b/apps/web/app/api/store/helper.ts index 2dc42125..6ab7fb23 100644 --- a/apps/web/app/api/store/helper.ts +++ b/apps/web/app/api/store/helper.ts @@ -2,21 +2,21 @@ import { z } from "zod"; import { db } from "@/server/db"; import { contentToSpace, space, storedContent } from "@repo/db/schema"; import { and, eq, inArray } from "drizzle-orm"; -import { LIMITS } from "@/lib/constants"; -import { limit } from "@/app/actions/doers"; +// import { LIMITS } from "@repo/shared-types"; +// import { limit } from "@/app/actions/doers"; import { type AddFromAPIType } from "@repo/shared-types"; export const createMemoryFromAPI = async (input: { data: AddFromAPIType; userId: string; }) => { - if (!(await limit(input.userId, input.data.type))) { - return { - success: false, - data: 0, - error: `You have exceeded the limit of ${LIMITS[input.data.type as keyof typeof LIMITS]} ${input.data.type}s.`, - }; - } + // if (!(await limit(input.userId, input.data.type))) { + // return { + // success: false, + // data: 0, + // error: `You have exceeded the limit of ${LIMITS[input.data.type as keyof typeof LIMITS]} ${input.data.type}s.`, + // }; + // } const vectorSaveResponse = await fetch( `${process.env.BACKEND_BASE_URL}/api/add`, diff --git a/apps/web/lib/constants.ts b/apps/web/lib/constants.ts index 241a6a1d..73f9a83d 100644 --- a/apps/web/lib/constants.ts +++ b/apps/web/lib/constants.ts @@ -1,9 +1,3 @@ -export const LIMITS = { - page: 100, - tweet: 1000, - note: 1000, -}; - export const codeLanguageSubset = [ "python", "javascript", -- cgit v1.2.3 From 4a9565f91d67cbc57be5bb4370c3e640daf643fb Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Mon, 5 Aug 2024 18:36:26 -0700 Subject: changes for staging --- apps/cf-ai-backend/bun.lockb | Bin 0 -> 2390 bytes apps/cf-ai-backend/src/helper.ts | 1 - apps/cf-ai-backend/wrangler.toml | 10 +++++----- apps/web/app/(auth)/onboarding/page.tsx | 14 ++++++++------ apps/web/app/(dash)/dialogContentContainer.tsx | 2 +- apps/web/app/(dash)/menu.tsx | 2 +- apps/web/migrations/0001_Adding_jobs_table.sql | 19 +++++++++++++++++++ apps/web/wrangler.toml | 6 +++--- 8 files changed, 37 insertions(+), 17 deletions(-) create mode 100755 apps/cf-ai-backend/bun.lockb create mode 100644 apps/web/migrations/0001_Adding_jobs_table.sql (limited to 'apps') diff --git a/apps/cf-ai-backend/bun.lockb b/apps/cf-ai-backend/bun.lockb new file mode 100755 index 00000000..601774bc Binary files /dev/null and b/apps/cf-ai-backend/bun.lockb differ diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index eadd9c21..af9e6442 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -184,7 +184,6 @@ export async function batchCreateChunksAndEmbeddings({ const results = []; for (let i = 0; i < newVectors.length; i += 20) { results.push(newVectors.slice(i, i + 20)); - console.log(JSON.stringify(newVectors[1].id)); } await Promise.all( diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index 665ca593..ce1f1948 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -1,4 +1,4 @@ -name = "new-cf-ai-backend" +name = "dev-cf-ai-backend" main = "src/index.ts" compatibility_date = "2024-02-23" node_compat = true @@ -31,8 +31,8 @@ binding = "AI" [[kv_namespaces]] binding = "KV" -id = "569c3f2a510943729019f5d1f798e7d9" -preview_id = "569c3f2a510943729019f5d1f798e7d9" +id = "a68a8182f08e46a1b5d11588fdda68dc" +preview_id = "a68a8182f08e46a1b5d11588fdda68dc" [placement] mode = "smart" @@ -50,5 +50,5 @@ mode = "smart" [[d1_databases]] binding = "DATABASE" -database_name = "supermemlocal" -database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" +database_name = "staging-d1-supermemory" +database_id = "40a3bb06-82bd-42f1-aef4-fde04f3635c5" \ No newline at end of file diff --git a/apps/web/app/(auth)/onboarding/page.tsx b/apps/web/app/(auth)/onboarding/page.tsx index 5b728928..c311ea13 100644 --- a/apps/web/app/(auth)/onboarding/page.tsx +++ b/apps/web/app/(auth)/onboarding/page.tsx @@ -26,11 +26,13 @@ export default function Home() { await completeOnboarding(); }; if (currStep > 3) { - updateDb().then(() => { - push("/home?q=what%20is%20supermemory"); - }).catch((e) => { - console.error(e); - }); + updateDb() + .then(() => { + push("/home?q=what%20is%20supermemory"); + }) + .catch((e) => { + console.error(e); + }); } }, [currStep]); @@ -292,7 +294,7 @@ function StepThree({ currStep }: { currStep: number }) { }); if (cont.success) { - toast.success("Memory created", { + toast.success("Memory queued", { richColors: true, }); } else { diff --git a/apps/web/app/(dash)/dialogContentContainer.tsx b/apps/web/app/(dash)/dialogContentContainer.tsx index c6cacd35..1a11ac6d 100644 --- a/apps/web/app/(dash)/dialogContentContainer.tsx +++ b/apps/web/app/(dash)/dialogContentContainer.tsx @@ -76,7 +76,7 @@ export function DialogContentContainer({ setSelectedSpaces([]); if (cont.success) { - toast.success("Memory created", { + toast.success("Memory queued", { richColors: true, }); } else { diff --git a/apps/web/app/(dash)/menu.tsx b/apps/web/app/(dash)/menu.tsx index 1acab2c8..783a4780 100644 --- a/apps/web/app/(dash)/menu.tsx +++ b/apps/web/app/(dash)/menu.tsx @@ -128,7 +128,7 @@ function Menu() { setSelectedSpaces([]); if (cont.success) { - toast.success("Memory created", { + toast.success("Memory queued", { richColors: true, }); } else { diff --git a/apps/web/migrations/0001_Adding_jobs_table.sql b/apps/web/migrations/0001_Adding_jobs_table.sql new file mode 100644 index 00000000..7a687f72 --- /dev/null +++ b/apps/web/migrations/0001_Adding_jobs_table.sql @@ -0,0 +1,19 @@ +-- Migration number: 0001 2024-08-05T18:05:16.793Z +CREATE TABLE `jobs` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `userId` text NOT NULL, + `url` text NOT NULL, + `status` text NOT NULL, + `attempts` integer DEFAULT 0 NOT NULL, + `lastAttemptAt` integer, + `error` blob, + `createdAt` integer NOT NULL, + `updatedAt` integer NOT NULL, + FOREIGN KEY (`userId`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade +); + + +CREATE INDEX `jobs_userId_idx` ON `jobs` (`userId`);--> statement-breakpoint +CREATE INDEX `jobs_status_idx` ON `jobs` (`status`);--> statement-breakpoint +CREATE INDEX `jobs_createdAt_idx` ON `jobs` (`createdAt`);--> statement-breakpoint +CREATE INDEX `jobs_url_idx` ON `jobs` (`url`);--> statement-breakpoint \ No newline at end of file diff --git a/apps/web/wrangler.toml b/apps/web/wrangler.toml index a6232450..2e991ad9 100644 --- a/apps/web/wrangler.toml +++ b/apps/web/wrangler.toml @@ -1,4 +1,4 @@ -name = "supermemory" +name = "dev-supermemory" compatibility_date = "2024-03-29" compatibility_flags = [ "nodejs_compat" ] pages_build_output_dir = ".vercel/output/static" @@ -31,8 +31,8 @@ database_id = "fc562605-157a-4f60-b439-2a24ffed5b4c" [[env.production.d1_databases]] binding = "DATABASE" -database_name = "prod-d1-supermemory" -database_id = "f527a727-c472-41d4-8eaf-3d7ba0f2f395" +database_name = "staging-d1-supermemory" +database_id = "40a3bb06-82bd-42f1-aef4-fde04f3635c5" [env.preview.ai] binding = "AI" -- cgit v1.2.3 From 1336da8aae05a0acdb3e03561fa7378f238d3eda Mon Sep 17 00:00:00 2001 From: Kush Thaker Date: Tue, 6 Aug 2024 20:48:13 +0530 Subject: Fix job errors not reflecting in D1; add delays on document insert in vectorize to help with open ai rate limits --- apps/cf-ai-backend/src/helper.ts | 60 ++++++++++++++++------ .../src/queueConsumer/helpers/processPage.ts | 2 +- apps/cf-ai-backend/src/queueConsumer/index.ts | 28 +++------- apps/cf-ai-backend/wrangler.toml | 2 +- apps/web/wrangler.toml | 5 +- 5 files changed, 54 insertions(+), 43 deletions(-) (limited to 'apps') diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index eadd9c21..998fd4b5 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -140,6 +140,7 @@ export async function batchCreateChunksAndEmbeddings({ //! If a user saves it through the extension, we don't want other users to be able to see it. // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; const random = seededRandom(ourID); const uuid = @@ -262,21 +263,30 @@ export async function batchCreateChunksAndEmbeddings({ }, {}); const ids = []; - const preparedDocuments = chunks.chunks.map((chunk, i) => { + console.log("Page hit moving on to the for loop"); + for (let i = 0; i < chunks.chunks.length; i++) { + const chunk = chunks.chunks[i]; const id = `${uuid}-${i}`; ids.push(id); - return { + const document = { pageContent: chunk, metadata: { - content: chunk, ...commonMetaData, ...spaceMetadata, }, }; - }); + const docs = await store.addDocuments([document], { ids: [id] }); + console.log("Docs added:", docs); + // Wait for a second after every 20 documents for open ai rate limit + console.log( + "This is the 20th thing in the list?", + (i + 1) % 20 === 0, + ); + if ((i + 1) % 20 === 0) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } - const docs = await store.addDocuments(preparedDocuments, { ids: ids }); - console.log("Docs added:", docs); const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, @@ -299,21 +309,29 @@ export async function batchCreateChunksAndEmbeddings({ }, {}); const ids = []; - const preparedDocuments = chunks.chunks.map((chunk, i) => { + for (let i = 0; i < chunks.chunks.length; i++) { + const chunk = chunks.chunks[i]; const id = `${uuid}-${i}`; ids.push(id); - return { + const document = { pageContent: chunk, metadata: { - content: chunk, ...commonMetaData, ...spaceMetadata, }, }; - }); + const docs = await store.addDocuments([document], { ids: [id] }); + console.log("Docs added:", docs); + // Wait for a second after every 20 documents for open ai rate limit + console.log( + "This is the 20th thing in the list?", + (i + 1) % 20 === 0, + ); + if ((i + 1) % 20 === 0) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } - const docs = await store.addDocuments(preparedDocuments, { ids: ids }); - console.log("Docs added:", docs); const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, @@ -335,20 +353,27 @@ export async function batchCreateChunksAndEmbeddings({ }, {}); const ids = []; - const preparedDocuments = chunks.chunks.map((chunk, i) => { + for (let i = 0; i < chunks.chunks.length; i++) { + const chunk = chunks.chunks[i]; const id = `${uuid}-${i}`; ids.push(id); - return { + const document = { pageContent: chunk, metadata: { ...commonMetaData, ...spaceMetadata, }, }; - }); + const docs = await store.addDocuments([document], { ids: [id] }); + console.log("Docs added:", docs); + // Wait for a second after every 20 documents for open ai rate limit + console.log("This is the 20th thing in the list?", (i + 1) % 20 === 0); + if ((i + 1) % 20 === 0) { + console.log("-----------waiting atm"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } - const docs = await store.addDocuments(preparedDocuments, { ids: ids }); - console.log("Docs added:", docs); const { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID } = env; await bulkInsertKv( { CF_KV_AUTH_TOKEN, CF_ACCOUNT_ID, KV_NAMESPACE_ID }, @@ -356,5 +381,6 @@ export async function batchCreateChunksAndEmbeddings({ ); } } + return; } diff --git a/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts index f967736e..9a50d701 100644 --- a/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts +++ b/apps/cf-ai-backend/src/queueConsumer/helpers/processPage.ts @@ -29,12 +29,12 @@ export async function processPage(input: { ), ); } - console.log("[This is the page content]", pageContent); const metadataResult = await getMetaData(input.url); if (isErr(metadataResult)) { throw metadataResult.error; } const metadata = metadataResult.value; + console.log("[this is the metadata]", metadata); return Ok({ pageContent, metadata }); } catch (e) { console.error("[Page Processing Error]", e); diff --git a/apps/cf-ai-backend/src/queueConsumer/index.ts b/apps/cf-ai-backend/src/queueConsumer/index.ts index 8ca23739..393f1fbf 100644 --- a/apps/cf-ai-backend/src/queueConsumer/index.ts +++ b/apps/cf-ai-backend/src/queueConsumer/index.ts @@ -45,7 +45,7 @@ const calculateExponentialBackoff = ( return baseDelaySeconds ** attempts; }; -const BASE_DELAY_SECONDS = 1.5; +const BASE_DELAY_SECONDS = 5; export async function queue( batch: MessageBatch<{ content: string; @@ -102,6 +102,7 @@ export async function queue( .set({ attempts: existingJob.value[0].attempts + 1, updatedAt: new Date(), + status: "Processing", }) .where(eq(jobs.id, jobId)), d1ErrorFactory, @@ -248,7 +249,7 @@ export async function queue( if (isErr(vectorResult)) { await db .update(jobs) - .set({ error: vectorResult.error }) + .set({ error: vectorResult.error.message, status: "error" }) .where(eq(jobs.id, jobId)); message.retry({ delaySeconds: calculateExponentialBackoff( @@ -288,7 +289,7 @@ export async function queue( if (isErr(insertResponse)) { await db .update(jobs) - .set({ error: insertResponse.error }) + .set({ error: insertResponse.error.message, status: "error" }) .where(eq(jobs.id, jobId)); message.retry({ delaySeconds: calculateExponentialBackoff( @@ -340,7 +341,7 @@ export async function queue( } } catch (e) { console.error("Error in simulated transaction", e.message); - console.log("Rooling back changes"); + message.retry({ delaySeconds: calculateExponentialBackoff( message.attempts, @@ -366,23 +367,6 @@ export async function queue( /* To do: -1. Abstract and shitft the entrie creatememory function to the queue consumer --> Hopefully done -2. Make the front end use that instead of whatever khichidi is going on right now -3. remove getMetada form the lib file as it's not being used anywhere else -4. Figure out the limit stuff ( server action for that seems fine because no use in limiting after they already in the queue rigth? ) -5. Figure out the initQuery stuff ( ;( ) --> This is a bad way of doing stuff :0 -6. How do I hande the content already exists wala use case? --> Also how do I figure out limits? - - - -8. Wrap the d1 thing in a transaction and then write to vectorize if d1 is sucessful if it's not then just error out ( if d1 fails dlq, recoverable failure --> retry ) - -Firt write to d1 in a transaction ( sotredContent + sapces ) --> write to vectorize --> vectorize failes --> reset d1 alternatively first we can also do the vectorise stuff if that suceeds then do the d1 stuff in a batch right? - - -DEBUG: -What's hapenning: -1. The stuff in the d1 is updating but nothing is hapenning in the vectorize for some reason - +Figure out rate limits!! */ diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index 665ca593..b5d67eb6 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -44,7 +44,7 @@ mode = "smart" [[queues.consumers]] queue = "embedchunks-queue" max_batch_size = 100 - max_retries = 10 + max_retries = 3 dead_letter_queue = "embedchunks-dlq" diff --git a/apps/web/wrangler.toml b/apps/web/wrangler.toml index a6232450..54b27325 100644 --- a/apps/web/wrangler.toml +++ b/apps/web/wrangler.toml @@ -26,8 +26,9 @@ bucket_name = "dev-r2-anycontext" [[d1_databases]] binding = "DATABASE" -database_name = "dev-d1-anycontext" -database_id = "fc562605-157a-4f60-b439-2a24ffed5b4c" +database_name = "supermemlocal" +database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" + [[env.production.d1_databases]] binding = "DATABASE" -- cgit v1.2.3 From 34451a7ad79d39da57dabfd36225f34200e41503 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Tue, 6 Aug 2024 10:55:07 -0700 Subject: changes for prod --- apps/cf-ai-backend/wrangler.toml | 10 +++++----- apps/extension/content/content.tsx | 11 ----------- apps/web/wrangler.toml | 11 +++++------ 3 files changed, 10 insertions(+), 22 deletions(-) (limited to 'apps') diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index eda26d71..ca4a5d14 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -1,4 +1,4 @@ -name = "dev-cf-ai-backend" +name = "new-cf-ai-backend" main = "src/index.ts" compatibility_date = "2024-02-23" node_compat = true @@ -31,8 +31,8 @@ binding = "AI" [[kv_namespaces]] binding = "KV" -id = "a68a8182f08e46a1b5d11588fdda68dc" -preview_id = "a68a8182f08e46a1b5d11588fdda68dc" +id = "569c3f2a510943729019f5d1f798e7d9" +preview_id = "569c3f2a510943729019f5d1f798e7d9" [placement] mode = "smart" @@ -50,5 +50,5 @@ mode = "smart" [[d1_databases]] binding = "DATABASE" -database_name = "staging-d1-supermemory" -database_id = "40a3bb06-82bd-42f1-aef4-fde04f3635c5" \ No newline at end of file +database_name = "prod-d1-supermemory" +database_id = "f527a727-c472-41d4-8eaf-3d7ba0f2f395" \ No newline at end of file diff --git a/apps/extension/content/content.tsx b/apps/extension/content/content.tsx index e97c06e6..1a80774c 100644 --- a/apps/extension/content/content.tsx +++ b/apps/extension/content/content.tsx @@ -4,9 +4,6 @@ import("./base.css"); setTimeout(initial, 1000); -const TAILWIND_URL = - "https://cdn.jsdelivr.net/npm/tailwindcss@^2.0/dist/tailwind.min.css"; - const appendTailwindStyleData = (shadowRoot: ShadowRoot) => { const styleSheet = document.createElement("style"); @@ -20,14 +17,6 @@ const appendTailwindStyleData = (shadowRoot: ShadowRoot) => { }); }; -const appendTailwindStyleLink = (shadowRoot: ShadowRoot) => { - // Import Tailwind CSS and inject it into the shadow DOM - const styleSheet = document.createElement("link"); - styleSheet.rel = "stylesheet"; - styleSheet.href = TAILWIND_URL; - shadowRoot.appendChild(styleSheet); -}; - function initial() { // Create a new div element to host the shadow root. // Styles for this div is in `content/content.css` diff --git a/apps/web/wrangler.toml b/apps/web/wrangler.toml index 6abe527d..a6232450 100644 --- a/apps/web/wrangler.toml +++ b/apps/web/wrangler.toml @@ -1,4 +1,4 @@ -name = "dev-supermemory" +name = "supermemory" compatibility_date = "2024-03-29" compatibility_flags = [ "nodejs_compat" ] pages_build_output_dir = ".vercel/output/static" @@ -26,14 +26,13 @@ bucket_name = "dev-r2-anycontext" [[d1_databases]] binding = "DATABASE" -database_name = "supermemlocal" -database_id = "0f93c990-72fb-489c-8563-57a7bb18dc43" - +database_name = "dev-d1-anycontext" +database_id = "fc562605-157a-4f60-b439-2a24ffed5b4c" [[env.production.d1_databases]] binding = "DATABASE" -database_name = "staging-d1-supermemory" -database_id = "40a3bb06-82bd-42f1-aef4-fde04f3635c5" +database_name = "prod-d1-supermemory" +database_id = "f527a727-c472-41d4-8eaf-3d7ba0f2f395" [env.preview.ai] binding = "AI" -- cgit v1.2.3 From 790d43ff822ac595e52972735e4d24214d024bc9 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Tue, 6 Aug 2024 11:10:23 -0700 Subject: change kv namespace ids --- apps/cf-ai-backend/wrangler.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apps') diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index ca4a5d14..6b275129 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -31,8 +31,8 @@ binding = "AI" [[kv_namespaces]] binding = "KV" -id = "569c3f2a510943729019f5d1f798e7d9" -preview_id = "569c3f2a510943729019f5d1f798e7d9" +id = "37a90353da63401e84e20e71165531d0" +preview_id = "c58b6202814f4224acea97627d0c18aa" [placement] mode = "smart" -- cgit v1.2.3 From 2dccd3ce15e3dd63f86c89b91d65bab332e0b0c5 Mon Sep 17 00:00:00 2001 From: Dhravya Shah Date: Tue, 6 Aug 2024 11:12:22 -0700 Subject: updated kv and queues --- apps/cf-ai-backend/wrangler.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'apps') diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index 6b275129..0d7ede62 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -38,14 +38,14 @@ preview_id = "c58b6202814f4224acea97627d0c18aa" mode = "smart" [[queues.producers]] - queue = "embedchunks-queue" + queue = "prod-embedchunks-queue" binding ="EMBEDCHUNKS_QUEUE" [[queues.consumers]] - queue = "embedchunks-queue" + queue = "prod-embedchunks-queue" max_batch_size = 100 max_retries = 3 - dead_letter_queue = "embedchunks-dlq" + dead_letter_queue = "prod-embedchunks-dlq" [[d1_databases]] -- cgit v1.2.3