diff options
Diffstat (limited to 'apps/cf-ai-backend')
| -rw-r--r-- | apps/cf-ai-backend/README.md | 74 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/helper.ts | 105 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/index.test.ts | 13 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/index.ts | 157 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/prompts/prompt1.ts | 8 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/types.ts | 5 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts | 24 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/utils/chonker.ts | 3 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/utils/seededRandom.ts | 7 | ||||
| -rw-r--r-- | apps/cf-ai-backend/tsconfig.json | 3 | ||||
| -rw-r--r-- | apps/cf-ai-backend/wrangler.toml | 2 |
11 files changed, 277 insertions, 124 deletions
diff --git a/apps/cf-ai-backend/README.md b/apps/cf-ai-backend/README.md index 86409f29..91d6b77e 100644 --- a/apps/cf-ai-backend/README.md +++ b/apps/cf-ai-backend/README.md @@ -1,58 +1,50 @@ -# Hono minimal project +baseURL: https://new-cf-ai-backend.dhravya.workers.dev -This is a minimal project with [Hono](https://github.com/honojs/hono/) for Cloudflare Workers. +Authentication: +You must authenticate with a header and `Authorization: bearer token` for each request in `/api/*` routes. -## Features +### Add content: -- Minimal -- TypeScript -- Wrangler to develop and deploy. -- [Jest](https://jestjs.io/ja/) for testing. - -## Usage - -Initialize +POST `/api/add` with ``` -npx create-cloudflare my-app https://github.com/honojs/hono-minimal +body { + pageContent: z.string(), + title: z.string().optional(), + description: z.string().optional(), + space: z.string().optional(), + url: z.string(), + user: z.string(), +} ``` -Install +### Query without user data -``` -yarn install -``` +GET `/api/ask` with +query `?query=testing` -Develop +(this is temp but works perfectly, will change soon for chat use cases specifically) -``` -yarn dev -``` +### Query vectorize and get results in natural language -Test +POST `/api/chat` with ``` -yarn test -``` - -Deploy +query paramters (?query=...&" { + query: z.string(), + topK: z.number().optional().default(10), + user: z.string(), + spaces: z.string().optional(), + sourcesOnly: z.string().optional().default("false"), + model: z.string().optional().default("gpt-4o"), + } +body z.object({ + chatHistory: z.array(contentObj).optional(), +}); ``` -yarn deploy -``` - -## Examples - -See: <https://github.com/honojs/examples> - -## For more information - -See: <https://honojs.dev> - -## Author - -Yusuke Wada <https://github.com/yusukebe> -## License +### Delete vectors -MIT +DELETE `/api/delete` with +query param websiteUrl, user diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts index 87495c59..cef781be 100644 --- a/apps/cf-ai-backend/src/helper.ts +++ b/apps/cf-ai-backend/src/helper.ts @@ -21,8 +21,6 @@ export async function initQuery( index: c.env.VECTORIZE_INDEX, }); - const DEFAULT_MODEL = "gpt-4o"; - let selectedModel: | ReturnType<ReturnType<typeof createOpenAI>> | ReturnType<ReturnType<typeof createGoogleGenerativeAI>> @@ -52,12 +50,6 @@ export async function initQuery( break; } - if (!selectedModel) { - throw new Error( - `Model ${model} not found and default model ${DEFAULT_MODEL} is also not available.`, - ); - } - return { store, model: selectedModel }; } @@ -72,19 +64,46 @@ export async function deleteDocument({ c: Context<{ Bindings: Env }>; store: CloudflareVectorizeStore; }) { - const toBeDeleted = `${url}-${user}`; + const toBeDeleted = `${url}#supermemory-web`; const random = seededRandom(toBeDeleted); const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - await c.env.KV.list({ prefix: uuid }).then(async (keys) => { - for (const key of keys.keys) { - await c.env.KV.delete(key.name); - await store.delete({ ids: [key.name] }); + const allIds = await c.env.KV.list({ prefix: uuid }); + + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await c.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + // We don't actually delete document directly, we just remove the user from the metadata. + // If there's no user left, we can delete the document. + const newVectors = vectors.map((vector) => { + delete vector.metadata[`user-${user}`]; + + // Get count of how many users are left + const userCount = Object.keys(vector.metadata).filter((key) => + key.startsWith("user-"), + ).length; + + // If there's no user left, we can delete the document. + // need to make sure that every chunk is deleted otherwise it would be problematic. + if (userCount === 0) { + store.delete({ ids: savedVectorIds }); + void Promise.all(savedVectorIds.map((id) => c.env.KV.delete(id))); + return null; + } + + return vector; + }); + + // If all vectors are null (deleted), we can delete the KV too. Otherwise, we update (upsert) the vectors. + if (newVectors.every((v) => v === null)) { + await c.env.KV.delete(uuid); + } else { + await c.env.VECTORIZE_INDEX.upsert(newVectors.filter((v) => v !== null)); } - }); + } } export async function batchCreateChunksAndEmbeddings({ @@ -98,19 +117,47 @@ export async function batchCreateChunksAndEmbeddings({ chunks: string[]; context: Context<{ Bindings: Env }>; }) { - const ourID = `${body.url}-${body.user}`; + //! NOTE that we use #supermemory-web to ensure that + //! If a user saves it through the extension, we don't want other users to be able to see it. + // Requests from the extension should ALWAYS have a unique ID with the USERiD in it. + // I cannot stress this enough, important for security. + const ourID = `${body.url}#supermemory-web`; + const random = seededRandom(ourID); + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); - await deleteDocument({ url: body.url, user: body.user, c: context, store }); + const allIds = await context.env.KV.list({ prefix: uuid }); - const random = seededRandom(ourID); + // If some chunks for that content already exist, we'll just update the metadata to include + // the user. + if (allIds.keys.length > 0) { + const savedVectorIds = allIds.keys.map((key) => key.name); + const vectors = await context.env.VECTORIZE_INDEX.getByIds(savedVectorIds); + + // Now, we'll update all vector metadatas with one more userId and all spaceIds + const newVectors = vectors.map((vector) => { + vector.metadata = { + ...vector.metadata, + [`user-${body.user}`]: 1, + + // For each space in body, add the spaceId to the vector metadata + ...(body.spaces ?? [])?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), + }; + + return vector; + }); + + await context.env.VECTORIZE_INDEX.upsert(newVectors); + return; + } for (let i = 0; i < chunks.length; i++) { const chunk = chunks[i]; - const uuid = - random().toString(36).substring(2, 15) + - random().toString(36).substring(2, 15) + - "-" + - i; + const chunkId = `${uuid}-${i}`; const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; @@ -121,19 +168,25 @@ export async function batchCreateChunksAndEmbeddings({ metadata: { title: body.title?.slice(0, 50) ?? "", description: body.description ?? "", - space: body.space ?? "", url: body.url, - user: body.user, + type: body.type ?? "page", + content: newPageContent, + + [`user-${body.user}`]: 1, + ...body.spaces?.reduce((acc, space) => { + acc[`space-${body.user}-${space}`] = 1; + return acc; + }, {}), }, }, ], { - ids: [uuid], + ids: [chunkId], }, ); console.log("Docs added: ", docs); - await context.env.KV.put(uuid, ourID); + await context.env.KV.put(chunkId, ourID); } } diff --git a/apps/cf-ai-backend/src/index.test.ts b/apps/cf-ai-backend/src/index.test.ts deleted file mode 100644 index bbf66fb5..00000000 --- a/apps/cf-ai-backend/src/index.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import app from "."; - -// TODO: write more tests -describe("Test the application", () => { - it("Should return 200 response", async () => { - const res = await app.request("http://localhost/"); - expect(res.status).toBe(200); - }), - it("Should return 404 response", async () => { - const res = await app.request("http://localhost/404"); - expect(res.status).toBe(404); - }); -}); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 19770dec..effdf517 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,6 +1,6 @@ import { z } from "zod"; import { Hono } from "hono"; -import { CoreMessage, streamText } from "ai"; +import { CoreMessage, generateText, streamText } from "ai"; import { chatObj, Env, vectorObj } from "./types"; import { batchCreateChunksAndEmbeddings, @@ -14,9 +14,17 @@ import { bearerAuth } from "hono/bearer-auth"; import { zValidator } from "@hono/zod-validator"; import chunkText from "./utils/chonker"; import { systemPrompt, template } from "./prompts/prompt1"; +import { swaggerUI } from "@hono/swagger-ui"; const app = new Hono<{ Bindings: Env }>(); +app.get( + "/ui", + swaggerUI({ + url: "/doc", + }), +); + // ------- MIDDLEWARES ------- app.use("*", poweredBy()); app.use("*", timing()); @@ -31,6 +39,17 @@ app.use("/api/", async (c, next) => { }); // ------- MIDDLEWARES END ------- +const fileSchema = z + .instanceof(File) + .refine( + (file) => file.size <= 10 * 1024 * 1024, + "File size should be less than 10MB", + ) // Validate file size + .refine( + (file) => ["image/jpeg", "image/png", "image/gif"].includes(file.type), + "Invalid file type", + ); // Validate file type + app.get("/", (c) => { return c.text("Supermemory backend API is running!"); }); @@ -54,6 +73,82 @@ app.post("/api/add", zValidator("json", vectorObj), async (c) => { return c.json({ status: "ok" }); }); +app.post( + "/api/add-with-image", + zValidator( + "form", + z.object({ + images: z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + "images[]": z + .array(fileSchema) + .min(1, "At least one image is required") + .optional(), + text: z.string().optional(), + spaces: z.array(z.string()).optional(), + url: z.string(), + user: z.string(), + }), + (c) => { + console.log(c); + }, + ), + async (c) => { + const body = c.req.valid("form"); + + const { store } = await initQuery(c); + + if (!(body.images || body["images[]"])) { + return c.json({ status: "error", message: "No images found" }, 400); + } + + const imagePromises = (body.images ?? body["images[]"]).map( + async (image) => { + const buffer = await image.arrayBuffer(); + const input = { + image: [...new Uint8Array(buffer)], + prompt: + "What's in this image? caption everything you see in great detail. If it has text, do an OCR and extract all of it.", + max_tokens: 1024, + }; + const response = await c.env.AI.run( + "@cf/llava-hf/llava-1.5-7b-hf", + input, + ); + console.log(response.description); + return response.description; + }, + ); + + const imageDescriptions = await Promise.all(imagePromises); + + await batchCreateChunksAndEmbeddings({ + store, + body: { + url: body.url, + user: body.user, + type: "image", + description: + imageDescriptions.length > 1 + ? `A group of ${imageDescriptions.length} images on ${body.url}` + : imageDescriptions[0], + spaces: body.spaces, + pageContent: imageDescriptions.join("\n"), + title: "Image content from the web", + }, + chunks: [ + imageDescriptions, + ...(body.text ? chunkText(body.text, 1536) : []), + ].flat(), + context: c, + }); + + return c.json({ status: "ok" }); + }, +); + app.get( "/api/ask", zValidator( @@ -85,8 +180,8 @@ app.post( "query", z.object({ query: z.string(), - topK: z.number().optional().default(10), user: z.string(), + topK: z.number().optional().default(10), spaces: z.string().optional(), sourcesOnly: z.string().optional().default("false"), model: z.string().optional().default("gpt-4o"), @@ -97,30 +192,29 @@ app.post( const query = c.req.valid("query"); const body = c.req.valid("json"); - if (body.chatHistory) { - body.chatHistory = body.chatHistory.map((i) => ({ - ...i, - content: i.parts.length > 0 ? i.parts.join(" ") : i.content, - })); - } - const sourcesOnly = query.sourcesOnly === "true"; - const spaces = query.spaces?.split(",") || [undefined]; + const spaces = query.spaces?.split(",") ?? [undefined]; // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); - const filter: VectorizeVectorMetadataFilter = { user: query.user }; + const filter: VectorizeVectorMetadataFilter = { + [`user-${query.user}`]: 1, + }; + console.log("Spaces", spaces); // Converting the query to a vector so that we can search for similar vectors const queryAsVector = await store.embeddings.embedQuery(query.query); const responses: VectorizeMatches = { matches: [], count: 0 }; + console.log("hello world", spaces); + // SLICED to 5 to avoid too many queries for (const space of spaces.slice(0, 5)) { - if (space !== undefined) { + console.log("space", space); + if (!space && spaces.length > 1) { // it's possible for space list to be [undefined] so we only add space filter conditionally - filter.space = space; + filter[`space-${query.user}-${space}`] = 1; } // Because there's no OR operator in the filter, we have to make multiple queries @@ -173,29 +267,20 @@ app.post( dataPoint.id.toString(), ); - // We are getting the content ID back, so that the frontend can show the actual sources properly. - // it IS a lot of DB calls, i completely agree. - // TODO: return metadata value here, so that the frontend doesn't have to re-fetch anything. const storedContent = await Promise.all( idsAsStrings.map(async (id) => await c.env.KV.get(id)), ); - return c.json({ ids: storedContent }); - } - - const vec = responses.matches.map((data) => ({ metadata: data.metadata })); + const metadata = normalizedData.map((datapoint) => datapoint.metadata); - const vecWithScores = vec.map((v, i) => ({ - ...v, - score: sortedHighScoreData[i].score, - normalisedScore: sortedHighScoreData[i].normalizedScore, - })); + return c.json({ ids: storedContent, metadata }); + } - const preparedContext = vecWithScores.map( - ({ metadata, score, normalisedScore }) => ({ + const preparedContext = normalizedData.map( + ({ metadata, score, normalizedScore }) => ({ context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, score, - normalisedScore, + normalizedScore, }), ); @@ -245,4 +330,20 @@ app.delete( }, ); +app.get('/api/editorai', zValidator( + "query", + z.object({ + context: z.string(), + request: z.string(), + }), +), async (c)=> { + const { context, request } = c.req.valid("query"); + + const { model } = await initQuery(c); + + const {text} = await generateText({ model, prompt: `${request}-${context}`, maxTokens: 224 }); + + return c.json({completion: text}); +}) + export default app; diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index aa7694d3..289495b6 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -6,28 +6,24 @@ To generate your answer: - Carefully analyze the question and identify the key information needed to address it - Locate the specific parts of each context that contain this key information - Compare the relevance scores of the provided contexts -- In the <justification> tags, provide a brief justification for which context(s) are more relevant to answering the question based on the scores - Concisely summarize the relevant information from the higher-scoring context(s) in your own words - Provide a direct answer to the question - Use markdown formatting in your answer, including bold, italics, and bullet points as appropriate to improve readability and highlight key points - Give detailed and accurate responses for things like 'write a blog' or long-form questions. - The normalisedScore is a value in which the scores are 'balanced' to give a better representation of the relevance of the context, between 1 and 100, out of the top 10 results - -Provide your justification between <justification> tags and your final answer between <answer> tags, formatting both in markdown. - +- provide your justification in the end, in a <justification> </justification> tag If no context is provided, introduce yourself and explain that the user can save content which will allow you to answer questions about that content in the future. Do not provide an answer if no context is provided.`; export const template = ({ contexts, question }) => { // Map over contexts to generate the context and score parts const contextParts = contexts .map( - ({ context, score, normalisedScore }) => ` + ({ context, normalisedScore }) => ` <context> ${context} </context> <context_score> - score: ${score} normalisedScore: ${normalisedScore} </context_score>`, ) diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index bea4bf80..417d6320 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -2,7 +2,7 @@ import { z } from "zod"; export type Env = { VECTORIZE_INDEX: VectorizeIndex; - AI: Fetcher; + AI: Ai; SECURITY_KEY: string; OPENAI_API_KEY: string; GOOGLE_AI_API_KEY: string; @@ -43,7 +43,8 @@ export const vectorObj = z.object({ pageContent: z.string(), title: z.string().optional(), description: z.string().optional(), - space: z.string().optional(), + spaces: z.array(z.string()).optional(), url: z.string(), user: z.string(), + type: z.string().optional().default("page"), }); diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts index 3514f579..be5839b1 100644 --- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + interface OpenAIEmbeddingsParams { apiKey: string; modelName: string; @@ -32,12 +34,22 @@ export class OpenAIEmbeddings { }), }); - const data = (await response.json()) as { - data: { - embedding: number[]; - }[]; - }; + const data = await response.json(); + + const zodTypeExpected = z.object({ + data: z.array( + z.object({ + embedding: z.array(z.number()), + }), + ), + }); + + const json = zodTypeExpected.safeParse(data); + + if (!json.success) { + throw new Error("Invalid response from OpenAI: " + json.error.message); + } - return data.data[0].embedding; + return json.data.data[0].embedding; } } diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index 39d4b458..c63020be 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -1,5 +1,8 @@ import nlp from "compromise"; +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ export default function chunkText( text: string, maxChunkSize: number, diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 36a1e4f9..9e315ee8 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -1,5 +1,9 @@ import { MersenneTwister19937, integer } from "random-js"; +/** + * Hashes a string to a 32-bit integer. + * @param {string} seed - The input string to hash. + */ function hashString(seed: string) { let hash = 0; for (let i = 0; i < seed.length; i++) { @@ -10,6 +14,9 @@ function hashString(seed: string) { return hash; } +/** + * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. + */ export function seededRandom(seed: string) { const seedHash = hashString(seed); const engine = MersenneTwister19937.seed(seedHash); diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index 2b75d5a0..fcdf6914 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,6 +1,7 @@ { "compilerOptions": { "lib": ["ES2020"], - "types": ["@cloudflare/workers-types"] + "types": ["@cloudflare/workers-types"], + "downlevelIteration": true } } diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index db0ae945..fa883195 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -5,7 +5,7 @@ node_compat = true [[vectorize]] binding = "VECTORIZE_INDEX" -index_name = "supermem-vector" +index_name = "supermem-vector-dev" [ai] binding = "AI" |