diff options
Diffstat (limited to 'apps/cf-ai-backend/src/utils')
| -rw-r--r-- | apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts | 24 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/utils/chonker.ts | 3 | ||||
| -rw-r--r-- | apps/cf-ai-backend/src/utils/seededRandom.ts | 7 |
3 files changed, 28 insertions, 6 deletions
diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts index 3514f579..be5839b1 100644 --- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -1,3 +1,5 @@ +import { z } from "zod"; + interface OpenAIEmbeddingsParams { apiKey: string; modelName: string; @@ -32,12 +34,22 @@ export class OpenAIEmbeddings { }), }); - const data = (await response.json()) as { - data: { - embedding: number[]; - }[]; - }; + const data = await response.json(); + + const zodTypeExpected = z.object({ + data: z.array( + z.object({ + embedding: z.array(z.number()), + }), + ), + }); + + const json = zodTypeExpected.safeParse(data); + + if (!json.success) { + throw new Error("Invalid response from OpenAI: " + json.error.message); + } - return data.data[0].embedding; + return json.data.data[0].embedding; } } diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts index 39d4b458..c63020be 100644 --- a/apps/cf-ai-backend/src/utils/chonker.ts +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -1,5 +1,8 @@ import nlp from "compromise"; +/** + * Split text into chunks of specified max size with some overlap for continuity. + */ export default function chunkText( text: string, maxChunkSize: number, diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts index 36a1e4f9..9e315ee8 100644 --- a/apps/cf-ai-backend/src/utils/seededRandom.ts +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -1,5 +1,9 @@ import { MersenneTwister19937, integer } from "random-js"; +/** + * Hashes a string to a 32-bit integer. + * @param {string} seed - The input string to hash. + */ function hashString(seed: string) { let hash = 0; for (let i = 0; i < seed.length; i++) { @@ -10,6 +14,9 @@ function hashString(seed: string) { return hash; } +/** + * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1. + */ export function seededRandom(seed: string) { const seedHash = hashString(seed); const engine = MersenneTwister19937.seed(seedHash); |