aboutsummaryrefslogtreecommitdiff
path: root/apps/cf-ai-backend/src/utils
diff options
context:
space:
mode:
Diffstat (limited to 'apps/cf-ai-backend/src/utils')
-rw-r--r--apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts24
-rw-r--r--apps/cf-ai-backend/src/utils/chonker.ts3
-rw-r--r--apps/cf-ai-backend/src/utils/seededRandom.ts7
3 files changed, 28 insertions, 6 deletions
diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts
index 3514f579..be5839b1 100644
--- a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts
+++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts
@@ -1,3 +1,5 @@
+import { z } from "zod";
+
interface OpenAIEmbeddingsParams {
apiKey: string;
modelName: string;
@@ -32,12 +34,22 @@ export class OpenAIEmbeddings {
}),
});
- const data = (await response.json()) as {
- data: {
- embedding: number[];
- }[];
- };
+ const data = await response.json();
+
+ const zodTypeExpected = z.object({
+ data: z.array(
+ z.object({
+ embedding: z.array(z.number()),
+ }),
+ ),
+ });
+
+ const json = zodTypeExpected.safeParse(data);
+
+ if (!json.success) {
+ throw new Error("Invalid response from OpenAI: " + json.error.message);
+ }
- return data.data[0].embedding;
+ return json.data.data[0].embedding;
}
}
diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts
index 39d4b458..c63020be 100644
--- a/apps/cf-ai-backend/src/utils/chonker.ts
+++ b/apps/cf-ai-backend/src/utils/chonker.ts
@@ -1,5 +1,8 @@
import nlp from "compromise";
+/**
+ * Split text into chunks of specified max size with some overlap for continuity.
+ */
export default function chunkText(
text: string,
maxChunkSize: number,
diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts
index 36a1e4f9..9e315ee8 100644
--- a/apps/cf-ai-backend/src/utils/seededRandom.ts
+++ b/apps/cf-ai-backend/src/utils/seededRandom.ts
@@ -1,5 +1,9 @@
import { MersenneTwister19937, integer } from "random-js";
+/**
+ * Hashes a string to a 32-bit integer.
+ * @param {string} seed - The input string to hash.
+ */
function hashString(seed: string) {
let hash = 0;
for (let i = 0; i < seed.length; i++) {
@@ -10,6 +14,9 @@ function hashString(seed: string) {
return hash;
}
+/**
+ * returns a funtion that generates same sequence of random numbers for a given seed between 0 and 1.
+ */
export function seededRandom(seed: string) {
const seedHash = hashString(seed);
const engine = MersenneTwister19937.seed(seedHash);