fix: deduplicate memories after returned to save tokens

5 files changed, 201 insertions, 38 deletions

diff --git a/packages/tools/package.json b/packages/tools/package.json
index 104e4cc2..3400576a 100644
--- a/packages/tools/package.json
+++ b/packages/tools/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@supermemory/tools",
   "type": "module",
-  "version": "1.3.52",
+  "version": "1.3.60",
   "description": "Memory tools for AI SDK and OpenAI function calling with supermemory",
   "scripts": {
     "build": "tsdown",
diff --git a/packages/tools/src/openai/middleware.ts b/packages/tools/src/openai/middleware.ts
index 29b66f70..018d1f4c 100644
--- a/packages/tools/src/openai/middleware.ts
+++ b/packages/tools/src/openai/middleware.ts
@@ -1,6 +1,7 @@
 import type OpenAI from "openai"
 import Supermemory from "supermemory"
 import { addConversation } from "../conversations-client"
+import { deduplicateMemories } from "../shared"
 import { createLogger, type Logger } from "../vercel/logger"
 import { convertProfileToMarkdown } from "../vercel/util"
 
@@ -180,26 +181,41 @@ const addSystemPrompt = async (
 		mode,
 	})
 
+	const deduplicated = deduplicateMemories({
+		static: memoriesResponse.profile.static,
+		dynamic: memoriesResponse.profile.dynamic,
+		searchResults: memoriesResponse.searchResults.results,
+	})
+
+	logger.debug("Memory deduplication completed for chat API", {
+		static: {
+			original: memoryCountStatic,
+			deduplicated: deduplicated.static.length,
+		},
+		dynamic: {
+			original: memoryCountDynamic,
+			deduplicated: deduplicated.dynamic.length,
+		},
+		searchResults: {
+			original: memoriesResponse.searchResults.results.length,
+			deduplicated: deduplicated.searchResults.length,
+		},
+	})
+
 	const profileData =
 		mode !== "query"
 			? convertProfileToMarkdown({
 					profile: {
-						static: memoriesResponse.profile.static?.map((item) => item.memory),
-						dynamic: memoriesResponse.profile.dynamic?.map(
-							(item) => item.memory,
-						),
-					},
-					searchResults: {
-						results: memoriesResponse.searchResults.results.map((item) => ({
-							memory: item.memory,
-						})) as [{ memory: string }],
+						static: deduplicated.static,
+						dynamic: deduplicated.dynamic,
 					},
+					searchResults: { results: [] },
 				})
 			: ""
 	const searchResultsMemories =
 		mode !== "profile"
-			? `Search results for user's recent message: \n${memoriesResponse.searchResults.results
-					.map((result) => `- ${result.memory}`)
+			? `Search results for user's recent message: \n${deduplicated.searchResults
+					.map((memory) => `- ${memory}`)
 					.join("\n")}`
 			: ""
 
@@ -448,28 +464,41 @@ export function createOpenAIMiddleware(
 			mode,
 		})
 
+		const deduplicated = deduplicateMemories({
+			static: memoriesResponse.profile.static,
+			dynamic: memoriesResponse.profile.dynamic,
+			searchResults: memoriesResponse.searchResults.results,
+		})
+
+		logger.debug(`Memory deduplication completed for ${context} API`, {
+			static: {
+				original: memoryCountStatic,
+				deduplicated: deduplicated.static.length,
+			},
+			dynamic: {
+				original: memoryCountDynamic,
+				deduplicated: deduplicated.dynamic.length,
+			},
+			searchResults: {
+				original: memoriesResponse.searchResults.results.length,
+				deduplicated: deduplicated.searchResults.length,
+			},
+		})
+
 		const profileData =
 			mode !== "query"
 				? convertProfileToMarkdown({
 						profile: {
-							static: memoriesResponse.profile.static?.map(
-								(item) => item.memory,
-							),
-							dynamic: memoriesResponse.profile.dynamic?.map(
-								(item) => item.memory,
-							),
-						},
-						searchResults: {
-							results: memoriesResponse.searchResults.results.map((item) => ({
-								memory: item.memory,
-							})) as [{ memory: string }],
+							static: deduplicated.static,
+							dynamic: deduplicated.dynamic,
 						},
+						searchResults: { results: [] },
 					})
 				: ""
 		const searchResultsMemories =
 			mode !== "profile"
-				? `Search results for user's ${context === "chat" ? "recent message" : "input"}: \n${memoriesResponse.searchResults.results
-						.map((result) => `- ${result.memory}`)
+				? `Search results for user's ${context === "chat" ? "recent message" : "input"}: \n${deduplicated.searchResults
+						.map((memory) => `- ${memory}`)
 						.join("\n")}`
 				: ""
 
diff --git a/packages/tools/src/shared.ts b/packages/tools/src/shared.ts
index 0ff14e86..91fc8917 100644
--- a/packages/tools/src/shared.ts
+++ b/packages/tools/src/shared.ts
@@ -45,3 +45,102 @@ export function getContainerTags(config?: {
 	}
 	return config?.containerTags ?? CONTAINER_TAG_CONSTANTS.defaultTags
 }
+
+/**
+ * Memory item interface representing a single memory with optional metadata
+ */
+export interface MemoryItem {
+	memory: string
+	metadata?: Record<string, unknown>
+}
+
+/**
+ * Profile data structure containing memory items from different sources
+ */
+export interface ProfileWithMemories {
+	static?: Array<MemoryItem>
+	dynamic?: Array<MemoryItem>
+	searchResults?: Array<MemoryItem>
+}
+
+/**
+ * Deduplicated memory strings organized by source
+ */
+export interface DeduplicatedMemories {
+	static: string[]
+	dynamic: string[]
+	searchResults: string[]
+}
+
+/**
+ * Deduplicates memory items across static, dynamic, and search result sources.
+ * Priority: Static > Dynamic > Search Results
+ *
+ * @param data - Profile data with memory items from different sources
+ * @returns Deduplicated memory strings for each source
+ *
+ * @example
+ * ```typescript
+ * const deduplicated = deduplicateMemories({
+ *   static: [{ memory: "User likes TypeScript" }],
+ *   dynamic: [{ memory: "User likes TypeScript" }, { memory: "User works remotely" }],
+ *   searchResults: [{ memory: "User prefers async/await" }]
+ * });
+ * // Returns:
+ * // {
+ * //   static: ["User likes TypeScript"],
+ * //   dynamic: ["User works remotely"],
+ * //   searchResults: ["User prefers async/await"]
+ * // }
+ * ```
+ */
+export function deduplicateMemories(
+	data: ProfileWithMemories,
+): DeduplicatedMemories {
+	const staticItems = data.static ?? []
+	const dynamicItems = data.dynamic ?? []
+	const searchItems = data.searchResults ?? []
+
+	const getMemoryString = (item: MemoryItem): string | null => {
+		if (!item || typeof item.memory !== "string") return null
+		const trimmed = item.memory.trim()
+		return trimmed.length > 0 ? trimmed : null
+	}
+
+	const staticMemories: string[] = []
+	const seenMemories = new Set<string>()
+
+	for (const item of staticItems) {
+		const memory = getMemoryString(item)
+		if (memory !== null) {
+			staticMemories.push(memory)
+			seenMemories.add(memory)
+		}
+	}
+
+	const dynamicMemories: string[] = []
+
+	for (const item of dynamicItems) {
+		const memory = getMemoryString(item)
+		if (memory !== null && !seenMemories.has(memory)) {
+			dynamicMemories.push(memory)
+			seenMemories.add(memory)
+		}
+	}
+
+	const searchMemories: string[] = []
+
+	for (const item of searchItems) {
+		const memory = getMemoryString(item)
+		if (memory !== null && !seenMemories.has(memory)) {
+			searchMemories.push(memory)
+			seenMemories.add(memory)
+		}
+	}
+
+	return {
+		static: staticMemories,
+		dynamic: dynamicMemories,
+		searchResults: searchMemories,
+	}
+}
diff --git a/packages/tools/src/vercel/memory-prompt.ts b/packages/tools/src/vercel/memory-prompt.ts
index d6dff44a..480d7b90 100644
--- a/packages/tools/src/vercel/memory-prompt.ts
+++ b/packages/tools/src/vercel/memory-prompt.ts
@@ -1,4 +1,5 @@
 import type { LanguageModelV2CallOptions } from "@ai-sdk/provider"
+import { deduplicateMemories } from "../shared"
 import type { Logger } from "./logger"
 import { convertProfileToMarkdown, type ProfileStructure } from "./util"
 
@@ -88,12 +89,41 @@ export const addSystemPrompt = async (
 		mode,
 	})
 
+	const deduplicated = deduplicateMemories({
+		static: memoriesResponse.profile.static,
+		dynamic: memoriesResponse.profile.dynamic,
+		searchResults: memoriesResponse.searchResults.results,
+	})
+
+	logger.debug("Memory deduplication completed", {
+		static: {
+			original: memoryCountStatic,
+			deduplicated: deduplicated.static.length,
+		},
+		dynamic: {
+			original: memoryCountDynamic,
+			deduplicated: deduplicated.dynamic.length,
+		},
+		searchResults: {
+			original: memoriesResponse.searchResults.results.length,
+			deduplicated: deduplicated.searchResults.length,
+		},
+	})
+
 	const profileData =
-		mode !== "query" ? convertProfileToMarkdown(memoriesResponse) : ""
+		mode !== "query"
+			? convertProfileToMarkdown({
+					profile: {
+						static: deduplicated.static,
+						dynamic: deduplicated.dynamic,
+					},
+					searchResults: { results: [] },
+				})
+			: ""
 	const searchResultsMemories =
 		mode !== "profile"
-			? `Search results for user's recent message: \n${memoriesResponse.searchResults.results
-					.map((result) => `- ${result.memory}`)
+			? `Search results for user's recent message: \n${deduplicated.searchResults
+					.map((memory) => `- ${memory}`)
 					.join("\n")}`
 			: ""
 
diff --git a/packages/tools/src/vercel/util.ts b/packages/tools/src/vercel/util.ts
index 1c06ec16..ac36280d 100644
--- a/packages/tools/src/vercel/util.ts
+++ b/packages/tools/src/vercel/util.ts
@@ -2,15 +2,21 @@ import type { LanguageModelV2CallOptions, LanguageModelV2Message } from "@ai-sdk
 
 export interface ProfileStructure {
 	profile: {
+		static?: Array<{ memory: string; metadata?: Record<string, unknown> }>
+		dynamic?: Array<{ memory: string; metadata?: Record<string, unknown> }>
+	}
+	searchResults: {
+		results: Array<{ memory: string; metadata?: Record<string, unknown> }>
+	}
+}
+
+export interface ProfileMarkdownData {
+	profile: {
 		static?: string[]
 		dynamic?: string[]
 	}
 	searchResults: {
-		results: [
-			{
-				memory: string
-			},
-		]
+		results: Array<{ memory: string }>
 	}
 }
 
@@ -32,12 +38,11 @@ export type OutputContentItem =
 	  }
 
 /**
- * Convert ProfileStructure to markdown
- * based on profile.static and profile.dynamic properties
- * @param data ProfileStructure
- * @returns Markdown string
+ * Convert profile data to markdown format
+ * @param data Profile data with string arrays for static and dynamic memories
+ * @returns Markdown string with profile sections
  */
-export function convertProfileToMarkdown(data: ProfileStructure): string {
+export function convertProfileToMarkdown(data: ProfileMarkdownData): string {
 	const sections: string[] = []
 
 	if (data.profile.static && data.profile.static.length > 0) {