Merge branch 'main' of https://github.com/Dhravya/supermemory

8 files changed, 444 insertions, 11 deletions

diff --git a/apps/cf-ai-backend/src/env.d.ts b/apps/cf-ai-backend/src/env.d.ts
index acbd6c43..65def4fd 100644
--- a/apps/cf-ai-backend/src/env.d.ts
+++ b/apps/cf-ai-backend/src/env.d.ts
@@ -4,4 +4,13 @@ interface Env {
 	SECURITY_KEY: string;
 	OPENAI_API_KEY: string;
 	GOOGLE_AI_API_KEY: string;
+	MY_QUEUE: Queue<TweetData>;
+}
+
+interface TweetData {
+	tweetText: string;
+	postUrl: string;
+	authorName: string;
+	handle: string;
+	time: string;
 }
diff --git a/apps/cf-ai-backend/src/routes/chat.ts b/apps/cf-ai-backend/src/routes/chat.ts
index 75e298b8..980e46a3 100644
--- a/apps/cf-ai-backend/src/routes/chat.ts
+++ b/apps/cf-ai-backend/src/routes/chat.ts
@@ -60,8 +60,9 @@ export async function POST(request: Request, _: CloudflareVectorizeStore, embedd
 	// if (responses.count === 0) {
 	// 	return new Response(JSON.stringify({ message: "No Results Found" }), { status: 404 });
 	// }
+	console.log(responses.matches);
 
-	const highScoreIds = responses.matches.filter(({ score }) => score > 0.35).map(({ id }) => id);
+	const highScoreIds = responses.matches.filter(({ score }) => score > 0.3).map(({ id }) => id);
 
 	if (sourcesOnly === 'true') {
 		return new Response(JSON.stringify({ ids: highScoreIds }), { status: 200 });
diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml
index 9e3ee21b..9a9effb6 100644
--- a/apps/cf-ai-backend/wrangler.toml
+++ b/apps/cf-ai-backend/wrangler.toml
@@ -9,6 +9,10 @@ index_name = "any-vector"
 [ai]
 binding = "AI"
 
+[[queues.producers]]
+ queue = "batch-vector-queue"
+ binding = "MY_QUEUE"
+
 # Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
 # Note: Use secrets to store sensitive data.
 # Docs: https://developers.cloudflare.com/workers/platform/environment-variables
diff --git a/apps/extension/src/App.tsx b/apps/extension/src/App.tsx
index f563664f..89227432 100644
--- a/apps/extension/src/App.tsx
+++ b/apps/extension/src/App.tsx
@@ -13,7 +13,7 @@ function App() {
     null,
   );
 
-  const doStuff = () => {
+  const getUserData = () => {
     chrome.runtime.sendMessage({ type: "getJwt" }, (response) => {
       const jwt = response.jwt;
       const loginButton = document.getElementById("login");
@@ -41,9 +41,69 @@ function App() {
   };
 
   useEffect(() => {
-    doStuff();
+    getUserData();
   }, []);
 
+  // TODO: Implement getting bookmarks from API directly
+  // const [status, setStatus] = useState('');
+  // const [bookmarks, setBookmarks] = useState<TweetData[]>([]);
+
+  // const fetchBookmarks = (e: React.MouseEvent<HTMLButtonElement>) => {
+  //   e.preventDefault();
+
+  //   chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
+  //     chrome.tabs.sendMessage(tabs[0].id!, { action: 'showProgressIndicator' });
+  //   });
+
+  //   chrome.tabs.create(
+  //     { url: 'https://twitter.com/i/bookmarks/all' },
+  //     function (tab) {
+  //       chrome.tabs.onUpdated.addListener(function listener(tabId, info) {
+  //         if (tabId === tab.id && info.status === 'complete') {
+  //           chrome.tabs.onUpdated.removeListener(listener);
+
+  //           chrome.runtime.sendMessage(
+  //             { action: 'getAuthData' },
+  //             function (response) {
+  //               const authorizationHeader = response.authorizationHeader;
+  //               const csrfToken = response.csrfToken;
+  //               const cookies = response.cookies;
+
+  //               if (authorizationHeader && csrfToken && cookies) {
+  //                 fetchAllBookmarks(authorizationHeader, csrfToken, cookies)
+  //                   .then((bookmarks) => {
+  //                     console.log('Bookmarks data:', bookmarks);
+  //                     setBookmarks(bookmarks);
+  //                     chrome.tabs.sendMessage(tabId, {
+  //                       action: 'hideProgressIndicator',
+  //                     });
+  //                     setStatus(
+  //                       `Fetched ${bookmarks.length} bookmarked tweets.`,
+  //                     );
+  //                   })
+  //                   .catch((error) => {
+  //                     console.error('Error:', error);
+  //                     chrome.tabs.sendMessage(tabId, {
+  //                       action: 'hideProgressIndicator',
+  //                     });
+  //                     setStatus(
+  //                       'Error fetching bookmarks. Please check the console for details.',
+  //                     );
+  //                   });
+  //               } else {
+  //                 chrome.tabs.sendMessage(tabId, {
+  //                   action: 'hideProgressIndicator',
+  //                 });
+  //                 setStatus('Missing authentication data');
+  //               }
+  //             },
+  //           );
+  //         }
+  //       });
+  //     },
+  //   );
+  // };
+
   return (
     <div className="p-8">
       <button
@@ -69,6 +129,19 @@ function App() {
               <h3>{userData.data.user.name}</h3>
               <p>{userData.data.user.email}</p>
             </div>
+            {/* TODO: Implement getting bookmarks from API directly */}
+            {/* <button onClick={(e) => fetchBookmarks(e)}>Fetch Bookmarks</button>
+            <div>{status}</div>
+
+            <div>
+              {bookmarks.map((bookmark) => (
+                <div key={bookmark.tweet_id}>
+                  <p>{bookmark.author}</p>
+                  <p>{bookmark.date}</p>
+                  <p>{bookmark.full_text}</p>
+                </div>
+              ))}
+            </div> */}
           </div>
         )}
       </div>
@@ -76,4 +149,160 @@ function App() {
   );
 }
 
+// TODO: Implement getting bookmarks from API directly
+// async function fetchAllBookmarks(
+//   authorizationHeader: string,
+//   csrfToken: string,
+//   cookies: string,
+// ): Promise<TweetData[]> {
+//   const baseUrl =
+//     'https://twitter.com/i/api/graphql/uJEL6XARgGmo2EAsO2Pfkg/Bookmarks';
+//   const params = new URLSearchParams({
+//     variables: JSON.stringify({
+//       count: 100,
+//       includePromotedContent: true,
+//     }),
+//     features: JSON.stringify({
+//       graphql_timeline_v2_bookmark_timeline: true,
+//       rweb_tipjar_consumption_enabled: false,
+//       responsive_web_graphql_exclude_directive_enabled: true,
+//       verified_phone_label_enabled: true,
+//       creator_subscriptions_tweet_preview_api_enabled: true,
+//       responsive_web_graphql_timeline_navigation_enabled: true,
+//       responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+//       communities_web_enable_tweet_community_results_fetch: true,
+//       c9s_tweet_anatomy_moderator_badge_enabled: true,
+//       tweetypie_unmention_optimization_enabled: true,
+//       responsive_web_edit_tweet_api_enabled: true,
+//       graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+//       view_counts_everywhere_api_enabled: true,
+//       longform_notetweets_consumption_enabled: true,
+//       responsive_web_twitter_article_tweet_consumption_enabled: true,
+//       tweet_awards_web_tipping_enabled: false,
+//       creator_subscriptions_quote_tweet_preview_enabled: false,
+//       freedom_of_speech_not_reach_fetch_enabled: true,
+//       standardized_nudges_misinfo: true,
+//       tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:
+//         true,
+//       tweet_with_visibility_results_prefer_gql_media_interstitial_enabled:
+//         false,
+//       rweb_video_timestamps_enabled: true,
+//       longform_notetweets_rich_text_read_enabled: true,
+//       longform_notetweets_inline_media_enabled: true,
+//       responsive_web_enhance_cards_enabled: false,
+//     }),
+//   });
+
+//   const requestUrl = `${baseUrl}?${params}`;
+
+//   const headers = {
+//     Authorization: authorizationHeader,
+//     'X-Csrf-Token': csrfToken,
+//     Cookie: cookies,
+//   };
+
+//   const bookmarks: TweetData[] = [];
+//   let nextCursor = null;
+//   let requestCount = 0;
+//   const maxRequestsPerWindow = 450;
+//   const windowDuration = 15 * 60 * 1000; // 15 minutes in milliseconds
+//   let windowStartTime = Date.now();
+
+//   do {
+//     if (nextCursor) {
+//       params.set(
+//         'variables',
+//         JSON.stringify({
+//           count: 100,
+//           cursor: nextCursor,
+//           includePromotedContent: true,
+//         }),
+//       );
+//     }
+
+//     // Check if the rate limit is exceeded
+//     if (requestCount >= maxRequestsPerWindow) {
+//       const elapsedTime = Date.now() - windowStartTime;
+//       if (elapsedTime < windowDuration) {
+//         const waitTime = windowDuration - elapsedTime;
+//         await new Promise((resolve) => setTimeout(resolve, waitTime));
+//       }
+//       requestCount = 0;
+//       windowStartTime = Date.now();
+//     }
+
+//     try {
+//       const response = await fetch(requestUrl, {
+//         method: 'GET',
+//         headers: headers,
+//       });
+
+//       requestCount++;
+
+//       if (!response.ok) {
+//         throw new Error(`HTTP error! status: ${response.status}`);
+//       }
+
+//       const data = await response.json();
+//       const timeline = data.data.bookmark_timeline_v2.timeline;
+
+//       timeline.instructions.forEach(
+//         (instruction: {
+//           type: string;
+//           entries: {
+//             content: {
+//               entryType: string;
+//               itemContent: {
+//                 tweet_results: {
+//                   result: {
+//                     legacy: {
+//                       full_text: string;
+//                       created_at: string;
+//                     };
+//                     core: {
+//                       user_results: {
+//                         result: {
+//                           legacy: {
+//                             screen_name: string;
+//                           };
+//                         };
+//                       };
+//                     };
+//                     rest_id: string;
+//                   };
+//                 };
+//               };
+//             };
+//           }[];
+//         }) => {
+//           if (instruction.type === 'TimelineAddEntries') {
+//             instruction.entries.forEach((entry) => {
+//               if (entry.content.entryType === 'TimelineTimelineItem') {
+//                 const tweet = entry.content.itemContent.tweet_results.result;
+//                 const tweetData = {
+//                   full_text: tweet.legacy.full_text,
+//                   url: `https://twitter.com/${tweet.core.user_results.result.legacy.screen_name}/status/${tweet.rest_id}`,
+//                   author: tweet.core.user_results.result.legacy.screen_name,
+//                   date: tweet.legacy.created_at,
+//                   tweet_id: tweet.rest_id,
+//                 };
+//                 bookmarks.push(tweetData);
+//               }
+//             });
+//           }
+//         },
+//       );
+
+//       nextCursor = timeline.instructions.find(
+//         (instruction: { type: string }) =>
+//           instruction.type === 'TimelineTerminateTimeline',
+//       )?.direction?.cursor;
+//     } catch (error) {
+//       console.error('Error fetching bookmarks:', error);
+//       throw error;
+//     }
+//   } while (nextCursor);
+
+//   return bookmarks;
+// }
 export default App;
diff --git a/apps/extension/src/SideBar.tsx b/apps/extension/src/SideBar.tsx
index 07d9b9f5..b3bfb08a 100644
--- a/apps/extension/src/SideBar.tsx
+++ b/apps/extension/src/SideBar.tsx
@@ -25,20 +25,176 @@ function sendUrlToAPI() {
 }
 
 function SideBar() {
+  // TODO: Implement getting bookmarks from API directly
+  // chrome.runtime.onMessage.addListener(function (request) {
+  //   if (request.action === 'showProgressIndicator') {
+  //     // TODO: SHOW PROGRESS INDICATOR
+  //     // showProgressIndicator();
+  //   } else if (request.action === 'hideProgressIndicator') {
+  //     // hideProgressIndicator();
+  //   }
+  // });
+
   const [savedWebsites, setSavedWebsites] = useState<string[]>([]);
 
   const [isSendingData, setIsSendingData] = useState(false);
 
+  interface TweetData {
+    tweetText: string;
+    postUrl: string;
+    authorName: string;
+    handle: string;
+    time: string;
+  }
+
+  const fetchBookmarks = () => {
+    const tweets: TweetData[] = []; // Initialize an empty array to hold all tweet elements
+
+    const scrollInterval = 1000;
+    const scrollStep = 5000; // Pixels to scroll on each step
+
+    let previousTweetCount = 0;
+    let unchangedCount = 0;
+
+    const scrollToEndIntervalID = setInterval(() => {
+      window.scrollBy(0, scrollStep);
+      const currentTweetCount = tweets.length;
+      if (currentTweetCount === previousTweetCount) {
+        unchangedCount++;
+        if (unchangedCount >= 2) {
+          // Stop if the count has not changed 5 times
+          console.log("Scraping complete");
+          console.log("Total tweets scraped: ", tweets.length);
+          console.log("Downloading tweets as JSON...");
+          clearInterval(scrollToEndIntervalID); // Stop scrolling
+          observer.disconnect(); // Stop observing DOM changes
+          downloadTweetsAsJson(tweets); // Download the tweets list as a JSON file
+        }
+      } else {
+        unchangedCount = 0; // Reset counter if new tweets were added
+      }
+      previousTweetCount = currentTweetCount; // Update previous count for the next check
+    }, scrollInterval);
+
+    function updateTweets() {
+      document
+        .querySelectorAll('article[data-testid="tweet"]')
+        .forEach((tweetElement) => {
+          const authorName = (
+            tweetElement.querySelector(
+              '[data-testid="User-Name"]',
+            ) as HTMLElement
+          )?.innerText;
+
+          const handle = (
+            tweetElement.querySelector('[role="link"]') as HTMLLinkElement
+          ).href
+            .split("/")
+            .pop();
+
+          const tweetText = (
+            tweetElement.querySelector(
+              '[data-testid="tweetText"]',
+            ) as HTMLElement
+          )?.innerText;
+          const time = (
+            tweetElement.querySelector("time") as HTMLTimeElement
+          ).getAttribute("datetime");
+          const postUrl = (
+            tweetElement.querySelector(
+              ".css-175oi2r.r-18u37iz.r-1q142lx a",
+            ) as HTMLLinkElement
+          )?.href;
+
+          const isTweetNew = !tweets.some((tweet) => tweet.postUrl === postUrl);
+          if (isTweetNew) {
+            tweets.push({
+              authorName,
+              handle: handle ?? "",
+              tweetText,
+              time: time ?? "",
+              postUrl,
+            });
+            console.log("Tweets capturados: ", tweets.length);
+          }
+        });
+    }
+
+    // Initially populate the tweets array
+    updateTweets();
+
+    // Create a MutationObserver to observe changes in the DOM
+    const observer = new MutationObserver((mutations) => {
+      mutations.forEach((mutation) => {
+        if (mutation.addedNodes.length) {
+          updateTweets(); // Call updateTweets whenever new nodes are added to the DOM
+        }
+      });
+    });
+
+    // Start observing the document body for child list changes
+    observer.observe(document.body, { childList: true, subtree: true });
+
+    function downloadTweetsAsJson(tweetsArray: TweetData[]) {
+      const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON
+
+      // TODO: SEND jsonData to server
+      console.log(jsonData);
+    }
+  };
+
   return (
     <>
       <TooltipProvider>
-        <div className="anycontext-flex anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
-          {/* <Tooltip delayDuration={300}>
-            <TooltipContent side="left">
-              <p>Open Sidebar</p>
-            </TooltipContent>
-          </Tooltip> */}
+        <div className="anycontext-flex anycontext-group anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
+          {window.location.href.includes("twitter.com") ||
+          window.location.href.includes("x.com") ? (
+            <Tooltip delayDuration={300}>
+              <TooltipTrigger
+                className="anycontext-bg-transparent
+                        anycontext-border-none anycontext-m-0 anycontext-p-0"
+              >
+                <button
+                  onClick={() => {
+                    if (window.location.href.endsWith("/i/bookmarks/all")) {
+                      fetchBookmarks();
+                    } else {
+                      window.location.href =
+                        "https://twitter.com/i/bookmarks/all";
 
+                      setTimeout(() => {
+                        fetchBookmarks();
+                      }, 2500);
+                    }
+                  }}
+                  className="anycontext-open-button disabled:anycontext-opacity-30 anycontext-bg-transparent
+                          anycontext-border-none anycontext-m-0 anycontext-p-0"
+                >
+                  <svg
+                    xmlns="http://www.w3.org/2000/svg"
+                    fill="none"
+                    viewBox="0 0 24 24"
+                    strokeWidth={1.5}
+                    stroke="currentColor"
+                    className="anycontext-w-6 anycontext-h-6"
+                  >
+                    <path
+                      strokeLinecap="round"
+                      strokeLinejoin="round"
+                      d="M17.593 3.322c1.1.128 1.907 1.077 1.907 2.185V21L12 17.25 4.5 21V5.507c0-1.108.806-2.057 1.907-2.185a48.507 48.507 0 0 1 11.186 0Z"
+                    />
+                  </svg>
+                </button>
+              </TooltipTrigger>
+              <TooltipContent className="anycontext-p-0" side="left">
+                <p className="anycontext-p-0 anycontext-m-0">
+                  Import twitter bookmarks
+                </p>
+              </TooltipContent>
+            </Tooltip>
+          ) : (
+            <></>
+          )}
           <Tooltip delayDuration={300}>
             <TooltipTrigger
               className="anycontext-bg-transparent
diff --git a/apps/extension/src/background.ts b/apps/extension/src/background.ts
index 2cf29f40..b3030e74 100644
--- a/apps/extension/src/background.ts
+++ b/apps/extension/src/background.ts
@@ -5,6 +5,30 @@ const backendUrl =
     ? "http://localhost:3000"
     : "https://supermemory.dhr.wtf";
 
+// TODO: Implement getting bookmarks from API directly
+// let authorizationHeader: string | null = null;
+// let csrfToken: string | null = null;
+// let cookies: string | null = null;
+
+// chrome.webRequest.onBeforeSendHeaders.addListener(
+//   (details) => {
+//     for (let i = 0; i < details.requestHeaders!.length; ++i) {
+//       const header = details.requestHeaders![i];
+//       if (header.name.toLowerCase() === 'authorization') {
+//         authorizationHeader = header.value || null;
+//       } else if (header.name.toLowerCase() === 'x-csrf-token') {
+//         csrfToken = header.value || null;
+//       } else if (header.name.toLowerCase() === 'cookie') {
+//         cookies = header.value || null;
+//       }
+
+//       console.log(header, authorizationHeader, csrfToken, cookies)
+//     }
+//   },
+//   { urls: ['https://twitter.com/*', 'https://x.com/*'] },
+//   ['requestHeaders']
+// );
+
 chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
   if (request.type === "getJwt") {
     chrome.storage.local.get(["jwt"], ({ jwt }) => {
@@ -73,4 +97,12 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
       return true;
     })();
   }
+  // TODO: Implement getting bookmarks from API directly
+  // else if (request.action === 'getAuthData') {
+  //   sendResponse({
+  //     authorizationHeader: authorizationHeader,
+  //     csrfToken: csrfToken,
+  //     cookies: cookies
+  //   });
+  // }
 });
diff --git a/apps/web/db/prepare.sql b/apps/web/db/prepare.sql
index a4f9951d..dcba4d40 100644
--- a/apps/web/db/prepare.sql
+++ b/apps/web/db/prepare.sql
@@ -34,7 +34,7 @@ CREATE TABLE `session` (
 --> statement-breakpoint
 CREATE TABLE `space` (
 	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`name` text DEFAULT 'all' NOT NULL,
+	`name` text DEFAULT 'none' NOT NULL,
 	`user` text(255),
 	FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action
 );
@@ -47,6 +47,7 @@ CREATE TABLE `storedContent` (
 	`url` text NOT NULL,
 	`savedAt` integer NOT NULL,
 	`baseUrl` text(255),
+	`type` text DEFAULT 'page',
 	`image` text(255),
 	`user` text(255),
 	FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action
@@ -69,6 +70,7 @@ CREATE TABLE `verificationToken` (
 --> statement-breakpoint
 CREATE INDEX `account_userId_idx` ON `account` (`userId`);--> statement-breakpoint
 CREATE INDEX `session_userId_idx` ON `session` (`userId`);--> statement-breakpoint
+CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint
 CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint
 CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint
 CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint
diff --git a/apps/web/src/app/api/store/route.ts b/apps/web/src/app/api/store/route.ts
index ebe23077..ca6921c4 100644
--- a/apps/web/src/app/api/store/route.ts
+++ b/apps/web/src/app/api/store/route.ts
@@ -67,7 +67,7 @@ export async function POST(req: NextRequest) {
   let storeToSpace = data.space;
 
   if (!storeToSpace) {
-    storeToSpace = "all";
+    storeToSpace = "none";
   }
 
   const storedContentId = await db.insert(storedContent).values({