import { Tweet } from "react-tweet/api"; import { features, transformTweetData } from "./helpers"; const tweetToMd = (tweet: Tweet) => { return `Tweet from @${tweet.user?.name ?? tweet.user?.screen_name ?? "Unknown"} ${tweet.text} Images: ${tweet.photos ? tweet.photos.map((photo) => photo.url).join(", ") : "none"} Time: ${tweet.created_at}, Likes: ${tweet.favorite_count}, Retweets: ${tweet.conversation_count} ${JSON.stringify(tweet)}`; }; const BOOKMARKS_URL = `https://x.com/i/api/graphql/xLjCVTqYWz8CGSprLU349w/Bookmarks?features=${encodeURIComponent(JSON.stringify(features))}`; const BACKEND_URL = "https://supermemory.ai"; // This is to prevent going over the rate limit let lastTwitterFetch = 0; const batchImportAll = async (cursor = "", totalImported = 0) => { chrome.storage.session.get(["cookie", "csrf", "auth"], (result) => { if (!result.cookie || !result.csrf || !result.auth) { console.log("cookie, csrf, or auth is missing"); return; } const myHeaders = new Headers(); myHeaders.append("Cookie", result.cookie); myHeaders.append("X-Csrf-token", result.csrf); myHeaders.append("Authorization", result.auth); const requestOptions: RequestInit = { method: "GET", headers: myHeaders, redirect: "follow", }; const variables = { count: 100, cursor: cursor, includePromotedContent: false, }; const urlWithCursor = cursor ? `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}` : BOOKMARKS_URL; fetch(urlWithCursor, requestOptions) .then((response) => response.json()) .then((data) => { const tweets = getAllTweets(data); let importedCount = 0; for (const tweet of tweets) { console.log(tweet); const tweetMd = tweetToMd(tweet); (async () => { chrome.storage.local.get(["jwt"], ({ jwt }) => { if (!jwt) { console.error("No JWT found"); return; } fetch(`${BACKEND_URL}/api/store`, { method: "POST", headers: { Authorization: `Bearer ${jwt}`, }, body: JSON.stringify({ pageContent: tweetMd, url: `https://twitter.com/supermemoryai/status/${tweet.id_str}`, title: `Tweet by ${tweet.user.name}`, description: tweet.text.slice(0, 200), type: "tweet", }), }).then(async (ers) => { console.log(ers.status); importedCount++; totalImported++; console.log(totalImported); chrome.tabs.query( { active: true, currentWindow: true }, async function (tabs) { if (tabs.length > 0) { let currentTabId = tabs[0].id; if (!currentTabId) { return; } await chrome.tabs.sendMessage(currentTabId, { type: "import-update", importedCount: totalImported, }); } }, ); }); }); })(); } console.log("tweets", tweets); console.log("data", data); const instructions = data.data?.bookmark_timeline_v2?.timeline?.instructions; const lastInstruction = instructions?.[0].entries.pop(); if (lastInstruction?.entryId.startsWith("cursor-bottom-")) { let nextCursor = lastInstruction?.content?.value; if (!nextCursor) { for (let i = instructions.length - 1; i >= 0; i--) { if (instructions[i].entryId.startsWith("cursor-bottom-")) { nextCursor = instructions[i].content.value; break; } } } if (nextCursor) { batchImportAll(nextCursor, totalImported); // Recursively call with new cursor } else { console.log("All bookmarks imported"); chrome.tabs.query( { active: true, currentWindow: true }, async function (tabs) { if (tabs.length > 0) { let currentTabId = tabs[0].id; if (!currentTabId) { return; } await chrome.runtime.sendMessage({ type: "import-done", importedCount: totalImported, }); } }, ); } } else { console.log("All bookmarks imported"); // Send a "done" message to the content script chrome.tabs.query( { active: true, currentWindow: true }, async function (tabs) { if (tabs.length > 0) { let currentTabId = tabs[0].id; if (!currentTabId) { return; } await chrome.runtime.sendMessage({ type: "import-done", importedCount: totalImported, }); } }, ); } }) .catch((error) => console.error(error)); }); }; chrome.webRequest.onBeforeSendHeaders.addListener( (details) => { if ( !(details.url.includes("x.com") || details.url.includes("twitter.com")) ) { return; } const authHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "authorization", ); const auth = authHeader ? authHeader.value : ""; const cookieHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "cookie", ); const cookie = cookieHeader ? cookieHeader.value : ""; const csrfHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "x-csrf-token", ); const csrf = csrfHeader ? csrfHeader.value : ""; if (!auth || !cookie || !csrf) { console.log("auth, cookie, or csrf is missing"); return; } chrome.storage.session.set({ cookie, csrf, auth }); chrome.storage.local.get(["twitterBookmarks"], (result) => { console.log("twitterBookmarks", result.twitterBookmarks); if (result.twitterBookmarks !== "true") { console.log("twitterBookmarks is NOT true"); } else { if ( !details.requestHeaders || details.requestHeaders.length === 0 || details.requestHeaders === undefined ) { return; } // Check cache first chrome.storage.local.get(["lastFetch", "cachedData"], (result) => { const now = new Date().getTime(); if (result.lastFetch && now - result.lastFetch < 30 * 60 * 1000) { // Cached data is less than 30 minutes old, use it console.log("Using cached data"); console.log(result.cachedData); return; } // No valid cache, proceed to fetch const authHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "authorization", ); const auth = authHeader ? authHeader.value : ""; const cookieHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "cookie", ); const cookie = cookieHeader ? cookieHeader.value : ""; const csrfHeader = details.requestHeaders!.find( (header) => header.name.toLowerCase() === "x-csrf-token", ); const csrf = csrfHeader ? csrfHeader.value : ""; if (!auth || !cookie || !csrf) { console.log("auth, cookie, or csrf is missing"); return; } chrome.storage.session.set({ cookie, csrf, auth }); const myHeaders = new Headers(); myHeaders.append("Cookie", cookie); myHeaders.append("X-Csrf-token", csrf); myHeaders.append("Authorization", auth); const requestOptions: RequestInit = { method: "GET", headers: myHeaders, redirect: "follow", }; const variables = { count: 200, includePromotedContent: false, }; // only fetch once in 1 minute if (now - lastTwitterFetch < 60 * 1000) { console.log("Waiting for ratelimits"); return; } fetch( `${BOOKMARKS_URL}&variables=${encodeURIComponent(JSON.stringify(variables))}`, requestOptions, ) .then((response) => response.text()) .then((result) => { const tweets = getAllTweets(JSON.parse(result)); console.log("tweets", tweets); // Cache the result along with the current timestamp chrome.storage.local.set({ lastFetch: new Date().getTime(), cachedData: tweets, }); lastTwitterFetch = now; }) .catch((error) => console.error(error)); }); return; } }); }, { urls: ["*://x.com/*", "*://twitter.com/*"] }, ["requestHeaders", "extraHeaders"], ); const getAllTweets = (rawJson: any): Tweet[] => { const entries = rawJson?.data?.bookmark_timeline_v2?.timeline?.instructions[0]?.entries; console.log("Entries: ", entries); if (!entries) { console.error("No entries found"); return []; } const tweets = entries .map((entry: any) => transformTweetData(entry)) .filter((tweet: Tweet | null) => tweet !== null) as Tweet[]; console.log(tweets); return tweets; }; chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { console.log(request); if (request.type === "getJwt") { chrome.storage.local.get(["jwt"], ({ jwt }) => { sendResponse({ jwt }); }); return true; } else if (request.type === "urlSave") { const content = request.content; const url = request.url; const title = request.title; const description = request.description; const ogImage = request.ogImage; const favicon = request.favicon; console.log(request.content, request.url); (async () => { chrome.storage.local.get(["jwt"], ({ jwt }) => { if (!jwt) { console.error("No JWT found"); return; } fetch(`${BACKEND_URL}/api/store`, { method: "POST", headers: { Authorization: `Bearer ${jwt}`, }, body: JSON.stringify({ pageContent: content, url: url + "#supermemory-user-" + Math.random(), title, spaces: request.spaces, description, ogImage, image: favicon, }), }).then((ers) => console.log(ers.status)); }); })(); } else if (request.type === "batchImportAll") { batchImportAll(); return true; } }); chrome.runtime.onInstalled.addListener(function (details) { if (details.reason === "install") { chrome.tabs.create({ url: "https://supermemory.ai/signin?extension=true", active: true, }); } }); chrome.runtime.onInstalled.addListener(() => { chrome.contextMenus.create({ id: "saveSelection", title: "Save note to Supermemory", contexts: ["selection"], }); chrome.contextMenus.create({ id: "savePage", title: "Save page to Supermemory", contexts: ["page"], }); // TODO // chrome.contextMenus.create({ // id: 'saveLink', // title: 'Save link to Supermemory', // contexts: ['link'], // }); }); interface FetchDataParams { content: string; url: string; title: string; description: string; ogImage: string; favicon: string; isExternalContent: boolean; // Indicates if the content is from an external API } const fetchData = ({ content, url, title, description, ogImage, favicon, isExternalContent, }: FetchDataParams) => { // Construct the URL const finalUrl = isExternalContent ? url : `${url}#supermemory-stuff-${Math.random()}`; // Construct the body const body = JSON.stringify({ pageContent: content, url: finalUrl, title, spaces: [], description, ogImage, image: favicon, }); // Make the fetch call fetch(`${BACKEND_URL}/api/store`, { method: "POST", headers: { "Content-Type": "application/json", }, body: body, }) .then((response) => { console.log("Data saved successfully"); }) .catch((error) => { console.error("Error saving data:", error); }); return Promise.resolve(); }; chrome.contextMenus.onClicked.addListener((info, tab) => { if (!tab || !tab.id) return; const tabId = tab.id; const sendMessageToTab = (message: string) => { chrome.tabs.sendMessage(tabId, { message, type: "supermemory-message" }); }; if (info.menuItemId === "saveSelection" && info.selectionText) { sendMessageToTab("Saving selection..."); fetchData({ content: info.selectionText || "No content", url: info.pageUrl, title: tab.title || "Selection Title", description: "User-selected content from the page", ogImage: "", favicon: "", isExternalContent: false, }) .then(() => { sendMessageToTab("Selection saved successfully."); }) .catch(() => { sendMessageToTab("Failed to save selection."); }); } else if (info.menuItemId === "savePage") { sendMessageToTab("Saving page..."); chrome.scripting.executeScript( { target: { tabId: tabId }, func: () => document.body.innerText, }, (results) => { if (results.length > 0 && results[0].result) { fetchData({ content: results[0].result as string, url: info.pageUrl, title: tab.title || "Page Title", description: "Full page content", ogImage: "", favicon: "", isExternalContent: false, }) .then(() => { sendMessageToTab("Page saved successfully."); }) .catch(() => { sendMessageToTab("Failed to save page."); }); } }, ); } });