From 7e447fd8f478fd3f980f9b44ace29abc7fdffb04 Mon Sep 17 00:00:00 2001 From: Fuwn Date: Fri, 27 Mar 2026 08:28:30 +0000 Subject: refactor(proxy): move manga chapter counts behind indexed cache --- apps/proxy/src/mangadex.js | 392 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 apps/proxy/src/mangadex.js (limited to 'apps/proxy/src/mangadex.js') diff --git a/apps/proxy/src/mangadex.js b/apps/proxy/src/mangadex.js new file mode 100644 index 00000000..d2a42154 --- /dev/null +++ b/apps/proxy/src/mangadex.js @@ -0,0 +1,392 @@ +import { + getMangadexRowsByMangadexIds, + getSyncState, + upsertMangadexRows, + upsertSyncState, +} from "./supabase.js"; + +const MANGADEX_API_ORIGIN = "https://api.mangadex.org"; +const SYNC_STATE_NAME = "chapter_en"; +const ALL_CONTENT_RATINGS = ["safe", "suggestive", "erotica", "pornographic"]; + +const sleep = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +const appendQueryValues = (url, query) => { + for (const [key, rawValue] of Object.entries(query)) { + if (rawValue === undefined || rawValue === null) continue; + + if (Array.isArray(rawValue)) { + for (const value of rawValue) url.searchParams.append(key, String(value)); + + continue; + } + + url.searchParams.set(key, String(rawValue)); + } +}; + +const mangadexUserAgent = (env) => + env?.MANGADEX_USER_AGENT || + "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"; + +const mangadexRequest = async ( + env, + path, + { query = {}, retries = 1, retryDelay = 1500 } = {}, +) => { + const url = new URL(path, MANGADEX_API_ORIGIN); + + appendQueryValues(url, query); + + const response = await fetch(url, { + headers: { + Accept: "application/json", + "User-Agent": mangadexUserAgent(env), + }, + }); + + if (response.status === 429 && retries > 0) { + const retryAfterHeader = + response.headers.get("Retry-After") || + response.headers.get("X-RateLimit-Retry-After"); + const retryAfterSeconds = Number.parseInt(retryAfterHeader || "", 10); + const retryAt = + Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 10_000 + ? retryAfterSeconds * 1000 - Date.now() + : retryAfterSeconds * 1000; + + await sleep( + Number.isFinite(retryAt) && retryAt > 0 + ? Math.min(retryAt, 30_000) + : retryDelay, + ); + + return mangadexRequest(env, path, { + query, + retries: retries - 1, + retryDelay: retryDelay * 2, + }); + } + + if (!response.ok) { + const message = await response.text(); + + throw new Error(`MangaDex request failed (${response.status}): ${message}`); + } + + return response.json(); +}; + +const statusMap = { + FINISHED: "completed", + RELEASING: "ongoing", + HIATUS: "hiatus", + CANCELLED: "cancelled", +}; + +const normaliseTitle = (value) => + String(value || "") + .trim() + .toLowerCase(); + +const parseOptionalNumber = (value) => { + if (value === null || value === undefined || value === "") return null; + + const number = Number.parseFloat(String(value)); + + return Number.isFinite(number) ? number : null; +}; + +const mangadexDateTime = (isoString) => { + const date = new Date(isoString); + + if (Number.isNaN(date.getTime())) return "1970-01-01T00:00:00"; + + return date.toISOString().slice(0, 19); +}; + +const scoreSearchResult = (manga, title) => { + const attributes = manga.attributes || {}; + const titleMap = attributes.title || {}; + const altTitles = (attributes.altTitles || []).flatMap((item) => + Object.values(item || {}), + ); + const titles = [...Object.values(titleMap || {}), ...altTitles] + .map(normaliseTitle) + .filter(Boolean); + const candidate = normaliseTitle(title); + + if (!candidate.length) return Number.MAX_SAFE_INTEGER; + if (titles.includes(candidate)) return 0; + if (titles.some((item) => item.includes(candidate))) return 1; + return 2; +}; + +const pickBestSearchResult = (results, title) => + [...results].sort( + (left, right) => + scoreSearchResult(left, title) - scoreSearchResult(right, title), + )[0]; + +const titleCandidates = (manga) => + [ + manga.nativeTitle, + manga.englishTitle, + manga.romajiTitle, + manga.nativeTitle === "null" ? null : manga.nativeTitle, + manga.englishTitle === "null" ? null : manga.englishTitle, + manga.romajiTitle === "null" ? null : manga.romajiTitle, + ] + .filter(Boolean) + .map((title) => String(title).trim()) + .filter((title, index, array) => array.indexOf(title) === index); + +const searchQueries = (manga, title) => { + const status = statusMap[manga.status]; + + return [ + { + title, + ...(manga.startYear ? { year: manga.startYear } : {}), + ...(status ? { "status[]": [status] } : {}), + limit: 5, + }, + { + title, + ...(status ? { "status[]": [status] } : {}), + limit: 5, + }, + { + title, + ...(manga.startYear ? { year: manga.startYear } : {}), + limit: 5, + }, + { + title, + limit: 5, + }, + ]; +}; + +const chooseLatestAggregateChapter = (volumes) => { + let latest = null; + + for (const [volumeKey, volumeEntry] of Object.entries(volumes || {})) { + const chapters = volumeEntry?.chapters || {}; + + for (const chapterEntry of Object.values(chapters)) { + const chapterText = chapterEntry?.chapter || null; + const chapterNumber = parseOptionalNumber(chapterText); + const volumeText = + chapterEntry?.volume || volumeEntry?.volume || volumeKey || null; + const volumeNumber = parseOptionalNumber(volumeText); + const candidate = { + chapterText, + chapterNumber, + volumeText, + volumeNumber, + chapterId: chapterEntry?.id || null, + }; + + if (!latest) { + latest = candidate; + + continue; + } + + if ( + candidate.chapterNumber !== null && + (latest.chapterNumber === null || + candidate.chapterNumber > latest.chapterNumber || + (candidate.chapterNumber === latest.chapterNumber && + (candidate.volumeNumber || -1) > (latest.volumeNumber || -1))) + ) { + latest = candidate; + + continue; + } + + if ( + candidate.chapterNumber === null && + latest.chapterNumber === null && + String(candidate.chapterText || "").localeCompare( + String(latest.chapterText || ""), + undefined, + { numeric: true }, + ) > 0 + ) + latest = candidate; + } + } + + return latest; +}; + +const buildIndexedRow = (manga, mangadexId, aggregateResponse) => { + const now = new Date().toISOString(); + const latest = chooseLatestAggregateChapter(aggregateResponse?.volumes); + + return { + anilist_id: manga.anilistId, + mangadex_id: mangadexId, + latest_en_chapter_text: latest?.chapterText || null, + latest_en_chapter_number: latest?.chapterNumber || null, + latest_en_volume_text: latest?.volumeText || null, + latest_en_chapter_id: latest?.chapterId || null, + latest_chapter_updated_at: now, + last_seen_at: now, + last_indexed_at: now, + is_releasing: manga.status === "RELEASING", + created_at: manga.createdAt || now, + updated_at: now, + }; +}; + +export const resolveMangadexId = async (env, manga) => { + const candidates = titleCandidates(manga); + + for (const title of candidates) { + for (const query of searchQueries(manga, title)) { + const payload = await mangadexRequest(env, "/manga", { query }); + const data = payload?.data || []; + + if (!data.length) continue; + + return pickBestSearchResult(data, title)?.id || data[0]?.id || null; + } + } + + return null; +}; + +export const bootstrapManga = async (env, manga) => { + const mangadexId = await resolveMangadexId(env, manga); + + if (!mangadexId) return null; + + const aggregateResponse = await mangadexRequest( + env, + `/manga/${mangadexId}/aggregate`, + { + query: { + "translatedLanguage[]": ["en"], + }, + }, + ); + + return buildIndexedRow(manga, mangadexId, aggregateResponse); +}; + +const extractMangaIdFromChapter = (chapter) => + chapter?.relationships?.find((relationship) => relationship.type === "manga") + ?.id || null; + +const currentCursor = () => new Date().toISOString(); + +const rewindCursor = (isoString, seconds = 30) => + new Date( + Math.max(new Date(isoString).getTime() - seconds * 1000, 0), + ).toISOString(); + +export const syncMangadexIndex = async ( + env, + { maxPages = 5, aggregateDelayMs = 250 } = {}, +) => { + const syncState = (await getSyncState(env, SYNC_STATE_NAME)) || { + cursor_updated_at: currentCursor(), + }; + const trackedMangadexIds = new Set(); + const cursor = syncState.cursor_updated_at; + let maxUpdatedAt = cursor; + let offset = 0; + let pageCount = 0; + let scannedChapterCount = 0; + + while (pageCount < maxPages) { + const payload = await mangadexRequest(env, "/chapter", { + query: { + "translatedLanguage[]": ["en"], + "contentRating[]": ALL_CONTENT_RATINGS, + includeFutureUpdates: 0, + includeFuturePublishAt: 0, + updatedAtSince: mangadexDateTime(cursor), + "order[updatedAt]": "asc", + limit: 100, + offset, + }, + }); + const chapters = payload?.data || []; + + if (!chapters.length) break; + + for (const chapter of chapters) { + scannedChapterCount += 1; + + const mangaId = extractMangaIdFromChapter(chapter); + const updatedAt = chapter?.attributes?.updatedAt; + + if (mangaId) trackedMangadexIds.add(mangaId); + if (updatedAt && updatedAt > maxUpdatedAt) maxUpdatedAt = updatedAt; + } + + pageCount += 1; + + if (chapters.length < 100) break; + + offset += chapters.length; + } + + if (!trackedMangadexIds.size) { + await upsertSyncState(env, SYNC_STATE_NAME, maxUpdatedAt); + + return { + cursor: maxUpdatedAt, + pageCount, + scannedChapterCount, + refreshedCount: 0, + }; + } + + const trackedRows = await getMangadexRowsByMangadexIds(env, [ + ...trackedMangadexIds, + ]); + const refreshedRows = []; + + for (const row of trackedRows) { + const aggregateResponse = await mangadexRequest( + env, + `/manga/${row.mangadex_id}/aggregate`, + { + query: { + "translatedLanguage[]": ["en"], + }, + }, + ); + + refreshedRows.push( + buildIndexedRow( + { + anilistId: row.anilist_id, + status: row.is_releasing ? "RELEASING" : "FINISHED", + createdAt: row.created_at, + }, + row.mangadex_id, + aggregateResponse, + ), + ); + + if (aggregateDelayMs > 0) await sleep(aggregateDelayMs); + } + + if (refreshedRows.length) await upsertMangadexRows(env, refreshedRows); + + await upsertSyncState(env, SYNC_STATE_NAME, rewindCursor(maxUpdatedAt)); + + return { + cursor: rewindCursor(maxUpdatedAt), + pageCount, + scannedChapterCount, + refreshedCount: refreshedRows.length, + }; +}; -- cgit v1.2.3