import { getMangadexRowsByMangadexIds, getSyncState, upsertMangadexRows, upsertSyncState, } from "./supabase.js"; const MANGADEX_API_ORIGIN = "https://api.mangadex.org"; const SYNC_STATE_NAME = "chapter_en"; const ALL_CONTENT_RATINGS = ["safe", "suggestive", "erotica", "pornographic"]; const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); const appendQueryValues = (url, query) => { for (const [key, rawValue] of Object.entries(query)) { if (rawValue === undefined || rawValue === null) continue; if (Array.isArray(rawValue)) { for (const value of rawValue) url.searchParams.append(key, String(value)); continue; } url.searchParams.set(key, String(rawValue)); } }; const mangadexUserAgent = (env) => env?.MANGADEX_USER_AGENT || "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"; const mangadexRequest = async ( env, path, { query = {}, retries = 1, retryDelay = 1500 } = {}, ) => { const url = new URL(path, MANGADEX_API_ORIGIN); appendQueryValues(url, query); const response = await fetch(url, { headers: { Accept: "application/json", "User-Agent": mangadexUserAgent(env), }, }); if (response.status === 429 && retries > 0) { const retryAfterHeader = response.headers.get("Retry-After") || response.headers.get("X-RateLimit-Retry-After"); const retryAfterSeconds = Number.parseInt(retryAfterHeader || "", 10); const retryAt = Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 10_000 ? retryAfterSeconds * 1000 - Date.now() : retryAfterSeconds * 1000; await sleep( Number.isFinite(retryAt) && retryAt > 0 ? Math.min(retryAt, 30_000) : retryDelay, ); return mangadexRequest(env, path, { query, retries: retries - 1, retryDelay: retryDelay * 2, }); } if (!response.ok) { const message = await response.text(); throw new Error(`MangaDex request failed (${response.status}): ${message}`); } return response.json(); }; const statusMap = { FINISHED: "completed", RELEASING: "ongoing", HIATUS: "hiatus", CANCELLED: "cancelled", }; const normaliseTitle = (value) => String(value || "") .trim() .toLowerCase(); const parseOptionalNumber = (value) => { if (value === null || value === undefined || value === "") return null; const number = Number.parseFloat(String(value)); return Number.isFinite(number) ? number : null; }; const mangadexDateTime = (isoString) => { const date = new Date(isoString); if (Number.isNaN(date.getTime())) return "1970-01-01T00:00:00"; return date.toISOString().slice(0, 19); }; const scoreSearchResult = (manga, title) => { const attributes = manga.attributes || {}; const titleMap = attributes.title || {}; const altTitles = (attributes.altTitles || []).flatMap((item) => Object.values(item || {}), ); const titles = [...Object.values(titleMap || {}), ...altTitles] .map(normaliseTitle) .filter(Boolean); const candidate = normaliseTitle(title); if (!candidate.length) return Number.MAX_SAFE_INTEGER; if (titles.includes(candidate)) return 0; if (titles.some((item) => item.includes(candidate))) return 1; return 2; }; const pickBestSearchResult = (results, title) => [...results].sort( (left, right) => scoreSearchResult(left, title) - scoreSearchResult(right, title), )[0]; const titleCandidates = (manga) => [ manga.nativeTitle, manga.englishTitle, manga.romajiTitle, manga.nativeTitle === "null" ? null : manga.nativeTitle, manga.englishTitle === "null" ? null : manga.englishTitle, manga.romajiTitle === "null" ? null : manga.romajiTitle, ] .filter(Boolean) .map((title) => String(title).trim()) .filter((title, index, array) => array.indexOf(title) === index); const searchQueries = (manga, title) => { const status = statusMap[manga.status]; return [ { title, ...(manga.startYear ? { year: manga.startYear } : {}), ...(status ? { "status[]": [status] } : {}), limit: 5, }, { title, ...(status ? { "status[]": [status] } : {}), limit: 5, }, { title, ...(manga.startYear ? { year: manga.startYear } : {}), limit: 5, }, { title, limit: 5, }, ]; }; const chooseLatestAggregateChapter = (volumes) => { let latest = null; for (const [volumeKey, volumeEntry] of Object.entries(volumes || {})) { const chapters = volumeEntry?.chapters || {}; for (const chapterEntry of Object.values(chapters)) { const chapterText = chapterEntry?.chapter || null; const chapterNumber = parseOptionalNumber(chapterText); const volumeText = chapterEntry?.volume || volumeEntry?.volume || volumeKey || null; const volumeNumber = parseOptionalNumber(volumeText); const candidate = { chapterText, chapterNumber, volumeText, volumeNumber, chapterId: chapterEntry?.id || null, }; if (!latest) { latest = candidate; continue; } if ( candidate.chapterNumber !== null && (latest.chapterNumber === null || candidate.chapterNumber > latest.chapterNumber || (candidate.chapterNumber === latest.chapterNumber && (candidate.volumeNumber || -1) > (latest.volumeNumber || -1))) ) { latest = candidate; continue; } if ( candidate.chapterNumber === null && latest.chapterNumber === null && String(candidate.chapterText || "").localeCompare( String(latest.chapterText || ""), undefined, { numeric: true }, ) > 0 ) latest = candidate; } } return latest; }; const buildVolumeChapterBoundaries = (volumes) => { const boundaries = {}; for (const [volumeKey, volumeEntry] of Object.entries(volumes || {})) { const chapters = volumeEntry?.chapters || {}; for (const chapterEntry of Object.values(chapters)) { const volumeText = chapterEntry?.volume || volumeEntry?.volume || volumeKey || null; const volumeNumber = parseOptionalNumber(volumeText); const chapterNumber = parseOptionalNumber(chapterEntry?.chapter || null); if (volumeNumber === null || chapterNumber === null) continue; const key = String(volumeNumber); const current = boundaries[key]; if (current === undefined || chapterNumber > current) boundaries[key] = chapterNumber; } } return Object.keys(boundaries).length ? boundaries : null; }; const buildIndexedRow = (manga, mangadexId, aggregateResponse) => { const now = new Date().toISOString(); const latest = chooseLatestAggregateChapter(aggregateResponse?.volumes); const volumeChapterBoundaries = buildVolumeChapterBoundaries( aggregateResponse?.volumes, ); return { anilist_id: manga.anilistId, mangadex_id: mangadexId, latest_en_chapter_text: latest?.chapterText || null, latest_en_chapter_number: latest?.chapterNumber || null, latest_en_volume_text: latest?.volumeText || null, volume_chapter_boundaries: volumeChapterBoundaries, latest_en_chapter_id: latest?.chapterId || null, latest_chapter_updated_at: now, last_seen_at: now, last_indexed_at: now, is_releasing: manga.status === "RELEASING", created_at: manga.createdAt || now, updated_at: now, }; }; export const resolveMangadexId = async (env, manga) => { const candidates = titleCandidates(manga); for (const title of candidates) { for (const query of searchQueries(manga, title)) { const payload = await mangadexRequest(env, "/manga", { query }); const data = payload?.data || []; if (!data.length) continue; return pickBestSearchResult(data, title)?.id || data[0]?.id || null; } } return null; }; export const bootstrapManga = async (env, manga) => { const mangadexId = await resolveMangadexId(env, manga); if (!mangadexId) return null; const aggregateResponse = await mangadexRequest( env, `/manga/${mangadexId}/aggregate`, { query: { "translatedLanguage[]": ["en"], }, }, ); return buildIndexedRow(manga, mangadexId, aggregateResponse); }; const extractMangaIdFromChapter = (chapter) => chapter?.relationships?.find((relationship) => relationship.type === "manga") ?.id || null; const currentCursor = () => new Date().toISOString(); const rewindCursor = (isoString, seconds = 30) => new Date( Math.max(new Date(isoString).getTime() - seconds * 1000, 0), ).toISOString(); export const syncMangadexIndex = async ( env, { maxPages = 5, aggregateDelayMs = 250 } = {}, ) => { const syncState = (await getSyncState(env, SYNC_STATE_NAME)) || { cursor_updated_at: currentCursor(), }; const trackedMangadexIds = new Set(); const cursor = syncState.cursor_updated_at; let maxUpdatedAt = cursor; let offset = 0; let pageCount = 0; let scannedChapterCount = 0; while (pageCount < maxPages) { const payload = await mangadexRequest(env, "/chapter", { query: { "translatedLanguage[]": ["en"], "contentRating[]": ALL_CONTENT_RATINGS, includeFutureUpdates: 0, includeFuturePublishAt: 0, updatedAtSince: mangadexDateTime(cursor), "order[updatedAt]": "asc", limit: 100, offset, }, }); const chapters = payload?.data || []; if (!chapters.length) break; for (const chapter of chapters) { scannedChapterCount += 1; const mangaId = extractMangaIdFromChapter(chapter); const updatedAt = chapter?.attributes?.updatedAt; if (mangaId) trackedMangadexIds.add(mangaId); if (updatedAt && updatedAt > maxUpdatedAt) maxUpdatedAt = updatedAt; } pageCount += 1; if (chapters.length < 100) break; offset += chapters.length; } if (!trackedMangadexIds.size) { await upsertSyncState(env, SYNC_STATE_NAME, maxUpdatedAt); return { cursor: maxUpdatedAt, pageCount, scannedChapterCount, refreshedCount: 0, }; } const trackedRows = await getMangadexRowsByMangadexIds(env, [ ...trackedMangadexIds, ]); const refreshedRows = []; for (const row of trackedRows) { const aggregateResponse = await mangadexRequest( env, `/manga/${row.mangadex_id}/aggregate`, { query: { "translatedLanguage[]": ["en"], }, }, ); refreshedRows.push( buildIndexedRow( { anilistId: row.anilist_id, status: row.is_releasing ? "RELEASING" : "FINISHED", createdAt: row.created_at, }, row.mangadex_id, aggregateResponse, ), ); if (aggregateDelayMs > 0) await sleep(aggregateDelayMs); } if (refreshedRows.length) await upsertMangadexRows(env, refreshedRows); await upsertSyncState(env, SYNC_STATE_NAME, rewindCursor(maxUpdatedAt)); return { cursor: rewindCursor(maxUpdatedAt), pageCount, scannedChapterCount, refreshedCount: refreshedRows.length, }; };