import { env } from "$env/dynamic/public"; import { type Media, recentMediaActivities } from "$lib/Data/AniList/media"; import { proxyRoute } from "$lib/Utility/proxy"; import settings from "$stores/settings"; import type { UserIdentity } from "../../Data/AniList/identity"; import { database } from "../../Database/IDB/chapters"; interface MangaDexChapterCount { chapter: number | null; volumeText?: string | null; } interface MangaDexChapterCountsResponse { data?: Record; pending?: number[]; retryAfterMs?: number; } interface NativeChapterCount { chapter: number | null; } interface NativeChapterCountsResponse { data?: Record; } const chapterMemoryCache = new Map(); const MAX_PENDING_RETRIES = 2; const DEFAULT_PENDING_RETRY_MS = 750; const NATIVE_PROGRESSIVE_CHUNK_SIZE = 10; const browserChapterCacheDisabled = () => env.PUBLIC_DISABLE_MANGA_BROWSER_CACHE === "true"; const parseOptionalNumber = (value: string | number | null | undefined) => { if (value === null || value === undefined || value === "") return null; const number = Number.parseFloat(String(value)); return Number.isFinite(number) ? number : null; }; const sleep = (milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds)); const readCachedChapterCount = async ( mangaId: number, allowNullCache = true, ) => { if (chapterMemoryCache.has(mangaId)) return chapterMemoryCache.get(mangaId); if (browserChapterCacheDisabled()) return undefined; const chapter = await database.chapters.get(mangaId); return chapter === undefined ? undefined : chapter.chapters === -1 ? allowNullCache ? null : undefined : chapter.chapters; }; const writeCachedChapterCount = async ( mangaId: number, chapters: number | null, volumes: number | null = null, ) => { chapterMemoryCache.set(mangaId, chapters); if (browserChapterCacheDisabled()) return; await database.chapters.put({ id: mangaId, chapters: chapters === null ? -1 : chapters, volumes, }); }; const writeNativeChapterCount = async ( mangaId: number, chapters: number | null, ) => { const existing = browserChapterCacheDisabled() ? undefined : await database.chapters.get(mangaId); await writeCachedChapterCount(mangaId, chapters, existing?.volumes ?? null); }; const getActivityChapterCount = async ( identity: UserIdentity, manga: Media, disableGuessing: boolean, ) => { if (disableGuessing) { await writeCachedChapterCount(manga.id, null); return null; } const anilistData = await recentMediaActivities( identity, manga, settings.get().calculateGuessMethod, ); await writeCachedChapterCount(manga.id, anilistData); return anilistData; }; const applyResolvedChapterCount = async ( identity: UserIdentity, manga: Media, disableGuessing: boolean, chapter: number, volumeText?: string | null, ) => { let resolvedChapter = chapter; if ( (manga.mediaListEntry || { progress: 0 }).progress > resolvedChapter && !disableGuessing ) { const activityChapterCount = await recentMediaActivities( identity, manga, settings.get().calculateGuessMethod, ); if (activityChapterCount !== null && activityChapterCount > resolvedChapter) resolvedChapter = activityChapterCount; } if (resolvedChapter === 0) resolvedChapter = -1; await writeCachedChapterCount( manga.id, resolvedChapter === -1 ? null : resolvedChapter, parseOptionalNumber(volumeText), ); return resolvedChapter === -1 ? null : resolvedChapter; }; const fetchMangaChapterCounts = async (manga: Media[]) => { const data: Record = {}; let rateLimited = false; let successfulResponse = false; const mangaById = new Map(manga.map((entry) => [entry.id, entry])); let pendingManga = manga; let retryAfterMs = DEFAULT_PENDING_RETRY_MS; for (let attempt = 0; attempt <= MAX_PENDING_RETRIES; attempt += 1) { const nextPendingIds = new Set(); for (let index = 0; index < pendingManga.length; index += 100) { const chunk = pendingManga.slice(index, index + 100); const response = await fetch(proxyRoute("/manga/chapter-counts"), { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ manga: chunk.map((entry) => ({ anilistId: entry.id, progress: entry.mediaListEntry?.progress || 0, status: entry.status, startYear: entry.startDate.year, nativeTitle: entry.title.native, englishTitle: entry.title.english, romajiTitle: entry.title.romaji, })), }), }).catch(() => null); if (!response) continue; if (!response.ok) { rateLimited = response.status === 429 || response.status === 503; continue; } const payload = (await response.json()) as MangaDexChapterCountsResponse; successfulResponse = true; retryAfterMs = payload.retryAfterMs || retryAfterMs; Object.assign(data, payload.data || {}); for (const id of payload.pending || []) nextPendingIds.add(id); } if (!nextPendingIds.size || attempt === MAX_PENDING_RETRIES) break; pendingManga = [...nextPendingIds] .map((id) => mangaById.get(id)) .filter((entry): entry is Media => entry !== undefined); if (!pendingManga.length) break; await sleep(retryAfterMs); } return { data, rateLimited: rateLimited && !successfulResponse }; }; const fetchNativeChapterCounts = async (manga: Media[]) => { const data: Record = {}; const failedIds = new Set(); for (let index = 0; index < manga.length; index += 100) { const chunk = manga.slice(index, index + 100); const response = await fetch(proxyRoute("/manga/native-chapter-counts"), { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ manga: chunk.map((entry) => ({ anilistId: entry.id, nativeTitle: entry.title.native, englishTitle: entry.title.english, romajiTitle: entry.title.romaji, })), }), }).catch(() => null); if (!response?.ok) { for (const entry of chunk) failedIds.add(entry.id); continue; } const payload = (await response.json()) as NativeChapterCountsResponse; Object.assign(data, payload.data || {}); for (const entry of chunk) if (!(String(entry.id) in (payload.data || {}))) failedIds.add(entry.id); } return { data, failedIds }; }; export const hydrateNativeChapterCountsProgressively = async ( manga: Media[], onChunkResolved?: () => void, ) => { const uniqueManga = manga .filter((entry) => entry.format !== "NOVEL") .filter( (entry, index, array) => array.findIndex((candidate) => candidate.id === entry.id) === index, ); for ( let index = 0; index < uniqueManga.length; index += NATIVE_PROGRESSIVE_CHUNK_SIZE ) { const chunk = uniqueManga.slice( index, index + NATIVE_PROGRESSIVE_CHUNK_SIZE, ); const { data: nativeCounts, failedIds } = await fetchNativeChapterCounts(chunk); let changed = false; for (const entry of chunk) { if (failedIds.has(entry.id)) continue; const nativeCount = nativeCounts[String(entry.id)]?.chapter; if (nativeCount === null || nativeCount === undefined) continue; await writeNativeChapterCount(entry.id, nativeCount); changed = true; } if (changed) onChunkResolved?.(); } }; export const hydrateChapterCounts = async ( identity: UserIdentity, manga: Media[], disableGuessing: boolean, options: { deferNative?: boolean; startDeferredNative?: boolean; onNativeChunkResolved?: () => void; } = {}, ): Promise<{ rateLimited: boolean }> => { const uniqueManga = manga.filter( (entry, index, array) => array.findIndex((candidate) => candidate.id === entry.id) === index, ); const nativeCountManga: Media[] = []; const unresolvedManga: Media[] = []; for (const entry of uniqueManga) { if ( (await readCachedChapterCount( entry.id, !settings.get().calculatePreferNativeChapterCount, )) !== undefined ) continue; if (entry.format === "NOVEL") { await getActivityChapterCount(identity, entry, disableGuessing); continue; } if (settings.get().calculatePreferNativeChapterCount) { nativeCountManga.push(entry); continue; } unresolvedManga.push(entry); } if (nativeCountManga.length) { if (options.deferNative) { unresolvedManga.push(...nativeCountManga); if (options.startDeferredNative) void hydrateNativeChapterCountsProgressively( nativeCountManga, options.onNativeChunkResolved, ); } else { const { data: nativeCounts, failedIds } = await fetchNativeChapterCounts(nativeCountManga); for (const entry of nativeCountManga) { if (failedIds.has(entry.id)) continue; const nativeCount = nativeCounts[String(entry.id)]?.chapter; if (nativeCount === null) { unresolvedManga.push(entry); continue; } if (nativeCount === undefined) continue; await writeNativeChapterCount(entry.id, nativeCount); } } } if (!unresolvedManga.length) return { rateLimited: false }; const { data, rateLimited } = await fetchMangaChapterCounts(unresolvedManga); for (const entry of unresolvedManga) { const resolved = data[String(entry.id)]; if (resolved?.chapter !== undefined && resolved.chapter !== null) { await applyResolvedChapterCount( identity, entry, disableGuessing, resolved.chapter, resolved.volumeText, ); continue; } await getActivityChapterCount(identity, entry, disableGuessing); } return { rateLimited }; }; export const chapterCount = async ( identity: UserIdentity, manga: Media, disableGuessing: boolean, ): Promise => { const cachedChapterCount = await readCachedChapterCount(manga.id); if (cachedChapterCount !== undefined) return cachedChapterCount; const { rateLimited } = await hydrateChapterCounts( identity, [manga], disableGuessing, ); if (rateLimited) return -22; return (await readCachedChapterCount(manga.id)) ?? null; };