From 7e447fd8f478fd3f980f9b44ace29abc7fdffb04 Mon Sep 17 00:00:00 2001
From: Fuwn <contact@fuwn.net>
Date: Fri, 27 Mar 2026 08:28:30 +0000
Subject: refactor(proxy): move manga chapter counts behind indexed cache

---
 src/lib/List/Manga/MangaListTemplate.svelte |  81 +++---
 src/lib/Media/Manga/chapters.ts             | 407 +++++++++++++++-------------
 src/lib/Utility/proxy.ts                    |  17 +-
 3 files changed, 270 insertions(+), 235 deletions(-)

(limited to 'src/lib')

diff --git a/src/lib/List/Manga/MangaListTemplate.svelte b/src/lib/List/Manga/MangaListTemplate.svelte
index eb7ffd83..7333d24e 100644
--- a/src/lib/List/Manga/MangaListTemplate.svelte
+++ b/src/lib/List/Manga/MangaListTemplate.svelte
@@ -1,25 +1,25 @@
 <script lang="ts">
-import { mediaListCollection, Type, type Media } from "$lib/Data/AniList/media";
-import type { AniListAuthorisation } from "$lib/Data/AniList/identity";
+import localforage from "localforage";
 import { onDestroy, onMount } from "svelte";
-import { chapterCount } from "$lib/Media/Manga/chapters";
-import { pruneAllManga } from "$lib/Media/Manga/cache";
-import manga from "$stores/manga";
+import { browser } from "$app/environment";
+import type { AniListAuthorisation } from "$lib/Data/AniList/identity";
+import { type Media, mediaListCollection, Type } from "$lib/Data/AniList/media";
 import { database } from "$lib/Database/IDB/chapters";
-import settings from "$stores/settings";
-import lastPruneTimes from "$stores/lastPruneTimes";
-import ListTitle from "../ListTitle.svelte";
 import RateLimitedError from "$lib/Error/RateLimited.svelte";
-import CleanMangaList from "./CleanMangaList.svelte";
+import Skeleton from "$lib/Loading/Skeleton.svelte";
 import { incrementMediaProgress } from "$lib/Media/Anime/cache";
-import { addNotification } from "$lib/Notification/store";
+import { pruneAllManga } from "$lib/Media/Manga/cache";
+import { chapterCount, hydrateChapterCounts } from "$lib/Media/Manga/chapters";
 import { options } from "$lib/Notification/options";
-import Skeleton from "$lib/Loading/Skeleton.svelte";
-import locale from "$stores/locale";
-import { browser } from "$app/environment";
-import identity from "$stores/identity";
+import { addNotification } from "$lib/Notification/store";
 import privilegedUser from "$lib/Utility/privilegedUser";
-import localforage from "localforage";
+import identity from "$stores/identity";
+import lastPruneTimes from "$stores/lastPruneTimes";
+import locale from "$stores/locale";
+import manga from "$stores/manga";
+import settings from "$stores/settings";
+import ListTitle from "../ListTitle.svelte";
+import CleanMangaList from "./CleanMangaList.svelte";
 
 export let user: AniListAuthorisation = {
 	accessToken: "",
@@ -78,7 +78,7 @@ onMount(async () => {
 			`last${due ? "" : "Completed"}MangaListLength`,
 		)) as number | null;
 
-		if (lastStoredList) lastListSize = parseInt(String(lastStoredList));
+		if (lastStoredList) lastListSize = parseInt(String(lastStoredList), 10);
 	}
 
 	startTime = performance.now();
@@ -143,6 +143,7 @@ const cleanMedia = async (
 	force: boolean,
 ) => {
 	progress = 0;
+	rateLimited = false;
 
 	if (manga && dummy) return manga;
 
@@ -157,7 +158,7 @@ const cleanMedia = async (
 		if ($lastPruneTimes.chapters === 1) {
 			refreshing = true;
 
-			lastPruneTimes.setKey("chapters", new Date().getTime());
+			lastPruneTimes.setKey("chapters", Date.now());
 		} else {
 			const currentDate = new Date();
 
@@ -196,33 +197,29 @@ const cleanMedia = async (
 				($settings.displayNotStarted === true ? 0 : 1),
 	);
 	let finalMedia = releasingMedia;
-	const progressStep = 100 / finalMedia.length / 2;
-	const chapterPromises = finalMedia.map((m: Media) =>
-		database.chapters.get(m.id).then((c) => {
-			if (progress < 100) progress += progressStep;
-
-			if (!due)
-				return new Promise((resolve) => resolve(m.chapters)) as Promise<
-					number | null
-				>;
-
-			if (c !== undefined)
-				return chapterCount($identity, m, $settings.calculateGuessingDisabled);
-			else {
-				// A = On 1 second interval,
-				// B = a maximum of 5 requests per second are allowed.
-				// C = chapterCount makes 3 requests per call.
-				// F = A / (B / C) = 0.6 seconds
-				return new Promise((resolve) => setTimeout(resolve, 600)).then(() =>
-					chapterCount($identity, m, $settings.calculateGuessingDisabled),
-				);
-			}
-		}),
-	);
 	const chapterCounts: (number | null)[] = [];
 
-	for (let i = 0; i < chapterPromises.length; i++) {
-		const count = await chapterPromises[i];
+	if (due && finalMedia.length > 0) {
+		const hydration = await hydrateChapterCounts(
+			$identity,
+			finalMedia,
+			$settings.calculateGuessingDisabled,
+		);
+
+		rateLimited = hydration.rateLimited;
+		progress = 50;
+	}
+
+	for (let i = 0; i < finalMedia.length; i++) {
+		const media = finalMedia[i];
+		const progressStep = finalMedia.length > 0 ? 50 / finalMedia.length : 0;
+		const count = due
+			? await chapterCount(
+					$identity,
+					media,
+					$settings.calculateGuessingDisabled,
+				)
+			: media.chapters;
 
 		if (count === -22) {
 			rateLimited = true;
diff --git a/src/lib/Media/Manga/chapters.ts b/src/lib/Media/Manga/chapters.ts
index e4ad9429..473a3ed4 100644
--- a/src/lib/Media/Manga/chapters.ts
+++ b/src/lib/Media/Manga/chapters.ts
@@ -1,237 +1,262 @@
-import { recentMediaActivities, type Media } from "$lib/Data/AniList/media";
+import { env } from "$env/dynamic/public";
+import { type Media, recentMediaActivities } from "$lib/Data/AniList/media";
 import { getChapterCount } from "$lib/Data/Manga/raw";
-import proxy from "$lib/Utility/proxy";
+import { proxyRoute } from "$lib/Utility/proxy";
 import settings from "$stores/settings";
 import type { UserIdentity } from "../../Data/AniList/identity";
 import { database } from "../../Database/IDB/chapters";
 
-const getManga = async (
-	statusIn: string,
-	year: number,
-	native: string | null,
-	english: string | null,
-	romaji: string | null,
-) => {
-	let status = "";
-	let error = false;
+interface MangaDexChapterCount {
+	chapter: number | null;
+	volumeText?: string | null;
+}
 
-	switch (statusIn) {
-		case "FINISHED":
-			{
-				status = "completed";
-			}
-			break;
-		case "RELEASING":
-			{
-				status = "ongoing";
-			}
-			break;
-		case "HIATUS":
-			{
-				status = "hiatus";
-			}
-			break;
-		case "CANCELLED":
-			{
-				status = "cancelled";
-			}
-			break;
-	}
+interface MangaDexChapterCountsResponse {
+	data?: Record<string, MangaDexChapterCount>;
+	pending?: number[];
+	retryAfterMs?: number;
+}
 
-	const nullIfNullString = (s: string | null) => (s === "null" ? null : s);
-	const get = async (title: string) => {
-		try {
-			return await (
-				await fetch(
-					proxy(
-						`https://api.mangadex.org/manga?title=${encodeURIComponent(
-							title,
-						)}&year=${year}&status[]=${status}`,
-					),
-				)
-			).json();
-		} catch {
-			error = true;
-		}
-	};
+const chapterMemoryCache = new Map<number, number | null>();
+const MAX_PENDING_RETRIES = 2;
+const DEFAULT_PENDING_RETRY_MS = 750;
 
-	let mangadexData = await get(
-		nullIfNullString(native) ||
-			nullIfNullString(english) ||
-			nullIfNullString(romaji) ||
-			"",
-	);
+const browserChapterCacheDisabled = () =>
+	env.PUBLIC_DISABLE_MANGA_BROWSER_CACHE === "true";
 
-	if (error) return new Response("rate-limited");
+const parseOptionalNumber = (value: string | number | null | undefined) => {
+	if (value === null || value === undefined || value === "") return null;
 
-	if (mangadexData["data"] === undefined || mangadexData["data"].length === 0) {
-		mangadexData = await get(nullIfNullString(english) || "");
+	const number = Number.parseFloat(String(value));
 
-		if (
-			mangadexData["data"] === undefined ||
-			mangadexData["data"].length === 0
-		) {
-			mangadexData = await get(nullIfNullString(romaji) || "");
-		}
-	}
+	return Number.isFinite(number) ? number : null;
+};
+
+const sleep = (milliseconds: number) =>
+	new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const readCachedChapterCount = async (mangaId: number) => {
+	if (chapterMemoryCache.has(mangaId)) return chapterMemoryCache.get(mangaId);
+
+	if (browserChapterCacheDisabled()) return undefined;
+
+	const chapter = await database.chapters.get(mangaId);
+
+	return chapter === undefined
+		? undefined
+		: chapter.chapters === -1
+			? null
+			: chapter.chapters;
+};
+
+const writeCachedChapterCount = async (
+	mangaId: number,
+	chapters: number | null,
+	volumes: number | null = null,
+) => {
+	chapterMemoryCache.set(mangaId, chapters);
+
+	if (browserChapterCacheDisabled()) return;
 
-	return Response.json(mangadexData, {
-		headers: {
-			"Cache-Control": "max-age=300",
-		},
+	await database.chapters.put({
+		id: mangaId,
+		chapters: chapters === null ? -1 : chapters,
+		volumes,
 	});
 };
 
-export const chapterCount = async (
+const getActivityChapterCount = async (
 	identity: UserIdentity,
 	manga: Media,
 	disableGuessing: boolean,
-	// preferActivity = false
-): Promise<number | null> => {
-	const chapters = await database.chapters.get(manga.id);
+) => {
+	if (disableGuessing) {
+		await writeCachedChapterCount(manga.id, null);
+
+		return null;
+	}
 
-	if (chapters !== undefined)
-		return chapters.chapters === -1 ? null : chapters.chapters;
+	const anilistData = await recentMediaActivities(
+		identity,
+		manga,
+		settings.get().calculateGuessMethod,
+	);
 
-	// if (preferActivity) {
-	// 	return await recentMediaActivities(identity, manga);
-	// }
+	await writeCachedChapterCount(manga.id, anilistData);
 
-	const tryRecentMediaActivities = async () => {
-		if (disableGuessing) {
-			await database.chapters.put({
-				id: manga.id,
-				chapters: -1,
-				volumes: null,
-			});
+	return anilistData;
+};
 
-			return null;
-		}
+const applyResolvedChapterCount = async (
+	identity: UserIdentity,
+	manga: Media,
+	disableGuessing: boolean,
+	chapter: number,
+	volumeText?: string | null,
+) => {
+	let resolvedChapter = chapter;
 
-		const anilistData = await recentMediaActivities(
+	if (
+		(manga.mediaListEntry || { progress: 0 }).progress > resolvedChapter &&
+		!disableGuessing
+	) {
+		const activityChapterCount = await recentMediaActivities(
 			identity,
 			manga,
 			settings.get().calculateGuessMethod,
 		);
 
-		await database.chapters.put({
-			id: manga.id,
-			chapters: anilistData ? anilistData : -1,
-			volumes: null,
-		});
+		if (activityChapterCount !== null && activityChapterCount > resolvedChapter)
+			resolvedChapter = activityChapterCount;
+	}
+
+	if (resolvedChapter === 0) resolvedChapter = -1;
 
-		return anilistData;
-	};
+	await writeCachedChapterCount(
+		manga.id,
+		resolvedChapter === -1 ? null : resolvedChapter,
+		parseOptionalNumber(volumeText),
+	);
 
-	if (manga.format === "NOVEL") return await tryRecentMediaActivities();
+	return resolvedChapter === -1 ? null : resolvedChapter;
+};
 
-	let lastChapter = 0;
-	let completedVolumes = null;
+const fetchMangaChapterCounts = async (manga: Media[]) => {
+	const data: Record<string, MangaDexChapterCount> = {};
+	let rateLimited = false;
+	let successfulResponse = false;
+	const mangaById = new Map(manga.map((entry) => [entry.id, entry]));
+	let pendingManga = manga;
+	let retryAfterMs = DEFAULT_PENDING_RETRY_MS;
+
+	for (let attempt = 0; attempt <= MAX_PENDING_RETRIES; attempt += 1) {
+		const nextPendingIds = new Set<number>();
+
+		for (let index = 0; index < pendingManga.length; index += 100) {
+			const chunk = pendingManga.slice(index, index + 100);
+			const response = await fetch(proxyRoute("/manga/chapter-counts"), {
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+				},
+				body: JSON.stringify({
+					manga: chunk.map((entry) => ({
+						anilistId: entry.id,
+						status: entry.status,
+						startYear: entry.startDate.year,
+						nativeTitle: entry.title.native,
+						englishTitle: entry.title.english,
+						romajiTitle: entry.title.romaji,
+					})),
+				}),
+			}).catch(() => null);
+
+			if (!response) continue;
+
+			if (!response.ok) {
+				rateLimited = response.status === 429 || response.status === 503;
+
+				continue;
+			}
 
-	if (!settings.get().calculatePreferNativeChapterCount) {
-		const mangadexData = await getManga(
-			manga.status,
-			manga.startDate.year,
-			manga.title.native,
-			manga.title.english,
-			manga.title.romaji,
-		);
+			const payload = (await response.json()) as MangaDexChapterCountsResponse;
+			successfulResponse = true;
+			retryAfterMs = payload.retryAfterMs || retryAfterMs;
 
-		if ((await mangadexData.clone().text()) === "rate-limited") return -22;
-
-		const mangadexDataJson = await mangadexData.json();
-
-		if (
-			mangadexDataJson["data"] === undefined ||
-			mangadexDataJson["data"].length === 0
-		)
-			return await tryRecentMediaActivities();
-
-		const mangadexId = mangadexDataJson["data"][0]["id"];
-		const lastChapterDataJson = await (
-			await fetch(
-				proxy(
-					`https://api.mangadex.org/manga/${mangadexId}/feed?order[chapter]=desc&translatedLanguage[]=en&limit=1&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic`,
-				),
-			)
-		).json();
-
-		if (
-			lastChapterDataJson["data"] === undefined ||
-			lastChapterDataJson["data"].length === 0
-		)
-			return await tryRecentMediaActivities();
-
-		lastChapter = lastChapterDataJson["data"][0]["attributes"]["chapter"];
-		completedVolumes = null;
-
-		if (
-			(manga.mediaListEntry || { progress: 0 }).progress > lastChapter &&
-			!disableGuessing
-		) {
-			const anilistData = await recentMediaActivities(
-				identity,
-				manga,
-				settings.get().calculateGuessMethod,
+			Object.assign(data, payload.data || {});
+
+			for (const id of payload.pending || []) nextPendingIds.add(id);
+		}
+
+		if (!nextPendingIds.size || attempt === MAX_PENDING_RETRIES) break;
+
+		pendingManga = [...nextPendingIds]
+			.map((id) => mangaById.get(id))
+			.filter((entry): entry is Media => entry !== undefined);
+
+		if (!pendingManga.length) break;
+
+		await sleep(retryAfterMs);
+	}
+
+	return { data, rateLimited: rateLimited && !successfulResponse };
+};
+
+export const hydrateChapterCounts = async (
+	identity: UserIdentity,
+	manga: Media[],
+	disableGuessing: boolean,
+): Promise<{ rateLimited: boolean }> => {
+	const uniqueManga = manga.filter(
+		(entry, index, array) =>
+			array.findIndex((candidate) => candidate.id === entry.id) === index,
+	);
+	const unresolvedManga: Media[] = [];
+
+	for (const entry of uniqueManga) {
+		if ((await readCachedChapterCount(entry.id)) !== undefined) continue;
+
+		if (entry.format === "NOVEL") {
+			await getActivityChapterCount(identity, entry, disableGuessing);
+
+			continue;
+		}
+
+		if (settings.get().calculatePreferNativeChapterCount) {
+			const nativeCount = (await getChapterCount(entry.title.native)) || 0;
+
+			await writeCachedChapterCount(
+				entry.id,
+				nativeCount === 0 ? null : nativeCount,
 			);
 
-			if (anilistData !== null && anilistData > lastChapter)
-				lastChapter = anilistData;
+			continue;
 		}
 
-		if (!settings.get().calculateDisableOutOfDateVolumeWarning) {
-			const volumeOfChapterData = await (
-				await fetch(
-					proxy(
-						`https://api.mangadex.org/chapter?manga=${mangadexId}&chapter=${manga.mediaListEntry?.progress}&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic&limit=1`,
-					),
-				)
-			).json();
-			let lastAvailableVolume =
-				lastChapterDataJson["data"][0]["attributes"]["volume"];
-
-			if (lastAvailableVolume === null) {
-				let chapterIndex = 0;
-
-				while (
-					chapterIndex < lastChapterDataJson["data"].length &&
-					lastAvailableVolume === null
-				) {
-					if (
-						lastChapterDataJson["data"][chapterIndex]["attributes"][
-							"volume"
-						] !== null
-					)
-						lastAvailableVolume =
-							lastChapterDataJson["data"][chapterIndex]["attributes"]["volume"];
-
-					chapterIndex += 1;
-				}
-			}
+		unresolvedManga.push(entry);
+	}
 
-			if (
-				volumeOfChapterData["data"] !== undefined &&
-				volumeOfChapterData["data"].length > 0
-			) {
-				const volumeOfChapter =
-					volumeOfChapterData["data"][0]["attributes"]["volume"];
+	if (!unresolvedManga.length) return { rateLimited: false };
 
-				if (volumeOfChapter !== null) completedVolumes = volumeOfChapter;
+	const { data, rateLimited } = await fetchMangaChapterCounts(unresolvedManga);
 
-				if (completedVolumes === volumeOfChapter) completedVolumes -= 1;
-			}
+	for (const entry of unresolvedManga) {
+		const resolved = data[String(entry.id)];
+
+		if (resolved?.chapter !== undefined && resolved.chapter !== null) {
+			await applyResolvedChapterCount(
+				identity,
+				entry,
+				disableGuessing,
+				resolved.chapter,
+				resolved.volumeText,
+			);
+
+			continue;
 		}
-	} else {
-		lastChapter = (await getChapterCount(manga.title.native)) || 0;
+
+		await getActivityChapterCount(identity, entry, disableGuessing);
 	}
 
-	if (lastChapter === 0) lastChapter = -1;
+	return { rateLimited };
+};
 
-	await database.chapters.put({
-		id: manga.id,
-		chapters: Number(lastChapter),
-		volumes: completedVolumes,
-	});
+export const chapterCount = async (
+	identity: UserIdentity,
+	manga: Media,
+	disableGuessing: boolean,
+): Promise<number | null> => {
+	const cachedChapterCount = await readCachedChapterCount(manga.id);
+
+	if (cachedChapterCount !== undefined) return cachedChapterCount;
+
+	const { rateLimited } = await hydrateChapterCounts(
+		identity,
+		[manga],
+		disableGuessing,
+	);
+
+	if (rateLimited) return -22;
 
-	return Number(lastChapter);
+	return (await readCachedChapterCount(manga.id)) ?? null;
 };
diff --git a/src/lib/Utility/proxy.ts b/src/lib/Utility/proxy.ts
index 32f31226..57883836 100644
--- a/src/lib/Utility/proxy.ts
+++ b/src/lib/Utility/proxy.ts
@@ -1,11 +1,24 @@
+import { env } from "$env/dynamic/public";
 import { isLocalApp } from "$lib/Utility/appOrigin";
 
+const DEFAULT_PROXY_ORIGIN = "https://proxy.due.moe";
+
+const normaliseOrigin = (origin: string) =>
+	origin.endsWith("/") ? origin.slice(0, -1) : origin;
+
+export const proxyOrigin = () =>
+	normaliseOrigin(env.PUBLIC_PROXY_ORIGIN || DEFAULT_PROXY_ORIGIN);
+
+export const proxyRoute = (path = "/") =>
+	`${proxyOrigin()}${path.startsWith("/") ? path : `/${path}`}`;
+
 export const proxy = (url: string, disable = false) => {
 	const randomKey = Math.floor(Math.random() * 90) + 10;
+	const forcedProxy = Boolean(env.PUBLIC_PROXY_ORIGIN);
 
-	return isLocalApp() && !disable
+	return isLocalApp() && !disable && !forcedProxy
 		? url
-		: `https://proxy.due.moe/?d2=${btoa(
+		: `${proxyOrigin()}/?d2=${btoa(
 				url
 					.split("")
 					.map((char) => char.charCodeAt(0) + randomKey)
-- 
cgit v1.2.3