1 files changed, 392 insertions, 0 deletions
diff --git a/apps/proxy/src/mangadex.js b/apps/proxy/src/mangadex.js
new file mode 100644
index 00000000..d2a42154
--- /dev/null
+++ b/apps/proxy/src/mangadex.js
@@ -0,0 +1,392 @@
+import {
+	getMangadexRowsByMangadexIds,
+	getSyncState,
+	upsertMangadexRows,
+	upsertSyncState,
+} from "./supabase.js";
+
+const MANGADEX_API_ORIGIN = "https://api.mangadex.org";
+const SYNC_STATE_NAME = "chapter_en";
+const ALL_CONTENT_RATINGS = ["safe", "suggestive", "erotica", "pornographic"];
+
+const sleep = (milliseconds) =>
+	new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const appendQueryValues = (url, query) => {
+	for (const [key, rawValue] of Object.entries(query)) {
+		if (rawValue === undefined || rawValue === null) continue;
+
+		if (Array.isArray(rawValue)) {
+			for (const value of rawValue) url.searchParams.append(key, String(value));
+
+			continue;
+		}
+
+		url.searchParams.set(key, String(rawValue));
+	}
+};
+
+const mangadexUserAgent = (env) =>
+	env?.MANGADEX_USER_AGENT ||
+	"Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0";
+
+const mangadexRequest = async (
+	env,
+	path,
+	{ query = {}, retries = 1, retryDelay = 1500 } = {},
+) => {
+	const url = new URL(path, MANGADEX_API_ORIGIN);
+
+	appendQueryValues(url, query);
+
+	const response = await fetch(url, {
+		headers: {
+			Accept: "application/json",
+			"User-Agent": mangadexUserAgent(env),
+		},
+	});
+
+	if (response.status === 429 && retries > 0) {
+		const retryAfterHeader =
+			response.headers.get("Retry-After") ||
+			response.headers.get("X-RateLimit-Retry-After");
+		const retryAfterSeconds = Number.parseInt(retryAfterHeader || "", 10);
+		const retryAt =
+			Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 10_000
+				? retryAfterSeconds * 1000 - Date.now()
+				: retryAfterSeconds * 1000;
+
+		await sleep(
+			Number.isFinite(retryAt) && retryAt > 0
+				? Math.min(retryAt, 30_000)
+				: retryDelay,
+		);
+
+		return mangadexRequest(env, path, {
+			query,
+			retries: retries - 1,
+			retryDelay: retryDelay * 2,
+		});
+	}
+
+	if (!response.ok) {
+		const message = await response.text();
+
+		throw new Error(`MangaDex request failed (${response.status}): ${message}`);
+	}
+
+	return response.json();
+};
+
+const statusMap = {
+	FINISHED: "completed",
+	RELEASING: "ongoing",
+	HIATUS: "hiatus",
+	CANCELLED: "cancelled",
+};
+
+const normaliseTitle = (value) =>
+	String(value || "")
+		.trim()
+		.toLowerCase();
+
+const parseOptionalNumber = (value) => {
+	if (value === null || value === undefined || value === "") return null;
+
+	const number = Number.parseFloat(String(value));
+
+	return Number.isFinite(number) ? number : null;
+};
+
+const mangadexDateTime = (isoString) => {
+	const date = new Date(isoString);
+
+	if (Number.isNaN(date.getTime())) return "1970-01-01T00:00:00";
+
+	return date.toISOString().slice(0, 19);
+};
+
+const scoreSearchResult = (manga, title) => {
+	const attributes = manga.attributes || {};
+	const titleMap = attributes.title || {};
+	const altTitles = (attributes.altTitles || []).flatMap((item) =>
+		Object.values(item || {}),
+	);
+	const titles = [...Object.values(titleMap || {}), ...altTitles]
+		.map(normaliseTitle)
+		.filter(Boolean);
+	const candidate = normaliseTitle(title);
+
+	if (!candidate.length) return Number.MAX_SAFE_INTEGER;
+	if (titles.includes(candidate)) return 0;
+	if (titles.some((item) => item.includes(candidate))) return 1;
+	return 2;
+};
+
+const pickBestSearchResult = (results, title) =>
+	[...results].sort(
+		(left, right) =>
+			scoreSearchResult(left, title) - scoreSearchResult(right, title),
+	)[0];
+
+const titleCandidates = (manga) =>
+	[
+		manga.nativeTitle,
+		manga.englishTitle,
+		manga.romajiTitle,
+		manga.nativeTitle === "null" ? null : manga.nativeTitle,
+		manga.englishTitle === "null" ? null : manga.englishTitle,
+		manga.romajiTitle === "null" ? null : manga.romajiTitle,
+	]
+		.filter(Boolean)
+		.map((title) => String(title).trim())
+		.filter((title, index, array) => array.indexOf(title) === index);
+
+const searchQueries = (manga, title) => {
+	const status = statusMap[manga.status];
+
+	return [
+		{
+			title,
+			...(manga.startYear ? { year: manga.startYear } : {}),
+			...(status ? { "status[]": [status] } : {}),
+			limit: 5,
+		},
+		{
+			title,
+			...(status ? { "status[]": [status] } : {}),
+			limit: 5,
+		},
+		{
+			title,
+			...(manga.startYear ? { year: manga.startYear } : {}),
+			limit: 5,
+		},
+		{
+			title,
+			limit: 5,
+		},
+	];
+};
+
+const chooseLatestAggregateChapter = (volumes) => {
+	let latest = null;
+
+	for (const [volumeKey, volumeEntry] of Object.entries(volumes || {})) {
+		const chapters = volumeEntry?.chapters || {};
+
+		for (const chapterEntry of Object.values(chapters)) {
+			const chapterText = chapterEntry?.chapter || null;
+			const chapterNumber = parseOptionalNumber(chapterText);
+			const volumeText =
+				chapterEntry?.volume || volumeEntry?.volume || volumeKey || null;
+			const volumeNumber = parseOptionalNumber(volumeText);
+			const candidate = {
+				chapterText,
+				chapterNumber,
+				volumeText,
+				volumeNumber,
+				chapterId: chapterEntry?.id || null,
+			};
+
+			if (!latest) {
+				latest = candidate;
+
+				continue;
+			}
+
+			if (
+				candidate.chapterNumber !== null &&
+				(latest.chapterNumber === null ||
+					candidate.chapterNumber > latest.chapterNumber ||
+					(candidate.chapterNumber === latest.chapterNumber &&
+						(candidate.volumeNumber || -1) > (latest.volumeNumber || -1)))
+			) {
+				latest = candidate;
+
+				continue;
+			}
+
+			if (
+				candidate.chapterNumber === null &&
+				latest.chapterNumber === null &&
+				String(candidate.chapterText || "").localeCompare(
+					String(latest.chapterText || ""),
+					undefined,
+					{ numeric: true },
+				) > 0
+			)
+				latest = candidate;
+		}
+	}
+
+	return latest;
+};
+
+const buildIndexedRow = (manga, mangadexId, aggregateResponse) => {
+	const now = new Date().toISOString();
+	const latest = chooseLatestAggregateChapter(aggregateResponse?.volumes);
+
+	return {
+		anilist_id: manga.anilistId,
+		mangadex_id: mangadexId,
+		latest_en_chapter_text: latest?.chapterText || null,
+		latest_en_chapter_number: latest?.chapterNumber || null,
+		latest_en_volume_text: latest?.volumeText || null,
+		latest_en_chapter_id: latest?.chapterId || null,
+		latest_chapter_updated_at: now,
+		last_seen_at: now,
+		last_indexed_at: now,
+		is_releasing: manga.status === "RELEASING",
+		created_at: manga.createdAt || now,
+		updated_at: now,
+	};
+};
+
+export const resolveMangadexId = async (env, manga) => {
+	const candidates = titleCandidates(manga);
+
+	for (const title of candidates) {
+		for (const query of searchQueries(manga, title)) {
+			const payload = await mangadexRequest(env, "/manga", { query });
+			const data = payload?.data || [];
+
+			if (!data.length) continue;
+
+			return pickBestSearchResult(data, title)?.id || data[0]?.id || null;
+		}
+	}
+
+	return null;
+};
+
+export const bootstrapManga = async (env, manga) => {
+	const mangadexId = await resolveMangadexId(env, manga);
+
+	if (!mangadexId) return null;
+
+	const aggregateResponse = await mangadexRequest(
+		env,
+		`/manga/${mangadexId}/aggregate`,
+		{
+			query: {
+				"translatedLanguage[]": ["en"],
+			},
+		},
+	);
+
+	return buildIndexedRow(manga, mangadexId, aggregateResponse);
+};
+
+const extractMangaIdFromChapter = (chapter) =>
+	chapter?.relationships?.find((relationship) => relationship.type === "manga")
+		?.id || null;
+
+const currentCursor = () => new Date().toISOString();
+
+const rewindCursor = (isoString, seconds = 30) =>
+	new Date(
+		Math.max(new Date(isoString).getTime() - seconds * 1000, 0),
+	).toISOString();
+
+export const syncMangadexIndex = async (
+	env,
+	{ maxPages = 5, aggregateDelayMs = 250 } = {},
+) => {
+	const syncState = (await getSyncState(env, SYNC_STATE_NAME)) || {
+		cursor_updated_at: currentCursor(),
+	};
+	const trackedMangadexIds = new Set();
+	const cursor = syncState.cursor_updated_at;
+	let maxUpdatedAt = cursor;
+	let offset = 0;
+	let pageCount = 0;
+	let scannedChapterCount = 0;
+
+	while (pageCount < maxPages) {
+		const payload = await mangadexRequest(env, "/chapter", {
+			query: {
+				"translatedLanguage[]": ["en"],
+				"contentRating[]": ALL_CONTENT_RATINGS,
+				includeFutureUpdates: 0,
+				includeFuturePublishAt: 0,
+				updatedAtSince: mangadexDateTime(cursor),
+				"order[updatedAt]": "asc",
+				limit: 100,
+				offset,
+			},
+		});
+		const chapters = payload?.data || [];
+
+		if (!chapters.length) break;
+
+		for (const chapter of chapters) {
+			scannedChapterCount += 1;
+
+			const mangaId = extractMangaIdFromChapter(chapter);
+			const updatedAt = chapter?.attributes?.updatedAt;
+
+			if (mangaId) trackedMangadexIds.add(mangaId);
+			if (updatedAt && updatedAt > maxUpdatedAt) maxUpdatedAt = updatedAt;
+		}
+
+		pageCount += 1;
+
+		if (chapters.length < 100) break;
+
+		offset += chapters.length;
+	}
+
+	if (!trackedMangadexIds.size) {
+		await upsertSyncState(env, SYNC_STATE_NAME, maxUpdatedAt);
+
+		return {
+			cursor: maxUpdatedAt,
+			pageCount,
+			scannedChapterCount,
+			refreshedCount: 0,
+		};
+	}
+
+	const trackedRows = await getMangadexRowsByMangadexIds(env, [
+		...trackedMangadexIds,
+	]);
+	const refreshedRows = [];
+
+	for (const row of trackedRows) {
+		const aggregateResponse = await mangadexRequest(
+			env,
+			`/manga/${row.mangadex_id}/aggregate`,
+			{
+				query: {
+					"translatedLanguage[]": ["en"],
+				},
+			},
+		);
+
+		refreshedRows.push(
+			buildIndexedRow(
+				{
+					anilistId: row.anilist_id,
+					status: row.is_releasing ? "RELEASING" : "FINISHED",
+					createdAt: row.created_at,
+				},
+				row.mangadex_id,
+				aggregateResponse,
+			),
+		);
+
+		if (aggregateDelayMs > 0) await sleep(aggregateDelayMs);
+	}
+
+	if (refreshedRows.length) await upsertMangadexRows(env, refreshedRows);
+
+	await upsertSyncState(env, SYNC_STATE_NAME, rewindCursor(maxUpdatedAt));
+
+	return {
+		cursor: rewindCursor(maxUpdatedAt),
+		pageCount,
+		scannedChapterCount,
+		refreshedCount: refreshedRows.length,
+	};
+};