aboutsummaryrefslogtreecommitdiff
path: root/src/lib
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib')
-rw-r--r--src/lib/List/Manga/MangaListTemplate.svelte81
-rw-r--r--src/lib/Media/Manga/chapters.ts407
-rw-r--r--src/lib/Utility/proxy.ts17
3 files changed, 270 insertions, 235 deletions
diff --git a/src/lib/List/Manga/MangaListTemplate.svelte b/src/lib/List/Manga/MangaListTemplate.svelte
index eb7ffd83..7333d24e 100644
--- a/src/lib/List/Manga/MangaListTemplate.svelte
+++ b/src/lib/List/Manga/MangaListTemplate.svelte
@@ -1,25 +1,25 @@
<script lang="ts">
-import { mediaListCollection, Type, type Media } from "$lib/Data/AniList/media";
-import type { AniListAuthorisation } from "$lib/Data/AniList/identity";
+import localforage from "localforage";
import { onDestroy, onMount } from "svelte";
-import { chapterCount } from "$lib/Media/Manga/chapters";
-import { pruneAllManga } from "$lib/Media/Manga/cache";
-import manga from "$stores/manga";
+import { browser } from "$app/environment";
+import type { AniListAuthorisation } from "$lib/Data/AniList/identity";
+import { type Media, mediaListCollection, Type } from "$lib/Data/AniList/media";
import { database } from "$lib/Database/IDB/chapters";
-import settings from "$stores/settings";
-import lastPruneTimes from "$stores/lastPruneTimes";
-import ListTitle from "../ListTitle.svelte";
import RateLimitedError from "$lib/Error/RateLimited.svelte";
-import CleanMangaList from "./CleanMangaList.svelte";
+import Skeleton from "$lib/Loading/Skeleton.svelte";
import { incrementMediaProgress } from "$lib/Media/Anime/cache";
-import { addNotification } from "$lib/Notification/store";
+import { pruneAllManga } from "$lib/Media/Manga/cache";
+import { chapterCount, hydrateChapterCounts } from "$lib/Media/Manga/chapters";
import { options } from "$lib/Notification/options";
-import Skeleton from "$lib/Loading/Skeleton.svelte";
-import locale from "$stores/locale";
-import { browser } from "$app/environment";
-import identity from "$stores/identity";
+import { addNotification } from "$lib/Notification/store";
import privilegedUser from "$lib/Utility/privilegedUser";
-import localforage from "localforage";
+import identity from "$stores/identity";
+import lastPruneTimes from "$stores/lastPruneTimes";
+import locale from "$stores/locale";
+import manga from "$stores/manga";
+import settings from "$stores/settings";
+import ListTitle from "../ListTitle.svelte";
+import CleanMangaList from "./CleanMangaList.svelte";
export let user: AniListAuthorisation = {
accessToken: "",
@@ -78,7 +78,7 @@ onMount(async () => {
`last${due ? "" : "Completed"}MangaListLength`,
)) as number | null;
- if (lastStoredList) lastListSize = parseInt(String(lastStoredList));
+ if (lastStoredList) lastListSize = parseInt(String(lastStoredList), 10);
}
startTime = performance.now();
@@ -143,6 +143,7 @@ const cleanMedia = async (
force: boolean,
) => {
progress = 0;
+ rateLimited = false;
if (manga && dummy) return manga;
@@ -157,7 +158,7 @@ const cleanMedia = async (
if ($lastPruneTimes.chapters === 1) {
refreshing = true;
- lastPruneTimes.setKey("chapters", new Date().getTime());
+ lastPruneTimes.setKey("chapters", Date.now());
} else {
const currentDate = new Date();
@@ -196,33 +197,29 @@ const cleanMedia = async (
($settings.displayNotStarted === true ? 0 : 1),
);
let finalMedia = releasingMedia;
- const progressStep = 100 / finalMedia.length / 2;
- const chapterPromises = finalMedia.map((m: Media) =>
- database.chapters.get(m.id).then((c) => {
- if (progress < 100) progress += progressStep;
-
- if (!due)
- return new Promise((resolve) => resolve(m.chapters)) as Promise<
- number | null
- >;
-
- if (c !== undefined)
- return chapterCount($identity, m, $settings.calculateGuessingDisabled);
- else {
- // A = On 1 second interval,
- // B = a maximum of 5 requests per second are allowed.
- // C = chapterCount makes 3 requests per call.
- // F = A / (B / C) = 0.6 seconds
- return new Promise((resolve) => setTimeout(resolve, 600)).then(() =>
- chapterCount($identity, m, $settings.calculateGuessingDisabled),
- );
- }
- }),
- );
const chapterCounts: (number | null)[] = [];
- for (let i = 0; i < chapterPromises.length; i++) {
- const count = await chapterPromises[i];
+ if (due && finalMedia.length > 0) {
+ const hydration = await hydrateChapterCounts(
+ $identity,
+ finalMedia,
+ $settings.calculateGuessingDisabled,
+ );
+
+ rateLimited = hydration.rateLimited;
+ progress = 50;
+ }
+
+ for (let i = 0; i < finalMedia.length; i++) {
+ const media = finalMedia[i];
+ const progressStep = finalMedia.length > 0 ? 50 / finalMedia.length : 0;
+ const count = due
+ ? await chapterCount(
+ $identity,
+ media,
+ $settings.calculateGuessingDisabled,
+ )
+ : media.chapters;
if (count === -22) {
rateLimited = true;
diff --git a/src/lib/Media/Manga/chapters.ts b/src/lib/Media/Manga/chapters.ts
index e4ad9429..473a3ed4 100644
--- a/src/lib/Media/Manga/chapters.ts
+++ b/src/lib/Media/Manga/chapters.ts
@@ -1,237 +1,262 @@
-import { recentMediaActivities, type Media } from "$lib/Data/AniList/media";
+import { env } from "$env/dynamic/public";
+import { type Media, recentMediaActivities } from "$lib/Data/AniList/media";
import { getChapterCount } from "$lib/Data/Manga/raw";
-import proxy from "$lib/Utility/proxy";
+import { proxyRoute } from "$lib/Utility/proxy";
import settings from "$stores/settings";
import type { UserIdentity } from "../../Data/AniList/identity";
import { database } from "../../Database/IDB/chapters";
-const getManga = async (
- statusIn: string,
- year: number,
- native: string | null,
- english: string | null,
- romaji: string | null,
-) => {
- let status = "";
- let error = false;
+interface MangaDexChapterCount {
+ chapter: number | null;
+ volumeText?: string | null;
+}
- switch (statusIn) {
- case "FINISHED":
- {
- status = "completed";
- }
- break;
- case "RELEASING":
- {
- status = "ongoing";
- }
- break;
- case "HIATUS":
- {
- status = "hiatus";
- }
- break;
- case "CANCELLED":
- {
- status = "cancelled";
- }
- break;
- }
+interface MangaDexChapterCountsResponse {
+ data?: Record<string, MangaDexChapterCount>;
+ pending?: number[];
+ retryAfterMs?: number;
+}
- const nullIfNullString = (s: string | null) => (s === "null" ? null : s);
- const get = async (title: string) => {
- try {
- return await (
- await fetch(
- proxy(
- `https://api.mangadex.org/manga?title=${encodeURIComponent(
- title,
- )}&year=${year}&status[]=${status}`,
- ),
- )
- ).json();
- } catch {
- error = true;
- }
- };
+const chapterMemoryCache = new Map<number, number | null>();
+const MAX_PENDING_RETRIES = 2;
+const DEFAULT_PENDING_RETRY_MS = 750;
- let mangadexData = await get(
- nullIfNullString(native) ||
- nullIfNullString(english) ||
- nullIfNullString(romaji) ||
- "",
- );
+const browserChapterCacheDisabled = () =>
+ env.PUBLIC_DISABLE_MANGA_BROWSER_CACHE === "true";
- if (error) return new Response("rate-limited");
+const parseOptionalNumber = (value: string | number | null | undefined) => {
+ if (value === null || value === undefined || value === "") return null;
- if (mangadexData["data"] === undefined || mangadexData["data"].length === 0) {
- mangadexData = await get(nullIfNullString(english) || "");
+ const number = Number.parseFloat(String(value));
- if (
- mangadexData["data"] === undefined ||
- mangadexData["data"].length === 0
- ) {
- mangadexData = await get(nullIfNullString(romaji) || "");
- }
- }
+ return Number.isFinite(number) ? number : null;
+};
+
+const sleep = (milliseconds: number) =>
+ new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+const readCachedChapterCount = async (mangaId: number) => {
+ if (chapterMemoryCache.has(mangaId)) return chapterMemoryCache.get(mangaId);
+
+ if (browserChapterCacheDisabled()) return undefined;
+
+ const chapter = await database.chapters.get(mangaId);
+
+ return chapter === undefined
+ ? undefined
+ : chapter.chapters === -1
+ ? null
+ : chapter.chapters;
+};
+
+const writeCachedChapterCount = async (
+ mangaId: number,
+ chapters: number | null,
+ volumes: number | null = null,
+) => {
+ chapterMemoryCache.set(mangaId, chapters);
+
+ if (browserChapterCacheDisabled()) return;
- return Response.json(mangadexData, {
- headers: {
- "Cache-Control": "max-age=300",
- },
+ await database.chapters.put({
+ id: mangaId,
+ chapters: chapters === null ? -1 : chapters,
+ volumes,
});
};
-export const chapterCount = async (
+const getActivityChapterCount = async (
identity: UserIdentity,
manga: Media,
disableGuessing: boolean,
- // preferActivity = false
-): Promise<number | null> => {
- const chapters = await database.chapters.get(manga.id);
+) => {
+ if (disableGuessing) {
+ await writeCachedChapterCount(manga.id, null);
+
+ return null;
+ }
- if (chapters !== undefined)
- return chapters.chapters === -1 ? null : chapters.chapters;
+ const anilistData = await recentMediaActivities(
+ identity,
+ manga,
+ settings.get().calculateGuessMethod,
+ );
- // if (preferActivity) {
- // return await recentMediaActivities(identity, manga);
- // }
+ await writeCachedChapterCount(manga.id, anilistData);
- const tryRecentMediaActivities = async () => {
- if (disableGuessing) {
- await database.chapters.put({
- id: manga.id,
- chapters: -1,
- volumes: null,
- });
+ return anilistData;
+};
- return null;
- }
+const applyResolvedChapterCount = async (
+ identity: UserIdentity,
+ manga: Media,
+ disableGuessing: boolean,
+ chapter: number,
+ volumeText?: string | null,
+) => {
+ let resolvedChapter = chapter;
- const anilistData = await recentMediaActivities(
+ if (
+ (manga.mediaListEntry || { progress: 0 }).progress > resolvedChapter &&
+ !disableGuessing
+ ) {
+ const activityChapterCount = await recentMediaActivities(
identity,
manga,
settings.get().calculateGuessMethod,
);
- await database.chapters.put({
- id: manga.id,
- chapters: anilistData ? anilistData : -1,
- volumes: null,
- });
+ if (activityChapterCount !== null && activityChapterCount > resolvedChapter)
+ resolvedChapter = activityChapterCount;
+ }
+
+ if (resolvedChapter === 0) resolvedChapter = -1;
- return anilistData;
- };
+ await writeCachedChapterCount(
+ manga.id,
+ resolvedChapter === -1 ? null : resolvedChapter,
+ parseOptionalNumber(volumeText),
+ );
- if (manga.format === "NOVEL") return await tryRecentMediaActivities();
+ return resolvedChapter === -1 ? null : resolvedChapter;
+};
- let lastChapter = 0;
- let completedVolumes = null;
+const fetchMangaChapterCounts = async (manga: Media[]) => {
+ const data: Record<string, MangaDexChapterCount> = {};
+ let rateLimited = false;
+ let successfulResponse = false;
+ const mangaById = new Map(manga.map((entry) => [entry.id, entry]));
+ let pendingManga = manga;
+ let retryAfterMs = DEFAULT_PENDING_RETRY_MS;
+
+ for (let attempt = 0; attempt <= MAX_PENDING_RETRIES; attempt += 1) {
+ const nextPendingIds = new Set<number>();
+
+ for (let index = 0; index < pendingManga.length; index += 100) {
+ const chunk = pendingManga.slice(index, index + 100);
+ const response = await fetch(proxyRoute("/manga/chapter-counts"), {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ manga: chunk.map((entry) => ({
+ anilistId: entry.id,
+ status: entry.status,
+ startYear: entry.startDate.year,
+ nativeTitle: entry.title.native,
+ englishTitle: entry.title.english,
+ romajiTitle: entry.title.romaji,
+ })),
+ }),
+ }).catch(() => null);
+
+ if (!response) continue;
+
+ if (!response.ok) {
+ rateLimited = response.status === 429 || response.status === 503;
+
+ continue;
+ }
- if (!settings.get().calculatePreferNativeChapterCount) {
- const mangadexData = await getManga(
- manga.status,
- manga.startDate.year,
- manga.title.native,
- manga.title.english,
- manga.title.romaji,
- );
+ const payload = (await response.json()) as MangaDexChapterCountsResponse;
+ successfulResponse = true;
+ retryAfterMs = payload.retryAfterMs || retryAfterMs;
- if ((await mangadexData.clone().text()) === "rate-limited") return -22;
-
- const mangadexDataJson = await mangadexData.json();
-
- if (
- mangadexDataJson["data"] === undefined ||
- mangadexDataJson["data"].length === 0
- )
- return await tryRecentMediaActivities();
-
- const mangadexId = mangadexDataJson["data"][0]["id"];
- const lastChapterDataJson = await (
- await fetch(
- proxy(
- `https://api.mangadex.org/manga/${mangadexId}/feed?order[chapter]=desc&translatedLanguage[]=en&limit=1&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic`,
- ),
- )
- ).json();
-
- if (
- lastChapterDataJson["data"] === undefined ||
- lastChapterDataJson["data"].length === 0
- )
- return await tryRecentMediaActivities();
-
- lastChapter = lastChapterDataJson["data"][0]["attributes"]["chapter"];
- completedVolumes = null;
-
- if (
- (manga.mediaListEntry || { progress: 0 }).progress > lastChapter &&
- !disableGuessing
- ) {
- const anilistData = await recentMediaActivities(
- identity,
- manga,
- settings.get().calculateGuessMethod,
+ Object.assign(data, payload.data || {});
+
+ for (const id of payload.pending || []) nextPendingIds.add(id);
+ }
+
+ if (!nextPendingIds.size || attempt === MAX_PENDING_RETRIES) break;
+
+ pendingManga = [...nextPendingIds]
+ .map((id) => mangaById.get(id))
+ .filter((entry): entry is Media => entry !== undefined);
+
+ if (!pendingManga.length) break;
+
+ await sleep(retryAfterMs);
+ }
+
+ return { data, rateLimited: rateLimited && !successfulResponse };
+};
+
+export const hydrateChapterCounts = async (
+ identity: UserIdentity,
+ manga: Media[],
+ disableGuessing: boolean,
+): Promise<{ rateLimited: boolean }> => {
+ const uniqueManga = manga.filter(
+ (entry, index, array) =>
+ array.findIndex((candidate) => candidate.id === entry.id) === index,
+ );
+ const unresolvedManga: Media[] = [];
+
+ for (const entry of uniqueManga) {
+ if ((await readCachedChapterCount(entry.id)) !== undefined) continue;
+
+ if (entry.format === "NOVEL") {
+ await getActivityChapterCount(identity, entry, disableGuessing);
+
+ continue;
+ }
+
+ if (settings.get().calculatePreferNativeChapterCount) {
+ const nativeCount = (await getChapterCount(entry.title.native)) || 0;
+
+ await writeCachedChapterCount(
+ entry.id,
+ nativeCount === 0 ? null : nativeCount,
);
- if (anilistData !== null && anilistData > lastChapter)
- lastChapter = anilistData;
+ continue;
}
- if (!settings.get().calculateDisableOutOfDateVolumeWarning) {
- const volumeOfChapterData = await (
- await fetch(
- proxy(
- `https://api.mangadex.org/chapter?manga=${mangadexId}&chapter=${manga.mediaListEntry?.progress}&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic&limit=1`,
- ),
- )
- ).json();
- let lastAvailableVolume =
- lastChapterDataJson["data"][0]["attributes"]["volume"];
-
- if (lastAvailableVolume === null) {
- let chapterIndex = 0;
-
- while (
- chapterIndex < lastChapterDataJson["data"].length &&
- lastAvailableVolume === null
- ) {
- if (
- lastChapterDataJson["data"][chapterIndex]["attributes"][
- "volume"
- ] !== null
- )
- lastAvailableVolume =
- lastChapterDataJson["data"][chapterIndex]["attributes"]["volume"];
-
- chapterIndex += 1;
- }
- }
+ unresolvedManga.push(entry);
+ }
- if (
- volumeOfChapterData["data"] !== undefined &&
- volumeOfChapterData["data"].length > 0
- ) {
- const volumeOfChapter =
- volumeOfChapterData["data"][0]["attributes"]["volume"];
+ if (!unresolvedManga.length) return { rateLimited: false };
- if (volumeOfChapter !== null) completedVolumes = volumeOfChapter;
+ const { data, rateLimited } = await fetchMangaChapterCounts(unresolvedManga);
- if (completedVolumes === volumeOfChapter) completedVolumes -= 1;
- }
+ for (const entry of unresolvedManga) {
+ const resolved = data[String(entry.id)];
+
+ if (resolved?.chapter !== undefined && resolved.chapter !== null) {
+ await applyResolvedChapterCount(
+ identity,
+ entry,
+ disableGuessing,
+ resolved.chapter,
+ resolved.volumeText,
+ );
+
+ continue;
}
- } else {
- lastChapter = (await getChapterCount(manga.title.native)) || 0;
+
+ await getActivityChapterCount(identity, entry, disableGuessing);
}
- if (lastChapter === 0) lastChapter = -1;
+ return { rateLimited };
+};
- await database.chapters.put({
- id: manga.id,
- chapters: Number(lastChapter),
- volumes: completedVolumes,
- });
+export const chapterCount = async (
+ identity: UserIdentity,
+ manga: Media,
+ disableGuessing: boolean,
+): Promise<number | null> => {
+ const cachedChapterCount = await readCachedChapterCount(manga.id);
+
+ if (cachedChapterCount !== undefined) return cachedChapterCount;
+
+ const { rateLimited } = await hydrateChapterCounts(
+ identity,
+ [manga],
+ disableGuessing,
+ );
+
+ if (rateLimited) return -22;
- return Number(lastChapter);
+ return (await readCachedChapterCount(manga.id)) ?? null;
};
diff --git a/src/lib/Utility/proxy.ts b/src/lib/Utility/proxy.ts
index 32f31226..57883836 100644
--- a/src/lib/Utility/proxy.ts
+++ b/src/lib/Utility/proxy.ts
@@ -1,11 +1,24 @@
+import { env } from "$env/dynamic/public";
import { isLocalApp } from "$lib/Utility/appOrigin";
+const DEFAULT_PROXY_ORIGIN = "https://proxy.due.moe";
+
+const normaliseOrigin = (origin: string) =>
+ origin.endsWith("/") ? origin.slice(0, -1) : origin;
+
+export const proxyOrigin = () =>
+ normaliseOrigin(env.PUBLIC_PROXY_ORIGIN || DEFAULT_PROXY_ORIGIN);
+
+export const proxyRoute = (path = "/") =>
+ `${proxyOrigin()}${path.startsWith("/") ? path : `/${path}`}`;
+
export const proxy = (url: string, disable = false) => {
const randomKey = Math.floor(Math.random() * 90) + 10;
+ const forcedProxy = Boolean(env.PUBLIC_PROXY_ORIGIN);
- return isLocalApp() && !disable
+ return isLocalApp() && !disable && !forcedProxy
? url
- : `https://proxy.due.moe/?d2=${btoa(
+ : `${proxyOrigin()}/?d2=${btoa(
url
.split("")
.map((char) => char.charCodeAt(0) + randomKey)
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-11 23:19:46 +0000