diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | apps/proxy/package.json | 25 | ||||
| -rw-r--r-- | apps/proxy/src/index.js | 390 | ||||
| -rw-r--r-- | apps/proxy/src/mangadex.js | 392 | ||||
| -rw-r--r-- | apps/proxy/src/supabase.js | 217 | ||||
| -rw-r--r-- | apps/proxy/wrangler.toml | 3 | ||||
| -rw-r--r-- | src/lib/List/Manga/MangaListTemplate.svelte | 81 | ||||
| -rw-r--r-- | src/lib/Media/Manga/chapters.ts | 407 | ||||
| -rw-r--r-- | src/lib/Utility/proxy.ts | 17 | ||||
| -rw-r--r-- | supabase/mangadex.sql | 39 |
10 files changed, 1236 insertions, 336 deletions
@@ -1,5 +1,6 @@ # Misc. .DS_Store +*.har # Node.js /node_modules diff --git a/apps/proxy/package.json b/apps/proxy/package.json index 62c13765..91ee6d7b 100644 --- a/apps/proxy/package.json +++ b/apps/proxy/package.json @@ -1,13 +1,14 @@ { - "name": "due-proxy", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev", - "start": "wrangler dev" - }, - "devDependencies": { - "wrangler": "^3.0.0" - } -}
\ No newline at end of file + "name": "due-proxy", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "deploy": "wrangler deploy", + "dev": "wrangler dev", + "start": "wrangler dev" + }, + "devDependencies": { + "wrangler": "^3.0.0" + } +} diff --git a/apps/proxy/src/index.js b/apps/proxy/src/index.js index f2a37111..f90adfa7 100644 --- a/apps/proxy/src/index.js +++ b/apps/proxy/src/index.js @@ -1,113 +1,325 @@ -const handleRequest = async (request) => { +import { bootstrapManga, syncMangadexIndex } from "./mangadex.js"; +import { + deleteMangadexFailureRows, + getMangadexFailureRowsByAniListIds, + getMangadexRowsByAniListIds, + hasSupabaseConfig, + upsertMangadexRows, + upsertMangadexFailureRows, +} from "./supabase.js"; + +const DEFAULT_ALLOWED_ORIGIN = "https://due.moe"; +const DEFAULT_BOOTSTRAP_RETRY_MINUTES = 360; +const DEFAULT_PENDING_RETRY_MS = 750; +const bootstrapInFlight = new Map(); + +const isPrivateHostname = (hostname) => + hostname === "localhost" || + hostname === "127.0.0.1" || + hostname.endsWith(".local") || + /^10\./.test(hostname) || + /^192\.168\./.test(hostname) || + /^172\.(1[6-9]|2\d|3[0-1])\./.test(hostname); + +const accessControlOrigin = (request) => { + const origin = request.headers.get("Origin"); + + if (!origin) return DEFAULT_ALLOWED_ORIGIN; + try { - const url = new URL(request.url); - let query; - let dropHeaders = false; + const url = new URL(origin); - if (url.search.includes('&dh')) { - url.search = url.search.replace('&dh', ''); - dropHeaders = true; - } + if ( + url.hostname === "due.moe" || + url.hostname.endsWith(".due.moe") || + isPrivateHostname(url.hostname) + ) + return origin; + } catch {} - if (url.search.includes('?q=')) { - query = url.search.split('?q=')[1]; - } else if (url.search.includes('?d=')) { - query = atob(url.search.split('?d=')[1]); - } else if (url.search.includes('?d2=')) { - const fullEncodedURL = url.search.split('?d2=')[1]; - const key = parseInt(fullEncodedURL.slice(-2)); - - query = atob(fullEncodedURL.slice(0, -2)) - .split(':') - .map((char) => String.fromCharCode(char - key)) - .join(''); - } else { - return new Response(null, { - status: 400, - statusText: 'Bad Request', - }); - } + return DEFAULT_ALLOWED_ORIGIN; +}; - request = new Request(query, request); +const appendCorsHeaders = (request, headers = new Headers()) => { + headers.set("Access-Control-Allow-Origin", accessControlOrigin(request)); + headers.set("Access-Control-Allow-Methods", "GET, HEAD, POST, OPTIONS"); + headers.set("Access-Control-Allow-Headers", "Authorization, Content-Type"); + headers.append("Vary", "Origin"); - request.headers.set('Host', new URL(query).origin); - request.headers.set('Referrer', new URL(query)); - request.headers.set('Origin', new URL(query)); - request.headers.set('Access-Control-Allow-Credentials', 'true'); - request.headers.delete('X-Content-Type-Options'); + return headers; +}; - let response = await fetch(request); +const jsonResponse = (request, body, init = {}) => { + const headers = appendCorsHeaders(request, new Headers(init.headers)); - response = new Response(response.body, response); + headers.set("Content-Type", "application/json"); - if (dropHeaders) response.headers.forEach((_, key) => response.headers.delete(key)); + return new Response(JSON.stringify(body), { + ...init, + headers, + }); +}; - response.headers.set('Access-Control-Allow-Origin', 'https://due.moe'); - response.headers.append('Vary', 'Origin'); - response.headers.set('Cache-Control', 'max-age=300'); +const textResponse = (request, body, init = {}) => + new Response(body, { + ...init, + headers: appendCorsHeaders(request, new Headers(init.headers)), + }); - return response; - } catch { - return new Response(null, { - status: 400, - statusText: 'Bad Request', - }); +const decodeProxyTarget = (url) => { + if (url.search.includes("?q=")) return url.search.split("?q=")[1]; + if (url.search.includes("?d=")) return atob(url.search.split("?d=")[1]); + if (url.search.includes("?d2=")) { + const fullEncodedUrl = url.search.split("?d2=")[1]; + const key = Number.parseInt(fullEncodedUrl.slice(-2), 10); + + return atob(fullEncodedUrl.slice(0, -2)) + .split(":") + .map((char) => String.fromCharCode(Number(char) - key)) + .join(""); } + + return null; }; -const handleOptions = async (request) => { - if ( - request.headers.get('Origin') !== null && - request.headers.get('Access-Control-Request-Method') !== null && - request.headers.get('Access-Control-Request-Headers') !== null - ) { - return new Response(null, { - headers: { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': 'GET, HEAD, POST, OPTIONS', - 'Access-Control-Allow-Headers': '*', - }, +const forwardProxyRequest = async (request) => { + const url = new URL(request.url); + const dropHeaders = url.search.includes("&dh"); + + if (dropHeaders) url.search = url.search.replace("&dh", ""); + + const target = decodeProxyTarget(url); + + if (!target) + return textResponse(request, null, { + status: 400, + statusText: "Bad Request", }); - } else { - return new Response(null, { - headers: { - Allow: 'GET, HEAD, POST, OPTIONS', + + const targetUrl = new URL(target); + const proxiedRequest = new Request(target, request); + + proxiedRequest.headers.set("Host", targetUrl.hostname); + proxiedRequest.headers.set("Referrer", targetUrl.toString()); + proxiedRequest.headers.set("Origin", targetUrl.origin); + proxiedRequest.headers.delete("X-Content-Type-Options"); + + let response = await fetch(proxiedRequest); + + response = new Response(response.body, response); + + if (dropHeaders) + for (const key of [...response.headers.keys()]) + response.headers.delete(key); + + appendCorsHeaders(request, response.headers); + response.headers.set("Cache-Control", "max-age=300"); + + return response; +}; + +const handleOptions = (request) => + new Response(null, { + headers: appendCorsHeaders(request), + }); + +const isMangadexIdConstraintConflict = (error) => + error instanceof Error && + error.message.includes("mangadex_manga_index_mangadex_id_key"); + +const parseMangaPayload = async (request) => { + const body = await request.json().catch(() => null); + const manga = Array.isArray(body?.manga) ? body.manga : []; + + return manga + .map((entry) => ({ + anilistId: Number(entry?.anilistId), + status: String(entry?.status || ""), + startYear: entry?.startYear ? Number(entry.startYear) : null, + nativeTitle: entry?.nativeTitle || null, + englishTitle: entry?.englishTitle || null, + romajiTitle: entry?.romajiTitle || null, + })) + .filter((entry) => Number.isFinite(entry.anilistId)); +}; + +const bootstrapRetryMinutes = (env) => { + const minutes = Number.parseInt(env.MANGADEX_BOOTSTRAP_RETRY_MINUTES || "", 10); + + return Number.isFinite(minutes) && minutes > 0 + ? minutes + : DEFAULT_BOOTSTRAP_RETRY_MINUTES; +}; + +const pendingRetryMs = (env) => { + const milliseconds = Number.parseInt(env.MANGA_CHAPTER_COUNTS_RETRY_MS || "", 10); + + return Number.isFinite(milliseconds) && milliseconds > 0 + ? milliseconds + : DEFAULT_PENDING_RETRY_MS; +}; + +const isRecentFailure = (row, retryMinutes) => + Date.now() - new Date(row.last_attempted_at || row.updated_at).getTime() < + retryMinutes * 60 * 1000; + +const queueBootstrap = (env, manga) => + manga + .map((entry) => { + const existing = bootstrapInFlight.get(entry.anilistId); + + if (existing) return existing; + + const promise = (async () => { + const row = await bootstrapManga(env, entry); + + if (row) { + try { + await upsertMangadexRows(env, [row]); + } catch (error) { + if (!isMangadexIdConstraintConflict(error)) throw error; + } + + await deleteMangadexFailureRows(env, [row.anilist_id]); + + return; + } + + await upsertMangadexFailureRows(env, [entry.anilistId]); + })().finally(() => { + bootstrapInFlight.delete(entry.anilistId); + }); + + bootstrapInFlight.set(entry.anilistId, promise); + + return promise; + }) + .filter(Boolean); + +const handleMangaChapterCounts = async (request, env, ctx) => { + if (!hasSupabaseConfig(env)) + return jsonResponse( + request, + { error: "Supabase is not configured for the proxy worker." }, + { status: 500 }, + ); + + const manga = await parseMangaPayload(request); + + if (!manga.length) return jsonResponse(request, { data: {} }); + + const anilistIds = manga.map((entry) => entry.anilistId); + const [existingRows, failureRows] = await Promise.all([ + getMangadexRowsByAniListIds(env, anilistIds), + getMangadexFailureRowsByAniListIds(env, anilistIds), + ]); + const existingIds = new Set(existingRows.map((row) => row.anilist_id)); + const recentFailures = new Set( + failureRows + .filter((row) => isRecentFailure(row, bootstrapRetryMinutes(env))) + .map((row) => row.anilist_id), + ); + const missingRows = manga.filter( + (entry) => + !existingIds.has(entry.anilistId) && !recentFailures.has(entry.anilistId), + ); + const pendingRows = missingRows.filter((entry) => + bootstrapInFlight.has(entry.anilistId), + ); + const queueableRows = missingRows.filter( + (entry) => !bootstrapInFlight.has(entry.anilistId), + ); + + if (queueableRows.length) + ctx.waitUntil( + Promise.all(queueBootstrap(env, queueableRows)).catch((error) => { + if (!isMangadexIdConstraintConflict(error)) throw error; + }), + ); + + const data = Object.fromEntries( + existingRows.map((row) => [ + String(row.anilist_id), + { + chapter: row.latest_en_chapter_number, + ...(row.latest_en_volume_text === null + ? {} + : { volumeText: row.latest_en_volume_text }), }, - }); - } + ]), + ); + const pending = [...new Set([...pendingRows, ...queueableRows].map((entry) => entry.anilistId))]; + + return jsonResponse(request, { + data, + ...(pending.length + ? { + pending, + retryAfterMs: pendingRetryMs(env), + } + : {}), + }); }; -addEventListener('fetch', (event) => { - const request = event.request; +const isAuthorisedSyncRequest = (request, env) => { + const token = env.MANGADEX_SYNC_TOKEN; - try { - switch (request.method) { - case 'OPTIONS': - event.respondWith(handleOptions(request)); + if (!token) return isPrivateHostname(new URL(request.url).hostname); + + return request.headers.get("Authorization") === `Bearer ${token}`; +}; + +const handleMangaSync = async (request, env) => { + if (!hasSupabaseConfig(env)) + return jsonResponse( + request, + { error: "Supabase is not configured for the proxy worker." }, + { status: 500 }, + ); + + if (!isAuthorisedSyncRequest(request, env)) + return jsonResponse(request, { error: "Forbidden" }, { status: 403 }); + + const result = await syncMangadexIndex(env); + + return jsonResponse(request, { data: result }); +}; + +export default { + async fetch(request, env, ctx) { + try { + const url = new URL(request.url); - break; + if (request.method === "OPTIONS") return handleOptions(request); - case 'GET': - case 'HEAD': - case 'POST': - event.respondWith(handleRequest(request)); + if ( + url.pathname === "/manga/chapter-counts" && + request.method === "POST" + ) + return handleMangaChapterCounts(request, env, ctx); - break; + if (url.pathname === "/manga/sync" && request.method === "POST") + return handleMangaSync(request, env); - default: - event.respondWith(async () => { - return new Response(null, { - status: 405, - statusText: 'Method Not Allowed', - }); - }); + if (["GET", "HEAD", "POST"].includes(request.method)) + return forwardProxyRequest(request); - break; + return textResponse(request, null, { + status: 405, + statusText: "Method Not Allowed", + }); + } catch (error) { + return jsonResponse( + request, + { error: error instanceof Error ? error.message : "Bad Request" }, + { status: 400 }, + ); } - } catch { - return new Response(null, { - status: 400, - statusText: 'Bad Request', - }); - } -}); + }, + + async scheduled(_controller, env, ctx) { + if (!hasSupabaseConfig(env)) return; + + ctx.waitUntil(syncMangadexIndex(env)); + }, +}; diff --git a/apps/proxy/src/mangadex.js b/apps/proxy/src/mangadex.js new file mode 100644 index 00000000..d2a42154 --- /dev/null +++ b/apps/proxy/src/mangadex.js @@ -0,0 +1,392 @@ +import { + getMangadexRowsByMangadexIds, + getSyncState, + upsertMangadexRows, + upsertSyncState, +} from "./supabase.js"; + +const MANGADEX_API_ORIGIN = "https://api.mangadex.org"; +const SYNC_STATE_NAME = "chapter_en"; +const ALL_CONTENT_RATINGS = ["safe", "suggestive", "erotica", "pornographic"]; + +const sleep = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +const appendQueryValues = (url, query) => { + for (const [key, rawValue] of Object.entries(query)) { + if (rawValue === undefined || rawValue === null) continue; + + if (Array.isArray(rawValue)) { + for (const value of rawValue) url.searchParams.append(key, String(value)); + + continue; + } + + url.searchParams.set(key, String(rawValue)); + } +}; + +const mangadexUserAgent = (env) => + env?.MANGADEX_USER_AGENT || + "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"; + +const mangadexRequest = async ( + env, + path, + { query = {}, retries = 1, retryDelay = 1500 } = {}, +) => { + const url = new URL(path, MANGADEX_API_ORIGIN); + + appendQueryValues(url, query); + + const response = await fetch(url, { + headers: { + Accept: "application/json", + "User-Agent": mangadexUserAgent(env), + }, + }); + + if (response.status === 429 && retries > 0) { + const retryAfterHeader = + response.headers.get("Retry-After") || + response.headers.get("X-RateLimit-Retry-After"); + const retryAfterSeconds = Number.parseInt(retryAfterHeader || "", 10); + const retryAt = + Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 10_000 + ? retryAfterSeconds * 1000 - Date.now() + : retryAfterSeconds * 1000; + + await sleep( + Number.isFinite(retryAt) && retryAt > 0 + ? Math.min(retryAt, 30_000) + : retryDelay, + ); + + return mangadexRequest(env, path, { + query, + retries: retries - 1, + retryDelay: retryDelay * 2, + }); + } + + if (!response.ok) { + const message = await response.text(); + + throw new Error(`MangaDex request failed (${response.status}): ${message}`); + } + + return response.json(); +}; + +const statusMap = { + FINISHED: "completed", + RELEASING: "ongoing", + HIATUS: "hiatus", + CANCELLED: "cancelled", +}; + +const normaliseTitle = (value) => + String(value || "") + .trim() + .toLowerCase(); + +const parseOptionalNumber = (value) => { + if (value === null || value === undefined || value === "") return null; + + const number = Number.parseFloat(String(value)); + + return Number.isFinite(number) ? number : null; +}; + +const mangadexDateTime = (isoString) => { + const date = new Date(isoString); + + if (Number.isNaN(date.getTime())) return "1970-01-01T00:00:00"; + + return date.toISOString().slice(0, 19); +}; + +const scoreSearchResult = (manga, title) => { + const attributes = manga.attributes || {}; + const titleMap = attributes.title || {}; + const altTitles = (attributes.altTitles || []).flatMap((item) => + Object.values(item || {}), + ); + const titles = [...Object.values(titleMap || {}), ...altTitles] + .map(normaliseTitle) + .filter(Boolean); + const candidate = normaliseTitle(title); + + if (!candidate.length) return Number.MAX_SAFE_INTEGER; + if (titles.includes(candidate)) return 0; + if (titles.some((item) => item.includes(candidate))) return 1; + return 2; +}; + +const pickBestSearchResult = (results, title) => + [...results].sort( + (left, right) => + scoreSearchResult(left, title) - scoreSearchResult(right, title), + )[0]; + +const titleCandidates = (manga) => + [ + manga.nativeTitle, + manga.englishTitle, + manga.romajiTitle, + manga.nativeTitle === "null" ? null : manga.nativeTitle, + manga.englishTitle === "null" ? null : manga.englishTitle, + manga.romajiTitle === "null" ? null : manga.romajiTitle, + ] + .filter(Boolean) + .map((title) => String(title).trim()) + .filter((title, index, array) => array.indexOf(title) === index); + +const searchQueries = (manga, title) => { + const status = statusMap[manga.status]; + + return [ + { + title, + ...(manga.startYear ? { year: manga.startYear } : {}), + ...(status ? { "status[]": [status] } : {}), + limit: 5, + }, + { + title, + ...(status ? { "status[]": [status] } : {}), + limit: 5, + }, + { + title, + ...(manga.startYear ? { year: manga.startYear } : {}), + limit: 5, + }, + { + title, + limit: 5, + }, + ]; +}; + +const chooseLatestAggregateChapter = (volumes) => { + let latest = null; + + for (const [volumeKey, volumeEntry] of Object.entries(volumes || {})) { + const chapters = volumeEntry?.chapters || {}; + + for (const chapterEntry of Object.values(chapters)) { + const chapterText = chapterEntry?.chapter || null; + const chapterNumber = parseOptionalNumber(chapterText); + const volumeText = + chapterEntry?.volume || volumeEntry?.volume || volumeKey || null; + const volumeNumber = parseOptionalNumber(volumeText); + const candidate = { + chapterText, + chapterNumber, + volumeText, + volumeNumber, + chapterId: chapterEntry?.id || null, + }; + + if (!latest) { + latest = candidate; + + continue; + } + + if ( + candidate.chapterNumber !== null && + (latest.chapterNumber === null || + candidate.chapterNumber > latest.chapterNumber || + (candidate.chapterNumber === latest.chapterNumber && + (candidate.volumeNumber || -1) > (latest.volumeNumber || -1))) + ) { + latest = candidate; + + continue; + } + + if ( + candidate.chapterNumber === null && + latest.chapterNumber === null && + String(candidate.chapterText || "").localeCompare( + String(latest.chapterText || ""), + undefined, + { numeric: true }, + ) > 0 + ) + latest = candidate; + } + } + + return latest; +}; + +const buildIndexedRow = (manga, mangadexId, aggregateResponse) => { + const now = new Date().toISOString(); + const latest = chooseLatestAggregateChapter(aggregateResponse?.volumes); + + return { + anilist_id: manga.anilistId, + mangadex_id: mangadexId, + latest_en_chapter_text: latest?.chapterText || null, + latest_en_chapter_number: latest?.chapterNumber || null, + latest_en_volume_text: latest?.volumeText || null, + latest_en_chapter_id: latest?.chapterId || null, + latest_chapter_updated_at: now, + last_seen_at: now, + last_indexed_at: now, + is_releasing: manga.status === "RELEASING", + created_at: manga.createdAt || now, + updated_at: now, + }; +}; + +export const resolveMangadexId = async (env, manga) => { + const candidates = titleCandidates(manga); + + for (const title of candidates) { + for (const query of searchQueries(manga, title)) { + const payload = await mangadexRequest(env, "/manga", { query }); + const data = payload?.data || []; + + if (!data.length) continue; + + return pickBestSearchResult(data, title)?.id || data[0]?.id || null; + } + } + + return null; +}; + +export const bootstrapManga = async (env, manga) => { + const mangadexId = await resolveMangadexId(env, manga); + + if (!mangadexId) return null; + + const aggregateResponse = await mangadexRequest( + env, + `/manga/${mangadexId}/aggregate`, + { + query: { + "translatedLanguage[]": ["en"], + }, + }, + ); + + return buildIndexedRow(manga, mangadexId, aggregateResponse); +}; + +const extractMangaIdFromChapter = (chapter) => + chapter?.relationships?.find((relationship) => relationship.type === "manga") + ?.id || null; + +const currentCursor = () => new Date().toISOString(); + +const rewindCursor = (isoString, seconds = 30) => + new Date( + Math.max(new Date(isoString).getTime() - seconds * 1000, 0), + ).toISOString(); + +export const syncMangadexIndex = async ( + env, + { maxPages = 5, aggregateDelayMs = 250 } = {}, +) => { + const syncState = (await getSyncState(env, SYNC_STATE_NAME)) || { + cursor_updated_at: currentCursor(), + }; + const trackedMangadexIds = new Set(); + const cursor = syncState.cursor_updated_at; + let maxUpdatedAt = cursor; + let offset = 0; + let pageCount = 0; + let scannedChapterCount = 0; + + while (pageCount < maxPages) { + const payload = await mangadexRequest(env, "/chapter", { + query: { + "translatedLanguage[]": ["en"], + "contentRating[]": ALL_CONTENT_RATINGS, + includeFutureUpdates: 0, + includeFuturePublishAt: 0, + updatedAtSince: mangadexDateTime(cursor), + "order[updatedAt]": "asc", + limit: 100, + offset, + }, + }); + const chapters = payload?.data || []; + + if (!chapters.length) break; + + for (const chapter of chapters) { + scannedChapterCount += 1; + + const mangaId = extractMangaIdFromChapter(chapter); + const updatedAt = chapter?.attributes?.updatedAt; + + if (mangaId) trackedMangadexIds.add(mangaId); + if (updatedAt && updatedAt > maxUpdatedAt) maxUpdatedAt = updatedAt; + } + + pageCount += 1; + + if (chapters.length < 100) break; + + offset += chapters.length; + } + + if (!trackedMangadexIds.size) { + await upsertSyncState(env, SYNC_STATE_NAME, maxUpdatedAt); + + return { + cursor: maxUpdatedAt, + pageCount, + scannedChapterCount, + refreshedCount: 0, + }; + } + + const trackedRows = await getMangadexRowsByMangadexIds(env, [ + ...trackedMangadexIds, + ]); + const refreshedRows = []; + + for (const row of trackedRows) { + const aggregateResponse = await mangadexRequest( + env, + `/manga/${row.mangadex_id}/aggregate`, + { + query: { + "translatedLanguage[]": ["en"], + }, + }, + ); + + refreshedRows.push( + buildIndexedRow( + { + anilistId: row.anilist_id, + status: row.is_releasing ? "RELEASING" : "FINISHED", + createdAt: row.created_at, + }, + row.mangadex_id, + aggregateResponse, + ), + ); + + if (aggregateDelayMs > 0) await sleep(aggregateDelayMs); + } + + if (refreshedRows.length) await upsertMangadexRows(env, refreshedRows); + + await upsertSyncState(env, SYNC_STATE_NAME, rewindCursor(maxUpdatedAt)); + + return { + cursor: rewindCursor(maxUpdatedAt), + pageCount, + scannedChapterCount, + refreshedCount: refreshedRows.length, + }; +}; diff --git a/apps/proxy/src/supabase.js b/apps/proxy/src/supabase.js new file mode 100644 index 00000000..314caf59 --- /dev/null +++ b/apps/proxy/src/supabase.js @@ -0,0 +1,217 @@ +const INDEX_COLUMNS = [ + "anilist_id", + "mangadex_id", + "latest_en_chapter_text", + "latest_en_chapter_number", + "latest_en_volume_text", + "latest_en_chapter_id", + "latest_chapter_updated_at", + "last_seen_at", + "last_indexed_at", + "is_releasing", + "created_at", + "updated_at", +].join(","); + +const chunk = (items, size) => { + const chunks = []; + + for (let index = 0; index < items.length; index += size) + chunks.push(items.slice(index, index + size)); + + return chunks; +}; + +const supabaseUrl = (env, path) => + `${String(env.SUPABASE_URL || "").replace(/\/$/, "")}/rest/v1/${path}`; + +const supabaseHeaders = (env, headers = {}) => ({ + apikey: env.SUPABASE_SERVICE_ROLE_KEY, + Authorization: `Bearer ${env.SUPABASE_SERVICE_ROLE_KEY}`, + ...headers, +}); + +const supabaseRequest = async ( + env, + path, + { + method = "GET", + query = {}, + body, + headers = {}, + prefer = [], + returnNullOn404 = false, + } = {}, +) => { + const url = new URL(supabaseUrl(env, path)); + + for (const [key, value] of Object.entries(query)) + if (value !== undefined && value !== null) url.searchParams.set(key, value); + + const response = await fetch(url, { + method, + headers: supabaseHeaders(env, { + ...(body ? { "Content-Type": "application/json" } : {}), + ...(prefer.length ? { Prefer: prefer.join(",") } : {}), + ...headers, + }), + body: body ? JSON.stringify(body) : undefined, + }); + + if (returnNullOn404 && response.status === 404) return null; + + if (!response.ok) { + const message = await response.text(); + + throw new Error(`Supabase request failed (${response.status}): ${message}`); + } + + if (response.status === 204) return null; + + const text = await response.text(); + + if (!text.length) return null; + + return JSON.parse(text); +}; + +const buildInFilter = (values) => `in.(${values.join(",")})`; + +export const hasSupabaseConfig = (env) => + Boolean(env.SUPABASE_URL && env.SUPABASE_SERVICE_ROLE_KEY); + +export const getMangadexRowsByAniListIds = async (env, anilistIds) => { + const rows = []; + + for (const ids of chunk([...new Set(anilistIds)], 100)) + rows.push( + ...((await supabaseRequest(env, "mangadex_manga_index", { + query: { + select: INDEX_COLUMNS, + anilist_id: buildInFilter(ids), + }, + })) || []), + ); + + return rows; +}; + +export const getMangadexRowsByMangadexIds = async (env, mangadexIds) => { + const rows = []; + + for (const ids of chunk([...new Set(mangadexIds)], 100)) + rows.push( + ...((await supabaseRequest(env, "mangadex_manga_index", { + query: { + select: INDEX_COLUMNS, + mangadex_id: buildInFilter(ids), + }, + })) || []), + ); + + return rows; +}; + +export const getMangadexFailureRowsByAniListIds = async (env, anilistIds) => { + const rows = []; + + for (const ids of chunk([...new Set(anilistIds)], 100)) + rows.push( + ...((await supabaseRequest(env, "mangadex_resolution_failures", { + query: { + select: "anilist_id,last_attempted_at,updated_at", + anilist_id: buildInFilter(ids), + }, + returnNullOn404: true, + })) || []), + ); + + return rows; +}; + +export const upsertMangadexRows = async (env, rows) => { + if (!rows.length) return []; + + return supabaseRequest(env, "mangadex_manga_index", { + method: "POST", + query: { + on_conflict: "anilist_id", + }, + body: rows, + prefer: ["resolution=merge-duplicates", "return=representation"], + }); +}; + +export const touchMangadexRows = async (env, rows) => { + if (!rows.length) return []; + + return upsertMangadexRows( + env, + rows.map((row) => ({ + anilist_id: row.anilist_id, + mangadex_id: row.mangadex_id, + is_releasing: row.is_releasing, + last_seen_at: row.last_seen_at, + updated_at: row.updated_at, + })), + ); +}; + +export const upsertMangadexFailureRows = async (env, anilistIds) => { + if (!anilistIds.length) return null; + + return supabaseRequest(env, "mangadex_resolution_failures", { + method: "POST", + query: { + on_conflict: "anilist_id", + }, + body: [...new Set(anilistIds)].map((anilistId) => ({ + anilist_id: anilistId, + last_attempted_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + })), + prefer: ["resolution=merge-duplicates", "return=minimal"], + returnNullOn404: true, + }); +}; + +export const deleteMangadexFailureRows = async (env, anilistIds) => { + if (!anilistIds.length) return null; + + return supabaseRequest(env, "mangadex_resolution_failures", { + method: "DELETE", + query: { + anilist_id: buildInFilter([...new Set(anilistIds)]), + }, + prefer: ["return=minimal"], + returnNullOn404: true, + }); +}; + +export const getSyncState = async (env, name) => { + const rows = await supabaseRequest(env, "mangadex_sync_state", { + query: { + select: "name,cursor_updated_at,updated_at", + name: `eq.${name}`, + limit: "1", + }, + }); + + return rows?.[0] || null; +}; + +export const upsertSyncState = async (env, name, cursorUpdatedAt) => + supabaseRequest(env, "mangadex_sync_state", { + method: "POST", + query: { + on_conflict: "name", + }, + body: [ + { + name, + cursor_updated_at: cursorUpdatedAt, + updated_at: new Date().toISOString(), + }, + ], + prefer: ["resolution=merge-duplicates", "return=minimal"], + }); diff --git a/apps/proxy/wrangler.toml b/apps/proxy/wrangler.toml index f65cc413..1115beeb 100644 --- a/apps/proxy/wrangler.toml +++ b/apps/proxy/wrangler.toml @@ -2,6 +2,9 @@ name = "due-proxy" main = "src/index.js" compatibility_date = "2023-12-18" +[triggers] +crons = ["* * * * *"] + # Variable bindings. These are arbitrary, plaintext strings (similar to environment variables) # Note: Use secrets to store sensitive data. # Docs: https://developers.cloudflare.com/workers/platform/environment-variables diff --git a/src/lib/List/Manga/MangaListTemplate.svelte b/src/lib/List/Manga/MangaListTemplate.svelte index eb7ffd83..7333d24e 100644 --- a/src/lib/List/Manga/MangaListTemplate.svelte +++ b/src/lib/List/Manga/MangaListTemplate.svelte @@ -1,25 +1,25 @@ <script lang="ts"> -import { mediaListCollection, Type, type Media } from "$lib/Data/AniList/media"; -import type { AniListAuthorisation } from "$lib/Data/AniList/identity"; +import localforage from "localforage"; import { onDestroy, onMount } from "svelte"; -import { chapterCount } from "$lib/Media/Manga/chapters"; -import { pruneAllManga } from "$lib/Media/Manga/cache"; -import manga from "$stores/manga"; +import { browser } from "$app/environment"; +import type { AniListAuthorisation } from "$lib/Data/AniList/identity"; +import { type Media, mediaListCollection, Type } from "$lib/Data/AniList/media"; import { database } from "$lib/Database/IDB/chapters"; -import settings from "$stores/settings"; -import lastPruneTimes from "$stores/lastPruneTimes"; -import ListTitle from "../ListTitle.svelte"; import RateLimitedError from "$lib/Error/RateLimited.svelte"; -import CleanMangaList from "./CleanMangaList.svelte"; +import Skeleton from "$lib/Loading/Skeleton.svelte"; import { incrementMediaProgress } from "$lib/Media/Anime/cache"; -import { addNotification } from "$lib/Notification/store"; +import { pruneAllManga } from "$lib/Media/Manga/cache"; +import { chapterCount, hydrateChapterCounts } from "$lib/Media/Manga/chapters"; import { options } from "$lib/Notification/options"; -import Skeleton from "$lib/Loading/Skeleton.svelte"; -import locale from "$stores/locale"; -import { browser } from "$app/environment"; -import identity from "$stores/identity"; +import { addNotification } from "$lib/Notification/store"; import privilegedUser from "$lib/Utility/privilegedUser"; -import localforage from "localforage"; +import identity from "$stores/identity"; +import lastPruneTimes from "$stores/lastPruneTimes"; +import locale from "$stores/locale"; +import manga from "$stores/manga"; +import settings from "$stores/settings"; +import ListTitle from "../ListTitle.svelte"; +import CleanMangaList from "./CleanMangaList.svelte"; export let user: AniListAuthorisation = { accessToken: "", @@ -78,7 +78,7 @@ onMount(async () => { `last${due ? "" : "Completed"}MangaListLength`, )) as number | null; - if (lastStoredList) lastListSize = parseInt(String(lastStoredList)); + if (lastStoredList) lastListSize = parseInt(String(lastStoredList), 10); } startTime = performance.now(); @@ -143,6 +143,7 @@ const cleanMedia = async ( force: boolean, ) => { progress = 0; + rateLimited = false; if (manga && dummy) return manga; @@ -157,7 +158,7 @@ const cleanMedia = async ( if ($lastPruneTimes.chapters === 1) { refreshing = true; - lastPruneTimes.setKey("chapters", new Date().getTime()); + lastPruneTimes.setKey("chapters", Date.now()); } else { const currentDate = new Date(); @@ -196,33 +197,29 @@ const cleanMedia = async ( ($settings.displayNotStarted === true ? 0 : 1), ); let finalMedia = releasingMedia; - const progressStep = 100 / finalMedia.length / 2; - const chapterPromises = finalMedia.map((m: Media) => - database.chapters.get(m.id).then((c) => { - if (progress < 100) progress += progressStep; - - if (!due) - return new Promise((resolve) => resolve(m.chapters)) as Promise< - number | null - >; - - if (c !== undefined) - return chapterCount($identity, m, $settings.calculateGuessingDisabled); - else { - // A = On 1 second interval, - // B = a maximum of 5 requests per second are allowed. - // C = chapterCount makes 3 requests per call. - // F = A / (B / C) = 0.6 seconds - return new Promise((resolve) => setTimeout(resolve, 600)).then(() => - chapterCount($identity, m, $settings.calculateGuessingDisabled), - ); - } - }), - ); const chapterCounts: (number | null)[] = []; - for (let i = 0; i < chapterPromises.length; i++) { - const count = await chapterPromises[i]; + if (due && finalMedia.length > 0) { + const hydration = await hydrateChapterCounts( + $identity, + finalMedia, + $settings.calculateGuessingDisabled, + ); + + rateLimited = hydration.rateLimited; + progress = 50; + } + + for (let i = 0; i < finalMedia.length; i++) { + const media = finalMedia[i]; + const progressStep = finalMedia.length > 0 ? 50 / finalMedia.length : 0; + const count = due + ? await chapterCount( + $identity, + media, + $settings.calculateGuessingDisabled, + ) + : media.chapters; if (count === -22) { rateLimited = true; diff --git a/src/lib/Media/Manga/chapters.ts b/src/lib/Media/Manga/chapters.ts index e4ad9429..473a3ed4 100644 --- a/src/lib/Media/Manga/chapters.ts +++ b/src/lib/Media/Manga/chapters.ts @@ -1,237 +1,262 @@ -import { recentMediaActivities, type Media } from "$lib/Data/AniList/media"; +import { env } from "$env/dynamic/public"; +import { type Media, recentMediaActivities } from "$lib/Data/AniList/media"; import { getChapterCount } from "$lib/Data/Manga/raw"; -import proxy from "$lib/Utility/proxy"; +import { proxyRoute } from "$lib/Utility/proxy"; import settings from "$stores/settings"; import type { UserIdentity } from "../../Data/AniList/identity"; import { database } from "../../Database/IDB/chapters"; -const getManga = async ( - statusIn: string, - year: number, - native: string | null, - english: string | null, - romaji: string | null, -) => { - let status = ""; - let error = false; +interface MangaDexChapterCount { + chapter: number | null; + volumeText?: string | null; +} - switch (statusIn) { - case "FINISHED": - { - status = "completed"; - } - break; - case "RELEASING": - { - status = "ongoing"; - } - break; - case "HIATUS": - { - status = "hiatus"; - } - break; - case "CANCELLED": - { - status = "cancelled"; - } - break; - } +interface MangaDexChapterCountsResponse { + data?: Record<string, MangaDexChapterCount>; + pending?: number[]; + retryAfterMs?: number; +} - const nullIfNullString = (s: string | null) => (s === "null" ? null : s); - const get = async (title: string) => { - try { - return await ( - await fetch( - proxy( - `https://api.mangadex.org/manga?title=${encodeURIComponent( - title, - )}&year=${year}&status[]=${status}`, - ), - ) - ).json(); - } catch { - error = true; - } - }; +const chapterMemoryCache = new Map<number, number | null>(); +const MAX_PENDING_RETRIES = 2; +const DEFAULT_PENDING_RETRY_MS = 750; - let mangadexData = await get( - nullIfNullString(native) || - nullIfNullString(english) || - nullIfNullString(romaji) || - "", - ); +const browserChapterCacheDisabled = () => + env.PUBLIC_DISABLE_MANGA_BROWSER_CACHE === "true"; - if (error) return new Response("rate-limited"); +const parseOptionalNumber = (value: string | number | null | undefined) => { + if (value === null || value === undefined || value === "") return null; - if (mangadexData["data"] === undefined || mangadexData["data"].length === 0) { - mangadexData = await get(nullIfNullString(english) || ""); + const number = Number.parseFloat(String(value)); - if ( - mangadexData["data"] === undefined || - mangadexData["data"].length === 0 - ) { - mangadexData = await get(nullIfNullString(romaji) || ""); - } - } + return Number.isFinite(number) ? number : null; +}; + +const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +const readCachedChapterCount = async (mangaId: number) => { + if (chapterMemoryCache.has(mangaId)) return chapterMemoryCache.get(mangaId); + + if (browserChapterCacheDisabled()) return undefined; + + const chapter = await database.chapters.get(mangaId); + + return chapter === undefined + ? undefined + : chapter.chapters === -1 + ? null + : chapter.chapters; +}; + +const writeCachedChapterCount = async ( + mangaId: number, + chapters: number | null, + volumes: number | null = null, +) => { + chapterMemoryCache.set(mangaId, chapters); + + if (browserChapterCacheDisabled()) return; - return Response.json(mangadexData, { - headers: { - "Cache-Control": "max-age=300", - }, + await database.chapters.put({ + id: mangaId, + chapters: chapters === null ? -1 : chapters, + volumes, }); }; -export const chapterCount = async ( +const getActivityChapterCount = async ( identity: UserIdentity, manga: Media, disableGuessing: boolean, - // preferActivity = false -): Promise<number | null> => { - const chapters = await database.chapters.get(manga.id); +) => { + if (disableGuessing) { + await writeCachedChapterCount(manga.id, null); + + return null; + } - if (chapters !== undefined) - return chapters.chapters === -1 ? null : chapters.chapters; + const anilistData = await recentMediaActivities( + identity, + manga, + settings.get().calculateGuessMethod, + ); - // if (preferActivity) { - // return await recentMediaActivities(identity, manga); - // } + await writeCachedChapterCount(manga.id, anilistData); - const tryRecentMediaActivities = async () => { - if (disableGuessing) { - await database.chapters.put({ - id: manga.id, - chapters: -1, - volumes: null, - }); + return anilistData; +}; - return null; - } +const applyResolvedChapterCount = async ( + identity: UserIdentity, + manga: Media, + disableGuessing: boolean, + chapter: number, + volumeText?: string | null, +) => { + let resolvedChapter = chapter; - const anilistData = await recentMediaActivities( + if ( + (manga.mediaListEntry || { progress: 0 }).progress > resolvedChapter && + !disableGuessing + ) { + const activityChapterCount = await recentMediaActivities( identity, manga, settings.get().calculateGuessMethod, ); - await database.chapters.put({ - id: manga.id, - chapters: anilistData ? anilistData : -1, - volumes: null, - }); + if (activityChapterCount !== null && activityChapterCount > resolvedChapter) + resolvedChapter = activityChapterCount; + } + + if (resolvedChapter === 0) resolvedChapter = -1; - return anilistData; - }; + await writeCachedChapterCount( + manga.id, + resolvedChapter === -1 ? null : resolvedChapter, + parseOptionalNumber(volumeText), + ); - if (manga.format === "NOVEL") return await tryRecentMediaActivities(); + return resolvedChapter === -1 ? null : resolvedChapter; +}; - let lastChapter = 0; - let completedVolumes = null; +const fetchMangaChapterCounts = async (manga: Media[]) => { + const data: Record<string, MangaDexChapterCount> = {}; + let rateLimited = false; + let successfulResponse = false; + const mangaById = new Map(manga.map((entry) => [entry.id, entry])); + let pendingManga = manga; + let retryAfterMs = DEFAULT_PENDING_RETRY_MS; + + for (let attempt = 0; attempt <= MAX_PENDING_RETRIES; attempt += 1) { + const nextPendingIds = new Set<number>(); + + for (let index = 0; index < pendingManga.length; index += 100) { + const chunk = pendingManga.slice(index, index + 100); + const response = await fetch(proxyRoute("/manga/chapter-counts"), { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + manga: chunk.map((entry) => ({ + anilistId: entry.id, + status: entry.status, + startYear: entry.startDate.year, + nativeTitle: entry.title.native, + englishTitle: entry.title.english, + romajiTitle: entry.title.romaji, + })), + }), + }).catch(() => null); + + if (!response) continue; + + if (!response.ok) { + rateLimited = response.status === 429 || response.status === 503; + + continue; + } - if (!settings.get().calculatePreferNativeChapterCount) { - const mangadexData = await getManga( - manga.status, - manga.startDate.year, - manga.title.native, - manga.title.english, - manga.title.romaji, - ); + const payload = (await response.json()) as MangaDexChapterCountsResponse; + successfulResponse = true; + retryAfterMs = payload.retryAfterMs || retryAfterMs; - if ((await mangadexData.clone().text()) === "rate-limited") return -22; - - const mangadexDataJson = await mangadexData.json(); - - if ( - mangadexDataJson["data"] === undefined || - mangadexDataJson["data"].length === 0 - ) - return await tryRecentMediaActivities(); - - const mangadexId = mangadexDataJson["data"][0]["id"]; - const lastChapterDataJson = await ( - await fetch( - proxy( - `https://api.mangadex.org/manga/${mangadexId}/feed?order[chapter]=desc&translatedLanguage[]=en&limit=1&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic`, - ), - ) - ).json(); - - if ( - lastChapterDataJson["data"] === undefined || - lastChapterDataJson["data"].length === 0 - ) - return await tryRecentMediaActivities(); - - lastChapter = lastChapterDataJson["data"][0]["attributes"]["chapter"]; - completedVolumes = null; - - if ( - (manga.mediaListEntry || { progress: 0 }).progress > lastChapter && - !disableGuessing - ) { - const anilistData = await recentMediaActivities( - identity, - manga, - settings.get().calculateGuessMethod, + Object.assign(data, payload.data || {}); + + for (const id of payload.pending || []) nextPendingIds.add(id); + } + + if (!nextPendingIds.size || attempt === MAX_PENDING_RETRIES) break; + + pendingManga = [...nextPendingIds] + .map((id) => mangaById.get(id)) + .filter((entry): entry is Media => entry !== undefined); + + if (!pendingManga.length) break; + + await sleep(retryAfterMs); + } + + return { data, rateLimited: rateLimited && !successfulResponse }; +}; + +export const hydrateChapterCounts = async ( + identity: UserIdentity, + manga: Media[], + disableGuessing: boolean, +): Promise<{ rateLimited: boolean }> => { + const uniqueManga = manga.filter( + (entry, index, array) => + array.findIndex((candidate) => candidate.id === entry.id) === index, + ); + const unresolvedManga: Media[] = []; + + for (const entry of uniqueManga) { + if ((await readCachedChapterCount(entry.id)) !== undefined) continue; + + if (entry.format === "NOVEL") { + await getActivityChapterCount(identity, entry, disableGuessing); + + continue; + } + + if (settings.get().calculatePreferNativeChapterCount) { + const nativeCount = (await getChapterCount(entry.title.native)) || 0; + + await writeCachedChapterCount( + entry.id, + nativeCount === 0 ? null : nativeCount, ); - if (anilistData !== null && anilistData > lastChapter) - lastChapter = anilistData; + continue; } - if (!settings.get().calculateDisableOutOfDateVolumeWarning) { - const volumeOfChapterData = await ( - await fetch( - proxy( - `https://api.mangadex.org/chapter?manga=${mangadexId}&chapter=${manga.mediaListEntry?.progress}&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic&limit=1`, - ), - ) - ).json(); - let lastAvailableVolume = - lastChapterDataJson["data"][0]["attributes"]["volume"]; - - if (lastAvailableVolume === null) { - let chapterIndex = 0; - - while ( - chapterIndex < lastChapterDataJson["data"].length && - lastAvailableVolume === null - ) { - if ( - lastChapterDataJson["data"][chapterIndex]["attributes"][ - "volume" - ] !== null - ) - lastAvailableVolume = - lastChapterDataJson["data"][chapterIndex]["attributes"]["volume"]; - - chapterIndex += 1; - } - } + unresolvedManga.push(entry); + } - if ( - volumeOfChapterData["data"] !== undefined && - volumeOfChapterData["data"].length > 0 - ) { - const volumeOfChapter = - volumeOfChapterData["data"][0]["attributes"]["volume"]; + if (!unresolvedManga.length) return { rateLimited: false }; - if (volumeOfChapter !== null) completedVolumes = volumeOfChapter; + const { data, rateLimited } = await fetchMangaChapterCounts(unresolvedManga); - if (completedVolumes === volumeOfChapter) completedVolumes -= 1; - } + for (const entry of unresolvedManga) { + const resolved = data[String(entry.id)]; + + if (resolved?.chapter !== undefined && resolved.chapter !== null) { + await applyResolvedChapterCount( + identity, + entry, + disableGuessing, + resolved.chapter, + resolved.volumeText, + ); + + continue; } - } else { - lastChapter = (await getChapterCount(manga.title.native)) || 0; + + await getActivityChapterCount(identity, entry, disableGuessing); } - if (lastChapter === 0) lastChapter = -1; + return { rateLimited }; +}; - await database.chapters.put({ - id: manga.id, - chapters: Number(lastChapter), - volumes: completedVolumes, - }); +export const chapterCount = async ( + identity: UserIdentity, + manga: Media, + disableGuessing: boolean, +): Promise<number | null> => { + const cachedChapterCount = await readCachedChapterCount(manga.id); + + if (cachedChapterCount !== undefined) return cachedChapterCount; + + const { rateLimited } = await hydrateChapterCounts( + identity, + [manga], + disableGuessing, + ); + + if (rateLimited) return -22; - return Number(lastChapter); + return (await readCachedChapterCount(manga.id)) ?? null; }; diff --git a/src/lib/Utility/proxy.ts b/src/lib/Utility/proxy.ts index 32f31226..57883836 100644 --- a/src/lib/Utility/proxy.ts +++ b/src/lib/Utility/proxy.ts @@ -1,11 +1,24 @@ +import { env } from "$env/dynamic/public"; import { isLocalApp } from "$lib/Utility/appOrigin"; +const DEFAULT_PROXY_ORIGIN = "https://proxy.due.moe"; + +const normaliseOrigin = (origin: string) => + origin.endsWith("/") ? origin.slice(0, -1) : origin; + +export const proxyOrigin = () => + normaliseOrigin(env.PUBLIC_PROXY_ORIGIN || DEFAULT_PROXY_ORIGIN); + +export const proxyRoute = (path = "/") => + `${proxyOrigin()}${path.startsWith("/") ? path : `/${path}`}`; + export const proxy = (url: string, disable = false) => { const randomKey = Math.floor(Math.random() * 90) + 10; + const forcedProxy = Boolean(env.PUBLIC_PROXY_ORIGIN); - return isLocalApp() && !disable + return isLocalApp() && !disable && !forcedProxy ? url - : `https://proxy.due.moe/?d2=${btoa( + : `${proxyOrigin()}/?d2=${btoa( url .split("") .map((char) => char.charCodeAt(0) + randomKey) diff --git a/supabase/mangadex.sql b/supabase/mangadex.sql new file mode 100644 index 00000000..79ea0408 --- /dev/null +++ b/supabase/mangadex.sql @@ -0,0 +1,39 @@ +create table if not exists public.mangadex_manga_index ( + anilist_id bigint primary key, + mangadex_id uuid not null, + latest_en_chapter_text text, + latest_en_chapter_number double precision, + latest_en_volume_text text, + latest_en_chapter_id uuid, + latest_chapter_updated_at timestamp with time zone, + last_seen_at timestamp with time zone default (now() at time zone 'utc'::text) not null, + last_indexed_at timestamp with time zone, + is_releasing boolean default true not null, + created_at timestamp with time zone default (now() at time zone 'utc'::text) not null, + updated_at timestamp with time zone default (now() at time zone 'utc'::text) not null +); + +create index if not exists mangadex_manga_index_mangadex_id_idx + on public.mangadex_manga_index (mangadex_id); + +create index if not exists mangadex_manga_index_last_indexed_at_idx + on public.mangadex_manga_index (last_indexed_at); + +create table if not exists public.mangadex_resolution_failures ( + anilist_id bigint primary key, + last_attempted_at timestamp with time zone default (now() at time zone 'utc'::text) not null, + updated_at timestamp with time zone default (now() at time zone 'utc'::text) not null +); + +create index if not exists mangadex_resolution_failures_last_attempted_at_idx + on public.mangadex_resolution_failures (last_attempted_at); + +create table if not exists public.mangadex_sync_state ( + name text primary key, + cursor_updated_at timestamp with time zone not null, + updated_at timestamp with time zone default (now() at time zone 'utc'::text) not null +); + +insert into public.mangadex_sync_state (name, cursor_updated_at) +values ('chapter_en', now() at time zone 'utc'::text) +on conflict (name) do nothing; |