From c32dd8832580d5850dbe6d2146a0d1d50a14756d Mon Sep 17 00:00:00 2001 From: Fuwn Date: Fri, 28 Jul 2023 21:34:34 -0700 Subject: refactor(due): like-functions to modules --- src/due/__init__.py | 2 +- src/due/anilist/__init__.py | 0 src/due/anilist/collection.py | 52 ++++++++ src/due/anilist/media.py | 31 +++++ src/due/anilist/utilities.py | 48 +++++++ src/due/html.py | 282 ------------------------------------------ src/due/html/__init__.py | 0 src/due/html/anime.py | 51 ++++++++ src/due/html/manga.py | 162 ++++++++++++++++++++++++ src/due/html/utilities.py | 72 +++++++++++ src/due/media.py | 129 ------------------- src/due/routes/index.py | 7 +- 12 files changed, 422 insertions(+), 414 deletions(-) create mode 100644 src/due/anilist/__init__.py create mode 100644 src/due/anilist/collection.py create mode 100644 src/due/anilist/media.py create mode 100644 src/due/anilist/utilities.py delete mode 100644 src/due/html.py create mode 100644 src/due/html/__init__.py create mode 100644 src/due/html/anime.py create mode 100644 src/due/html/manga.py create mode 100644 src/due/html/utilities.py delete mode 100644 src/due/media.py (limited to 'src') diff --git a/src/due/__init__.py b/src/due/__init__.py index 6b10591..863a817 100644 --- a/src/due/__init__.py +++ b/src/due/__init__.py @@ -1,6 +1,6 @@ from flask import Flask from due.routes import auth, oauth, index -from due.html import page +from due.html.utilities import page from flask_cors import CORS from dotenv import load_dotenv from due.cache import cache diff --git a/src/due/anilist/__init__.py b/src/due/anilist/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/due/anilist/collection.py b/src/due/anilist/collection.py new file mode 100644 index 0000000..0e67602 --- /dev/null +++ b/src/due/anilist/collection.py @@ -0,0 +1,52 @@ +import requests +from .utilities import user_id + + +def create_collection(anilist, type, username=None): + current_collection = media_list_collection(anilist, type, username) + current = [] + + for list in current_collection["MediaListCollection"]["lists"]: + current += list["entries"] + + return (current, current_collection["MediaListCollection"]["user"]["name"]) + + +def media_list_collection(anilist, type, username=None): + return requests.post( + "https://graphql.anilist.co", + json={ + "query": media_list_collection_query( + user_id(anilist) if username is None else user_name_to_id(username), + type, + ) + }, + headers={ + "Authorization": anilist["token_type"] + " " + anilist["access_token"], + "Content-Type": "application/json", + "Accept": "application/json", + }, + ).json()["data"] + + +def media_list_collection_query(user_id: int, type) -> str: + return f"""{{ + MediaListCollection(userId: {user_id}, type: {"ANIME" if type == "ANIME" else "MANGA"}, status_not_in: [COMPLETED]) {{ + hasNextChunk + lists {{ + entries {{ + media {{ + id + status + type + episodes + title {{ romaji english native }} + nextAiringEpisode {{ episode }} + mediaListEntry {{ progress }} + startDate {{ year }} + }} + }} + }} + user {{ name }} + }} + }}""" diff --git a/src/due/anilist/media.py b/src/due/anilist/media.py new file mode 100644 index 0000000..b8fadfa --- /dev/null +++ b/src/due/anilist/media.py @@ -0,0 +1,31 @@ +# import requests + +# def media_list(anilist, pageNumber): +# return requests.post( +# "https://graphql.anilist.co", +# json={"query": media_list_query(user_id(anilist), pageNumber)}, +# headers={ +# "Authorization": anilist["token_type"] + " " + anilist["access_token"], +# "Content-Type": "application/json", +# "Accept": "application/json", +# }, +# ).json()["data"] + + +# def media_list_query(user_id: int, page_number: int) -> str: +# return f"""{{ +# Page(page: {page_number}) {{ +# mediaList(userId: {user_id}, status_not_in: [COMPLETED]) {{ +# media {{ +# id +# status +# type +# title {{ romaji english }} +# nextAiringEpisode {{ episode }} +# mediaListEntry {{ progress }} +# startDate {{ year }} +# }} +# }} +# pageInfo {{ hasNextPage }} +# }} +# }}""" diff --git a/src/due/anilist/utilities.py b/src/due/anilist/utilities.py new file mode 100644 index 0000000..033939e --- /dev/null +++ b/src/due/anilist/utilities.py @@ -0,0 +1,48 @@ +import requests + + +def user_id(anilist): + viewer = requests.post( + "https://graphql.anilist.co", + json={"query": "{ Viewer { id } }"}, + headers={ + "Authorization": anilist["token_type"] + " " + anilist["access_token"], + "Content-Type": "application/json", + "Accept": "application/json", + }, + ).json()["data"]["Viewer"] + + if viewer is None: + return -1 + + return int(viewer["id"]) + + +def last_activity(id): + return int( + requests.post( + "https://graphql.anilist.co", + json={ + "query": f"""{{ Activity(userId: {id}, type: MEDIA_LIST, sort: ID_DESC) {{ + __typename ... on ListActivity {{ createdAt }} + }} }}""" + }, + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + }, + ).json()["data"]["Activity"]["createdAt"] + ) + + +def user_name_to_id(name): + return int( + requests.post( + "https://graphql.anilist.co", + json={"query": f'{{ User(name: "{name}") {{ id }} }}'}, + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + }, + ).json()["data"]["User"]["id"] + ) diff --git a/src/due/html.py b/src/due/html.py deleted file mode 100644 index a43753b..0000000 --- a/src/due/html.py +++ /dev/null @@ -1,282 +0,0 @@ -import requests -import joblib -from due.cache import cache -from flask import request -import math -import re -import os - - -def seen(element, manga=False): - read = 0 - - if manga: - available_matches = re.search(r"\[(\d+)<\/a>\]", element) - read_matches = re.search(r"\[(?=\d+)<\/a>\]", element) - - if read_matches: - read = int(read_matches.group()) - print(read) - - if available_matches: - return int(available_matches.group(1)) - read - else: - return 0 - - available_matches = re.findall(r"\d+\]|\[\d+", element) - seen_matches = re.search( - r"\s(\d+)*?(/(\d+))*\s", element - ) - - if seen_matches: - read = int(seen_matches.group(1)) - - if len(available_matches) > 1: - return int(available_matches[1].strip("[]")) - read - elif len(available_matches) == 1: - return int(available_matches[0].strip("[]")) - read - else: - return 0 - - -def anime_to_html(releasing_outdated_anime): - current_html = [] - ids = [] - - for media in releasing_outdated_anime: - anime = media["media"] - title = anime["title"]["english"] - id = anime["id"] - - if id in ids: - continue - else: - ids.append(id) - - progress = (anime["mediaListEntry"] or {"progress": 0})["progress"] - available = ( - {"episode": 0} - if media["media"]["nextAiringEpisode"] is None - else media["media"]["nextAiringEpisode"] - )["episode"] - 1 - - if available <= 0: - available = "?" - - if title is None: - title = anime["title"]["romaji"] - - if request.cookies.get("show_missing") is not None and str(available)[0] == "?": - ids.pop() - - continue - - episodes = anime["episodes"] - total_html = ( - "" if episodes is None else f'/{episodes}' - ) - - current_html.append( - f'
  • {title} {progress}{total_html} [{available}]
    • ' - ) - - current_html = sorted(current_html, key=seen) - - current_html.insert(0, "
      ") - current_html.append("
    ") - - return ("".join(current_html), len(ids)) - - -def manga_to_html(releasing_outdated_manga, show_missing): - current_html = [] - ids = [] - - def process(media): - manga = media["media"] - title = manga["title"]["native"] - id = manga["id"] - previous_volume_largest_chapter = 0 - - if id in ids: - return - else: - ids.append(id) - - progress = manga["mediaListEntry"]["progress"] - available = ( - {"episode": 0} - if media["media"]["nextAiringEpisode"] is None - else media["media"]["nextAiringEpisode"] - )["episode"] - 1 - - if available <= 0: - available = "?" - - mangadex_data = cache.get(str(manga["id"]) + "id") - mangadex_id = None - - if mangadex_data is None: - mangadex_data = requests.get( - "https://api.mangadex.org/manga", - params={"title": title, "year": manga["startDate"]["year"]}, - ).json()["data"] - - # This is very stupid. It should never get this far, anyway. - if len(mangadex_data) == 0: - mangadex_data = requests.get( - "https://api.mangadex.org/manga", - params={ - "title": manga["title"]["romaji"], - "year": manga["startDate"]["year"], - }, - ).json()["data"] - - if len(mangadex_data) == 0: - mangadex_data = requests.get( - "https://api.mangadex.org/manga", - params={ - "title": manga["title"]["romaji"], - "year": manga["startDate"]["year"], - }, - ).json()["data"] - - cache.set(str(manga["id"]) + "id", mangadex_data) - - if len(mangadex_data) == 0: - available = "?" - else: - mangadex_id = mangadex_data[0]["id"] - manga_chapter_aggregate = cache.get(str(manga["id"]) + "ag") - - if manga_chapter_aggregate is None: - manga_chapter_aggregate = requests.get( - f"https://api.mangadex.org/manga/{mangadex_id}/aggregate", - ).json() - - cache.set(str(manga["id"]) + "ag", manga_chapter_aggregate) - - if "none" in manga_chapter_aggregate["volumes"]: - previous_volume_largest_chapter = list( - manga_chapter_aggregate["volumes"][ - str( - dict(enumerate(manga_chapter_aggregate["volumes"])).get(1) - or "none" - ) - ]["chapters"] - )[0] - available = list( - manga_chapter_aggregate["volumes"]["none"]["chapters"] - )[0] - - if str(available) == "none": - available = list( - manga_chapter_aggregate["volumes"]["none"]["chapters"] - )[1] - else: - try: - available = list( - manga_chapter_aggregate["volumes"][ - str(list(manga_chapter_aggregate["volumes"])[0]) - ]["chapters"] - )[0] - except Exception: - available = "?" - - if not str(available)[0].isdigit(): - if len(manga_chapter_aggregate["volumes"]) == 0: - ids.pop() - - return - - available = math.floor( - float( - list(manga_chapter_aggregate["volumes"]["1"]["chapters"])[0] - ) - ) - - if show_missing is not None and str(available)[0] == "?": - ids.pop() - - return - - # Useful when debugging - # if str(available)[0] != "?": - # ids.pop() - - # return - - if str(available)[0] == "{": - ids.pop() - - return - - if str(available)[0].isdigit(): - available = math.floor(float(available)) - - if math.floor(float(previous_volume_largest_chapter)) > available: - available = math.floor(float(previous_volume_largest_chapter)) - - if str(available) != "?" and int(progress) >= int(available): - ids.pop() - - return - - available_link = ( - available - if mangadex_id is None - else f'{available}' - ) - - current_html.append( - f'
  • {manga["title"]["english"] or manga["title"]["romaji"] or title} {progress} [{available_link}]
    • ' - ) - - joblib.Parallel(n_jobs=int(os.getenv("CONCURRENT_JOBS")) or 4, require="sharedmem")( - joblib.delayed(process)(media) for media in releasing_outdated_manga - ) - - current_html = sorted(current_html, key=lambda x: seen(x, manga=True)) - - current_html.insert(0, "
      ") - current_html.append("
    ") - - return ("".join(current_html), len(ids)) - - -def page(main_content, footer): - message = '
    Slow loads? If your media hasn\'t been cached in a while, the first load will take a couple seconds longer than the rest. Subsequent requests on cached media should be faster. Hide forever
    ' - - if request.cookies.get("hide_message") == "1": - message = "" - - return f""" - - - - 期限 - - - - - - - - - - - -

    期限

    - - {main_content} - -

    - -
    - -

    {footer}

    - - {message} - - -""" diff --git a/src/due/html/__init__.py b/src/due/html/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/due/html/anime.py b/src/due/html/anime.py new file mode 100644 index 0000000..1113965 --- /dev/null +++ b/src/due/html/anime.py @@ -0,0 +1,51 @@ +from flask import request +from .utilities import seen + + +def anime_to_html(releasing_outdated_anime): + current_html = [] + ids = [] + + for media in releasing_outdated_anime: + anime = media["media"] + title = anime["title"]["english"] + id = anime["id"] + + if id in ids: + continue + else: + ids.append(id) + + progress = (anime["mediaListEntry"] or {"progress": 0})["progress"] + available = ( + {"episode": 0} + if media["media"]["nextAiringEpisode"] is None + else media["media"]["nextAiringEpisode"] + )["episode"] - 1 + + if available <= 0: + available = "?" + + if title is None: + title = anime["title"]["romaji"] + + if request.cookies.get("show_missing") is not None and str(available)[0] == "?": + ids.pop() + + continue + + episodes = anime["episodes"] + total_html = ( + "" if episodes is None else f'/{episodes}' + ) + + current_html.append( + f'
  • {title} {progress}{total_html} [{available}]
    • ' + ) + + current_html = sorted(current_html, key=seen) + + current_html.insert(0, "
      ") + current_html.append("
    ") + + return ("".join(current_html), len(ids)) diff --git a/src/due/html/manga.py b/src/due/html/manga.py new file mode 100644 index 0000000..8e74a63 --- /dev/null +++ b/src/due/html/manga.py @@ -0,0 +1,162 @@ +import requests +import joblib +from due.cache import cache +import math +import os +from .utilities import seen + + +def manga_to_html(releasing_outdated_manga, show_missing): + current_html = [] + ids = [] + + def process(media): + manga = media["media"] + title = manga["title"]["native"] + id = manga["id"] + previous_volume_largest_chapter = 0 + + if id in ids: + return + else: + ids.append(id) + + progress = manga["mediaListEntry"]["progress"] + available = ( + {"episode": 0} + if media["media"]["nextAiringEpisode"] is None + else media["media"]["nextAiringEpisode"] + )["episode"] - 1 + + if available <= 0: + available = "?" + + mangadex_data = cache.get(str(manga["id"]) + "id") + mangadex_id = None + + if mangadex_data is None: + mangadex_data = requests.get( + "https://api.mangadex.org/manga", + params={"title": title, "year": manga["startDate"]["year"]}, + ).json()["data"] + + # This is very stupid. It should never get this far, anyway. + if len(mangadex_data) == 0: + mangadex_data = requests.get( + "https://api.mangadex.org/manga", + params={ + "title": manga["title"]["romaji"], + "year": manga["startDate"]["year"], + }, + ).json()["data"] + + if len(mangadex_data) == 0: + mangadex_data = requests.get( + "https://api.mangadex.org/manga", + params={ + "title": manga["title"]["romaji"], + "year": manga["startDate"]["year"], + }, + ).json()["data"] + + cache.set(str(manga["id"]) + "id", mangadex_data) + + if len(mangadex_data) == 0: + available = "?" + else: + mangadex_id = mangadex_data[0]["id"] + manga_chapter_aggregate = cache.get(str(manga["id"]) + "ag") + + if manga_chapter_aggregate is None: + manga_chapter_aggregate = requests.get( + f"https://api.mangadex.org/manga/{mangadex_id}/aggregate", + ).json() + + cache.set(str(manga["id"]) + "ag", manga_chapter_aggregate) + + if "none" in manga_chapter_aggregate["volumes"]: + previous_volume_largest_chapter = list( + manga_chapter_aggregate["volumes"][ + str( + dict(enumerate(manga_chapter_aggregate["volumes"])).get(1) + or "none" + ) + ]["chapters"] + )[0] + available = list( + manga_chapter_aggregate["volumes"]["none"]["chapters"] + )[0] + + if str(available) == "none": + available = list( + manga_chapter_aggregate["volumes"]["none"]["chapters"] + )[1] + else: + try: + available = list( + manga_chapter_aggregate["volumes"][ + str(list(manga_chapter_aggregate["volumes"])[0]) + ]["chapters"] + )[0] + except Exception: + available = "?" + + if not str(available)[0].isdigit(): + if len(manga_chapter_aggregate["volumes"]) == 0: + ids.pop() + + return + + available = math.floor( + float( + list(manga_chapter_aggregate["volumes"]["1"]["chapters"])[0] + ) + ) + + if show_missing is not None and str(available)[0] == "?": + ids.pop() + + return + + # Useful when debugging + # if str(available)[0] != "?": + # ids.pop() + + # return + + if str(available)[0] == "{": + ids.pop() + + return + + if str(available)[0].isdigit(): + available = math.floor(float(available)) + + if math.floor(float(previous_volume_largest_chapter)) > available: + available = math.floor(float(previous_volume_largest_chapter)) + + if str(available) != "?" and int(progress) >= int(available): + ids.pop() + + return + + available_link = ( + available + if mangadex_id is None + else f'{available}' + ) + + current_html.append( + f'
  • {manga["title"]["english"] or manga["title"]["romaji"] or title} {progress} [{available_link}]
    • ' + ) + + joblib.Parallel(n_jobs=int(os.getenv("CONCURRENT_JOBS")) or 4, require="sharedmem")( + joblib.delayed(process)(media) for media in releasing_outdated_manga + ) + + current_html = sorted(current_html, key=lambda x: seen(x, manga=True)) + + current_html.insert(0, "
      ") + current_html.append("
    ") + + return ("".join(current_html), len(ids)) diff --git a/src/due/html/utilities.py b/src/due/html/utilities.py new file mode 100644 index 0000000..367b434 --- /dev/null +++ b/src/due/html/utilities.py @@ -0,0 +1,72 @@ +from flask import request +import re + + +def seen(element, manga=False): + read = 0 + + if manga: + available_matches = re.search(r"\[(\d+)<\/a>\]", element) + read_matches = re.search(r"\[(?=\d+)<\/a>\]", element) + + if read_matches: + read = int(read_matches.group()) + print(read) + + if available_matches: + return int(available_matches.group(1)) - read + else: + return 0 + + available_matches = re.findall(r"\d+\]|\[\d+", element) + seen_matches = re.search( + r"\s(\d+)*?(/(\d+))*\s", element + ) + + if seen_matches: + read = int(seen_matches.group(1)) + + if len(available_matches) > 1: + return int(available_matches[1].strip("[]")) - read + elif len(available_matches) == 1: + return int(available_matches[0].strip("[]")) - read + else: + return 0 + + +def page(main_content, footer): + message = '
    Slow loads? If your media hasn\'t been cached in a while, the first load will take a couple seconds longer than the rest. Subsequent requests on cached media should be faster. Hide forever
    ' + + if request.cookies.get("hide_message") == "1": + message = "" + + return f""" + + + + 期限 + + + + + + + + + + + +

    期限

    + + {main_content} + +

    + +
    + +

    {footer}

    + + {message} + + +""" diff --git a/src/due/media.py b/src/due/media.py deleted file mode 100644 index e97cc83..0000000 --- a/src/due/media.py +++ /dev/null @@ -1,129 +0,0 @@ -import requests - - -def user_id(anilist): - viewer = requests.post( - "https://graphql.anilist.co", - json={"query": "{ Viewer { id } }"}, - headers={ - "Authorization": anilist["token_type"] + " " + anilist["access_token"], - "Content-Type": "application/json", - "Accept": "application/json", - }, - ).json()["data"]["Viewer"] - - if viewer is None: - return -1 - - return int(viewer["id"]) - - -def last_activity(id): - return int( - requests.post( - "https://graphql.anilist.co", - json={ - "query": f"""{{ Activity(userId: {id}, type: MEDIA_LIST, sort: ID_DESC) {{ - __typename ... on ListActivity {{ createdAt }} - }} }}""" - }, - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - }, - ).json()["data"]["Activity"]["createdAt"] - ) - - -def user_name_to_id(name): - return int( - requests.post( - "https://graphql.anilist.co", - json={"query": f'{{ User(name: "{name}") {{ id }} }}'}, - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - }, - ).json()["data"]["User"]["id"] - ) - - -def create_collection(anilist, type, username=None): - current_collection = media_list_collection(anilist, type, username) - current = [] - - for list in current_collection["MediaListCollection"]["lists"]: - current += list["entries"] - - return (current, current_collection["MediaListCollection"]["user"]["name"]) - - -# def media_list(anilist, pageNumber): -# return requests.post( -# "https://graphql.anilist.co", -# json={"query": media_list_query(user_id(anilist), pageNumber)}, -# headers={ -# "Authorization": anilist["token_type"] + " " + anilist["access_token"], -# "Content-Type": "application/json", -# "Accept": "application/json", -# }, -# ).json()["data"] - - -# def media_list_query(user_id: int, page_number: int) -> str: -# return f"""{{ -# Page(page: {page_number}) {{ -# mediaList(userId: {user_id}, status_not_in: [COMPLETED]) {{ -# media {{ -# id -# status -# type -# title {{ romaji english }} -# nextAiringEpisode {{ episode }} -# mediaListEntry {{ progress }} -# startDate {{ year }} -# }} -# }} -# pageInfo {{ hasNextPage }} -# }} -# }}""" - - -def media_list_collection(anilist, type, username=None): - return requests.post( - "https://graphql.anilist.co", - json={ - "query": media_list_collection_query( - user_id(anilist) if username is None else user_name_to_id(username), - type, - ) - }, - headers={ - "Authorization": anilist["token_type"] + " " + anilist["access_token"], - "Content-Type": "application/json", - "Accept": "application/json", - }, - ).json()["data"] - - -def media_list_collection_query(user_id: int, type) -> str: - return f"""{{ - MediaListCollection(userId: {user_id}, type: {"ANIME" if type == "ANIME" else "MANGA"}, status_not_in: [COMPLETED]) {{ - hasNextChunk - lists {{ - entries {{ - media {{ - id - status - type - episodes - title {{ romaji english native }} - nextAiringEpisode {{ episode }} - mediaListEntry {{ progress }} - startDate {{ year }} - }} - }} - }} - user {{ name }} - }} - }}""" diff --git a/src/due/routes/index.py b/src/due/routes/index.py index 49f5acc..c029e01 100644 --- a/src/due/routes/index.py +++ b/src/due/routes/index.py @@ -1,6 +1,9 @@ import time -from due.html import anime_to_html, manga_to_html, page -from due.media import create_collection, last_activity, user_id, user_name_to_id +from due.html.utilities import page +from due.html.anime import anime_to_html +from due.html.manga import manga_to_html +from due.anilist.collection import create_collection +from due.anilist.utilities import last_activity, user_id, user_name_to_id from flask import make_response, redirect, request, Blueprint import json import os -- cgit v1.2.3