From 10bd5e74122ef4434b13e1dcafcf1b2ff7bf7f13 Mon Sep 17 00:00:00 2001 From: projectdx Date: Wed, 1 Mar 2023 18:01:39 +0900 Subject: [PATCH] =?UTF-8?q?main=20->=202023.03.01=20ohli24=20=EB=B2=84?= =?UTF-8?q?=EA=B7=B8=20=ED=94=BD=EC=8A=A4=20(.01.=20code=20cleanup)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/plugin/ffmpeg_queue.py | 7 +- lib/utils.py | 2 +- logic_linkkf.py | 145 ++++++++++++++---- logic_ohli24.py | 77 +++++++++- .../anime_downloader_linkkf_request.html | 3 + .../anime_downloader_ohli24_setting.html | 1 + 6 files changed, 201 insertions(+), 34 deletions(-) diff --git a/lib/plugin/ffmpeg_queue.py b/lib/plugin/ffmpeg_queue.py index 6b29b4b..e1839dd 100644 --- a/lib/plugin/ffmpeg_queue.py +++ b/lib/plugin/ffmpeg_queue.py @@ -142,8 +142,11 @@ class FfmpegQueue(object): # os.makedirs(save_path) # except: # logger.debug('program path make fail!!') + # 파일 존재여부 체크 - filepath = entity.get_video_filepath() + filepath = str(entity.get_video_filepath()) + self.P.logger.debug(filepath) + self.P.logger.debug(entity.get_video_filepath()) if os.path.exists(filepath): entity.ffmpeg_status_kor = "파일 있음" entity.ffmpeg_percent = 100 @@ -151,6 +154,8 @@ class FfmpegQueue(object): # plugin.socketio_list_refresh() continue dirname = os.path.dirname(filepath) + self.P.logger.debug(type(dirname)) + self.P.logger.debug(dirname) if not os.path.exists(dirname): os.makedirs(dirname) f = ffmpeg.Ffmpeg( diff --git a/lib/utils.py b/lib/utils.py index 7ef1fb0..2162fad 100644 --- a/lib/utils.py +++ b/lib/utils.py @@ -18,7 +18,7 @@ def yommi_timeit(func): total_time = end_time - start_time # print(f"Function {func.__name__}{args} {kwargs} Took {total_time:.4f} secs") logger.opt(colors=True).debug( - f"{func.__name__}{args} {kwargs} function took {total_time:.4f}secs" + f"{func.__name__}{args} {kwargs} function took {total_time:.4f} secs" ) return result diff --git a/logic_linkkf.py b/logic_linkkf.py index c3fd741..a2091bb 100644 --- a/logic_linkkf.py +++ b/logic_linkkf.py @@ -39,6 +39,7 @@ from plugin import ( # 철자가 틀린 부분이 있어서 분리함 # from .lib.plugin import FfmpegQueue, FfmpegQueueEntity +from .lib.utils import yommi_timeit packages = ["beautifulsoup4", "requests-cache", "cloudscraper"] @@ -241,6 +242,7 @@ class LogicLinkkf(LogicModuleBase): P.logger.error(traceback.format_exc()) @staticmethod + @yommi_timeit def get_html(url: str, timeout: int = 10, cached=False): try: @@ -310,7 +312,7 @@ class LogicLinkkf(LogicModuleBase): def add_whitelist(*args): ret = {} - logger.debug(f"args: {args}") + # logger.debug(f"args: {args}") try: if len(args) == 0: @@ -710,7 +712,7 @@ class LogicLinkkf(LogicModuleBase): data = {"ret": "success", "page": page} response_data = LogicLinkkf.get_html(url, timeout=10) # P.logger.debug(response_data) - P.logger.debug("debug.....................") + # P.logger.debug("debug.....................") tree = html.fromstring(response_data) tmp_items = tree.xpath(items_xpath) @@ -758,7 +760,7 @@ class LogicLinkkf(LogicModuleBase): url = "%s/%s" % (P.ModelSetting.get("linkkf_url"), code) logger.info(url) - logger.debug(LogicLinkkf.headers) + # logger.debug(LogicLinkkf.headers) html_content = LogicLinkkf.get_html(url, cached=False) # html_content = LogicLinkkf.get_html_playwright(url) # html_content = LogicLinkkf.get_html_cloudflare(url, cached=False) @@ -862,7 +864,7 @@ class LogicLinkkf(LogicModuleBase): else: tags = soup.select("ul > a") - logger.debug(len(tags)) + logger.debug(f"count: {len(tags)}") # logger.info("tags", tags) # re1 = re.compile(r'\/(?P\d+)') @@ -878,14 +880,16 @@ class LogicLinkkf(LogicModuleBase): idx = 1 for t in tags: entity = { - "_id": data["code"], + "code": data["code"], "program_code": data["code"], "program_title": data["title"], + "day": "", "save_folder": Util.change_text_for_use_filename( data["save_folder"] ), "title": t.text.strip(), - # "title": t.text_content().strip(), + "episode_no": t.text.strip() + # "title": data["title"], } # entity['code'] = re1.search(t.attrib['href']).group('code') @@ -902,9 +906,9 @@ class LogicLinkkf(LogicModuleBase): # logger.debug(f"m_obj::> {m_obj}") if m_obj is not None: episode_code = m_obj.group(1) - entity["code"] = data["code"] + episode_code.zfill(4) + entity["_id"] = data["code"] + episode_code.zfill(4) else: - entity["code"] = data["code"] + entity["_id"] = data["code"] aa = t["href"] if "/player" in aa: @@ -932,6 +936,7 @@ class LogicLinkkf(LogicModuleBase): data["episode"].append(entity) idx = idx + 1 + # logger.debug(f"{data}") data["ret"] = True # logger.info('data', data) self.current_data = data @@ -977,7 +982,7 @@ class LogicLinkkf(LogicModuleBase): ret = "%s.S%sE%s.720p-LK.mp4" % (maintitle, season, epi_no) else: logger.debug("NOT MATCH") - ret = "%s.720p-SA.mp4" % maintitle + ret = "%s.720p-LK.mp4" % maintitle return Util.change_text_for_use_filename(ret) except Exception as e: @@ -985,22 +990,22 @@ class LogicLinkkf(LogicModuleBase): logger.error(traceback.format_exc()) def add(self, episode_info): - logger.debug("episode_info") - logger.debug(episode_info) + # logger.debug("episode_info") + # logger.debug(episode_info) if self.is_exist(episode_info): return "queue_exist" else: - db_entity = ModelLinkkfItem.get_by_linkkf_id(episode_info["_id"]) + db_entity = ModelLinkkfItem.get_by_linkkf_id(episode_info["code"]) - logger.debug("db_entity:::> %s", db_entity) + # logger.debug("db_entity:::> %s", db_entity) # logger.debug("db_entity.status ::: %s", db_entity.status) if db_entity is None: entity = LinkkfQueueEntity(P, self, episode_info) - logger.debug("entity:::> %s", entity.as_dict()) + # logger.debug("entity:::> %s", entity.as_dict()) ModelLinkkfItem.append(entity.as_dict()) # # logger.debug("entity:: type >> %s", type(entity)) # @@ -1036,7 +1041,7 @@ class LogicLinkkf(LogicModuleBase): logger.debug(f"linkkf scheduler_function:: =========================") content_code_list = P.ModelSetting.get_list("linkkf_auto_code_list", "|") - logger.debug(content_code_list) + # logger.debug(content_code_list) if "all" in content_code_list: url = f'{P.ModelSetting.get("linkkf_url")}/dailyani' @@ -1050,7 +1055,7 @@ class LogicLinkkf(LogicModuleBase): # ret_data = LogicOhli24.get_auto_anime_info(self, url=url) content_info = self.get_series_info(item) - logger.debug(content_info["episode"]) + # logger.debug(content_info["episode"]) # exit() for episode_info in content_info["episode"]: @@ -1127,28 +1132,33 @@ class LinkkfQueueEntity(FfmpegQueueEntity): db_entity = ModelLinkkfItem.get_by_linkkf_id(self.info["_id"]) if db_entity is not None: db_entity.status = "completed" - db_entity.complated_time = datetime.now() + db_entity.completed_time = datetime.now() db_entity.save() def donwload_completed(self): db_entity = ModelLinkkfItem.get_by_linkkf_id(self.info["_id"]) if db_entity is not None: db_entity.status = "completed" - db_entity.complated_time = datetime.now() + db_entity.completed_time = datetime.now() db_entity.save() # Get episode info from site def make_episode_info(self): + logger.debug("call make_episode_info(): ") url2s = [] url = None - logger.debug(self.info) - logger.debug(f'self.info:: {self.info["url"]}') - + # logger.debug(self) + # print("") + # logger.debug(self.info) + # logger.debug(f'self.info:: {self.info["url"]}') + # exit() try: # logger.debug(self) # logger.debug(self.url) data = LogicLinkkf.get_html_episode_content(self.info["url"]) + # logger.debug(f"data:: {data}") + # exit() tree = html.fromstring(data) xpath_select_query = '//*[@id="body"]/div/span/center/select/option' @@ -1198,7 +1208,7 @@ class LinkkfQueueEntity(FfmpegQueueEntity): continue # logger.debug(f"url: {url}, url2: {url2}") ret = LogicLinkkf.get_video_url_from_url(url, url2) - logger.debug(f"ret::::> {ret}") + # logger.debug(f"ret::::> {ret}") if ret is not None: video_url = ret @@ -1209,11 +1219,93 @@ class LinkkfQueueEntity(FfmpegQueueEntity): # logger.info(video_url) # return [video_url, referer_url] - return video_url + # return video_url + + logger.debug(video_url) + logger.info("dx: urls2:: %s", url2s) - video_url = None - referer_url = None # dx + self.url = video_url[0] + base_url = "https://kfani.me" + self.srt_url = base_url + video_url[2] + + match = re.compile( + r"(?P.*?)\s*((?P<season>\d+)%s)?\s*((?P<epi_no>\d+)%s)" + % ("기", "화") + ).search(self.info["program_title"]) + # + # epi_no 초기값 + epi_no = 1 + # + + logger.debug(match) + + if match: + self.content_title = match.group("title").strip() + # if "season" in match.groupdict() and match.group("season") is not None: + # self.season = int(match.group("season")) + # + # # epi_no = 1 + # epi_no = int(match.group("epi_no")) + # ret = "%s.S%sE%s.%s-LK.mp4" % ( + # self.content_title, + # "0%s" % self.season if self.season < 10 else self.season, + # "0%s" % epi_no if epi_no < 10 else epi_no, + # self.quality, + # ) + else: + self.content_title = self.info["program_title"] + # P.logger.debug("NOT MATCH") + # ret = "%s.720p-LK.mp4" % self.info["program_title"] + + # logger.info('self.content_title:: %s', self.content_title) + self.epi_queue = epi_no + # self.filename = Util.change_text_for_use_filename(ret) + self.filename = self.info["filename"] + logger.info(f"self.filename::> {self.filename}") + self.savepath = P.ModelSetting.get("linkkf_download_path") + logger.info(f"self.savepath::> {self.savepath}") + + # TODO: 완결 처리 + + folder_name = None + + if P.ModelSetting.get_bool("linkkf_auto_make_folder"): + if self.info["day"].find("완결") != -1: + folder_name = "%s %s" % ( + P.ModelSetting.get("linkkf_finished_insert"), + self.content_title, + ) + else: + folder_name = self.content_title + + # logger.debug(f"folder_name:: {folder_name}") + # logger.debug(f"self.content_title:: {self.content_title}") + + folder_name = Util.change_text_for_use_filename(folder_name.strip()) + self.savepath = os.path.join(self.savepath, folder_name) + if P.ModelSetting.get_bool("linkkf_auto_make_season_folder"): + self.savepath = os.path.join( + self.savepath, "Season %s" % int(self.season) + ) + + self.filepath = os.path.join(self.savepath, self.filename) + if not os.path.exists(self.savepath): + os.makedirs(self.savepath) + + from framework.common.util import write_file, convert_vtt_to_srt + + srt_filepath = os.path.join( + self.savepath, self.filename.replace(".mp4", ".ko.srt") + ) + + if ( + self.srt_url is not None + and not os.path.exists(srt_filepath) + and not ("thumbnails.vtt" in self.srt_url) + ): + srt_data = requests.get(self.srt_url, headers=headers).text + write_file(srt_data, srt_filepath) except Exception as e: logger.error(f"Exception: {str(e)}") @@ -1234,6 +1326,7 @@ class ModelLinkkfItem(db.Model): title = db.Column(db.String) episode_title = db.Column(db.String) # linkkf_va = db.Column(db.String) + linkkf_code = db.Column(db.String) linkkf_vi = db.Column(db.String) linkkf_id = db.Column(db.String) quality = db.Column(db.String) @@ -1329,7 +1422,7 @@ class ModelLinkkfItem(db.Model): @classmethod def append(cls, q): - logger.debug(q) + # logger.debug(q) item = ModelLinkkfItem() item.content_code = q["program_code"] item.season = q["season"] diff --git a/logic_ohli24.py b/logic_ohli24.py index 4e298f6..42eb7f2 100644 --- a/logic_ohli24.py +++ b/logic_ohli24.py @@ -7,12 +7,16 @@ # @Software: PyCharm import os, sys, traceback, re, json, threading +import time from datetime import datetime, date import copy import hashlib +import discord + # third-party import requests +from discord_webhook import DiscordWebhook, DiscordEmbed from lxml import html from urllib import parse import urllib @@ -62,7 +66,7 @@ logger = P.logger class LogicOhli24(LogicModuleBase): db_default = { - "ohli24_db_version": "1", + "ohli24_db_version": "1.1", "ohli24_url": "https://ohli24.org", "ohli24_download_path": os.path.join(path_data, P.package_name, "ohli24"), "ohli24_auto_make_folder": "True", @@ -105,6 +109,11 @@ class LogicOhli24(LogicModuleBase): super(LogicOhli24, self).__init__(P, "setting", scheduler_desc="ohli24 자동 다운로드") self.name = "ohli24" self.queue = None + self.last_post_title = "" + self.discord_webhook_url = "https://discord.com/api/webhooks/1071430127860334663/viCiM5ssS-U1_ONWgdWa-64KgvPfU5jJ8WQAym-4vkiyASB0e8IcnlLnxG4F40nj10kZ" + self.discord_color = "242424" + self.discord_title = "새로운 애니" + self.DISCORD_CHANNEL_ID = "1071430054023798958" default_route_socketio(P, self) @staticmethod @@ -503,10 +512,7 @@ class LogicOhli24(LogicModuleBase): # print(code) whitelist_program = P.ModelSetting.get("ohli24_auto_code_list") - # whitelist_programs = [ - # str(x.strip().replace(" ", "")) - # for x in whitelist_program.replace("\n", "|").split("|") - # ] + whitelist_programs = [ str(x.strip()) for x in whitelist_program.replace("\n", "|").split("|") ] @@ -596,6 +602,8 @@ class LogicOhli24(LogicModuleBase): # ret_data = LogicOhli24.get_auto_anime_info(self, url=url) content_info = self.get_series_info(item, "", "") + logger.debug(content_info) + for episode_info in content_info["episode"]: add_ret = self.add(episode_info) if add_ret.startswith("enqueue"): @@ -666,7 +674,7 @@ class LogicOhli24(LogicModuleBase): else: pass - logger.debug("url:::> %s", url) + # logger.debug("url:::> %s", url) # self.current_headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) # AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/96.0.4664.110 Whale/3.12.129.46 Safari/537.36', @@ -933,6 +941,59 @@ class LogicOhli24(LogicModuleBase): P.logger.error(traceback.format_exc()) return {"ret": "exception", "log": str(e)} + def check_for_new_post(self): + # Get the HTML content of the page + res = requests.get("https://ohli24.org/bbs/board.php?bo_table=ing") + soup = BeautifulSoup(res.content, "html.parser") + + # Find the latest post on the page + latest_post = soup.find("div", class_="post-title").text + latest_post_image = ( + soup.find("div", class_="img-item") + .find("img", class_="wr-img") + .get("src") + .replace("..", "https://ohli24.org") + ) + + logger.debug(f"latest_post:: {latest_post}") + logger.debug(f"self.last_post_title:: {self.last_post_title}") + logger.debug(f"latest_post_image:: {latest_post_image}") + + # Compare the latest post with the last recorded post + if latest_post != self.last_post_title: + # If there is a new post, update the last recorded post + self.last_post_title = latest_post + + # Send a notification to Discord channel + # discord_client = discord.Client() + # discord_client.run(self.DISCORD_BOT_TOKEN) + # + # async def on_ready(): + # channel = discord_client.get_channel(self.DISCORD_CHANNEL_ID) + # await channel.send(f"A new post has been added: {latest_post}") + # + # discord_client.close() + + webhook = DiscordWebhook(url=self.discord_webhook_url) + embed = DiscordEmbed(title=self.discord_title, color=self.discord_color) + embed.set_timestamp() + path = self.last_post_title + embed.set_image(url=latest_post_image) + embed.add_embed_field(name="", value=path, inline=True) + embed.set_timestamp() + webhook.add_embed(embed) + response = webhook.execute() + + return self.last_post_title + return self.last_post_title + + def send_notify(self): + logger.debug("send_notify() routine") + while True: + self.last_post_title = self.check_for_new_post() + logger.debug(self.last_post_title) + time.sleep(600) + # @staticmethod def plugin_load(self): try: @@ -943,6 +1004,10 @@ class LogicOhli24(LogicModuleBase): self.current_data = None self.queue.queue_start() + logger.debug(P.ModelSetting.get_bool("ohli24_discord_notify")) + if P.ModelSetting.get_bool("ohli24_discord_notify"): + self.send_notify() + except Exception as e: logger.error("Exception:%s", e) logger.error(traceback.format_exc()) diff --git a/templates/anime_downloader_linkkf_request.html b/templates/anime_downloader_linkkf_request.html index a2fcc34..a8942cc 100644 --- a/templates/anime_downloader_linkkf_request.html +++ b/templates/anime_downloader_linkkf_request.html @@ -243,7 +243,9 @@ $("#analysis_btn").unbind("click").bind('click', function (e) { e.preventDefault(); e.stopPropagation() + const button = document.getElementById('analysis_btn'); const code = document.getElementById("code").value + button.setAttribute("disabled", "disabled"); console.log(code) $.ajax({ url: '/' + package_name + '/ajax/' + sub + '/analysis', @@ -256,6 +258,7 @@ // {#console.log(ret.code)#} console.log(ret.data) make_program(ret.data) + button.removeAttribute("disabled"); } else { $.notify('<strong>분석 실패</strong><br>' + ret.log, {type: 'warning'}); } diff --git a/templates/anime_downloader_ohli24_setting.html b/templates/anime_downloader_ohli24_setting.html index 3898d5f..820abd7 100644 --- a/templates/anime_downloader_ohli24_setting.html +++ b/templates/anime_downloader_ohli24_setting.html @@ -24,6 +24,7 @@ {{ macros.setting_checkbox('ohli24_auto_make_season_folder', '시즌 폴더 생성', value=arg['ohli24_auto_make_season_folder'], desc=['On : Season 번호 폴더를 만듭니다.']) }} </div> {{ macros.setting_checkbox('ohli24_uncompleted_auto_enqueue', '자동으로 다시 받기', value=arg['ohli24_uncompleted_auto_enqueue'], desc=['On : 플러그인 로딩시 미완료인 항목은 자동으로 다시 받습니다.']) }} + {{ macros.setting_checkbox('ohli24_discord_notify', '디스 코드 알림 받기', value=arg['ohli24_discord_notify'], desc=['On : 새로운 글이 올라올때 디스코드 알림을 보냅니다.']) }} {{ macros.m_tab_content_end() }} {{ macros.m_tab_content_start('auto', false) }}