projectdx/inflearn
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

인프런 업로드

Browse Source
This commit is contained in:
projectdx
2022-05-13 20:24:03 +09:00
parent 8352abf485
commit 9b39921a1a
13 changed files with 2272 additions and 1410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

292
logic_queue.py
View File

@@ -16,6 +16,7 @@ from datetime import datetime
import requests
# third-party
from .api_youtube_dl import APIYoutubeDL
# sjva 공용
from framework import db, scheduler, path_data
@@ -26,7 +27,7 @@ from framework.logger import get_logger
# 패키지
# from .plugin import package_name, logger
import system
from .model import ModelSetting, ModelLinkkf
from .model import ModelSetting, ModelInflearn
# from plugin import LogicModuleBase, FfmpegQueueEntity, FfmpegQueue, default_route_socketio
@@ -75,7 +76,7 @@ class QueueEntity:
class LogicQueue(object):
download_queue = None
download_thread = None
current_ffmpeg_count = 0
current_youtube_count = 0
def refresh_status(self):
self.module_logic.socketio_callback("status", self.as_dict())
@@ -96,22 +97,8 @@ class LogicQueue(object):
logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
# @staticmethod
# def download_thread_function():
# while True:
# try:
# entity = LogicQueue.download_queue.get()
# logger.debug(
# "Queue receive item:%s %s", entity.title_id, entity.episode_id
# )
# # LogicAni.process(entity)
# LogicQueue.download_queue.task_done()
# except Exception as e:
# logger.error("Exception:%s", e)
# logger.error(traceback.format_exc())
@staticmethod
def download_thread_function():
def download_thread_function() -> None:
headers = None
from . import plugin
@@ -119,42 +106,30 @@ class LogicQueue(object):
while True:
try:
while True:
if LogicQueue.current_ffmpeg_count < int(
if LogicQueue.current_youtube_count < int(
ModelSetting.get("max_ffmpeg_process_count")
):
break
# logger.debug(LogicQueue.current_ffmpeg_count)
time.sleep(5)
entity = LogicQueue.download_queue.get()
# Todo: 고찰
# if entity.cancel:
# continue
# logger.debug(
# "download_thread_function()::entity.info['code'] >> %s", entity
# )
logger.debug(f"entity:: {entity.info}")
if entity is None:
continue
# db에 해당 에피소드가 존재하는 확인
db_entity = ModelLinkkf.get_by_inflearn_id(entity.info["code"])
db_entity = ModelInflearn.get_by_inflearn_id(entity.info["code"])
if db_entity is None:
episode = ModelLinkkf("auto", info=entity.info)
episode = ModelInflearn("auto", info=entity.info)
db.session.add(episode)
db.session.commit()
else:
# episode = ModelLinkkf("auto", info=entity.info)
# query = db.session.query(ModelLinkkf).filter_by(episodecode=entity.info.episodecode).with_for_update().first()
pass
from .logic_inflearn import LogicInflearn
# entity.url = LogicLinkkfYommi.get_video_url(
# entity.info['code'])
# logger.debug(f"entity.info[url] = {entity.info['url']}")
entity.url = LogicInflearn.get_video_url(entity.info["url"])
entity.url = entity.info["url"]
# logger.info('entity.info: %s', entity.info['url'])
logger.debug(f"entity.url: {entity.url}")
@@ -163,8 +138,8 @@ class LogicQueue(object):
# logger.info('entity: %s', entity.__dict__)
# logger.info('entity.url:::> %s', entity.url)
if entity.url[0] is None:
self.ffmpeg_status_kor = "URL실패"
if entity.url is None:
QueueEntity.ffmpeg_status_kor = "URL실패"
plugin.socketio_list_refresh()
continue
@@ -187,8 +162,8 @@ class LogicQueue(object):
logger.debug("program path make fail!!")
# 파일 존재여부 체크
if entity.url[1] is not None:
referer = entity.url[1]
if entity.url is not None:
referer = "http://www.inflearn.com"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
# 'Accept':
@@ -215,7 +190,7 @@ class LogicQueue(object):
# logger.debug(f"referer: {referer}")
f = ffmpeg.Ffmpeg(
entity.url[0],
entity.url,
entity.info["filename"],
plugin_id=entity.entity_id,
listener=LogicQueue.ffmpeg_listener,
@@ -224,36 +199,37 @@ class LogicQueue(object):
call_plugin=package_name,
save_path=save_path,
headers=headers,
# proxy=f" -correct_ts_overflow 0",
)
f.start()
LogicQueue.current_ffmpeg_count += 1
LogicQueue.current_youtube_count += 1
LogicQueue.download_queue.task_done()
# vtt file to srt file
from framework.common.util import write_file, convert_vtt_to_srt
from urllib import parse
ourls = parse.urlparse(entity.url[1])
# print(ourls)
# logger.info('ourls:::>', ourls)
base_url = f"{ourls.scheme}://{ourls.netloc}"
# logger.info('base_url:::>', base_url)
# Todo: 임시 커밋 로직 해결하면 다시 처리
# if "linkkf.app" in base_url:
# base_url = f"{ourls.scheme}://kfani.me"
vtt_url = base_url + entity.url[2]
logger.debug(f"srt:url => {vtt_url}")
srt_filepath = os.path.join(
save_path, entity.info["filename"].replace(".mp4", ".ko.srt")
)
# logger.info('srt_filepath::: %s', srt_filepath)
if entity.url[2] is not None and not os.path.exists(srt_filepath):
vtt_data = requests.get(vtt_url, headers=headers).text
srt_data = convert_vtt_to_srt(vtt_data)
write_file(srt_data, srt_filepath)
# from framework.common.util import write_file, convert_vtt_to_srt
# from urllib import parse
#
# ourls = parse.urlparse(entity.url[1])
# # print(ourls)
# # logger.info('ourls:::>', ourls)
# base_url = f"{ourls.scheme}://{ourls.netloc}"
# # logger.info('base_url:::>', base_url)
#
# # Todo: 임시 커밋 로직 해결하면 다시 처리
# # if "linkkf.app" in base_url:
# # base_url = f"{ourls.scheme}://kfani.me"
#
# vtt_url = base_url + entity.url[2]
# logger.debug(f"srt:url => {vtt_url}")
# srt_filepath = os.path.join(
# save_path, entity.info["filename"].replace(".mp4", ".ko.srt")
# )
# # logger.info('srt_filepath::: %s', srt_filepath)
# if entity.url[2] is not None and not os.path.exists(srt_filepath):
# vtt_data = requests.get(vtt_url, headers=headers).text
# srt_data = convert_vtt_to_srt(vtt_data)
# write_file(srt_data, srt_filepath)
except Exception as e:
logger.error("Exception:%s", e)
@@ -269,7 +245,7 @@ class LogicQueue(object):
if arg["type"] == "status_change":
if arg["status"] == ffmpeg.Status.DOWNLOADING:
episode = (
db.session.query(ModelLinkkf)
db.session.query(ModelInflearn)
.filter_by(episodecode=arg["plugin_id"])
.with_for_update()
.first()
@@ -285,7 +261,7 @@ class LogicQueue(object):
elif arg["type"] == "last":
LogicQueue.current_ffmpeg_count += -1
episode = (
db.session.query(ModelLinkkf)
db.session.query(ModelInflearn)
.filter_by(episodecode=arg["plugin_id"])
.with_for_update()
.first()
@@ -351,6 +327,111 @@ class LogicQueue(object):
# logger.error("Exception:%s", e)
# logger.error(traceback.format_exc())
# return False
@staticmethod
def add_youtube_queue(info):
# logger.debug(type(info))
logger.debug(f"add_youtube_queue():info {info}")
while True:
try:
_temp = info["data"]
if not isinstance(_temp, dict):
_info = json.loads(_temp.replace("'", '"'))
else:
_info = _temp
logger.debug(_info)
_info["code"] = _info["data_id"]
# _info["url"] = _info["hlsUrl"]
# _info["filename"] = _info["name"]
# while True:
# # if LogicQueue.current_youtube_count < int(
# # ModelSetting.get("max_ffmpeg_process_count")
# # ):
# if LogicQueue.current_youtube_count < 3:
# logger.debug("break!!!")
# break
# # logger.debug(LogicQueue.current_ffmpeg_count)
# time.sleep(2)
# entity = LogicQueue.download_queue.get()
# logger.debug(f"entity:: {entity}")
#
# if entity is None:
# continue
logger.debug("add_queue()::info >> %s", _info)
logger.debug(type(_info))
# episode[] code (episode_code)
db_entity = ModelInflearn.get_by_inflearn_id(_info["code"])
if db_entity is None:
# entity = QueueEntity.create(_info)
# download = APIYoutubeDL.download(
# package_name,
# db_entity.code,
# url,
# filename=entity.filename,
# save_path=entity.save_path,
# format_code=entity.format,
# preferredcodec="mp3" if entity.convert_mp3 else None,
# dateafter=date_after,
# playlist="reverse" if entity.playlistreverse else None,
# start=False,
# cookiefile=ModelSetting.get("cookiefile_path"),
# )
# LogicQueue.download_queue.put(entity)
download = APIYoutubeDL.download(
package_name,
_info["code"],
_info["url"],
filename=_info["filename"],
save_path=_info["save_path"],
format_code="mp4",
# preferredcodec="mp3" if entity.convert_mp3 else None,
preferredcodec=None,
dateafter=None,
playlist=None,
start=True,
cookiefile=None,
)
LogicQueue.current_youtube_count += 1
# download.start()
return "enqueue_db_append"
elif db_entity.status != "completed":
logger.debug(f"db_entity.status: {db_entity.status}")
# entity = QueueEntity.create(_info)
# return "Debugging"
# LogicQueue.download_queue.put(entity)
download = APIYoutubeDL.download(
package_name,
_info["code"],
_info["url"],
filename=_info["filename"],
save_path=_info["save_path"],
format_code="mp4",
# preferredcodec="mp3" if entity.convert_mp3 else None,
preferredcodec=None,
dateafter=None,
playlist=None,
start=True,
cookiefile=None,
)
download.start()
logger.debug("add_queue()::enqueue_db_exist")
return "enqueue_db_exist"
else:
return "db_completed"
except Exception as e:
logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
@staticmethod
def add_queue(info):
try:
@@ -358,21 +439,92 @@ class LogicQueue(object):
# Todo:
# if is_exist(info):
# return 'queue_exist'
# logger.debug("add_queue()::info >> %s", info)
# logger.debug("info::", info["_id"])
# logger.debug()
logger.debug(info)
logger.debug(type(info))
_temp = info
_info = json.loads(_temp.replace("'", '"'))
_info["code"] = _info["data_id"]
# _info["url"] = _info["hlsUrl"]
_info["filename"] = _info["name"]
logger.debug("add_queue()::info >> %s", _info)
logger.debug(type(_info))
# episode[] code (episode_code)
db_entity = ModelLinkkf.get_by_inflearn_id(info["code"])
# logger.debug("add_queue:: db_entity >> %s", db_entity)
db_entity = ModelInflearn.get_by_inflearn_id(_info["code"])
logger.debug("add_queue:: db_entity >> %s", db_entity)
if db_entity is None:
entity = QueueEntity.create(info)
entity = QueueEntity.create(_info)
# logger.debug("add_queue()::entity >> %s", entity)
LogicQueue.download_queue.put(entity)
return "enqueue_db_append"
elif db_entity.status != "completed":
entity = QueueEntity.create(info)
logger.debug(f"db_entity.status: {db_entity.status}")
entity = QueueEntity.create(_info)
# return "Debugging"
LogicQueue.download_queue.put(entity)
logger.debug("add_queue()::enqueue_db_exist")
return "enqueue_db_exist"
else:
return "db_completed"
except Exception as e:
logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
return False
@staticmethod
def add_checked_youtube_queue(info):
logger.debug(f"add_checked_youtube_queue():info {info}")
try:
_temp = info["data"]
if not isinstance(_temp, dict):
_info = json.loads(_temp.replace("'", '"'))
else:
_info = _temp
logger.debug(_info)
_info["code"] = _info["data_id"]
logger.debug("add_queue()::info >> %s", _info)
logger.debug(type(_info))
# episode[] code (episode_code)
db_entity = ModelInflearn.get_by_inflearn_id(_info["code"])
logger.debug("add_queue:: db_entity >> %s", db_entity)
if db_entity is None:
# entity = QueueEntity.create(_info)
entity = APIYoutubeDL.download(
package_name,
_info["code"],
_info["url"],
filename=_info["filename"],
save_path=_info["save_path"],
format_code="mp4",
# preferredcodec="mp3" if entity.convert_mp3 else None,
preferredcodec=None,
dateafter=None,
playlist=None,
start=True,
cookiefile=None,
)
# logger.debug("add_queue()::entity >> %s", entity)
LogicQueue.download_queue.put(entity)
return "enqueue_db_append"
elif db_entity.status != "completed":
logger.debug(f"db_entity.status: {db_entity.status}")
entity = QueueEntity.create(_info)
# return "Debugging"
LogicQueue.download_queue.put(entity)