projectdx/anime_downloader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

anilife update 2022.10.23(01.)

Browse Source
This commit is contained in:
projectdx
2022-10-24 03:59:23 +09:00
parent 6e9b71f4cc
commit 3102ba50ad
1 changed files with 231 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

455
logic_anilife.py
View File

@@ -102,7 +102,7 @@ class LogicAniLife(LogicModuleBase):
} }
useragent = { useragent = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, " "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, "
"like Gecko) Chrome/96.0.4664.110 Whale/3.12.129.46 Safari/537.36" "like Gecko) Chrome/96.0.4664.110 Whale/3.12.129.46 Safari/537.36"
} }
def __init__(self, P): def __init__(self, P):
@@ -112,7 +112,7 @@ class LogicAniLife(LogicModuleBase):
default_route_socketio(P, self) default_route_socketio(P, self)
@staticmethod @staticmethod
def get_html(url, referer=None, stream=False, timeout=5): def get_html(url: str, referer: str = None, stream: bool = False, timeout: int = 5) -> str:
data = "" data = ""
try: try:
print("cloudflare protection bypass ==================") print("cloudflare protection bypass ==================")
@@ -143,7 +143,7 @@ class LogicAniLife(LogicModuleBase):
return data return data
@staticmethod @staticmethod
def get_html_requests(url, referer=None, stream=False, timeout=5): def get_html_requests(url: str, referer: str = None, stream: str = False, timeout: int = 5) -> str:
data = "" data = ""
try: try:
print("get_html_requests ==================") print("get_html_requests ==================")
@@ -173,153 +173,172 @@ class LogicAniLife(LogicModuleBase):
@staticmethod @staticmethod
async def get_html_playwright( async def get_html_playwright(
url, headless=False, referer=None, engine="chrome", stealth=False url: str, headless: bool = False, referer: str = None, engine: str = "chrome", stealth: bool = False
): ) -> str:
from playwright.sync_api import sync_playwright try:
from playwright.async_api import async_playwright from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync, stealth_async from playwright.async_api import async_playwright
from playwright_stealth import stealth_sync, stealth_async
import time import time
browser_args = [ cookie = None
"--window-size=1300,570", browser_args = [
"--window-position=000,000", "--window-size=1300,570",
"--disable-dev-shm-usage", "--window-position=000,000",
"--no-sandbox", "--disable-dev-shm-usage",
"--disable-web-security", "--no-sandbox",
"--disable-features=site-per-process", "--disable-web-security",
"--disable-setuid-sandbox", "--disable-features=site-per-process",
"--disable-accelerated-2d-canvas", "--disable-setuid-sandbox",
"--no-first-run", "--disable-accelerated-2d-canvas",
"--no-zygote", "--no-first-run",
# '--single-process', "--no-zygote",
"--disable-gpu", # '--single-process',
"--use-gl=egl", "--disable-gpu",
"--disable-blink-features=AutomationControlled", "--use-gl=egl",
"--disable-background-networking", "--disable-blink-features=AutomationControlled",
"--enable-features=NetworkService,NetworkServiceInProcess", "--disable-background-networking",
"--disable-background-timer-throttling", "--enable-features=NetworkService,NetworkServiceInProcess",
"--disable-backgrounding-occluded-windows", "--disable-background-timer-throttling",
"--disable-breakpad", "--disable-backgrounding-occluded-windows",
"--disable-client-side-phishing-detection", "--disable-breakpad",
"--disable-component-extensions-with-background-pages", "--disable-client-side-phishing-detection",
"--disable-default-apps", "--disable-component-extensions-with-background-pages",
"--disable-extensions", "--disable-default-apps",
"--disable-features=Translate", "--disable-extensions",
"--disable-hang-monitor", "--disable-features=Translate",
"--disable-ipc-flooding-protection", "--disable-hang-monitor",
"--disable-popup-blocking", "--disable-ipc-flooding-protection",
"--disable-prompt-on-repost", "--disable-popup-blocking",
"--disable-renderer-backgrounding", "--disable-prompt-on-repost",
"--disable-sync", "--disable-renderer-backgrounding",
"--force-color-profile=srgb", "--disable-sync",
"--metrics-recording-only", "--force-color-profile=srgb",
"--enable-automation", "--metrics-recording-only",
"--password-store=basic", "--enable-automation",
"--use-mock-keychain", "--password-store=basic",
"--hide-scrollbars", "--use-mock-keychain",
"--mute-audio", "--hide-scrollbars",
] "--mute-audio",
# scraper = cloudscraper.create_scraper( ]
# browser={"browser": "chrome", "platform": "windows", "desktop": True}, # scraper = cloudscraper.create_scraper(
# debug=False, # browser={"browser": "chrome", "platform": "windows", "desktop": True},
# # sess=LogicAniLife.session, # debug=False,
# delay=10, # # sess=LogicAniLife.session,
# ) # delay=10,
#
# cookie_value, user_agent = scraper.get_cookie_string(url)
#
# logger.debug(f"cookie_value:: {cookie_value}")
start = time.time()
ua = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/69.0.3497.100 Safari/537.36"
)
# from playwright_stealth import stealth_sync
cookie = None
def set_cookie(req):
nonlocal cookie
if "cookie" in req.headers:
cookie = req.headers["cookie"]
async with async_playwright() as p:
if engine == "chrome":
browser = await p.chromium.launch(
channel="chrome", args=browser_args, headless=headless
)
elif engine == "webkit":
browser = await p.webkit.launch(
headless=headless,
args=browser_args,
)
else:
browser = await p.firefox.launch(
headless=headless,
args=browser_args,
)
# context = browser.new_context(
# user_agent=ua,
# ) # )
#
# cookie_value, user_agent = scraper.get_cookie_string(url)
#
# logger.debug(f"cookie_value:: {cookie_value}")
LogicAniLife.headers["Referer"] = "https://anilife.live/detail/id/471" start = time.time()
# print(LogicAniLife.headers) ua = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
LogicAniLife.headers["Referer"] = LogicAniLife.episode_url "AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/69.0.3497.100 Safari/537.36"
if referer is not None:
LogicAniLife.headers["Referer"] = referer
logger.debug(f"LogicAniLife.headers::: {LogicAniLife.headers}")
context = await browser.new_context(extra_http_headers=LogicAniLife.headers)
await context.add_cookies(LogicAniLife.cookies)
# LogicAniLife.headers["Cookie"] = cookie_value
# context.set_extra_http_headers(LogicAniLife.headers)
page = await context.new_page()
# page.set_extra_http_headers(LogicAniLife.headers)
if stealth:
await stealth_async(page)
# page.on("request", set_cookie)
# stealth_sync(page)
print(LogicAniLife.headers["Referer"])
page.on("request", set_cookie)
print(f'Referer:: {LogicAniLife.headers["Referer"]}')
# await page.set_extra_http_headers(LogicAniLife.headers)
await page.goto(
url, wait_until="load", referer=LogicAniLife.headers["Referer"]
) )
# page.wait_for_timeout(10000)
await asyncio.sleep(2.9)
# await page.reload() # from playwright_stealth import stealth_sync
# time.sleep(10) def set_cookie(req):
# cookies = context.cookies nonlocal cookie
# print(cookies) if "cookie" in req.headers:
cookie = req.headers["cookie"]
print(f"page.url:: {page.url}") async with async_playwright() as p:
LogicAniLife.origin_url = page.url try:
if engine == "chrome":
browser = await p.chromium.launch(
channel="chrome", args=browser_args, headless=headless
)
elif engine == "webkit":
browser = await p.webkit.launch(
headless=headless,
args=browser_args,
)
else:
browser = await p.firefox.launch(
headless=headless,
args=browser_args,
)
# context = browser.new_context(
# user_agent=ua,
# )
# print(page.content()) LogicAniLife.headers[
"Referer"
] = "https://anilife.live/detail/id/471"
# print(LogicAniLife.headers)
print(f"run at {time.time() - start} sec") LogicAniLife.headers["Referer"] = LogicAniLife.episode_url
return await page.content() if referer is not None:
LogicAniLife.headers["Referer"] = referer
logger.debug(f"LogicAniLife.headers::: {LogicAniLife.headers}")
context = await browser.new_context(
extra_http_headers=LogicAniLife.headers
)
await context.add_cookies(LogicAniLife.cookies)
# LogicAniLife.headers["Cookie"] = cookie_value
# context.set_extra_http_headers(LogicAniLife.headers)
page = await context.new_page()
# page.set_extra_http_headers(LogicAniLife.headers)
if stealth:
await stealth_async(page)
# page.on("request", set_cookie)
# stealth_sync(page)
print(LogicAniLife.headers["Referer"])
page.on("request", set_cookie)
print(f'Referer:: {LogicAniLife.headers["Referer"]}')
# await page.set_extra_http_headers(LogicAniLife.headers)
await page.goto(
url, wait_until="load", referer=LogicAniLife.headers["Referer"]
)
# page.wait_for_timeout(10000)
await asyncio.sleep(2.9)
# await page.reload()
# time.sleep(10)
# cookies = context.cookies
# print(cookies)
print(f"page.url:: {page.url}")
LogicAniLife.origin_url = page.url
# print(page.content())
print(f"run at {time.time() - start} sec")
return await page.content()
except Exception as e:
logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
finally:
await browser.close()
except Exception as e:
logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
finally:
# browser.close()
pass
@staticmethod @staticmethod
async def get_vod_url_v1( async def get_vod_url_v1(
url, headless=False, referer=None, engine="chrome", stealth=False url, headless=False, referer=None, engine="chrome", stealth=False
): ):
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
@@ -421,7 +440,7 @@ class LogicAniLife(LogicModuleBase):
return await page.content() return await page.content()
@staticmethod @staticmethod
async def get_vod_url(url, headless=False): async def get_vod_url(url: str, headless: bool = False) -> str:
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
from playwright_stealth import stealth_async from playwright_stealth import stealth_async
@@ -488,110 +507,96 @@ class LogicAniLife(LogicModuleBase):
# from playwright_stealth import stealth_sync # from playwright_stealth import stealth_sync
async with async_playwright() as p: async with async_playwright() as p:
# browser = await p.chromium.launch(headless=headless, args=browser_args) try:
browser = await p.chromium.launch(headless=headless, args=browser_args) # browser = await p.chromium.launch(headless=headless, args=browser_args)
browser = await p.chromium.launch(headless=headless, args=browser_args)
# browser = await p.webkit.launch(headless=headless) # browser = await p.webkit.launch(headless=headless)
# context = browser.new_context( # context = browser.new_context(
# user_agent=ua, # user_agent=ua,
# ) # )
LogicAniLife.headers[ LogicAniLife.headers[
"Referer" "Referer"
] = "https://anilife.live/g/l?id=14344143-040a-4e40-9399-a7d22d94554b" ] = "https://anilife.live/g/l?id=14344143-040a-4e40-9399-a7d22d94554b"
# print(LogicAniLife.headers) # print(LogicAniLife.headers)
# context = await browser.new_context(extra_http_headers=LogicAniLife.headers) # context = await browser.new_context(extra_http_headers=LogicAniLife.headers)
context = await browser.new_context() context = await browser.new_context()
await context.set_extra_http_headers(LogicAniLife.headers) await context.set_extra_http_headers(LogicAniLife.headers)
# await context.add_cookies(LogicAniLife.cookies) # await context.add_cookies(LogicAniLife.cookies)
# tracer = HarTracer(context=context, browser_name=p.chromium.name) # tracer = HarTracer(context=context, browser_name=p.chromium.name)
tracer = HarTracer(context=context, browser_name=p.webkit.name) tracer = HarTracer(context=context, browser_name=p.webkit.name)
# LogicAniLife.headers["Cookie"] = cookie_value # LogicAniLife.headers["Cookie"] = cookie_value
# context.set_extra_http_headers(LogicAniLife.headers) # context.set_extra_http_headers(LogicAniLife.headers)
page = await context.new_page() page = await context.new_page()
# await page.set_extra_http_headers(LogicAniLife.headers) # await page.set_extra_http_headers(LogicAniLife.headers)
# await stealth_async(page) # await stealth_async(page)
# logger.debug(url) # logger.debug(url)
# page.on("request", set_cookie) # page.on("request", set_cookie)
# stealth_sync(page) # stealth_sync(page)
# await page.goto( # await page.goto(
# url, wait_until="load", referer=LogicAniLife.headers["Referer"] # url, wait_until="load", referer=LogicAniLife.headers["Referer"]
# ) # )
# await page.goto(url, wait_until="load") # await page.goto(url, wait_until="load")
await page.goto(url, wait_until="domcontentloaded") await page.goto(url, wait_until="domcontentloaded")
har = await tracer.flush() har = await tracer.flush()
# page.wait_for_timeout(10000) # page.wait_for_timeout(10000)
await asyncio.sleep(2) await asyncio.sleep(2)
# logger.debug(har) # logger.debug(har)
# page.reload() # page.reload()
# time.sleep(10) # time.sleep(10)
# cookies = context.cookies # cookies = context.cookies
# print(cookies) # print(cookies)
# print(page.content()) # print(page.content())
# vod_url = page.evaluate( # vod_url = page.evaluate(
# """() => { # """() => {
# return console.log(vodUrl_1080p) }""" # return console.log(vodUrl_1080p) }"""
# ) # )
# vod_url = page.evaluate( # vod_url = page.evaluate(
# """async () =>{ # """async () =>{
# return _0x55265f(0x99) + alJson[_0x55265f(0x91)] # return _0x55265f(0x99) + alJson[_0x55265f(0x91)]
# }""" # }"""
# ) # )
result_har_json = har.to_json() result_har_json = har.to_json()
result_har_dict = har.to_dict() result_har_dict = har.to_dict()
# logger.debug(result_har_dict) # logger.debug(result_har_dict)
tmp_video_url = [] tmp_video_url = []
for i, elem in enumerate(result_har_dict["log"]["entries"]): for i, elem in enumerate(result_har_dict["log"]["entries"]):
if "m3u8" in elem["request"]["url"]: if "m3u8" in elem["request"]["url"]:
logger.debug(elem["request"]["url"]) logger.debug(elem["request"]["url"])
tmp_video_url.append(elem["request"]["url"]) tmp_video_url.append(elem["request"]["url"])
vod_url = tmp_video_url[-1] vod_url = tmp_video_url[-1]
logger.debug(f"vod_url:: {vod_url}") logger.debug(f"vod_url:: {vod_url}")
logger.debug(f"run at {time.time() - start} sec") logger.debug(f"run at {time.time() - start} sec")
# html_content = LogicAniLife.get_html_selenium( return vod_url
# vod_url, "https://anilife.live" except Exception as e:
# ) logger.error("Exception:%s", e)
logger.error(traceback.format_exc())
# html_content = LogicAniLife.get_html_playwright( finally:
# vod_url, False, referer="https://anilife.live" await browser.close()
# )
# html_content = LogicAniLife.get_html(
# vod_url, referer="https://anilife.live"
# )
# html_content = LogicAniLife.get_html_requests(
# vod_url, referer="https://anilife.live"
# )
# print(f"html_content:: {html_content}")
# output_json = html_to_json.convert(html_content)
# resolution = output_json["html"][0]["body"][0]["_value"]
# logger.debug(f"output_json:: {resolution}")
return vod_url
@staticmethod @staticmethod
def get_html_selenium(url, referer): def get_html_selenium(url: str, referer: str) -> str:
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium import webdriver from selenium import webdriver
from selenium_stealth import stealth from selenium_stealth import stealth
@@ -654,6 +659,8 @@ class LogicAniLife(LogicModuleBase):
elem = driver.find_element(By.XPATH, "//*") elem = driver.find_element(By.XPATH, "//*")
source_code = elem.get_attribute("outerHTML") source_code = elem.get_attribute("outerHTML")
driver.close()
return source_code.encode("utf-8") return source_code.encode("utf-8")
# Create a request interceptor # Create a request interceptor
@@ -846,7 +853,7 @@ class LogicAniLife(LogicModuleBase):
def setting_save_after(self): def setting_save_after(self):
if self.queue.get_max_ffmpeg_count() != P.ModelSetting.get_int( if self.queue.get_max_ffmpeg_count() != P.ModelSetting.get_int(
"anilife_max_ffmpeg_process_count" "anilife_max_ffmpeg_process_count"
): ):
self.queue.set_max_ffmpeg_count( self.queue.set_max_ffmpeg_count(
P.ModelSetting.get_int("anilife_max_ffmpeg_process_count") P.ModelSetting.get_int("anilife_max_ffmpeg_process_count")
@@ -1025,16 +1032,16 @@ class LogicAniLife(LogicModuleBase):
) )
elif cate == "theater": elif cate == "theater":
url = ( url = (
P.ModelSetting.get("anilife_url") P.ModelSetting.get("anilife_url")
+ "/vodtype/categorize/Movie/" + "/vodtype/categorize/Movie/"
+ page + page
) )
wrapper_xpath = '//div[@class="bsx"]' wrapper_xpath = '//div[@class="bsx"]'
else: else:
url = ( url = (
P.ModelSetting.get("anilife_url") P.ModelSetting.get("anilife_url")
+ "/vodtype/categorize/Movie/" + "/vodtype/categorize/Movie/"
+ page + page
) )
# cate == "complete": # cate == "complete":
logger.info("url:::> %s", url) logger.info("url:::> %s", url)