first commit
This commit is contained in:
446
plugin/ffmpeg/interface_program_ffmpeg.py
Executable file
446
plugin/ffmpeg/interface_program_ffmpeg.py
Executable file
@@ -0,0 +1,446 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#########################################################
|
||||
# python
|
||||
import os
|
||||
import traceback
|
||||
import threading
|
||||
import subprocess
|
||||
import platform
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pytz import timezone
|
||||
|
||||
# third-party
|
||||
import requests
|
||||
|
||||
|
||||
#from flask import Blueprint, request, Response, send_file, render_template, redirect, jsonify
|
||||
|
||||
# sjva 공용
|
||||
from framework.logger import get_logger
|
||||
from framework.util import Util
|
||||
from framework import app
|
||||
|
||||
# 패키지
|
||||
from ffmpeg.logic import Logic, Status
|
||||
from ffmpeg.model import ModelSetting
|
||||
|
||||
# 로그
|
||||
package_name = __name__.split('.')[0]
|
||||
logger = get_logger(package_name)
|
||||
#########################################################
|
||||
|
||||
|
||||
class Ffmpeg(object):
|
||||
instance_list = []
|
||||
idx = 1
|
||||
# retry : 재시도 횟수
|
||||
# max_error_packet_count : 이 숫자 초과시 중단
|
||||
# where : 호출 모듈
|
||||
def __init__(self, url, filename, plugin_id=None, listener=None, max_pf_count=None, call_plugin=None, temp_path=None, save_path=None, proxy=None, headers=None):
|
||||
self.thread = None
|
||||
self.url = url
|
||||
self.filename = filename
|
||||
self.plugin_id = plugin_id
|
||||
self.listener = listener
|
||||
self.max_pf_count = int(ModelSetting.query.filter_by(key='max_pf_count').first().value if max_pf_count is None else max_pf_count)
|
||||
self.call_plugin = call_plugin
|
||||
self.process = None
|
||||
self.temp_path = ModelSetting.query.filter_by(key='temp_path').first().value if temp_path is None else temp_path
|
||||
self.save_path = ModelSetting.query.filter_by(key='save_path').first().value if save_path is None else save_path
|
||||
self.proxy = proxy
|
||||
self.temp_fullpath = os.path.join(self.temp_path, filename)
|
||||
self.save_fullpath = os.path.join(self.save_path, filename)
|
||||
self.log_thread = None
|
||||
self.status = Status.READY
|
||||
self.duration = 0
|
||||
self.duration_str = ''
|
||||
self.current_duration = 0
|
||||
self.percent = 0
|
||||
#self.log = []
|
||||
self.current_pf_count = 0
|
||||
self.idx = str(Ffmpeg.idx)
|
||||
Ffmpeg.idx += 1
|
||||
self.current_bitrate = ''
|
||||
self.current_speed = ''
|
||||
self.start_time = None
|
||||
self.end_time = None
|
||||
self.download_time = None
|
||||
self.start_event = threading.Event()
|
||||
self.exist = False
|
||||
self.filesize = 0
|
||||
self.filesize_str = ''
|
||||
self.download_speed = ''
|
||||
self.headers = headers
|
||||
|
||||
Ffmpeg.instance_list.append(self)
|
||||
#logger.debug('Ffmpeg.instance_list LEN:%s', len(Ffmpeg.instance_list))
|
||||
if len(Ffmpeg.instance_list) > 30:
|
||||
for f in Ffmpeg.instance_list:
|
||||
if f.thread is None and f.status != Status.READY:
|
||||
Ffmpeg.instance_list.remove(f)
|
||||
break
|
||||
else:
|
||||
logger.debug('remove fail %s %s', f.thread, self.status)
|
||||
|
||||
def start(self):
|
||||
self.thread = threading.Thread(target=self.thread_fuction, args=())
|
||||
self.thread.start()
|
||||
#self.start_event.wait(timeout=10)
|
||||
self.start_time = datetime.now()
|
||||
return self.get_data()
|
||||
|
||||
def start_and_wait(self):
|
||||
self.start()
|
||||
self.thread.join(timeout=60*70)
|
||||
|
||||
def stop(self):
|
||||
try:
|
||||
self.status = Status.USER_STOP
|
||||
self.kill()
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
def kill(self):
|
||||
try:
|
||||
if self.process is not None and self.process.poll() is None:
|
||||
import psutil
|
||||
process = psutil.Process(self.process.pid)
|
||||
for proc in process.children(recursive=True):
|
||||
proc.kill()
|
||||
process.kill()
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
|
||||
def thread_fuction(self):
|
||||
try:
|
||||
import system
|
||||
user = str(system.ModelSetting.get('sjva_me_user_id'))
|
||||
try:
|
||||
from framework.common.util import AESCipher
|
||||
user = AESCipher.encrypt(user)
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
if self.proxy is None:
|
||||
if self.headers is None:
|
||||
command = [ModelSetting.get('ffmpeg_path'), '-y', '-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-metadata', 'network=%s' % user]
|
||||
else:
|
||||
headers_command = []
|
||||
for key, value in self.headers.items():
|
||||
if key.lower() == 'user-agent':
|
||||
headers_command.append('-user_agent')
|
||||
headers_command.append(value)
|
||||
pass
|
||||
else:
|
||||
headers_command.append('-headers')
|
||||
#headers_command.append('\'%s: "%s"\''%(key,value))
|
||||
if platform.system() == 'Windows':
|
||||
headers_command.append('\'%s:%s\''%(key,value))
|
||||
else:
|
||||
headers_command.append(f'{key}:{value}')
|
||||
command = [ModelSetting.get('ffmpeg_path'), '-y'] + headers_command + ['-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-metadata', 'network=%s' % user]
|
||||
else:
|
||||
command = [ModelSetting.get('ffmpeg_path'), '-y', '-http_proxy', self.proxy, '-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-metadata', 'network=%s' % user]
|
||||
#command = [ModelSetting.get('ffmpeg_path'), '-y', '-i', self.url, '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
|
||||
if platform.system() == 'Windows':
|
||||
#if self.call_plugin.startswith('pooq'):
|
||||
now = str(datetime.now()).replace(':', '').replace('-', '').replace(' ', '-')
|
||||
#filename = ('%s' % now).split('.')[0] + '.mp4'
|
||||
#동시에 요청시 구분이 안됨. 2019-10-11
|
||||
filename = ('%s' % now) + '.mp4'
|
||||
self.temp_fullpath = os.path.join(self.temp_path, filename)
|
||||
command.append(self.temp_fullpath)
|
||||
#else:
|
||||
# command.append(self.temp_fullpath.encode('cp949'))
|
||||
else:
|
||||
command.append(self.temp_fullpath)
|
||||
|
||||
try:
|
||||
logger.debug(' '.join(command))
|
||||
#logger.debug(command)
|
||||
if os.path.exists(self.temp_fullpath):
|
||||
for f in Ffmpeg.instance_list:
|
||||
if f.idx != self.idx and f.temp_fullpath == self.temp_fullpath and f.status in [Status.DOWNLOADING, Status.READY]:
|
||||
self.status = Status.ALREADY_DOWNLOADING
|
||||
#logger.debug('temp_fullpath ALREADY_DOWNLOADING')
|
||||
return
|
||||
except:
|
||||
pass
|
||||
#self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1)
|
||||
self.process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8')
|
||||
|
||||
self.status = Status.READY
|
||||
self.log_thread = threading.Thread(target=self.log_thread_fuction, args=())
|
||||
self.log_thread.start()
|
||||
self.start_event.wait(timeout=60)
|
||||
#logger.debug('start_event awake.. ')
|
||||
if self.log_thread is None:
|
||||
#logger.debug('log_thread is none')
|
||||
#logger.debug(self.log)
|
||||
if self.status == Status.READY:
|
||||
self.status = Status.ERROR
|
||||
self.kill()
|
||||
elif self.status == Status.READY:
|
||||
#logger.debug('still status.ready kill')
|
||||
#logger.debug(self.log)
|
||||
self.status = Status.ERROR
|
||||
self.kill()
|
||||
else:
|
||||
#logger.debug('normally process wait()')
|
||||
#process_ret = self.process.wait(timeout=60*60)
|
||||
process_ret = self.process.wait(timeout=60*ModelSetting.get_int('timeout_minute'))
|
||||
#logger.debug('process_ret :%s' % process_ret)
|
||||
if process_ret is None: # timeout
|
||||
if self.status != Status.COMPLETED and self.status != Status.USER_STOP and self.status != Status.PF_STOP:
|
||||
self.status = Status.TIME_OVER
|
||||
self.kill()
|
||||
else:
|
||||
#logger.debug('process end')
|
||||
if self.status == Status.DOWNLOADING:
|
||||
self.status = Status.FORCE_STOP
|
||||
self.end_time = datetime.now()
|
||||
self.download_time = self.end_time - self.start_time
|
||||
try:
|
||||
if self.status == Status.COMPLETED:
|
||||
if self.save_fullpath != self.temp_fullpath:
|
||||
if os.path.exists(self.save_fullpath):
|
||||
os.remove(self.save_fullpath)
|
||||
#if app.config['config']['is_py2']:
|
||||
# os.chmod(self.temp_fullpath, 0777)
|
||||
#else:
|
||||
# os.chmod(self.temp_fullpath, 777)
|
||||
os.system('chmod 777 "%s"' % self.temp_fullpath)
|
||||
# 2020-03-19 씹힌다 ㅠ
|
||||
#import framework.common.celery as celery_task
|
||||
#celery_task.move(self.temp_fullpath, self.save_fullpath)
|
||||
import shutil
|
||||
shutil.move(self.temp_fullpath, self.save_fullpath)
|
||||
|
||||
# 2019-05-16
|
||||
#os.chmod(self.save_fullpath, 0777)
|
||||
self.filesize = os.stat(self.save_fullpath).st_size
|
||||
else:
|
||||
if os.path.exists(self.temp_fullpath):
|
||||
os.remove(self.temp_fullpath)
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
arg = {'type':'last', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
self.process = None
|
||||
self.thread = None
|
||||
|
||||
#파일처리
|
||||
|
||||
#logger.debug('ffmpeg thread end')
|
||||
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
try:
|
||||
self.status = Status.EXCEPTION
|
||||
arg = {'type':'last', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
except:
|
||||
pass
|
||||
#self.start_event.set()
|
||||
|
||||
|
||||
def log_thread_fuction(self):
|
||||
with self.process.stdout:
|
||||
iter_arg = b'' if app.config['config']['is_py2'] else ''
|
||||
for line in iter(self.process.stdout.readline, iter_arg):
|
||||
try:
|
||||
#logger.debug(line)
|
||||
#logger.debug('XXXXXXXXXXXXXX %s', self.status)
|
||||
#self.log.append(line.strip())
|
||||
#arg = {'type':'log', 'status':self.status, 'data' : {'idx':self.idx, 'line':line}}
|
||||
#self.send_to_listener(**arg)
|
||||
if self.status == Status.READY:
|
||||
if line.find('Server returned 404 Not Found') != -1 or line.find('Unknown error') != -1:
|
||||
self.status = Status.WRONG_URL
|
||||
self.start_event.set()
|
||||
#logger.debug('start_event set by 404 not found')
|
||||
elif line.find('No such file or directory') != -1:
|
||||
self.status = Status.WRONG_DIRECTORY
|
||||
self.start_event.set()
|
||||
#logger.debug('start_event set by WRONG_DIR')
|
||||
else:
|
||||
match = re.compile(r'Duration\:\s(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\,\sstart').search(line)
|
||||
if match:
|
||||
self.duration_str = '%s:%s:%s' % ( match.group(1), match.group(2), match.group(3))
|
||||
self.duration = int(match.group(4))
|
||||
self.duration += int(match.group(3)) * 100
|
||||
self.duration += int(match.group(2)) * 100 * 60
|
||||
self.duration += int(match.group(1)) * 100 * 60 * 60
|
||||
#logger.debug('Duration : %s', self.duration)
|
||||
if match:
|
||||
self.status = Status.READY
|
||||
arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
continue
|
||||
#if self.listener is not None:
|
||||
# #arg = {'status':self.status, 'duration' : self.duration}
|
||||
# arg = {'status':self.status, 'data' : self.get_data()}
|
||||
# self.listener(**arg)
|
||||
#
|
||||
#self.status = 'DOWNLOADING'
|
||||
match = re.compile(r'time\=(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\sbitrate\=\s*(?P<bitrate>\d+).*?[$|\s](\s?speed\=\s*(?P<speed>.*?)x)?').search(line)
|
||||
if match:
|
||||
self.status = Status.DOWNLOADING
|
||||
arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
self.start_event.set()
|
||||
#logger.debug('start_event set by DOWNLOADING')
|
||||
elif self.status == Status.DOWNLOADING:
|
||||
if line.find('PES packet size mismatch') != -1:
|
||||
self.current_pf_count += 1
|
||||
if self.current_pf_count > self.max_pf_count:
|
||||
#logger.debug('%s : PF_STOP!', self.idx)
|
||||
self.status = Status.PF_STOP
|
||||
self.kill()
|
||||
continue
|
||||
if line.find('HTTP error 403 Forbidden') != -1:
|
||||
#logger.debug('HTTP error 403 Forbidden')
|
||||
self.status = Status.HTTP_FORBIDDEN
|
||||
self.kill()
|
||||
continue
|
||||
match = re.compile(r'time\=(\d{2})\:(\d{2})\:(\d{2})\.(\d{2})\sbitrate\=\s*(?P<bitrate>\d+).*?[$|\s](\s?speed\=\s*(?P<speed>.*?)x)?').search(line)
|
||||
if match:
|
||||
self.current_duration = int(match.group(4))
|
||||
self.current_duration += int(match.group(3)) * 100
|
||||
self.current_duration += int(match.group(2)) * 100 * 60
|
||||
self.current_duration += int(match.group(1)) * 100 * 60 * 60
|
||||
try:
|
||||
self.percent = int(self.current_duration * 100 / self.duration)
|
||||
except: pass
|
||||
#logger.debug('Current Duration : %s %s %s%%', self.duration, self.current_duration, self.percent)
|
||||
self.current_bitrate = match.group('bitrate')
|
||||
self.current_speed = match.group('speed')
|
||||
self.download_time = datetime.now() - self.start_time
|
||||
arg = {'type':'normal', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
continue
|
||||
match = re.compile(r'video\:\d*kB\saudio\:\d*kB').search(line)
|
||||
if match:
|
||||
self.status = Status.COMPLETED
|
||||
self.end_time = datetime.now()
|
||||
self.download_time = self.end_time - self.start_time
|
||||
self.percent = 100
|
||||
arg = {'type':'status_change', 'status':self.status, 'data' : self.get_data()}
|
||||
self.send_to_listener(**arg)
|
||||
continue
|
||||
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
#logger.debug('ffmpeg log thread end')
|
||||
self.start_event.set()
|
||||
self.log_thread = None
|
||||
|
||||
def get_data(self):
|
||||
data = {
|
||||
'url' : self.url,
|
||||
'filename' : self.filename,
|
||||
'max_pf_count' : self.max_pf_count,
|
||||
'call_plugin' : self.call_plugin,
|
||||
'temp_path' : self.temp_path,
|
||||
'save_path' : self.save_path,
|
||||
'temp_fullpath' : self.temp_fullpath,
|
||||
'save_fullpath' : self.save_fullpath,
|
||||
'status' : int(self.status),
|
||||
'status_str' : self.status.name,
|
||||
'status_kor' : str(self.status),
|
||||
'duration' : self.duration,
|
||||
'duration_str' : self.duration_str,
|
||||
'current_duration' : self.current_duration,
|
||||
'percent' : self.percent,
|
||||
'current_pf_count' : self.current_pf_count,
|
||||
'idx' : self.idx,
|
||||
#'log' : self.log,
|
||||
'current_bitrate' : self.current_bitrate,
|
||||
'current_speed' : self.current_speed,
|
||||
'start_time' : '' if self.start_time is None else str(self.start_time).split('.')[0][5:],
|
||||
'end_time' : '' if self.end_time is None else str(self.end_time).split('.')[0][5:],
|
||||
'download_time' : '' if self.download_time is None else '%02d:%02d' % (self.download_time.seconds/60, self.download_time.seconds%60),
|
||||
'exist' : os.path.exists(self.save_fullpath),
|
||||
}
|
||||
if self.status == Status.COMPLETED:
|
||||
data['filesize'] = self.filesize
|
||||
data['filesize_str'] = Util.sizeof_fmt(self.filesize)
|
||||
data['download_speed'] = Util.sizeof_fmt(self.filesize/self.download_time.seconds, suffix='Bytes/Second')
|
||||
return data
|
||||
|
||||
def send_to_listener(self, **arg):
|
||||
Logic.ffmpeg_listener(**arg)
|
||||
if self.listener is not None:
|
||||
arg['plugin_id'] = self.plugin_id
|
||||
self.listener(**arg)
|
||||
|
||||
@staticmethod
|
||||
def get_version():
|
||||
try:
|
||||
command = u'%s -version' % (ModelSetting.get('ffmpeg_path'))
|
||||
command = command.split(' ')
|
||||
#logger.debug(command)
|
||||
if app.config['config']['is_py2']:
|
||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1)
|
||||
else:
|
||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8')
|
||||
iter_arg = b'' if app.config['config']['is_py2'] else ''
|
||||
ret = []
|
||||
with process.stdout:
|
||||
for line in iter(process.stdout.readline, iter_arg):
|
||||
ret.append(line)
|
||||
process.wait() # wait for the subprocess to exit
|
||||
return ret
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
@staticmethod
|
||||
def stop_by_idx(idx):
|
||||
try:
|
||||
for f in Ffmpeg.instance_list:
|
||||
if f.idx == idx:
|
||||
f.stop()
|
||||
break
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
@staticmethod
|
||||
def ffmpeg_by_idx(idx):
|
||||
try:
|
||||
for f in Ffmpeg.instance_list:
|
||||
if f.idx == idx:
|
||||
return f
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_ffmpeg_by_caller(caller, caller_id):
|
||||
try:
|
||||
for f in Ffmpeg.instance_list:
|
||||
if f.plugin_id == caller_id and f.call_plugin == caller:
|
||||
return f
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
|
||||
@staticmethod
|
||||
def plugin_unload():
|
||||
try:
|
||||
for f in Ffmpeg.instance_list:
|
||||
f.stop()
|
||||
except Exception as exception:
|
||||
logger.error('Exception:%s', exception)
|
||||
logger.error(traceback.format_exc())
|
||||
Reference in New Issue
Block a user