mirror of https://github.com/ytdl-org/youtube-dl
Compare commits
25 Commits
16a37db1dd
...
ca6f85e396
Author | SHA1 | Date |
---|---|---|
dirkf | ca6f85e396 | |
dirkf | e0727e4ab6 | |
Ori Avtalion | 4ea59c6107 | |
dirkf | 21792b88b7 | |
dirkf | d8f134a664 | |
dirkf | 31a15a7c8d | |
dirkf | 19dc10b986 | |
dirkf | 182f63e82a | |
gy-chen | 71211e7db7 | |
Zizheng Guo | a96a45b2cd | |
hatsomatt | 820fae3b3a | |
dirkf | aef24d97e9 | |
dirkf | f7b30e3f73 | |
dirkf | f66372403f | |
dirkf | 7216fa2ac4 | |
dirkf | acc383b9e3 | |
Hubert Hirtz | f0812d7848 | |
Aaron Tan | 40bd5c1815 | |
dirkf | 70f230f9cf | |
dirkf | 48ddab1f3a | |
dirkf | 7687389f08 | |
dirkf | 108737d600 | |
dirkf | 015954f21a | |
dirkf | ebf0fcd916 | |
dirkf | b6a24ad93d |
|
@ -18,6 +18,7 @@ from test.helper import (
|
|||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.compat import (
|
||||
compat_contextlib_suppress,
|
||||
compat_http_cookiejar_Cookie,
|
||||
compat_http_server,
|
||||
compat_kwargs,
|
||||
|
@ -35,6 +36,9 @@ from youtube_dl.downloader.external import (
|
|||
HttpieFD,
|
||||
WgetFD,
|
||||
)
|
||||
from youtube_dl.postprocessor import (
|
||||
FFmpegPostProcessor,
|
||||
)
|
||||
import threading
|
||||
|
||||
TEST_SIZE = 10 * 1024
|
||||
|
@ -227,7 +231,17 @@ class TestAria2cFD(unittest.TestCase):
|
|||
self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
|
||||
|
||||
|
||||
@ifExternalFDAvailable(FFmpegFD)
|
||||
# Handle delegated availability
|
||||
def ifFFmpegFDAvailable(externalFD):
|
||||
# raise SkipTest, or set False!
|
||||
avail = ifExternalFDAvailable(externalFD) and False
|
||||
with compat_contextlib_suppress(Exception):
|
||||
avail = FFmpegPostProcessor(downloader=None).available
|
||||
return unittest.skipUnless(
|
||||
avail, externalFD.get_basename() + ' not found')
|
||||
|
||||
|
||||
@ifFFmpegFDAvailable(FFmpegFD)
|
||||
class TestFFmpegFD(unittest.TestCase):
|
||||
_args = []
|
||||
|
||||
|
|
|
@ -81,6 +81,7 @@ from youtube_dl.utils import (
|
|||
sanitize_filename,
|
||||
sanitize_path,
|
||||
sanitize_url,
|
||||
sanitized_Request,
|
||||
shell_quote,
|
||||
smuggle_url,
|
||||
str_or_none,
|
||||
|
@ -255,6 +256,18 @@ class TestUtil(unittest.TestCase):
|
|||
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
||||
self.assertEqual(sanitize_url('foo bar'), 'foo bar')
|
||||
|
||||
def test_sanitized_Request(self):
|
||||
self.assertFalse(sanitized_Request('http://foo.bar').has_header('Authorization'))
|
||||
self.assertFalse(sanitized_Request('http://:foo.bar').has_header('Authorization'))
|
||||
self.assertEqual(sanitized_Request('http://@foo.bar').get_header('Authorization'),
|
||||
'Basic Og==')
|
||||
self.assertEqual(sanitized_Request('http://:pass@foo.bar').get_header('Authorization'),
|
||||
'Basic OnBhc3M=')
|
||||
self.assertEqual(sanitized_Request('http://user:@foo.bar').get_header('Authorization'),
|
||||
'Basic dXNlcjo=')
|
||||
self.assertEqual(sanitized_Request('http://user:pass@foo.bar').get_header('Authorization'),
|
||||
'Basic dXNlcjpwYXNz')
|
||||
|
||||
def test_expand_path(self):
|
||||
def env(var):
|
||||
return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
|
||||
|
@ -512,11 +525,14 @@ class TestUtil(unittest.TestCase):
|
|||
self.assertEqual(float_or_none(set()), None)
|
||||
|
||||
def test_int_or_none(self):
|
||||
self.assertEqual(int_or_none(42), 42)
|
||||
self.assertEqual(int_or_none('42'), 42)
|
||||
self.assertEqual(int_or_none(''), None)
|
||||
self.assertEqual(int_or_none(None), None)
|
||||
self.assertEqual(int_or_none([]), None)
|
||||
self.assertEqual(int_or_none(set()), None)
|
||||
self.assertEqual(int_or_none('42', base=8), 34)
|
||||
self.assertRaises(TypeError, int_or_none(42, base=8))
|
||||
|
||||
def test_str_to_int(self):
|
||||
self.assertEqual(str_to_int('123,456'), 123456)
|
||||
|
|
|
@ -2421,29 +2421,26 @@ except ImportError: # Python 2
|
|||
compat_urllib_request_urlretrieve = compat_urlretrieve
|
||||
|
||||
try:
|
||||
from HTMLParser import (
|
||||
HTMLParser as compat_HTMLParser,
|
||||
HTMLParseError as compat_HTMLParseError)
|
||||
except ImportError: # Python 3
|
||||
from html.parser import HTMLParser as compat_HTMLParser
|
||||
except ImportError: # Python 2
|
||||
from HTMLParser import HTMLParser as compat_HTMLParser
|
||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||
|
||||
try: # Python 2
|
||||
from HTMLParser import HTMLParseError as compat_HTMLParseError
|
||||
except ImportError: # Python <3.4
|
||||
try:
|
||||
from html.parser import HTMLParseError as compat_HTMLParseError
|
||||
except ImportError: # Python >3.4
|
||||
|
||||
# HTMLParseError has been deprecated in Python 3.3 and removed in
|
||||
# HTMLParseError was deprecated in Python 3.3 and removed in
|
||||
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
||||
# and uniform cross-version exception handling
|
||||
class compat_HTMLParseError(Exception):
|
||||
pass
|
||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||
compat_html_parser_HTMLParseError = compat_HTMLParseError
|
||||
|
||||
try:
|
||||
from subprocess import DEVNULL
|
||||
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
||||
except ImportError:
|
||||
_DEVNULL = subprocess.DEVNULL
|
||||
compat_subprocess_get_DEVNULL = lambda: _DEVNULL
|
||||
except AttributeError:
|
||||
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
|
||||
|
||||
try:
|
||||
|
@ -2943,6 +2940,51 @@ else:
|
|||
compat_socket_create_connection = socket.create_connection
|
||||
|
||||
|
||||
try:
|
||||
from contextlib import suppress as compat_contextlib_suppress
|
||||
except ImportError:
|
||||
class compat_contextlib_suppress(object):
|
||||
_exceptions = None
|
||||
|
||||
def __init__(self, *exceptions):
|
||||
super(compat_contextlib_suppress, self).__init__()
|
||||
# TODO: [Base]ExceptionGroup (3.12+)
|
||||
self._exceptions = exceptions
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
|
||||
|
||||
|
||||
# subprocess.Popen context manager
|
||||
# avoids leaking handles if .communicate() is not called
|
||||
try:
|
||||
_Popen = subprocess.Popen
|
||||
# check for required context manager attributes
|
||||
_Popen.__enter__ and _Popen.__exit__
|
||||
compat_subprocess_Popen = _Popen
|
||||
except AttributeError:
|
||||
# not a context manager - make one
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def compat_subprocess_Popen(*args, **kwargs):
|
||||
popen = None
|
||||
try:
|
||||
popen = _Popen(*args, **kwargs)
|
||||
yield popen
|
||||
finally:
|
||||
if popen:
|
||||
for f in (popen.stdin, popen.stdout, popen.stderr):
|
||||
if f:
|
||||
# repeated .close() is OK, but just in case
|
||||
with compat_contextlib_suppress(EnvironmentError):
|
||||
f.close()
|
||||
popen.wait()
|
||||
|
||||
|
||||
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
||||
# See http://bugs.python.org/issue9161 for what is broken
|
||||
def workaround_optparse_bug9161():
|
||||
|
@ -3263,6 +3305,7 @@ __all__ = [
|
|||
'compat_http_cookiejar_Cookie',
|
||||
'compat_http_cookies',
|
||||
'compat_http_cookies_SimpleCookie',
|
||||
'compat_contextlib_suppress',
|
||||
'compat_ctypes_WINFUNCTYPE',
|
||||
'compat_etree_fromstring',
|
||||
'compat_filter',
|
||||
|
@ -3298,6 +3341,7 @@ __all__ = [
|
|||
'compat_struct_pack',
|
||||
'compat_struct_unpack',
|
||||
'compat_subprocess_get_DEVNULL',
|
||||
'compat_subprocess_Popen',
|
||||
'compat_tokenize_tokenize',
|
||||
'compat_urllib_error',
|
||||
'compat_urllib_parse',
|
||||
|
|
|
@ -11,8 +11,14 @@ from .common import FileDownloader
|
|||
from ..compat import (
|
||||
compat_setenv,
|
||||
compat_str,
|
||||
compat_subprocess_Popen,
|
||||
)
|
||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||
|
||||
try:
|
||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||
except ImportError:
|
||||
FFmpegPostProcessor = None
|
||||
|
||||
from ..utils import (
|
||||
cli_option,
|
||||
cli_valueless_option,
|
||||
|
@ -206,7 +212,10 @@ class WgetFD(ExternalFD):
|
|||
retry[1] = '0'
|
||||
cmd += retry
|
||||
cmd += self._option('--bind-address', 'source_address')
|
||||
cmd += self._option('--proxy', 'proxy')
|
||||
proxy = self.params.get('proxy')
|
||||
if proxy:
|
||||
for var in ('http_proxy', 'https_proxy'):
|
||||
cmd += ['--execute', '%s=%s' % (var, proxy)]
|
||||
cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
|
||||
cmd += self._configuration_args()
|
||||
cmd += ['--', info_dict['url']]
|
||||
|
@ -358,13 +367,14 @@ class FFmpegFD(ExternalFD):
|
|||
|
||||
@classmethod
|
||||
def available(cls):
|
||||
return FFmpegPostProcessor().available
|
||||
# actual availability can only be confirmed for an instance
|
||||
return bool(FFmpegPostProcessor)
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
url = info_dict['url']
|
||||
ffpp = FFmpegPostProcessor(downloader=self)
|
||||
# `downloader` means the parent `YoutubeDL`
|
||||
ffpp = FFmpegPostProcessor(downloader=self.ydl)
|
||||
if not ffpp.available:
|
||||
self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
|
||||
self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
|
||||
return False
|
||||
ffpp.check_version()
|
||||
|
||||
|
@ -393,6 +403,7 @@ class FFmpegFD(ExternalFD):
|
|||
# if end_time:
|
||||
# args += ['-t', compat_str(end_time - start_time)]
|
||||
|
||||
url = info_dict['url']
|
||||
cookies = self.ydl.cookiejar.get_cookies_for_url(url)
|
||||
if cookies:
|
||||
args.extend(['-cookies', ''.join(
|
||||
|
@ -480,21 +491,25 @@ class FFmpegFD(ExternalFD):
|
|||
|
||||
self._debug_cmd(args)
|
||||
|
||||
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
|
||||
try:
|
||||
retval = proc.wait()
|
||||
except BaseException as e:
|
||||
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||
# produces a file that is playable (this is mostly useful for live
|
||||
# streams). Note that Windows is not affected and produces playable
|
||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||
process_communicate_or_kill(proc, b'q')
|
||||
else:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
# From [1], a PIPE opened in Popen() should be closed, unless
|
||||
# .communicate() is called. Avoid leaking any PIPEs by using Popen
|
||||
# as a context manager (newer Python 3.x and compat)
|
||||
# Fixes "Resource Warning" in test/test_downloader_external.py
|
||||
# [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
|
||||
with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
|
||||
try:
|
||||
retval = proc.wait()
|
||||
except BaseException as e:
|
||||
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||
# produces a file that is playable (this is mostly useful for live
|
||||
# streams). Note that Windows is not affected and produces playable
|
||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||
process_communicate_or_kill(proc, b'q')
|
||||
else:
|
||||
proc.kill()
|
||||
raise
|
||||
return retval
|
||||
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@ from ..utils import (
|
|||
intlist_to_bytes,
|
||||
int_or_none,
|
||||
strip_jsonp,
|
||||
try_get,
|
||||
unescapeHTML,
|
||||
unsmuggle_url,
|
||||
)
|
||||
|
@ -203,23 +204,29 @@ class AnvatoIE(InfoExtractor):
|
|||
'telemundo': 'anvato_mcp_telemundo_web_prod_c5278d51ad46fda4b6ca3d0ea44a7846a054f582'
|
||||
}
|
||||
|
||||
_API_PREFIX = 'https://tkx.mp.lura.live/rest/v2/'
|
||||
_API_KEY = '3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA'
|
||||
|
||||
_ANVP_RE = r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1'
|
||||
_ANVP_RE = (
|
||||
r'<script[^>]*>[^<]*?\bAnvatoPlayer\s*\(\s*["\w]+\s*\)\s*\.\s*init\s*\(\s*(?P<anvp>{[^<]+?})\s*\);',
|
||||
r'<script[^>]+\bdata-anvp\s*=\s*(["\'])(?P<anvp>(?:(?!\1).)+)\1')
|
||||
_AUTH_KEY = b'\x31\xc2\x42\x84\x9e\x73\xa0\xce'
|
||||
|
||||
_TESTS = [{
|
||||
# from https://www.boston25news.com/news/watch-humpback-whale-breaches-right-next-to-fishing-boat-near-nh/817484874
|
||||
'url': 'anvato:8v9BEynrwx8EFLYpgfOWcG1qJqyXKlRM:4465496',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# from https://miami.cbslocal.com/2022/02/12/no-appetite-for-new-miami-restaurant-glorifying-castro-communism/
|
||||
'url': 'anvato:5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl:6197559', # 8v9BEynrwx8EFLYpgfOWcG1qJqyXKlRM:4465496',
|
||||
'info_dict': {
|
||||
'id': '4465496',
|
||||
'id': '6197559',
|
||||
'ext': 'mp4',
|
||||
'title': 'VIDEO: Humpback whale breaches right next to NH boat',
|
||||
'description': 'VIDEO: Humpback whale breaches right next to NH boat. Footage courtesy: Zach Fahey.',
|
||||
'duration': 22,
|
||||
'timestamp': 1534855680,
|
||||
'upload_date': '20180821',
|
||||
'uploader': 'ANV',
|
||||
'upload_date': '20220209',
|
||||
'uploader': 'CBS',
|
||||
'description': 'CBS4\'s Joel Waldman has more on the backlash Cafe Habana is receiving.',
|
||||
'timestamp': 1644381300,
|
||||
'title': 'Miamians Want No Part Of New Restaurant Set To Open In Brickell That Glorifies Fidel Castro & Communism',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
|
@ -228,46 +235,85 @@ class AnvatoIE(InfoExtractor):
|
|||
# from https://sanfrancisco.cbslocal.com/2016/06/17/source-oakland-cop-on-leave-for-having-girlfriend-help-with-police-reports/
|
||||
'url': 'anvato:DVzl9QRzox3ZZsP9bNu5Li3X7obQOnqP:3417601',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
# from https://sanfrancisco.cbslocal.com/2022/02/16/san-francisco-voters-recall-embattled-school-board-members/
|
||||
'url': 'anvato:5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl:6201051',
|
||||
'info_dict': {
|
||||
'id': '6201051',
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20220216',
|
||||
'uploader': 'CBS',
|
||||
'description': 'Voters were successful in their high-profile effort to recall three San Francisco school board members. Anne Makovec reports.',
|
||||
'timestamp': 1645043880,
|
||||
'title': 'Voters Recall San Francisco School Board Members',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}]
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(AnvatoIE, self).__init__(*args, **kwargs)
|
||||
self.__server_time = None
|
||||
|
||||
def _server_time(self, access_key, video_id):
|
||||
def _server_time(self, access_key, video_id, server_url=None):
|
||||
if self.__server_time is not None:
|
||||
return self.__server_time
|
||||
|
||||
if not server_url:
|
||||
server_url = self._API_PREFIX + 'server_time?anvack={ANVACK}'
|
||||
|
||||
self.__server_time = int(self._download_json(
|
||||
self._api_prefix(access_key) + 'server_time?anvack=' + access_key, video_id,
|
||||
server_url.format(ANVACK=access_key), video_id,
|
||||
note='Fetching server time')['server_time'])
|
||||
|
||||
return self.__server_time
|
||||
|
||||
def _api_prefix(self, access_key):
|
||||
return 'https://tkx2-%s.anvato.net/rest/v2/' % ('prod' if 'prod' in access_key else 'stage')
|
||||
|
||||
def _get_video_json(self, access_key, video_id):
|
||||
# See et() in anvplayer.min.js, which is an alias of getVideoJSON()
|
||||
video_data_url = self._api_prefix(access_key) + 'mcp/video/%s?anvack=%s' % (video_id, access_key)
|
||||
server_time = self._server_time(access_key, video_id)
|
||||
|
||||
def fix_template_vars(template):
|
||||
return re.sub(r'\{(\{\w+})}', r'\1', template)
|
||||
|
||||
# https://access.mp.lura.live/anvacks/5VD6Eyd6djewbCmNwBFnsJj17YAvGRwl?apikey=3hwbSuqqT690uxjNYBktSQpa5ZrpYYR0Iofx7NcJHyA
|
||||
access_info = self._download_json(
|
||||
'https://access.mp.lura.live/anvacks/%s?apikey=%s'
|
||||
% (access_key, self._API_KEY),
|
||||
video_id, note='Downloading access details')
|
||||
server_time_url = try_get(access_info, lambda x: x['api']['time'])
|
||||
|
||||
server_time_url = (
|
||||
server_time_url
|
||||
and fix_template_vars(server_time_url).format(ANVACK=access_key))
|
||||
server_time = self._server_time(access_key, video_id, server_time_url)
|
||||
|
||||
video_data_url = access_info['api'].get('video')
|
||||
|
||||
if not video_data_url:
|
||||
# use special knowledge
|
||||
video_data_url = self._API_PREFIX + 'mcp/video/{{VIDEO_ID}}?anvack={{ANVACK}}'
|
||||
|
||||
video_data_url = fix_template_vars(video_data_url).format(ANVACK=access_key, VIDEO_ID=video_id)
|
||||
|
||||
input_data = '%d~%s~%s' % (server_time, md5_text(video_data_url), md5_text(server_time))
|
||||
|
||||
auth_secret = intlist_to_bytes(aes_encrypt(
|
||||
bytes_to_intlist(input_data[:64]), bytes_to_intlist(self._AUTH_KEY)))
|
||||
|
||||
video_data_url += '&X-Anvato-Adst-Auth=' + base64.b64encode(auth_secret).decode('ascii')
|
||||
anvrid = md5_text(time.time() * 1000 * random.random())[:30]
|
||||
|
||||
query = {
|
||||
'X-Anvato-Adst-Auth': base64.b64encode(auth_secret).decode('ascii'),
|
||||
'rtyp': 'fp',
|
||||
}
|
||||
api = {
|
||||
'anvrid': anvrid,
|
||||
'anvts': server_time,
|
||||
'anvstk2': 'default',
|
||||
}
|
||||
api['anvstk'] = md5_text('%s|%s|%d|%s' % (
|
||||
access_key, anvrid, server_time,
|
||||
self._ANVACK_TABLE.get(access_key, self._API_KEY)))
|
||||
|
||||
return self._download_json(
|
||||
video_data_url, video_id, transform_source=strip_jsonp,
|
||||
note='Downloading video details',
|
||||
query=query,
|
||||
data=json.dumps({'api': api}).encode('utf-8'))
|
||||
|
||||
def _get_anvato_videos(self, access_key, video_id):
|
||||
|
@ -337,26 +383,28 @@ class AnvatoIE(InfoExtractor):
|
|||
@staticmethod
|
||||
def _extract_urls(ie, webpage, video_id):
|
||||
entries = []
|
||||
for mobj in re.finditer(AnvatoIE._ANVP_RE, webpage):
|
||||
anvplayer_data = ie._parse_json(
|
||||
mobj.group('anvp'), video_id, transform_source=unescapeHTML,
|
||||
fatal=False)
|
||||
if not anvplayer_data:
|
||||
continue
|
||||
video = anvplayer_data.get('video')
|
||||
if not isinstance(video, compat_str) or not video.isdigit():
|
||||
continue
|
||||
access_key = anvplayer_data.get('accessKey')
|
||||
if not access_key:
|
||||
mcp = anvplayer_data.get('mcp')
|
||||
if mcp:
|
||||
access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
|
||||
mcp.lower())
|
||||
if not access_key:
|
||||
continue
|
||||
entries.append(ie.url_result(
|
||||
'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
|
||||
video_id=video))
|
||||
anvp_res = AnvatoIE._ANVP_RE
|
||||
for anvp_re in anvp_res if isinstance(anvp_res, (list, tuple, )) else (anvp_res, ):
|
||||
for mobj in re.finditer(anvp_re, webpage):
|
||||
anvplayer_data = ie._parse_json(
|
||||
mobj.group('anvp'), video_id, transform_source=unescapeHTML,
|
||||
fatal=False)
|
||||
if not anvplayer_data:
|
||||
continue
|
||||
video = anvplayer_data.get('video')
|
||||
if not isinstance(video, compat_str) or not video.isdigit():
|
||||
continue
|
||||
access_key = anvplayer_data.get('accessKey')
|
||||
if not access_key:
|
||||
mcp = anvplayer_data.get('mcp')
|
||||
if mcp:
|
||||
access_key = AnvatoIE._MCP_TO_ACCESS_KEY_TABLE.get(
|
||||
mcp.lower())
|
||||
if not access_key:
|
||||
continue
|
||||
entries.append(ie.url_result(
|
||||
'anvato:%s:%s' % (access_key, video), ie=AnvatoIE.ie_key(),
|
||||
video_id=video))
|
||||
return entries
|
||||
|
||||
def _extract_anvato_videos(self, webpage, video_id):
|
||||
|
|
|
@ -0,0 +1,79 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
merge_dicts,
|
||||
parse_iso8601,
|
||||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class CaffeineTVIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?caffeine\.tv/[^/]+/video/(?P<id>[0-9a-f-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.caffeine.tv/TsuSurf/video/cffc0a00-e73f-11ec-8080-80017d29f26e',
|
||||
'info_dict': {
|
||||
'id': 'cffc0a00-e73f-11ec-8080-80017d29f26e',
|
||||
'ext': 'mp4',
|
||||
'title': 'GOOOOD MORNINNNNN #highlights',
|
||||
'timestamp': 1654702180,
|
||||
'upload_date': '20220608',
|
||||
'uploader': 'TsuSurf',
|
||||
'duration': 3145,
|
||||
'age_limit': 17,
|
||||
},
|
||||
'params': {
|
||||
'format': 'bestvideo',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
json_data = self._download_json(
|
||||
'https://api.caffeine.tv/social/public/activity/' + video_id,
|
||||
video_id)
|
||||
broadcast_info = traverse_obj(json_data, ('broadcast_info', T(dict))) or {}
|
||||
title = broadcast_info['broadcast_title']
|
||||
video_url = broadcast_info['video_url']
|
||||
|
||||
ext = determine_ext(video_url)
|
||||
if ext == 'm3u8':
|
||||
formats = self._extract_m3u8_formats(
|
||||
video_url, video_id, 'mp4', entry_protocol='m3u8',
|
||||
fatal=False)
|
||||
else:
|
||||
formats = [{'url': video_url}]
|
||||
self._sort_formats(formats)
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
}, traverse_obj(json_data, {
|
||||
'uploader': ((None, 'user'), 'username'),
|
||||
}, get_all=False), traverse_obj(json_data, {
|
||||
'like_count': ('like_count', T(int_or_none)),
|
||||
'view_count': ('view_count', T(int_or_none)),
|
||||
'comment_count': ('comment_count', T(int_or_none)),
|
||||
'tags': ('tags', Ellipsis, T(txt_or_none)),
|
||||
'is_live': 'is_live',
|
||||
'uploader': ('user', 'name'),
|
||||
}), traverse_obj(broadcast_info, {
|
||||
'duration': ('content_duration', T(int_or_none)),
|
||||
'timestamp': ('broadcast_start_time', T(parse_iso8601)),
|
||||
'thumbnail': ('preview_image_path', T(lambda u: urljoin(url, u))),
|
||||
'age_limit': ('content_rating', T(lambda r: r and {
|
||||
# assume Apple Store ratings [1]
|
||||
# 1. https://en.wikipedia.org/wiki/Mobile_software_content_rating_system
|
||||
'FOUR_PLUS': 0,
|
||||
'NINE_PLUS': 9,
|
||||
'TWELVE_PLUS': 12,
|
||||
'SEVENTEEN_PLUS': 17,
|
||||
}.get(r, 17))),
|
||||
}))
|
|
@ -5,6 +5,7 @@ from .anvato import AnvatoIE
|
|||
from .sendtonews import SendtoNewsIE
|
||||
from ..compat import compat_urlparse
|
||||
from ..utils import (
|
||||
merge_dicts,
|
||||
parse_iso8601,
|
||||
unified_timestamp,
|
||||
)
|
||||
|
@ -14,6 +15,8 @@ class CBSLocalIE(AnvatoIE):
|
|||
_VALID_URL_BASE = r'https?://[a-z]+\.cbslocal\.com/'
|
||||
_VALID_URL = _VALID_URL_BASE + r'video/(?P<id>\d+)'
|
||||
|
||||
_OLD_ANVATO_KEY = 'anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://newyork.cbslocal.com/video/3580809-a-very-blue-anniversary/',
|
||||
'info_dict': {
|
||||
|
@ -30,10 +33,6 @@ class CBSLocalIE(AnvatoIE):
|
|||
},
|
||||
'categories': [
|
||||
'Stations\\Spoken Word\\WCBSTV',
|
||||
'Syndication\\AOL',
|
||||
'Syndication\\MSN',
|
||||
'Syndication\\NDN',
|
||||
'Syndication\\Yahoo',
|
||||
'Content\\News',
|
||||
'Content\\News\\Local News',
|
||||
],
|
||||
|
@ -42,12 +41,21 @@ class CBSLocalIE(AnvatoIE):
|
|||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
'expected_warnings': ('Failed to download m3u8 information', ),
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
||||
mcp_id = self._match_id(url)
|
||||
return self.url_result(
|
||||
'anvato:anvato_cbslocal_app_web_prod_547f3e49241ef0e5d30c79b2efbca5d92c698f67:' + mcp_id, 'Anvato', mcp_id)
|
||||
webpage = self._download_webpage(url, mcp_id)
|
||||
|
||||
json_ld = self._search_json_ld(webpage, mcp_id, fatal=False) or {}
|
||||
json_ld.pop('url', None)
|
||||
|
||||
return merge_dicts(
|
||||
self._extract_anvato_videos(webpage, mcp_id)
|
||||
or self.url_result(self._OLD_ANVATO_KEY + ':' + mcp_id, 'Anvato', mcp_id),
|
||||
json_ld)
|
||||
|
||||
|
||||
class CBSLocalArticleIE(AnvatoIE):
|
||||
|
@ -56,30 +64,25 @@ class CBSLocalArticleIE(AnvatoIE):
|
|||
_TESTS = [{
|
||||
# Anvato backend
|
||||
'url': 'http://losangeles.cbslocal.com/2016/05/16/safety-advocates-say-fatal-car-seat-failures-are-public-health-crisis',
|
||||
'md5': 'f0ee3081e3843f575fccef901199b212',
|
||||
'only_matching': True
|
||||
}, {
|
||||
'url': 'https://losangeles.cbslocal.com/2022/02/16/rams-super-bowl-parade-to-take-place-wednesday/',
|
||||
'md5': '36bdac3fb24ec8a6d7790218a0357b08',
|
||||
'info_dict': {
|
||||
'id': '3401037',
|
||||
'id': '6201053',
|
||||
'ext': 'mp4',
|
||||
'title': 'Safety Advocates Say Fatal Car Seat Failures Are \'Public Health Crisis\'',
|
||||
'description': 'Collapsing seats have been the focus of scrutiny for decades, though experts say remarkably little has been done to address the issue. Randy Paige reports.',
|
||||
'thumbnail': 're:^https?://.*',
|
||||
'timestamp': 1463440500,
|
||||
'upload_date': '20160516',
|
||||
'display_id': 'rams-super-bowl-parade-to-take-place-wednesday',
|
||||
'upload_date': '20220216',
|
||||
'uploader': 'CBS',
|
||||
'subtitles': {
|
||||
'en': 'mincount:5',
|
||||
},
|
||||
'description': 'Jeff Nguyen is live from outside the L.A. Memorial Coliseum where fans cheered on the Los Angeles Rams.',
|
||||
'timestamp': 1645044990,
|
||||
'title': 'Rams Fans Gather Outside The LA Memorial Coliseum',
|
||||
'categories': [
|
||||
'Stations\\Spoken Word\\KCBSTV',
|
||||
'Syndication\\MSN',
|
||||
'Syndication\\NDN',
|
||||
'Syndication\\AOL',
|
||||
'Syndication\\Yahoo',
|
||||
'Syndication\\Tribune',
|
||||
'Syndication\\Curb.tv',
|
||||
'Content\\News'
|
||||
'Stations\\Spoken Word\\KCALTV',
|
||||
'Content\\News',
|
||||
'Content\\Top Story',
|
||||
],
|
||||
'tags': ['CBS 2 News Evening'],
|
||||
'tags': ['KCAL 9 News Afternoon'],
|
||||
},
|
||||
}, {
|
||||
# SendtoNews embed
|
||||
|
@ -92,18 +95,24 @@ class CBSLocalArticleIE(AnvatoIE):
|
|||
# m3u8 download
|
||||
'skip_download': True,
|
||||
},
|
||||
'skip': 'Redirects to CBS News home page',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
json_ld = self._search_json_ld(webpage, display_id, fatal=False) or {}
|
||||
json_ld.pop('url', None)
|
||||
|
||||
sendtonews_url = SendtoNewsIE._extract_url(webpage)
|
||||
if sendtonews_url:
|
||||
return self.url_result(
|
||||
result = self.url_result(
|
||||
compat_urlparse.urljoin(url, sendtonews_url),
|
||||
ie=SendtoNewsIE.ie_key())
|
||||
return merge_dicts(result, json_ld)
|
||||
|
||||
# returns a dict, or raises
|
||||
info_dict = self._extract_anvato_videos(webpage, display_id)
|
||||
|
||||
timestamp = unified_timestamp(self._html_search_regex(
|
||||
|
@ -111,9 +120,10 @@ class CBSLocalArticleIE(AnvatoIE):
|
|||
'released date', default=None)) or parse_iso8601(
|
||||
self._html_search_meta('uploadDate', webpage))
|
||||
|
||||
info_dict.update({
|
||||
'display_id': display_id,
|
||||
'timestamp': timestamp,
|
||||
})
|
||||
|
||||
return info_dict
|
||||
return merge_dicts(
|
||||
info_dict,
|
||||
json_ld,
|
||||
{
|
||||
'display_id': display_id,
|
||||
'timestamp': timestamp,
|
||||
})
|
||||
|
|
|
@ -25,6 +25,7 @@ from ..compat import (
|
|||
compat_getpass,
|
||||
compat_integer_types,
|
||||
compat_http_client,
|
||||
compat_kwargs,
|
||||
compat_map as map,
|
||||
compat_open as open,
|
||||
compat_os_name,
|
||||
|
@ -1102,6 +1103,60 @@ class InfoExtractor(object):
|
|||
self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
|
||||
return None
|
||||
|
||||
def _search_json(self, start_pattern, string, name, video_id, **kwargs):
|
||||
"""Searches string for the JSON object specified by start_pattern"""
|
||||
|
||||
# self, start_pattern, string, name, video_id, *, end_pattern='',
|
||||
# contains_pattern=r'{(?s:.+)}', fatal=True, default=NO_DEFAULT
|
||||
# NB: end_pattern is only used to reduce the size of the initial match
|
||||
end_pattern = kwargs.pop('end_pattern', '')
|
||||
# (?:[\s\S]) simulates (?(s):.) (eg)
|
||||
contains_pattern = kwargs.pop('contains_pattern', r'{[\s\S]+}')
|
||||
fatal = kwargs.pop('fatal', True)
|
||||
default = kwargs.pop('default', NO_DEFAULT)
|
||||
|
||||
if default is NO_DEFAULT:
|
||||
default, has_default = {}, False
|
||||
else:
|
||||
fatal, has_default = False, True
|
||||
|
||||
json_string = self._search_regex(
|
||||
r'(?:{0})\s*(?P<json>{1})\s*(?:{2})'.format(
|
||||
start_pattern, contains_pattern, end_pattern),
|
||||
string, name, group='json', fatal=fatal, default=None if has_default else NO_DEFAULT)
|
||||
if not json_string:
|
||||
return default
|
||||
|
||||
# yt-dlp has a special JSON parser that allows trailing text.
|
||||
# Until that arrives here, the diagnostic from the exception
|
||||
# raised by json.loads() is used to extract the wanted text.
|
||||
# Either way, it's a problem if a transform_source() can't
|
||||
# handle the trailing text.
|
||||
|
||||
# force an exception
|
||||
kwargs['fatal'] = True
|
||||
|
||||
# self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
|
||||
for _ in range(2):
|
||||
try:
|
||||
# return self._parse_json(json_string, video_id, ignore_extra=True, **kwargs)
|
||||
transform_source = kwargs.pop('transform_source', None)
|
||||
if transform_source:
|
||||
json_string = transform_source(json_string)
|
||||
return self._parse_json(json_string, video_id, **compat_kwargs(kwargs))
|
||||
except ExtractorError as e:
|
||||
end = int_or_none(self._search_regex(r'\(char\s+(\d+)', error_to_compat_str(e), 'end', default=None))
|
||||
if end is not None:
|
||||
json_string = json_string[:end]
|
||||
continue
|
||||
msg = 'Unable to extract {0} - Failed to parse JSON'.format(name)
|
||||
if fatal:
|
||||
raise ExtractorError(msg, cause=e.cause, video_id=video_id)
|
||||
elif not has_default:
|
||||
self.report_warning(
|
||||
'{0}: {1}'.format(msg, error_to_compat_str(e)), video_id=video_id)
|
||||
return default
|
||||
|
||||
def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
|
||||
"""
|
||||
Like _search_regex, but strips HTML tags and unescapes entities.
|
||||
|
@ -2966,25 +3021,22 @@ class InfoExtractor(object):
|
|||
return formats
|
||||
|
||||
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
|
||||
mobj = re.search(
|
||||
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
|
||||
webpage)
|
||||
if mobj:
|
||||
try:
|
||||
jwplayer_data = self._parse_json(mobj.group('options'),
|
||||
video_id=video_id,
|
||||
transform_source=transform_source)
|
||||
except ExtractorError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(jwplayer_data, dict):
|
||||
return jwplayer_data
|
||||
return self._search_json(
|
||||
r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
|
||||
webpage, 'JWPlayer data', video_id,
|
||||
# must be a {...} or sequence, ending
|
||||
contains_pattern=r'\{[\s\S]*}(?(load)(?:\s*,\s*\{[\s\S]*})*)', end_pattern=r'(?(load)\]|\))',
|
||||
transform_source=transform_source, default=None)
|
||||
|
||||
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
|
||||
jwplayer_data = self._find_jwplayer_data(
|
||||
webpage, video_id, transform_source=js_to_json)
|
||||
return self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, *args, **kwargs)
|
||||
|
||||
# allow passing `transform_source` through to _find_jwplayer_data()
|
||||
transform_source = kwargs.pop('transform_source', None)
|
||||
kwfind = compat_kwargs({'transform_source': transform_source}) if transform_source else {}
|
||||
|
||||
jwplayer_data = self._find_jwplayer_data(webpage, video_id, **kwfind)
|
||||
|
||||
return self._parse_jwplayer_data(jwplayer_data, video_id, *args, **kwargs)
|
||||
|
||||
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
|
||||
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
|
||||
|
@ -3018,22 +3070,14 @@ class InfoExtractor(object):
|
|||
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
|
||||
|
||||
subtitles = {}
|
||||
tracks = video_data.get('tracks')
|
||||
if tracks and isinstance(tracks, list):
|
||||
for track in tracks:
|
||||
if not isinstance(track, dict):
|
||||
continue
|
||||
track_kind = track.get('kind')
|
||||
if not track_kind or not isinstance(track_kind, compat_str):
|
||||
continue
|
||||
if track_kind.lower() not in ('captions', 'subtitles'):
|
||||
continue
|
||||
track_url = urljoin(base_url, track.get('file'))
|
||||
if not track_url:
|
||||
continue
|
||||
subtitles.setdefault(track.get('label') or 'en', []).append({
|
||||
'url': self._proto_relative_url(track_url)
|
||||
})
|
||||
for track in traverse_obj(video_data, (
|
||||
'tracks', lambda _, t: t.get('kind').lower() in ('captions', 'subtitles'))):
|
||||
track_url = urljoin(base_url, track.get('file'))
|
||||
if not track_url:
|
||||
continue
|
||||
subtitles.setdefault(track.get('label') or 'en', []).append({
|
||||
'url': self._proto_relative_url(track_url)
|
||||
})
|
||||
|
||||
entry = {
|
||||
'id': this_video_id,
|
||||
|
|
|
@ -159,6 +159,7 @@ from .businessinsider import BusinessInsiderIE
|
|||
from .buzzfeed import BuzzFeedIE
|
||||
from .byutv import BYUtvIE
|
||||
from .c56 import C56IE
|
||||
from .caffeine import CaffeineTVIE
|
||||
from .callin import CallinIE
|
||||
from .camdemy import (
|
||||
CamdemyIE,
|
||||
|
@ -382,7 +383,6 @@ from .fc2 import (
|
|||
FC2EmbedIE,
|
||||
)
|
||||
from .fczenit import FczenitIE
|
||||
from .filemoon import FileMoonIE
|
||||
from .fifa import FifaIE
|
||||
from .filmon import (
|
||||
FilmOnIE,
|
||||
|
@ -443,6 +443,7 @@ from .gamespot import GameSpotIE
|
|||
from .gamestar import GameStarIE
|
||||
from .gaskrank import GaskrankIE
|
||||
from .gazeta import GazetaIE
|
||||
from .gbnews import GBNewsIE
|
||||
from .gdcvault import GDCVaultIE
|
||||
from .gedidigital import GediDigitalIE
|
||||
from .generic import GenericIE
|
||||
|
|
|
@ -1,43 +0,0 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
decode_packed_codes,
|
||||
js_to_json,
|
||||
)
|
||||
|
||||
|
||||
class FileMoonIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?filemoon\.sx/./(?P<id>\w+)'
|
||||
_TEST = {
|
||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||
'info_dict': {
|
||||
'id': 'dw40rxrzruqz',
|
||||
'title': 'dw40rxrzruqz',
|
||||
'ext': 'mp4'
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
matches = re.findall(r'(?s)(eval.*?)</script>', webpage)
|
||||
packed = matches[-1]
|
||||
unpacked = decode_packed_codes(packed)
|
||||
jwplayer_sources = self._parse_json(
|
||||
self._search_regex(
|
||||
r'(?s)player\s*\.\s*setup\s*\(\s*\{\s*sources\s*:\s*(.*?])', unpacked, 'jwplayer sources'),
|
||||
video_id, transform_source=js_to_json)
|
||||
|
||||
formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._generic_title(url) or video_id,
|
||||
'formats': formats
|
||||
}
|
|
@ -0,0 +1,139 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class GBNewsIE(InfoExtractor):
|
||||
IE_DESC = 'GB News clips, features and live stream'
|
||||
|
||||
# \w+ is normally shows or news, but apparently any word redirects to the correct URL
|
||||
_VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
|
||||
|
||||
_PLATFORM = 'safari'
|
||||
_SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889',
|
||||
'info_dict': {
|
||||
'id': '106889',
|
||||
'ext': 'mp4',
|
||||
'title': "Andrew Neil's message to companies choosing to boycott GB News",
|
||||
'description': 'md5:b281f5d22fd6d5eda64a4e3ba771b351',
|
||||
},
|
||||
'skip': '404 not found',
|
||||
}, {
|
||||
'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
|
||||
'info_dict': {
|
||||
'id': '52264136',
|
||||
'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
|
||||
'ext': 'mp4',
|
||||
'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
|
||||
'description': 'The post was criticised by former employers of the broadcaster',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.gbnews.uk/watchlive',
|
||||
'info_dict': {
|
||||
'id': '1069',
|
||||
'display_id': 'watchlive',
|
||||
'ext': 'mp4',
|
||||
'title': 'GB News Live',
|
||||
'is_live': True,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url).split('/')[-1]
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
# extraction based on https://github.com/ytdl-org/youtube-dl/issues/29341
|
||||
'''
|
||||
<div id="video-106908"
|
||||
class="simplestream"
|
||||
data-id="GB001"
|
||||
data-type="vod"
|
||||
data-key="3Li3Nt2Qs8Ct3Xq9Fi5Uy0Mb2Bj0Qs"
|
||||
data-token="f9c317c727dc07f515b20036c8ef14a6"
|
||||
data-expiry="1624300052"
|
||||
data-uvid="37900558"
|
||||
data-poster="https://thumbnails.simplestreamcdn.com/gbnews/ondemand/37900558.jpg?width=700&"
|
||||
data-npaw="false"
|
||||
data-env="production">
|
||||
'''
|
||||
# exception if no match
|
||||
video_data = self._search_regex(
|
||||
r'(<div\s[^>]*\bclass\s*=\s*(\'|")(?!.*sidebar\b)simplestream(?:\s[\s\w$-]*)?\2[^>]*>)',
|
||||
webpage, 'video data')
|
||||
|
||||
video_data = extract_attributes(video_data)
|
||||
ss_id = video_data.get('data-id')
|
||||
if not ss_id:
|
||||
raise ExtractorError('Simplestream ID not found')
|
||||
|
||||
json_data = self._download_json(
|
||||
self._SSMP_URL, display_id,
|
||||
note='Downloading Simplestream JSON metadata',
|
||||
errnote='Unable to download Simplestream JSON metadata',
|
||||
query={
|
||||
'id': ss_id,
|
||||
'env': video_data.get('data-env', 'production'),
|
||||
}, fatal=False)
|
||||
|
||||
meta_url = traverse_obj(json_data, ('response', 'api_hostname'))
|
||||
if not meta_url:
|
||||
raise ExtractorError('No API host found')
|
||||
|
||||
uvid = video_data['data-uvid']
|
||||
dtype = video_data.get('data-type')
|
||||
stream_data = self._download_json(
|
||||
'%s/api/%s/stream/%s' % (meta_url, 'show' if dtype == 'vod' else dtype, uvid),
|
||||
uvid,
|
||||
query={
|
||||
'key': video_data.get('data-key'),
|
||||
'platform': self._PLATFORM,
|
||||
},
|
||||
headers={
|
||||
'Token': video_data.get('data-token'),
|
||||
'Token-Expiry': video_data.get('data-expiry'),
|
||||
'Uvid': uvid,
|
||||
}, fatal=False)
|
||||
|
||||
stream_url = traverse_obj(stream_data, (
|
||||
'response', 'stream', T(url_or_none)))
|
||||
if not stream_url:
|
||||
raise ExtractorError('No stream data/URL')
|
||||
|
||||
# now known to be a dict
|
||||
stream_data = stream_data['response']
|
||||
drm = stream_data.get('drm')
|
||||
if drm:
|
||||
self.report_drm(uvid)
|
||||
|
||||
formats = self._extract_m3u8_formats(
|
||||
stream_url, uvid, ext='mp4', entry_protocol='m3u8_native',
|
||||
fatal=False)
|
||||
# exception if no formats
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': uvid,
|
||||
'display_id': display_id,
|
||||
'title': (traverse_obj(stream_data, ('title', T(txt_or_none)))
|
||||
or self._og_search_title(webpage, default=None)
|
||||
or display_id.replace('-', ' ').capitalize()),
|
||||
'description': self._og_search_description(webpage, default=None),
|
||||
'thumbnail': (traverse_obj(video_data, ('data-poster', T(url_or_none)))
|
||||
or self._og_search_thumbnail(webpage)),
|
||||
'formats': formats,
|
||||
'is_live': (dtype == 'live') or None,
|
||||
}
|
|
@ -5,43 +5,45 @@ import re
|
|||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
dict_get,
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
parse_iso8601,
|
||||
update_url_query,
|
||||
int_or_none,
|
||||
determine_ext,
|
||||
determine_protocol,
|
||||
strip_or_none,
|
||||
try_get,
|
||||
unescapeHTML,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class SendtoNewsIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'
|
||||
|
||||
_TEST = {
|
||||
# TODO handle items with ?fk=XXXX6789&cid=1234 -> SC=XXXX6789-???????-1234
|
||||
_VALID_URL = r'https?://embed\.sendtonews\.com/(?:player\d/embed(?:player|code)\.(?:php|js)|oembed/?)\?.*\bSC=(?P<id>[\w-]+)'
|
||||
_TESTS = [{
|
||||
# From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
|
||||
'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',
|
||||
'info_dict': {
|
||||
'id': 'GxfCe0Zo7D-175909-5588'
|
||||
},
|
||||
'playlist_count': 8,
|
||||
# test the first video only to prevent lengthy tests
|
||||
'playlist': [{
|
||||
'info_dict': {
|
||||
'id': '240385',
|
||||
'ext': 'mp4',
|
||||
'title': 'Indians introduce Encarnacion',
|
||||
'description': 'Indians president of baseball operations Chris Antonetti and Edwin Encarnacion discuss the slugger\'s three-year contract with Cleveland',
|
||||
'duration': 137.898,
|
||||
'thumbnail': r're:https?://.*\.jpg$',
|
||||
'upload_date': '20170105',
|
||||
'timestamp': 1483649762,
|
||||
},
|
||||
}],
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
'playlist_count': 10,
|
||||
}, {
|
||||
'url': 'https://embed.sendtonews.com/player4/embedplayer.php?SC=mq3wIKSb68-1206898-8402&type=single',
|
||||
'info_dict': {
|
||||
'id': '1752278',
|
||||
'ext': 'mp4',
|
||||
'title': 'Las vegas homebuilders had banner sales year in 2021, and other top stories from January 24, 2022.',
|
||||
'description': 'LAS VEGAS HOMEBUILDERS HAD BANNER SALES YEAR IN 2021., and other top stories from January 24, 2022.',
|
||||
'timestamp': 1643063702,
|
||||
'upload_date': '20220124',
|
||||
'thumbnail': r're:https?://.*\.(?:png|jpg)$',
|
||||
'categories': ['Business'],
|
||||
'tags': list,
|
||||
},
|
||||
}
|
||||
}]
|
||||
|
||||
_URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'
|
||||
|
||||
|
@ -59,17 +61,62 @@ class SendtoNewsIE(InfoExtractor):
|
|||
playlist_id = self._match_id(url)
|
||||
|
||||
data_url = update_url_query(
|
||||
url.replace('embedplayer.php', 'data_read.php'),
|
||||
{'cmd': 'loadInitial'})
|
||||
re.sub(
|
||||
r'(?P<player>player\d)?(?<!/)/(?P<embed>embed.+?|oembed/)\?',
|
||||
lambda m: '/%s/data_read.php?' % ((m.group('player') or 'player4'), ),
|
||||
url),
|
||||
{'cmd': 'loadInitial', 'type': 'single', })
|
||||
playlist_data = self._download_json(data_url, playlist_id)
|
||||
playlist = try_get(playlist_data, lambda x: x['playlistData'][0], (dict, list)) or {}
|
||||
if isinstance(playlist, dict):
|
||||
err = playlist.get('error', 'No or invalid data returned from API')
|
||||
raise ExtractorError(err)
|
||||
|
||||
entries = []
|
||||
for video in playlist_data['playlistData'][0]:
|
||||
info_dict = self._parse_jwplayer_data(
|
||||
video['jwconfiguration'],
|
||||
require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True})
|
||||
info_dict = {}
|
||||
for video in playlist:
|
||||
try:
|
||||
err = video.get('error')
|
||||
if err and video.get('S_ID') is not None:
|
||||
e = ExtractorError(err)
|
||||
e.msg = err
|
||||
raise e
|
||||
except AttributeError:
|
||||
continue
|
||||
except ExtractorError as e:
|
||||
self.report_warning(e.msg, playlist_id)
|
||||
continue
|
||||
if 'jwconfiguration' in video:
|
||||
info_dict.update(self._parse_jwplayer_data(
|
||||
video['jwconfiguration'],
|
||||
require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True}))
|
||||
elif 'configuration' not in video:
|
||||
continue
|
||||
else:
|
||||
fmt_url = urljoin(
|
||||
url,
|
||||
try_get(video, lambda x: x['configuration']['sources']['src']))
|
||||
if not fmt_url:
|
||||
continue
|
||||
video_id = strip_or_none(video.get('SM_ID') or video['configuration']['mediaid'])
|
||||
title = strip_or_none(video.get('S_headLine') or video['configuration']['title'])
|
||||
if not video_id or not title:
|
||||
continue
|
||||
ext = determine_ext(fmt_url)
|
||||
if ext == 'm3u8':
|
||||
formats = self._extract_m3u8_formats(
|
||||
fmt_url, playlist_id, 'mp4', entry_protocol='m3u8_native',
|
||||
m3u8_id='hls', fatal=False)
|
||||
else:
|
||||
formats = [{
|
||||
'url': fmt_url,
|
||||
'ext': ext,
|
||||
'width': int_or_none(video.get('SM_M_VIDEO_WIDTH')),
|
||||
'height': int_or_none(video.get('SM_M_VIDEO_HEIGHT')),
|
||||
}]
|
||||
info_dict['formats'] = formats
|
||||
|
||||
for f in info_dict['formats']:
|
||||
for f in info_dict.get('formats') or []:
|
||||
if f.get('tbr'):
|
||||
continue
|
||||
tbr = int_or_none(self._search_regex(
|
||||
|
@ -83,23 +130,31 @@ class SendtoNewsIE(InfoExtractor):
|
|||
self._sort_formats(info_dict['formats'], ('tbr', 'height', 'width', 'format_id'))
|
||||
|
||||
thumbnails = []
|
||||
if video.get('thumbnailUrl'):
|
||||
for tn_id, tn in (('poster', video['configuration'].get('poster')),
|
||||
('normal', video.get('thumbnailUrl')),
|
||||
('small', video.get('smThumbnailUrl'))):
|
||||
tn = urljoin(url, tn)
|
||||
if not tn:
|
||||
continue
|
||||
thumbnails.append({
|
||||
'id': 'normal',
|
||||
'url': video['thumbnailUrl'],
|
||||
})
|
||||
if video.get('smThumbnailUrl'):
|
||||
thumbnails.append({
|
||||
'id': 'small',
|
||||
'url': video['smThumbnailUrl'],
|
||||
'id': tn_id,
|
||||
'url': tn,
|
||||
})
|
||||
info_dict.update({
|
||||
'title': video['S_headLine'].strip(),
|
||||
'description': unescapeHTML(video.get('S_fullStory')),
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': unescapeHTML(dict_get(video, ('S_fullStory', 'S_shortSummary'))),
|
||||
'thumbnails': thumbnails,
|
||||
'duration': float_or_none(video.get('SM_length')),
|
||||
'duration': float_or_none(
|
||||
dict_get(video, ('SM_length', 'SM_M_LENGTH'))
|
||||
or video['configuration'].get('duration')),
|
||||
'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
|
||||
'tags': [t for t in video.get('S_tags', '').split(',') if t],
|
||||
'categories': [c for c in video.get('S_category', '').split(',') if c],
|
||||
})
|
||||
entries.append(info_dict)
|
||||
|
||||
if len(entries) == 1:
|
||||
entries[0]['display_id'] = playlist_id
|
||||
return entries[0]
|
||||
return self.playlist_result(entries, playlist_id)
|
||||
|
|
|
@ -7,6 +7,7 @@ import time
|
|||
from .common import InfoExtractor
|
||||
from ..compat import compat_kwargs
|
||||
from ..utils import (
|
||||
base_url,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
|
@ -14,6 +15,7 @@ from ..utils import (
|
|||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
url_basename,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
@ -33,8 +35,8 @@ class Vbox7IE(InfoExtractor):
|
|||
'''
|
||||
_EMBED_REGEX = [r'<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)']
|
||||
_GEO_COUNTRIES = ['BG']
|
||||
_GEO_BYPASS = False
|
||||
_TESTS = [{
|
||||
# the http: URL just redirects here
|
||||
'url': 'https://vbox7.com/play:0946fff23c',
|
||||
'md5': '50ca1f78345a9c15391af47d8062d074',
|
||||
'info_dict': {
|
||||
|
@ -42,17 +44,19 @@ class Vbox7IE(InfoExtractor):
|
|||
'ext': 'mp4',
|
||||
'title': 'Борисов: Притеснен съм за бъдещето на България',
|
||||
'description': 'По думите му е опасно страната ни да бъде обявена за "сигурна"',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'timestamp': 1470982814,
|
||||
'upload_date': '20160812',
|
||||
'uploader': 'zdraveibulgaria',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'view_count': int,
|
||||
'duration': 2640,
|
||||
},
|
||||
'expected_warnings': [
|
||||
'Unable to download webpage',
|
||||
],
|
||||
}, {
|
||||
'url': 'http://vbox7.com/play:249bb972c2',
|
||||
'md5': 'aaf19465e37ec0b30b918df83ec32c50',
|
||||
'md5': '99f65c0c9ef9b682b97313e052734c3f',
|
||||
'info_dict': {
|
||||
'id': '249bb972c2',
|
||||
'ext': 'mp4',
|
||||
|
@ -61,7 +65,11 @@ class Vbox7IE(InfoExtractor):
|
|||
'timestamp': 1360215023,
|
||||
'upload_date': '20130207',
|
||||
'uploader': 'svideteliat_ot_varshava',
|
||||
'thumbnail': 'https://i49.vbox7.com/o/249/249bb972c20.jpg',
|
||||
'view_count': int,
|
||||
'duration': 83,
|
||||
},
|
||||
'expected_warnings': ['Failed to download m3u8 information'],
|
||||
}, {
|
||||
'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
|
||||
'only_matching': True,
|
||||
|
@ -76,6 +84,9 @@ class Vbox7IE(InfoExtractor):
|
|||
if mobj:
|
||||
return mobj.group('url')
|
||||
|
||||
# specialisation to transform what looks like ld+json that
|
||||
# may contain invalid character combinations
|
||||
|
||||
# transform_source=None, fatal=True
|
||||
def _parse_json(self, json_string, video_id, *args, **kwargs):
|
||||
if '"@context"' in json_string[:30]:
|
||||
|
@ -103,49 +114,64 @@ class Vbox7IE(InfoExtractor):
|
|||
|
||||
now = time.time()
|
||||
response = self._download_json(
|
||||
'https://www.vbox7.com/aj/player/item/options?vid=%s' % (video_id,),
|
||||
video_id, headers={'Referer': url})
|
||||
'https://www.vbox7.com/aj/player/item/options', video_id,
|
||||
query={'vid': video_id}, headers={'Referer': url})
|
||||
# estimate time to which possible `ago` member is relative
|
||||
now = now + 0.5 * (time.time() - now)
|
||||
|
||||
if 'error' in response:
|
||||
if traverse_obj(response, 'error'):
|
||||
raise ExtractorError(
|
||||
'%s said: %s' % (self.IE_NAME, response['error']), expected=True)
|
||||
|
||||
video_url = traverse_obj(response, ('options', 'src', T(url_or_none)))
|
||||
src_url = traverse_obj(response, ('options', 'src', T(url_or_none))) or ''
|
||||
|
||||
if '/na.mp4' in video_url or '':
|
||||
fmt_base = url_basename(src_url).rsplit('.', 1)[0].rsplit('_', 1)[0]
|
||||
if fmt_base in ('na', 'vn'):
|
||||
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
|
||||
|
||||
ext = determine_ext(video_url)
|
||||
ext = determine_ext(src_url)
|
||||
if ext == 'mpd':
|
||||
# In case MPD cannot be parsed, or anyway, get mp4 combined
|
||||
# formats usually provided to Safari, iOS, and old Windows
|
||||
# extract MPD
|
||||
try:
|
||||
formats, subtitles = self._extract_mpd_formats_and_subtitles(
|
||||
video_url, video_id, 'dash', fatal=False)
|
||||
except KeyError:
|
||||
src_url, video_id, 'dash', fatal=False)
|
||||
except KeyError: # fatal doesn't catch this
|
||||
self.report_warning('Failed to parse MPD manifest')
|
||||
formats, subtitles = [], {}
|
||||
elif ext != 'm3u8':
|
||||
formats = [{
|
||||
'url': src_url,
|
||||
}] if src_url else []
|
||||
subtitles = {}
|
||||
|
||||
if src_url:
|
||||
# possibly extract HLS, based on https://github.com/yt-dlp/yt-dlp/pull/9100
|
||||
fmt_base = base_url(src_url) + fmt_base
|
||||
# prepare for _extract_m3u8_formats_and_subtitles()
|
||||
# hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(
|
||||
hls_formats = self._extract_m3u8_formats(
|
||||
'{0}.m3u8'.format(fmt_base), video_id, m3u8_id='hls', fatal=False)
|
||||
formats.extend(hls_formats)
|
||||
# self._merge_subtitles(hls_subs, target=subtitles)
|
||||
|
||||
# In case MPD/HLS cannot be parsed, or anyway, get mp4 combined
|
||||
# formats usually provided to Safari, iOS, and old Windows
|
||||
video = response['options']
|
||||
resolutions = (1080, 720, 480, 240, 144)
|
||||
highest_res = traverse_obj(video, ('highestRes', T(int))) or resolutions[0]
|
||||
for res in traverse_obj(video, ('resolutions', lambda _, r: int(r) > 0)) or resolutions:
|
||||
if res > highest_res:
|
||||
continue
|
||||
formats.append({
|
||||
'url': video_url.replace('.mpd', '_%d.mp4' % res),
|
||||
'format_id': '%dp' % res,
|
||||
highest_res = traverse_obj(video, (
|
||||
'highestRes', T(int))) or resolutions[0]
|
||||
resolutions = traverse_obj(video, (
|
||||
'resolutions', lambda _, r: highest_res >= int(r) > 0)) or resolutions
|
||||
mp4_formats = traverse_obj(resolutions, (
|
||||
Ellipsis, T(lambda res: {
|
||||
'url': '{0}_{1}.mp4'.format(fmt_base, res),
|
||||
'format_id': 'http-{0}'.format(res),
|
||||
'height': res,
|
||||
})
|
||||
})))
|
||||
# if above formats are flaky, enable the line below
|
||||
# self._check_formats(formats, video_id)
|
||||
else:
|
||||
formats = [{
|
||||
'url': video_url,
|
||||
}]
|
||||
subtitles = {}
|
||||
# self._check_formats(mp4_formats, video_id)
|
||||
formats.extend(mp4_formats)
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
webpage = self._download_webpage(url, video_id, fatal=False) or ''
|
||||
|
|
|
@ -6,22 +6,31 @@ import re
|
|||
import string
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import (
|
||||
compat_b64decode,
|
||||
compat_ord,
|
||||
compat_struct_pack,
|
||||
)
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
mimetype2ext,
|
||||
parse_codecs,
|
||||
parse_qs,
|
||||
update_url_query,
|
||||
urljoin,
|
||||
xpath_element,
|
||||
xpath_text,
|
||||
)
|
||||
from ..compat import (
|
||||
compat_b64decode,
|
||||
compat_ord,
|
||||
compat_struct_pack,
|
||||
compat_urlparse,
|
||||
)
|
||||
|
||||
|
||||
def compat_random_choices(population, *args, **kwargs):
|
||||
# weights=None, *, cum_weights=None, k=1
|
||||
# limited implementation needed here
|
||||
weights = args[0] if args else kwargs.get('weights')
|
||||
assert all(w is None for w in (weights, kwargs.get('cum_weights')))
|
||||
k = kwargs.get('k', 1)
|
||||
return ''.join(random.choice(population) for _ in range(k))
|
||||
|
||||
|
||||
class VideaIE(InfoExtractor):
|
||||
|
@ -35,6 +44,7 @@ class VideaIE(InfoExtractor):
|
|||
)
|
||||
(?P<id>[^?#&]+)
|
||||
'''
|
||||
_EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1']
|
||||
_TESTS = [{
|
||||
'url': 'http://videa.hu/videok/allatok/az-orult-kigyasz-285-kigyot-kigyo-8YfIAjxwWGwT8HVQ',
|
||||
'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
|
||||
|
@ -44,6 +54,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 21,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
|
||||
|
@ -54,6 +65,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Supercars előzés',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 64,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
|
||||
|
@ -64,6 +76,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 21,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
|
||||
|
@ -80,11 +93,14 @@ class VideaIE(InfoExtractor):
|
|||
}]
|
||||
_STATIC_SECRET = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p'
|
||||
|
||||
@staticmethod
|
||||
def _extract_urls(webpage):
|
||||
return [url for _, url in re.findall(
|
||||
r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1',
|
||||
webpage)]
|
||||
@classmethod
|
||||
def _extract_urls(cls, webpage):
|
||||
def yield_urls():
|
||||
for pattern in cls._EMBED_REGEX:
|
||||
for m in re.finditer(pattern, webpage):
|
||||
yield m.group('url')
|
||||
|
||||
return list(yield_urls())
|
||||
|
||||
@staticmethod
|
||||
def rc4(cipher_text, key):
|
||||
|
@ -130,13 +146,13 @@ class VideaIE(InfoExtractor):
|
|||
for i in range(0, 32):
|
||||
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
|
||||
|
||||
query = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
|
||||
random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
|
||||
query = parse_qs(player_url)
|
||||
random_seed = ''.join(compat_random_choices(string.ascii_letters + string.digits, k=8))
|
||||
query['_s'] = random_seed
|
||||
query['_t'] = result[:16]
|
||||
|
||||
b64_info, handle = self._download_webpage_handle(
|
||||
'http://videa.hu/videaplayer_get_xml.php', video_id, query=query)
|
||||
'http://videa.hu/player/xml', video_id, query=query)
|
||||
if b64_info.startswith('<?xml'):
|
||||
info = self._parse_xml(b64_info, video_id)
|
||||
else:
|
||||
|
|
|
@ -673,8 +673,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
|
|||
raise
|
||||
|
||||
if '//player.vimeo.com/video/' in url:
|
||||
config = self._parse_json(self._search_regex(
|
||||
r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
|
||||
config = self._search_json(
|
||||
r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
|
||||
if config.get('view') == 4:
|
||||
config = self._verify_player_video_password(
|
||||
redirect_url, video_id, headers)
|
||||
|
|
|
@ -4,20 +4,28 @@ from __future__ import unicode_literals
|
|||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_chr
|
||||
from ..compat import (
|
||||
compat_chr,
|
||||
compat_zip as zip,
|
||||
)
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
decode_packed_codes,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
merge_dicts,
|
||||
T,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
# based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
|
||||
def aa_decode(aa_code):
|
||||
symbol_table = [
|
||||
symbol_table = (
|
||||
('7', '((゚ー゚) + (o^_^o))'),
|
||||
('6', '((o^_^o) +(o^_^o))'),
|
||||
('5', '((゚ー゚) + (゚Θ゚))'),
|
||||
|
@ -26,84 +34,180 @@ def aa_decode(aa_code):
|
|||
('3', '(o^_^o)'),
|
||||
('1', '(゚Θ゚)'),
|
||||
('0', '(c^_^o)'),
|
||||
]
|
||||
('+', ''),
|
||||
)
|
||||
delim = '(゚Д゚)[゚ε゚]+'
|
||||
ret = ''
|
||||
for aa_char in aa_code.split(delim):
|
||||
|
||||
def chr_from_code(c):
|
||||
for val, pat in symbol_table:
|
||||
aa_char = aa_char.replace(pat, val)
|
||||
aa_char = aa_char.replace('+ ', '')
|
||||
m = re.match(r'^\d+', aa_char)
|
||||
if m:
|
||||
ret += compat_chr(int(m.group(0), 8))
|
||||
c = c.replace(pat, val)
|
||||
if c.startswith(('u', 'U')):
|
||||
base = 16
|
||||
c = c[1:]
|
||||
else:
|
||||
m = re.match(r'^u([\da-f]+)', aa_char)
|
||||
if m:
|
||||
ret += compat_chr(int(m.group(1), 16))
|
||||
return ret
|
||||
base = 10
|
||||
c = int_or_none(c, base=base)
|
||||
return '' if c is None else compat_chr(c)
|
||||
|
||||
return ''.join(
|
||||
chr_from_code(aa_char)
|
||||
for aa_char in aa_code.split(delim))
|
||||
|
||||
|
||||
class XFileShareIE(InfoExtractor):
|
||||
_SITES = (
|
||||
(r'aparat\.cam', 'Aparat'),
|
||||
(r'clipwatching\.com', 'ClipWatching'),
|
||||
(r'gounlimited\.to', 'GoUnlimited'),
|
||||
(r'govid\.me', 'GoVid'),
|
||||
(r'holavid\.com', 'HolaVid'),
|
||||
(r'streamty\.com', 'Streamty'),
|
||||
(r'thevideobee\.to', 'TheVideoBee'),
|
||||
(r'uqload\.com', 'Uqload'),
|
||||
(r'vidbom\.com', 'VidBom'),
|
||||
(r'vidlo\.us', 'vidlo'),
|
||||
(r'vidlocker\.xyz', 'VidLocker'),
|
||||
(r'vidshare\.tv', 'VidShare'),
|
||||
(r'vup\.to', 'VUp'),
|
||||
# status check 2024-02: site availability, G site: search
|
||||
(r'aparat\.cam', 'Aparat'), # Cloudflare says host error 522, apparently changed to wolfstreeam.tv
|
||||
(r'filemoon\.sx/.', 'FileMoon'),
|
||||
(r'gounlimited\.to', 'GoUnlimited'), # no media pages listed
|
||||
(r'govid\.me', 'GoVid'), # no media pages listed
|
||||
(r'highstream\.tv', 'HighStream'), # clipwatching.com redirects here
|
||||
(r'holavid\.com', 'HolaVid'), # Cloudflare says host error 522
|
||||
# (r'streamty\.com', 'Streamty'), # no media pages listed, connection timeout
|
||||
# (r'thevideobee\.to', 'TheVideoBee'), # no pages listed, refuses connection
|
||||
(r'uqload\.to', 'Uqload'), # .com, .co redirect here
|
||||
(r'(?:vedbam\.xyz|vadbam.net)', 'V?dB?m'), # vidbom.com redirects here, but no valid media pages listed
|
||||
(r'vidlo\.us', 'vidlo'), # no valid media pages listed
|
||||
(r'vidlocker\.xyz', 'VidLocker'), # no media pages listed
|
||||
(r'(?:w\d\.)?viidshar\.com', 'VidShare'), # vidshare.tv redirects here
|
||||
# (r'vup\.to', 'VUp'), # domain not found
|
||||
(r'wolfstream\.tv', 'WolfStream'),
|
||||
(r'xvideosharing\.com', 'XVideoSharing'),
|
||||
(r'xvideosharing\.com', 'XVideoSharing'), # just started showing 'maintenance mode'
|
||||
)
|
||||
|
||||
IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||
IE_DESC = 'XFileShare-based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
||||
% '|'.join(site for site in list(zip(*_SITES))[0]))
|
||||
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1' % '|'.join(site for site in list(zip(*_SITES))[0])]
|
||||
|
||||
_FILE_NOT_FOUND_REGEXES = (
|
||||
r'>(?:404 - )?File Not Found<',
|
||||
r'>The file was removed by administrator<',
|
||||
)
|
||||
_TITLE_REGEXES = (
|
||||
r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||
r'<td nowrap>([^<]+)</td>',
|
||||
r'h4-fine[^>]*>([^<]+)<',
|
||||
r'>Watch (.+)[ <]',
|
||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to (dead)
|
||||
r'title\s*:\s*"([^"]+)"', # govid.me
|
||||
)
|
||||
_SOURCE_URL_REGEXES = (
|
||||
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
||||
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
||||
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
)
|
||||
_THUMBNAIL_REGEXES = (
|
||||
r'<video[^>]+poster="([^"]+)"',
|
||||
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
||||
)
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://xvideosharing.com/fq65f94nd2ve',
|
||||
'md5': '4181f63957e8fe90ac836fa58dc3c8a6',
|
||||
'note': 'link in `sources`',
|
||||
'url': 'https://uqload.to/dcsu06gdb45o',
|
||||
'md5': '7f8db187b254379440bf4fcad094ae86',
|
||||
'info_dict': {
|
||||
'id': 'fq65f94nd2ve',
|
||||
'id': 'dcsu06gdb45o',
|
||||
'ext': 'mp4',
|
||||
'title': 'sample',
|
||||
'thumbnail': r're:http://.*\.jpg',
|
||||
'title': 'f2e31015957e74c8c8427982e161c3fc mp4',
|
||||
'thumbnail': r're:https://.*\.jpg'
|
||||
},
|
||||
'params': {
|
||||
'nocheckcertificate': True,
|
||||
},
|
||||
'expected_warnings': ['Unable to extract JWPlayer data'],
|
||||
}, {
|
||||
'note': 'link in decoded `sources`',
|
||||
'url': 'https://xvideosharing.com/1tlg6agrrdgc',
|
||||
'md5': '2608ce41932c1657ae56258a64e647d9',
|
||||
'info_dict': {
|
||||
'id': '1tlg6agrrdgc',
|
||||
'ext': 'mp4',
|
||||
'title': '0121',
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
},
|
||||
'skip': 'This server is in maintenance mode.',
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||
'info_dict': {
|
||||
'id': 'dw40rxrzruqz',
|
||||
'title': 'dw40rxrzruqz',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://vadbam.net/6lnbkci96wly.html',
|
||||
'md5': 'a1616800076177e2ac769203957c54bc',
|
||||
'info_dict': {
|
||||
'id': '6lnbkci96wly',
|
||||
'title': 'Heart Crime S01 E03 weciima autos',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://w1.viidshar.com/nnibe0xf0h79.html',
|
||||
'md5': 'f0a580ce9df06cc61b4a5c979d672367',
|
||||
'info_dict': {
|
||||
'id': 'nnibe0xf0h79',
|
||||
'title': 'JaGa 68ar',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'ffmpeg',
|
||||
},
|
||||
'expected_warnings': ['hlsnative has detected features it does not support'],
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://wolfstream.tv/a3drtehyrg52.html',
|
||||
'md5': '1901d86a79c5e0c6a51bdc9a4cfd3769',
|
||||
'info_dict': {
|
||||
'id': 'a3drtehyrg52',
|
||||
'title': 'NFL 2023 W04 DET@GB',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'url': 'https://aparat.cam/n4d6dh0wvlpr',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://wolfstream.tv/nthme29v9u2x',
|
||||
'url': 'https://uqload.to/ug5somm0ctnk.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://highstream.tv/2owiyz3sjoux',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://vedbam.xyz/6lnbkci96wly.html',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _extract_urls(webpage):
|
||||
return [
|
||||
mobj.group('url')
|
||||
for mobj in re.finditer(
|
||||
r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1'
|
||||
% '|'.join(site for site in list(zip(*XFileShareIE._SITES))[0]),
|
||||
webpage)]
|
||||
@classmethod
|
||||
def _extract_urls(cls, webpage):
|
||||
|
||||
def yield_urls():
|
||||
for regex in cls._EMBED_REGEX:
|
||||
for mobj in re.finditer(regex, webpage):
|
||||
yield mobj.group('url')
|
||||
|
||||
return list(yield_urls())
|
||||
|
||||
def _real_extract(self, url):
|
||||
host, video_id = re.match(self._VALID_URL, url).groups()
|
||||
host, video_id = self._match_valid_url(url).group('host', 'id')
|
||||
|
||||
url = 'https://%s/' % host + ('embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||
url = 'https://%s/%s' % (
|
||||
host,
|
||||
'embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
|
||||
container_div = get_element_by_id('container', webpage) or webpage
|
||||
if self._search_regex(
|
||||
r'>This server is in maintenance mode\.', container_div,
|
||||
'maint error', group=0, default=None):
|
||||
raise ExtractorError(clean_html(container_div), expected=True)
|
||||
if self._search_regex(
|
||||
self._FILE_NOT_FOUND_REGEXES, container_div,
|
||||
'missing video error', group=0, default=None):
|
||||
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
|
||||
|
||||
fields = self._hidden_inputs(webpage)
|
||||
|
@ -122,59 +226,43 @@ class XFileShareIE(InfoExtractor):
|
|||
'Content-type': 'application/x-www-form-urlencoded',
|
||||
})
|
||||
|
||||
title = (self._search_regex(
|
||||
(r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||
r'<td nowrap>([^<]+)</td>',
|
||||
r'h4-fine[^>]*>([^<]+)<',
|
||||
r'>Watch (.+)[ <]',
|
||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to
|
||||
r'title\s*:\s*"([^"]+)"'), # govid.me
|
||||
webpage, 'title', default=None) or self._og_search_title(
|
||||
webpage, default=None) or video_id).strip()
|
||||
title = (
|
||||
self._search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
|
||||
or self._og_search_title(webpage, default=None)
|
||||
or video_id).strip()
|
||||
|
||||
for regex, func in (
|
||||
(r'(eval\(function\(p,a,c,k,e,d\){.+)', decode_packed_codes),
|
||||
(r'(゚.+)', aa_decode)):
|
||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||
if obf_code:
|
||||
webpage = webpage.replace(obf_code, func(obf_code))
|
||||
obf_code = True
|
||||
while obf_code:
|
||||
for regex, func in (
|
||||
(r'(?s)(?<!-)\b(eval\(function\(p,a,c,k,e,d\)\{(?:(?!</script>).)+\)\))',
|
||||
decode_packed_codes),
|
||||
(r'(゚.+)', aa_decode)):
|
||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||
if obf_code:
|
||||
webpage = webpage.replace(obf_code, func(obf_code))
|
||||
break
|
||||
|
||||
formats = []
|
||||
jwplayer_data = self._find_jwplayer_data(
|
||||
webpage.replace(r'\'', '\''), video_id)
|
||||
result = self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, require_title=False,
|
||||
m3u8_id='hls', mpd_id='dash')
|
||||
|
||||
jwplayer_data = self._search_regex(
|
||||
[
|
||||
r'jwplayer\("[^"]+"\)\.load\(\[({.+?})\]\);',
|
||||
r'jwplayer\("[^"]+"\)\.setup\(({.+?})\);',
|
||||
], webpage,
|
||||
'jwplayer data', default=None)
|
||||
if jwplayer_data:
|
||||
jwplayer_data = self._parse_json(
|
||||
jwplayer_data.replace(r"\'", "'"), video_id, js_to_json)
|
||||
if not traverse_obj(result, 'formats'):
|
||||
if jwplayer_data:
|
||||
formats = self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, False,
|
||||
m3u8_id='hls', mpd_id='dash')['formats']
|
||||
|
||||
if not formats:
|
||||
urls = []
|
||||
for regex in (
|
||||
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
||||
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
||||
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
|
||||
self.report_warning(
|
||||
'Failed to extract JWPlayer formats', video_id=video_id)
|
||||
urls = set()
|
||||
for regex in self._SOURCE_URL_REGEXES:
|
||||
for mobj in re.finditer(regex, webpage):
|
||||
video_url = mobj.group('url')
|
||||
if video_url not in urls:
|
||||
urls.append(video_url)
|
||||
urls.add(mobj.group('url'))
|
||||
|
||||
sources = self._search_regex(
|
||||
r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
|
||||
if sources:
|
||||
urls.extend(self._parse_json(sources, video_id))
|
||||
urls.update(traverse_obj(sources, (T(lambda s: self._parse_json(s, video_id)), Ellipsis)))
|
||||
|
||||
formats = []
|
||||
for video_url in urls:
|
||||
for video_url in traverse_obj(urls, (Ellipsis, T(url_or_none))):
|
||||
if determine_ext(video_url) == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
video_url, video_id, 'mp4',
|
||||
|
@ -185,17 +273,19 @@ class XFileShareIE(InfoExtractor):
|
|||
'url': video_url,
|
||||
'format_id': 'sd',
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
result = {'formats': formats}
|
||||
|
||||
self._sort_formats(result['formats'])
|
||||
|
||||
thumbnail = self._search_regex(
|
||||
[
|
||||
r'<video[^>]+poster="([^"]+)"',
|
||||
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
||||
], webpage, 'thumbnail', default=None)
|
||||
self._THUMBNAIL_REGEXES, webpage, 'thumbnail', default=None)
|
||||
|
||||
return {
|
||||
if not (title or result.get('title')):
|
||||
title = self._generic_title(url) or video_id
|
||||
|
||||
return merge_dicts(result, {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'title': title or None,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
}
|
||||
'http_headers': {'Referer': url}
|
||||
})
|
||||
|
|
|
@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
|||
except JSInterpreter.Exception as e:
|
||||
self.report_warning(
|
||||
'%s (%s %s)' % (
|
||||
self.__ie_msg(
|
||||
'Unable to decode n-parameter: download likely to be throttled'),
|
||||
'Unable to decode n-parameter: download likely to be throttled',
|
||||
error_to_compat_str(e),
|
||||
traceback.format_exc()))
|
||||
traceback.format_exc()),
|
||||
video_id=video_id)
|
||||
return
|
||||
|
||||
self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
|
||||
|
|
|
@ -74,8 +74,11 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
return FFmpegPostProcessor(downloader)._versions
|
||||
|
||||
def _determine_executables(self):
|
||||
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
|
||||
# ordered to match prefer_ffmpeg!
|
||||
convs = ['ffmpeg', 'avconv']
|
||||
probes = ['ffprobe', 'avprobe']
|
||||
prefer_ffmpeg = True
|
||||
programs = convs + probes
|
||||
|
||||
def get_ffmpeg_version(path):
|
||||
ver = get_exe_version(path, args=['-version'])
|
||||
|
@ -96,6 +99,7 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
|
||||
self._paths = None
|
||||
self._versions = None
|
||||
location = None
|
||||
if self._downloader:
|
||||
prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
|
||||
location = self._downloader.params.get('ffmpeg_location')
|
||||
|
@ -118,33 +122,21 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
location = os.path.dirname(os.path.abspath(location))
|
||||
if basename in ('ffmpeg', 'ffprobe'):
|
||||
prefer_ffmpeg = True
|
||||
self._paths = dict(
|
||||
(p, p if location is None else os.path.join(location, p))
|
||||
for p in programs)
|
||||
self._versions = dict(
|
||||
x for x in (
|
||||
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
||||
if x[1] is not None)
|
||||
|
||||
self._paths = dict(
|
||||
(p, os.path.join(location, p)) for p in programs)
|
||||
self._versions = dict(
|
||||
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
||||
if self._versions is None:
|
||||
self._versions = dict(
|
||||
(p, get_ffmpeg_version(p)) for p in programs)
|
||||
self._paths = dict((p, p) for p in programs)
|
||||
|
||||
if prefer_ffmpeg is False:
|
||||
prefs = ('avconv', 'ffmpeg')
|
||||
else:
|
||||
prefs = ('ffmpeg', 'avconv')
|
||||
for p in prefs:
|
||||
if self._versions[p]:
|
||||
self.basename = p
|
||||
break
|
||||
|
||||
if prefer_ffmpeg is False:
|
||||
prefs = ('avprobe', 'ffprobe')
|
||||
else:
|
||||
prefs = ('ffprobe', 'avprobe')
|
||||
for p in prefs:
|
||||
if self._versions[p]:
|
||||
self.probe_basename = p
|
||||
break
|
||||
basenames = [None, None]
|
||||
for i, progs in enumerate((convs, probes)):
|
||||
for p in progs[::-1 if prefer_ffmpeg is False else 1]:
|
||||
if self._versions.get(p):
|
||||
basenames[i] = p
|
||||
break
|
||||
self.basename, self.probe_basename = basenames
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
|
|
|
@ -45,6 +45,7 @@ from .compat import (
|
|||
compat_casefold,
|
||||
compat_chr,
|
||||
compat_collections_abc,
|
||||
compat_contextlib_suppress,
|
||||
compat_cookiejar,
|
||||
compat_ctypes_WINFUNCTYPE,
|
||||
compat_datetime_timedelta_total_seconds,
|
||||
|
@ -1855,25 +1856,18 @@ def write_json_file(obj, fn):
|
|||
try:
|
||||
with tf:
|
||||
json.dump(obj, tf)
|
||||
if sys.platform == 'win32':
|
||||
# Need to remove existing file on Windows, else os.rename raises
|
||||
# WindowsError or FileExistsError.
|
||||
try:
|
||||
with compat_contextlib_suppress(OSError):
|
||||
if sys.platform == 'win32':
|
||||
# Need to remove existing file on Windows, else os.rename raises
|
||||
# WindowsError or FileExistsError.
|
||||
os.unlink(fn)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
mask = os.umask(0)
|
||||
os.umask(mask)
|
||||
os.chmod(tf.name, 0o666 & ~mask)
|
||||
except OSError:
|
||||
pass
|
||||
os.rename(tf.name, fn)
|
||||
except Exception:
|
||||
try:
|
||||
with compat_contextlib_suppress(OSError):
|
||||
os.remove(tf.name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
|
@ -2033,14 +2027,13 @@ def extract_attributes(html_element):
|
|||
NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
|
||||
but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
|
||||
"""
|
||||
parser = HTMLAttributeParser()
|
||||
try:
|
||||
parser.feed(html_element)
|
||||
parser.close()
|
||||
# Older Python may throw HTMLParseError in case of malformed HTML
|
||||
except compat_HTMLParseError:
|
||||
pass
|
||||
return parser.attrs
|
||||
ret = None
|
||||
# Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
|
||||
with compat_contextlib_suppress(compat_HTMLParseError):
|
||||
with contextlib.closing(HTMLAttributeParser()) as parser:
|
||||
parser.feed(html_element)
|
||||
ret = parser.attrs
|
||||
return ret or {}
|
||||
|
||||
|
||||
def clean_html(html):
|
||||
|
@ -2182,8 +2175,28 @@ def sanitize_url(url):
|
|||
return url
|
||||
|
||||
|
||||
def extract_basic_auth(url):
|
||||
parts = compat_urllib_parse.urlsplit(url)
|
||||
if parts.username is None:
|
||||
return url, None
|
||||
url = compat_urllib_parse.urlunsplit(parts._replace(netloc=(
|
||||
parts.hostname if parts.port is None
|
||||
else '%s:%d' % (parts.hostname, parts.port))))
|
||||
auth_payload = base64.b64encode(
|
||||
('%s:%s' % (parts.username, parts.password or '')).encode('utf-8'))
|
||||
return url, 'Basic {0}'.format(auth_payload.decode('ascii'))
|
||||
|
||||
|
||||
def sanitized_Request(url, *args, **kwargs):
|
||||
return compat_urllib_request.Request(escape_url(sanitize_url(url)), *args, **kwargs)
|
||||
url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
|
||||
if auth_header is not None:
|
||||
headers = args[1] if len(args) > 1 else kwargs.get('headers')
|
||||
headers = headers or {}
|
||||
headers['Authorization'] = auth_header
|
||||
if len(args) <= 1 and kwargs.get('headers') is None:
|
||||
kwargs['headers'] = headers
|
||||
kwargs = compat_kwargs(kwargs)
|
||||
return compat_urllib_request.Request(url, *args, **kwargs)
|
||||
|
||||
|
||||
def expand_path(s):
|
||||
|
@ -2221,7 +2234,8 @@ def _htmlentity_transform(entity_with_semicolon):
|
|||
numstr = '0%s' % numstr
|
||||
else:
|
||||
base = 10
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/7518
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/7518\
|
||||
# Also, weirdly, compat_contextlib_suppress fails here in 2.6
|
||||
try:
|
||||
return compat_chr(int(numstr, base))
|
||||
except ValueError:
|
||||
|
@ -2328,11 +2342,9 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
# Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
|
||||
# https://github.com/python/cpython/issues/85140
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/3878
|
||||
try:
|
||||
with compat_contextlib_suppress(AttributeError, NotImplementedError):
|
||||
# fails for Python < 2.7.10, not ssl.HAS_ALPN
|
||||
ctx.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||
except (AttributeError, NotImplementedError):
|
||||
# Python < 2.7.10, not ssl.HAS_ALPN
|
||||
pass
|
||||
|
||||
opts_no_check_certificate = params.get('nocheckcertificate', False)
|
||||
if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
|
||||
|
@ -2342,12 +2354,10 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
|
||||
try:
|
||||
with compat_contextlib_suppress(TypeError):
|
||||
# Fails with Python 2.7.8 (create_default_context present
|
||||
# but HTTPSHandler has no context=)
|
||||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||
except TypeError:
|
||||
# Python 2.7.8
|
||||
# (create_default_context present but HTTPSHandler has no context=)
|
||||
pass
|
||||
|
||||
if sys.version_info < (3, 2):
|
||||
return YoutubeDLHTTPSHandler(params, **kwargs)
|
||||
|
@ -2361,15 +2371,24 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||
|
||||
|
||||
def bug_reports_message():
|
||||
def bug_reports_message(before=';'):
|
||||
if ytdl_is_updateable():
|
||||
update_cmd = 'type youtube-dl -U to update'
|
||||
else:
|
||||
update_cmd = 'see https://yt-dl.org/update on how to update'
|
||||
msg = '; please report this issue on https://yt-dl.org/bug .'
|
||||
msg += ' Make sure you are using the latest version; %s.' % update_cmd
|
||||
msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
|
||||
return msg
|
||||
update_cmd = 'see https://github.com/ytdl-org/youtube-dl/#user-content-installation on how to update'
|
||||
|
||||
msg = (
|
||||
'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
|
||||
' using the appropriate issue template.'
|
||||
' Make sure you are using the latest version; %s.'
|
||||
' Be sure to call youtube-dl with the --verbose option and include the complete output.'
|
||||
) % update_cmd
|
||||
|
||||
before = (before or '').rstrip()
|
||||
if not before or before.endswith(('.', '!', '?')):
|
||||
msg = msg[0].title() + msg[1:]
|
||||
|
||||
return (before + ' ' if before else '') + msg
|
||||
|
||||
|
||||
class YoutubeDLError(Exception):
|
||||
|
@ -3156,12 +3175,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
|
|||
if timezone is None:
|
||||
timezone, date_str = extract_timezone(date_str)
|
||||
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
|
||||
dt = datetime.datetime.strptime(date_str, date_format) - timezone
|
||||
return calendar.timegm(dt.timetuple())
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def date_formats(day_first=True):
|
||||
|
@ -3181,17 +3198,13 @@ def unified_strdate(date_str, day_first=True):
|
|||
_, date_str = extract_timezone(date_str)
|
||||
|
||||
for expression in date_formats(day_first):
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
||||
except ValueError:
|
||||
pass
|
||||
if upload_date is None:
|
||||
timetuple = email.utils.parsedate_tz(date_str)
|
||||
if timetuple:
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
|
||||
except ValueError:
|
||||
pass
|
||||
if upload_date is not None:
|
||||
return compat_str(upload_date)
|
||||
|
||||
|
@ -3220,11 +3233,9 @@ def unified_timestamp(date_str, day_first=True):
|
|||
date_str = m.group(1)
|
||||
|
||||
for expression in date_formats(day_first):
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
|
||||
return calendar.timegm(dt.timetuple())
|
||||
except ValueError:
|
||||
pass
|
||||
timetuple = email.utils.parsedate_tz(date_str)
|
||||
if timetuple:
|
||||
return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)
|
||||
|
@ -3832,14 +3843,15 @@ class PUTRequest(compat_urllib_request.Request):
|
|||
return 'PUT'
|
||||
|
||||
|
||||
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
|
||||
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1, base=None):
|
||||
if get_attr:
|
||||
if v is not None:
|
||||
v = getattr(v, get_attr, None)
|
||||
if v in (None, ''):
|
||||
return default
|
||||
try:
|
||||
return int(v) * invscale // scale
|
||||
# like int, raise if base is specified and v is not a string
|
||||
return (int(v) if base is None else int(v, base=base)) * invscale // scale
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return default
|
||||
|
||||
|
|
Loading…
Reference in New Issue