mirror of
https://github.com/ytdl-org/youtube-dl
synced 2025-05-11 16:24:50 +09:00
Compare commits
28 Commits
c0ff5c3da3
...
b26c2bdf3a
| Author | SHA1 | Date | |
|---|---|---|---|
|
b26c2bdf3a | ||
|
|
c052a16f72 | ||
|
|
bd2ded59f2 | ||
|
|
16b7e97afa | ||
|
|
d21717978c | ||
|
|
7513413794 | ||
|
|
67dbfa65f2 | ||
|
|
6eb6d6dff5 | ||
|
|
6c40d9f847 | ||
|
|
1b08d3281d | ||
|
|
32b8d31780 | ||
|
|
570b868078 | ||
|
|
2190e89260 | ||
|
|
7e136639db | ||
|
|
cedeeed56f | ||
|
|
add4622870 | ||
|
|
9a6ddece4d | ||
|
|
f7dc45d3a6 | ||
|
|
06996aca12 | ||
|
|
13b0e81f17 | ||
|
|
01b80a0802 | ||
|
|
ae0cbb84f2 | ||
|
|
94f181f9f5 | ||
|
|
620298e0ff | ||
|
|
c445489a46 | ||
|
|
ac0c9c8f9f | ||
|
|
515c8b85b1 | ||
|
|
48082c9091 |
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.class
|
||||
@ -5,6 +6,7 @@
|
||||
*.DS_Store
|
||||
wine-py2exe/
|
||||
py2exe.log
|
||||
.pytest_cache/
|
||||
*.kate-swp
|
||||
build/
|
||||
dist/
|
||||
|
||||
@ -63,7 +63,7 @@ class TestCache(unittest.TestCase):
|
||||
obj = {'x': 1, 'y': ['ä', '\\a', True]}
|
||||
c.store('test_cache', 'k.', obj)
|
||||
self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
|
||||
new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
|
||||
new_version = '.'.join(('%0.2d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
|
||||
self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
|
||||
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ from __future__ import unicode_literals
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import math
|
||||
@ -146,6 +147,25 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/32815
|
||||
self._test('function f(){return 0 - 7 * - 6;}', 42)
|
||||
|
||||
def test_bitwise_operators_typecast(self):
|
||||
# madness
|
||||
self._test('function f(){return null << 5}', 0)
|
||||
self._test('function f(){return undefined >> 5}', 0)
|
||||
self._test('function f(){return 42 << NaN}', 42)
|
||||
self._test('function f(){return 42 << Infinity}', 42)
|
||||
self._test('function f(){return 0.0 << null}', 0)
|
||||
self._test('function f(){return NaN << 42}', 0)
|
||||
self._test('function f(){return "21.9" << 1}', 42)
|
||||
self._test('function f(){return true << "5";}', 32)
|
||||
self._test('function f(){return true << true;}', 2)
|
||||
self._test('function f(){return "19" & "21.9";}', 17)
|
||||
self._test('function f(){return "19" & false;}', 0)
|
||||
self._test('function f(){return "11.0" >> "2.1";}', 2)
|
||||
self._test('function f(){return 5 ^ 9;}', 12)
|
||||
self._test('function f(){return 0.0 << NaN}', 0)
|
||||
self._test('function f(){return null << undefined}', 0)
|
||||
self._test('function f(){return 21 << 4294967297}', 42)
|
||||
|
||||
def test_array_access(self):
|
||||
self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
|
||||
|
||||
@ -160,6 +180,7 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
|
||||
self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
|
||||
self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
|
||||
self._test('function f(){var x = 2; var y = ["a", "b"]; y[x%y["length"]]="z"; return y}', ['z', 'b'])
|
||||
|
||||
def test_comments(self):
|
||||
self._test('''
|
||||
@ -351,6 +372,13 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
|
||||
self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
|
||||
|
||||
def test_not(self):
|
||||
self._test('function f() { return ! undefined; }', True)
|
||||
self._test('function f() { return !0; }', True)
|
||||
self._test('function f() { return !!0; }', False)
|
||||
self._test('function f() { return ![]; }', False)
|
||||
self._test('function f() { return !0 !== false; }', True)
|
||||
|
||||
def test_void(self):
|
||||
self._test('function f() { return void 42; }', JS_Undefined)
|
||||
|
||||
@ -435,6 +463,7 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
|
||||
def test_regex(self):
|
||||
self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
|
||||
self._test('function f() { let a=/,,[/,913,/](,)}/; return a.source; }', ',,[/,913,/](,)}')
|
||||
|
||||
jsi = JSInterpreter('''
|
||||
function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
|
||||
@ -482,25 +511,6 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test('function f(){return -524999584 << 5}', 379882496)
|
||||
self._test('function f(){return 1236566549 << 5}', 915423904)
|
||||
|
||||
def test_bitwise_operators_typecast(self):
|
||||
# madness
|
||||
self._test('function f(){return null << 5}', 0)
|
||||
self._test('function f(){return undefined >> 5}', 0)
|
||||
self._test('function f(){return 42 << NaN}', 42)
|
||||
self._test('function f(){return 42 << Infinity}', 42)
|
||||
self._test('function f(){return 0.0 << null}', 0)
|
||||
self._test('function f(){return NaN << 42}', 0)
|
||||
self._test('function f(){return "21.9" << 1}', 42)
|
||||
self._test('function f(){return 21 << 4294967297}', 42)
|
||||
self._test('function f(){return true << "5";}', 32)
|
||||
self._test('function f(){return true << true;}', 2)
|
||||
self._test('function f(){return "19" & "21.9";}', 17)
|
||||
self._test('function f(){return "19" & false;}', 0)
|
||||
self._test('function f(){return "11.0" >> "2.1";}', 2)
|
||||
self._test('function f(){return 5 ^ 9;}', 12)
|
||||
self._test('function f(){return 0.0 << NaN}', 0)
|
||||
self._test('function f(){return null << undefined}', 0)
|
||||
|
||||
def test_negative(self):
|
||||
self._test('function f(){return 2 * -2.0 ;}', -4)
|
||||
self._test('function f(){return 2 - - -2 ;}', 0)
|
||||
@ -543,6 +553,8 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
test_result = list('test')
|
||||
tests = [
|
||||
'function f(a, b){return a.split(b)}',
|
||||
'function f(a, b){return a["split"](b)}',
|
||||
'function f(a, b){let x = ["split"]; return a[x[0]](b)}',
|
||||
'function f(a, b){return String.prototype.split.call(a, b)}',
|
||||
'function f(a, b){return String.prototype.split.apply(a, [b])}',
|
||||
]
|
||||
@ -593,6 +605,9 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test('function f(){return "012345678".slice(-1, 1)}', '')
|
||||
self._test('function f(){return "012345678".slice(-3, -1)}', '67')
|
||||
|
||||
def test_splice(self):
|
||||
self._test('function f(){var T = ["0", "1", "2"]; T["splice"](2, 1, "0")[0]; return T }', ['0', '1', '0'])
|
||||
|
||||
def test_pop(self):
|
||||
# pop
|
||||
self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.pop(), a]}',
|
||||
@ -627,6 +642,16 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}',
|
||||
[2, 4, 1, [4, 2]])
|
||||
|
||||
def test_extract_function(self):
|
||||
jsi = JSInterpreter('function a(b) { return b + 1; }')
|
||||
func = jsi.extract_function('a')
|
||||
self.assertEqual(func([2]), 3)
|
||||
|
||||
def test_extract_function_with_global_stack(self):
|
||||
jsi = JSInterpreter('function c(d) { return d + e + f + g; }')
|
||||
func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000})
|
||||
self.assertEqual(func([1]), 1111)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@ -1603,11 +1603,14 @@ Line 1
|
||||
|
||||
def test_get_element_by_class(self):
|
||||
html = '''
|
||||
<span class="foo bar">nice</span>
|
||||
<span class="foo bar baz-bam">nice</span>
|
||||
'''
|
||||
|
||||
self.assertEqual(get_element_by_class('foo', html), 'nice')
|
||||
self.assertEqual(get_element_by_class('bar', html), 'nice')
|
||||
self.assertEqual(get_element_by_class('no-such-class', html), None)
|
||||
self.assertEqual(get_element_by_class('baz', html), None)
|
||||
self.assertEqual(get_element_by_class('bam', html), None)
|
||||
|
||||
def test_get_element_by_attribute(self):
|
||||
html = '''
|
||||
@ -1626,10 +1629,13 @@ Line 1
|
||||
|
||||
def test_get_elements_by_class(self):
|
||||
html = '''
|
||||
<span class="foo bar">nice</span><span class="foo bar">also nice</span>
|
||||
<span class="foo bar baz-bam">nice</span><span class="foo bar">also nice</span>
|
||||
'''
|
||||
|
||||
self.assertEqual(get_elements_by_class('foo', html), ['nice', 'also nice'])
|
||||
self.assertEqual(get_elements_by_class('bar', html), ['nice', 'also nice'])
|
||||
self.assertEqual(get_elements_by_class('baz', html), [])
|
||||
self.assertEqual(get_elements_by_class('bam', html), [])
|
||||
self.assertEqual(get_elements_by_class('no-such-class', html), [])
|
||||
|
||||
def test_get_elements_by_attribute(self):
|
||||
|
||||
@ -94,11 +94,51 @@ _SIG_TESTS = [
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/363db69b/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player_ias.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
|
||||
'2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
|
||||
'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
|
||||
),
|
||||
]
|
||||
|
||||
_NSIG_TESTS = [
|
||||
@ -272,7 +312,7 @@ _NSIG_TESTS = [
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
|
||||
'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
|
||||
'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
|
||||
@ -286,6 +326,26 @@ _NSIG_TESTS = [
|
||||
'https://www.youtube.com/s/player/4fcd6e4a/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/tv-player-ias.vflset/tv-player-ias.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
|
||||
'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
|
||||
'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
|
||||
'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@ -335,7 +395,7 @@ def t_factory(name, sig_func, url_pattern):
|
||||
test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id'))
|
||||
|
||||
def test_func(self):
|
||||
basename = 'player-{0}-{1}.js'.format(name, test_id)
|
||||
basename = 'player-{0}.js'.format(test_id)
|
||||
fn = os.path.join(self.TESTDATA_DIR, basename)
|
||||
|
||||
if not os.path.exists(fn):
|
||||
|
||||
@ -540,10 +540,14 @@ class YoutubeDL(object):
|
||||
"""Print message to stdout if not in quiet mode."""
|
||||
return self.to_stdout(message, skip_eol, check_quiet=True)
|
||||
|
||||
def _write_string(self, s, out=None):
|
||||
def _write_string(self, s, out=None, only_once=False, _cache=set()):
|
||||
if only_once and s in _cache:
|
||||
return
|
||||
write_string(s, out=out, encoding=self.params.get('encoding'))
|
||||
if only_once:
|
||||
_cache.add(s)
|
||||
|
||||
def to_stdout(self, message, skip_eol=False, check_quiet=False):
|
||||
def to_stdout(self, message, skip_eol=False, check_quiet=False, only_once=False):
|
||||
"""Print message to stdout if not in quiet mode."""
|
||||
if self.params.get('logger'):
|
||||
self.params['logger'].debug(message)
|
||||
@ -552,9 +556,9 @@ class YoutubeDL(object):
|
||||
terminator = ['\n', ''][skip_eol]
|
||||
output = message + terminator
|
||||
|
||||
self._write_string(output, self._screen_file)
|
||||
self._write_string(output, self._screen_file, only_once=only_once)
|
||||
|
||||
def to_stderr(self, message):
|
||||
def to_stderr(self, message, only_once=False):
|
||||
"""Print message to stderr."""
|
||||
assert isinstance(message, compat_str)
|
||||
if self.params.get('logger'):
|
||||
@ -562,7 +566,7 @@ class YoutubeDL(object):
|
||||
else:
|
||||
message = self._bidi_workaround(message)
|
||||
output = message + '\n'
|
||||
self._write_string(output, self._err_file)
|
||||
self._write_string(output, self._err_file, only_once=only_once)
|
||||
|
||||
def to_console_title(self, message):
|
||||
if not self.params.get('consoletitle', False):
|
||||
@ -641,18 +645,11 @@ class YoutubeDL(object):
|
||||
raise DownloadError(message, exc_info)
|
||||
self._download_retcode = 1
|
||||
|
||||
def report_warning(self, message, only_once=False, _cache={}):
|
||||
def report_warning(self, message, only_once=False):
|
||||
'''
|
||||
Print the message to stderr, it will be prefixed with 'WARNING:'
|
||||
If stderr is a tty file the 'WARNING:' will be colored
|
||||
'''
|
||||
if only_once:
|
||||
m_hash = hash((self, message))
|
||||
m_cnt = _cache.setdefault(m_hash, 0)
|
||||
_cache[m_hash] = m_cnt + 1
|
||||
if m_cnt > 0:
|
||||
return
|
||||
|
||||
if self.params.get('logger') is not None:
|
||||
self.params['logger'].warning(message)
|
||||
else:
|
||||
@ -663,7 +660,7 @@ class YoutubeDL(object):
|
||||
else:
|
||||
_msg_header = 'WARNING:'
|
||||
warning_message = '%s %s' % (_msg_header, message)
|
||||
self.to_stderr(warning_message)
|
||||
self.to_stderr(warning_message, only_once=only_once)
|
||||
|
||||
def report_error(self, message, *args, **kwargs):
|
||||
'''
|
||||
@ -677,6 +674,16 @@ class YoutubeDL(object):
|
||||
kwargs['message'] = '%s %s' % (_msg_header, message)
|
||||
self.trouble(*args, **kwargs)
|
||||
|
||||
def write_debug(self, message, only_once=False):
|
||||
'''Log debug message or Print message to stderr'''
|
||||
if not self.params.get('verbose', False):
|
||||
return
|
||||
message = '[debug] {0}'.format(message)
|
||||
if self.params.get('logger'):
|
||||
self.params['logger'].debug(message)
|
||||
else:
|
||||
self.to_stderr(message, only_once)
|
||||
|
||||
def report_unscoped_cookies(self, *args, **kwargs):
|
||||
# message=None, tb=False, is_error=False
|
||||
if len(args) <= 2:
|
||||
@ -2514,7 +2521,7 @@ class YoutubeDL(object):
|
||||
self.get_encoding()))
|
||||
write_string(encoding_str, encoding=None)
|
||||
|
||||
writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))
|
||||
writeln_debug = lambda *s: self.write_debug(''.join(s))
|
||||
writeln_debug('youtube-dl version ', __version__)
|
||||
if _LAZY_LOADER:
|
||||
writeln_debug('Lazy loading extractors enabled')
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@ -8,14 +8,17 @@ import shutil
|
||||
import traceback
|
||||
|
||||
from .compat import (
|
||||
compat_contextlib_suppress,
|
||||
compat_getenv,
|
||||
compat_open as open,
|
||||
compat_os_makedirs,
|
||||
)
|
||||
from .utils import (
|
||||
error_to_compat_str,
|
||||
escape_rfc3986,
|
||||
expand_path,
|
||||
is_outdated_version,
|
||||
try_get,
|
||||
traverse_obj,
|
||||
write_json_file,
|
||||
)
|
||||
from .version import __version__
|
||||
@ -30,23 +33,35 @@ class Cache(object):
|
||||
def __init__(self, ydl):
|
||||
self._ydl = ydl
|
||||
|
||||
def _write_debug(self, *args, **kwargs):
|
||||
self._ydl.write_debug(*args, **kwargs)
|
||||
|
||||
def _report_warning(self, *args, **kwargs):
|
||||
self._ydl.report_warning(*args, **kwargs)
|
||||
|
||||
def _to_screen(self, *args, **kwargs):
|
||||
self._ydl.to_screen(*args, **kwargs)
|
||||
|
||||
def _get_param(self, k, default=None):
|
||||
return self._ydl.params.get(k, default)
|
||||
|
||||
def _get_root_dir(self):
|
||||
res = self._ydl.params.get('cachedir')
|
||||
res = self._get_param('cachedir')
|
||||
if res is None:
|
||||
cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
|
||||
res = os.path.join(cache_root, self._YTDL_DIR)
|
||||
return expand_path(res)
|
||||
|
||||
def _get_cache_fn(self, section, key, dtype):
|
||||
assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
|
||||
assert re.match(r'^[\w.-]+$', section), \
|
||||
'invalid section %r' % section
|
||||
assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
|
||||
key = escape_rfc3986(key, safe='').replace('%', ',') # encode non-ascii characters
|
||||
return os.path.join(
|
||||
self._get_root_dir(), section, '%s.%s' % (key, dtype))
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
return self._ydl.params.get('cachedir') is not False
|
||||
return self._get_param('cachedir') is not False
|
||||
|
||||
def store(self, section, key, data, dtype='json'):
|
||||
assert dtype in ('json',)
|
||||
@ -56,61 +71,55 @@ class Cache(object):
|
||||
|
||||
fn = self._get_cache_fn(section, key, dtype)
|
||||
try:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(fn))
|
||||
except OSError as ose:
|
||||
if ose.errno != errno.EEXIST:
|
||||
raise
|
||||
compat_os_makedirs(os.path.dirname(fn), exist_ok=True)
|
||||
self._write_debug('Saving {section}.{key} to cache'.format(section=section, key=key))
|
||||
write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
|
||||
except Exception:
|
||||
tb = traceback.format_exc()
|
||||
self._ydl.report_warning(
|
||||
'Writing cache to %r failed: %s' % (fn, tb))
|
||||
self._report_warning('Writing cache to {fn!r} failed: {tb}'.format(fn=fn, tb=tb))
|
||||
|
||||
def _validate(self, data, min_ver):
|
||||
version = try_get(data, lambda x: x[self._VERSION_KEY])
|
||||
version = traverse_obj(data, self._VERSION_KEY)
|
||||
if not version: # Backward compatibility
|
||||
data, version = {'data': data}, self._DEFAULT_VERSION
|
||||
if not is_outdated_version(version, min_ver or '0', assume_new=False):
|
||||
return data['data']
|
||||
self._ydl.to_screen(
|
||||
'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
|
||||
self._write_debug('Discarding old cache from version {version} (needs {min_ver})'.format(version=version, min_ver=min_ver))
|
||||
|
||||
def load(self, section, key, dtype='json', default=None, min_ver=None):
|
||||
def load(self, section, key, dtype='json', default=None, **kw_min_ver):
|
||||
assert dtype in ('json',)
|
||||
min_ver = kw_min_ver.get('min_ver')
|
||||
|
||||
if not self.enabled:
|
||||
return default
|
||||
|
||||
cache_fn = self._get_cache_fn(section, key, dtype)
|
||||
with compat_contextlib_suppress(IOError): # If no cache available
|
||||
try:
|
||||
try:
|
||||
with open(cache_fn, 'r', encoding='utf-8') as cachef:
|
||||
with open(cache_fn, encoding='utf-8') as cachef:
|
||||
self._write_debug('Loading {section}.{key} from cache'.format(section=section, key=key), only_once=True)
|
||||
return self._validate(json.load(cachef), min_ver)
|
||||
except ValueError:
|
||||
except (ValueError, KeyError):
|
||||
try:
|
||||
file_size = os.path.getsize(cache_fn)
|
||||
except (OSError, IOError) as oe:
|
||||
file_size = error_to_compat_str(oe)
|
||||
self._ydl.report_warning(
|
||||
'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||
except IOError:
|
||||
pass # No cache available
|
||||
self._report_warning('Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
|
||||
|
||||
return default
|
||||
|
||||
def remove(self):
|
||||
if not self.enabled:
|
||||
self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
|
||||
self._to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
|
||||
return
|
||||
|
||||
cachedir = self._get_root_dir()
|
||||
if not any((term in cachedir) for term in ('cache', 'tmp')):
|
||||
raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
|
||||
raise Exception('Not removing directory %s - this does not look like a cache dir' % (cachedir,))
|
||||
|
||||
self._ydl.to_screen(
|
||||
'Removing cache dir %s .' % cachedir, skip_eol=True)
|
||||
self._to_screen(
|
||||
'Removing cache dir %s .' % (cachedir,), skip_eol=True, ),
|
||||
if os.path.exists(cachedir):
|
||||
self._ydl.to_screen('.', skip_eol=True)
|
||||
self._to_screen('.', skip_eol=True)
|
||||
shutil.rmtree(cachedir)
|
||||
self._ydl.to_screen('.')
|
||||
self._to_screen('.')
|
||||
|
||||
@ -2498,8 +2498,7 @@ try:
|
||||
from urllib.parse import urlencode as compat_urllib_parse_urlencode
|
||||
from urllib.parse import parse_qs as compat_parse_qs
|
||||
except ImportError: # Python 2
|
||||
_asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
|
||||
else re.compile(r'([\x00-\x7f]+)'))
|
||||
_asciire = getattr(compat_urllib_parse, '_asciire', None) or re.compile(r'([\x00-\x7f]+)')
|
||||
|
||||
# HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
|
||||
# implementations from cpython 3.4.3's stdlib. Python 2's version
|
||||
@ -2567,24 +2566,21 @@ except ImportError: # Python 2
|
||||
# Possible solutions are to either port it from python 3 with all
|
||||
# the friends or manually ensure input query contains only byte strings.
|
||||
# We will stick with latter thus recursively encoding the whole query.
|
||||
def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'):
|
||||
def compat_urllib_parse_urlencode(query, doseq=0, safe='', encoding='utf-8', errors='strict'):
|
||||
|
||||
def encode_elem(e):
|
||||
if isinstance(e, dict):
|
||||
e = encode_dict(e)
|
||||
elif isinstance(e, (list, tuple,)):
|
||||
list_e = encode_list(e)
|
||||
e = tuple(list_e) if isinstance(e, tuple) else list_e
|
||||
e = type(e)(encode_elem(el) for el in e)
|
||||
elif isinstance(e, compat_str):
|
||||
e = e.encode(encoding)
|
||||
e = e.encode(encoding, errors)
|
||||
return e
|
||||
|
||||
def encode_dict(d):
|
||||
return dict((encode_elem(k), encode_elem(v)) for k, v in d.items())
|
||||
return tuple((encode_elem(k), encode_elem(v)) for k, v in d.items())
|
||||
|
||||
def encode_list(l):
|
||||
return [encode_elem(e) for e in l]
|
||||
|
||||
return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq)
|
||||
return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq).decode('ascii')
|
||||
|
||||
# HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
|
||||
# Python 2's version is apparently totally broken
|
||||
@ -2639,6 +2635,57 @@ except ImportError: # Python 2
|
||||
('parse_qs', compat_parse_qs)):
|
||||
setattr(compat_urllib_parse, name, fix)
|
||||
|
||||
try:
|
||||
all(chr(i) in b'' for i in range(256))
|
||||
except TypeError:
|
||||
# not all chr(i) are str: patch Python2 quote
|
||||
|
||||
_safemaps = getattr(compat_urllib_parse, '_safemaps', {})
|
||||
_always_safe = frozenset(compat_urllib_parse.always_safe)
|
||||
|
||||
def _quote(s, safe='/'):
|
||||
"""quote('abc def') -> 'abc%20def'"""
|
||||
|
||||
if not s and s is not None: # fast path
|
||||
return s
|
||||
safe = frozenset(safe)
|
||||
cachekey = (safe, _always_safe)
|
||||
try:
|
||||
safe_map = _safemaps[cachekey]
|
||||
except KeyError:
|
||||
safe = _always_safe | safe
|
||||
safe_map = {}
|
||||
for i in range(256):
|
||||
c = chr(i)
|
||||
safe_map[c] = (
|
||||
c if (i < 128 and c in safe)
|
||||
else b'%{0:02X}'.format(i))
|
||||
_safemaps[cachekey] = safe_map
|
||||
|
||||
if safe.issuperset(s):
|
||||
return s
|
||||
return ''.join(safe_map[c] for c in s)
|
||||
|
||||
# linked code
|
||||
def _quote_plus(s, safe=''):
|
||||
return (
|
||||
_quote(s, safe + b' ').replace(b' ', b'+') if b' ' in s
|
||||
else _quote(s, safe))
|
||||
|
||||
# linked code
|
||||
def _urlcleanup():
|
||||
if compat_urllib_parse._urlopener:
|
||||
compat_urllib_parse._urlopener.cleanup()
|
||||
_safemaps.clear()
|
||||
compat_urllib_parse.ftpcache.clear()
|
||||
|
||||
for name, fix in (
|
||||
('quote', _quote),
|
||||
('quote_plus', _quote_plus),
|
||||
('urlcleanup', _urlcleanup)):
|
||||
setattr(compat_urllib_parse, '_' + name, getattr(compat_urllib_parse, name))
|
||||
setattr(compat_urllib_parse, name, fix)
|
||||
|
||||
compat_urllib_parse_parse_qs = compat_parse_qs
|
||||
|
||||
|
||||
@ -3120,6 +3167,21 @@ else:
|
||||
compat_os_path_expanduser = compat_expanduser
|
||||
|
||||
|
||||
# compat_os_makedirs
|
||||
try:
|
||||
os.makedirs('.', exist_ok=True)
|
||||
compat_os_makedirs = os.makedirs
|
||||
except TypeError: # < Py3.2
|
||||
from errno import EEXIST as _errno_EEXIST
|
||||
|
||||
def compat_os_makedirs(name, mode=0o777, exist_ok=False):
|
||||
try:
|
||||
return os.makedirs(name, mode=mode)
|
||||
except OSError as ose:
|
||||
if not (exist_ok and ose.errno == _errno_EEXIST):
|
||||
raise
|
||||
|
||||
|
||||
# compat_os_path_realpath
|
||||
if compat_os_name == 'nt' and sys.version_info < (3, 8):
|
||||
# os.path.realpath on Windows does not follow symbolic links
|
||||
@ -3637,6 +3699,7 @@ __all__ = [
|
||||
'compat_numeric_types',
|
||||
'compat_open',
|
||||
'compat_ord',
|
||||
'compat_os_makedirs',
|
||||
'compat_os_name',
|
||||
'compat_os_path_expanduser',
|
||||
'compat_os_path_realpath',
|
||||
|
||||
@ -505,7 +505,7 @@ class InfoExtractor(object):
|
||||
if not self._x_forwarded_for_ip:
|
||||
|
||||
# Geo bypass mechanism is explicitly disabled by user
|
||||
if not self._downloader.params.get('geo_bypass', True):
|
||||
if not self.get_param('geo_bypass', True):
|
||||
return
|
||||
|
||||
if not geo_bypass_context:
|
||||
@ -527,7 +527,7 @@ class InfoExtractor(object):
|
||||
|
||||
# Explicit IP block specified by user, use it right away
|
||||
# regardless of whether extractor is geo bypassable or not
|
||||
ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
|
||||
ip_block = self.get_param('geo_bypass_ip_block', None)
|
||||
|
||||
# Otherwise use random IP block from geo bypass context but only
|
||||
# if extractor is known as geo bypassable
|
||||
@ -538,8 +538,8 @@ class InfoExtractor(object):
|
||||
|
||||
if ip_block:
|
||||
self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
|
||||
if self._downloader.params.get('verbose', False):
|
||||
self._downloader.to_screen(
|
||||
if self.get_param('verbose', False):
|
||||
self.to_screen(
|
||||
'[debug] Using fake IP %s as X-Forwarded-For.'
|
||||
% self._x_forwarded_for_ip)
|
||||
return
|
||||
@ -548,7 +548,7 @@ class InfoExtractor(object):
|
||||
|
||||
# Explicit country code specified by user, use it right away
|
||||
# regardless of whether extractor is geo bypassable or not
|
||||
country = self._downloader.params.get('geo_bypass_country', None)
|
||||
country = self.get_param('geo_bypass_country', None)
|
||||
|
||||
# Otherwise use random country code from geo bypass context but
|
||||
# only if extractor is known as geo bypassable
|
||||
@ -559,8 +559,8 @@ class InfoExtractor(object):
|
||||
|
||||
if country:
|
||||
self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
|
||||
if self._downloader.params.get('verbose', False):
|
||||
self._downloader.to_screen(
|
||||
if self.get_param('verbose', False):
|
||||
self.to_screen(
|
||||
'[debug] Using fake IP %s (%s) as X-Forwarded-For.'
|
||||
% (self._x_forwarded_for_ip, country.upper()))
|
||||
|
||||
@ -586,9 +586,9 @@ class InfoExtractor(object):
|
||||
raise ExtractorError('An extractor error has occurred.', cause=e)
|
||||
|
||||
def __maybe_fake_ip_and_retry(self, countries):
|
||||
if (not self._downloader.params.get('geo_bypass_country', None)
|
||||
if (not self.get_param('geo_bypass_country', None)
|
||||
and self._GEO_BYPASS
|
||||
and self._downloader.params.get('geo_bypass', True)
|
||||
and self.get_param('geo_bypass', True)
|
||||
and not self._x_forwarded_for_ip
|
||||
and countries):
|
||||
country_code = random.choice(countries)
|
||||
@ -698,7 +698,7 @@ class InfoExtractor(object):
|
||||
if fatal:
|
||||
raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
|
||||
else:
|
||||
self._downloader.report_warning(errmsg)
|
||||
self.report_warning(errmsg)
|
||||
return False
|
||||
|
||||
def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
|
||||
@ -770,11 +770,11 @@ class InfoExtractor(object):
|
||||
webpage_bytes = prefix + webpage_bytes
|
||||
if not encoding:
|
||||
encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
|
||||
if self._downloader.params.get('dump_intermediate_pages', False):
|
||||
if self.get_param('dump_intermediate_pages', False):
|
||||
self.to_screen('Dumping request to ' + urlh.geturl())
|
||||
dump = base64.b64encode(webpage_bytes).decode('ascii')
|
||||
self._downloader.to_screen(dump)
|
||||
if self._downloader.params.get('write_pages', False):
|
||||
self.to_screen(dump)
|
||||
if self.get_param('write_pages', False):
|
||||
basen = '%s_%s' % (video_id, urlh.geturl())
|
||||
if len(basen) > 240:
|
||||
h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
|
||||
@ -976,19 +976,9 @@ class InfoExtractor(object):
|
||||
"""Print msg to screen, prefixing it with '[ie_name]'"""
|
||||
self._downloader.to_screen(self.__ie_msg(msg))
|
||||
|
||||
def write_debug(self, msg, only_once=False, _cache=[]):
|
||||
def write_debug(self, msg, only_once=False):
|
||||
'''Log debug message or Print message to stderr'''
|
||||
if not self.get_param('verbose', False):
|
||||
return
|
||||
message = '[debug] ' + self.__ie_msg(msg)
|
||||
logger = self.get_param('logger')
|
||||
if logger:
|
||||
logger.debug(message)
|
||||
else:
|
||||
if only_once and hash(message) in _cache:
|
||||
return
|
||||
self._downloader.to_stderr(message)
|
||||
_cache.append(hash(message))
|
||||
self._downloader.write_debug(self.__ie_msg(msg), only_once=only_once)
|
||||
|
||||
# name, default=None, *args, **kwargs
|
||||
def get_param(self, name, *args, **kwargs):
|
||||
@ -1084,7 +1074,7 @@ class InfoExtractor(object):
|
||||
if mobj:
|
||||
break
|
||||
|
||||
if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
|
||||
if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
|
||||
_name = '\033[0;34m%s\033[0m' % name
|
||||
else:
|
||||
_name = name
|
||||
@ -1102,7 +1092,7 @@ class InfoExtractor(object):
|
||||
elif fatal:
|
||||
raise RegexNotFoundError('Unable to extract %s' % _name)
|
||||
else:
|
||||
self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
|
||||
self.report_warning('unable to extract %s' % _name + bug_reports_message())
|
||||
return None
|
||||
|
||||
def _search_json(self, start_pattern, string, name, video_id, **kwargs):
|
||||
@ -1172,7 +1162,7 @@ class InfoExtractor(object):
|
||||
username = None
|
||||
password = None
|
||||
|
||||
if self._downloader.params.get('usenetrc', False):
|
||||
if self.get_param('usenetrc', False):
|
||||
try:
|
||||
netrc_machine = netrc_machine or self._NETRC_MACHINE
|
||||
info = netrc.netrc().authenticators(netrc_machine)
|
||||
@ -1183,7 +1173,7 @@ class InfoExtractor(object):
|
||||
raise netrc.NetrcParseError(
|
||||
'No authenticators for %s' % netrc_machine)
|
||||
except (AttributeError, IOError, netrc.NetrcParseError) as err:
|
||||
self._downloader.report_warning(
|
||||
self.report_warning(
|
||||
'parsing .netrc: %s' % error_to_compat_str(err))
|
||||
|
||||
return username, password
|
||||
@ -1220,10 +1210,10 @@ class InfoExtractor(object):
|
||||
"""
|
||||
if self._downloader is None:
|
||||
return None
|
||||
downloader_params = self._downloader.params
|
||||
|
||||
if downloader_params.get('twofactor') is not None:
|
||||
return downloader_params['twofactor']
|
||||
twofactor = self.get_param('twofactor')
|
||||
if twofactor is not None:
|
||||
return twofactor
|
||||
|
||||
return compat_getpass('Type %s and press [Return]: ' % note)
|
||||
|
||||
@ -1358,7 +1348,7 @@ class InfoExtractor(object):
|
||||
elif fatal:
|
||||
raise RegexNotFoundError('Unable to extract JSON-LD')
|
||||
else:
|
||||
self._downloader.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
|
||||
self.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
|
||||
return {}
|
||||
|
||||
def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
|
||||
@ -1589,7 +1579,7 @@ class InfoExtractor(object):
|
||||
|
||||
if f.get('vcodec') == 'none': # audio only
|
||||
preference -= 50
|
||||
if self._downloader.params.get('prefer_free_formats'):
|
||||
if self.get_param('prefer_free_formats'):
|
||||
ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
|
||||
else:
|
||||
ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
|
||||
@ -1601,7 +1591,7 @@ class InfoExtractor(object):
|
||||
else:
|
||||
if f.get('acodec') == 'none': # video only
|
||||
preference -= 40
|
||||
if self._downloader.params.get('prefer_free_formats'):
|
||||
if self.get_param('prefer_free_formats'):
|
||||
ORDER = ['flv', 'mp4', 'webm']
|
||||
else:
|
||||
ORDER = ['webm', 'flv', 'mp4']
|
||||
@ -1667,7 +1657,7 @@ class InfoExtractor(object):
|
||||
""" Either "http:" or "https:", depending on the user's preferences """
|
||||
return (
|
||||
'http:'
|
||||
if self._downloader.params.get('prefer_insecure', False)
|
||||
if self.get_param('prefer_insecure', False)
|
||||
else 'https:')
|
||||
|
||||
def _proto_relative_url(self, url, scheme=None):
|
||||
@ -3130,7 +3120,8 @@ class InfoExtractor(object):
|
||||
continue
|
||||
urls.add(source_url)
|
||||
source_type = source.get('type') or ''
|
||||
ext = mimetype2ext(source_type) or determine_ext(source_url)
|
||||
# https://github.com/yt-dlp/yt-dlp/pull/10956
|
||||
ext = determine_ext(source_url, default_ext=mimetype2ext(source_type))
|
||||
if source_type == 'hls' or ext == 'm3u8' or 'format=m3u8-aapl' in source_url:
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
source_url, video_id, 'mp4', entry_protocol='m3u8_native',
|
||||
@ -3199,7 +3190,7 @@ class InfoExtractor(object):
|
||||
if fatal:
|
||||
raise ExtractorError(msg)
|
||||
else:
|
||||
self._downloader.report_warning(msg)
|
||||
self.report_warning(msg)
|
||||
return res
|
||||
|
||||
def _float(self, v, name, fatal=False, **kwargs):
|
||||
@ -3209,7 +3200,7 @@ class InfoExtractor(object):
|
||||
if fatal:
|
||||
raise ExtractorError(msg)
|
||||
else:
|
||||
self._downloader.report_warning(msg)
|
||||
self.report_warning(msg)
|
||||
return res
|
||||
|
||||
def _set_cookie(self, domain, name, value, expire_time=None, port=None,
|
||||
@ -3218,12 +3209,12 @@ class InfoExtractor(object):
|
||||
0, name, value, port, port is not None, domain, True,
|
||||
domain.startswith('.'), path, True, secure, expire_time,
|
||||
discard, None, None, rest)
|
||||
self._downloader.cookiejar.set_cookie(cookie)
|
||||
self.cookiejar.set_cookie(cookie)
|
||||
|
||||
def _get_cookies(self, url):
|
||||
""" Return a compat_cookies_SimpleCookie with the cookies for the url """
|
||||
req = sanitized_Request(url)
|
||||
self._downloader.cookiejar.add_cookie_header(req)
|
||||
self.cookiejar.add_cookie_header(req)
|
||||
return compat_cookies_SimpleCookie(req.get_header('Cookie'))
|
||||
|
||||
def _apply_first_set_cookie_header(self, url_handle, cookie):
|
||||
@ -3283,8 +3274,8 @@ class InfoExtractor(object):
|
||||
return not any_restricted
|
||||
|
||||
def extract_subtitles(self, *args, **kwargs):
|
||||
if (self._downloader.params.get('writesubtitles', False)
|
||||
or self._downloader.params.get('listsubtitles')):
|
||||
if (self.get_param('writesubtitles', False)
|
||||
or self.get_param('listsubtitles')):
|
||||
return self._get_subtitles(*args, **kwargs)
|
||||
return {}
|
||||
|
||||
@ -3305,7 +3296,11 @@ class InfoExtractor(object):
|
||||
""" Merge subtitle dictionaries, language by language. """
|
||||
|
||||
# ..., * , target=None
|
||||
target = kwargs.get('target') or dict(subtitle_dict1)
|
||||
target = kwargs.get('target')
|
||||
if target is None:
|
||||
target = dict(subtitle_dict1)
|
||||
else:
|
||||
subtitle_dicts = (subtitle_dict1,) + subtitle_dicts
|
||||
|
||||
for subtitle_dict in subtitle_dicts:
|
||||
for lang in subtitle_dict:
|
||||
@ -3313,8 +3308,8 @@ class InfoExtractor(object):
|
||||
return target
|
||||
|
||||
def extract_automatic_captions(self, *args, **kwargs):
|
||||
if (self._downloader.params.get('writeautomaticsub', False)
|
||||
or self._downloader.params.get('listsubtitles')):
|
||||
if (self.get_param('writeautomaticsub', False)
|
||||
or self.get_param('listsubtitles')):
|
||||
return self._get_automatic_captions(*args, **kwargs)
|
||||
return {}
|
||||
|
||||
@ -3322,9 +3317,9 @@ class InfoExtractor(object):
|
||||
raise NotImplementedError('This method must be implemented by subclasses')
|
||||
|
||||
def mark_watched(self, *args, **kwargs):
|
||||
if (self._downloader.params.get('mark_watched', False)
|
||||
if (self.get_param('mark_watched', False)
|
||||
and (self._get_login_info()[0] is not None
|
||||
or self._downloader.params.get('cookiefile') is not None)):
|
||||
or self.get_param('cookiefile') is not None)):
|
||||
self._mark_watched(*args, **kwargs)
|
||||
|
||||
def _mark_watched(self, *args, **kwargs):
|
||||
@ -3332,7 +3327,7 @@ class InfoExtractor(object):
|
||||
|
||||
def geo_verification_headers(self):
|
||||
headers = {}
|
||||
geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
|
||||
geo_verification_proxy = self.get_param('geo_verification_proxy')
|
||||
if geo_verification_proxy:
|
||||
headers['Ytdl-request-proxy'] = geo_verification_proxy
|
||||
return headers
|
||||
|
||||
@ -31,10 +31,14 @@ from ..utils import (
|
||||
parse_resolution,
|
||||
sanitized_Request,
|
||||
smuggle_url,
|
||||
strip_or_none,
|
||||
T,
|
||||
traverse_obj,
|
||||
unescapeHTML,
|
||||
unified_timestamp,
|
||||
unsmuggle_url,
|
||||
UnsupportedError,
|
||||
update_url_query,
|
||||
url_or_none,
|
||||
urljoin,
|
||||
xpath_attr,
|
||||
@ -2237,6 +2241,7 @@ class GenericIE(InfoExtractor):
|
||||
'display_id': 'kelis-4th-of-july',
|
||||
'ext': 'mp4',
|
||||
'title': 'Kelis - 4th Of July',
|
||||
'description': 'Kelis - 4th Of July',
|
||||
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
|
||||
},
|
||||
}, {
|
||||
@ -2246,7 +2251,7 @@ class GenericIE(InfoExtractor):
|
||||
'id': '105',
|
||||
'display_id': 'kelis-4th-of-july',
|
||||
'ext': 'mp4',
|
||||
'title': 'Kelis - 4th Of July / Embed Player',
|
||||
'title': r're:Kelis - 4th Of July(?: / Embed Player)?$',
|
||||
'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
|
||||
},
|
||||
'params': {
|
||||
@ -2297,6 +2302,32 @@ class GenericIE(InfoExtractor):
|
||||
'title': 'Syren De Mer onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
|
||||
'thumbnail': r're:https?://www\.camhub\.world/contents/videos_screenshots/389000/389508/preview\.mp4\.jpg',
|
||||
},
|
||||
'skip': 'needs Referer ?',
|
||||
}, {
|
||||
# KVS Player v10
|
||||
'url': 'https://www.cambro.tv/588174/marleny-1/',
|
||||
'md5': '759d2050590986c6fc341da0592c4d8e',
|
||||
'info_dict': {
|
||||
'id': '588174',
|
||||
'display_id': 'marleny-1',
|
||||
'ext': 'mp4',
|
||||
'title': 'marleny 1',
|
||||
'description': 'la maestra de tic toc',
|
||||
'thumbnail': r're:https?://www\.cambro\.tv/contents/videos_screenshots/588000/588174/preview\.jpg',
|
||||
'age_limit': 18,
|
||||
},
|
||||
}, {
|
||||
# KVS Player v10 embed, NSFW
|
||||
'url': 'https://www.cambro.tv/embed/436185',
|
||||
'md5': '24338dc8b182900a2c9eda075a0a46c0',
|
||||
'info_dict': {
|
||||
'id': '436185',
|
||||
'display_id': 'jaeandbailey-chaturbate-webcam-porn-videos',
|
||||
'ext': 'mp4',
|
||||
'title': 'jaeandbailey Chaturbate webcam porn videos',
|
||||
'thumbnail': r're:https?://www\.cambro\.tv/contents/videos_screenshots/436000/436185/preview\.jpg',
|
||||
'age_limit': 18,
|
||||
},
|
||||
}, {
|
||||
'url': 'https://mrdeepfakes.com/video/5/selena-gomez-pov-deep-fakes',
|
||||
'md5': 'fec4ad5ec150f655e0c74c696a4a2ff4',
|
||||
@ -2309,14 +2340,16 @@ class GenericIE(InfoExtractor):
|
||||
'height': 720,
|
||||
'age_limit': 18,
|
||||
},
|
||||
# 'skip': 'Geo-blocked in some mjurisdictions',
|
||||
}, {
|
||||
# KVS Player v2
|
||||
'url': 'https://shooshtime.com/videos/284002/just-out-of-the-shower-joi/',
|
||||
'md5': 'e2f0a4c329f7986280b7328e24036d60',
|
||||
'info_dict': {
|
||||
'id': '284002',
|
||||
'display_id': 'just-out-of-the-shower-joi',
|
||||
'ext': 'mp4',
|
||||
'title': 'Just Out Of The Shower JOI - Shooshtime',
|
||||
'title': r're:Just Out Of The Shower JOI(?: - Shooshtime)?$',
|
||||
'height': 720,
|
||||
'age_limit': 18,
|
||||
},
|
||||
@ -2482,9 +2515,12 @@ class GenericIE(InfoExtractor):
|
||||
return '/'.join(urlparts) + '?' + url_query
|
||||
|
||||
flashvars = self._search_regex(
|
||||
r'(?s)<script\b[^>]*>.*?var\s+flashvars\s*=\s*(\{.+?\});.*?</script>',
|
||||
webpage, 'flashvars')
|
||||
flashvars = self._parse_json(flashvars, video_id, transform_source=js_to_json)
|
||||
r'''(?<![=!+*-])=\s*kt_player\s*\(\s*'kt_player'\s*,\s*[^)]+,\s*([\w$]+)\s*\)''',
|
||||
webpage, 'flashvars name', default='flashvars')
|
||||
flashvars = self._search_json(
|
||||
r'<script(?:\s[^>]*)?>[\s\S]*?var\s+%s\s*=' % (flashvars,),
|
||||
webpage, 'flashvars', video_id, end_pattern=r';[\s\S]*?</script>',
|
||||
transform_source=js_to_json)
|
||||
|
||||
# extract the part after the last / as the display_id from the
|
||||
# canonical URL.
|
||||
@ -2493,12 +2529,7 @@ class GenericIE(InfoExtractor):
|
||||
r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
|
||||
webpage, 'display_id', fatal=False
|
||||
)
|
||||
title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
|
||||
|
||||
thumbnail = flashvars['preview_url']
|
||||
if thumbnail.startswith('//'):
|
||||
protocol, _, _ = url.partition('/')
|
||||
thumbnail = protocol + thumbnail
|
||||
title = flashvars.get('video_title') or self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
|
||||
|
||||
url_keys = list(filter(re.compile(r'^video_(?:url|alt_url\d*)$').match, flashvars.keys()))
|
||||
formats = []
|
||||
@ -2506,9 +2537,13 @@ class GenericIE(InfoExtractor):
|
||||
if '/get_file/' not in flashvars[key]:
|
||||
continue
|
||||
format_id = flashvars.get(key + '_text', key)
|
||||
f_url = urljoin(url, getrealurl(flashvars[key], flashvars['license_code']))
|
||||
rnd = flashvars.get('rnd', key)
|
||||
if rnd:
|
||||
f_url = update_url_query(f_url, {'rnd': rnd})
|
||||
formats.append(merge_dicts(
|
||||
parse_resolution(format_id) or parse_resolution(flashvars[key]), {
|
||||
'url': urljoin(url, getrealurl(flashvars[key], flashvars['license_code'])),
|
||||
'url': f_url,
|
||||
'format_id': format_id,
|
||||
'ext': 'mp4',
|
||||
'http_headers': {'Referer': url},
|
||||
@ -2518,13 +2553,31 @@ class GenericIE(InfoExtractor):
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
csv2list = (T(lambda s: s.split(',')), Ellipsis, T(strip_or_none))
|
||||
info = traverse_obj(flashvars, {
|
||||
'tags': ('video_tags',) + csv2list,
|
||||
'categories': ('video_categories',) + csv2list,
|
||||
'thumbnails': (
|
||||
T(dict.items), lambda _, k_v: k_v[0].startswith('preview_url'), {
|
||||
'url': (1, T(lambda u: urljoin(url, u))),
|
||||
'preference': (0, T(lambda k: 100 - len(k))),
|
||||
}),
|
||||
})
|
||||
info = merge_dicts(info, {
|
||||
'id': flashvars['video_id'],
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
}
|
||||
})
|
||||
|
||||
# check-porn test for embed pages
|
||||
if 'age_limit' not in info and traverse_obj(info, (
|
||||
('title', (('tags', 'categories'), Ellipsis) or []),
|
||||
T(lambda t: bool(re.search(r'(?i)(?:^|\s+)porn(?:$|\s+)', t)) or None)),
|
||||
get_all=False):
|
||||
info['age_limit'] = 18
|
||||
|
||||
return info
|
||||
|
||||
def _real_extract(self, url):
|
||||
if url.startswith('//'):
|
||||
@ -3598,7 +3651,7 @@ class GenericIE(InfoExtractor):
|
||||
), webpage, 'KVS player', group='ver', default=False)
|
||||
if found:
|
||||
self.report_extraction('%s: KVS Player' % (video_id, ))
|
||||
if found.split('.')[0] not in ('4', '5', '6'):
|
||||
if found.split('.')[0] not in ('2', '4', '5', '6', '10'):
|
||||
self.report_warning('Untested major version (%s) in player engine - download may fail.' % (found, ))
|
||||
return merge_dicts(
|
||||
self._extract_kvs(url, webpage, video_id),
|
||||
|
||||
@ -1,6 +1,11 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
merge_dicts,
|
||||
traverse_obj,
|
||||
)
|
||||
|
||||
|
||||
class HentaiStigmaIE(InfoExtractor):
|
||||
@ -24,16 +29,17 @@ class HentaiStigmaIE(InfoExtractor):
|
||||
title = self._html_search_regex(
|
||||
r'<h2[^>]+class="posttitle"[^>]*><a[^>]*>([^<]+)</a>',
|
||||
webpage, 'title')
|
||||
wrap_url = self._html_search_regex(
|
||||
|
||||
wrap_url = self._search_regex(
|
||||
r'<iframe[^>]+src="([^"]+mp4)"', webpage, 'wrapper url')
|
||||
wrap_webpage = self._download_webpage(wrap_url, video_id)
|
||||
|
||||
video_url = self._html_search_regex(
|
||||
r'file\s*:\s*"([^"]+)"', wrap_webpage, 'video url')
|
||||
vid_page = self._download_webpage(wrap_url, video_id)
|
||||
|
||||
return {
|
||||
entries = self._parse_html5_media_entries(wrap_url, vid_page, video_id)
|
||||
self._sort_formats(traverse_obj(entries, (0, 'formats')) or [])
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
'age_limit': 18,
|
||||
}
|
||||
}, entries[0])
|
||||
|
||||
@ -23,7 +23,7 @@ class KalturaIE(InfoExtractor):
|
||||
(?:
|
||||
kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)|
|
||||
https?://
|
||||
(:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com(?::\d+)?/
|
||||
(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com(?::\d+)?/
|
||||
(?:
|
||||
(?:
|
||||
# flash player
|
||||
|
||||
@ -13,7 +13,7 @@ from ..utils import (
|
||||
|
||||
class MgoonIE(InfoExtractor):
|
||||
_VALID_URL = r'''(?x)https?://(?:www\.)?
|
||||
(?:(:?m\.)?mgoon\.com/(?:ch/(?:.+)/v|play/view)|
|
||||
(?:(?:m\.)?mgoon\.com/(?:ch/(?:.+)/v|play/view)|
|
||||
video\.mgoon\.com)/(?P<id>[0-9]+)'''
|
||||
_API_URL = 'http://mpos.mgoon.com/player/video?id={0:}'
|
||||
_TESTS = [
|
||||
|
||||
@ -112,7 +112,7 @@ class ORFRadioIE(ORFRadioBase):
|
||||
|
||||
_VALID_URL = (
|
||||
r'https?://sound\.orf\.at/radio/(?P<station>{0})/sendung/(?P<id>\d+)(?:/(?P<show>\w+))?'.format(_STATION_RE),
|
||||
r'https?://(?P<station>{0})\.orf\.at/player/(?P<date>\d{{8}})/(?P<id>\d+)'.format(_STATION_RE),
|
||||
r'https?://(?P<station>{0})\.orf\.at/(?:player|programm)/(?P<date>\d{{8}})/(?P<id>\d+)'.format(_STATION_RE),
|
||||
)
|
||||
|
||||
_TESTS = [{
|
||||
@ -150,6 +150,10 @@ class ORFRadioIE(ORFRadioBase):
|
||||
'duration': 1500,
|
||||
},
|
||||
'skip': 'Shows from ORF Sound are only available for 30 days.'
|
||||
}, {
|
||||
# yt-dlp/yt-dlp#11014
|
||||
'url': 'https://oe1.orf.at/programm/20240916/769302/Playgrounds',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
||||
@ -12,7 +12,9 @@ from ..utils import (
|
||||
clean_html,
|
||||
decode_packed_codes,
|
||||
determine_ext,
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
get_element_by_class,
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
merge_dicts,
|
||||
@ -56,39 +58,40 @@ def aa_decode(aa_code):
|
||||
|
||||
class XFileShareIE(InfoExtractor):
|
||||
_SITES = (
|
||||
# status check 2024-02: site availability, G site: search
|
||||
(r'aparat\.cam', 'Aparat'), # Cloudflare says host error 522, apparently changed to wolfstreeam.tv
|
||||
(r'filemoon\.sx/.', 'FileMoon'),
|
||||
(r'gounlimited\.to', 'GoUnlimited'), # no media pages listed
|
||||
# status check 2024-10: site availability, G site: search
|
||||
(r'aparat\.cam', 'Aparat'), # Cloudflare says host error 522, apparently changed to wolfstream.tv
|
||||
(r'filemoon\.(?:sx|to|in)', 'FileMoon'),
|
||||
# (r'gounlimited\.to', 'GoUnlimited'), # domain not found
|
||||
(r'govid\.me', 'GoVid'), # no media pages listed
|
||||
(r'highstream\.tv', 'HighStream'), # clipwatching.com redirects here
|
||||
(r'holavid\.com', 'HolaVid'), # Cloudflare says host error 522
|
||||
# (r'streamty\.com', 'Streamty'), # no media pages listed, connection timeout
|
||||
# (r'thevideobee\.to', 'TheVideoBee'), # no pages listed, refuses connection
|
||||
(r'uqload\.to', 'Uqload'), # .com, .co redirect here
|
||||
(r'(?:vedbam\.xyz|vadbam.net)', 'V?dB?m'), # vidbom.com redirects here, but no valid media pages listed
|
||||
(r'highstream\.tv', 'HighStream'), # Cloudflare says host error 522, clipwatching.com now dead
|
||||
(r'holavid\.com', 'HolaVid'), # hoster default home page
|
||||
# (r'streamty\.com', 'Streamty'), # spam parking domain
|
||||
# (r'thevideobee\.to', 'TheVideoBee'), # domain for sale
|
||||
(r'uqload\.ws', 'Uqload'), # .com, .co, .to redirect here
|
||||
# (r'(vadbam.net', 'VadBam'), # domain not found
|
||||
(r'(?:vedbam\.xyz|vadbam\.net|vbn\d\.vdbtm\.shop)', 'V?dB?m'), # vidbom.com redirects here, but no valid media pages listed
|
||||
(r'vidlo\.us', 'vidlo'), # no valid media pages listed
|
||||
(r'vidlocker\.xyz', 'VidLocker'), # no media pages listed
|
||||
(r'(?:w\d\.)?viidshar\.com', 'VidShare'), # vidshare.tv redirects here
|
||||
(r'(?:w\d\.)?viidshar\.com', 'VidShare'), # vidshare.tv parked
|
||||
# (r'vup\.to', 'VUp'), # domain not found
|
||||
(r'wolfstream\.tv', 'WolfStream'),
|
||||
(r'xvideosharing\.com', 'XVideoSharing'), # just started showing 'maintenance mode'
|
||||
# (r'wolfstream\.tv', 'WolfStream'), # domain not found
|
||||
(r'xvideosharing\.com', 'XVideoSharing'),
|
||||
)
|
||||
|
||||
IE_DESC = 'XFileShare-based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
||||
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?P<sub>[a-z]/|)(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
||||
% '|'.join(site for site in list(zip(*_SITES))[0]))
|
||||
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1' % '|'.join(site for site in list(zip(*_SITES))[0])]
|
||||
|
||||
_FILE_NOT_FOUND_REGEXES = (
|
||||
r'>(?:404 - )?File Not Found<',
|
||||
r'>The file was removed by administrator<',
|
||||
r'>\s*(?:404 - )?File Not Found\s*<',
|
||||
r'>\s*The file was removed by administrator\s*<',
|
||||
)
|
||||
_TITLE_REGEXES = (
|
||||
r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||
r'<td nowrap>([^<]+)</td>',
|
||||
r'h4-fine[^>]*>([^<]+)<',
|
||||
r'>Watch (.+)[ <]',
|
||||
r'>Watch (.+?)(?: mp4)?(?: The Ultimate Free Video Hosting Solution for Webmasters and Bloggers)?<',
|
||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to (dead)
|
||||
r'title\s*:\s*"([^"]+)"', # govid.me
|
||||
@ -106,38 +109,41 @@ class XFileShareIE(InfoExtractor):
|
||||
|
||||
_TESTS = [{
|
||||
'note': 'link in `sources`',
|
||||
'url': 'https://uqload.to/dcsu06gdb45o',
|
||||
'md5': '7f8db187b254379440bf4fcad094ae86',
|
||||
'url': 'https://uqload.ws/4sah252totrk.html',
|
||||
'md5': '1f11151b5044862fbc3c112732f9f7d8',
|
||||
'info_dict': {
|
||||
'id': 'dcsu06gdb45o',
|
||||
'id': '4sah252totrk',
|
||||
'ext': 'mp4',
|
||||
'title': 'f2e31015957e74c8c8427982e161c3fc mp4',
|
||||
'title': 'JEONGHAN WONWOO Interview With Allure Korea Arabic Sub',
|
||||
'thumbnail': r're:https://.*\.jpg'
|
||||
},
|
||||
'params': {
|
||||
'nocheckcertificate': True,
|
||||
},
|
||||
'expected_warnings': ['Unable to extract JWPlayer data'],
|
||||
# 'expected_warnings': ['Unable to extract JWPlayer data'],
|
||||
}, {
|
||||
'note': 'link in decoded `sources`',
|
||||
'url': 'https://xvideosharing.com/1tlg6agrrdgc',
|
||||
'md5': '2608ce41932c1657ae56258a64e647d9',
|
||||
'note': 'link in Playerjs', # need test with 'link in decoded `sources`'
|
||||
'url': 'https://xvideosharing.com/8cnupzc1z8xq.html',
|
||||
'md5': '9725ca7229e8f3046f2417da3bd5eddc',
|
||||
'info_dict': {
|
||||
'id': '1tlg6agrrdgc',
|
||||
'id': '8cnupzc1z8xq',
|
||||
'ext': 'mp4',
|
||||
'title': '0121',
|
||||
'title': 'HEVC X265 Big Buck Bunny 1080 10s 20MB',
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
},
|
||||
'skip': 'This server is in maintenance mode.',
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS, in player frame',
|
||||
'url': 'https://filemoon.sx/d/fbsxidybremo',
|
||||
'md5': '82007a71661630f60e866f0d6ed31b2a',
|
||||
'info_dict': {
|
||||
'id': 'dw40rxrzruqz',
|
||||
'title': 'dw40rxrzruqz',
|
||||
'id': 'fbsxidybremo',
|
||||
'title': 'Uchouten',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'ffmpeg',
|
||||
},
|
||||
'expected_warnings': ['hlsnative has detected features it does not support'],
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://vadbam.net/6lnbkci96wly.html',
|
||||
@ -150,7 +156,7 @@ class XFileShareIE(InfoExtractor):
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://w1.viidshar.com/nnibe0xf0h79.html',
|
||||
'md5': 'f0a580ce9df06cc61b4a5c979d672367',
|
||||
'md5': 'b95b97978093bc287c322307c689bd94',
|
||||
'info_dict': {
|
||||
'id': 'nnibe0xf0h79',
|
||||
'title': 'JaGa 68ar',
|
||||
@ -160,27 +166,21 @@ class XFileShareIE(InfoExtractor):
|
||||
'skip_download': 'ffmpeg',
|
||||
},
|
||||
'expected_warnings': ['hlsnative has detected features it does not support'],
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://wolfstream.tv/a3drtehyrg52.html',
|
||||
'md5': '1901d86a79c5e0c6a51bdc9a4cfd3769',
|
||||
'info_dict': {
|
||||
'id': 'a3drtehyrg52',
|
||||
'title': 'NFL 2023 W04 DET@GB',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'url': 'https://aparat.cam/n4d6dh0wvlpr',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://uqload.to/ug5somm0ctnk.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://highstream.tv/2owiyz3sjoux',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://vedbam.xyz/6lnbkci96wly.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://vbn2.vdbtm.shop/6lnbkci96wly.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://filemoon.in/e/5abn1ze9jifb',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@classmethod
|
||||
@ -194,17 +194,26 @@ class XFileShareIE(InfoExtractor):
|
||||
return list(yield_urls())
|
||||
|
||||
def _real_extract(self, url):
|
||||
host, video_id = self._match_valid_url(url).group('host', 'id')
|
||||
host, sub, video_id = self._match_valid_url(url).group('host', 'sub', 'id')
|
||||
|
||||
url = 'https://%s/%s' % (
|
||||
host,
|
||||
url = 'https://%s/%s%s' % (
|
||||
host, sub,
|
||||
'embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
container_div = get_element_by_id('container', webpage) or webpage
|
||||
main = self._search_regex(
|
||||
r'(?s)<main>(.+)</main>', webpage, 'main', default=webpage)
|
||||
container_div = (
|
||||
get_element_by_id('container', main)
|
||||
or get_element_by_class('container', main)
|
||||
or webpage)
|
||||
if self._search_regex(
|
||||
r'>This server is in maintenance mode\.', container_div,
|
||||
'maint error', group=0, default=None):
|
||||
raise ExtractorError(clean_html(container_div), expected=True)
|
||||
if self._search_regex(
|
||||
'not available in your country', container_div,
|
||||
'geo block', group=0, default=None):
|
||||
self.raise_geo_restricted()
|
||||
if self._search_regex(
|
||||
self._FILE_NOT_FOUND_REGEXES, container_div,
|
||||
'missing video error', group=0, default=None):
|
||||
@ -228,38 +237,41 @@ class XFileShareIE(InfoExtractor):
|
||||
|
||||
title = (
|
||||
self._search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
|
||||
or self._og_search_title(webpage, default=None)
|
||||
or video_id).strip()
|
||||
or self._og_search_title(webpage, default='')).strip()
|
||||
|
||||
obf_code = True
|
||||
while obf_code:
|
||||
def deobfuscate(html):
|
||||
for regex, func in (
|
||||
(r'(?s)(?<!-)\b(eval\(function\(p,a,c,k,e,d\)\{(?:(?!</script>).)+\)\))',
|
||||
decode_packed_codes),
|
||||
(r'(ﾟ.+)', aa_decode)):
|
||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||
obf_code = self._search_regex(regex, html, 'obfuscated code', default=None)
|
||||
if obf_code:
|
||||
webpage = webpage.replace(obf_code, func(obf_code))
|
||||
break
|
||||
return html.replace(obf_code, func(obf_code))
|
||||
|
||||
def jw_extract(html):
|
||||
jwplayer_data = self._find_jwplayer_data(
|
||||
webpage.replace(r'\'', '\''), video_id)
|
||||
html.replace(r'\'', '\''), video_id)
|
||||
result = self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, require_title=False,
|
||||
m3u8_id='hls', mpd_id='dash')
|
||||
|
||||
if not traverse_obj(result, 'formats'):
|
||||
if jwplayer_data:
|
||||
result = traverse_obj(result, (
|
||||
(None, ('entries', 0)), T(lambda r: r if r['formats'] else None)),
|
||||
get_all=False) or {}
|
||||
if not result and jwplayer_data:
|
||||
self.report_warning(
|
||||
'Failed to extract JWPlayer formats', video_id=video_id)
|
||||
return result
|
||||
|
||||
def extract_from_links(html):
|
||||
urls = set()
|
||||
for regex in self._SOURCE_URL_REGEXES:
|
||||
for mobj in re.finditer(regex, webpage):
|
||||
for mobj in re.finditer(regex, html):
|
||||
urls.add(mobj.group('url'))
|
||||
|
||||
sources = self._search_regex(
|
||||
r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
|
||||
urls.update(traverse_obj(sources, (T(lambda s: self._parse_json(s, video_id)), Ellipsis)))
|
||||
sources = self._search_json(
|
||||
r'\bsources\s*:', webpage, 'sources', video_id,
|
||||
contains_pattern=r'\[(?!{)[^\]]+\]', default=[])
|
||||
urls.update(sources)
|
||||
|
||||
formats = []
|
||||
for video_url in traverse_obj(urls, (Ellipsis, T(url_or_none))):
|
||||
@ -273,7 +285,33 @@ class XFileShareIE(InfoExtractor):
|
||||
'url': video_url,
|
||||
'format_id': 'sd',
|
||||
})
|
||||
result = {'formats': formats}
|
||||
return {'formats': formats}
|
||||
|
||||
def extract_info(html):
|
||||
html = deobfuscate(html) or html
|
||||
result = jw_extract(html)
|
||||
if not result.get('formats'):
|
||||
result = extract_from_links(html)
|
||||
return result
|
||||
|
||||
def pages_to_extract(html):
|
||||
yield html
|
||||
# page with separate protected download page also has player link
|
||||
player_iframe = self._search_regex(
|
||||
r'(<iframe\s[^>]+>)',
|
||||
get_element_by_id('iframe-holder', html) or '',
|
||||
'player iframe', default='')
|
||||
player_url = extract_attributes(player_iframe).get('src')
|
||||
if player_url:
|
||||
html = self._download_webpage(player_url, video_id, note='Downloading player page', fatal=False)
|
||||
if html:
|
||||
yield html
|
||||
|
||||
result = {}
|
||||
for html in pages_to_extract(webpage):
|
||||
result = extract_info(html)
|
||||
if result.get('formats'):
|
||||
break
|
||||
|
||||
self._sort_formats(result['formats'])
|
||||
|
||||
|
||||
@ -30,17 +30,20 @@ class YandexMusicBaseIE(InfoExtractor):
|
||||
@staticmethod
|
||||
def _raise_captcha():
|
||||
raise ExtractorError(
|
||||
'YandexMusic has considered youtube-dl requests automated and '
|
||||
'asks you to solve a CAPTCHA. You can either wait for some '
|
||||
'time until unblocked and optionally use --sleep-interval '
|
||||
'in future or alternatively you can go to https://music.yandex.ru/ '
|
||||
'solve CAPTCHA, then export cookies and pass cookie file to '
|
||||
'youtube-dl with --cookies',
|
||||
'YandexMusic has considered youtube-dl requests automated '
|
||||
'and asks you to solve a CAPTCHA. You can wait for some time '
|
||||
'until unblocked and optionally use --sleep-interval in future; '
|
||||
'otherwise solve the CAPTCHA at https://music.yandex.ru/, '
|
||||
'then export cookies and pass the cookie file to youtube-dl '
|
||||
'with --cookies.',
|
||||
expected=True)
|
||||
|
||||
def _download_webpage_handle(self, *args, **kwargs):
|
||||
webpage = super(YandexMusicBaseIE, self)._download_webpage_handle(*args, **kwargs)
|
||||
if 'Нам очень жаль, но запросы, поступившие с вашего IP-адреса, похожи на автоматические.' in webpage:
|
||||
blocked_ip_msg = (
|
||||
'Нам очень жаль, но запросы, поступившие с '
|
||||
'вашего IP-адреса, похожи на автоматические.')
|
||||
if blocked_ip_msg in (webpage or [''])[0]:
|
||||
self._raise_captcha()
|
||||
return webpage
|
||||
|
||||
|
||||
@ -342,14 +342,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
if not self._login():
|
||||
return
|
||||
|
||||
_DEFAULT_API_DATA = {
|
||||
'context': {
|
||||
'client': {
|
||||
'clientName': 'WEB',
|
||||
'clientVersion': '2.20201021.03.00',
|
||||
},
|
||||
},
|
||||
}
|
||||
_DEFAULT_API_DATA = {'context': _INNERTUBE_CLIENTS['web']['INNERTUBE_CONTEXT']}
|
||||
|
||||
_YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
|
||||
_YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
|
||||
@ -497,11 +490,15 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
data['params'] = params
|
||||
for page_num in itertools.count(1):
|
||||
search = self._download_json(
|
||||
'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
|
||||
'https://www.youtube.com/youtubei/v1/search',
|
||||
video_id='query "%s"' % query,
|
||||
note='Downloading page %s' % page_num,
|
||||
errnote='Unable to download API page', fatal=False,
|
||||
data=json.dumps(data).encode('utf8'),
|
||||
query={
|
||||
# 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
|
||||
'prettyPrint': 'false',
|
||||
},
|
||||
headers={'content-type': 'application/json'})
|
||||
if not search:
|
||||
break
|
||||
@ -1655,7 +1652,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
assert os.path.basename(func_id) == func_id
|
||||
|
||||
self.write_debug('Extracting signature function {0}'.format(func_id))
|
||||
cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
|
||||
cache_spec, code = self.cache.load('youtube-sigfuncs', func_id, min_ver='2025.04.07'), None
|
||||
|
||||
if not cache_spec:
|
||||
code = self._load_player(video_id, player_url, player_id)
|
||||
@ -1816,6 +1813,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
return ret
|
||||
|
||||
def _extract_n_function_name(self, jscode):
|
||||
func_name, idx = None, None
|
||||
# these special cases are redundant and probably obsolete (2025-04):
|
||||
# they make the tests run ~10% faster without fallback warnings
|
||||
r"""
|
||||
func_name, idx = self._search_regex(
|
||||
# (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}};
|
||||
# (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}};
|
||||
@ -1842,9 +1843,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
\(\s*[\w$]+\s*\)
|
||||
''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
|
||||
default=(None, None))
|
||||
"""
|
||||
|
||||
if not func_name:
|
||||
# nfunc=function(x){...}|function nfunc(x); ...
|
||||
# ... var y=[nfunc]|y[idx]=nfunc);
|
||||
# obvious REs hang, so use a two-stage tactic
|
||||
for m in re.finditer(r'''(?x)
|
||||
[\n;]var\s(?:(?:(?!,).)+,|\s)*?(?!\d)[\w$]+(?:\[(?P<idx>\d+)\])?\s*=\s*
|
||||
(?(idx)|\[\s*)(?P<nfunc>(?!\d)[\w$]+)(?(idx)|\s*\])
|
||||
\s*?[;\n]
|
||||
''', jscode):
|
||||
func_name = self._search_regex(
|
||||
r'[;,]\s*(function\s+)?({0})(?(1)|\s*=\s*function)\s*\((?!\d)[\w$]+\)\s*\{1}(?!\s*return\s)'.format(
|
||||
re.escape(m.group('nfunc')), '{'),
|
||||
jscode, 'Initial JS player n function name (2)', group=2, default=None)
|
||||
if func_name:
|
||||
idx = m.group('idx')
|
||||
break
|
||||
|
||||
# thx bashonly: yt-dlp/yt-dlp/pull/10611
|
||||
if not func_name:
|
||||
self.report_warning('Falling back to generic n function search')
|
||||
self.report_warning('Falling back to generic n function search', only_once=True)
|
||||
return self._search_regex(
|
||||
r'''(?xs)
|
||||
(?:(?<=[^\w$])|^) # instead of \b, which ignores $
|
||||
@ -1858,14 +1878,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
return func_name
|
||||
|
||||
return self._search_json(
|
||||
r'var\s+{0}\s*='.format(re.escape(func_name)), jscode,
|
||||
r'(?<![\w-])var\s(?:(?:(?!,).)+,|\s)*?{0}\s*='.format(re.escape(func_name)), jscode,
|
||||
'Initial JS player n function list ({0}.{1})'.format(func_name, idx),
|
||||
func_name, contains_pattern=r'\[[\s\S]+\]', end_pattern='[,;]',
|
||||
func_name, contains_pattern=r'\[.+\]', end_pattern='[,;]',
|
||||
transform_source=js_to_json)[int(idx)]
|
||||
|
||||
def _extract_n_function_code(self, video_id, player_url):
|
||||
player_id = self._extract_player_info(player_url)
|
||||
func_code = self.cache.load('youtube-nsig', player_id)
|
||||
func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.04.07')
|
||||
jscode = func_code or self._load_player(video_id, player_url)
|
||||
jsi = JSInterpreter(jscode)
|
||||
|
||||
@ -3339,6 +3359,20 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
'thumbnailViewModel', 'image'), final_key='sources'),
|
||||
})
|
||||
|
||||
def _extract_shorts_lockup_view_model(self, view_model):
|
||||
content_id = traverse_obj(view_model, (
|
||||
'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId',
|
||||
T(lambda v: v if YoutubeIE.suitable(v) else None)))
|
||||
if not content_id:
|
||||
return
|
||||
return merge_dicts(self.url_result(
|
||||
content_id, ie=YoutubeIE.ie_key(), video_id=content_id), {
|
||||
'title': traverse_obj(view_model, (
|
||||
'overlayMetadata', 'primaryText', 'content', T(compat_str))),
|
||||
'thumbnails': self._extract_thumbnails(
|
||||
view_model, 'thumbnail', final_key='sources'),
|
||||
})
|
||||
|
||||
def _video_entry(self, video_renderer):
|
||||
video_id = video_renderer.get('videoId')
|
||||
if video_id:
|
||||
@ -3385,10 +3419,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
yield entry
|
||||
|
||||
def _rich_grid_entries(self, contents):
|
||||
for content in contents:
|
||||
content = traverse_obj(
|
||||
content, ('richItemRenderer', 'content'),
|
||||
expected_type=dict) or {}
|
||||
for content in traverse_obj(
|
||||
contents, (Ellipsis, 'richItemRenderer', 'content'),
|
||||
expected_type=dict):
|
||||
video_renderer = traverse_obj(
|
||||
content, 'videoRenderer', 'reelItemRenderer',
|
||||
expected_type=dict)
|
||||
@ -3396,6 +3429,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
entry = self._video_entry(video_renderer)
|
||||
if entry:
|
||||
yield entry
|
||||
# shorts item
|
||||
shorts_lockup_view_model = content.get('shortsLockupViewModel')
|
||||
if shorts_lockup_view_model:
|
||||
entry = self._extract_shorts_lockup_view_model(shorts_lockup_view_model)
|
||||
if entry:
|
||||
yield entry
|
||||
# playlist
|
||||
renderer = traverse_obj(
|
||||
content, 'playlistRenderer', expected_type=dict) or {}
|
||||
@ -3434,23 +3473,15 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
next_continuation = cls._extract_next_continuation_data(renderer)
|
||||
if next_continuation:
|
||||
return next_continuation
|
||||
contents = []
|
||||
for key in ('contents', 'items'):
|
||||
contents.extend(try_get(renderer, lambda x: x[key], list) or [])
|
||||
for content in contents:
|
||||
if not isinstance(content, dict):
|
||||
continue
|
||||
continuation_ep = try_get(
|
||||
content, lambda x: x['continuationItemRenderer']['continuationEndpoint'],
|
||||
dict)
|
||||
if not continuation_ep:
|
||||
continue
|
||||
continuation = try_get(
|
||||
continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
|
||||
for command in traverse_obj(renderer, (
|
||||
('contents', 'items', 'rows'), Ellipsis, 'continuationItemRenderer',
|
||||
('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
|
||||
(('commandExecutorCommand', 'commands', Ellipsis), None), T(dict))):
|
||||
continuation = traverse_obj(command, ('continuationCommand', 'token', T(compat_str)))
|
||||
if not continuation:
|
||||
continue
|
||||
ctp = continuation_ep.get('clickTrackingParams')
|
||||
return YoutubeTabIE._build_continuation_query(continuation, ctp)
|
||||
ctp = command.get('clickTrackingParams')
|
||||
return cls._build_continuation_query(continuation, ctp)
|
||||
|
||||
def _entries(self, tab, item_id, webpage):
|
||||
tab_content = try_get(tab, lambda x: x['content'], dict)
|
||||
@ -3499,6 +3530,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
entry = self._video_entry(renderer)
|
||||
if entry:
|
||||
yield entry
|
||||
renderer = isr_content.get('richGridRenderer')
|
||||
if renderer:
|
||||
for from_ in self._rich_grid_entries(
|
||||
traverse_obj(renderer, ('contents', Ellipsis, T(dict)))):
|
||||
yield from_
|
||||
continuation = self._extract_continuation(renderer)
|
||||
continue
|
||||
|
||||
if not continuation:
|
||||
continuation = self._extract_continuation(is_renderer)
|
||||
@ -3508,8 +3546,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
rich_grid_renderer = tab_content.get('richGridRenderer')
|
||||
if not rich_grid_renderer:
|
||||
return
|
||||
for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
|
||||
yield entry
|
||||
for from_ in self._rich_grid_entries(
|
||||
traverse_obj(rich_grid_renderer, ('contents', Ellipsis, T(dict)))):
|
||||
yield from_
|
||||
|
||||
continuation = self._extract_continuation(rich_grid_renderer)
|
||||
|
||||
@ -3555,8 +3594,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
# Downloading page may result in intermittent 5xx HTTP error
|
||||
# that is usually worked around with a retry
|
||||
response = self._download_json(
|
||||
'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
|
||||
'https://www.youtube.com/youtubei/v1/browse',
|
||||
None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
|
||||
query={
|
||||
# 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
|
||||
'prettyPrint': 'false',
|
||||
},
|
||||
headers=headers, data=json.dumps(data).encode('utf8'))
|
||||
break
|
||||
except ExtractorError as e:
|
||||
|
||||
@ -240,7 +240,7 @@ def _js_ternary(cndn, if_true=True, if_false=False):
|
||||
def _js_unary_op(op):
|
||||
|
||||
@wraps_op(op)
|
||||
def wrapped(_, a):
|
||||
def wrapped(a, _):
|
||||
return op(a)
|
||||
|
||||
return wrapped
|
||||
@ -283,17 +283,6 @@ _OPERATORS = (
|
||||
('**', _js_exp),
|
||||
)
|
||||
|
||||
_COMP_OPERATORS = (
|
||||
('===', _js_id_op(operator.is_)),
|
||||
('!==', _js_id_op(operator.is_not)),
|
||||
('==', _js_eq),
|
||||
('!=', _js_neq),
|
||||
('<=', _js_comp_op(operator.le)),
|
||||
('>=', _js_comp_op(operator.ge)),
|
||||
('<', _js_comp_op(operator.lt)),
|
||||
('>', _js_comp_op(operator.gt)),
|
||||
)
|
||||
|
||||
_LOG_OPERATORS = (
|
||||
('|', _js_bit_op(operator.or_)),
|
||||
('^', _js_bit_op(operator.xor)),
|
||||
@ -310,13 +299,27 @@ _SC_OPERATORS = (
|
||||
_UNARY_OPERATORS_X = (
|
||||
('void', _js_unary_op(lambda _: JS_Undefined)),
|
||||
('typeof', _js_unary_op(_js_typeof)),
|
||||
# avoid functools.partial here since Py2 update_wrapper(partial) -> no __module__
|
||||
('!', _js_unary_op(lambda x: _js_ternary(x, if_true=False, if_false=True))),
|
||||
)
|
||||
|
||||
_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
|
||||
_COMP_OPERATORS = (
|
||||
('===', _js_id_op(operator.is_)),
|
||||
('!==', _js_id_op(operator.is_not)),
|
||||
('==', _js_eq),
|
||||
('!=', _js_neq),
|
||||
('<=', _js_comp_op(operator.le)),
|
||||
('>=', _js_comp_op(operator.ge)),
|
||||
('<', _js_comp_op(operator.lt)),
|
||||
('>', _js_comp_op(operator.gt)),
|
||||
)
|
||||
|
||||
_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS + _SC_OPERATORS))
|
||||
|
||||
_NAME_RE = r'[a-zA-Z_$][\w$]*'
|
||||
_MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
|
||||
_QUOTES = '\'"/'
|
||||
_NESTED_BRACKETS = r'[^[\]]+(?:\[[^[\]]+(?:\[[^\]]+\])?\])?'
|
||||
|
||||
|
||||
class JS_Break(ExtractorError):
|
||||
@ -353,7 +356,7 @@ class LocalNameSpace(ChainMap):
|
||||
raise NotImplementedError('Deleting is not supported')
|
||||
|
||||
def __repr__(self):
|
||||
return 'LocalNameSpace%s' % (self.maps, )
|
||||
return 'LocalNameSpace({0!r})'.format(self.maps)
|
||||
|
||||
|
||||
class Debugger(object):
|
||||
@ -374,6 +377,9 @@ class Debugger(object):
|
||||
|
||||
@classmethod
|
||||
def wrap_interpreter(cls, f):
|
||||
if not cls.ENABLED:
|
||||
return f
|
||||
|
||||
@wraps(f)
|
||||
def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs):
|
||||
if cls.ENABLED and stmt.strip():
|
||||
@ -414,7 +420,17 @@ class JSInterpreter(object):
|
||||
msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
|
||||
super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
|
||||
|
||||
class JS_RegExp(object):
|
||||
class JS_Object(object):
|
||||
def __getitem__(self, key):
|
||||
if hasattr(self, key):
|
||||
return getattr(self, key)
|
||||
raise KeyError(key)
|
||||
|
||||
def dump(self):
|
||||
"""Serialise the instance"""
|
||||
raise NotImplementedError
|
||||
|
||||
class JS_RegExp(JS_Object):
|
||||
RE_FLAGS = {
|
||||
# special knowledge: Python's re flags are bitmask values, current max 128
|
||||
# invent new bitmask values well above that for literal parsing
|
||||
@ -435,16 +451,24 @@ class JSInterpreter(object):
|
||||
def __init__(self, pattern_txt, flags=0):
|
||||
if isinstance(flags, compat_str):
|
||||
flags, _ = self.regex_flags(flags)
|
||||
# First, avoid https://github.com/python/cpython/issues/74534
|
||||
self.__self = None
|
||||
pattern_txt = str_or_none(pattern_txt) or '(?:)'
|
||||
self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
|
||||
# escape unintended embedded flags
|
||||
pattern_txt = re.sub(
|
||||
r'(\(\?)([aiLmsux]*)(-[imsx]+:|(?<!\?)\))',
|
||||
lambda m: ''.join(
|
||||
(re.escape(m.group(1)), m.group(2), re.escape(m.group(3)))
|
||||
if m.group(3) == ')'
|
||||
else ('(?:', m.group(2), m.group(3))),
|
||||
pattern_txt)
|
||||
# Avoid https://github.com/python/cpython/issues/74534
|
||||
self.source = pattern_txt.replace('[[', r'[\[')
|
||||
self.__flags = flags
|
||||
|
||||
def __instantiate(self):
|
||||
if self.__self:
|
||||
return
|
||||
self.__self = re.compile(self.__pattern_txt, self.__flags)
|
||||
self.__self = re.compile(self.source, self.__flags)
|
||||
# Thx: https://stackoverflow.com/questions/44773522/setattr-on-python2-sre-sre-pattern
|
||||
for name in dir(self.__self):
|
||||
# Only these? Obviously __class__, __init__.
|
||||
@ -452,16 +476,15 @@ class JSInterpreter(object):
|
||||
# that can't be setattr'd but also can't need to be copied.
|
||||
if name in ('__class__', '__init__', '__weakref__'):
|
||||
continue
|
||||
if name == 'flags':
|
||||
setattr(self, name, getattr(self.__self, name, self.__flags))
|
||||
else:
|
||||
setattr(self, name, getattr(self.__self, name))
|
||||
|
||||
def __getattr__(self, name):
|
||||
self.__instantiate()
|
||||
# make Py 2.6 conform to its lying documentation
|
||||
if name == 'flags':
|
||||
self.flags = self.__flags
|
||||
return self.flags
|
||||
elif name == 'pattern':
|
||||
self.pattern = self.__pattern_txt
|
||||
if name == 'pattern':
|
||||
self.pattern = self.source
|
||||
return self.pattern
|
||||
elif hasattr(self.__self, name):
|
||||
v = getattr(self.__self, name)
|
||||
@ -469,6 +492,26 @@ class JSInterpreter(object):
|
||||
return v
|
||||
elif name in ('groupindex', 'groups'):
|
||||
return 0 if name == 'groupindex' else {}
|
||||
else:
|
||||
flag_attrs = ( # order by 2nd elt
|
||||
('hasIndices', 'd'),
|
||||
('global', 'g'),
|
||||
('ignoreCase', 'i'),
|
||||
('multiline', 'm'),
|
||||
('dotAll', 's'),
|
||||
('unicode', 'u'),
|
||||
('unicodeSets', 'v'),
|
||||
('sticky', 'y'),
|
||||
)
|
||||
for k, c in flag_attrs:
|
||||
if name == k:
|
||||
return bool(self.RE_FLAGS[c] & self.__flags)
|
||||
else:
|
||||
if name == 'flags':
|
||||
return ''.join(
|
||||
(c if self.RE_FLAGS[c] & self.__flags else '')
|
||||
for _, c in flag_attrs)
|
||||
|
||||
raise AttributeError('{0} has no attribute named {1}'.format(self, name))
|
||||
|
||||
@classmethod
|
||||
@ -482,7 +525,16 @@ class JSInterpreter(object):
|
||||
flags |= cls.RE_FLAGS[ch]
|
||||
return flags, expr[idx + 1:]
|
||||
|
||||
class JS_Date(object):
|
||||
def dump(self):
|
||||
return '(/{0}/{1})'.format(
|
||||
re.sub(r'(?<!\\)/', r'\/', self.source),
|
||||
self.flags)
|
||||
|
||||
@staticmethod
|
||||
def escape(string_):
|
||||
return re.escape(string_)
|
||||
|
||||
class JS_Date(JS_Object):
|
||||
_t = None
|
||||
|
||||
@staticmethod
|
||||
@ -549,6 +601,9 @@ class JSInterpreter(object):
|
||||
def valueOf(self):
|
||||
return _NaN if self._t is None else self._t
|
||||
|
||||
def dump(self):
|
||||
return '(new Date({0}))'.format(self.toString())
|
||||
|
||||
@classmethod
|
||||
def __op_chars(cls):
|
||||
op_chars = set(';,[')
|
||||
@ -652,6 +707,68 @@ class JSInterpreter(object):
|
||||
_SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS, _UNARY_OPERATORS_X))
|
||||
return _cached
|
||||
|
||||
def _separate_at_op(self, expr, max_split=None):
|
||||
|
||||
for op, _ in self._all_operators():
|
||||
# hackety: </> have higher priority than <</>>, but don't confuse them
|
||||
skip_delim = (op + op) if op in '<>*?' else None
|
||||
if op == '?':
|
||||
skip_delim = (skip_delim, '?.')
|
||||
separated = list(self._separate(expr, op, skip_delims=skip_delim))
|
||||
if len(separated) < 2:
|
||||
continue
|
||||
|
||||
right_expr = separated.pop()
|
||||
# handle operators that are both unary and binary, minimal BODMAS
|
||||
if op in ('+', '-'):
|
||||
# simplify/adjust consecutive instances of these operators
|
||||
undone = 0
|
||||
separated = [s.strip() for s in separated]
|
||||
while len(separated) > 1 and not separated[-1]:
|
||||
undone += 1
|
||||
separated.pop()
|
||||
if op == '-' and undone % 2 != 0:
|
||||
right_expr = op + right_expr
|
||||
elif op == '+':
|
||||
while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
|
||||
right_expr = separated.pop() + right_expr
|
||||
if separated[-1][-1:] in self.OP_CHARS:
|
||||
right_expr = separated.pop() + right_expr
|
||||
# hanging op at end of left => unary + (strip) or - (push right)
|
||||
separated.append(right_expr)
|
||||
dm_ops = ('*', '%', '/', '**')
|
||||
dm_chars = set(''.join(dm_ops))
|
||||
|
||||
def yield_terms(s):
|
||||
skip = False
|
||||
for i, term in enumerate(s[:-1]):
|
||||
if skip:
|
||||
skip = False
|
||||
continue
|
||||
if not (dm_chars & set(term)):
|
||||
yield term
|
||||
continue
|
||||
for dm_op in dm_ops:
|
||||
bodmas = list(self._separate(term, dm_op, skip_delims=skip_delim))
|
||||
if len(bodmas) > 1 and not bodmas[-1].strip():
|
||||
bodmas[-1] = (op if op == '-' else '') + s[i + 1]
|
||||
yield dm_op.join(bodmas)
|
||||
skip = True
|
||||
break
|
||||
else:
|
||||
if term:
|
||||
yield term
|
||||
|
||||
if not skip and s[-1]:
|
||||
yield s[-1]
|
||||
|
||||
separated = list(yield_terms(separated))
|
||||
right_expr = separated.pop() if len(separated) > 1 else None
|
||||
expr = op.join(separated)
|
||||
if right_expr is None:
|
||||
continue
|
||||
return op, separated, right_expr
|
||||
|
||||
def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
|
||||
if op in ('||', '&&'):
|
||||
if (op == '&&') ^ _js_ternary(left_val):
|
||||
@ -662,7 +779,7 @@ class JSInterpreter(object):
|
||||
elif op == '?':
|
||||
right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1))
|
||||
|
||||
right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
|
||||
right_val = self.interpret_expression(right_expr, local_vars, allow_recursion) if right_expr else left_val
|
||||
opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
|
||||
if not opfunc:
|
||||
return right_val
|
||||
@ -707,51 +824,9 @@ class JSInterpreter(object):
|
||||
_FINALLY_RE = re.compile(r'finally\s*\{')
|
||||
_SWITCH_RE = re.compile(r'switch\s*\(')
|
||||
|
||||
def handle_operators(self, expr, local_vars, allow_recursion):
|
||||
|
||||
for op, _ in self._all_operators():
|
||||
# hackety: </> have higher priority than <</>>, but don't confuse them
|
||||
skip_delim = (op + op) if op in '<>*?' else None
|
||||
if op == '?':
|
||||
skip_delim = (skip_delim, '?.')
|
||||
separated = list(self._separate(expr, op, skip_delims=skip_delim))
|
||||
if len(separated) < 2:
|
||||
continue
|
||||
|
||||
right_expr = separated.pop()
|
||||
# handle operators that are both unary and binary, minimal BODMAS
|
||||
if op in ('+', '-'):
|
||||
# simplify/adjust consecutive instances of these operators
|
||||
undone = 0
|
||||
separated = [s.strip() for s in separated]
|
||||
while len(separated) > 1 and not separated[-1]:
|
||||
undone += 1
|
||||
separated.pop()
|
||||
if op == '-' and undone % 2 != 0:
|
||||
right_expr = op + right_expr
|
||||
elif op == '+':
|
||||
while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
|
||||
right_expr = separated.pop() + right_expr
|
||||
if separated[-1][-1:] in self.OP_CHARS:
|
||||
right_expr = separated.pop() + right_expr
|
||||
# hanging op at end of left => unary + (strip) or - (push right)
|
||||
left_val = separated[-1] if separated else ''
|
||||
for dm_op in ('*', '%', '/', '**'):
|
||||
bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
|
||||
if len(bodmas) > 1 and not bodmas[-1].strip():
|
||||
expr = op.join(separated) + op + right_expr
|
||||
if len(separated) > 1:
|
||||
separated.pop()
|
||||
right_expr = op.join((left_val, right_expr))
|
||||
else:
|
||||
separated = [op.join((left_val, right_expr))]
|
||||
right_expr = None
|
||||
break
|
||||
if right_expr is None:
|
||||
continue
|
||||
|
||||
left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
|
||||
return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), True
|
||||
def _eval_operator(self, op, left_expr, right_expr, expr, local_vars, allow_recursion):
|
||||
left_val = self.interpret_expression(left_expr, local_vars, allow_recursion)
|
||||
return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion)
|
||||
|
||||
@Debugger.wrap_interpreter
|
||||
def interpret_statement(self, stmt, local_vars, allow_recursion=100):
|
||||
@ -807,15 +882,19 @@ class JSInterpreter(object):
|
||||
else:
|
||||
raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr)
|
||||
|
||||
# apply unary operators (see new above)
|
||||
for op, _ in _UNARY_OPERATORS_X:
|
||||
if not expr.startswith(op):
|
||||
continue
|
||||
operand = expr[len(op):]
|
||||
if not operand or operand[0] != ' ':
|
||||
if not operand or (op.isalpha() and operand[0] != ' '):
|
||||
continue
|
||||
op_result = self.handle_operators(expr, local_vars, allow_recursion)
|
||||
if op_result:
|
||||
return op_result[0], should_return
|
||||
separated = self._separate_at_op(operand, max_split=1)
|
||||
if separated:
|
||||
next_op, separated, right_expr = separated
|
||||
separated.append(right_expr)
|
||||
operand = next_op.join(separated)
|
||||
return self._eval_operator(op, operand, '', expr, local_vars, allow_recursion), should_return
|
||||
|
||||
if expr.startswith('{'):
|
||||
inner, outer = self._separate_at_paren(expr)
|
||||
@ -1010,15 +1089,18 @@ class JSInterpreter(object):
|
||||
|
||||
m = re.match(r'''(?x)
|
||||
(?P<assign>
|
||||
(?P<out>{_NAME_RE})(?:\[(?P<out_idx>(?:.+?\]\s*\[)*.+?)\])?\s*
|
||||
(?P<out>{_NAME_RE})(?P<out_idx>(?:\[{_NESTED_BRACKETS}\])+)?\s*
|
||||
(?P<op>{_OPERATOR_RE})?
|
||||
=(?!=)(?P<expr>.*)$
|
||||
)|(?P<return>
|
||||
(?!if|return|true|false|null|undefined|NaN|Infinity)(?P<name>{_NAME_RE})$
|
||||
)|(?P<indexing>
|
||||
(?P<in>{_NAME_RE})\[(?P<in_idx>(?:.+?\]\s*\[)*.+?)\]$
|
||||
)|(?P<attribute>
|
||||
(?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
|
||||
(?P<var>{_NAME_RE})(?:
|
||||
(?P<nullish>\?)?\.(?P<member>[^(]+)|
|
||||
\[(?P<member2>{_NESTED_BRACKETS})\]
|
||||
)\s*
|
||||
)|(?P<indexing>
|
||||
(?P<in>{_NAME_RE})(?P<in_idx>\[.+\])$
|
||||
)|(?P<function>
|
||||
(?P<fname>{_NAME_RE})\((?P<args>.*)\)$
|
||||
)'''.format(**globals()), expr)
|
||||
@ -1033,10 +1115,11 @@ class JSInterpreter(object):
|
||||
elif left_val in (None, JS_Undefined):
|
||||
raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
|
||||
|
||||
indexes = re.split(r'\]\s*\[', m.group('out_idx'))
|
||||
for i, idx in enumerate(indexes, 1):
|
||||
indexes = md['out_idx']
|
||||
while indexes:
|
||||
idx, indexes = self._separate_at_paren(indexes)
|
||||
idx = self.interpret_expression(idx, local_vars, allow_recursion)
|
||||
if i < len(indexes):
|
||||
if indexes:
|
||||
left_val = self._index(left_val, idx)
|
||||
if isinstance(idx, float):
|
||||
idx = int(idx)
|
||||
@ -1081,14 +1164,17 @@ class JSInterpreter(object):
|
||||
|
||||
if md.get('indexing'):
|
||||
val = local_vars[m.group('in')]
|
||||
for idx in re.split(r'\]\s*\[', m.group('in_idx')):
|
||||
indexes = m.group('in_idx')
|
||||
while indexes:
|
||||
idx, indexes = self._separate_at_paren(indexes)
|
||||
idx = self.interpret_expression(idx, local_vars, allow_recursion)
|
||||
val = self._index(val, idx)
|
||||
return val, should_return
|
||||
|
||||
op_result = self.handle_operators(expr, local_vars, allow_recursion)
|
||||
if op_result:
|
||||
return op_result[0], should_return
|
||||
separated = self._separate_at_op(expr)
|
||||
if separated:
|
||||
op, separated, right_expr = separated
|
||||
return self._eval_operator(op, op.join(separated), right_expr, expr, local_vars, allow_recursion), should_return
|
||||
|
||||
if md.get('attribute'):
|
||||
variable, member, nullish = m.group('var', 'member', 'nullish')
|
||||
@ -1109,13 +1195,15 @@ class JSInterpreter(object):
|
||||
def eval_method(variable, member):
|
||||
if (variable, member) == ('console', 'debug'):
|
||||
if Debugger.ENABLED:
|
||||
Debugger.write(self.interpret_expression('[{}]'.format(arg_str), local_vars, allow_recursion))
|
||||
Debugger.write(self.interpret_expression('[{0}]'.format(arg_str), local_vars, allow_recursion))
|
||||
return
|
||||
types = {
|
||||
'String': compat_str,
|
||||
'Math': float,
|
||||
'Array': list,
|
||||
'Date': self.JS_Date,
|
||||
'RegExp': self.JS_RegExp,
|
||||
# 'Error': self.Exception, # has no std static methods
|
||||
}
|
||||
obj = local_vars.get(variable)
|
||||
if obj in (JS_Undefined, None):
|
||||
@ -1123,7 +1211,7 @@ class JSInterpreter(object):
|
||||
if obj is JS_Undefined:
|
||||
try:
|
||||
if variable not in self._objects:
|
||||
self._objects[variable] = self.extract_object(variable)
|
||||
self._objects[variable] = self.extract_object(variable, local_vars)
|
||||
obj = self._objects[variable]
|
||||
except self.Exception:
|
||||
if not nullish:
|
||||
@ -1134,7 +1222,7 @@ class JSInterpreter(object):
|
||||
|
||||
# Member access
|
||||
if arg_str is None:
|
||||
return self._index(obj, member)
|
||||
return self._index(obj, member, nullish)
|
||||
|
||||
# Function call
|
||||
argvals = [
|
||||
@ -1277,7 +1365,8 @@ class JSInterpreter(object):
|
||||
assertion(len(argvals) == 2, 'takes exactly two arguments')
|
||||
# TODO: argvals[1] callable, other Py vs JS edge cases
|
||||
if isinstance(argvals[0], self.JS_RegExp):
|
||||
count = 0 if argvals[0].flags & self.JS_RegExp.RE_FLAGS['g'] else 1
|
||||
# access JS member with Py reserved name
|
||||
count = 0 if self._index(argvals[0], 'global') else 1
|
||||
assertion(member != 'replaceAll' or count == 0,
|
||||
'replaceAll must be called with a global RegExp')
|
||||
return argvals[0].sub(argvals[1], obj, count=count)
|
||||
@ -1318,7 +1407,7 @@ class JSInterpreter(object):
|
||||
for v in self._separate(list_txt):
|
||||
yield self.interpret_expression(v, local_vars, allow_recursion)
|
||||
|
||||
def extract_object(self, objname):
|
||||
def extract_object(self, objname, *global_stack):
|
||||
_FUNC_NAME_RE = r'''(?:{n}|"{n}"|'{n}')'''.format(n=_NAME_RE)
|
||||
obj = {}
|
||||
fields = next(filter(None, (
|
||||
@ -1339,7 +1428,8 @@ class JSInterpreter(object):
|
||||
fields):
|
||||
argnames = self.build_arglist(f.group('args'))
|
||||
name = remove_quotes(f.group('key'))
|
||||
obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), 'F<{0}>'.format(name))
|
||||
obj[name] = function_with_repr(
|
||||
self.build_function(argnames, f.group('code'), *global_stack), 'F<{0}>'.format(name))
|
||||
|
||||
return obj
|
||||
|
||||
|
||||
@ -1960,7 +1960,7 @@ def get_element_by_attribute(attribute, value, html, escape_value=True):
|
||||
def get_elements_by_class(class_name, html):
|
||||
"""Return the content of all tags with the specified class in the passed HTML document as a list"""
|
||||
return get_elements_by_attribute(
|
||||
'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
|
||||
'class', r'[^\'"]*(?<!-)\b%s\b(?!-)[^\'"]*' % re.escape(class_name),
|
||||
html, escape_value=False)
|
||||
|
||||
|
||||
@ -4204,12 +4204,16 @@ def lowercase_escape(s):
|
||||
s)
|
||||
|
||||
|
||||
def escape_rfc3986(s):
|
||||
def escape_rfc3986(s, safe=None):
|
||||
"""Escape non-ASCII characters as suggested by RFC 3986"""
|
||||
if sys.version_info < (3, 0):
|
||||
s = _encode_compat_str(s, 'utf-8')
|
||||
if safe is not None:
|
||||
safe = _encode_compat_str(safe, 'utf-8')
|
||||
if safe is None:
|
||||
safe = b"%/;:@&=+$,!~*'()?#[]"
|
||||
# ensure unicode: after quoting, it can always be converted
|
||||
return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]"))
|
||||
return compat_str(compat_urllib_parse.quote(s, safe))
|
||||
|
||||
|
||||
def escape_url(url):
|
||||
@ -4498,7 +4502,7 @@ def strip_jsonp(code):
|
||||
def js_to_json(code, *args, **kwargs):
|
||||
|
||||
# vars is a dict of (var, val) pairs to substitute
|
||||
vars = args[0] if len(args) > 0 else kwargs.get('vars', {})
|
||||
js_vars = args[0] if len(args) > 0 else kwargs.get('vars', {})
|
||||
strict = kwargs.get('strict', False)
|
||||
|
||||
STRING_QUOTES = '\'"`'
|
||||
@ -4523,9 +4527,13 @@ def js_to_json(code, *args, **kwargs):
|
||||
else escape)
|
||||
|
||||
def template_substitute(match):
|
||||
evaluated = js_to_json(match.group(1), vars, strict=strict)
|
||||
evaluated = js_to_json(match.group(1), js_vars, strict=strict)
|
||||
if evaluated[0] == '"':
|
||||
try:
|
||||
return json.loads(evaluated)
|
||||
except JSONDecodeError:
|
||||
if strict:
|
||||
raise
|
||||
return evaluated
|
||||
|
||||
def fix_kv(m):
|
||||
@ -4559,14 +4567,14 @@ def js_to_json(code, *args, **kwargs):
|
||||
i = int(im.group(1), base)
|
||||
return ('"%s":' if v.endswith(':') else '%s') % inv(i)
|
||||
|
||||
if v in vars:
|
||||
if v in js_vars:
|
||||
try:
|
||||
if not strict:
|
||||
json.loads(vars[v])
|
||||
json.loads(js_vars[v])
|
||||
except JSONDecodeError:
|
||||
return inv(json.dumps(vars[v]))
|
||||
return inv(json.dumps(js_vars[v]))
|
||||
else:
|
||||
return inv(vars[v])
|
||||
return inv(js_vars[v])
|
||||
|
||||
if not strict:
|
||||
v = try_call(inv, args=(v,), default=v)
|
||||
@ -4577,7 +4585,7 @@ def js_to_json(code, *args, **kwargs):
|
||||
raise ValueError('Unknown value: ' + v)
|
||||
|
||||
def create_map(mobj):
|
||||
return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=vars))))
|
||||
return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=js_vars))))
|
||||
|
||||
code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
|
||||
if not strict:
|
||||
@ -6715,3 +6723,8 @@ class _UnsafeExtensionError(Exception):
|
||||
raise cls(extension)
|
||||
|
||||
return extension
|
||||
|
||||
|
||||
def json_stringify(json_data, **kwargs):
|
||||
kwargs.setdefault('separators', (',', ':'))
|
||||
return json.dumps(json_data, **kwargs).decode('utf-8')
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
__version__ = '2021.12.17'
|
||||
__version__ = '2025.04.07'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user