Merge f297981f0975adac573c184661908756a8c6de12 into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

README.md

@@ -3,17 +3,94 @@
 
 youtube-dl - download videos from youtube.com or other video platforms
 
+
 - [INSTALLATION](#installation)
 - [DESCRIPTION](#description)
 - [OPTIONS](#options)
+  - [Network Options:](#network-options)
+  - [Geo Restriction:](#geo-restriction)
+  - [Video Selection:](#video-selection)
+  - [Download Options:](#download-options)
+  - [Filesystem Options:](#filesystem-options)
+  - [Thumbnail Options:](#thumbnail-options)
+  - [Verbosity / Simulation Options:](#verbosity--simulation-options)
+  - [Workarounds:](#workarounds)
+  - [Video Format Options:](#video-format-options)
+  - [Subtitle Options:](#subtitle-options)
+  - [Authentication Options:](#authentication-options)
+  - [Adobe Pass Options:](#adobe-pass-options)
+  - [Post-processing Options:](#post-processing-options)
 - [CONFIGURATION](#configuration)
+    - [Authentication with `.netrc` file](#authentication-with-netrc-file)
 - [OUTPUT TEMPLATE](#output-template)
+      - [Output template and Windows batch files](#output-template-and-windows-batch-files)
+      - [Output template examples](#output-template-examples)
 - [FORMAT SELECTION](#format-selection)
-- [VIDEO SELECTION](#video-selection)
+      - [Format selection examples](#format-selection-examples)
+- [VIDEO SELECTION](#video-selection-1)
 - [FAQ](#faq)
+    - [How do I update youtube-dl?](#how-do-i-update-youtube-dl)
+    - [youtube-dl is extremely slow to start on Windows](#youtube-dl-is-extremely-slow-to-start-on-windows)
+    - [I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists](#im-getting-an-error-unable-to-extract-opengraph-title-on-youtube-playlists)
+    - [I'm getting an error when trying to use output template: `error: using output template conflicts with using title, video ID or auto number`](#im-getting-an-error-when-trying-to-use-output-template-error-using-output-template-conflicts-with-using-title-video-id-or-auto-number)
+    - [Do I always have to pass `-citw`?](#do-i-always-have-to-pass--citw)
+    - [Can you please put the `-b` option back?](#can-you-please-put-the--b-option-back)
+    - [I get HTTP error 402 when trying to download a video. What's this?](#i-get-http-error-402-when-trying-to-download-a-video-whats-this)
+    - [Do I need any other programs?](#do-i-need-any-other-programs)
+    - [I have downloaded a video but how can I play it?](#i-have-downloaded-a-video-but-how-can-i-play-it)
+    - [I extracted a video URL with `-g`, but it does not play on another machine / in my web browser.](#i-extracted-a-video-url-with--g-but-it-does-not-play-on-another-machine--in-my-web-browser)
+    - [ERROR: no fmt_url_map or conn information found in video info](#error-no-fmt_url_map-or-conn-information-found-in-video-info)
+    - [ERROR: unable to download video](#error-unable-to-download-video)
+    - [Video URL contains an ampersand and I'm getting some strange output `[1] 2839` or `'v' is not recognized as an internal or external command`](#video-url-contains-an-ampersand-and-im-getting-some-strange-output-1-2839-or-v-is-not-recognized-as-an-internal-or-external-command)
+    - [ExtractorError: Could not find JS function u'OF'](#extractorerror-could-not-find-js-function-uof)
+    - [HTTP Error 429: Too Many Requests or 402: Payment Required](#http-error-429-too-many-requests-or-402-payment-required)
+    - [SyntaxError: Non-ASCII character](#syntaxerror-non-ascii-character)
+    - [What is this binary file? Where has the code gone?](#what-is-this-binary-file-where-has-the-code-gone)
+    - [The exe throws an error due to missing `MSVCR100.dll`](#the-exe-throws-an-error-due-to-missing-msvcr100dll)
+    - [On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?](#on-windows-how-should-i-set-up-ffmpeg-and-youtube-dl-where-should-i-put-the-exe-files)
+    - [How do I put downloads into a specific folder?](#how-do-i-put-downloads-into-a-specific-folder)
+    - [How do I download a video starting with a `-`?](#how-do-i-download-a-video-starting-with-a--)
+    - [How do I pass cookies to youtube-dl?](#how-do-i-pass-cookies-to-youtube-dl)
+    - [How do I stream directly to media player?](#how-do-i-stream-directly-to-media-player)
+    - [How do I download only new videos from a playlist?](#how-do-i-download-only-new-videos-from-a-playlist)
+    - [Should I add `--hls-prefer-native` into my config?](#should-i-add---hls-prefer-native-into-my-config)
+    - [Can you add support for this anime video site, or site which shows current movies for free?](#can-you-add-support-for-this-anime-video-site-or-site-which-shows-current-movies-for-free)
+    - [How can I speed up work on my issue?](#how-can-i-speed-up-work-on-my-issue)
+    - [How can I detect whether a given URL is supported by youtube-dl?](#how-can-i-detect-whether-a-given-url-is-supported-by-youtube-dl)
+- [Why do I need to go through that much red tape when filing bugs?](#why-do-i-need-to-go-through-that-much-red-tape-when-filing-bugs)
 - [DEVELOPER INSTRUCTIONS](#developer-instructions)
+    - [Adding support for a new site](#adding-support-for-a-new-site)
+  - [youtube-dl coding conventions](#youtube-dl-coding-conventions)
+    - [Mandatory and optional metafields](#mandatory-and-optional-metafields)
+      - [Example](#example)
+    - [Provide fallbacks](#provide-fallbacks)
+      - [Example](#example-1)
+    - [Regular expressions](#regular-expressions)
+      - [Don't capture groups you don't use](#dont-capture-groups-you-dont-use)
+        - [Example](#example-2)
+      - [Make regular expressions relaxed and flexible](#make-regular-expressions-relaxed-and-flexible)
+        - [Example](#example-3)
+    - [Long lines policy](#long-lines-policy)
+    - [Inline values](#inline-values)
+      - [Example](#example-4)
+    - [Collapse fallbacks](#collapse-fallbacks)
+      - [Example](#example-5)
+    - [Trailing parentheses](#trailing-parentheses)
+      - [Example](#example-6)
+    - [Use convenience conversion and parsing functions](#use-convenience-conversion-and-parsing-functions)
+      - [More examples](#more-examples)
+        - [Safely extract optional description from parsed JSON](#safely-extract-optional-description-from-parsed-json)
+        - [Safely extract more optional metadata](#safely-extract-more-optional-metadata)
 - [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl)
 - [BUGS](#bugs)
+    - [Is the description of the issue itself sufficient?](#is-the-description-of-the-issue-itself-sufficient)
+    - [Are you using the latest version?](#are-you-using-the-latest-version)
+    - [Is the issue already documented?](#is-the-issue-already-documented)
+    - [Why are existing options not enough?](#why-are-existing-options-not-enough)
+    - [Is there enough context in your bug report?](#is-there-enough-context-in-your-bug-report)
+    - [Does the issue involve one problem, and one problem only?](#does-the-issue-involve-one-problem-and-one-problem-only)
+    - [Is anyone going to need the feature?](#is-anyone-going-to-need-the-feature)
+    - [Is your question about youtube-dl?](#is-your-question-about-youtube-dl)
 - [COPYRIGHT](#copyright)
 
 # INSTALLATION

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: