Merge 0292587bf3a7eb291fe515b46279a9ff1d819782 into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

.github/workflows/build.yml

@@ -0,0 +1,187 @@
+name: Build
+
+on: workflow_dispatch
+
+jobs:
+  build_unix:
+    runs-on: ubuntu-latest
+    outputs:
+      version_suffix: ${{ steps.version_suffix.outputs.version_suffix }}
+      ytdl_version: ${{ steps.bump_version.outputs.ytdl_version }}
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+      sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
+      sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }}
+      sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }}
+      sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }}
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+          python-version: '3.8'
+    - name: Install packages
+      run: sudo apt-get -y install zip pandoc man
+    - name: Set version suffix
+      id: version_suffix
+      env:
+        PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }}
+      if: "env.PUSH_VERSION_COMMIT == ''"
+      run: echo ::set-output name=version_suffix::$(date -u +"%H%M%S")
+    - name: Bump version
+      id: bump_version
+      run: |
+        python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }}
+        make issuetemplates
+    - name: Push to release
+      id: push_release
+      run: echo ::set-output name=head_sha::$(git rev-parse HEAD)
+    - name: Get Changelog
+      id: get_changelog
+      run: |
+        changelog=$(cat ChangeLog | grep -oPz '(?s)(?<=version ${{ steps.bump_version.outputs.ytdl_version }}\n{2}).+?(?=\n{2,3}version)') || true
+        echo "changelog<<EOF" >> $GITHUB_ENV
+        echo "$changelog" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
+
+    - name: Run Make
+      run: make all tar
+    - name: Get SHA2-256SUMS for youtube-dl
+      id: sha256_bin
+      run: echo "::set-output name=sha256_bin::$(sha256sum youtube-dl | awk '{print $1}')"
+    - name: Get SHA2-256SUMS for youtube-dl.tar.gz
+      id: sha256_tar
+      run: echo "::set-output name=sha256_tar::$(sha256sum youtube-dl.tar.gz | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for youtube-dl
+      id: sha512_bin
+      run: echo "::set-output name=sha512_bin::$(sha512sum youtube-dl | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for youtube-dl.tar.gz
+      id: sha512_tar
+      run: echo "::set-output name=sha512_tar::$(sha512sum youtube-dl.tar.gz | awk '{print $1}')"
+
+    - name: Create Release
+      id: create_release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ steps.bump_version.outputs.ytdl_version }}
+        release_name: youtube-dl ${{ steps.bump_version.outputs.ytdl_version }}
+        commitish: ${{ steps.push_release.outputs.head_sha }}
+        body: ${{ env.changelog }}
+        draft: false
+        prerelease: false
+    - name: Upload youtube-dl Unix binary
+      id: upload-release-asset
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./youtube-dl
+        asset_name: youtube-dl
+        asset_content_type: application/octet-stream
+    - name: Upload Source tar
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./youtube-dl.tar.gz
+        asset_name: youtube-dl-${{ steps.bump_version.outputs.ytdl_version }}.tar.gz
+        asset_content_type: application/gzip
+
+  build_windows:
+    runs-on: windows-2022
+    needs: build_unix
+    outputs:
+      sha256_win: ${{ steps.sha256_win.outputs.sha256_win }}
+      sha512_win: ${{ steps.sha512_win.outputs.sha512_win }}
+
+    steps:
+    - uses: actions/checkout@v2
+    # reason to choose 3.4: https://media.discordapp.net/attachments/807245652072857613/942409077701619742/unknown.png
+    - name: Set up Python 3.4
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.4'
+        architecture: 'x86'
+    - name: Install packages
+      # https://setuptools.pypa.io/en/latest/history.html#v44-0-0
+      # https://pypi.org/project/py2exe/0.9.2.2/
+      # https://pip.pypa.io/en/stable/news/#v19-2
+      # https://wheel.readthedocs.io/en/stable/news.html
+      run: python -m pip install --upgrade "pip<19.2" "setuptools<44" "wheel<0.34.0" py2exe==0.9.2.2
+    - name: Bump version
+      id: bump_version
+      env:
+        version_suffix: ${{ needs.build_unix.outputs.version_suffix }}
+      run: python devscripts/update-version.py ${{ env.version_suffix }}
+    # - name: Run PyInstaller Script
+    #   run: python -m PyInstaller --onefile --console --distpath dist/ -n youtube-dl youtube_dl\__main__.py
+    - name: Build EXE file
+      run: python setup.py py2exe
+    - name: Upload youtube-dl.exe Windows binary
+      id: upload-release-windows
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./youtube-dl.exe
+        asset_name: youtube-dl.exe
+        asset_content_type: application/vnd.microsoft.portable-executable
+    - name: Get SHA2-256SUMS for youtube-dl.exe
+      id: sha256_win
+      run: echo "::set-output name=sha256_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA256).Hash.ToLower())"
+    - name: Get SHA2-512SUMS for youtube-dl.exe
+      id: sha512_win
+      run: echo "::set-output name=sha512_win::$((Get-FileHash youtube-dl.exe -Algorithm SHA512).Hash.ToLower())"
+
+  finish:
+    runs-on: ubuntu-latest
+    needs: [build_unix, build_windows]
+    env:
+      YTDL_VERSION: ${{ needs.build_unix.outputs.ytdl_version }}
+
+    steps:
+    - name: Make SHA2-256SUMS file
+      env:
+        SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }}
+        SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }}
+        SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }}
+      run: |
+        echo "${{ env.SHA256_BIN }}  youtube-dl" >> SHA2-256SUMS
+        echo "${{ env.SHA256_TAR }}  youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-256SUMS
+        echo "${{ env.SHA256_WIN }}  youtube-dl.exe" >> SHA2-256SUMS
+    - name: Upload 256SUMS file
+      id: upload-sums
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./SHA2-256SUMS
+        asset_name: SHA2-256SUMS
+        asset_content_type: text/plain
+    - name: Make SHA2-512SUMS file
+      env:
+        SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }}
+        SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }}
+        SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }}
+      run: |
+        echo "${{ env.SHA512_BIN }}  youtube-dl" >> SHA2-512SUMS
+        echo "${{ env.SHA512_TAR }}  youtube-dl-${YTDL_VERSION}.tar.gz" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WIN }}  youtube-dl.exe" >> SHA2-512SUMS
+    - name: Upload 512SUMS file
+      id: upload-512sums
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./SHA2-512SUMS
+        asset_name: SHA2-512SUMS
+        asset_content_type: text/plain

devscripts/update-version.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+from __future__ import unicode_literals
+
+from datetime import datetime
+import sys
+
+
+with open('youtube_dl/version.py', 'rt') as f:
+    exec(compile(f.read(), 'youtube_dl/version.py', 'exec'))
+old_version = locals()['__version__']
+
+old_version_list = old_version.split('.')
+
+old_ver = '.'.join(old_version_list[:3])
+old_rev = old_version_list[3] if len(old_version_list) > 3 else ''
+
+ver = datetime.utcnow().strftime("%Y.%m.%d")
+
+rev = (sys.argv[1:] or [''])[0]  # Use first argument, if present as revision number
+if not rev:
+    rev = str(int(old_rev or 0) + 1) if old_ver == ver else ''
+
+VERSION = '.'.join((ver, rev)) if rev else ver
+
+VERSION_FILE = '''# Autogenerated by devscripts/update-version.py
+
+__version__ = {!r}
+'''.format(VERSION)
+
+with open('youtube_dl/version.py', 'wt') as f:
+    f.write(VERSION_FILE)
+
+print('::set-output name=ytdl_version::' + VERSION)
+print('\nVersion = %s' % VERSION)

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: