diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..c93cdeb05 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,172 @@ +name: Build Artifacts + +on: + workflow_call: + inputs: + version: + required: true + type: string + unix: + default: true + type: boolean + windows32: + default: true + type: boolean + + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + required: true + type: string + unix: + description: youtube-dl, youtube-dl-py3, youtube-dl.tar.gz + default: true + type: boolean + windows32: + description: youtube-dl.exe + default: true + type: boolean + +permissions: + contents: read + +jobs: + unix: + if: inputs.unix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Needed for changelog + + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install Requirements + run: | + sudo apt -y install zip pandoc man sed + + - name: Prepare + run: | + python devscripts/update_version.py "${{ inputs.version }}" + python devscripts/changelog.py --update + python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py + + - name: Build Unix platform-independent binary + run: | + make all tar + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + youtube-dl + youtube-dl-py3 + youtube-dl.tar.gz + compression-level: 0 + + windows32: + if: inputs.windows32 + runs-on: windows-2022 + env: + PYCRYPTO: pycrypto-2.6.1-cp34-none-win32 + # Workaround for Python 3.4/5 PyPi certificate verification failures - May 2024 + PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + # required for Windows XP support + python-version: "3.4" + architecture: x86 + + - name: Install packages + # https://pip.pypa.io/en/stable/news/#v19-2 + # https://setuptools.pypa.io/en/latest/history.html#v44-0-0 + # https://wheel.readthedocs.io/en/stable/news.html + # https://pypi.org/project/py2exe/0.9.2.2 + shell: bash + run: | + python -m pip install --upgrade \ + "pip<19.2" \ + "setuptools<44" \ + "wheel<0.34.0" \ + "py2exe==0.9.2.2" \ + ; + + - name: PyCrypto cache + id: cache_pycrypto + uses: actions/cache@v4 + with: + key: ${{ env.PYCRYPTO }} + path: ./${{ env.PYCRYPTO }} + + - name: PyCrypto download + if: | + steps.cache_pycrypto.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p "${PYCRYPTO}" + cd "${PYCRYPTO}" + curl -L -O "https://web.archive.org/web/20200627032153/http://www.voidspace.org.uk/python/pycrypto-2.6.1/${PYCRYPTO}.whl" + + - name: PyCrypto install + shell: bash + run: | + python -m pip install "./${PYCRYPTO}/${PYCRYPTO}.whl" + + - name: Prepare + run: | + python devscripts/update_version.py "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py + + - name: Build binary + run: python setup.py py2exe + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + youtube-dl.exe + compression-level: 0 + + meta_files: + if: always() && !cancelled() + needs: + - unix + - windows32 + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-bin-* + merge-multiple: true + + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + # make sure SHA sums are also printed to stdout + sha256sum -- * | tee ../SHA2-256SUMS + sha512sum -- * | tee ../SHA2-512SUMS + # also print as permanent annotations to the summary page + while read -r shasum; do + echo "::notice title=${shasum##* }::sha256: ${shasum% *}" + done < ../SHA2-256SUMS + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-${{ github.job }} + path: | + SHA*SUMS* + compression-level: 0 + overwrite: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..2a441a3a8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,147 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + (default: auto-generated) + required: false + default: "" + type: string + prerelease: + description: Pre-release + default: false + type: boolean + +jobs: + prepare: + permissions: + contents: write + runs-on: ubuntu-latest + outputs: + version: ${{ steps.setup_variables.outputs.version }} + head_sha: ${{ steps.get_target.outputs.head_sha }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Setup variables + id: setup_variables + run: | + revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}" + version="$( + python devscripts/update_version.py \ + ${{ inputs.version || '"${revision}"' }} )" + echo "::group::Output variables" + cat << EOF | tee -a "$GITHUB_OUTPUT" + version=${version} + EOF + echo "::endgroup::" + + - name: Update documentation + env: + version: ${{ steps.setup_variables.outputs.version }} + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + !inputs.prerelease + run: | + python devscripts/changelog.py --update + make README.md + make issuetemplates + make supportedsites + + - name: Push to release + id: push_release + env: + version: ${{ steps.setup_variables.outputs.version }} + creator: ${{ github.event.sender.login }} + if: | + !inputs.prerelease + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -u + git commit -m "Release ${version}" \ + -m "Created by: ${creator}" \ + -m ":ci skip all" + git push origin --force master:release + + - name: Get target commitish + id: get_target + run: | + echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Update master + env: + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease + run: | + git push origin ${{ github.event.ref }} + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + permissions: + contents: read + + publish: + needs: [prepare, build] + permissions: + contents: write + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-* + merge-multiple: true + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Generate release notes + run: | + cat >> ./RELEASE_NOTES << EOF +

Changelog

 + + $(python devscripts/changelog.py) + +
+ EOF + cat > ./PRERELEASE_NOTES << EOF + **This is a pre-release build** + --- + + $(cat ./RELEASE_NOTES) + EOF + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + version: ${{ needs.prepare.outputs.version }} + head_sha: ${{ needs.prepare.outputs.head_sha }} + run: | + gh release create \ + --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \ + --target ${{ env.head_sha }} \ + --title "youtube-dl ${version}" \ + ${{ inputs.prerelease && '--prerelease' || '' }} \ + "${version}" \ + artifact/* diff --git a/Makefile b/Makefile index 3e17365b8..c623d747b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites +all: youtube-dl youtube-dl-py3 README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe + rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish youtube_dl/extractor/lazy_extractors.py *.dump *.part* *.ytdl *.info.json *.mp4 *.m4a *.flv *.mp3 *.avi *.mkv *.webm *.3gp *.wav *.ape *.swf *.jpg *.png CONTRIBUTING.md.tmp youtube-dl youtube-dl-py3 youtube-dl.zip youtube-dl.exe find . -name "*.pyc" -delete find . -name "*.class" -delete @@ -10,6 +10,7 @@ BINDIR ?= $(PREFIX)/bin MANDIR ?= $(PREFIX)/man SHAREDIR ?= $(PREFIX)/share PYTHON ?= /usr/bin/env python +PYTHON3 ?= /usr/bin/env python3 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi) @@ -57,7 +58,7 @@ tar: youtube-dl.tar.gz pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish -youtube-dl: youtube_dl/*.py youtube_dl/*/*.py +youtube-dl.zip: youtube_dl/*.py youtube_dl/*/*.py mkdir -p zip for d in youtube_dl youtube_dl/downloader youtube_dl/extractor youtube_dl/postprocessor ; do \ mkdir -p zip/$$d ;\ @@ -67,11 +68,17 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py mv zip/youtube_dl/__main__.py zip/ cd zip ; zip -q ../youtube-dl youtube_dl/*.py youtube_dl/*/*.py __main__.py rm -rf zip + +youtube-dl: youtube-dl.zip echo '#!$(PYTHON)' > youtube-dl cat youtube-dl.zip >> youtube-dl - rm youtube-dl.zip chmod a+x youtube-dl +youtube-dl-py3: youtube-dl.zip + echo '#!$(PYTHON3)' > youtube-dl-py3 + cat youtube-dl.zip >> youtube-dl-py3 + chmod a+x youtube-dl-py3 + README.md: youtube_dl/*.py youtube_dl/*/*.py COLUMNS=80 $(PYTHON) youtube_dl/__main__.py --help | $(PYTHON) devscripts/make_readme.py diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py deleted file mode 100644 index 4a4295ba9..000000000 --- a/devscripts/buildserver.py +++ /dev/null @@ -1,433 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import ctypes -import functools -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback -import os.path - -sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) -from youtube_dl.compat import ( - compat_input, - compat_http_server, - compat_str, - compat_urlparse, -) - -# These are not used outside of buildserver.py thus not in compat.py - -try: - import winreg as compat_winreg -except ImportError: # Python 2 - import _winreg as compat_winreg - -try: - import socketserver as compat_socketserver -except ImportError: # Python 2 - import SocketServer as compat_socketserver - - -class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer): - allow_reuse_address = True - - -advapi32 = ctypes.windll.advapi32 - -SC_MANAGER_ALL_ACCESS = 0xf003f -SC_MANAGER_CREATE_SERVICE = 0x02 -SERVICE_WIN32_OWN_PROCESS = 0x10 -SERVICE_AUTO_START = 0x2 -SERVICE_ERROR_NORMAL = 0x1 -DELETE = 0x00010000 -SERVICE_STATUS_START_PENDING = 0x00000002 -SERVICE_STATUS_RUNNING = 0x00000004 -SERVICE_ACCEPT_STOP = 0x1 - -SVCNAME = 'youtubedl_builder' - -LPTSTR = ctypes.c_wchar_p -START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR)) - - -class SERVICE_TABLE_ENTRY(ctypes.Structure): - _fields_ = [ - ('lpServiceName', LPTSTR), - ('lpServiceProc', START_CALLBACK) - ] - - -HandlerEx = ctypes.WINFUNCTYPE( - ctypes.c_int, # return - ctypes.c_int, # dwControl - ctypes.c_int, # dwEventType - ctypes.c_void_p, # lpEventData, - ctypes.c_void_p, # lpContext, -) - - -def _ctypes_array(c_type, py_array): - ar = (c_type * len(py_array))() - ar[:] = py_array - return ar - - -def win_OpenSCManager(): - res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS) - if not res: - raise Exception('Opening service manager failed - ' - 'are you running this as administrator?') - return res - - -def win_install_service(service_name, cmdline): - manager = win_OpenSCManager() - try: - h = advapi32.CreateServiceW( - manager, service_name, None, - SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, - cmdline, None, None, None, None, None) - if not h: - raise OSError('Service creation failed: %s' % ctypes.FormatError()) - - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_uninstall_service(service_name): - manager = win_OpenSCManager() - try: - h = advapi32.OpenServiceW(manager, service_name, DELETE) - if not h: - raise OSError('Could not find service %s: %s' % ( - service_name, ctypes.FormatError())) - - try: - if not advapi32.DeleteService(h): - raise OSError('Deletion failed: %s' % ctypes.FormatError()) - finally: - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_service_report_event(service_name, msg, is_error=True): - with open('C:/sshkeys/log', 'a', encoding='utf-8') as f: - f.write(msg + '\n') - - event_log = advapi32.RegisterEventSourceW(None, service_name) - if not event_log: - raise OSError('Could not report event: %s' % ctypes.FormatError()) - - try: - type_id = 0x0001 if is_error else 0x0004 - event_id = 0xc0000000 if is_error else 0x40000000 - lines = _ctypes_array(LPTSTR, [msg]) - - if not advapi32.ReportEventW( - event_log, type_id, 0, event_id, None, len(lines), 0, - lines, None): - raise OSError('Event reporting failed: %s' % ctypes.FormatError()) - finally: - advapi32.DeregisterEventSource(event_log) - - -def win_service_handler(stop_event, *args): - try: - raise ValueError('Handler called with args ' + repr(args)) - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_set_status(handle, status_code): - svcStatus = SERVICE_STATUS() - svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS - svcStatus.dwCurrentState = status_code - svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP - - svcStatus.dwServiceSpecificExitCode = 0 - - if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)): - raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError()) - - -def win_service_main(service_name, real_main, argc, argv_raw): - try: - # args = [argv_raw[i].value for i in range(argc)] - stop_event = threading.Event() - handler = HandlerEx(functools.partial(stop_event, win_service_handler)) - h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None) - if not h: - raise OSError('Handler registration failed: %s' % - ctypes.FormatError()) - - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_start(service_name, real_main): - try: - cb = START_CALLBACK( - functools.partial(win_service_main, service_name, real_main)) - dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [ - SERVICE_TABLE_ENTRY( - service_name, - cb - ), - SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK)) - ]) - - if not advapi32.StartServiceCtrlDispatcherW(dispatch_table): - raise OSError('ctypes start failed: %s' % ctypes.FormatError()) - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--install', - action='store_const', dest='action', const='install', - help='Launch at Windows startup') - parser.add_argument('-u', '--uninstall', - action='store_const', dest='action', const='uninstall', - help='Remove Windows service') - parser.add_argument('-s', '--service', - action='store_const', dest='action', const='service', - help='Run as a Windows service') - parser.add_argument('-b', '--bind', metavar='', - action='store', default='0.0.0.0:8142', - help='Bind to host:port (default %default)') - options = parser.parse_args(args=args) - - if options.action == 'install': - fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox') - cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind) - win_install_service(SVCNAME, cmdline) - return - - if options.action == 'uninstall': - win_uninstall_service(SVCNAME) - return - - if options.action == 'service': - win_service_start(SVCNAME, main) - return - - host, port_str = options.bind.split(':') - port = int(port_str) - - print('Listening on %s:%d' % (host, port)) - srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler) - thr = threading.Thread(target=srv.serve_forever) - thr.start() - compat_input('Press ENTER to shut down') - srv.shutdown() - thr.join() - - -def rmtree(path): - for name in os.listdir(path): - fname = os.path.join(path, name) - if os.path.isdir(fname): - rmtree(fname) - else: - os.chmod(fname, 0o666) - os.remove(fname) - os.rmdir(path) - - -class BuildError(Exception): - def __init__(self, output, code=500): - self.output = output - self.code = code - - def __str__(self): - return self.output - - -class HTTPError(BuildError): - pass - - -class PythonBuilder(object): - def __init__(self, **kwargs): - python_version = kwargs.pop('python', '3.4') - python_path = None - for node in ('Wow6432Node\\', ''): - try: - key = compat_winreg.OpenKey( - compat_winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version)) - try: - python_path, _ = compat_winreg.QueryValueEx(key, '') - finally: - compat_winreg.CloseKey(key) - break - except Exception: - pass - - if not python_path: - raise BuildError('No such Python version: %s' % python_version) - - self.pythonPath = python_path - - super(PythonBuilder, self).__init__(**kwargs) - - -class GITInfoBuilder(object): - def __init__(self, **kwargs): - try: - self.user, self.repoName = kwargs['path'][:2] - self.rev = kwargs.pop('rev') - except ValueError: - raise BuildError('Invalid path') - except KeyError as e: - raise BuildError('Missing mandatory parameter "%s"' % e.args[0]) - - path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user) - if not os.path.exists(path): - os.makedirs(path) - self.basePath = tempfile.mkdtemp(dir=path) - self.buildPath = os.path.join(self.basePath, 'build') - - super(GITInfoBuilder, self).__init__(**kwargs) - - -class GITBuilder(GITInfoBuilder): - def build(self): - try: - subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath]) - subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(GITBuilder, self).build() - - -class YoutubeDLBuilder(object): - authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] - - def __init__(self, **kwargs): - if self.repoName != 'youtube-dl': - raise BuildError('Invalid repository "%s"' % self.repoName) - if self.user not in self.authorizedUsers: - raise HTTPError('Unauthorized user "%s"' % self.user, 401) - - super(YoutubeDLBuilder, self).__init__(**kwargs) - - def build(self): - try: - proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath) - proc.wait() - #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], - # cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(YoutubeDLBuilder, self).build() - - -class DownloadBuilder(object): - def __init__(self, **kwargs): - self.handler = kwargs.pop('handler') - self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:])) - self.srcPath = os.path.abspath(os.path.normpath(self.srcPath)) - if not self.srcPath.startswith(self.buildPath): - raise HTTPError(self.srcPath, 401) - - super(DownloadBuilder, self).__init__(**kwargs) - - def build(self): - if not os.path.exists(self.srcPath): - raise HTTPError('No such file', 404) - if os.path.isdir(self.srcPath): - raise HTTPError('Is a directory: %s' % self.srcPath, 401) - - self.handler.send_response(200) - self.handler.send_header('Content-Type', 'application/octet-stream') - self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1]) - self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size)) - self.handler.end_headers() - - with open(self.srcPath, 'rb') as src: - shutil.copyfileobj(src, self.handler.wfile) - - super(DownloadBuilder, self).build() - - -class CleanupTempDir(object): - def build(self): - try: - rmtree(self.basePath) - except Exception as e: - print('WARNING deleting "%s": %s' % (self.basePath, e)) - - super(CleanupTempDir, self).build() - - -class Null(object): - def __init__(self, **kwargs): - pass - - def start(self): - pass - - def close(self): - pass - - def build(self): - pass - - -class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null): - pass - - -class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler): - actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. - - def do_GET(self): - path = compat_urlparse.urlparse(self.path) - paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()]) - action, _, path = path.path.strip('/').partition('/') - if path: - path = path.split('/') - if action in self.actionDict: - try: - builder = self.actionDict[action](path=path, handler=self, **paramDict) - builder.start() - try: - builder.build() - finally: - builder.close() - except BuildError as e: - self.send_response(e.code) - msg = compat_str(e).encode('UTF-8') - self.send_header('Content-Type', 'text/plain; charset=UTF-8') - self.send_header('Content-Length', len(msg)) - self.end_headers() - self.wfile.write(msg) - else: - self.send_response(500, 'Unknown build method "%s"' % action) - else: - self.send_response(500, 'Malformed URL') - -if __name__ == '__main__': - main() diff --git a/devscripts/changelog.py b/devscripts/changelog.py new file mode 100755 index 000000000..097328ede --- /dev/null +++ b/devscripts/changelog.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import os +import subprocess +import sys + + +def run(args): + process = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) + return process.communicate()[0].strip() + + +def is_core(short): + prefix = None + if ']' in short: + prefix = short.partition(']')[0][1:] + elif ': ' in short: + prefix = short.partition(': ')[0] + + if not prefix or ' ' in prefix: + return True + + prefix = prefix.partition(':')[0].lower() + if prefix.startswith('extractor/'): + prefix = prefix[len('extractor/'):] + if prefix.endswith('ie'): + prefix = prefix[:-len('ie')] + return not os.path.exists('youtube_dl/extractor/%s.py' % prefix) + + +def format_line(markdown, short, sha): + if not markdown: + return '* ' + short + + return '* [%s](https://github.com/ytdl-org/youtube-dl/commit/%s)' % (short, sha) + + +def generate_changelog(markdown): + most_recent_tag = run([ + 'git', 'tag', '--list', '--sort=-v:refname', + '????.??.??', '????.??.??.?', + ]).split('\n')[0] + lines = run([ + 'git', 'log', + '--format=format:%H%n%s', '--no-merges', '-z', + most_recent_tag + '..HEAD', + ]).split('\x00') + + core = [] + extractor = [] + for line in lines: + if not line: + continue + sha, short = line.split('\n') + + if ' * ' in short: + short = short.partition(' * ')[0] + + target = core if is_core(short) else extractor + target.append((sha, short)) + + result = [] + if core: + result.append('#### Core' if markdown else 'Core') + for sha, short in core: + result.append(format_line(markdown, short, sha)) + result.append('') + + if extractor: + result.append('#### Extractor' if markdown else 'Extractor') + for sha, short in extractor: + result.append(format_line(markdown, short, sha)) + result.append('') + + return '\n'.join(result) + + +def read_version(): + with open('youtube_dl/version.py', 'r') as f: + exec(compile(f.read(), 'youtube_dl/version.py', 'exec')) + + return locals()['__version__'] + + +update_in_place = len(sys.argv) > 1 and sys.argv[1] == '--update' +changelog = generate_changelog(not update_in_place) + +if not update_in_place: + print(changelog) + sys.exit() + +with open('ChangeLog', 'rb') as file: + data = file.read() + +with open('ChangeLog', 'wb') as file: + file.write(('version %s\n\n' % read_version()).encode('utf-8')) + file.write(changelog.encode('utf-8')) + file.write('\n\n'.encode('utf-8')) + file.write(data) diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py deleted file mode 100644 index 320bcfc27..000000000 --- a/devscripts/create-github-release.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -from __future__ import unicode_literals - -import json -import mimetypes -import netrc -import optparse -import os -import re -import sys - -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from youtube_dl.compat import ( - compat_basestring, - compat_getpass, - compat_print, - compat_urllib_request, -) -from youtube_dl.utils import ( - make_HTTPS_handler, - sanitized_Request, -) -from utils import read_file - - -class GitHubReleaser(object): - _API_URL = 'https://api.github.com/repos/ytdl-org/youtube-dl/releases' - _UPLOADS_URL = 'https://uploads.github.com/repos/ytdl-org/youtube-dl/releases/%s/assets?name=%s' - _NETRC_MACHINE = 'github.com' - - def __init__(self, debuglevel=0): - self._init_github_account() - https_handler = make_HTTPS_handler({}, debuglevel=debuglevel) - self._opener = compat_urllib_request.build_opener(https_handler) - - def _init_github_account(self): - try: - info = netrc.netrc().authenticators(self._NETRC_MACHINE) - if info is not None: - self._token = info[2] - compat_print('Using GitHub credentials found in .netrc...') - return - else: - compat_print('No GitHub credentials found in .netrc') - except (IOError, netrc.NetrcParseError): - compat_print('Unable to parse .netrc') - self._token = compat_getpass( - 'Type your GitHub PAT (personal access token) and press [Return]: ') - - def _call(self, req): - if isinstance(req, compat_basestring): - req = sanitized_Request(req) - req.add_header('Authorization', 'token %s' % self._token) - response = self._opener.open(req).read().decode('utf-8') - return json.loads(response) - - def list_releases(self): - return self._call(self._API_URL) - - def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False): - data = { - 'tag_name': tag_name, - 'target_commitish': 'master', - 'name': name, - 'body': body, - 'draft': draft, - 'prerelease': prerelease, - } - req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8')) - return self._call(req) - - def create_asset(self, release_id, asset): - asset_name = os.path.basename(asset) - url = self._UPLOADS_URL % (release_id, asset_name) - # Our files are small enough to be loaded directly into memory. - data = open(asset, 'rb').read() - req = sanitized_Request(url, data) - mime_type, _ = mimetypes.guess_type(asset_name) - req.add_header('Content-Type', mime_type or 'application/octet-stream') - return self._call(req) - - -def main(): - parser = optparse.OptionParser(usage='%prog CHANGELOG VERSION BUILDPATH') - options, args = parser.parse_args() - if len(args) != 3: - parser.error('Expected a version and a build directory') - - changelog_file, version, build_path = args - - changelog = read_file(changelog_file) - - mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog) - body = mobj.group(1) if mobj else '' - - releaser = GitHubReleaser() - - new_release = releaser.create_release( - version, name='youtube-dl %s' % version, body=body) - release_id = new_release['id'] - - for asset in os.listdir(build_path): - compat_print('Uploading %s...' % asset) - releaser.create_asset(release_id, os.path.join(build_path, asset)) - - -if __name__ == '__main__': - main() diff --git a/devscripts/release.sh b/devscripts/release.sh deleted file mode 100755 index f2411c927..000000000 --- a/devscripts/release.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# IMPORTANT: the following assumptions are made -# * the GH repo is on the origin remote -# * the gh-pages branch is named so locally -# * the git config user.signingkey is properly set - -# You will need -# pip install coverage nose rsa wheel - -# TODO -# release notes -# make hash on local files - -set -e - -skip_tests=true -gpg_sign_commits="" -buildserver='localhost:8142' - -while true -do -case "$1" in - --run-tests) - skip_tests=false - shift - ;; - --gpg-sign-commits|-S) - gpg_sign_commits="-S" - shift - ;; - --buildserver) - buildserver="$2" - shift 2 - ;; - --*) - echo "ERROR: unknown option $1" - exit 1 - ;; - *) - break - ;; -esac -done - -if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi -version="$1" -major_version=$(echo "$version" | sed -n 's#^\([0-9]*\.[0-9]*\.[0-9]*\).*#\1#p') -if test "$major_version" '!=' "$(date '+%Y.%m.%d')"; then - echo "$version does not start with today's date!" - exit 1 -fi - -if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi -if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi -useless_files=$(find youtube_dl -type f -not -name '*.py') -if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi -if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi -if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi -if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi -if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi - -read -p "Is ChangeLog up to date? (y/n) " -n 1 -if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi - -/bin/echo -e "\n### First of all, testing..." -make clean -if $skip_tests ; then - echo 'SKIPPING TESTS' -else - nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1 -fi - -/bin/echo -e "\n### Changing version in version.py..." -sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py - -/bin/echo -e "\n### Changing version in ChangeLog..." -sed -i "s//$version/" ChangeLog - -/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..." -make README.md CONTRIBUTING.md issuetemplates supportedsites -git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md youtube_dl/version.py ChangeLog -git commit $gpg_sign_commits -m "release $version" - -/bin/echo -e "\n### Now tagging, signing and pushing..." -git tag -s -m "Release $version" "$version" -git show "$version" -read -p "Is it good, can I push? (y/n) " -n 1 -if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi -echo -MASTER=$(git rev-parse --abbrev-ref HEAD) -git push origin $MASTER:master -git push origin "$version" - -/bin/echo -e "\n### OK, now it is time to build the binaries..." -REV=$(git rev-parse HEAD) -make youtube-dl youtube-dl.tar.gz -read -p "VM running? (y/n) " -n 1 -wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe -mkdir -p "build/$version" -mv youtube-dl youtube-dl.exe "build/$version" -mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz" -RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz" -(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS) -(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS) -(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS) -(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS) - -/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..." -for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done - -ROOT=$(pwd) -python devscripts/create-github-release.py ChangeLog $version "$ROOT/build/$version" - -ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" - -/bin/echo -e "\n### Now switching to gh-pages..." -git clone --branch gh-pages --single-branch . build/gh-pages -( - set -e - ORIGIN_URL=$(git config --get remote.origin.url) - cd build/gh-pages - "$ROOT/devscripts/gh-pages/add-version.py" $version - "$ROOT/devscripts/gh-pages/update-feed.py" - "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem" - "$ROOT/devscripts/gh-pages/generate-download.py" - "$ROOT/devscripts/gh-pages/update-copyright.py" - "$ROOT/devscripts/gh-pages/update-sites.py" - git add *.html *.html.in update - git commit $gpg_sign_commits -m "release $version" - git push "$ROOT" gh-pages - git push "$ORIGIN_URL" gh-pages -) -rm -rf build - -make pypi-files -echo "Uploading to PyPi ..." -python setup.py sdist bdist_wheel upload -make clean - -/bin/echo -e "\n### DONE!" diff --git a/devscripts/update_version.py b/devscripts/update_version.py new file mode 100755 index 000000000..c39c98966 --- /dev/null +++ b/devscripts/update_version.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import datetime as dt +import sys + + +VERSION_FILE_FORMAT = '''\ +# Autogenerated by devscripts/update_version.py +from __future__ import unicode_literals + +__version__ = {!r} +''' + + +def split_version(version): + if '.' not in version: + return None, version + + version_list = version.split('.') + version = '.'.join(version_list[:3]) + revision = version_list[3] if len(version_list) > 3 else None + + return version, revision + + +with open('youtube_dl/version.py', 'r') as f: + exec(compile(f.read(), 'youtube_dl/version.py', 'exec')) + +old_ver, old_rev = split_version(locals()['__version__']) +ver, rev = split_version(sys.argv[1]) if len(sys.argv) > 1 else (None, None) + +if not ver: + ver = ( + dt.datetime.now(dt.timezone.utc) if sys.version_info >= (3,) + else dt.datetime.utcnow()).strftime('%Y.%m.%d') + if not rev and old_ver == ver: + rev = str(int(old_rev or 0) + 1) + +if rev: + ver = ver + '.' + rev + +with open('youtube_dl/version.py', 'w') as f: + f.write(VERSION_FILE_FORMAT.format(ver)) + +print(ver) diff --git a/devscripts/wine-py2exe.sh b/devscripts/wine-py2exe.sh deleted file mode 100755 index dc2d6501a..000000000 --- a/devscripts/wine-py2exe.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Run with as parameter a setup.py that works in the current directory -# e.g. no os.chdir() -# It will run twice, the first time will crash - -set -e - -SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" - -if [ ! -d wine-py2exe ]; then - - sudo apt-get install wine1.3 axel bsdiff - - mkdir wine-py2exe - cd wine-py2exe - export WINEPREFIX=`pwd` - - axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi" - axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" - #axel -a "http://winetricks.org/winetricks" - - # http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957 - echo "Follow python setup on screen" - wine msiexec /i python-2.7.msi - - echo "Follow py2exe setup on screen" - wine py2exe-0.6.9.win32-py2.7.exe - - #echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen" - #bash winetricks vcrun2008 - - rm py2exe-0.6.9.win32-py2.7.exe - rm python-2.7.msi - #rm winetricks - - # http://bugs.winehq.org/show_bug.cgi?id=3591 - - mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup - bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch" - mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup - bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch" - - cd - - -else - - export WINEPREFIX="$( cd wine-py2exe && pwd )" - -fi - -wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true -echo '# Copying python27.dll' >> "py2exe.log" -cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/ -wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1 -