From aac33155e40af3da96a2467dd05faea201815989 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 24 Jul 2023 23:43:36 +0100 Subject: [PATCH] [build] Add and use `devscripts/utils` --- devscripts/__init__.py | 1 + devscripts/make_lazy_extractors.py | 22 +++++------ devscripts/utils.py | 62 ++++++++++++++++++++++++++++++ test/test_execution.py | 10 +++-- 4 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 devscripts/__init__.py create mode 100644 devscripts/utils.py diff --git a/devscripts/__init__.py b/devscripts/__init__.py new file mode 100644 index 000000000..750dbdca7 --- /dev/null +++ b/devscripts/__init__.py @@ -0,0 +1 @@ +# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index dee9d6d91..5b8b123a4 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals, print_function from inspect import getsource -import io import os from os.path import dirname as dirn import re @@ -9,17 +8,20 @@ import sys print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) # Py2: may be confused by leftover lazy_extractors.pyc -try: - os.remove(lazy_extractors_filename + 'c') -except OSError: - pass +if sys.version_info[0] < 3: + for c in ('c', 'o'): + try: + os.remove(lazy_extractors_filename + 'c') + except OSError: + pass +from devscripts.utils import read_file, write_file from youtube_dl.compat import compat_register_utf8 compat_register_utf8() @@ -27,8 +29,7 @@ compat_register_utf8() from youtube_dl.extractor import _ALL_CLASSES from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor -with open('devscripts/lazy_load_template.py', 'rt') as f: - module_template = f.read() +module_template = read_file('devscripts/lazy_load_template.py') def get_source(m): @@ -114,10 +115,9 @@ for ie in ordered_cls: module_contents.append( '_ALL_CLASSES = [{0}]'.format(', '.join(names))) -module_src = '\n'.join(module_contents) + '\n' +module_src = '\n'.join(module_contents) -with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: - f.write(module_src) +write_file(lazy_extractors_filename, module_src + '\n') # work around JVM byte code module limit in Jython if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7): diff --git a/devscripts/utils.py b/devscripts/utils.py new file mode 100644 index 000000000..2d072d2e0 --- /dev/null +++ b/devscripts/utils.py @@ -0,0 +1,62 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import argparse +import functools +import os.path +import subprocess +import sys + +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +from youtube_dl.compat import ( + compat_kwargs, + compat_open as open, +) + + +def read_file(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +def write_file(fname, content, mode='w'): + with open(fname, mode, encoding='utf-8') as f: + return f.write(content) + + +def read_version(fname='youtube_dl/version.py'): + """Get the version without importing the package""" + exec(compile(read_file(fname), fname, 'exec')) + return locals()['__version__'] + + +def get_filename_args(has_infile=False, default_outfile=None): + parser = argparse.ArgumentParser() + if has_infile: + parser.add_argument('infile', help='Input file') + kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} + kwargs['help'] = 'Output file' + parser.add_argument('outfile', **compat_kwargs(kwargs)) + + opts = parser.parse_args() + if has_infile: + return opts.infile, opts.outfile + return opts.outfile + + +def compose_functions(*functions): + return lambda x: functools.reduce(lambda y, f: f(y), functions, x) + + +def run_process(*args, **kwargs): + kwargs.setdefault('text', True) + kwargs.setdefault('check', True) + kwargs.setdefault('capture_output', True) + if kwargs['text']: + kwargs.setdefault('encoding', 'utf-8') + kwargs.setdefault('errors', 'replace') + kwargs = compat_kwargs(kwargs) + return subprocess.run(args, **kwargs) diff --git a/test/test_execution.py b/test/test_execution.py index 56e1b679d..9daaafa6c 100644 --- a/test/test_execution.py +++ b/test/test_execution.py @@ -8,14 +8,16 @@ import unittest import sys import os import subprocess -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +sys.path.insert(0, rootDir) from youtube_dl.compat import compat_register_utf8, compat_subprocess_get_DEVNULL from youtube_dl.utils import encodeArgument compat_register_utf8() -rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) _DEV_NULL = compat_subprocess_get_DEVNULL() @@ -49,10 +51,10 @@ class TestExecution(unittest.TestCase): subprocess.check_call([sys.executable, os.path.normpath('devscripts/make_lazy_extractors.py'), lazy_extractors], cwd=rootDir, stdout=_DEV_NULL) subprocess.check_call([sys.executable, os.path.normpath('test/test_all_urls.py')], cwd=rootDir, stdout=_DEV_NULL) finally: - for x in ['', 'c'] if sys.version_info[0] < 3 else ['']: + for x in ('', 'c') if sys.version_info[0] < 3 else ('',): try: os.remove(lazy_extractors + x) - except (IOError, OSError): + except OSError: pass