From 2e8984e8c221e719eddf587e1cc1ed7bcfcd19d0 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 4 May 2022 17:30:07 +0100 Subject: [PATCH 1/3] [common] Extract series name from ld+json types such as TVSeries --- youtube_dl/extractor/common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 797c35fd5..526d3483e 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1319,6 +1319,11 @@ class InfoExtractor(object): part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries') if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'): info['series'] = unescapeHTML(part_of_series.get('name')) + elif item_type in ('TVSeries', 'Series', 'CreativeWorkSeries'): + series_name = unescapeHTML(e.get('name')) + info.update({ + 'series': series_name, + }) elif item_type == 'Movie': info.update({ 'title': unescapeHTML(e.get('name')), From 48c366cd89d321ea9fdc52a10946951634ff042f Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 4 May 2022 19:25:41 +0100 Subject: [PATCH 2/3] [common] Avoid erasing extracted ld+json data with None values --- youtube_dl/extractor/common.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 526d3483e..b07b7f9c4 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1228,6 +1228,10 @@ class InfoExtractor(object): if isinstance(json_ld, dict): json_ld = [json_ld] + def valued_dict(items): + """Return dict from dict or iterable of pairs omitting None values""" + return dict((k, v) for k, v in (items.items() if isinstance(items, dict) else items) if v is not None) + INTERACTION_TYPE_MAP = { 'CommentAction': 'comment', 'AgreeAction': 'like', @@ -1325,18 +1329,19 @@ class InfoExtractor(object): 'series': series_name, }) elif item_type == 'Movie': - info.update({ + # here and in the next, don't erase existing value with None + info.update(valued_dict({ 'title': unescapeHTML(e.get('name')), 'description': unescapeHTML(e.get('description')), 'duration': parse_duration(e.get('duration')), 'timestamp': unified_timestamp(e.get('dateCreated')), - }) + })) elif item_type in ('Article', 'NewsArticle'): - info.update({ + info.update(valued_dict({ 'timestamp': parse_iso8601(e.get('datePublished')), 'title': unescapeHTML(e.get('headline')), 'description': unescapeHTML(e.get('articleBody')), - }) + })) elif item_type == 'VideoObject': extract_video_object(e) if expected_type is None: @@ -1350,7 +1355,7 @@ class InfoExtractor(object): continue else: break - return dict((k, v) for k, v in info.items() if v is not None) + return valued_dict(info) @staticmethod def _hidden_inputs(html): From 66ee6aa2da2d71ababcbce6b7604c29dc83d2c82 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 6 May 2022 03:27:45 +0100 Subject: [PATCH 3/3] [TRT] Add extractor for TRT (Turkish TV/radio) catch-up and live streams --- youtube_dl/extractor/extractors.py | 4 + youtube_dl/extractor/trt.py | 325 +++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 youtube_dl/extractor/trt.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 452caeade..7d6a05053 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1293,6 +1293,10 @@ from .trovo import ( TrovoIE, TrovoVodIE, ) +from .trt import ( + TRTIE, + TRTLiveIE, +) from .trunews import TruNewsIE from .trutv import TruTVIE from .tube8 import Tube8IE diff --git a/youtube_dl/extractor/trt.py b/youtube_dl/extractor/trt.py new file mode 100644 index 000000000..65553427d --- /dev/null +++ b/youtube_dl/extractor/trt.py @@ -0,0 +1,325 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import json +import re +import time + +from .common import InfoExtractor + +from ..compat import ( + compat_urlparse, +) +from ..utils import ( + determine_ext, + dict_get, + merge_dicts, + parse_iso8601, + strip_or_none, + try_get, + url_or_none, + urljoin, +) + + +class TRTIE(InfoExtractor): + IE_DESC = 'TRT (Turkish State TV) programs and series' + _VALID_URL = r'''(?x) + (?Phttps?://(?:www\.)trtizle\.com/ + (?:diziler|programlar|belgesel|filmler|cocuk|trtarsiv|engelsiz)/ + (?P[\w-]+))(?:/(?P[\w-]+))?''' + _TESTS = [{ + 'url': 'https://www.trtizle.com/belgesel/dunya-tarihinin-donum-noktalari/dunya-tarihinin-donum-noktalari-1-bolum-125583', + 'md5': 'c46dc0b9b53ad372c4ac6b3982805f05', + 'info_dict': { + 'id': 'dunya-tarihinin-donum-noktalari-1-bolum-125583', + 'ext': 'mp4', + 'title': 'Dünya Tarihinin Dönüm Noktaları 1.Bölüm', + 'description': 'Bedelini insanların ödeyeceği bir imparatorluk çekişmesinde Persler, Yunanlara karşı...', + 'timestamp': 1617148800, + 'upload_date': '20210331', + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': float, + 'series': 'Dünya Tarihinin Dönüm Noktaları', + }, + 'params': { + # adaptive download + 'skip_download': True, + } + }, { + 'url': 'https://www.trtizle.com/belgesel/dunya-tarihinin-donum-noktalari', + 'info_dict': { + 'id': 'dunya-tarihinin-donum-noktalari', + 'title': 'Dünya Tarihinin Dönüm Noktaları', + }, + 'playlist_mincount': 22, + }, { + 'url': 'https://www.trtizle.com/diziler/yol-ayrimi/yol-ayrimi-1-bolum-5774583', + 'md5': '67ada6b2020b5dd0d3e24646b2725676', + 'info_dict': { + 'id': 'yol-ayrimi-1-bolum-5774583', + 'ext': 'mp4', + 'title': 'Yol Ayrımı 1.Bölüm', + 'description': 'Seyrisefain balosunda, herkes bir haberin akıbetini beklemektedir…', + 'timestamp': 1623888000, + 'upload_date': '20210617', + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': float, + 'series': 'Yol Ayrımı', + }, + 'params': { + # adaptive download + 'skip_download': True, + }, + }, { + 'url': 'https://www.trtizle.com/diziler/yol-ayrimi/', + 'info_dict': { + 'id': 'yol-ayrimi', + 'title': 'Yol Ayrımı', + }, + 'playlist_mincount': 5, + }, { + 'url': 'https://www.trtizle.com/programlar/sade-saz/sade-saz-1-bolum-7646201', + 'md5': '8f416e64379ea4d1d3ea0a65dc922f5c', + 'info_dict': { + 'id': 'sade-saz-1-bolum-7646201', + 'ext': 'mp4', + 'title': 'Sade Saz 1.Bölüm', + 'description': 'Sade Saz’ın ilk bölümünün konuğu, tanbur icracısı K. Alper Uzkur.', + 'timestamp': 1641772800, + 'upload_date': '20220110', + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': float, + 'series': 'Sade Saz', + }, + 'params': { + # adaptive download + 'skip_download': True, + }, + }, { + 'url': 'https://www.trtizle.com/programlar/sade-saz', + 'info_dict': { + 'id': 'sade-saz', + 'title': 'Sade Saz', + }, + 'playlist_mincount': 6, + }, { + 'url': 'https://www.trtizle.com/filmler/looking-for-eric/looking-for-eric-8414201', + 'md5': '833d61e4a10606d71b3903295cfa3c63', + 'info_dict': { + 'id': 'looking-for-eric-8414201', + 'ext': 'mp4', + 'title': 'Looking for Eric', + 'description': 'Postacı Eric\'in hayatı krize sürüklenirken gerçek ve hayal birbirine karışır...', + 'upload_date': '20220401', + 'timestamp': 1648771200, + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': float, + }, + 'params': { + # adaptive download + 'skip_download': True, + }, + }, { + 'url': 'https://www.trtizle.com/cocuk/kaptan-pengu-ve-arkadaslari/kaptan-pengu-ve-arkadaslari-okul-aciliyor-6034815', + 'md5': '551c479d1a6bc7c538356907d4ea5d19', + 'info_dict': { + 'id': 'kaptan-pengu-ve-arkadaslari-okul-aciliyor-6034815', + 'ext': 'mp4', + 'title': 'Kaptan Pengu ve Arkadaşları 1.Bölüm', + 'description': 'Hayvanlar Konseyi\'nden Kaptan Pengu\'ya bir mektup vardır...', + 'timestamp': 1626134400, + 'upload_date': '20210713', + 'thumbnail': r're:https?://.+\.jpe?g', + 'duration': float, + 'series': 'Kaptan Pengu ve Arkadaşları', + }, + 'params': { + # adaptive download + 'skip_download': True, + }, + }, { + 'url': 'https://www.trtizle.com/cocuk/kaptan-pengu-ve-arkadaslari', + 'info_dict': { + 'id': 'kaptan-pengu-ve-arkadaslari', + 'title': 'Kaptan Pengu ve Arkadaşları', + }, + 'playlist_mincount': 41, + }, + ] + + def _extract_formats(self, fmt_url, video_id): + formats = [] + ext = determine_ext(fmt_url) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + fmt_url, video_id, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False)) + elif ext == 'mpd': + formats.extend(self._extract_mpd_formats( + fmt_url, video_id, mpd_id='dash', fatal=False)) + else: + formats.append({ + 'url': fmt_url, + }) + return formats + + def _extract_list(self, playlist_id, url): + webpage = self._download_webpage(url, playlist_id) + LIST_RE = ( + r''']*?\b%s\s*=\s*['"](%s(?:(?<=/)|/)[\w-]+)''' + % ('data-path' if 'data-path' in webpage else 'href', + re.escape(compat_urlparse.urlparse(url).path), )) + + def entries(): + for item_url in re.finditer(LIST_RE, webpage): + item_url = urljoin(url, item_url.group(1)) + yield self._extract_video(self._match_id(item_url), item_url) + + series = self._search_json_ld(webpage, playlist_id, default={}, expected_type='TVSeries') + return self.playlist_result(entries(), playlist_id, series.get('series')) + + def _extract_video(self, video_id, url): + webpage = self._download_webpage(url, video_id) + result = self._search_json_ld(webpage, video_id, default={}) + result['id'] = video_id + if 'title' not in result: + result['title'] = ( + self._html_search_meta(('title', 'og:title', 'twitter:title'), webpage) + or self._html_search_regex(r']*>([^<]+)[\w-]+)' + _TESTS = [{ + 'url': 'https://www.trtizle.com/canli/tv/trt-world', + 'info_dict': { + 'id': 'trtworld', + 'ext': 'mp4', + 'title': r're:TRT WORLD .+', + 'description': 'TRT World', + 'is_live': True, + }, + 'params': { + # adaptive download + 'skip_download': True, + } + }, + ] + + def _real_extract(self, url): + chan_id = self._match_id(url) + webpage = self._download_webpage(url, chan_id) + chan_id = self._search_regex( + r'\blivePlayer\s*\.\s*openPlayer\s*\([^)]*?\btrt\.com\.tr/trtportal/(?:[^/]+/)+thumbnails/([\w-]+)\.(?:jp|png)', + webpage, 'slug', fatal=False) or chan_id + chan_url = self._search_regex( + r'''\blivePlayerConfig\s*\.\s*baseEpgUrl\s*=\s*(?P'|")(?Phttps?://(?:(?!(?P=q)).)+)(?P=q)''', + webpage, 'player config', group='url') + chan_url = '%s%s.json' % (chan_url, chan_id) + + def maybe_xml2json(src): + """Turn unexpected XML returned from an API URL into JSON""" + m = re.match(r'''^\s*<\?xml\b(?:[^/>]*?\bencoding\s*=\s*['"](?P[\w-]+))?[^/>]*\?>\s*(?P<.+>)$''', src) + if m: + + # Thanks https://stackoverflow.com/a/63556250 for inspiration + ATTR_RE = ( + r"""(?s)(?P\S ?)(?:\s*=\s*(?P['"])(?P.*?)(?\S )(?P[^/>]*)(?:(?:>(?P.*?))|(?:/>))", + xml_str, + ) + + if elts: + elts = [{i[0]: elt_value(i[1], xml2dict(i[2]))} for i in elts] + if len(elts) == 1: + return elts[0] + return elts + return xml_str + + try: + return json.dumps(xml2dict(m.group('xml').encode(m.group('enc') or 'utf-8'))) + except Exception: + pass + return src + + chan_info = self._download_json( + chan_url, chan_id, fatal=False, + note='Downloading player EPG JSON', + query={'_': int(time.time() * 1000)}, + expected_status=403, + # errors are returned as XML + transform_source=maybe_xml2json) + if not isinstance(chan_info, dict) or 'Error' in chan_info: + chan_info = self._download_json( + 'https://trtizle-api.cdn.wp.trt.com.tr/trttv/v3/livestream', + chan_id, fatal=False, + note='Downloading livestream API JSON', + query={'path': compat_urlparse.urlparse(url).path}) or {} + + title = chan_info['channel']['title'] + + current = try_get(chan_info, lambda x: x['current'], dict) or {} + if current.get('geo_block'): + self._downloader.report_warning( + '[%s] %s' % (self.IE_NAME, 'Stream is geo-blocked')) + + chan_info = chan_info['channel'] + fmt_url = dict_get(chan_info, ('url', 'noneDvrUrl')) + formats = [] + if fmt_url: + formats = self._extract_formats(fmt_url, chan_id) + self._sort_formats(formats) + + start_end = [parse_iso8601(current.get(x)) for x in ('starttime', 'endtime')] + if None in start_end: + start_end = None + + return { + 'id': chan_id, + 'title': self._live_title(current.get('title') or title), + 'is_live': True, + 'formats': formats, + 'description': strip_or_none(chan_info.get('description')), + 'thumbnail': next((url_or_none(chan_info.get(x)) + for x in ('thumbnail', 'thumbnailYoutubeUrl', 'square_logo', 'livestreamLogoUrl')), + None), + 'timestamp': start_end and start_end[0], + 'duration': start_end and (start_end[1] - time.time()), + }