From d53fec697ea4afad261a8a1832bd9be205f9aecd Mon Sep 17 00:00:00 2001 From: nixxo Date: Mon, 7 Sep 2020 14:21:21 +0200 Subject: [PATCH 1/2] [mailru] fixed extraction improved valid_url regex to match more urls like >https://my.mail.ru/mail/7bw7/video/embed/_myvideo/6778 >https://videoapi.my.mail.ru/videos/embed/mail/7bw7/_myvideo/6778.html previously not matched. Fixed json extraction to download metadata and video url. --- youtube_dl/extractor/mailru.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py index 65cc474db..6fdf70aa6 100644 --- a/youtube_dl/extractor/mailru.py +++ b/youtube_dl/extractor/mailru.py @@ -20,10 +20,10 @@ class MailRuIE(InfoExtractor): IE_DESC = 'Видео@Mail.Ru' _VALID_URL = r'''(?x) https?:// - (?:(?:www|m)\.)?my\.mail\.ru/+ + (?:(?:www|m|videoapi)\.)?my\.mail\.ru/+ (?: video/.*\#video=/?(?P(?:[^/]+/){3}\d+)| - (?:(?P(?:[^/]+/+){2})video/(?P[^/]+/\d+))\.html| + (?:videos/embed/)?(?:(?P(?:[^/]+/+){2})(?:video/(?:embed/)?)?(?P[^/]+/\d+))(?:\.html)?| (?:video/embed|\+/video/meta)/(?P\d+) ) ''' @@ -108,15 +108,21 @@ class MailRuIE(InfoExtractor): if not video_id: video_id = mobj.group('idv2prefix') + mobj.group('idv2suffix') webpage = self._download_webpage(url, video_id) - page_config = self._parse_json(self._search_regex( + page_config = self._parse_json(self._search_regex([ r'(?s)]+class="sp-video__page-config"[^>]*>(.+?)', + r'(?s)"video":\s*(\{.+?\}),'], webpage, 'page config', default='{}'), video_id, fatal=False) if page_config: - meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl') + meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl') or page_config.get('metadataUrl') else: meta_url = None video_data = None + + # fix meta_url if missing the host address + if re.match(r'^\/\+\/', meta_url): + meta_url = 'https://my.mail.ru' + meta_url + if meta_url: video_data = self._download_json( meta_url, video_id or meta_id, 'Downloading video meta JSON', From f3f5b3939eede223537193b710e845a6d3e7c051 Mon Sep 17 00:00:00 2001 From: nixxo Date: Sun, 20 Sep 2020 08:57:09 +0200 Subject: [PATCH 2/2] [mailru] removed escaped braces, use urljoin, added tests --- youtube_dl/extractor/mailru.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py index 6fdf70aa6..5bfe40649 100644 --- a/youtube_dl/extractor/mailru.py +++ b/youtube_dl/extractor/mailru.py @@ -12,6 +12,7 @@ from ..utils import ( parse_duration, remove_end, try_get, + urljoin, ) @@ -93,6 +94,14 @@ class MailRuIE(InfoExtractor): { 'url': 'https://my.mail.ru//list//sinyutin10/video/_myvideo/4.html', 'only_matching': True, + }, + { + 'url': 'https://my.mail.ru/mail/cloud-strife/video/embed/Games/2009', + 'only_matching': True, + }, + { + 'url': 'https://videoapi.my.mail.ru/videos/embed/mail/cloud-strife/Games/2009.html', + 'only_matching': True, } ] @@ -110,7 +119,7 @@ class MailRuIE(InfoExtractor): webpage = self._download_webpage(url, video_id) page_config = self._parse_json(self._search_regex([ r'(?s)]+class="sp-video__page-config"[^>]*>(.+?)', - r'(?s)"video":\s*(\{.+?\}),'], + r'(?s)"video":\s*({.+?}),'], webpage, 'page config', default='{}'), video_id, fatal=False) if page_config: meta_url = page_config.get('metaUrl') or page_config.get('video', {}).get('metaUrl') or page_config.get('metadataUrl') @@ -121,7 +130,7 @@ class MailRuIE(InfoExtractor): # fix meta_url if missing the host address if re.match(r'^\/\+\/', meta_url): - meta_url = 'https://my.mail.ru' + meta_url + meta_url = urljoin('https://my.mail.ru', meta_url) if meta_url: video_data = self._download_json(