From 47478f876f72919741e34ae48a5c138ecaaf8a1f Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Tue, 26 Oct 2021 13:45:37 +0100 Subject: [PATCH 01/10] Fixed playlist description/added playlist description when using `youtube-dl --flat-playlist -J ` --- youtube_dl/extractor/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index dc4bd4a77..774c34bc4 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2785,6 +2785,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): data, lambda x: x['metadata']['playlistMetadataRenderer'], dict) if renderer: title = renderer.get('title') + description = renderer.get('description') else: renderer = try_get( data, lambda x: x['header']['hashtagHeaderRenderer'], dict) From dc7db7123452a4107a099bf6aca62c0096d69c87 Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:13:50 +0100 Subject: [PATCH 02/10] Added `view_count` & `last_updated` details about playlist to the output of: `youtube-dl -J ` --- youtube_dl/extractor/common.py | 7 ++++++- youtube_dl/extractor/youtube.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 797c35fd5..037acb6d9 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -968,7 +968,8 @@ class InfoExtractor(object): urls, playlist_id=playlist_id, playlist_title=playlist_title) @staticmethod - def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None): + def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, + playlist_view_count=None, playlist_last_update=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} @@ -978,6 +979,10 @@ class InfoExtractor(object): video_info['title'] = playlist_title if playlist_description: video_info['description'] = playlist_description + if playlist_view_count: + video_info['view_count'] = playlist_view_count + if playlist_last_update: + video_info['last_updated'] = playlist_last_update return video_info def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 774c34bc4..91f558883 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2786,6 +2786,19 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if renderer: title = renderer.get('title') description = renderer.get('description') + + stats = try_get( + data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'][0]['playlistSidebarPrimaryInfoRenderer']['stats'] + ) + view_count_text = try_get( + stats, lambda x: x[1]['simpleText'], compat_str) or '' + view_count = str_to_int(self._search_regex( + r'^([\d,]+)', re.sub(r'\s', '', view_count_text), + 'view count', default=None)) + + last_updated_text = try_get(stats, lambda x: x[2]['runs'][1]['text']) + last_updated = unified_strdate(last_updated_text) + else: renderer = try_get( data, lambda x: x['header']['hashtagHeaderRenderer'], dict) @@ -2794,7 +2807,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): playlist = self.playlist_result( self._entries(selected_tab, item_id, webpage), playlist_id=playlist_id, playlist_title=title, - playlist_description=description) + playlist_description=description, + playlist_view_count=view_count, + playlist_last_update=last_updated + ) playlist.update(self._extract_uploader(data)) return playlist From 9dc25a936722f736d661850ab625737ab77da7a3 Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 27 Oct 2021 15:13:50 +0100 Subject: [PATCH 03/10] Added `view_count` & `last_updated` details about playlist to the output of: `youtube-dl -J ` --- youtube_dl/extractor/common.py | 7 ++++++- youtube_dl/extractor/youtube.py | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 797c35fd5..037acb6d9 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -968,7 +968,8 @@ class InfoExtractor(object): urls, playlist_id=playlist_id, playlist_title=playlist_title) @staticmethod - def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None): + def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, + playlist_view_count=None, playlist_last_update=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} @@ -978,6 +979,10 @@ class InfoExtractor(object): video_info['title'] = playlist_title if playlist_description: video_info['description'] = playlist_description + if playlist_view_count: + video_info['view_count'] = playlist_view_count + if playlist_last_update: + video_info['last_updated'] = playlist_last_update return video_info def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 774c34bc4..b08da9956 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2786,6 +2786,18 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if renderer: title = renderer.get('title') description = renderer.get('description') + + stats = try_get( + data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'][0]['playlistSidebarPrimaryInfoRenderer']['stats']) + view_count_text = try_get( + stats, lambda x: x[1]['simpleText'], compat_str) or '' + view_count = str_to_int(self._search_regex( + r'^([\d,]+)', re.sub(r'\s', '', view_count_text), + 'view count', default=None)) + + last_updated_text = try_get(stats, lambda x: x[2]['runs'][1]['text']) + last_updated = unified_strdate(last_updated_text) + else: renderer = try_get( data, lambda x: x['header']['hashtagHeaderRenderer'], dict) @@ -2794,7 +2806,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): playlist = self.playlist_result( self._entries(selected_tab, item_id, webpage), playlist_id=playlist_id, playlist_title=title, - playlist_description=description) + playlist_description=description, + playlist_view_count=view_count, + playlist_last_update=last_updated) playlist.update(self._extract_uploader(data)) return playlist From ac6d433c80bfd32da8491a3714eaa83250002891 Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 27 Oct 2021 22:10:35 +0100 Subject: [PATCH 04/10] Changed method `playlist_result` to use `**kwargs` instead of acquiring a potentially unlimited argument list --- youtube_dl/extractor/common.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 037acb6d9..83b6db2f1 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -968,21 +968,11 @@ class InfoExtractor(object): urls, playlist_id=playlist_id, playlist_title=playlist_title) @staticmethod - def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, - playlist_view_count=None, playlist_last_update=None): + def playlist_result(entries, **kwargs): """Returns a playlist""" video_info = {'_type': 'playlist', - 'entries': entries} - if playlist_id: - video_info['id'] = playlist_id - if playlist_title: - video_info['title'] = playlist_title - if playlist_description: - video_info['description'] = playlist_description - if playlist_view_count: - video_info['view_count'] = playlist_view_count - if playlist_last_update: - video_info['last_updated'] = playlist_last_update + 'entries': entries, + **kwargs} return video_info def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None): From 530361126f556f8ff4a9ab1eab3f7fffdb59d89a Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 27 Oct 2021 22:55:55 +0100 Subject: [PATCH 05/10] Modified code to account for extractors that do not specify field names. --- youtube_dl/extractor/common.py | 12 +++++++++--- youtube_dl/extractor/youtube.py | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 83b6db2f1..5b9844474 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -968,11 +968,17 @@ class InfoExtractor(object): urls, playlist_id=playlist_id, playlist_title=playlist_title) @staticmethod - def playlist_result(entries, **kwargs): + def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None, **kwargs): """Returns a playlist""" video_info = {'_type': 'playlist', - 'entries': entries, - **kwargs} + 'entries': entries} + video_info.update(kwargs) + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title + if playlist_description is not None: + video_info['description'] = playlist_description return video_info def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 0b8093446..0983357f5 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2806,8 +2806,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): self._entries(selected_tab, item_id, webpage), playlist_id=playlist_id, playlist_title=title, playlist_description=description, - playlist_view_count=view_count, - playlist_last_update=last_updated) + view_count=view_count, + last_update=last_updated) playlist.update(self._extract_uploader(data)) return playlist From 0056f6d8806d9fa83ba4a1c7272904ddf99ee7f6 Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 27 Oct 2021 23:01:03 +0100 Subject: [PATCH 06/10] Corrected typo & modified if statement --- youtube_dl/extractor/common.py | 2 +- youtube_dl/extractor/youtube.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 5b9844474..f5ad7ae06 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -977,7 +977,7 @@ class InfoExtractor(object): video_info['id'] = playlist_id if playlist_title: video_info['title'] = playlist_title - if playlist_description is not None: + if playlist_description: video_info['description'] = playlist_description return video_info diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 0983357f5..7f482ed3a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2807,7 +2807,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): playlist_id=playlist_id, playlist_title=title, playlist_description=description, view_count=view_count, - last_update=last_updated) + last_updated=last_updated) playlist.update(self._extract_uploader(data)) return playlist From 165646d7e2b0f16ba0a04dd1b4b12282585225f0 Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Thu, 28 Oct 2021 19:14:37 +0100 Subject: [PATCH 07/10] Fixed bug where if playlist was edited under 7 days ago, it would show `last_updated` as null due to position of view_count in stats changing location & format --- youtube_dl/extractor/youtube.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 7f482ed3a..6a9baa96e 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -38,6 +38,7 @@ from ..utils import ( try_get, unescapeHTML, unified_strdate, + date_from_str, unsmuggle_url, update_url_query, url_or_none, @@ -2795,8 +2796,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): r'^([\d,]+)', re.sub(r'\s', '', view_count_text), 'view count', default=None)) - last_updated_text = try_get(stats, lambda x: x[2]['runs'][1]['text']) - last_updated = unified_strdate(last_updated_text) + last_updated_text = try_get(stats, lambda x: x[2]['runs'][1]['text']) or try_get(stats, lambda x: x[2]['runs'][0]['text']) + last_updated_text = last_updated_text.replace('Updated ', '') if 'Updated ' in last_updated_text else last_updated_text + try: + last_updated = unified_strdate(last_updated_text) + if last_updated is None: + last_updated = date_from_str(last_updated_text).strftime("%Y%m%d") + except: + last_updated = None else: renderer = try_get( data, lambda x: x['header']['hashtagHeaderRenderer'], dict) From 9d9d429fa485362ae4a6e03987f7c3ad680ceaad Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Thu, 28 Oct 2021 19:22:56 +0100 Subject: [PATCH 08/10] Specified error for except to look for --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 6a9baa96e..30e79ccfa 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2802,7 +2802,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): last_updated = unified_strdate(last_updated_text) if last_updated is None: last_updated = date_from_str(last_updated_text).strftime("%Y%m%d") - except: + except ValueError: last_updated = None else: renderer = try_get( From 8385c8d047fe4fb97f738d4fc0c3b26d8872b57e Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 17 Nov 2021 15:00:06 +0000 Subject: [PATCH 09/10] playlist_result no longer adds kwarg values that contain None --- youtube_dl/extractor/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f5ad7ae06..c97d81b90 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -972,7 +972,8 @@ class InfoExtractor(object): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} - video_info.update(kwargs) + video_info.update((key, value) for key, value in kwargs.items() if value is not None) + if playlist_id: video_info['id'] = playlist_id if playlist_title: From 8d657f3efc6b36753cec07832bdd4fa9274145bb Mon Sep 17 00:00:00 2001 From: nose-gnome <71675376+nose-gnome@users.noreply.github.com> Date: Wed, 17 Nov 2021 15:12:04 +0000 Subject: [PATCH 10/10] made declared view_count & last_updated_text as None in-case they are missing in result. --- youtube_dl/extractor/youtube.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 30e79ccfa..2e5b978e0 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2773,6 +2773,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): data, lambda x: x['metadata']['channelMetadataRenderer'], dict) playlist_id = item_id title = description = None + view_count = None + last_updated = None if renderer: channel_title = renderer.get('title') or item_id tab_title = selected_tab.get('title')