pepepper/youtube-dl
1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl synced 2024-12-23 12:40:11 +09:00
Code Issues Projects Releases Wiki Activity

Fix OnDemandPagedList underflow on slice end

Browse Source 
Also migrate towards yt-dlp structure
This commit is contained in:
dirkf 2021-12-30 21:33:51 +00:00
parent 5014bd67c2
commit 3efdb2758d
1 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
youtube_dl/utils.py
View File

@ -3810,6 +3810,12 @@ class PagedList(object):
# This is only useful for tests # This is only useful for tests
return len(self.getslice()) return len(self.getslice())
def _getslice(self, start, end):
raise NotImplementedError('This method must be implemented by subclasses')
def getslice(self, start=0, end=None):
return list(self._getslice(start, end))
class OnDemandPagedList(PagedList): class OnDemandPagedList(PagedList):
def __init__(self, pagefunc, pagesize, use_cache=True): def __init__(self, pagefunc, pagesize, use_cache=True):
@ -3819,11 +3825,12 @@ class OnDemandPagedList(PagedList):
if use_cache: if use_cache:
self._cache = {} self._cache = {}
def getslice(self, start=0, end=None): def _getslice(self, start=0, end=None):
res = [] firstpage = start // self._pagesize
for pagenum in itertools.count(start // self._pagesize): nextfirstid = firstpage * self._pagesize
firstid = pagenum * self._pagesize for pagenum in itertools.count(firstpage):
nextfirstid = pagenum * self._pagesize + self._pagesize firstid = nextfirstid
nextfirstid += self._pagesize
if start >= nextfirstid: if start >= nextfirstid:
continue continue
@ -3836,18 +3843,19 @@ class OnDemandPagedList(PagedList):
self._cache[pagenum] = page_results self._cache[pagenum] = page_results
startv = ( startv = (
start % self._pagesize start - firstid
if firstid <= start < nextfirstid if firstid <= start < nextfirstid
else 0) else 0)
endv = ( endv = (
((end - 1) % self._pagesize) + 1 end - firstid
if (end is not None and firstid <= end <= nextfirstid) if (end is not None and firstid <= end <= nextfirstid)
else None) else None)
if startv != 0 or endv is not None: if startv != 0 or endv is not None:
page_results = page_results[startv:endv] page_results = page_results[startv:endv]
res.extend(page_results) for item in page_results:
yield item
# A little optimization - if current page is not "full", ie. does # A little optimization - if current page is not "full", ie. does
# not contain page_size videos then we can assume that this page # not contain page_size videos then we can assume that this page
@ -3860,7 +3868,7 @@ class OnDemandPagedList(PagedList):
# break out early as well # break out early as well
if end == nextfirstid: if end == nextfirstid:
break break
return res return
class InAdvancePagedList(PagedList): class InAdvancePagedList(PagedList):