From be540fe77d00f22cf6a4f0f6865a109cf8011bdc Mon Sep 17 00:00:00 2001 From: garret Date: Sun, 11 Aug 2024 11:27:57 +0100 Subject: persons: only check the extractor arg once --- yt_dlp_plugins/extractor/radiko.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 7d06d37..618e1cb 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -719,10 +719,10 @@ class RadikoPersonIE(InfoExtractor): person_api = self._download_json(person_api_url, person_id) def entries(): + key_station_only = len(self._configuration_arg("key_station_only", ie_key="rajiko")) > 0 for episode in person_api.get("data"): - if len(self._configuration_arg("key_station_only", ie_key="rajiko")) > 0: - if episode.get("key_station_id") != episode.get("station_id"): - continue + if key_station_only and episode.get("key_station_id") != episode.get("station_id"): + continue share_url = traverse_obj(episode, ("radiko_url", ("pc", "sp", "android", "ios", "app"), {url_or_none}), get_all=False) -- cgit v1.2.3-70-g09d2 From 032c7951143e6452aeda34863bb38fed7ad26d3e Mon Sep 17 00:00:00 2001 From: garret Date: Sun, 11 Aug 2024 11:29:02 +0100 Subject: fix RadikoSiteTime test (forgot) changed func in eb94441420845970d48fe7b4e30cc5eed78ac95a , forgot to change the test also --- yt_dlp_plugins/extractor/radiko_time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_time.py b/yt_dlp_plugins/extractor/radiko_time.py index 9e77008..4d51ce5 100755 --- a/yt_dlp_plugins/extractor/radiko_time.py +++ b/yt_dlp_plugins/extractor/radiko_time.py @@ -79,7 +79,7 @@ if __name__ == "__main__": # cursed (no seconds) - seems to do -1s assert RadikoSiteTime('202308240100').timestring() == "20230824005959" # broadcast day starts at 05:00, ends at 04:59 (29:59) - assert RadikoSiteTime('20230824030000').broadcast_day() == '20230823' + assert RadikoSiteTime('20230824030000').broadcast_day_string() == '20230823' assert RadikoSiteTime('20230824130000').broadcast_day_end() == datetime.datetime(2023, 8, 25, 5, 0, 0, tzinfo=JST) assert RadikoSiteTime('20230824030000').broadcast_day_end() == datetime.datetime(2023, 8, 24, 5, 0, 0, tzinfo=JST) # checking timezone -- cgit v1.2.3-70-g09d2 From bd51e96ddabc48de64ac91970d4bdf2f576b0509 Mon Sep 17 00:00:00 2001 From: garret Date: Sun, 11 Aug 2024 11:58:31 +0100 Subject: get rid of unnecessary "else"s (+ fix indentation) --- yt_dlp_plugins/extractor/radiko.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 618e1cb..fa5a15c 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -230,9 +230,9 @@ class _RadikoBaseIE(InfoExtractor): "meta": meta }) return meta - else: - self.to_screen(f"{station_id}: Using cached station metadata") - return cachedata.get("meta") + + self.to_screen(f"{station_id}: Using cached station metadata") + return cachedata.get("meta") def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None): device = self._configuration_arg('device', ['aSmartPhone7a'], casesense=True, ie_key="rajiko")[0] # aSmartPhone7a formats = always happy path @@ -274,13 +274,13 @@ class _RadikoBaseIE(InfoExtractor): entry_protocol = 'm3u8' if domain in self._DOESNT_WORK_WITH_FFMPEG: - self.write_debug(f"skipping {domain} (known not working)") - continue - elif domain in self._DELIVERED_ONDEMAND: - # override the defaults for delivered as on-demand - delivered_live = False - preference = 1 - entry_protocol = None + self.write_debug(f"skipping {domain} (known not working)") + continue + if domain in self._DELIVERED_ONDEMAND: + # override the defaults for delivered as on-demand + delivered_live = False + preference = 1 + entry_protocol = None formats += self._extract_m3u8_formats( playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data, -- cgit v1.2.3-70-g09d2 From 2fda4016aa082da057d08627c10f3ec49475c8b9 Mon Sep 17 00:00:00 2001 From: garret Date: Sun, 11 Aug 2024 12:08:39 +0100 Subject: update timefree tests --- yt_dlp_plugins/extractor/radiko.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index fa5a15c..6c27973 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -375,22 +375,22 @@ class RadikoLiveIE(_RadikoBaseIE): class RadikoTimeFreeIE(_RadikoBaseIE): _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/ts/(?P[A-Z0-9-_]+)/(?P\d+)" _TESTS = [{ - "url": "https://radiko.jp/#!/ts/INT/20240802230000", + "url": "https://radiko.jp/#!/ts/INT/20240809230000", "info_dict": { "live_status": "was_live", "ext": "m4a", - "id": "INT-20240802230000", + "id": "INT-20240809230000", "title": "TOKYO MOON", "series": "Tokyo Moon", "description": "md5:20e68d2f400a391fa34d4e7c8c702cb8", - "chapters": "count:15", + "chapters": "count:14", "thumbnail": "https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg", - "upload_date": "20240802", - "timestamp": 1722607200.0, - "release_date": "20240802", - "release_timestamp": 1722610800.0, + "upload_date": "20240809", + "timestamp": 1723212000.0, + "release_date": "20240809", + "release_timestamp": 1723215600.0, "duration": 3600, "channel": "interfm", @@ -405,22 +405,22 @@ class RadikoTimeFreeIE(_RadikoBaseIE): }, }, { # late-night/early-morning show to test broadcast day checking - "url": "https://radiko.jp/#!/ts/TBS/20240803033000", + "url": "https://radiko.jp/#!/ts/TBS/20240810033000", "info_dict": { "live_status": "was_live", "ext": "m4a", - "id": "TBS-20240803033000", + "id": "TBS-20240810033000", "title": "CITY CHILL CLUB", "series": "CITY CHILL CLUB", "description": "md5:3fba2c1125059bed27247c0be90e58fa", - "chapters": "count:24", + "chapters": "count:22", "thumbnail": "https://program-static.cf.radiko.jp/ku7t4ztnaq.jpg", - "upload_date": "20240802", - "timestamp": 1722623400.0, - "release_date": "20240802", - "release_timestamp": 1722628800.0, + "upload_date": "20240809", + "timestamp": 1723228200.0, + "release_date": "20240809", + "release_timestamp": 1723233600.0, "duration": 5400, "channel": "TBSラジオ", -- cgit v1.2.3-70-g09d2 From 9fb1d45638e2e2b792fa4861c2d33209a69c9706 Mon Sep 17 00:00:00 2001 From: garret Date: Mon, 26 Aug 2024 22:06:48 +0100 Subject: remove self._region and self._user variables region wasnt used at all we can make the token stuff into a dict and include the user as well ...which is what i was already doing in the cache one day i want to change the cache format to use better var names but i will deal with that later --- yt_dlp_plugins/extractor/radiko.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 6c27973..f9d9dd4 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -84,9 +84,6 @@ class _RadikoBaseIE(InfoExtractor): _DELIVERED_ONDEMAND = ('radiko.jp',) _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp') - _region = None - _user = None - def _index_regions(self): region_data = {} @@ -175,30 +172,28 @@ class _RadikoBaseIE(InfoExtractor): self.report_warning(auth2.strip()) self.report_warning(auth2_headers) - token = { - "X-Radiko-AreaId": actual_region, - "X-Radiko-AuthToken": auth_token, + auth_data = { + "token": { + "X-Radiko-AreaId": actual_region, + "X-Radiko-AuthToken": auth_token, + }, + "user": auth2_headers["X-Radiko-User"], } - self._user = auth2_headers["X-Radiko-User"] if not region_mismatch: - self.cache.store("rajiko", station_region, { - "token": token, - "user": self._user, - }) - return token + self.cache.store("rajiko", station_region, auth_data) + return auth_data def _auth(self, station_region): cachedata = self.cache.load("rajiko", station_region) self.write_debug(cachedata) if cachedata is not None: - token = cachedata.get("token") - self._user = cachedata.get("user") + auth_headers = cachedata.get("token") response = self._download_webpage("https://radiko.jp/v2/api/auth_check", station_region, "Checking cached token", - headers=token, expected_status=401) + headers=auth_headers, expected_status=401) self.write_debug(response) if response == "OK": - return token + return cachedata return self._negotiate_token(station_region) def _get_station_meta(self, region, station_id): @@ -253,7 +248,7 @@ class _RadikoBaseIE(InfoExtractor): playlist_url = update_url_query(url, { "station_id": station, "l": "15", # l = length, ie how many seconds in the live m3u8 (max 300) - "lsid": self._user, + "lsid": auth_data["user"], "type": "b", # it is a mystery }) @@ -283,7 +278,7 @@ class _RadikoBaseIE(InfoExtractor): entry_protocol = None formats += self._extract_m3u8_formats( - playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data, + playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data["token"], live=delivered_live, preference=preference, entry_protocol=entry_protocol, note=f"Downloading m3u8 information from {domain}") return formats -- cgit v1.2.3-70-g09d2 From b1d5e613cda58eb4503abda367c2770588ad7fe9 Mon Sep 17 00:00:00 2001 From: garret Date: Tue, 17 Sep 2024 01:12:59 +0100 Subject: personsIE: fix API usage that excluded programmes aired today api ignores end_at_lt, it just sets start_at to the start of the broadcast day the idea was to only get stuff that's actually finished, so only stuff where the end is before now with start_at it'll get ongoing stuff as well, but that's better than not getting the day at all lol --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index f9d9dd4..caaa020 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -709,7 +709,7 @@ class RadikoPersonIE(InfoExtractor): person_api_url = update_url_query("https://api.radiko.jp/program/api/v1/programs", { "person_id": person_id, "start_at_gte": min_start.isoformat(), - "end_at_lt": now.isoformat(), + "start_at_lt": now.isoformat(), }) person_api = self._download_json(person_api_url, person_id) -- cgit v1.2.3-70-g09d2 From e9655881313b9b7ca1e402a82f51bbeed8a81e04 Mon Sep 17 00:00:00 2001 From: garret Date: Tue, 17 Sep 2024 01:18:09 +0100 Subject: prepare for "timefree 30" https://prtimes.jp/main/html/rd/p/000000032.000007490.html "radiko to launch new service "Timefree 30" this autumn, allows listening to programmes from the past 30 days, no 3-hour time limit" probably can't spoof having the plan, that's fine it might work if you pass cookies of a timefree30 account though, so i'm adapting the time stuff to account for that the plan doesn't exist yet, and i don't know how i would go about detecting it yet, so i'm just hardcoding to False for now --- yt_dlp_plugins/extractor/radiko.py | 4 ++-- yt_dlp_plugins/extractor/radiko_time.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index caaa020..d12f203 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -493,7 +493,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): end = times[1] now = datetime.datetime.now(tz=rtime.JST) - if end.broadcast_day_end() < now - datetime.timedelta(days=7): + if end.expiry(False) < now: self.raise_no_formats("Programme is no longer available.", video_id=meta["id"], expected=True) elif start > now: self.raise_no_formats("Programme has not aired yet.", video_id=meta["id"], expected=True) @@ -701,7 +701,7 @@ class RadikoPersonIE(InfoExtractor): now = rtime.RadikoTime.now(tz=rtime.JST) - min_start = (now - datetime.timedelta(days=7)).broadcast_day_start() + min_start = rtime.earliest_available(False) # we set the earliest time as the earliest we can get, # so, the start of the broadcast day 1 week ago # that way we can get everything we can actually download, including stuff that aired at eg "26:00" diff --git a/yt_dlp_plugins/extractor/radiko_time.py b/yt_dlp_plugins/extractor/radiko_time.py index 4d51ce5..b383098 100755 --- a/yt_dlp_plugins/extractor/radiko_time.py +++ b/yt_dlp_plugins/extractor/radiko_time.py @@ -36,6 +36,17 @@ class RadikoTime(datetime.datetime): dt = datetime.datetime(date.year, date.month, date.day, 5, 0, 0, tzinfo=JST) return dt + def expiry(self, tf30): + available_days = 30 if tf30 else 7 + return self.broadcast_day_end() + datetime.timedelta(days=available_days) + # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↓↓earliest_available↓↓ AS WELL! + + +def earliest_available(tf30): + available_days = 30 if tf30 else 7 + return (RadikoTime.now(tz=JST) - datetime.timedelta(days=available_days)).broadcast_day_start() + # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↑↑expiry↑↑ AS WELL! + class RadikoSiteTime(RadikoTime): -- cgit v1.2.3-70-g09d2 From 8805fd0326aff58332c626107625e80a597b6e45 Mon Sep 17 00:00:00 2001 From: garret Date: Fri, 1 Nov 2024 09:10:15 +0000 Subject: more right-thinking way of handling timefree 30 still hardcoded to not work though --- yt_dlp_plugins/extractor/radiko.py | 12 ++++++++---- yt_dlp_plugins/extractor/radiko_time.py | 13 ++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index d12f203..6b8a4c3 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -492,9 +492,13 @@ class RadikoTimeFreeIE(_RadikoBaseIE): start = times[0] end = times[1] now = datetime.datetime.now(tz=rtime.JST) + expiry_free, expiry_tf30 = end.expiry() + have_tf30 = False - if end.expiry(False) < now: + if expiry_tf30 < now: self.raise_no_formats("Programme is no longer available.", video_id=meta["id"], expected=True) + elif not have_tf30 and expiry_free < now: + self.raise_login_required("Programme is only available with a Timefree 30 subscription") elif start > now: self.raise_no_formats("Programme has not aired yet.", video_id=meta["id"], expected=True) live_status = "is_upcoming" @@ -701,9 +705,9 @@ class RadikoPersonIE(InfoExtractor): now = rtime.RadikoTime.now(tz=rtime.JST) - min_start = rtime.earliest_available(False) - # we set the earliest time as the earliest we can get, - # so, the start of the broadcast day 1 week ago + min_start = now - datetime.timedelta(days=30).broadcast_day_start() + # we set the earliest time as the earliest we can get (or at least, that it's possible to get), + # so, the start of the broadcast day 30 days ago # that way we can get everything we can actually download, including stuff that aired at eg "26:00" person_api_url = update_url_query("https://api.radiko.jp/program/api/v1/programs", { diff --git a/yt_dlp_plugins/extractor/radiko_time.py b/yt_dlp_plugins/extractor/radiko_time.py index b383098..d2084aa 100755 --- a/yt_dlp_plugins/extractor/radiko_time.py +++ b/yt_dlp_plugins/extractor/radiko_time.py @@ -36,16 +36,11 @@ class RadikoTime(datetime.datetime): dt = datetime.datetime(date.year, date.month, date.day, 5, 0, 0, tzinfo=JST) return dt - def expiry(self, tf30): - available_days = 30 if tf30 else 7 - return self.broadcast_day_end() + datetime.timedelta(days=available_days) - # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↓↓earliest_available↓↓ AS WELL! + def expiry(self): + free = self.broadcast_day_end() + datetime.timedelta(days=7) + tf30 = self.broadcast_day_end() + datetime.timedelta(days=30) - -def earliest_available(tf30): - available_days = 30 if tf30 else 7 - return (RadikoTime.now(tz=JST) - datetime.timedelta(days=available_days)).broadcast_day_start() - # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↑↑expiry↑↑ AS WELL! + return free, tf30 class RadikoSiteTime(RadikoTime): -- cgit v1.2.3-70-g09d2 From dc2c2e2413ce4c698dae3141f94a9fdd3f9207c0 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 22 Nov 2024 11:45:41 +0000 Subject: fix broken time calculation in PersonsIE the .broadcast_day_start() was trying to run on the datetime.timedelta, not the RadikoTime that we calculate so it just ERROR: 'datetime.timedelta' object has no attribute 'broadcast_day_start' --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 6b8a4c3..c6cea37 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -705,7 +705,7 @@ class RadikoPersonIE(InfoExtractor): now = rtime.RadikoTime.now(tz=rtime.JST) - min_start = now - datetime.timedelta(days=30).broadcast_day_start() + min_start = (now - datetime.timedelta(days=30)).broadcast_day_start() # we set the earliest time as the earliest we can get (or at least, that it's possible to get), # so, the start of the broadcast day 30 days ago # that way we can get everything we can actually download, including stuff that aired at eg "26:00" -- cgit v1.2.3-70-g09d2 From d84e4dbe4f4b1be71e17072dc2a2f3685b9a2997 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 22 Nov 2024 11:40:28 +0000 Subject: support logging in with accounts for timefree30 --- yt_dlp_plugins/extractor/radiko.py | 44 +++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index c6cea37..dc51dd9 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -7,12 +7,14 @@ import pkgutil from yt_dlp.extractor.common import InfoExtractor from yt_dlp.utils import ( + ExtractorError, OnDemandPagedList, clean_html, int_or_none, join_nonempty, parse_qs, traverse_obj, + urlencode_postdata, url_or_none, update_url_query, ) @@ -229,8 +231,16 @@ class _RadikoBaseIE(InfoExtractor): self.to_screen(f"{station_id}: Using cached station metadata") return cachedata.get("meta") - def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None): - device = self._configuration_arg('device', ['aSmartPhone7a'], casesense=True, ie_key="rajiko")[0] # aSmartPhone7a formats = always happy path + def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30_override=False): + config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0) + + if not tf30_override: + device = config_device or "aSmartPhone7a" # the only one that works for timefree with this is the on-demand one + # that's good imo - we just get the one that works, and don't bother with probing the rest as well + else: + device = config_device or "pc_html5" # the on-demand one doesnt work with timefree30 stuff sadly + # so just use pc_html5 which has everything + url_data = self._download_xml(f"https://radiko.jp/v3/station/stream/{device}/{station}.xml", station, note=f"Downloading {device} stream information") @@ -368,6 +378,7 @@ class RadikoLiveIE(_RadikoBaseIE): class RadikoTimeFreeIE(_RadikoBaseIE): + _NETRC_MACHINE = "rajiko" _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/ts/(?P[A-Z0-9-_]+)/(?P\d+)" _TESTS = [{ "url": "https://radiko.jp/#!/ts/INT/20240809230000", @@ -430,6 +441,29 @@ class RadikoTimeFreeIE(_RadikoBaseIE): }, }] + _has_tf30 = None + + def _perform_login(self, username, password): + try: + login_info = self._download_json('https://radiko.jp/ap/member/webapi/member/login', None, note='Logging in', + data=urlencode_postdata({'mail': username, 'pass': password})) + self._has_tf30 = '2' in login_info.get('privileges') + # areafree = 1, timefree30 = 2, double plan = both + except ExtractorError as error: + if isinstance(error.cause, HTTPError) and error.cause.status == 401: + raise ExtractorError('Invalid username and/or password', expected=True) + raise + + def _check_tf30(self): + if self._has_tf30 is not None: + return self._has_tf30 + if self._get_cookies('https://radiko.jp').get('radiko_session') is None: + return + account_info = self._download_json('https://radiko.jp/ap/member/webapi/v2/member/login/check', + None, note='Checking account status from cookies', expected_status=400) + self._has_tf30 = account_info.get('timefreeplus') == '1' + return self._has_tf30 + def _get_programme_meta(self, station_id, url_time): day = url_time.broadcast_day_string() meta = self._download_json(f"https://radiko.jp/v4/program/station/date/{day}/{station_id}.json", station_id, @@ -493,11 +527,11 @@ class RadikoTimeFreeIE(_RadikoBaseIE): end = times[1] now = datetime.datetime.now(tz=rtime.JST) expiry_free, expiry_tf30 = end.expiry() - have_tf30 = False if expiry_tf30 < now: self.raise_no_formats("Programme is no longer available.", video_id=meta["id"], expected=True) - elif not have_tf30 and expiry_free < now: + needs_tf30 = expiry_free < now + if needs_tf30 and not self._check_tf30(): self.raise_login_required("Programme is only available with a Timefree 30 subscription") elif start > now: self.raise_no_formats("Programme has not aired yet.", video_id=meta["id"], expected=True) @@ -510,7 +544,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): station_meta = self._get_station_meta(region, station) chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) auth_data = self._auth(region) - formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end) + formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30_override=needs_tf30) return { **station_meta, -- cgit v1.2.3-70-g09d2 From 6661c60ece44309a6a28802d3d55f76d7c332148 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 22 Nov 2024 21:59:45 +0000 Subject: Force new token if cached one doesn't have tf30 --- yt_dlp_plugins/extractor/radiko.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index dc51dd9..93f0c93 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -86,6 +86,8 @@ class _RadikoBaseIE(InfoExtractor): _DELIVERED_ONDEMAND = ('radiko.jp',) _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp') + _has_tf30 = None + def _index_regions(self): region_data = {} @@ -180,16 +182,20 @@ class _RadikoBaseIE(InfoExtractor): "X-Radiko-AuthToken": auth_token, }, "user": auth2_headers["X-Radiko-User"], + "tf30": self._has_tf30, } if not region_mismatch: self.cache.store("rajiko", station_region, auth_data) return auth_data - def _auth(self, station_region): + def _auth(self, station_region, tf30=False): cachedata = self.cache.load("rajiko", station_region) self.write_debug(cachedata) if cachedata is not None: + if tf30 and not cachedata.get("tf30"): + return self._negotiate_token(station_region, tf30=True) + auth_headers = cachedata.get("token") response = self._download_webpage("https://radiko.jp/v2/api/auth_check", station_region, "Checking cached token", headers=auth_headers, expected_status=401) @@ -231,10 +237,10 @@ class _RadikoBaseIE(InfoExtractor): self.to_screen(f"{station_id}: Using cached station metadata") return cachedata.get("meta") - def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30_override=False): + def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30=False): config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0) - if not tf30_override: + if not tf30: device = config_device or "aSmartPhone7a" # the only one that works for timefree with this is the on-demand one # that's good imo - we just get the one that works, and don't bother with probing the rest as well else: @@ -441,8 +447,6 @@ class RadikoTimeFreeIE(_RadikoBaseIE): }, }] - _has_tf30 = None - def _perform_login(self, username, password): try: login_info = self._download_json('https://radiko.jp/ap/member/webapi/member/login', None, note='Logging in', @@ -543,8 +547,8 @@ class RadikoTimeFreeIE(_RadikoBaseIE): region = self._get_station_region(station) station_meta = self._get_station_meta(region, station) chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) - auth_data = self._auth(region) - formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30_override=needs_tf30) + auth_data = self._auth(region, tf30=needs_tf30) + formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30=needs_tf30) return { **station_meta, -- cgit v1.2.3-70-g09d2 From 8c2f5d394d28a683e8f3e6ddf233be84f1ae73c3 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 22 Nov 2024 22:16:19 +0000 Subject: add debug logs for account sign-in process --- yt_dlp_plugins/extractor/radiko.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 93f0c93..3996c60 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -194,6 +194,7 @@ class _RadikoBaseIE(InfoExtractor): self.write_debug(cachedata) if cachedata is not None: if tf30 and not cachedata.get("tf30"): + self.write_debug("Cached token doesn't have timefree 30, getting a new one") return self._negotiate_token(station_region, tf30=True) auth_headers = cachedata.get("token") @@ -453,6 +454,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): data=urlencode_postdata({'mail': username, 'pass': password})) self._has_tf30 = '2' in login_info.get('privileges') # areafree = 1, timefree30 = 2, double plan = both + self.write_debug({**login_info, "radiko_session": "PRIVATE", "member_ukey": "PRIVATE"}) except ExtractorError as error: if isinstance(error.cause, HTTPError) and error.cause.status == 401: raise ExtractorError('Invalid username and/or password', expected=True) @@ -465,6 +467,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): return account_info = self._download_json('https://radiko.jp/ap/member/webapi/v2/member/login/check', None, note='Checking account status from cookies', expected_status=400) + self.write_debug({**account_info, "user_key": "PRIVATE"}) self._has_tf30 = account_info.get('timefreeplus') == '1' return self._has_tf30 -- cgit v1.2.3-70-g09d2 From 1b7830f9e2bf97eaeaee1e6a3936e4361481e961 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 22 Nov 2024 22:45:07 +0000 Subject: clearer variable names/comments + fix getting tf30 token --- yt_dlp_plugins/extractor/radiko.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 3996c60..9c70a1b 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -182,20 +182,20 @@ class _RadikoBaseIE(InfoExtractor): "X-Radiko-AuthToken": auth_token, }, "user": auth2_headers["X-Radiko-User"], - "tf30": self._has_tf30, + "has_tf30": self._has_tf30, } if not region_mismatch: self.cache.store("rajiko", station_region, auth_data) return auth_data - def _auth(self, station_region, tf30=False): + def _auth(self, station_region, need_tf30=False): cachedata = self.cache.load("rajiko", station_region) self.write_debug(cachedata) if cachedata is not None: - if tf30 and not cachedata.get("tf30"): + if need_tf30 and not cachedata.get("has_tf30"): self.write_debug("Cached token doesn't have timefree 30, getting a new one") - return self._negotiate_token(station_region, tf30=True) + return self._negotiate_token(station_region) auth_headers = cachedata.get("token") response = self._download_webpage("https://radiko.jp/v2/api/auth_check", station_region, "Checking cached token", @@ -238,11 +238,11 @@ class _RadikoBaseIE(InfoExtractor): self.to_screen(f"{station_id}: Using cached station metadata") return cachedata.get("meta") - def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30=False): + def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30_override=False): config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0) - if not tf30: - device = config_device or "aSmartPhone7a" # the only one that works for timefree with this is the on-demand one + if not tf30_override: + device = config_device or "aSmartPhone7a" # this device only gives us the on-demand one for timefree # that's good imo - we just get the one that works, and don't bother with probing the rest as well else: device = config_device or "pc_html5" # the on-demand one doesnt work with timefree30 stuff sadly @@ -537,8 +537,8 @@ class RadikoTimeFreeIE(_RadikoBaseIE): if expiry_tf30 < now: self.raise_no_formats("Programme is no longer available.", video_id=meta["id"], expected=True) - needs_tf30 = expiry_free < now - if needs_tf30 and not self._check_tf30(): + need_tf30 = expiry_free < now + if need_tf30 and not self._check_tf30(): self.raise_login_required("Programme is only available with a Timefree 30 subscription") elif start > now: self.raise_no_formats("Programme has not aired yet.", video_id=meta["id"], expected=True) @@ -550,8 +550,8 @@ class RadikoTimeFreeIE(_RadikoBaseIE): region = self._get_station_region(station) station_meta = self._get_station_meta(region, station) chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) - auth_data = self._auth(region, tf30=needs_tf30) - formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30=needs_tf30) + auth_data = self._auth(region, need_tf30=need_tf30) + formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30_override=need_tf30) return { **station_meta, -- cgit v1.2.3-70-g09d2 From 5b069ee39394d927fa553902cd9767a7d5b1bfc2 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 21 Jan 2025 17:53:06 +0000 Subject: missed an import --- yt_dlp_plugins/extractor/radiko.py | 1 + 1 file changed, 1 insertion(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 9c70a1b..3839aa9 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -6,6 +6,7 @@ import urllib.parse import pkgutil from yt_dlp.extractor.common import InfoExtractor +from yt_dlp.networking.exceptions import HTTPError from yt_dlp.utils import ( ExtractorError, OnDemandPagedList, -- cgit v1.2.3-70-g09d2 From d7aaaaf00397634eeaf7dec7b91d57d1b64bd30a Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 6 Mar 2025 07:31:41 +0000 Subject: filter out "cul_area_id" in search params - HTTP Error 415 site used to use "cul_area_id" in the search url, now it uses "cur_area_id" (with an r) and it outright rejects the old one with HTTP Error 415: Unsupported Media Type probably better to patch in the extractor than to break peoples scripts etc - site does this too idk why they felt the need to break the old one --- yt_dlp_plugins/extractor/radiko.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 3839aa9..9ca353b 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -609,6 +609,11 @@ class RadikoSearchIE(InfoExtractor): # urllib.parse interprets the path as just one giant fragment because of the #, so we hack it away queries = parse_qs(url) + if queries.get("cul_area_id"): + queries["cur_area_id"] = queries.pop("cul_area_id") + # site used to use "cul_area_id" in the search url, now it uses "cur_area_id" (with an r) + # and outright rejects the old one with HTTP Error 415: Unsupported Media Type + search_url = update_url_query("https://radiko.jp/v3/api/program/search", { **queries, "uid": "".join(random.choices("0123456789abcdef", k=32)), -- cgit v1.2.3-70-g09d2 From 23effbcd8c423018b2b104125509806b36ee0985 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 6 Mar 2025 07:33:54 +0000 Subject: switch to different search API url observed on site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit no change in the response structure*, just the URL afaict (stuff doesnt break at least) interestingly the headers have started including GRPC stuff though *but the search _results_ seem to have changed somewhat had to change my search for ジャズ倶楽部 to specifically ジャズ倶楽部 CRT because loads of results had "ジャズ・クラブ" - not katakana クラブ not kanji 倶楽部 maybe its not changed and i just got unlucky lol --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 9ca353b..a06b1e4 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -614,7 +614,7 @@ class RadikoSearchIE(InfoExtractor): # site used to use "cul_area_id" in the search url, now it uses "cur_area_id" (with an r) # and outright rejects the old one with HTTP Error 415: Unsupported Media Type - search_url = update_url_query("https://radiko.jp/v3/api/program/search", { + search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/programs/legacy/perl/program/search", { **queries, "uid": "".join(random.choices("0123456789abcdef", k=32)), "app_id": "pc", -- cgit v1.2.3-70-g09d2 From 1083eada23729aa799826f517c282f36db6e9d3c Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 27 Mar 2025 20:08:55 +0000 Subject: add new alliance-stream-radiko.smartstream.ne.jp format to ffmpeg blacklist github issue #25 --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index a06b1e4..3f28f6b 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -85,7 +85,7 @@ class _RadikoBaseIE(InfoExtractor): _APP_VERSIONS = ["7.5.0", "7.4.17", "7.4.16", "7.4.15", "7.4.14", "7.4.13", "7.4.12", "7.4.11", "7.4.10", "7.4.9", "7.4.8", "7.4.7", "7.4.6", "7.4.5", "7.4.4", "7.4.3", "7.4.2", "7.4.1", "7.4.0", "7.3.8", "7.3.7", "7.3.6", "7.3.1", "7.3.0", "7.2.11", "7.2.10"] _DELIVERED_ONDEMAND = ('radiko.jp',) - _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp') + _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp', 'alliance-stream-radiko.smartstream.ne.jp') _has_tf30 = None -- cgit v1.2.3-70-g09d2 From fff631e63dfaa8bf3ec33c3fe4b5d04031b508bf Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 30 Mar 2025 16:20:53 +0100 Subject: add extractor arg to disable the blacklist --- yt_dlp_plugins/extractor/radiko.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 3f28f6b..9a74f34 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -256,6 +256,7 @@ class _RadikoBaseIE(InfoExtractor): formats = [] timefree_int = 1 if timefree else 0 + do_blacklist_streams = not len(self._configuration_arg("no_stream_blacklist", ie_key="rajiko")) > 0 for element in url_data.findall(f".//url[@timefree='{timefree_int}'][@areafree='0']/playlist_create_url"): # find s with matching timefree and no areafree, then get their url = element.text @@ -286,7 +287,8 @@ class _RadikoBaseIE(InfoExtractor): preference = -1 entry_protocol = 'm3u8' - if domain in self._DOESNT_WORK_WITH_FFMPEG: + + if domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams: self.write_debug(f"skipping {domain} (known not working)") continue if domain in self._DELIVERED_ONDEMAND: -- cgit v1.2.3-70-g09d2 From b879d27a7b4fa15ecb41483c25f762eb5d4cb6fa Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 17 May 2025 19:29:02 +0100 Subject: update "type" comment with new information --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 9a74f34..c9d2756 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -268,7 +268,7 @@ class _RadikoBaseIE(InfoExtractor): "station_id": station, "l": "15", # l = length, ie how many seconds in the live m3u8 (max 300) "lsid": auth_data["user"], - "type": "b", # it is a mystery + "type": "b", # a/b = in-region, c = areafree }) if timefree: -- cgit v1.2.3-70-g09d2 From c028c876287e7dbbd3fe7e1350f36c03d84b40f5 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 17 May 2025 22:20:20 +0100 Subject: add quick-downloading support for as-live formats closes #24 --- yt_dlp_plugins/extractor/radiko.py | 23 ++++++++++++++---- yt_dlp_plugins/extractor/radiko_hacks.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 yt_dlp_plugins/extractor/radiko_hacks.py (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index c9d2756..e371878 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -21,6 +21,7 @@ from yt_dlp.utils import ( ) import yt_dlp_plugins.extractor.radiko_time as rtime +import yt_dlp_plugins.extractor.radiko_hacks as hacks class _RadikoBaseIE(InfoExtractor): @@ -287,7 +288,6 @@ class _RadikoBaseIE(InfoExtractor): preference = -1 entry_protocol = 'm3u8' - if domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams: self.write_debug(f"skipping {domain} (known not working)") continue @@ -297,10 +297,23 @@ class _RadikoBaseIE(InfoExtractor): preference = 1 entry_protocol = None - formats += self._extract_m3u8_formats( - playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data["token"], - live=delivered_live, preference=preference, entry_protocol=entry_protocol, - note=f"Downloading m3u8 information from {domain}") + if delivered_live and timefree: + chunks = hacks._generate_as_live_chunks(playlist_url, start_at, end_at) + + formats.append({ + "url": playlist_url, # fallback to live for ffmpeg etc + "format_id": join_nonempty(domain, "chunked"), + "live": False, + "hls_media_playlist_data": hacks._playlist_from_chunks(self, chunks, domain, auth_data["token"]), + "preference": preference, + "ext": "m4a", + }) + else: + + formats += self._extract_m3u8_formats( + playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data["token"], + live=delivered_live, preference=preference, entry_protocol=entry_protocol, + note=f"Downloading m3u8 information from {domain}") return formats diff --git a/yt_dlp_plugins/extractor/radiko_hacks.py b/yt_dlp_plugins/extractor/radiko_hacks.py new file mode 100644 index 0000000..7b20654 --- /dev/null +++ b/yt_dlp_plugins/extractor/radiko_hacks.py @@ -0,0 +1,41 @@ +import datetime + +from yt_dlp.extractor.common import InfoExtractor +from yt_dlp.utils import ( + join_nonempty, + update_url_query, + traverse_obj, +) + +# "hacks" as in great jank/schizo shit that works anyway + +def _generate_as_live_chunks(playlist_url, start_at, end_at): + chunks = [] + chunk_length = 300 # max the api allows + + duration = int(end_at.timestamp() - start_at.timestamp()) + cursor = 0 + while cursor < duration: + chunk_length = min(chunk_length, duration - cursor) + chunk_start = start_at + datetime.timedelta(seconds=cursor) + chunk_url = update_url_query(playlist_url, { + "seek": chunk_start.timestring(), + "l": chunk_length, + }) + chunks.append(chunk_url) + cursor += chunk_length + + return chunks + +def _playlist_from_chunks(self, chunks, src_id, headers={}): + playlist = "" + for i, chunk in enumerate(chunks): + i +=1 # for more friendly cli output, it gets reset each loop so it shouldnt effect anything + chunk_id = join_nonempty(src_id, i) + base_format = self._extract_m3u8_formats( + chunk, chunk_id, fatal=False, headers=headers, + note=f"Preparing {src_id} chunk {i}" + ) + m3u8_url = traverse_obj(base_format, (..., "url",), get_all=False) + playlist += self._download_webpage(m3u8_url, chunk_id, note=f"Getting {src_id} chunk {i} fragments") + return playlist -- cgit v1.2.3-70-g09d2 From 3d9a1d81293797a68dd8b2b4a45ad31418180477 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 17 May 2025 22:39:44 +0100 Subject: add extractor arg to disable as-live workaround --- yt_dlp_plugins/extractor/radiko.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index e371878..b20f9bb 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -258,6 +258,7 @@ class _RadikoBaseIE(InfoExtractor): timefree_int = 1 if timefree else 0 do_blacklist_streams = not len(self._configuration_arg("no_stream_blacklist", ie_key="rajiko")) > 0 + do_as_live_chunks = not len(self._configuration_arg("no_as_live_chunks", ie_key="rajiko")) > 0 for element in url_data.findall(f".//url[@timefree='{timefree_int}'][@areafree='0']/playlist_create_url"): # find s with matching timefree and no areafree, then get their url = element.text @@ -297,7 +298,7 @@ class _RadikoBaseIE(InfoExtractor): preference = 1 entry_protocol = None - if delivered_live and timefree: + if delivered_live and timefree and do_as_live_chunks: chunks = hacks._generate_as_live_chunks(playlist_url, start_at, end_at) formats.append({ -- cgit v1.2.3-70-g09d2 From d56d025e635dfe83dd1897956e99349f8b125ead Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 18 May 2025 12:10:06 +0100 Subject: track real chunk duration to prevent dupes when chunks are longer than expected closes #24 for real this time --- yt_dlp_plugins/extractor/radiko.py | 7 ++-- yt_dlp_plugins/extractor/radiko_hacks.py | 56 +++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 18 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index b20f9bb..e8174c6 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -299,13 +299,16 @@ class _RadikoBaseIE(InfoExtractor): entry_protocol = None if delivered_live and timefree and do_as_live_chunks: - chunks = hacks._generate_as_live_chunks(playlist_url, start_at, end_at) + + chunks_playlist = hacks._generate_as_live_playlist( + self, playlist_url, start_at, end_at, domain, auth_data["token"] + ) formats.append({ "url": playlist_url, # fallback to live for ffmpeg etc "format_id": join_nonempty(domain, "chunked"), "live": False, - "hls_media_playlist_data": hacks._playlist_from_chunks(self, chunks, domain, auth_data["token"]), + "hls_media_playlist_data": chunks_playlist, "preference": preference, "ext": "m4a", }) diff --git a/yt_dlp_plugins/extractor/radiko_hacks.py b/yt_dlp_plugins/extractor/radiko_hacks.py index 7b20654..eb68de1 100644 --- a/yt_dlp_plugins/extractor/radiko_hacks.py +++ b/yt_dlp_plugins/extractor/radiko_hacks.py @@ -1,4 +1,5 @@ import datetime +import re from yt_dlp.extractor.common import InfoExtractor from yt_dlp.utils import ( @@ -9,33 +10,56 @@ from yt_dlp.utils import ( # "hacks" as in great jank/schizo shit that works anyway -def _generate_as_live_chunks(playlist_url, start_at, end_at): - chunks = [] +def _generate_as_live_playlist(self, playlist_url, start_at, end_at, domain, headers={}): + playlist = "" chunk_length = 300 # max the api allows duration = int(end_at.timestamp() - start_at.timestamp()) cursor = 0 + chunk_num = 1 while cursor < duration: chunk_length = min(chunk_length, duration - cursor) + chunk_start = start_at + datetime.timedelta(seconds=cursor) chunk_url = update_url_query(playlist_url, { "seek": chunk_start.timestring(), "l": chunk_length, }) - chunks.append(chunk_url) - cursor += chunk_length - return chunks + chunk_playlist, real_chunk_length = _get_chunk_playlist(self, chunk_url, domain, chunk_num, headers) + + playlist += chunk_playlist + cursor += real_chunk_length + chunk_num += 1 -def _playlist_from_chunks(self, chunks, src_id, headers={}): - playlist = "" - for i, chunk in enumerate(chunks): - i +=1 # for more friendly cli output, it gets reset each loop so it shouldnt effect anything - chunk_id = join_nonempty(src_id, i) - base_format = self._extract_m3u8_formats( - chunk, chunk_id, fatal=False, headers=headers, - note=f"Preparing {src_id} chunk {i}" - ) - m3u8_url = traverse_obj(base_format, (..., "url",), get_all=False) - playlist += self._download_webpage(m3u8_url, chunk_id, note=f"Getting {src_id} chunk {i} fragments") return playlist + +def _get_chunk_playlist(self, chunk_url, src_id, chunk_num, headers={}): + EXTINF_duration = re.compile("^#EXTINF:([\d.]+),", flags=re.MULTILINE) + + playlist = "" + chunk_id = join_nonempty(src_id, chunk_num) + base_format = self._extract_m3u8_formats( + chunk_url, chunk_id, fatal=False, headers=headers, + note=f"Preparing {src_id} chunk {chunk_num}" + ) + m3u8_url = traverse_obj(base_format, (..., "url",), get_all=False) + playlist = self._download_webpage(m3u8_url, chunk_id, note=f"Getting {src_id} chunk {chunk_num} fragments") + + real_duration = 0 + for i in EXTINF_duration.findall(playlist): + real_duration += float(i) + real_duration = round(real_duration) + + # playlists can sometimes be longer than they should + # wowza stream does some strange things + # it goes along fine with every fragment 5s long as normal + # and then during the ad break it does one with a different length (2s here) + # i assume so they have a clean split to do ad insertion in? idk + + # but anyway now the chunks aren't always a clean 5mins long + # and we get a repeated fragment going into the next chunk + + # so to work around this, we track the real duration from the #EXTINF tags + + return playlist, real_duration -- cgit v1.2.3-70-g09d2 From d2b9505aa7bf0128e1c2f6bc062f41afa149c41c Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 19 May 2025 02:05:46 +0100 Subject: cleanup m3u8 request headers into a variable --- yt_dlp_plugins/extractor/radiko.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index e8174c6..acad1c9 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -298,10 +298,12 @@ class _RadikoBaseIE(InfoExtractor): preference = 1 entry_protocol = None + auth_headers = auth_data["token"] + if delivered_live and timefree and do_as_live_chunks: chunks_playlist = hacks._generate_as_live_playlist( - self, playlist_url, start_at, end_at, domain, auth_data["token"] + self, playlist_url, start_at, end_at, domain, auth_headers ) formats.append({ @@ -315,7 +317,7 @@ class _RadikoBaseIE(InfoExtractor): else: formats += self._extract_m3u8_formats( - playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data["token"], + playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_headers, live=delivered_live, preference=preference, entry_protocol=entry_protocol, note=f"Downloading m3u8 information from {domain}") return formats -- cgit v1.2.3-70-g09d2 From 6822dcb17ae217d0edef74970f4dff3fd0982381 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 19 May 2025 02:06:31 +0100 Subject: chunked: properly fallback to live for ffmpeg etc --- yt_dlp_plugins/extractor/radiko.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index acad1c9..06c85fa 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -307,12 +307,14 @@ class _RadikoBaseIE(InfoExtractor): ) formats.append({ - "url": playlist_url, # fallback to live for ffmpeg etc "format_id": join_nonempty(domain, "chunked"), - "live": False, "hls_media_playlist_data": chunks_playlist, "preference": preference, "ext": "m4a", + + # fallback to live for ffmpeg etc + "url": playlist_url, + "http_headers": auth_headers, }) else: -- cgit v1.2.3-70-g09d2 From f857c544a0d1c56d3d5b7a05e18afd44a34ed17d Mon Sep 17 00:00:00 2001 From: garret1317 Date: Wed, 21 May 2025 16:15:54 +0100 Subject: make tracklist downloading non-fatal so MAJAL works MUSIC AWARDS JAPAN AUDIO LIVE pop-up stream --- yt_dlp_plugins/extractor/radiko.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 06c85fa..5f22451 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -532,10 +532,12 @@ class RadikoTimeFreeIE(_RadikoBaseIE): "start_time_gte": start.isoformat(), "end_time_lt": end.isoformat(), }) - data = self._download_json(api_url, video_id, note="Downloading tracklist").get("data") + data_json = self._download_json( + api_url, video_id, note="Downloading tracklist", errnote="Downloading tracklist", fatal=False + ) chapters = [] - for track in data: + for track in traverse_obj(data_json, "data") or []: artist = traverse_obj(track, ("artist", "name")) or track.get("artist_name") chapters.append({ "title": join_nonempty(artist, track.get("title"), delim=" - "), -- cgit v1.2.3-70-g09d2 From a659b48a20b6241a68fc046c86c9ad0f71c9bd86 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 24 May 2025 19:28:00 +0100 Subject: make SearchIE return an id in advance, for download archives to prevent un-needed extractions when somethings in the archive already absolutely kicking myself that it was this simple and i only just did it --- yt_dlp_plugins/extractor/radiko.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 5f22451..8f84bd7 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -619,15 +619,25 @@ class RadikoSearchIE(InfoExtractor): }] def _strip_date(self, date): + # lazy way of making a timestring (from eg 2025-05-20 01:00:00) return date.replace(" ", "").replace("-", "").replace(":", "") def _pagefunc(self, url, idx): url = update_url_query(url, {"page_idx": idx}) data = self._download_json(url, None, note=f"Downloading page {idx+1}") - return [self.url_result("https://radiko.jp/#!/ts/{station}/{time}".format( - station = i.get("station_id"), time = self._strip_date(i.get("start_time")))) - for i in data.get("data")] + results = [] + for r in data.get("data"): + station = r.get("station_id") + timestring = self._strip_date(r.get("start_time")) + + results.append( + self.url_result( + f"https://radiko.jp/#!/ts/{station}/{timestring}", + id=join_nonempty(station, timestring) + ) + ) + return results def _real_extract(self, url): url = url.replace("/#!/", "/!/", 1) -- cgit v1.2.3-70-g09d2 From 83ff35b140638c665547f55b1e4b876482ec3ce8 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 25 May 2025 12:16:13 +0100 Subject: make ShareIE return an id in advance as well in my excitement i may have prematurely released (har har) v1.5 with the return-id-in-advance trick only added to SearchIE oh well, it can be in ShareIE for next release adding to ShareIE has knock-on effects for PersonsIE as it returns a share url two birds with one stone --- yt_dlp_plugins/extractor/radiko.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 8f84bd7..fc09724 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -711,7 +711,10 @@ class RadikoShareIE(InfoExtractor): station = traverse_obj(queries, ("sid", 0)) time = traverse_obj(queries, ("t", 0)) time = rtime.RadikoShareTime(time).timestring() - return self.url_result(f"https://radiko.jp/#!/ts/{station}/{time}", RadikoTimeFreeIE) + return self.url_result( + f"https://radiko.jp/#!/ts/{station}/{time}", RadikoTimeFreeIE, + id=join_nonempty(station, time) + ) class RadikoStationButtonIE(InfoExtractor): -- cgit v1.2.3-70-g09d2 From 596c845c9d439e67d4a39faf7dc9f6e7b1d8d86e Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 25 May 2025 17:32:28 +0100 Subject: switch programme data api to api.radiko.jp as used on site identical content --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index fc09724..b9bb056 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -497,7 +497,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): def _get_programme_meta(self, station_id, url_time): day = url_time.broadcast_day_string() - meta = self._download_json(f"https://radiko.jp/v4/program/station/date/{day}/{station_id}.json", station_id, + meta = self._download_json(f"https://api.radiko.jp/program/v4/date/{day}/station/{station_id}.json", station_id, note="Downloading programme data") programmes = traverse_obj(meta, ("stations", lambda _, v: v["station_id"] == station_id, "programs", "program"), get_all=False) -- cgit v1.2.3-70-g09d2 From 543de39a20b8a4c19baf468e0a4946a4610d30de Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 30 May 2025 21:47:52 +0100 Subject: fix SyntaxWarning: invalid escape sequence '\d' in py3.12 --- yt_dlp_plugins/extractor/radiko_hacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_hacks.py b/yt_dlp_plugins/extractor/radiko_hacks.py index eb68de1..6486034 100644 --- a/yt_dlp_plugins/extractor/radiko_hacks.py +++ b/yt_dlp_plugins/extractor/radiko_hacks.py @@ -35,7 +35,7 @@ def _generate_as_live_playlist(self, playlist_url, start_at, end_at, domain, hea return playlist def _get_chunk_playlist(self, chunk_url, src_id, chunk_num, headers={}): - EXTINF_duration = re.compile("^#EXTINF:([\d.]+),", flags=re.MULTILINE) + EXTINF_duration = re.compile(r"^#EXTINF:([\d.]+),", flags=re.MULTILINE) playlist = "" chunk_id = join_nonempty(src_id, chunk_num) -- cgit v1.2.3-70-g09d2 From bc161b9a04a735ac43dc44bccbd6e4d5639ab2bd Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 1 Jun 2025 12:12:42 +0100 Subject: set device to pc_html5 for live as well aSmartPhone7a removed working stream --- yt_dlp_plugins/extractor/radiko.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index b9bb056..f58e744 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -240,10 +240,10 @@ class _RadikoBaseIE(InfoExtractor): self.to_screen(f"{station_id}: Using cached station metadata") return cachedata.get("meta") - def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, tf30_override=False): + def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, use_pc_html5=False): config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0) - if not tf30_override: + if not use_pc_html5: device = config_device or "aSmartPhone7a" # this device only gives us the on-demand one for timefree # that's good imo - we just get the one that works, and don't bother with probing the rest as well else: @@ -398,7 +398,7 @@ class RadikoLiveIE(_RadikoBaseIE): region = self._get_station_region(station) station_meta = self._get_station_meta(region, station) auth_data = self._auth(region) - formats = self._get_station_formats(station, False, auth_data) + formats = self._get_station_formats(station, False, auth_data, use_pc_html5=True) return { "is_live": True, @@ -577,7 +577,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): station_meta = self._get_station_meta(region, station) chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) auth_data = self._auth(region, need_tf30=need_tf30) - formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, tf30_override=need_tf30) + formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, use_pc_html5=need_tf30) return { **station_meta, -- cgit v1.2.3-70-g09d2 From b2a004a66e697bbb24af60ebc8221986fa519c19 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 6 Jun 2025 02:22:30 +0100 Subject: fix search only returning one result rather embarrassing bug introduced in a659b48a20b6241a68fc046c86c9ad0f71c9bd86 / v1.5 only caught it now because i only listen to weekly programmes so i only get 1 result anyway --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index f58e744..ca9ab6f 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -637,7 +637,7 @@ class RadikoSearchIE(InfoExtractor): id=join_nonempty(station, timestring) ) ) - return results + return results def _real_extract(self, url): url = url.replace("/#!/", "/!/", 1) -- cgit v1.2.3-70-g09d2 From 957b10b831015d55f376a34707b851f533146c03 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 9 Jun 2025 07:29:22 +0100 Subject: Extract station logo as thumbnail for live now that banner is gone well not technically _gone_, but replaced with a useless placeholder image --- yt_dlp_plugins/extractor/radiko.py | 39 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index ca9ab6f..c87b1b7 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -216,6 +216,17 @@ class _RadikoBaseIE(InfoExtractor): station = region.find(f'.//station/id[.="{station_id}"]/..') # a with an of our station_id station_name = station.find("name").text station_url = url_or_none(station.find("href").text) + + thumbnails = [] + for logo in station.findall("logo"): + thumbnails.append({ + "url": logo.text, + **traverse_obj(logo.attrib, ({ + "width": ("width", {int_or_none}), + "height": ("height", {int_or_none}), + })) + }) + meta = { "id": station_id, "title": station_name, @@ -229,7 +240,7 @@ class _RadikoBaseIE(InfoExtractor): "uploader_id": station_id, "uploader_url": station_url, - "thumbnail": url_or_none(station.find("banner").text), + "thumbnails": thumbnails, } self.cache.store("rajiko", station_id, { "expiry": (now + datetime.timedelta(days=1)).timestamp(), @@ -340,7 +351,7 @@ class RadikoLiveIE(_RadikoBaseIE): "id": "FMT", "title": "re:^TOKYO FM.+$", "alt_title": "TOKYO FM", - "thumbnail": "https://radiko.jp/res/banner/FMT/20220512162447.jpg", + "thumbnail": "https://radiko.jp/v2/static/station/logo/FMT/lrtrim/688x160.png", "channel": "TOKYO FM", "channel_id": "FMT", @@ -360,7 +371,7 @@ class RadikoLiveIE(_RadikoBaseIE): "id": "NORTHWAVE", "title": "re:^FM NORTH WAVE.+$", "alt_title": "FM NORTH WAVE", - "thumbnail": "https://radiko.jp/res/banner/NORTHWAVE/20150731161543.png", + "thumbnail": "https://radiko.jp/v2/static/station/logo/NORTHWAVE/lrtrim/688x160.png", "uploader": "FM NORTH WAVE", "uploader_url": "https://www.fmnorth.co.jp/", @@ -381,7 +392,7 @@ class RadikoLiveIE(_RadikoBaseIE): "id": "RN1", "title": "re:^ラジオNIKKEI第1.+$", "alt_title": "RADIONIKKEI", - "thumbnail": "https://radiko.jp/res/banner/RN1/20120802154152.png", + "thumbnail": "https://radiko.jp/v2/static/station/logo/RN1/lrtrim/688x160.png", "channel": "ラジオNIKKEI第1", "channel_url": "http://www.radionikkei.jp/", @@ -581,7 +592,9 @@ class RadikoTimeFreeIE(_RadikoBaseIE): return { **station_meta, - "alt_title": None, + "alt_title": None, # override from station metadata + "thumbnails": None, + **meta, "chapters": chapters, "formats": formats, @@ -727,19 +740,9 @@ class RadikoStationButtonIE(InfoExtractor): "info_dict": { "ext": "m4a", 'live_status': 'is_live', - "id": "QRR", - "title": "re:^文化放送.+$", - 'alt_title': 'JOQR BUNKA HOSO', - 'thumbnail': 'https://radiko.jp/res/banner/QRR/20240423144553.png', - 'channel': '文化放送', - 'channel_id': 'QRR', - 'channel_url': 'http://www.joqr.co.jp/', - 'uploader': '文化放送', - 'uploader_id': 'QRR', - 'uploader_url': 'http://www.joqr.co.jp/', - - } + }, + 'only_matching': True, }] _WEBPAGE_TESTS = [{ @@ -750,7 +753,7 @@ class RadikoStationButtonIE(InfoExtractor): 'id': 'CCL', "title": "re:^FM COCOLO.+$", 'alt_title': 'FM COCOLO', - 'thumbnail': 'https://radiko.jp/res/banner/CCL/20161014144826.png', + 'thumbnail': 'https://radiko.jp/v2/static/station/logo/CCL/lrtrim/688x160.png', 'channel': 'FM COCOLO', 'channel_id': 'CCL', -- cgit v1.2.3-70-g09d2 From a77e8c29ed85378768b97f7fc61d1b1fb21aedaa Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 9 Jun 2025 07:51:32 +0100 Subject: set count -> mincount for persons key station test --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index c87b1b7..5b79fc6 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -782,7 +782,7 @@ class RadikoPersonIE(InfoExtractor): },{ "url": "https://radiko.jp/persons/11421", "params": {'extractor_args': {'rajiko': {'key_station_only': ['']}}}, - "playlist_count": 1, + "playlist_mincount": 1, "info_dict": { "id": "person-11421", }, -- cgit v1.2.3-70-g09d2 From 12aced7d2d34d3e2ba1b2929aa421adcaabef434 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 22 Jun 2025 01:03:45 +0100 Subject: Move RadikoTimeFreeIE tests to test script --- misc/test_extractors.py | 25 ++++++++++++++++ yt_dlp_plugins/extractor/radiko.py | 61 +------------------------------------- 2 files changed, 26 insertions(+), 60 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/misc/test_extractors.py b/misc/test_extractors.py index 4ae5313..41dedb4 100755 --- a/misc/test_extractors.py +++ b/misc/test_extractors.py @@ -76,7 +76,32 @@ RadikoTimeFreeIE._TESTS.append({ }) +# late-night/v. early morning show, to test broadcast day handling +# this should be monday 27:00 / tuesday 03:00 +airtime, release_time = get_latest_airtimes(now, TUE, 3, 0, datetime.timedelta(hours=2)) +RadikoTimeFreeIE._TESTS.append({ + "url": f"https://radiko.jp/#!/ts/TBS/{airtime.timestring()}", + "info_dict": { + "ext": "m4a", + "id": f"TBS-{airtime.timestring()}", + **get_test_timefields(airtime, release_time), + 'title': 'CITY CHILL CLUB', + 'description': r"re:^目を閉じて…リラックスして[\S\s]+chill@tbs.co.jp$", + 'uploader': 'TBSラジオ', + 'uploader_id': 'TBS', + 'uploader_url': 'https://www.tbsradio.jp/', + 'channel': 'TBSラジオ', + 'channel_id': 'TBS', + 'channel_url': 'https://www.tbsradio.jp/', + 'thumbnail': 'https://program-static.cf.radiko.jp/nrf8fowbjo.jpg', + 'chapters': list, + 'tags': ['CCC905', '音楽との出会いが楽しめる', '人気アーティストトーク', '音楽プロデューサー出演', 'ドライブ中におすすめ', '寝る前におすすめ', '学生におすすめ'], + 'cast': list, + 'series': 'CITY CHILL CLUB', + 'live_status': 'was_live', + }, +}) diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 5b79fc6..c4e4846 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -422,66 +422,7 @@ class RadikoLiveIE(_RadikoBaseIE): class RadikoTimeFreeIE(_RadikoBaseIE): _NETRC_MACHINE = "rajiko" _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/ts/(?P[A-Z0-9-_]+)/(?P\d+)" - _TESTS = [{ - "url": "https://radiko.jp/#!/ts/INT/20240809230000", - "info_dict": { - "live_status": "was_live", - "ext": "m4a", - "id": "INT-20240809230000", - - "title": "TOKYO MOON", - "series": "Tokyo Moon", - "description": "md5:20e68d2f400a391fa34d4e7c8c702cb8", - "chapters": "count:14", - "thumbnail": "https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg", - - "upload_date": "20240809", - "timestamp": 1723212000.0, - "release_date": "20240809", - "release_timestamp": 1723215600.0, - "duration": 3600, - - "channel": "interfm", - "channel_id": "INT", - "channel_url": "https://www.interfm.co.jp/", - "uploader": "interfm", - "uploader_id": "INT", - "uploader_url": "https://www.interfm.co.jp/", - - "cast": ["松浦\u3000俊夫"], - "tags": ["松浦俊夫"], - }, - }, { - # late-night/early-morning show to test broadcast day checking - "url": "https://radiko.jp/#!/ts/TBS/20240810033000", - "info_dict": { - "live_status": "was_live", - "ext": "m4a", - "id": "TBS-20240810033000", - - "title": "CITY CHILL CLUB", - "series": "CITY CHILL CLUB", - "description": "md5:3fba2c1125059bed27247c0be90e58fa", - "chapters": "count:22", - "thumbnail": "https://program-static.cf.radiko.jp/ku7t4ztnaq.jpg", - - "upload_date": "20240809", - "timestamp": 1723228200.0, - "release_date": "20240809", - "release_timestamp": 1723233600.0, - "duration": 5400, - - "channel": "TBSラジオ", - "channel_url": "https://www.tbsradio.jp/", - "channel_id": "TBS", - "uploader": "TBSラジオ", - "uploader_url": "https://www.tbsradio.jp/", - "uploader_id": "TBS", - - "tags": ["CCC905", "音楽との出会いが楽しめる", "人気アーティストトーク", "音楽プロデューサー出演", "ドライブ中におすすめ", "寝る前におすすめ", "学生におすすめ"], - "cast": ["PES"], - }, - }] + # TESTS use a custom-ish script that updates the airdates automatically, see misc/test_extractors.py def _perform_login(self, username, password): try: -- cgit v1.2.3-70-g09d2 From 781ce8e7a74aa19df8793a3f29176d82c44f6b6b Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 22 Jun 2025 01:25:24 +0100 Subject: move ShareIE tests to test script --- misc/test_extractors.py | 36 ++++++++++++++++++++++++++++++++++-- yt_dlp_plugins/extractor/radiko.py | 31 ------------------------------- 2 files changed, 34 insertions(+), 33 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/misc/test_extractors.py b/misc/test_extractors.py index 41dedb4..7c1a2c7 100755 --- a/misc/test_extractors.py +++ b/misc/test_extractors.py @@ -42,7 +42,7 @@ def get_test_timefields(airtime, release_time): -from yt_dlp_plugins.extractor.radiko import RadikoTimeFreeIE +from yt_dlp_plugins.extractor.radiko import RadikoTimeFreeIE, RadikoShareIE RadikoTimeFreeIE._TESTS = [] @@ -104,8 +104,40 @@ RadikoTimeFreeIE._TESTS.append({ }) +# testing 29-hour clock handling +airtime, release_time = get_latest_airtimes(now, WED, 0, 0, datetime.timedelta(minutes=55)) +share_timestring = (airtime - datetime.timedelta(days=1)).strftime("%Y%m%d") + "240000" -IEs = [RadikoTimeFreeIE] +RadikoShareIE._TESTS = [{ + "url": f"http://radiko.jp/share/?sid=FMT&t={share_timestring}", + "info_dict": { + "live_status": "was_live", + "ext": "m4a", + "id": f"FMT-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + + "title": "JET STREAM", + "series": "JET STREAM", + "description": r"re:^JET STREAM・・・作家が描く世界への旅。[\s\S]+https://www.tfm.co.jp/f/jetstream/message$", + "chapters": list, + "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg", + + "channel": "TOKYO FM", + "channel_id": "FMT", + "channel_url": "https://www.tfm.co.jp/", + "uploader": "TOKYO FM", + "uploader_id": "FMT", + "uploader_url": "https://www.tfm.co.jp/", + + "cast": ["福山雅治"], + "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"], + }, + }] + + + +IEs = [RadikoTimeFreeIE, RadikoShareIE] import test.helper as th diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index c4e4846..2996290 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -628,37 +628,6 @@ class RadikoSearchIE(InfoExtractor): class RadikoShareIE(InfoExtractor): _VALID_URL = r"https?://(?:www\.)?radiko\.jp/share/" - _TESTS = [{ - # 29-hour time -> 24-hour time - "url": "http://radiko.jp/share/?sid=FMT&t=20240802240000", - "info_dict": { - "live_status": "was_live", - "ext": "m4a", - "id": "FMT-20240803000000", # the time given (24:00) works out to 00:00 the next day - - "title": "JET STREAM", - "series": "JET STREAM", - "description": "md5:c1a2172036ebb7a54eeafb47e0a08a50", - "chapters": "count:9", - "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg", - - "upload_date": "20240802", - "timestamp": 1722610800.0, - "release_date": "20240802", - "release_timestamp": 1722614100.0, - "duration": 3300, - - "channel": "TOKYO FM", - "channel_id": "FMT", - "channel_url": "https://www.tfm.co.jp/", - "uploader": "TOKYO FM", - "uploader_id": "FMT", - "uploader_url": "https://www.tfm.co.jp/", - - "cast": ["福山雅治"], - "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"], - } - }] def _real_extract(self, url): queries = parse_qs(url) -- cgit v1.2.3-70-g09d2 From 794ec2b5256a8219a0e50f3e14165f0914189db4 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 10 Jul 2025 22:12:28 +0100 Subject: Add basic radiko podcast extractors --- yt_dlp_plugins/extractor/radiko_podcast.py | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 yt_dlp_plugins/extractor/radiko_podcast.py (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py new file mode 100644 index 0000000..93e1408 --- /dev/null +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -0,0 +1,114 @@ +from yt_dlp.extractor.common import InfoExtractor +from yt_dlp.utils import ( + clean_html, + traverse_obj, + url_or_none, + str_or_none, +) + +# nice simple one for a change... +# the app uses a similar system to regular programmes, thankfully the site doesn't +# but it does need protobufs to get more than 20 items... + +class _RadikoPodcastBaseIE(InfoExtractor): + + def _extract_episode(self, episode_info): + return { + **traverse_obj(episode_info, { + "id": ("id", {str_or_none}), + "url": ("audio", "url"), + "duration": ("audio", "durationSec"), + + "title": "title", + "description": ("description", {clean_html}), + "timestamp": ("startAt", "seconds"), + + "series": "channelTitle", + "series_id": "channelId", + "channel": "stationName", + "uploader": "stationName", + }), + "thumbnail": traverse_obj(episode_info, ("imageUrl", {url_or_none})) + or traverse_obj(episode_info, ("channelImageUrl", {url_or_none})), + + # so that --download-archive still works if you download from the playlist page + "webpage_url": "https://radiko.jp/podcast/episodes/{id}".format(id=episode_info.get("id")), + 'extractor_key': RadikoPodcastEpisodeIE.ie_key(), + 'extractor': 'RadikoPodcastEpisode', + } + + +class RadikoPodcastEpisodeIE(_RadikoPodcastBaseIE): + _VALID_URL = r"https?://radiko\.jp/podcast/episodes/(?P[a-f0-9-]+)" + + _TESTS = [{ + "url": "https://radiko.jp/podcast/episodes/cc8cf709-a50b-4846-aa0e-91ab10cf8bff", + "info_dict": { + "id": "cc8cf709-a50b-4846-aa0e-91ab10cf8bff", + "ext": "mp3", + 'title': '2025.6.26 おしゃべり技術くん', + 'description': 'md5:1c4048025f68d6da053dd879a5d62304', + 'duration': 717, + 'thumbnail': 'https://podcast-static.cf.radiko.jp/09f27a48-ae04-4ce7-a024-572460e46eb7-20240214160012.png', + 'series': 'おしゃべり技術くん', + 'series_id': '09f27a48-ae04-4ce7-a024-572460e46eb7', + 'timestamp': 1751554800, + 'upload_date': '20250703', + }, + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + next_data = self._search_nextjs_data(webpage, video_id)["props"]["pageProps"] + + episode_info = next_data["podcastEpisode"] + + return self._extract_episode(episode_info) + + +class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): + _VALID_URL = r"https?://radiko\.jp/podcast/channels/(?P[a-f0-9-]+)" + + _TESTS = [{ + "url": "https://radiko.jp/podcast/channels/09f27a48-ae04-4ce7-a024-572460e46eb7", + "info_dict": { + "id": "09f27a48-ae04-4ce7-a024-572460e46eb7" + }, + 'playlist_mincount': 20, + 'expected_warnings': ['Currently this extractor can only extract the latest 20 episodes'], + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + next_data = self._search_nextjs_data(webpage, video_id)["props"]["pageProps"] + + channel_info = next_data["podcastChannel"] + episode_list_response = next_data["listPodcastEpisodesResponse"] + + + def entries(): + for episode in episode_list_response["episodesList"]: + yield self._extract_episode(episode) + + if traverse_obj(episode_list_response, "hasNextPage"): + self.report_warning(f'Currently this extractor can only extract the latest {len(episode_list_response["episodesList"])} episodes') + + # TODO: GRPC/protobuf stuff to get the next page + # https://api.annex.radiko.jp/radiko.PodcastService/ListPodcastEpisodes + # see さらに表示 button on site + + + return { + "_type": "playlist", + "id": video_id, + **traverse_obj(channel_info, { + "playlist_title": "title", + "playlist_id": "id", + "playlist_description": ("description", {clean_html}), + "playlist_thumbnail": ("imageUrl", {url_or_none}), + + }), + "entries": entries(), + } -- cgit v1.2.3-70-g09d2 From fad5614bf07dae50be5850f28e845af7893967dc Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 10 Jul 2025 23:13:39 +0100 Subject: fix channel/uploader field in podcast extractor --- yt_dlp_plugins/extractor/radiko_podcast.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 93e1408..af66f6a 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -25,8 +25,8 @@ class _RadikoPodcastBaseIE(InfoExtractor): "series": "channelTitle", "series_id": "channelId", - "channel": "stationName", - "uploader": "stationName", + "channel": "channelStationName", + "uploader": "channelStationName", }), "thumbnail": traverse_obj(episode_info, ("imageUrl", {url_or_none})) or traverse_obj(episode_info, ("channelImageUrl", {url_or_none})), @@ -54,6 +54,8 @@ class RadikoPodcastEpisodeIE(_RadikoPodcastBaseIE): 'series_id': '09f27a48-ae04-4ce7-a024-572460e46eb7', 'timestamp': 1751554800, 'upload_date': '20250703', + 'uploader': 'IBCラジオ', + 'channel': 'IBCラジオ', }, }] -- cgit v1.2.3-70-g09d2 From 42cd450ab7578f723a5590003f751052d0a83ad7 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 11 Aug 2025 09:10:14 +0100 Subject: Add podcast protobufs + functions to use them --- yt_dlp_plugins/extractor/radiko_protobufs.py | 152 +++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 yt_dlp_plugins/extractor/radiko_protobufs.py (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_protobufs.py b/yt_dlp_plugins/extractor/radiko_protobufs.py new file mode 100755 index 0000000..2336f10 --- /dev/null +++ b/yt_dlp_plugins/extractor/radiko_protobufs.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +try: + import protobug +except ImportError: + protobug = None + +import base64 +import struct + +import random +import requests + +if protobug: # i suppose it works lmao + + + def add_grpc_header(protobuf_data): + compression_flag = 0 + message_length = len(protobuf_data) + header = struct.pack('>BI', compression_flag, message_length) + return header + protobuf_data + + def strip_grpc_response(response): + return response[5:].rpartition(b"grpc-status:")[0] + + def _download_grpc(self, url_or_request, video_id, response_message, note="Downloading GRPC information", *args, **kwargs): + urlh = self._request_webpage(url_or_request, video_id, + headers={ + 'Content-Type': 'application/grpc-web+proto', + 'X-User-Agent': 'grpc-web-javascript/0.1', + 'X-Grpc-Web': '1', + **kwargs.pop('headers') + }, + data=add_grpc_header(protobug.dumps(kwargs.pop('data'))), note=note, + *args, **kwargs, + ) + response = urlh.read() + + protobuf = strip_grpc_response(response) + if len(protobuf) > 0: + return protobug.loads(protobuf, response_message) + + + @protobug.message + class SignUpRequest: + lsid: protobug.String = protobug.field(1) + + def sign_up(self): + lsid = ''.join(random.choices('0123456789abcdef', k=32)) + + signup = _download_grpc(self, "https://api.annex.radiko.jp/radiko.UserService/SignUp", + "UserService", None, note="Registering ID", headers={'Origin': 'https://radiko.jp'}, + data=SignUpRequest(lsid=lsid), + ) + # youre meant to only do the sign up ^ once and then keep your lsid for later + # so that you can sign in and get the token for the API to work + return lsid + + + @protobug.message + class SignInRequest: + lsid: protobug.String = protobug.field(2) + area: protobug.String = protobug.field(3) + + @protobug.message + class SignInResponse: + jwt: protobug.String = protobug.field(1) + + + def sign_in(self, lsid): + sign_in = _download_grpc(self, "https://api.annex.radiko.jp/radiko.UserService/SignIn", + "UserService", SignInResponse, note="Getting auth token", headers={'Origin': 'https://radiko.jp'}, + data=SignInRequest(lsid=lsid, area="JP13"), + ) + return sign_in.jwt + + + def auth_userservice(self): + cachedata = self.cache.load("rajiko", "UserService") + if cachedata is not None: + lsid = cachedata.get("lsid") + else: + lsid = sign_up(self) + self.cache.store("rajiko", "UserService", {"lsid": lsid}) + jwt = sign_in(self, lsid) + return jwt + + + @protobug.message + class ListPodcastEpisodesRequest: + channel_id: protobug.String = protobug.field(1) + dontknow: protobug.Int32 = protobug.field(2) + page_length: protobug.Int32 = protobug.field(4) + cursor: protobug.String = protobug.field(5, default=None) + + + @protobug.message + class Audio: + revision: protobug.Int32 = protobug.field(1) + url: protobug.String = protobug.field(2) + fileSize: protobug.Int64 = protobug.field(3) + durationSec: protobug.Int64 = protobug.field(4) + transcoded: protobug.Bool = protobug.field(5) + + @protobug.message + class EpisodeStartAt: + seconds: protobug.UInt64 = protobug.field(1) + nanos: protobug.UInt64 = protobug.field(2, default=0) + + + @protobug.message + class PodcastEpisode: + id: protobug.String = protobug.field(1) + workspaceId: protobug.String = protobug.field(2) + channelId: protobug.String = protobug.field(3) + title: protobug.String = protobug.field(4) + description: protobug.String = protobug.field(5) + + audio: Audio = protobug.field(8) + channelImageUrl: protobug.String = protobug.field(16) + channelTitle: protobug.String = protobug.field(17) + channelStationName: protobug.String = protobug.field(18) + channelAuthor: protobug.String = protobug.field(19) + + channelThumbnailImageUrl: protobug.String = protobug.field(21) + channelStationType: protobug.UInt32 = protobug.field(22) + startAt: EpisodeStartAt = protobug.field(27) + isEnabled: protobug.Bool = protobug.field(29) + hasTranscription: protobug.Bool = protobug.field(32) + + imageUrl: protobug.String = protobug.field(7, default=None) + thumbnailImageUrl: protobug.String = protobug.field(20, default=None) + + @protobug.message + class ListPodcastEpisodesResponse: + episodes: list[PodcastEpisode] = protobug.field(1) + hasNextPage: protobug.Bool = protobug.field(2, default=False) + + + def get_podcast_episodes(self, channel_id, jwt, cursor, page_length=20): + # site uses 20 items + # cursor is the id of the last episode you've seen in the list + + return _download_grpc(self, 'https://api.annex.radiko.jp/radiko.PodcastService/ListPodcastEpisodes', + channel_id, ListPodcastEpisodesResponse, note="Downloading episode listings", + headers={'Authorization': f'Bearer {jwt}'}, + data=ListPodcastEpisodesRequest( + channel_id=channel_id, + dontknow=1, + page_length=page_length, + cursor=cursor, + ) + ) -- cgit v1.2.3-70-g09d2 From ac94bad6ed14f32adfeceac35cc60d39680508dd Mon Sep 17 00:00:00 2001 From: garret1317 Date: Mon, 11 Aug 2025 09:22:50 +0100 Subject: Implement multi-page podcasts with protobug needs core change in upstream to work (traverse_obj doesnt work with dataclasses) --- yt_dlp_plugins/extractor/radiko_podcast.py | 39 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index af66f6a..904bb62 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -6,9 +6,12 @@ from yt_dlp.utils import ( str_or_none, ) -# nice simple one for a change... -# the app uses a similar system to regular programmes, thankfully the site doesn't -# but it does need protobufs to get more than 20 items... +try: + import protobug + import yt_dlp_plugins.extractor.radiko_protobufs as pb +except ImportError: + protobug = None + class _RadikoPodcastBaseIE(InfoExtractor): @@ -32,7 +35,7 @@ class _RadikoPodcastBaseIE(InfoExtractor): or traverse_obj(episode_info, ("channelImageUrl", {url_or_none})), # so that --download-archive still works if you download from the playlist page - "webpage_url": "https://radiko.jp/podcast/episodes/{id}".format(id=episode_info.get("id")), + "webpage_url": "https://radiko.jp/podcast/episodes/{id}".format(id=traverse_obj(episode_info, "id")), 'extractor_key': RadikoPodcastEpisodeIE.ie_key(), 'extractor': 'RadikoPodcastEpisode', } @@ -82,29 +85,35 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): }] def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - next_data = self._search_nextjs_data(webpage, video_id)["props"]["pageProps"] + channel_id = self._match_id(url) + webpage = self._download_webpage(url, channel_id) + next_data = self._search_nextjs_data(webpage, channel_id)["props"]["pageProps"] channel_info = next_data["podcastChannel"] episode_list_response = next_data["listPodcastEpisodesResponse"] def entries(): + has_next_page = episode_list_response.get("hasNextPage") for episode in episode_list_response["episodesList"]: + cursor = episode.get("id") yield self._extract_episode(episode) - if traverse_obj(episode_list_response, "hasNextPage"): - self.report_warning(f'Currently this extractor can only extract the latest {len(episode_list_response["episodesList"])} episodes') - - # TODO: GRPC/protobuf stuff to get the next page - # https://api.annex.radiko.jp/radiko.PodcastService/ListPodcastEpisodes - # see さらに表示 button on site - + if has_next_page: + if protobug: + userservice_token = pb.auth_userservice(self) + while has_next_page: + page = pb.get_podcast_episodes(self, channel_id, userservice_token, cursor) + has_next_page = page.hasNextPage + for episode in page.episodes: + cursor = episode.id + yield self._extract_episode(episode) + else: + self.report_warning(f'Only extracting the latest {len(episode_list_response["episodesList"])} episodes. Install protobug for more.') return { "_type": "playlist", - "id": video_id, + "id": channel_id, **traverse_obj(channel_info, { "playlist_title": "title", "playlist_id": "id", -- cgit v1.2.3-70-g09d2 From d5f824093b0748889916a1ba820398aecaa184c8 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Wed, 13 Aug 2025 02:49:46 +0100 Subject: convert protobug obj to dict, for traverse_obj --- yt_dlp_plugins/extractor/radiko_podcast.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 904bb62..84bc288 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -6,6 +6,7 @@ from yt_dlp.utils import ( str_or_none, ) +import dataclasses try: import protobug import yt_dlp_plugins.extractor.radiko_protobufs as pb @@ -107,7 +108,7 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): has_next_page = page.hasNextPage for episode in page.episodes: cursor = episode.id - yield self._extract_episode(episode) + yield self._extract_episode(dataclasses.asdict(episode)) else: self.report_warning(f'Only extracting the latest {len(episode_list_response["episodesList"])} episodes. Install protobug for more.') -- cgit v1.2.3-70-g09d2 From 9e91cb5ee32a47eb05dc2d3885e13d274cdadd03 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Wed, 13 Aug 2025 07:29:19 +0100 Subject: ListPodcastEpisodesRequest: dontknow -> sort_by_latest --- yt_dlp_plugins/extractor/radiko_protobufs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_protobufs.py b/yt_dlp_plugins/extractor/radiko_protobufs.py index 2336f10..ff4531e 100755 --- a/yt_dlp_plugins/extractor/radiko_protobufs.py +++ b/yt_dlp_plugins/extractor/radiko_protobufs.py @@ -88,7 +88,7 @@ if protobug: # i suppose it works lmao @protobug.message class ListPodcastEpisodesRequest: channel_id: protobug.String = protobug.field(1) - dontknow: protobug.Int32 = protobug.field(2) + sort_by_latest: protobug.Bool = protobug.field(2) page_length: protobug.Int32 = protobug.field(4) cursor: protobug.String = protobug.field(5, default=None) @@ -145,7 +145,7 @@ if protobug: # i suppose it works lmao headers={'Authorization': f'Bearer {jwt}'}, data=ListPodcastEpisodesRequest( channel_id=channel_id, - dontknow=1, + sort_by_latest=True, page_length=page_length, cursor=cursor, ) -- cgit v1.2.3-70-g09d2 From 4da1432900abacc270d615314475c284c60b69ae Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 24 Aug 2025 12:13:13 +0100 Subject: set vcodec none for chunked formats so they get marked as audio only --- yt_dlp_plugins/extractor/radiko.py | 1 + 1 file changed, 1 insertion(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 2996290..9601c32 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -322,6 +322,7 @@ class _RadikoBaseIE(InfoExtractor): "hls_media_playlist_data": chunks_playlist, "preference": preference, "ext": "m4a", + "vcodec": "none", # fallback to live for ffmpeg etc "url": playlist_url, -- cgit v1.2.3-70-g09d2 From 221cefe4810e0bcd5c6faf199ae2589f62edbb77 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Tue, 26 Aug 2025 00:37:05 +0100 Subject: Set ffmpeg args to not send Range header github issue #29 --- yt_dlp_plugins/extractor/radiko.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 9601c32..fb705dd 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -317,7 +317,7 @@ class _RadikoBaseIE(InfoExtractor): self, playlist_url, start_at, end_at, domain, auth_headers ) - formats.append({ + m3u8_formats = [{ "format_id": join_nonempty(domain, "chunked"), "hls_media_playlist_data": chunks_playlist, "preference": preference, @@ -327,13 +327,20 @@ class _RadikoBaseIE(InfoExtractor): # fallback to live for ffmpeg etc "url": playlist_url, "http_headers": auth_headers, - }) + }] else: - formats += self._extract_m3u8_formats( + m3u8_formats = self._extract_m3u8_formats( playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_headers, live=delivered_live, preference=preference, entry_protocol=entry_protocol, note=f"Downloading m3u8 information from {domain}") + + for f in m3u8_formats: + # ffmpeg sends a Range header which some streams reject. here we disable that (and also some icecast header as well) + f['downloader_options'] = {'ffmpeg_args': ['-seekable', '0', '-http_seekable', '0', '-icy', '0']} + f['format_note'] = ", ".join(format_note) + formats.append(f) + return formats -- cgit v1.2.3-70-g09d2 From a12ae8fe4636399ce4582c6cdb0bc67352b2caf3 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 16 Aug 2025 06:32:40 +0100 Subject: Flag and deprioritise formats with ad insertion --- yt_dlp_plugins/extractor/radiko.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index fb705dd..1f5bf30 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -87,6 +87,7 @@ class _RadikoBaseIE(InfoExtractor): _DELIVERED_ONDEMAND = ('radiko.jp',) _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp', 'alliance-stream-radiko.smartstream.ne.jp') + _AD_INSERTION = ('si-f-radiko.smartstream.ne.jp', ) _has_tf30 = None @@ -299,6 +300,7 @@ class _RadikoBaseIE(InfoExtractor): delivered_live = True preference = -1 entry_protocol = 'm3u8' + format_note=[] if domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams: self.write_debug(f"skipping {domain} (known not working)") @@ -306,8 +308,12 @@ class _RadikoBaseIE(InfoExtractor): if domain in self._DELIVERED_ONDEMAND: # override the defaults for delivered as on-demand delivered_live = False - preference = 1 + preference += 2 entry_protocol = None + if domain in self._AD_INSERTION: + preference -= 3 + format_note.append("Ad insertion") + auth_headers = auth_data["token"] @@ -328,6 +334,7 @@ class _RadikoBaseIE(InfoExtractor): "url": playlist_url, "http_headers": auth_headers, }] + format_note.append("Chunked") else: m3u8_formats = self._extract_m3u8_formats( -- cgit v1.2.3-70-g09d2 From cbb46ec14339cd938f96446440ea2bd32a2ff1ba Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 29 Aug 2025 20:35:13 +0100 Subject: PersonsIE: construct a timefree url_result directly so that --download-archive etc will quickly skip without a full extraction also removes reliance on `radiko_url` always being a share url --- yt_dlp_plugins/extractor/radiko.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 1f5bf30..94df945 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -733,19 +733,16 @@ class RadikoPersonIE(InfoExtractor): def entries(): key_station_only = len(self._configuration_arg("key_station_only", ie_key="rajiko")) > 0 for episode in person_api.get("data"): - if key_station_only and episode.get("key_station_id") != episode.get("station_id"): - continue - share_url = traverse_obj(episode, ("radiko_url", ("pc", "sp", "android", "ios", "app"), - {url_or_none}), get_all=False) - # they're all identical share links at the moment (5th aug 2024) but they might not be in the future + station = episode.get("station_id") + if key_station_only and episode.get("key_station_id") != station: + continue - # predictions: - # pc will probably stay the same - # don't know what sp is, possibly "SmartPhone"?, anyway seems reasonably generic - # android is easier for me to reverse-engineer than ios (no ithing) - # i assume "app" would be some internal tell-it-to-do-something link, not a regular web link + start = episode.get("start_at") + timestring = rtime.RadikoTime.fromisoformat(start).timestring() - yield self.url_result(share_url, ie=RadikoShareIE, video_title=episode.get("title")) + timefree_id = join_nonempty(station, timestring) + timefree_url = f"https://radiko.jp/#!/ts/{station}/{timestring}" + yield self.url_result(timefree_url, ie=RadikoTimeFreeIE, video_id=timefree_id) return self.playlist_result(entries(), playlist_id=join_nonempty("person", person_id)) -- cgit v1.2.3-70-g09d2 From bd04585fa66131530227456b9f8c2717c0d23a23 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 29 Aug 2025 20:57:50 +0100 Subject: add support for "r_seasons" programme pages --- misc/test_extractors.py | 4 +++- yt_dlp_plugins/extractor/radiko.py | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/misc/test_extractors.py b/misc/test_extractors.py index f6ee0dd..21800c5 100755 --- a/misc/test_extractors.py +++ b/misc/test_extractors.py @@ -44,7 +44,8 @@ def get_test_timefields(airtime, release_time): from yt_dlp_plugins.extractor.radiko import ( RadikoTimeFreeIE, RadikoShareIE, - RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE + RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, + RadikoRSeasonsIE ) from yt_dlp_plugins.extractor.radiko_podcast import ( @@ -147,6 +148,7 @@ IEs = [ RadikoTimeFreeIE, RadikoShareIE, RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, + RadikoRSeasonsIE, ] import test.helper as th diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 94df945..e81a1ca 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -746,3 +746,45 @@ class RadikoPersonIE(InfoExtractor): yield self.url_result(timefree_url, ie=RadikoTimeFreeIE, video_id=timefree_id) return self.playlist_result(entries(), playlist_id=join_nonempty("person", person_id)) + + +class RadikoRSeasonsIE(InfoExtractor): + _VALID_URL = r"https?://(?:www\.)?radiko\.jp/(?:mobile/)?r_seasons/(?P\d+$)" + _TESTS = [{ + "url": "https://radiko.jp/r_seasons/10012302", + "playlist_mincount": 4, + "info_dict": { + "id": '10012302', + "title": '山下達郎の楽天カード サンデー・ソングブック', + } + }, { + "url": "https://radiko.jp/r_seasons/10002831", + "playlist_mincount": 4, + "info_dict": { + "id": "10002831", + "title": "Tokyo Moon", + } + }] + + def _real_extract(self, url): + season_id = self._match_id(url) + html = self._download_webpage(url, season_id) + pageProps = self._search_nextjs_data(html, season_id)["props"]["pageProps"] + season_id = traverse_obj(pageProps, ("rSeason", "id")) or season_id + + def entries(): + for episode in pageProps.get("pastPrograms"): + station = traverse_obj(episode, ("stationId")) + start = traverse_obj(episode, ("startAt", "seconds")) + timestring = rtime.RadikoTime.fromtimestamp(start, tz=rtime.JST).timestring() + + timefree_id = join_nonempty(station, timestring) + timefree_url = f"https://radiko.jp/#!/ts/{station}/{timestring}" + + yield self.url_result(timefree_url, ie=RadikoTimeFreeIE, video_id=timefree_id) + + return self.playlist_result( + entries(), + playlist_id=season_id, + playlist_title=traverse_obj(pageProps, ("rSeason", "rSeasonName")), + ) -- cgit v1.2.3-70-g09d2 From 205bcb84e106126fedf419ab869c68bbee739441 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 11 Sep 2025 17:09:35 +0100 Subject: update some comments --- yt_dlp_plugins/extractor/radiko.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index e81a1ca..d387a79 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -256,8 +256,7 @@ class _RadikoBaseIE(InfoExtractor): config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0) if not use_pc_html5: - device = config_device or "aSmartPhone7a" # this device only gives us the on-demand one for timefree - # that's good imo - we just get the one that works, and don't bother with probing the rest as well + device = config_device or "aSmartPhone7a" # still has the radiko.jp on-demand one for timefree else: device = config_device or "pc_html5" # the on-demand one doesnt work with timefree30 stuff sadly # so just use pc_html5 which has everything @@ -273,6 +272,7 @@ class _RadikoBaseIE(InfoExtractor): do_as_live_chunks = not len(self._configuration_arg("no_as_live_chunks", ie_key="rajiko")) > 0 for element in url_data.findall(f".//url[@timefree='{timefree_int}'][@areafree='0']/playlist_create_url"): # find s with matching timefree and no areafree, then get their + # we don't want areafree here because we should always be in-region url = element.text if url in seen_urls: # there are always dupes, even with ^ specific filtering continue -- cgit v1.2.3-70-g09d2 From 3d8d8b1cb6e3a7792077b413d705c8337ae9c014 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 11 Sep 2025 18:14:59 +0100 Subject: Don't return any formats when we don't actually have any --- yt_dlp_plugins/extractor/radiko.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index d387a79..ae0ee49 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -542,9 +542,13 @@ class RadikoTimeFreeIE(_RadikoBaseIE): region = self._get_station_region(station) station_meta = self._get_station_meta(region, station) - chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) - auth_data = self._auth(region, need_tf30=need_tf30) - formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, use_pc_html5=need_tf30) + if live_status == "was_live": + chapters = self._extract_chapters(station, start, end, video_id=meta["id"]) + auth_data = self._auth(region, need_tf30=need_tf30) + formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, use_pc_html5=need_tf30) + else: + chapters = None + formats = None return { **station_meta, -- cgit v1.2.3-70-g09d2 From a88ca60ee514c5347193fc679933a31117bac7ea Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sat, 13 Sep 2025 15:27:57 +0100 Subject: clarify "indexing (station) regions" text --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index ae0ee49..0566b6d 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -94,7 +94,7 @@ class _RadikoBaseIE(InfoExtractor): def _index_regions(self): region_data = {} - tree = self._download_xml("https://radiko.jp/v3/station/region/full.xml", None, note="Indexing regions") + tree = self._download_xml("https://radiko.jp/v3/station/region/full.xml", None, note="Indexing station regions") for stations in tree: for station in stations: area = station.find("area_id").text -- cgit v1.2.3-70-g09d2 From f3a16083b9f09c132dce6f262a1fb30f672b9441 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 14 Sep 2025 15:04:02 +0100 Subject: rename "misc" -> "contrib" --- .github/workflows/download.yml | 2 +- contrib/generate_html.py | 84 +++++++++++++++++ contrib/how to do a release | 41 +++++++++ contrib/old_generate_changelog.py | 116 +++++++++++++++++++++++ contrib/protostuff.py | 154 +++++++++++++++++++++++++++++++ contrib/randominfo.py | 12 +++ contrib/streammon.py | 66 +++++++++++++ contrib/test_areas.py | 26 ++++++ contrib/test_extractors.py | 183 +++++++++++++++++++++++++++++++++++++ misc/generate_html.py | 84 ----------------- misc/how to do a release | 41 --------- misc/old_generate_changelog.py | 116 ----------------------- misc/protostuff.py | 154 ------------------------------- misc/randominfo.py | 12 --- misc/streammon.py | 66 ------------- misc/test_areas.py | 26 ------ misc/test_extractors.py | 183 ------------------------------------- yt_dlp_plugins/extractor/radiko.py | 2 +- 18 files changed, 684 insertions(+), 684 deletions(-) create mode 100755 contrib/generate_html.py create mode 100644 contrib/how to do a release create mode 100755 contrib/old_generate_changelog.py create mode 100755 contrib/protostuff.py create mode 100755 contrib/randominfo.py create mode 100755 contrib/streammon.py create mode 100755 contrib/test_areas.py create mode 100755 contrib/test_extractors.py delete mode 100755 misc/generate_html.py delete mode 100644 misc/how to do a release delete mode 100755 misc/old_generate_changelog.py delete mode 100755 misc/protostuff.py delete mode 100755 misc/randominfo.py delete mode 100755 misc/streammon.py delete mode 100755 misc/test_areas.py delete mode 100755 misc/test_extractors.py (limited to 'yt_dlp_plugins/extractor') diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 2a39a3b..115ae4e 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -45,4 +45,4 @@ jobs: - name: Run tests env: PYTHONPATH: ${{ github.workspace }}/yt-dlp${{ runner.os == 'Windows' && ';' || ':' }}${{ env.PYTHONPATH }} - run: python ./yt-dlp-plugins/yt-dlp-rajiko/misc/test_extractors.py + run: python ./yt-dlp-plugins/yt-dlp-rajiko/contrib/test_extractors.py diff --git a/contrib/generate_html.py b/contrib/generate_html.py new file mode 100755 index 0000000..0e15d6a --- /dev/null +++ b/contrib/generate_html.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import os +import hashlib +import re + +pip_index = open("index.html", "w") + +pip_index.write(""" + + + yt-dlp-rajiko pip index + + + + +
    +""") + +site_sha256 = [] + +tarballs = [] +wheels = [] + +for item in sorted(os.listdir()):#, key=lambda x: x.name): + if os.path.islink(item): + continue + + if item.endswith(".tar.gz"): + tarballs.append(item) + elif item.endswith(".whl"): + wheels.append(item) + else: + continue + + pip_index.write("\t
  • ") + pip_index.write('') + pip_index.write(item) + pip_index.write("\n") + + site_string = checksum + "  " + '' + item + "
    " + site_sha256.append(site_string) + +pip_index.write("""
+ + +""") + +latest_tarball = tarballs[-1] +latest_wheel = wheels[-1] +print(latest_tarball, latest_wheel) + +os.remove("yt_dlp_rajiko-latest.tar.gz") +os.symlink(latest_tarball, "yt_dlp_rajiko-latest.tar.gz") + +os.remove("yt_dlp_rajiko-latest.whl") +os.symlink(latest_wheel, "yt_dlp_rajiko-latest.whl") + +site_sha256.reverse() + +latest_list = site_sha256[:2] +previous_list = site_sha256[2:] + +latest = "\n".join(["", "", "\n".join(latest_list), "", ""]) + +previous = "\n".join(["", "", "\n".join(previous_list), "", ""]) + +for i in ["../../index.html", "../../index.ja.html"]: + with open(i, "r+") as f: + page = f.read() + + page = re.sub(r".+", latest, page, flags=re.DOTALL) + page = re.sub(r".+", previous, page, flags=re.DOTALL) + + f.seek(0) + f.truncate(0) + f.write(page) diff --git a/contrib/how to do a release b/contrib/how to do a release new file mode 100644 index 0000000..6e91e14 --- /dev/null +++ b/contrib/how to do a release @@ -0,0 +1,41 @@ +putting this here because i'll forget how to do it otherwise + +update the pyproject.toml +tag it in git, eg v1.0 + +## build the builds +python3 -m build + +and then put BOTH items from `dist` into the pip index dir - ~/site2/yt-dlp-rajiko/pip/yt-dlp-rajiko/ +because without the .whl pip has to "build" it itself, with all the stuff that needs to be installed for that to work + +run script to update the pip index html and the dl/ "latest" symlinks +this also updates the sha256s on the site + +## update the changelog file + +write in html, paste into the feed xml like +make sure to set the link, date +to get date use: +git log --pretty --date=rfc2822 + +include the pip instructions, sha256sum etc + +now push to the server + +!!NEW!! +upload to pip proper as well +go to dl/ dir and do +twine upload yt_dlp_rajiko-1.x* + + +## update github + +paste the changelog output into a github release, upload the new builds +change link at the bottom to just "below" + +post in the radiko thread on 5ch if i can be bothered + +and thats probably all diff --git a/contrib/old_generate_changelog.py b/contrib/old_generate_changelog.py new file mode 100755 index 0000000..1bce073 --- /dev/null +++ b/contrib/old_generate_changelog.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +import email.utils +import feedgenerator + +def parse_changelog(lines): + got_version = False + got_date = False + got_url = False + done_remarks = False + releases = [] + release = {} + release_remarks = [] + release_changes = [] + current_change = "" + + for idx, line in enumerate(lines): + line = line.rstrip() + + if not got_version: + got_version = True + release["version"] = line + continue + + if not got_date: + release["date"] = email.utils.parsedate_to_datetime(line) + got_date = True + continue + + key, sep, val = line.partition(": ") + if key in ["url", "sha256", "released"] and val != "": + release[key] = val + continue + + if not done_remarks: + if line == "": + done_remarks = True + release["remarks"] = release_remarks + release_remarks = [] + continue + else: + release_remarks.append(line) + continue + + if line != "": + release_changes.append(line.rstrip()) + + if idx + 1 != len(lines): + continue + + release["changes"] = release_changes + if release.get("released") != "no": + releases.append(release) + + got_version = False + got_date = False + done_remarks = False + release = {} + release_changes = [] + + return releases + +def generate_rss_feed(releases): + feed = feedgenerator.Rss201rev2Feed( + title="yt-dlp-rajiko changelog", + description="Notifications for new yt-dlp-rajiko releases, with changelogs", + link="https://427738.xyz/yt-dlp-rajiko/", + language="en-GB", + ttl=180, # 3 hours + ) + + for release in releases: + title = "yt-dlp-rajiko " + release["version"] + " has been released" + description = "" + description += "

" + for remark in release["remarks"]: + description += remark + description += "
" + description += "

" + description += "

This release:

\n" + description += "
    " + for change in release["changes"]: + description += "
  • " + description += change + description += "
    • \n" + description += "

" + + if release.get("url"): + if release["version"] != "1.0": + description += "\n

If you use pip, you should be able to upgrade with pip install yt-dlp-rajiko --upgrade --extra-index-url https://427738.xyz/yt-dlp-rajiko/pip/.
" + description += "If you installed manually, you can download the updated .whl from this post's link." + if release.get("sha256"): + description += " The SHA256 checksum should be " + description += release.get("sha256") + description += "." + description += "

" + else: + description += '\n

Please see the homepage for initial installation instructions.

' + + feed.add_item( + title=title, + description=description, + link=release.get("url"), + pubdate=release["date"] + ) + return feed + +if __name__ == "__main__": + with open("CHANGELOG") as f: + releases = parse_changelog(f.readlines()) + + feed = generate_rss_feed(releases) + feed_contents = feed.writeString("utf-8") + feed_contents = feed_contents.replace("\nBI', compression_flag, message_length) + return header + protobuf_data + +def strip_grpc_response(response): + return response[5:].rpartition(b"grpc-status:")[0] + +print("SIGNUP") +# why do they have to make it so bloody complicated + +lsid = ''.join(random.choices('0123456789abcdef', k=32)) +big_funny = ("\n " + lsid).encode() + +signup = requests.post("https://api.annex.radiko.jp/radiko.UserService/SignUp", headers={ + 'Origin': 'https://radiko.jp', + 'Content-Type': 'application/grpc-web+proto', + 'X-User-Agent': 'grpc-web-javascript/0.1', + 'X-Grpc-Web': '1', + }, data=( add_grpc_header(big_funny)), +) + +print(signup.content) + +# youre meant to only do the sign up ^ once and then keep your id for later +# so that you can V sign in and get the token for the API to work + +print("SIGNIN") + +si=add_grpc_header(protobug.dumps(SignInRequest( + lsid=lsid, + area="JP13", +))) + +print(si) +print(base64.b64encode(si)) + +signin = requests.post("https://api.annex.radiko.jp/radiko.UserService/SignIn", headers={ + 'Origin': 'https://radiko.jp', + 'Content-Type': 'application/grpc-web+proto', + 'X-User-Agent': 'grpc-web-javascript/0.1', + 'X-Grpc-Web': '1', +}, data=si) + +print(signin.content) + +signin_result = protobug.loads(strip_grpc_response(signin.content), SignInResponse) + + +headers = { + 'Origin': 'https://radiko.jp', + 'Authorization': f'Bearer {signin_result.jwt}', + 'x-annex-proto-version': '1.0.0', + 'Content-Type': 'application/grpc-web+proto', + 'X-User-Agent': 'grpc-web-javascript/0.1', + 'X-Grpc-Web': '1', +} + +response = requests.post('https://api.annex.radiko.jp/radiko.PodcastService/ListPodcastEpisodes', headers=headers, + data=add_grpc_header(protobug.dumps(ListPodcastEpisodesRequest( + channel_id="0ce1d2d7-5e07-4ec5-901a-d0eacdacc332", + dontknow=1, + page_length=200, # site uses 20 +# cursor="ef693874-0ad2-48cc-8c52-ac4de31cbf54" # here you put the id of the last episode you've seen in the list + ))) +) + +print(response) + +episodes = strip_grpc_response(response.content) + + +with open("ListPodcastEpisodes.bin", "wb") as f: + f.write(episodes) + + +@protobug.message +class Audio: + revision: protobug.Int32 = protobug.field(1) + url: protobug.String = protobug.field(2) + fileSize: protobug.Int64 = protobug.field(3) + durationSec: protobug.Int64 = protobug.field(4) + transcoded: protobug.Bool = protobug.field(5) + +@protobug.message +class EpisodeStartAt: + seconds: protobug.UInt64 = protobug.field(1) + nanos: protobug.UInt64 = protobug.field(2, default=0) + + +@protobug.message +class PodcastEpisode: + id: protobug.String = protobug.field(1) + workspaceId: protobug.String = protobug.field(2) + channelId: protobug.String = protobug.field(3) + title: protobug.String = protobug.field(4) + description: protobug.String = protobug.field(5) + + audio: Audio = protobug.field(8) + channelImageUrl: protobug.String = protobug.field(16) + channelTitle: protobug.String = protobug.field(17) + channelStationName: protobug.String = protobug.field(18) + channelAuthor: protobug.String = protobug.field(19) + + channelThumbnailImageUrl: protobug.String = protobug.field(21) + channelStationType: protobug.UInt32 = protobug.field(22) + startAt: EpisodeStartAt = protobug.field(27) + isEnabled: protobug.Bool = protobug.field(29) + hasTranscription: protobug.Bool = protobug.field(32) + + imageUrl: protobug.String = protobug.field(7, default=None) + thumbnailImageUrl: protobug.String = protobug.field(20, default=None) + +@protobug.message +class ListPodcastEpisodesResponse: + episodes: list[PodcastEpisode] = protobug.field(1) + hasNextPage: protobug.Bool = protobug.field(2, default=False) + + +episodes_response = protobug.loads(episodes, ListPodcastEpisodesResponse) + +print(episodes_response) + +for e in episodes_response.episodes: + print(e.title, e.id) +print(episodes_response.hasNextPage) diff --git a/contrib/randominfo.py b/contrib/randominfo.py new file mode 100755 index 0000000..bdb7660 --- /dev/null +++ b/contrib/randominfo.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +from yt_dlp_plugins.extractor import radiko +from yt_dlp import YoutubeDL + + +ie = radiko._RadikoBaseIE() +ydl = YoutubeDL(auto_init=False) +ie.set_downloader(ydl) + +info = ie._generate_random_info() +print("random device info") +print(info) diff --git a/contrib/streammon.py b/contrib/streammon.py new file mode 100755 index 0000000..8f52bb4 --- /dev/null +++ b/contrib/streammon.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# monitor stream APIs for any changes, so I can check they don't break anything +# run via cronjob every now and then + +import difflib +import os +import sys +import xml.etree.ElementTree as ET +from datetime import datetime + +import requests + +s = requests.Session() + +DISCORD_WEBHOOK = "PUT WEBHOOK HERE" +STREAMS_API = "https://radiko.jp/v3/station/stream/{device}/{station}.xml" + +if len(sys.argv) > 1: + PATH = sys.argv[1] +else: + PATH = "" + +devices = ('pc_html5', 'aSmartPhone7a', 'aSmartPhone8') +stations = ('FMT', 'CCL', 'NORTHWAVE', 'TBS') + +def format_xml(txt): + root = ET.fromstring(txt) + res = "" + for el in root.findall("url"): + res += el.find("playlist_create_url").text + for k, v in el.attrib.items(): + res += f" {k}:{v}" + + res += "\n" + return res + +for device in devices: + for station in stations: + url = STREAMS_API.format(device=device, station=station) + now_response = s.get(url) + now = now_response.text + now_modified = now_response.headers["last-modified"] + now_datetime = datetime.strptime(now_modified, "%a, %d %b %Y %H:%M:%S %Z") + + + filename = f"{PATH}{station}-{device}.xml" + with open(filename, "a+") as f: + f.seek(0) + past = f.read() + + modtime = datetime.fromtimestamp(os.path.getmtime(filename)) + diff = difflib.unified_diff( + format_xml(past).splitlines(), format_xml(now).splitlines(), + fromfile=url, tofile=url, + fromfiledate=str(modtime), tofiledate=str(now_datetime.now()), + ) + + diff_str = "\n".join(diff) + if diff_str != "": + f.truncate(0) + f.write(now) + + s.post(DISCORD_WEBHOOK, json={ + "content": f"**Streams changed: {station} {device}**\n" + "\n".join(("```diff", diff_str, "```")), + }) + os.utime(filename, (now_datetime.timestamp(), now_datetime.timestamp())) diff --git a/contrib/test_areas.py b/contrib/test_areas.py new file mode 100755 index 0000000..ba6475f --- /dev/null +++ b/contrib/test_areas.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import unittest + +from yt_dlp_plugins.extractor import radiko +from yt_dlp import YoutubeDL + + +class test_tokens(unittest.TestCase): + + def setUp(self): + self.ie = radiko._RadikoBaseIE() + ydl = YoutubeDL(auto_init=False) + self.ie.set_downloader(ydl) + + def test_area(self): + # check areas etc work + for i in range(1, 48): + area = "JP" + str(i) + with self.subTest(f"Negotiating token for {area}", area=area): + token = self.ie._negotiate_token(area) + self.assertEqual(token.get("X-Radiko-AreaId"), area) + + +if __name__ == '__main__': + unittest.main() + # may wish to set failfast=True diff --git a/contrib/test_extractors.py b/contrib/test_extractors.py new file mode 100755 index 0000000..21800c5 --- /dev/null +++ b/contrib/test_extractors.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +# programmes expire, so i have to update the times in the tests every time i run them +# but thats a massive ballache, so i end up just not running them, which leads to cockups +# so, this script has the tests automatically use the latest episode as you run it, by setting dynamically generated time values +# everything else is always the same so it should be fine lol + + +import datetime +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, "/home/g/Downloads/yt-dlp/") # TODO: un-hardcode. has to be the source/git repo because pip doesnt carry the tests + +from yt_dlp_plugins.extractor import radiko_time as rtime + +MON, TUE, WED, THU, FRI, SAT, SUN = range(7) +weekdays = {0: "MON", 1: "TUE", 2: "WED", 3: "THU", 4: "FRI", 5: "SAT", 6: "SUN"} + +now = rtime.RadikoTime.now(tz = rtime.JST) +UTC = datetime.timezone.utc + +def get_latest_airtimes(now, weekday, hour, minute, duration): + days_after_weekday = (7 - (now.weekday() - weekday)) % 7 + latest_airdate = (now + datetime.timedelta(days=days_after_weekday)).replace(hour=hour, minute=minute, second=0, microsecond=0) + if (latest_airdate + duration) > now: + latest_airdate -= datetime.timedelta(days=7) + return latest_airdate, latest_airdate + duration + +def get_test_timefields(airtime, release_time): + return { + "timestamp": airtime.timestamp(), + "release_timestamp": release_time.timestamp(), + "upload_date": airtime.astimezone(UTC).strftime("%Y%m%d"), + "release_date": release_time.astimezone(UTC).strftime("%Y%m%d"), + + "duration": (release_time - airtime).total_seconds(), + } + + + + +from yt_dlp_plugins.extractor.radiko import ( + RadikoTimeFreeIE, RadikoShareIE, + RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, + RadikoRSeasonsIE +) + +from yt_dlp_plugins.extractor.radiko_podcast import ( + RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, +) +RadikoTimeFreeIE._TESTS = [] + + + +# TOKYO MOON - interfm - EVERY FRI 2300 +airtime, release_time = get_latest_airtimes(now, FRI, 23, 0, datetime.timedelta(hours=1)) +RadikoTimeFreeIE._TESTS.append({ + "url": f"https://radiko.jp/#!/ts/INT/{airtime.timestring()}", + "info_dict": { + "ext": "m4a", + "id": f"INT-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + + 'title': 'TOKYO MOON', + 'description': r're:[\S\s]+Xハッシュタグは「#tokyomoon」$', + 'uploader': 'interfm', + 'uploader_id': 'INT', + 'uploader_url': 'https://www.interfm.co.jp/', + 'channel': 'interfm', + 'channel_id': 'INT', + 'channel_url': 'https://www.interfm.co.jp/', + 'thumbnail': 'https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg', + 'chapters': list, + 'tags': ['松浦俊夫', 'ジャズの魅力を楽しめる'], + 'cast': ['松浦\u3000俊夫'], + 'series': 'Tokyo Moon', + 'live_status': 'was_live', + } +}) + + +# late-night/v. early morning show, to test broadcast day handling +# this should be monday 27:00 / tuesday 03:00 +airtime, release_time = get_latest_airtimes(now, TUE, 3, 0, datetime.timedelta(hours=2)) +RadikoTimeFreeIE._TESTS.append({ + "url": f"https://radiko.jp/#!/ts/TBS/{airtime.timestring()}", + "info_dict": { + "ext": "m4a", + "id": f"TBS-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + 'title': 'CITY CHILL CLUB', + 'description': r"re:^目を閉じて…リラックスして[\S\s]+chill@tbs.co.jp$", + 'uploader': 'TBSラジオ', + 'uploader_id': 'TBS', + 'uploader_url': 'https://www.tbsradio.jp/', + 'channel': 'TBSラジオ', + 'channel_id': 'TBS', + 'channel_url': 'https://www.tbsradio.jp/', + 'thumbnail': 'https://program-static.cf.radiko.jp/nrf8fowbjo.jpg', + 'chapters': list, + 'tags': ['CCC905', '音楽との出会いが楽しめる', '人気アーティストトーク', '音楽プロデューサー出演', 'ドライブ中におすすめ', '寝る前におすすめ', '学生におすすめ'], + 'cast': list, + 'series': 'CITY CHILL CLUB', + 'live_status': 'was_live', + }, +}) + + +# testing 29-hour clock handling +airtime, release_time = get_latest_airtimes(now, WED, 0, 0, datetime.timedelta(minutes=55)) +share_timestring = (airtime - datetime.timedelta(days=1)).strftime("%Y%m%d") + "240000" + +RadikoShareIE._TESTS = [{ + "url": f"http://radiko.jp/share/?sid=FMT&t={share_timestring}", + "info_dict": { + "live_status": "was_live", + "ext": "m4a", + "id": f"FMT-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + + "title": "JET STREAM", + "series": "JET STREAM", + "description": r"re:^JET STREAM・・・[\s\S]+https://www.tfm.co.jp/f/jetstream/message$", + "chapters": list, + "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg", + + "channel": "TOKYO FM", + "channel_id": "FMT", + "channel_url": "https://www.tfm.co.jp/", + "uploader": "TOKYO FM", + "uploader_id": "FMT", + "uploader_url": "https://www.tfm.co.jp/", + + "cast": ["福山雅治"], + "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"], + }, + }] + + + +IEs = [ + RadikoTimeFreeIE, RadikoShareIE, + RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, + RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, + RadikoRSeasonsIE, +] + +import test.helper as th + +# override to only get testcases from our IEs + +def _new_gettestcases(include_onlymatching=False): + import yt_dlp.plugins as plugins + plugins.load_all_plugins() + + for ie in IEs: + yield from ie.get_testcases(include_onlymatching) + +def _new_getwebpagetestcases(): + import yt_dlp.plugins as plugins + plugins.load_all_plugins() + + for ie in IEs: + for tc in ie.get_webpage_testcases(): + tc.setdefault('add_ie', []).append('Generic') + yield tc + +th.gettestcases = _new_gettestcases +th.getwebpagetestcases = _new_getwebpagetestcases + +import test.test_download as td + +class TestDownload(td.TestDownload): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/misc/generate_html.py b/misc/generate_html.py deleted file mode 100755 index 0e15d6a..0000000 --- a/misc/generate_html.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -import os -import hashlib -import re - -pip_index = open("index.html", "w") - -pip_index.write(""" - - - yt-dlp-rajiko pip index - - - - -
    -""") - -site_sha256 = [] - -tarballs = [] -wheels = [] - -for item in sorted(os.listdir()):#, key=lambda x: x.name): - if os.path.islink(item): - continue - - if item.endswith(".tar.gz"): - tarballs.append(item) - elif item.endswith(".whl"): - wheels.append(item) - else: - continue - - pip_index.write("\t
  • ") - pip_index.write('') - pip_index.write(item) - pip_index.write("\n") - - site_string = checksum + "  " + '' + item + "
    " - site_sha256.append(site_string) - -pip_index.write("""
- - -""") - -latest_tarball = tarballs[-1] -latest_wheel = wheels[-1] -print(latest_tarball, latest_wheel) - -os.remove("yt_dlp_rajiko-latest.tar.gz") -os.symlink(latest_tarball, "yt_dlp_rajiko-latest.tar.gz") - -os.remove("yt_dlp_rajiko-latest.whl") -os.symlink(latest_wheel, "yt_dlp_rajiko-latest.whl") - -site_sha256.reverse() - -latest_list = site_sha256[:2] -previous_list = site_sha256[2:] - -latest = "\n".join(["", "", "\n".join(latest_list), "", ""]) - -previous = "\n".join(["", "", "\n".join(previous_list), "", ""]) - -for i in ["../../index.html", "../../index.ja.html"]: - with open(i, "r+") as f: - page = f.read() - - page = re.sub(r".+", latest, page, flags=re.DOTALL) - page = re.sub(r".+", previous, page, flags=re.DOTALL) - - f.seek(0) - f.truncate(0) - f.write(page) diff --git a/misc/how to do a release b/misc/how to do a release deleted file mode 100644 index 6e91e14..0000000 --- a/misc/how to do a release +++ /dev/null @@ -1,41 +0,0 @@ -putting this here because i'll forget how to do it otherwise - -update the pyproject.toml -tag it in git, eg v1.0 - -## build the builds -python3 -m build - -and then put BOTH items from `dist` into the pip index dir - ~/site2/yt-dlp-rajiko/pip/yt-dlp-rajiko/ -because without the .whl pip has to "build" it itself, with all the stuff that needs to be installed for that to work - -run script to update the pip index html and the dl/ "latest" symlinks -this also updates the sha256s on the site - -## update the changelog file - -write in html, paste into the feed xml like -make sure to set the link, date -to get date use: -git log --pretty --date=rfc2822 - -include the pip instructions, sha256sum etc - -now push to the server - -!!NEW!! -upload to pip proper as well -go to dl/ dir and do -twine upload yt_dlp_rajiko-1.x* - - -## update github - -paste the changelog output into a github release, upload the new builds -change link at the bottom to just "below" - -post in the radiko thread on 5ch if i can be bothered - -and thats probably all diff --git a/misc/old_generate_changelog.py b/misc/old_generate_changelog.py deleted file mode 100755 index 1bce073..0000000 --- a/misc/old_generate_changelog.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -import email.utils -import feedgenerator - -def parse_changelog(lines): - got_version = False - got_date = False - got_url = False - done_remarks = False - releases = [] - release = {} - release_remarks = [] - release_changes = [] - current_change = "" - - for idx, line in enumerate(lines): - line = line.rstrip() - - if not got_version: - got_version = True - release["version"] = line - continue - - if not got_date: - release["date"] = email.utils.parsedate_to_datetime(line) - got_date = True - continue - - key, sep, val = line.partition(": ") - if key in ["url", "sha256", "released"] and val != "": - release[key] = val - continue - - if not done_remarks: - if line == "": - done_remarks = True - release["remarks"] = release_remarks - release_remarks = [] - continue - else: - release_remarks.append(line) - continue - - if line != "": - release_changes.append(line.rstrip()) - - if idx + 1 != len(lines): - continue - - release["changes"] = release_changes - if release.get("released") != "no": - releases.append(release) - - got_version = False - got_date = False - done_remarks = False - release = {} - release_changes = [] - - return releases - -def generate_rss_feed(releases): - feed = feedgenerator.Rss201rev2Feed( - title="yt-dlp-rajiko changelog", - description="Notifications for new yt-dlp-rajiko releases, with changelogs", - link="https://427738.xyz/yt-dlp-rajiko/", - language="en-GB", - ttl=180, # 3 hours - ) - - for release in releases: - title = "yt-dlp-rajiko " + release["version"] + " has been released" - description = "" - description += "

" - for remark in release["remarks"]: - description += remark - description += "
" - description += "

" - description += "

This release:

\n" - description += "
    " - for change in release["changes"]: - description += "
  • " - description += change - description += "
    • \n" - description += "

" - - if release.get("url"): - if release["version"] != "1.0": - description += "\n

If you use pip, you should be able to upgrade with pip install yt-dlp-rajiko --upgrade --extra-index-url https://427738.xyz/yt-dlp-rajiko/pip/.
" - description += "If you installed manually, you can download the updated .whl from this post's link." - if release.get("sha256"): - description += " The SHA256 checksum should be " - description += release.get("sha256") - description += "." - description += "

" - else: - description += '\n

Please see the homepage for initial installation instructions.

' - - feed.add_item( - title=title, - description=description, - link=release.get("url"), - pubdate=release["date"] - ) - return feed - -if __name__ == "__main__": - with open("CHANGELOG") as f: - releases = parse_changelog(f.readlines()) - - feed = generate_rss_feed(releases) - feed_contents = feed.writeString("utf-8") - feed_contents = feed_contents.replace("\nBI', compression_flag, message_length) - return header + protobuf_data - -def strip_grpc_response(response): - return response[5:].rpartition(b"grpc-status:")[0] - -print("SIGNUP") -# why do they have to make it so bloody complicated - -lsid = ''.join(random.choices('0123456789abcdef', k=32)) -big_funny = ("\n " + lsid).encode() - -signup = requests.post("https://api.annex.radiko.jp/radiko.UserService/SignUp", headers={ - 'Origin': 'https://radiko.jp', - 'Content-Type': 'application/grpc-web+proto', - 'X-User-Agent': 'grpc-web-javascript/0.1', - 'X-Grpc-Web': '1', - }, data=( add_grpc_header(big_funny)), -) - -print(signup.content) - -# youre meant to only do the sign up ^ once and then keep your id for later -# so that you can V sign in and get the token for the API to work - -print("SIGNIN") - -si=add_grpc_header(protobug.dumps(SignInRequest( - lsid=lsid, - area="JP13", -))) - -print(si) -print(base64.b64encode(si)) - -signin = requests.post("https://api.annex.radiko.jp/radiko.UserService/SignIn", headers={ - 'Origin': 'https://radiko.jp', - 'Content-Type': 'application/grpc-web+proto', - 'X-User-Agent': 'grpc-web-javascript/0.1', - 'X-Grpc-Web': '1', -}, data=si) - -print(signin.content) - -signin_result = protobug.loads(strip_grpc_response(signin.content), SignInResponse) - - -headers = { - 'Origin': 'https://radiko.jp', - 'Authorization': f'Bearer {signin_result.jwt}', - 'x-annex-proto-version': '1.0.0', - 'Content-Type': 'application/grpc-web+proto', - 'X-User-Agent': 'grpc-web-javascript/0.1', - 'X-Grpc-Web': '1', -} - -response = requests.post('https://api.annex.radiko.jp/radiko.PodcastService/ListPodcastEpisodes', headers=headers, - data=add_grpc_header(protobug.dumps(ListPodcastEpisodesRequest( - channel_id="0ce1d2d7-5e07-4ec5-901a-d0eacdacc332", - dontknow=1, - page_length=200, # site uses 20 -# cursor="ef693874-0ad2-48cc-8c52-ac4de31cbf54" # here you put the id of the last episode you've seen in the list - ))) -) - -print(response) - -episodes = strip_grpc_response(response.content) - - -with open("ListPodcastEpisodes.bin", "wb") as f: - f.write(episodes) - - -@protobug.message -class Audio: - revision: protobug.Int32 = protobug.field(1) - url: protobug.String = protobug.field(2) - fileSize: protobug.Int64 = protobug.field(3) - durationSec: protobug.Int64 = protobug.field(4) - transcoded: protobug.Bool = protobug.field(5) - -@protobug.message -class EpisodeStartAt: - seconds: protobug.UInt64 = protobug.field(1) - nanos: protobug.UInt64 = protobug.field(2, default=0) - - -@protobug.message -class PodcastEpisode: - id: protobug.String = protobug.field(1) - workspaceId: protobug.String = protobug.field(2) - channelId: protobug.String = protobug.field(3) - title: protobug.String = protobug.field(4) - description: protobug.String = protobug.field(5) - - audio: Audio = protobug.field(8) - channelImageUrl: protobug.String = protobug.field(16) - channelTitle: protobug.String = protobug.field(17) - channelStationName: protobug.String = protobug.field(18) - channelAuthor: protobug.String = protobug.field(19) - - channelThumbnailImageUrl: protobug.String = protobug.field(21) - channelStationType: protobug.UInt32 = protobug.field(22) - startAt: EpisodeStartAt = protobug.field(27) - isEnabled: protobug.Bool = protobug.field(29) - hasTranscription: protobug.Bool = protobug.field(32) - - imageUrl: protobug.String = protobug.field(7, default=None) - thumbnailImageUrl: protobug.String = protobug.field(20, default=None) - -@protobug.message -class ListPodcastEpisodesResponse: - episodes: list[PodcastEpisode] = protobug.field(1) - hasNextPage: protobug.Bool = protobug.field(2, default=False) - - -episodes_response = protobug.loads(episodes, ListPodcastEpisodesResponse) - -print(episodes_response) - -for e in episodes_response.episodes: - print(e.title, e.id) -print(episodes_response.hasNextPage) diff --git a/misc/randominfo.py b/misc/randominfo.py deleted file mode 100755 index bdb7660..0000000 --- a/misc/randominfo.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -from yt_dlp_plugins.extractor import radiko -from yt_dlp import YoutubeDL - - -ie = radiko._RadikoBaseIE() -ydl = YoutubeDL(auto_init=False) -ie.set_downloader(ydl) - -info = ie._generate_random_info() -print("random device info") -print(info) diff --git a/misc/streammon.py b/misc/streammon.py deleted file mode 100755 index 8f52bb4..0000000 --- a/misc/streammon.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# monitor stream APIs for any changes, so I can check they don't break anything -# run via cronjob every now and then - -import difflib -import os -import sys -import xml.etree.ElementTree as ET -from datetime import datetime - -import requests - -s = requests.Session() - -DISCORD_WEBHOOK = "PUT WEBHOOK HERE" -STREAMS_API = "https://radiko.jp/v3/station/stream/{device}/{station}.xml" - -if len(sys.argv) > 1: - PATH = sys.argv[1] -else: - PATH = "" - -devices = ('pc_html5', 'aSmartPhone7a', 'aSmartPhone8') -stations = ('FMT', 'CCL', 'NORTHWAVE', 'TBS') - -def format_xml(txt): - root = ET.fromstring(txt) - res = "" - for el in root.findall("url"): - res += el.find("playlist_create_url").text - for k, v in el.attrib.items(): - res += f" {k}:{v}" - - res += "\n" - return res - -for device in devices: - for station in stations: - url = STREAMS_API.format(device=device, station=station) - now_response = s.get(url) - now = now_response.text - now_modified = now_response.headers["last-modified"] - now_datetime = datetime.strptime(now_modified, "%a, %d %b %Y %H:%M:%S %Z") - - - filename = f"{PATH}{station}-{device}.xml" - with open(filename, "a+") as f: - f.seek(0) - past = f.read() - - modtime = datetime.fromtimestamp(os.path.getmtime(filename)) - diff = difflib.unified_diff( - format_xml(past).splitlines(), format_xml(now).splitlines(), - fromfile=url, tofile=url, - fromfiledate=str(modtime), tofiledate=str(now_datetime.now()), - ) - - diff_str = "\n".join(diff) - if diff_str != "": - f.truncate(0) - f.write(now) - - s.post(DISCORD_WEBHOOK, json={ - "content": f"**Streams changed: {station} {device}**\n" + "\n".join(("```diff", diff_str, "```")), - }) - os.utime(filename, (now_datetime.timestamp(), now_datetime.timestamp())) diff --git a/misc/test_areas.py b/misc/test_areas.py deleted file mode 100755 index ba6475f..0000000 --- a/misc/test_areas.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -import unittest - -from yt_dlp_plugins.extractor import radiko -from yt_dlp import YoutubeDL - - -class test_tokens(unittest.TestCase): - - def setUp(self): - self.ie = radiko._RadikoBaseIE() - ydl = YoutubeDL(auto_init=False) - self.ie.set_downloader(ydl) - - def test_area(self): - # check areas etc work - for i in range(1, 48): - area = "JP" + str(i) - with self.subTest(f"Negotiating token for {area}", area=area): - token = self.ie._negotiate_token(area) - self.assertEqual(token.get("X-Radiko-AreaId"), area) - - -if __name__ == '__main__': - unittest.main() - # may wish to set failfast=True diff --git a/misc/test_extractors.py b/misc/test_extractors.py deleted file mode 100755 index 21800c5..0000000 --- a/misc/test_extractors.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 - -# programmes expire, so i have to update the times in the tests every time i run them -# but thats a massive ballache, so i end up just not running them, which leads to cockups -# so, this script has the tests automatically use the latest episode as you run it, by setting dynamically generated time values -# everything else is always the same so it should be fine lol - - -import datetime -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -sys.path.insert(0, "/home/g/Downloads/yt-dlp/") # TODO: un-hardcode. has to be the source/git repo because pip doesnt carry the tests - -from yt_dlp_plugins.extractor import radiko_time as rtime - -MON, TUE, WED, THU, FRI, SAT, SUN = range(7) -weekdays = {0: "MON", 1: "TUE", 2: "WED", 3: "THU", 4: "FRI", 5: "SAT", 6: "SUN"} - -now = rtime.RadikoTime.now(tz = rtime.JST) -UTC = datetime.timezone.utc - -def get_latest_airtimes(now, weekday, hour, minute, duration): - days_after_weekday = (7 - (now.weekday() - weekday)) % 7 - latest_airdate = (now + datetime.timedelta(days=days_after_weekday)).replace(hour=hour, minute=minute, second=0, microsecond=0) - if (latest_airdate + duration) > now: - latest_airdate -= datetime.timedelta(days=7) - return latest_airdate, latest_airdate + duration - -def get_test_timefields(airtime, release_time): - return { - "timestamp": airtime.timestamp(), - "release_timestamp": release_time.timestamp(), - "upload_date": airtime.astimezone(UTC).strftime("%Y%m%d"), - "release_date": release_time.astimezone(UTC).strftime("%Y%m%d"), - - "duration": (release_time - airtime).total_seconds(), - } - - - - -from yt_dlp_plugins.extractor.radiko import ( - RadikoTimeFreeIE, RadikoShareIE, - RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, - RadikoRSeasonsIE -) - -from yt_dlp_plugins.extractor.radiko_podcast import ( - RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, -) -RadikoTimeFreeIE._TESTS = [] - - - -# TOKYO MOON - interfm - EVERY FRI 2300 -airtime, release_time = get_latest_airtimes(now, FRI, 23, 0, datetime.timedelta(hours=1)) -RadikoTimeFreeIE._TESTS.append({ - "url": f"https://radiko.jp/#!/ts/INT/{airtime.timestring()}", - "info_dict": { - "ext": "m4a", - "id": f"INT-{airtime.timestring()}", - - **get_test_timefields(airtime, release_time), - - 'title': 'TOKYO MOON', - 'description': r're:[\S\s]+Xハッシュタグは「#tokyomoon」$', - 'uploader': 'interfm', - 'uploader_id': 'INT', - 'uploader_url': 'https://www.interfm.co.jp/', - 'channel': 'interfm', - 'channel_id': 'INT', - 'channel_url': 'https://www.interfm.co.jp/', - 'thumbnail': 'https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg', - 'chapters': list, - 'tags': ['松浦俊夫', 'ジャズの魅力を楽しめる'], - 'cast': ['松浦\u3000俊夫'], - 'series': 'Tokyo Moon', - 'live_status': 'was_live', - } -}) - - -# late-night/v. early morning show, to test broadcast day handling -# this should be monday 27:00 / tuesday 03:00 -airtime, release_time = get_latest_airtimes(now, TUE, 3, 0, datetime.timedelta(hours=2)) -RadikoTimeFreeIE._TESTS.append({ - "url": f"https://radiko.jp/#!/ts/TBS/{airtime.timestring()}", - "info_dict": { - "ext": "m4a", - "id": f"TBS-{airtime.timestring()}", - - **get_test_timefields(airtime, release_time), - 'title': 'CITY CHILL CLUB', - 'description': r"re:^目を閉じて…リラックスして[\S\s]+chill@tbs.co.jp$", - 'uploader': 'TBSラジオ', - 'uploader_id': 'TBS', - 'uploader_url': 'https://www.tbsradio.jp/', - 'channel': 'TBSラジオ', - 'channel_id': 'TBS', - 'channel_url': 'https://www.tbsradio.jp/', - 'thumbnail': 'https://program-static.cf.radiko.jp/nrf8fowbjo.jpg', - 'chapters': list, - 'tags': ['CCC905', '音楽との出会いが楽しめる', '人気アーティストトーク', '音楽プロデューサー出演', 'ドライブ中におすすめ', '寝る前におすすめ', '学生におすすめ'], - 'cast': list, - 'series': 'CITY CHILL CLUB', - 'live_status': 'was_live', - }, -}) - - -# testing 29-hour clock handling -airtime, release_time = get_latest_airtimes(now, WED, 0, 0, datetime.timedelta(minutes=55)) -share_timestring = (airtime - datetime.timedelta(days=1)).strftime("%Y%m%d") + "240000" - -RadikoShareIE._TESTS = [{ - "url": f"http://radiko.jp/share/?sid=FMT&t={share_timestring}", - "info_dict": { - "live_status": "was_live", - "ext": "m4a", - "id": f"FMT-{airtime.timestring()}", - - **get_test_timefields(airtime, release_time), - - "title": "JET STREAM", - "series": "JET STREAM", - "description": r"re:^JET STREAM・・・[\s\S]+https://www.tfm.co.jp/f/jetstream/message$", - "chapters": list, - "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg", - - "channel": "TOKYO FM", - "channel_id": "FMT", - "channel_url": "https://www.tfm.co.jp/", - "uploader": "TOKYO FM", - "uploader_id": "FMT", - "uploader_url": "https://www.tfm.co.jp/", - - "cast": ["福山雅治"], - "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"], - }, - }] - - - -IEs = [ - RadikoTimeFreeIE, RadikoShareIE, - RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, - RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, - RadikoRSeasonsIE, -] - -import test.helper as th - -# override to only get testcases from our IEs - -def _new_gettestcases(include_onlymatching=False): - import yt_dlp.plugins as plugins - plugins.load_all_plugins() - - for ie in IEs: - yield from ie.get_testcases(include_onlymatching) - -def _new_getwebpagetestcases(): - import yt_dlp.plugins as plugins - plugins.load_all_plugins() - - for ie in IEs: - for tc in ie.get_webpage_testcases(): - tc.setdefault('add_ie', []).append('Generic') - yield tc - -th.gettestcases = _new_gettestcases -th.getwebpagetestcases = _new_getwebpagetestcases - -import test.test_download as td - -class TestDownload(td.TestDownload): - pass - -if __name__ == "__main__": - unittest.main() diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 0566b6d..5bda1de 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -437,7 +437,7 @@ class RadikoLiveIE(_RadikoBaseIE): class RadikoTimeFreeIE(_RadikoBaseIE): _NETRC_MACHINE = "rajiko" _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/ts/(?P[A-Z0-9-_]+)/(?P\d+)" - # TESTS use a custom-ish script that updates the airdates automatically, see misc/test_extractors.py + # TESTS use a custom-ish script that updates the airdates automatically, see contrib/test_extractors.py def _perform_login(self, username, password): try: -- cgit v1.2.3-70-g09d2 From bcb5df38d71f2b6d2092797201ad62638b6d3ef0 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 14 Sep 2025 15:17:26 +0100 Subject: disable stream blacklist for live ref https://github.com/garret1317/yt-dlp-rajiko/issues/29#issuecomment-3289577318 github issue #29 --- yt_dlp_plugins/extractor/radiko.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 5bda1de..ad5c77f 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -302,7 +302,9 @@ class _RadikoBaseIE(InfoExtractor): entry_protocol = 'm3u8' format_note=[] - if domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams: + if timefree and domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams: + # TODO: remove this completely + # https://github.com/garret1317/yt-dlp-rajiko/issues/29 self.write_debug(f"skipping {domain} (known not working)") continue if domain in self._DELIVERED_ONDEMAND: -- cgit v1.2.3-70-g09d2 From e1c74fc8eb027d40acc9d5114f352670b435c23b Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 14 Sep 2025 17:12:00 +0100 Subject: clean up stray imports in protobufs file --- yt_dlp_plugins/extractor/radiko_protobufs.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_protobufs.py b/yt_dlp_plugins/extractor/radiko_protobufs.py index ff4531e..4eb4f8b 100755 --- a/yt_dlp_plugins/extractor/radiko_protobufs.py +++ b/yt_dlp_plugins/extractor/radiko_protobufs.py @@ -4,11 +4,8 @@ try: except ImportError: protobug = None -import base64 import struct - import random -import requests if protobug: # i suppose it works lmao -- cgit v1.2.3-70-g09d2 From 8337d2b164759181777f64f12b985e4fad769ab7 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 14 Sep 2025 17:28:08 +0100 Subject: Add support for bundled protobug library github: closes #29 --- yt_dlp_plugins/extractor/radiko_dependencies.py | 29 +++++++++++++++++++++++++ yt_dlp_plugins/extractor/radiko_podcast.py | 7 +++--- yt_dlp_plugins/extractor/radiko_protobufs.py | 7 ++---- 3 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 yt_dlp_plugins/extractor/radiko_dependencies.py (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_dependencies.py b/yt_dlp_plugins/extractor/radiko_dependencies.py new file mode 100644 index 0000000..769a5e3 --- /dev/null +++ b/yt_dlp_plugins/extractor/radiko_dependencies.py @@ -0,0 +1,29 @@ +# Bundle importing code Copyright (c) 2021-2022 Grub4K, from yt-dont-lock-p. +# https://github.com/Grub4K/yt-dont-lock-p/blob/ff3b6e1d42ce8584153ae27544d2c05b50ab5954/yt_dlp_plugins/postprocessor/yt_dont_lock_p/__init__.py#L23-L46 +# Used under 0BSD with permission + +# https://discord.com/channels/807245652072857610/1112613156934668338/1416816007732920430 (yt-dlp discord server, https://discord.gg/H5MNcFW63r ) +# [17:00] garret1317: @Grub4K can i pinch your MIT-licensed dependency bundling code to use in my 0BSD-licensed plugin? +# I will credit of course but i can't require that anyone else does the same +# (Any response to this message will be considered a written consent or refusal of the request) +# [17:04] Grub4K: Feel free to use that part under 0BSD +# [17:05] garret1317: 👍 cheers + +try: + import protobug +except ImportError: + import sys + from pathlib import Path + + # Try importing from zip file bundle + search_path = str(Path(__file__).parent.parent) + sys.path.append(search_path) + try: + import protobug + except ImportError: + protobug = None + except Exception: + protobug = None + + finally: + sys.path.remove(search_path) diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 84bc288..a984be3 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -7,11 +7,10 @@ from yt_dlp.utils import ( ) import dataclasses -try: - import protobug + +from yt_dlp_plugins.extractor.radiko_dependencies import protobug +if protobug: import yt_dlp_plugins.extractor.radiko_protobufs as pb -except ImportError: - protobug = None class _RadikoPodcastBaseIE(InfoExtractor): diff --git a/yt_dlp_plugins/extractor/radiko_protobufs.py b/yt_dlp_plugins/extractor/radiko_protobufs.py index 4eb4f8b..a8bbec1 100755 --- a/yt_dlp_plugins/extractor/radiko_protobufs.py +++ b/yt_dlp_plugins/extractor/radiko_protobufs.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 -try: - import protobug -except ImportError: - protobug = None - import struct import random +from yt_dlp_plugins.extractor.radiko_dependencies import protobug + if protobug: # i suppose it works lmao -- cgit v1.2.3-70-g09d2 From 68660d1dfe550be845033cd0301cd2e46144afdd Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 14 Sep 2025 17:33:57 +0100 Subject: Add instructions for obtaining protobug in the warning message --- yt_dlp_plugins/extractor/radiko_podcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index a984be3..10579b5 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -109,7 +109,7 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): cursor = episode.id yield self._extract_episode(dataclasses.asdict(episode)) else: - self.report_warning(f'Only extracting the latest {len(episode_list_response["episodesList"])} episodes. Install protobug for more.') + self.report_warning(f'protobug is required to extract more than the latest {len(episode_list_response["episodesList"])} episodes.\nIf you installed yt-dlp-rajiko manually, use the .zip bundle instead. If you installed with pip, install protobug as well.') return { "_type": "playlist", -- cgit v1.2.3-70-g09d2 From 25d4c20be556825a5b3e2b6a66c719ad2c5d8c1f Mon Sep 17 00:00:00 2001 From: garret1317 Date: Thu, 18 Sep 2025 23:11:24 +0100 Subject: clarify protobug error message --- yt_dlp_plugins/extractor/radiko_podcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 10579b5..21e3dfd 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -109,7 +109,7 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): cursor = episode.id yield self._extract_episode(dataclasses.asdict(episode)) else: - self.report_warning(f'protobug is required to extract more than the latest {len(episode_list_response["episodesList"])} episodes.\nIf you installed yt-dlp-rajiko manually, use the .zip bundle instead. If you installed with pip, install protobug as well.') + self.report_warning(f'protobug is required to extract more than the latest {len(episode_list_response["episodesList"])} episodes.\nIf you installed yt-dlp-rajiko manually (with the .whl), use the .zip bundle instead. If you installed with pip, pip install protobug .') return { "_type": "playlist", -- cgit v1.2.3-70-g09d2 From ca5cf0703d7a3171c7c9e5b1a9c89a96e0b8d6e8 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 00:48:09 +0100 Subject: add podcast search IE --- yt_dlp_plugins/extractor/radiko_podcast.py | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 21e3dfd..67d6475 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -1,12 +1,16 @@ from yt_dlp.extractor.common import InfoExtractor from yt_dlp.utils import ( clean_html, + OnDemandPagedList, + parse_qs, traverse_obj, + update_url_query, url_or_none, str_or_none, ) import dataclasses +import random from yt_dlp_plugins.extractor.radiko_dependencies import protobug if protobug: @@ -123,3 +127,40 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): }), "entries": entries(), } + + +class RadikoPodcastSearchIE(InfoExtractor): + _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/search/podcast/(?:timeshift|live)\?" + + def _pagefunc(self, url, idx): + url = update_url_query(url, {"pageIdx": idx}) + data = self._download_json(url, None, note=f"Downloading page {idx+1}") + + results = [] + for channel in data.get("channels"): + results.append( + self.url_result( + channel.get("channelUrl"), + id=channel.get("id"), + ie=RadikoPodcastChannelIE, + ) + ) + return results + + + def _real_extract(self, url): + # hack away the # so urllib.parse will work (same as normal RadikoSearchIE) + url = url.replace("/#!/", "/!/", 1) + queries = parse_qs(url) + + keywords = traverse_obj(queries, ("key", 0)) + search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/podcasts/channels/search_with_keywords_by_offset", { + "keywords": keywords, + "uid": "".join(random.choices("0123456789abcdef", k=32)), + "limit": 50, # result limit. the actual limit before the api errors is 5000, but that seems a bit rude so i'll leave as 50 like the radio one + }) + + return self.playlist_result( + OnDemandPagedList(lambda idx: self._pagefunc(search_url, idx), 50), + title=keywords, + ) -- cgit v1.2.3-70-g09d2 From d9c78c18dbc3f6afc3994cd0060ab7b2f55d3af0 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 01:22:49 +0100 Subject: Fix search filtering + add support for podcast filtering --- yt_dlp_plugins/extractor/radiko.py | 44 +++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index ad5c77f..6654b57 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -19,6 +19,7 @@ from yt_dlp.utils import ( url_or_none, update_url_query, ) +from yt_dlp_plugins.extractor.radiko_podcast import RadikoPodcastSearchIE import yt_dlp_plugins.extractor.radiko_time as rtime import yt_dlp_plugins.extractor.radiko_hacks as hacks @@ -566,7 +567,7 @@ class RadikoTimeFreeIE(_RadikoBaseIE): class RadikoSearchIE(InfoExtractor): - _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/search/(?:timeshift|live|history)\?" + _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/search/(?:radio/)?(?:timeshift|live|history)\?" _TESTS = [{ # timefree, specific area "url": "https://radiko.jp/#!/search/live?key=city%20chill%20club&filter=past&start_day=&end_day=®ion_id=&area_id=JP13&cul_area_id=JP13&page_idx=0", @@ -609,23 +610,50 @@ class RadikoSearchIE(InfoExtractor): results.append( self.url_result( f"https://radiko.jp/#!/ts/{station}/{timestring}", - id=join_nonempty(station, timestring) + id=join_nonempty(station, timestring), + ie=RadikoTimeFreeIE, ) ) return results def _real_extract(self, url): - url = url.replace("/#!/", "/!/", 1) # urllib.parse interprets the path as just one giant fragment because of the #, so we hack it away + url = url.replace("/#!/", "/!/", 1) queries = parse_qs(url) + key = traverse_obj(queries, ("key", 0)) - if queries.get("cul_area_id"): - queries["cur_area_id"] = queries.pop("cul_area_id") # site used to use "cul_area_id" in the search url, now it uses "cur_area_id" (with an r) # and outright rejects the old one with HTTP Error 415: Unsupported Media Type + if queries.get("cul_area_id"): + queries["cur_area_id"] = queries.pop("cul_area_id") + + filter_str = "" + if queries.get("filter"): + filter_set = set(queries["filter"][0].split("|")) + del queries["filter"] + + if filter_set == {"channel"}: + podcast_search_url = update_url_query( + "https://radiko.jp/!/search/podcast/live", {"key": key} + ).replace("!", "#!", 1) # same shit with urllib.parse + return self.url_result(podcast_search_url, ie=RadikoPodcastSearchIE) + + if "channel" in filter_set: + self.report_warning("Skipping podcasts. If you really want EVERY EPISODE of EVERY RESULT, set your search filter to Podcasts only.") + + filter_set.discard("channel") + if filter_set == {"future", "past"}: + filter_str = "" + else: + filter_str = "|".join(filter_set) # there should be only one filter now, so this should be the same as filter_set[0] + # but if there's more than one, then we should at least try to pass it through as-is, in the hope that it works + if len(filter_set) != 1: + # but also kick up a stink about it so it's clear it probably won't + self.report_warning("Your search has an unknkown combination of filters, so this request will probably fail!") search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/programs/legacy/perl/program/search", { **queries, + "filter": filter_str, "uid": "".join(random.choices("0123456789abcdef", k=32)), "app_id": "pc", "row_limit": 50, # higher row_limit = more results = less requests = more good @@ -633,20 +661,20 @@ class RadikoSearchIE(InfoExtractor): results = OnDemandPagedList(lambda idx: self._pagefunc(search_url, idx), 50) - key = traverse_obj(queries, ("key", 0)) day = traverse_obj(queries, ("start_day", 0)) or "all" region = traverse_obj(queries, ("region_id", 0)) or traverse_obj(queries, ("area_id", 0)) - status_filter = traverse_obj(queries, ("filter", 0)) or "all" + status_filter = filter_str or "all" playlist_id = join_nonempty(key, status_filter, day, region) return { "_type": "playlist", - "title": traverse_obj(queries, ("key", 0)), + "title": key, "id": playlist_id, "entries": results, } + class RadikoShareIE(InfoExtractor): _VALID_URL = r"https?://(?:www\.)?radiko\.jp/share/" -- cgit v1.2.3-70-g09d2 From b8807109ba7e4515ffc5d6d3d0fcf8888acd13a3 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 01:28:17 +0100 Subject: Add PodcastSearch test --- contrib/test_extractors.py | 4 ++-- yt_dlp_plugins/extractor/radiko_podcast.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/contrib/test_extractors.py b/contrib/test_extractors.py index 1ef63d0..0b505b8 100755 --- a/contrib/test_extractors.py +++ b/contrib/test_extractors.py @@ -49,7 +49,7 @@ from yt_dlp_plugins.extractor.radiko import ( ) from yt_dlp_plugins.extractor.radiko_podcast import ( - RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, + RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, RadikoPodcastSearchIE, ) RadikoTimeFreeIE._TESTS = [] @@ -148,7 +148,7 @@ IEs = [ RadikoTimeFreeIE, RadikoShareIE, RadikoLiveIE, RadikoPersonIE, RadikoStationButtonIE, RadikoPodcastEpisodeIE, RadikoPodcastChannelIE, - RadikoSearchIE, RadikoRSeasonsIE, + RadikoSearchIE, RadikoPodcastSearchIE, RadikoRSeasonsIE, ] import test.helper as th diff --git a/yt_dlp_plugins/extractor/radiko_podcast.py b/yt_dlp_plugins/extractor/radiko_podcast.py index 67d6475..27b91ad 100644 --- a/yt_dlp_plugins/extractor/radiko_podcast.py +++ b/yt_dlp_plugins/extractor/radiko_podcast.py @@ -131,6 +131,14 @@ class RadikoPodcastChannelIE(_RadikoPodcastBaseIE): class RadikoPodcastSearchIE(InfoExtractor): _VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/search/podcast/(?:timeshift|live)\?" + _TESTS = [{ + "url": "https://radiko.jp/#!/search/podcast/live?key=ドラマ", + "playlist_mincount": 51, + "info_dict": { + "id": "ドラマ", + "title": "ドラマ", + }, + }] def _pagefunc(self, url, idx): url = update_url_query(url, {"pageIdx": idx}) @@ -163,4 +171,5 @@ class RadikoPodcastSearchIE(InfoExtractor): return self.playlist_result( OnDemandPagedList(lambda idx: self._pagefunc(search_url, idx), 50), title=keywords, + id=keywords, # i have to put some kind of id or the tests fail ) -- cgit v1.2.3-70-g09d2 From e98426f316bf9d60d6e131a9163c8b872d663a92 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 11:18:02 +0100 Subject: oh, absence of a filter means every option doesnt it --- yt_dlp_plugins/extractor/radiko.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 6654b57..ec260ab 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -627,29 +627,30 @@ class RadikoSearchIE(InfoExtractor): if queries.get("cul_area_id"): queries["cur_area_id"] = queries.pop("cul_area_id") - filter_str = "" if queries.get("filter"): filter_set = set(queries["filter"][0].split("|")) del queries["filter"] + else: + filter_set = {"future", "past", "channel"} - if filter_set == {"channel"}: - podcast_search_url = update_url_query( - "https://radiko.jp/!/search/podcast/live", {"key": key} - ).replace("!", "#!", 1) # same shit with urllib.parse - return self.url_result(podcast_search_url, ie=RadikoPodcastSearchIE) + if filter_set == {"channel"}: + podcast_search_url = update_url_query( + "https://radiko.jp/!/search/podcast/live", {"key": key} + ).replace("!", "#!", 1) # same shit with urllib.parse + return self.url_result(podcast_search_url, ie=RadikoPodcastSearchIE) - if "channel" in filter_set: - self.report_warning("Skipping podcasts. If you really want EVERY EPISODE of EVERY RESULT, set your search filter to Podcasts only.") + if "channel" in filter_set: + self.report_warning("Skipping podcasts. If you really want EVERY EPISODE of EVERY RESULT, set your search filter to Podcasts only.") + filter_set.discard("channel") - filter_set.discard("channel") - if filter_set == {"future", "past"}: - filter_str = "" - else: - filter_str = "|".join(filter_set) # there should be only one filter now, so this should be the same as filter_set[0] - # but if there's more than one, then we should at least try to pass it through as-is, in the hope that it works - if len(filter_set) != 1: - # but also kick up a stink about it so it's clear it probably won't - self.report_warning("Your search has an unknkown combination of filters, so this request will probably fail!") + if filter_set == {"future", "past"}: + filter_str = "" + else: + filter_str = "|".join(filter_set) # there should be only one filter now, so this should be the same as filter_set[0] + # but if there's more than one, then we should at least try to pass it through as-is, in the hope that it works + if len(filter_set) != 1: + # but also kick up a stink about it so it's clear it probably won't + self.report_warning("Your search has an unknkown combination of filters, so this request will probably fail!") search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/programs/legacy/perl/program/search", { **queries, -- cgit v1.2.3-70-g09d2 From adbc8de0b8707ef8af14b545d11b99317d5e171e Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 11:39:07 +0100 Subject: Fix search test --- yt_dlp_plugins/extractor/radiko.py | 1 + 1 file changed, 1 insertion(+) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index ec260ab..8b6b623 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -592,6 +592,7 @@ class RadikoSearchIE(InfoExtractor): "id": "ニュース-all-all", "title": "ニュース" }, + 'expected_warnings': ['Skipping podcasts. If you really want EVERY EPISODE of EVERY RESULT, set your search filter to Podcasts only.'], }] def _strip_date(self, date): -- cgit v1.2.3-70-g09d2 From 4678f793475241d395157eaee6f375a8b1f24976 Mon Sep 17 00:00:00 2001 From: garret1317 Date: Fri, 19 Sep 2025 11:50:30 +0100 Subject: fix typo --- yt_dlp_plugins/extractor/radiko.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 8b6b623..922c0eb 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -651,7 +651,7 @@ class RadikoSearchIE(InfoExtractor): # but if there's more than one, then we should at least try to pass it through as-is, in the hope that it works if len(filter_set) != 1: # but also kick up a stink about it so it's clear it probably won't - self.report_warning("Your search has an unknkown combination of filters, so this request will probably fail!") + self.report_warning("Your search has an unknown combination of filters, so this request will probably fail!") search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/programs/legacy/perl/program/search", { **queries, -- cgit v1.2.3-70-g09d2 From 9b5db8364faf25cf3019e92ed4110b7958a4da9e Mon Sep 17 00:00:00 2001 From: garret1317 Date: Sun, 21 Sep 2025 18:06:15 +0100 Subject: RSeasons: extract description and thumbnail --- yt_dlp_plugins/extractor/radiko.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'yt_dlp_plugins/extractor') diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py index 922c0eb..3fd19d9 100644 --- a/yt_dlp_plugins/extractor/radiko.py +++ b/yt_dlp_plugins/extractor/radiko.py @@ -792,6 +792,7 @@ class RadikoRSeasonsIE(InfoExtractor): "info_dict": { "id": '10012302', "title": '山下達郎の楽天カード サンデー・ソングブック', + 'thumbnail': 'https://program-static.cf.radiko.jp/935a87fc-4a52-48e5-9468-7b2ef9448d9f.jpeg', } }, { "url": "https://radiko.jp/r_seasons/10002831", @@ -799,6 +800,8 @@ class RadikoRSeasonsIE(InfoExtractor): "info_dict": { "id": "10002831", "title": "Tokyo Moon", + 'description': 'md5:3eef525003bbe96ccf33ec647c43d904', + 'thumbnail': 'https://program-static.cf.radiko.jp/0368ee85-5d5f-41c9-8ee1-6c1035d87b3f.jpeg', } }] @@ -822,5 +825,9 @@ class RadikoRSeasonsIE(InfoExtractor): return self.playlist_result( entries(), playlist_id=season_id, - playlist_title=traverse_obj(pageProps, ("rSeason", "rSeasonName")), + **traverse_obj(pageProps, ("rSeason", { + "playlist_title": "rSeasonName", + "thumbnail": "backgroundImageUrl", + "description": ("summary", filter), + })), ) -- cgit v1.2.3-70-g09d2