aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
-rw-r--r--.gitignore2
-rw-r--r--LICENCE2
-rw-r--r--README.md69
-rwxr-xr-xmisc/generate_html.py84
-rw-r--r--misc/how to do a release42
-rwxr-xr-xmisc/old_generate_changelog.py (renamed from misc/generate_changelog.py)0
-rwxr-xr-xmisc/streammon.py53
-rwxr-xr-xmisc/test_areas.py (renamed from misc/test-tokens.py)0
-rwxr-xr-xmisc/test_extractors.py174
-rw-r--r--pyproject.toml11
-rw-r--r--yt_dlp_plugins/extractor/radiko.py272
-rw-r--r--yt_dlp_plugins/extractor/radiko_hacks.py65
-rwxr-xr-xyt_dlp_plugins/extractor/radiko_time.py13
13 files changed, 593 insertions, 194 deletions
diff --git a/.gitignore b/.gitignore
index 04a43b4..23d37d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@ __pycache__
*.pyc
wiki/
dist/
+*.m4a*
+*.m3u8
diff --git a/LICENCE b/LICENCE
index 9041052..ba3d837 100644
--- a/LICENCE
+++ b/LICENCE
@@ -1,6 +1,6 @@
BSD Zero Clause License
-Copyright (c) 2023, 2024 garret1317
+Copyright (c) 2023, 2024, 2025 garret1317
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted.
diff --git a/README.md b/README.md
index 31b8601..1546867 100644
--- a/README.md
+++ b/README.md
@@ -4,20 +4,19 @@ yt-dlp-rajiko is an improved [radiko.jp](https://radiko.jp) extractor plugin for
## Installation
-[Download the Python wheel](https://427738.xyz/yt-dlp-rajiko/dl/yt_dlp_rajiko-latest.whl) or `pip install
---extra-index-url https://427738.xyz/yt-dlp-rajiko/pip/ yt-dlp-rajiko`
+[Download the Python wheel](https://427738.xyz/yt-dlp-rajiko/dl/yt_dlp_rajiko-latest.whl) or `pip install yt-dlp-rajiko`
-Requires yt-dlp 2023.06.22 or above.
+Requires yt-dlp 2025.02.19 or above.
Use the pip command if you installed yt-dlp with pip. If you installed
-yt-dlp with `pipx`, use `pipx inject --index-url
-https://427738.xyz/yt-dlp-rajiko/pip/ yt-dlp yt-dlp-rajiko` to install
+yt-dlp with `pipx`, use `pipx inject yt-dlp yt-dlp-rajiko` to install
the plugin in yt-dlp's environment.
Otherwise, download the wheel, and place it in one of these locations:
- `~/.config/yt-dlp/plugins/` (on Linux and Mac)
- `%appdata%/yt-dlp/plugins/` (on Windows)
+ - a `yt-dlp-plugins` folder next to your `yt-dlp.exe` (<a href="https://427738.xyz/yt-dlp-rajiko/i/installation-pluginsfolder.png">like this</a>)
- anywhere else listed in [the yt-dlp
documentation](https://github.com/yt-dlp/yt-dlp#installing-plugins).
@@ -28,28 +27,52 @@ More information about yt-dlp plugins is available from [yt-dlp's documentation]
## Usage
simply:
+```
+# timefree download
+yt-dlp 'https://radiko.jp/#!/ts/INT/20240308230000'
+# live recording
+yt-dlp 'https://radiko.jp/#!/live/CCL'
+# live shorthand
+yt-dlp 'https://radiko.jp/#FMT'
+```
- # timefree download
- yt-dlp 'https://radiko.jp/#!/ts/INT/20240308230000'
- # live recording
- yt-dlp 'https://radiko.jp/#!/live/CCL'
- # live shorthand
- yt-dlp 'https://radiko.jp/#FMT'
+You can semi-automatically grab the latest episodes of programmes by using the search page, like this:
+```
+# single-station programme yt-dlp
+'https://radiko.jp/#!/search/live?key=World Jazz Warehouse&filter=past'
+# area filtering also works
+yt-dlp 'https://radiko.jp/#!/search/live?key=tokyo speakeasy&filter=past&area_id=JP13'
+```
+(though of course you still need to be there to press the button)
-You can somewhat automate downloading programmes by using the search
-page.
+If you can reliably get it in the search, you can somewhat-automate downloading it.
+If there's a programme that airs on multiple stations, the best way to filter down to the station you want is to use the search's 地域 (region) filter.
- # all programmes related to Toshiki Kadomatsu
- yt-dlp 'https://radiko.jp/#!/search/live?key=角松敏生&filter=past&region_id=all'
- # specific programme from Osaka
- yt-dlp 'https://radiko.jp/#!/search/live?key=world%20jazz%20warehouse&filter=past&area_id=JP27'
+You can also get programmes that a person has appeared in, using the links from those little boxes on the side:
+`yt-dlp 'https://radiko.jp/persons/3363'`
-Just copying from the browser URL bar should work with no changes.
+you can limit it to the "key station" only, like so:
+`yt-dlp 'https://radiko.jp/persons/33635' --extractor-args rajiko:key-station-only`
-----
+This is meant as an alternative to the region filter, for networked programmes where the same thing airs on multiple stations.
+Currently /persons/ URLs are the only place where that option works, as the information just isn't available for normal search results.
-[Please see the website for more information](https://427738.xyz/yt-dlp-rajiko/).
-If the website is down, an archived copy may be available on [the Internet Archive's Wayback Machine](https://web.archive.org/web/*/https://427738.xyz/yt-dlp-rajiko/).
+As a general rule, just copying from the browser URL bar should work with no changes. (if it doesn't, [let me know!](https://github.com/garret1317/yt-dlp-rajiko/issues))
-[日本語訳は公式サイトをご覧ください。](https://427738.xyz/yt-dlp-rajiko/index.ja.html)
-サイトが無くなった場合は、多分[Internet ArchiveのWayback Machineにアーカイブされています](https://web.archive.org/web/*/https://427738.xyz/yt-dlp-rajiko/index.ja.html)。
+(Apparently on Windows it won't work unless you use "double quotes", but on Linux it won't work unless you use 'single quotes'. If the command doesnt work with one quote type then try the other.)
+
+**[You can find more usage tips on the website](https://427738.xyz/yt-dlp-rajiko/)**
+
+## How do I get help?
+
+[Open an issue on github](https://github.com/garret1317/yt-dlp-rajiko/issues) or [send a message via the "hate mail" form](https://427738.xyz/hate-mail.html).
+You can also ping me [in the yt-dlp discord server](https://discord.gg/H5MNcFW63r), username `garret1317`.
+
+Please try to include a verbose log if you're reporting a problem, it helps with diagnosing the issue.
+
+## notes about this repository
+
+this is just where the source code and bug tracker live. most of the info is on the website.
+
+Generally you should use the release versions.
+`master` branch usually works, but should be considered experimental and may have bugs
diff --git a/misc/generate_html.py b/misc/generate_html.py
new file mode 100755
index 0000000..0e15d6a
--- /dev/null
+++ b/misc/generate_html.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import os
+import hashlib
+import re
+
+pip_index = open("index.html", "w")
+
+pip_index.write("""<!DOCTYPE HTML>
+<html lang="en-GB">
+<head>
+ <title>yt-dlp-rajiko pip index</title>
+ <link rel="canonical" href="https://427738.xyz/yt-dlp-rajiko/pip/yt-dlp-rajiko/">
+</head>
+<body>
+
+<ul>
+""")
+
+site_sha256 = []
+
+tarballs = []
+wheels = []
+
+for item in sorted(os.listdir()):#, key=lambda x: x.name):
+ if os.path.islink(item):
+ continue
+
+ if item.endswith(".tar.gz"):
+ tarballs.append(item)
+ elif item.endswith(".whl"):
+ wheels.append(item)
+ else:
+ continue
+
+ pip_index.write("\t<li>")
+ pip_index.write('<a href="')
+ pip_index.write(item)
+
+ with open(item, "rb") as f:
+ checksum = hashlib.sha256(f.read()).hexdigest()
+
+ pip_index.write("#sha256=")
+ pip_index.write(checksum)
+ pip_index.write('">')
+ pip_index.write(item)
+ pip_index.write("</a>\n")
+
+ site_string = checksum + "&nbsp;&nbsp;" + '<a href="dl/' + item + '">' + item + "</a><br>"
+ site_sha256.append(site_string)
+
+pip_index.write("""</ul>
+
+</body>
+</html>""")
+
+latest_tarball = tarballs[-1]
+latest_wheel = wheels[-1]
+print(latest_tarball, latest_wheel)
+
+os.remove("yt_dlp_rajiko-latest.tar.gz")
+os.symlink(latest_tarball, "yt_dlp_rajiko-latest.tar.gz")
+
+os.remove("yt_dlp_rajiko-latest.whl")
+os.symlink(latest_wheel, "yt_dlp_rajiko-latest.whl")
+
+site_sha256.reverse()
+
+latest_list = site_sha256[:2]
+previous_list = site_sha256[2:]
+
+latest = "\n".join(["<!-- LATEST SHA256 START -->", "<code>", "\n".join(latest_list), "</code>", "<!-- LATEST SHA256 END -->"])
+
+previous = "\n".join(["<!-- PREVIOUS SHA256 START -->", "<code>", "\n".join(previous_list), "</code>", "<!-- PREVIOUS SHA256 END -->"])
+
+for i in ["../../index.html", "../../index.ja.html"]:
+ with open(i, "r+") as f:
+ page = f.read()
+
+ page = re.sub(r"<!-- LATEST SHA256 START -->.+<!-- LATEST SHA256 END -->", latest, page, flags=re.DOTALL)
+ page = re.sub(r"<!-- PREVIOUS SHA256 START -->.+<!-- PREVIOUS SHA256 END -->", previous, page, flags=re.DOTALL)
+
+ f.seek(0)
+ f.truncate(0)
+ f.write(page)
diff --git a/misc/how to do a release b/misc/how to do a release
index f5eb85f..6e91e14 100644
--- a/misc/how to do a release
+++ b/misc/how to do a release
@@ -8,40 +8,34 @@ python3 -m build
and then put BOTH items from `dist` into the pip index dir - ~/site2/yt-dlp-rajiko/pip/yt-dlp-rajiko/
because without the .whl pip has to "build" it itself, with all the stuff that needs to be installed for that to work
-update the pip index html
-update the dl/ "latest" symlinks
-## update the changelog file
-
-~/site2/yt-dlp-rajiko/CHANGELOG
+run script to update the pip index html and the dl/ "latest" symlinks
+this also updates the sha256s on the site
-```
-version number
-date (git log v1.0 --pretty --date=rfc2822)
-url: whl download link
-sha256: sha256 of the whl
-brief summary of the release
-can span multiple lines
+## update the changelog file
-bullet points of changes, 1 per line
-simple present tense, third person singular - continue "this release...", eg..
-fixes a bug where the computer would explode
-makes downloading 5000x faster
-```
+write in html, paste into the feed xml like <![CDATA[
+stuff
+]]>
+make sure to set the link, date
+to get date use:
+git log --pretty --date=rfc2822
-./generate_changelog.py to make the new rss feed
+include the pip instructions, sha256sum etc
-## update the website
+now push to the server
-move the previous release into the "Previous releases" <details>
-update the sha256 (just sha256 command in the pip dir)
-update the whl link
-repeat for japanese version
+!!NEW!!
+upload to pip proper as well
+go to dl/ dir and do
+twine upload yt_dlp_rajiko-1.x*
-now push to the server
## update github
paste the changelog output into a github release, upload the new builds
+change link at the bottom to just "below"
+
+post in the radiko thread on 5ch if i can be bothered
and thats probably all
diff --git a/misc/generate_changelog.py b/misc/old_generate_changelog.py
index 1bce073..1bce073 100755
--- a/misc/generate_changelog.py
+++ b/misc/old_generate_changelog.py
diff --git a/misc/streammon.py b/misc/streammon.py
new file mode 100755
index 0000000..4051833
--- /dev/null
+++ b/misc/streammon.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# monitor stream APIs for any changes, so I can check they don't break anything
+# run via cronjob every now and then
+
+import difflib
+import os
+import sys
+from datetime import datetime
+
+import requests
+
+s = requests.Session()
+
+DISCORD_WEBHOOK = "PUT WEBHOOK HERE"
+STREAMS_API = "https://radiko.jp/v3/station/stream/{device}/{station}.xml"
+
+if len(sys.argv) > 1:
+ PATH = sys.argv[1]
+else:
+ PATH = ""
+
+devices = ('pc_html5', 'aSmartPhone7a', 'aSmartPhone8')
+stations = ('FMT', 'CCL', 'NORTHWAVE', 'TBS')
+
+for device in devices:
+ for station in stations:
+ url = STREAMS_API.format(device=device, station=station)
+ now = s.get(url).text
+
+ filename = f"{PATH}{station}-{device}.xml"
+ with open(filename, "a+") as f:
+ f.seek(0)
+ past = f.read()
+
+ modtime = datetime.fromtimestamp(os.path.getmtime(filename))
+ diff = difflib.unified_diff(
+ past.splitlines(), now.splitlines(),
+ fromfile=url, tofile=url,
+ fromfiledate=str(modtime), tofiledate=str(datetime.now()),
+ )
+
+ diff_str = "\n".join(diff)
+ if diff_str != "":
+ f.truncate(0)
+ f.write(now)
+
+ s.post(DISCORD_WEBHOOK, json={
+ "embeds": [{
+ "type": "rich",
+ "title": f"Streams changed: {station} {device}",
+ "description": "\n".join(("```diff", diff_str, "```"))
+ }]
+ })
diff --git a/misc/test-tokens.py b/misc/test_areas.py
index ba6475f..ba6475f 100755
--- a/misc/test-tokens.py
+++ b/misc/test_areas.py
diff --git a/misc/test_extractors.py b/misc/test_extractors.py
new file mode 100755
index 0000000..18e9783
--- /dev/null
+++ b/misc/test_extractors.py
@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+
+# programmes expire, so i have to update the times in the tests every time i run them
+# but thats a massive ballache, so i end up just not running them, which leads to cockups
+# so, this script has the tests automatically use the latest episode as you run it, by setting dynamically generated time values
+# everything else is always the same so it should be fine lol
+
+
+import datetime
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, "/home/g/Downloads/yt-dlp/") # TODO: un-hardcode. has to be the source/git repo because pip doesnt carry the tests
+
+from yt_dlp_plugins.extractor import radiko_time as rtime
+
+MON, TUE, WED, THU, FRI, SAT, SUN = range(7)
+weekdays = {0: "MON", 1: "TUE", 2: "WED", 3: "THU", 4: "FRI", 5: "SAT", 6: "SUN"}
+
+now = rtime.RadikoTime.now(tz = rtime.JST)
+UTC = datetime.timezone.utc
+
+def get_latest_airtimes(now, weekday, hour, minute, duration):
+ days_after_weekday = (7 - (now.weekday() - weekday)) % 7
+ latest_airdate = (now + datetime.timedelta(days=days_after_weekday)).replace(hour=hour, minute=minute, second=0, microsecond=0)
+ if (latest_airdate + duration) > now:
+ latest_airdate -= datetime.timedelta(days=7)
+ return latest_airdate, latest_airdate + duration
+
+def get_test_timefields(airtime, release_time):
+ return {
+ "timestamp": airtime.timestamp(),
+ "release_timestamp": release_time.timestamp(),
+ "upload_date": airtime.astimezone(UTC).strftime("%Y%m%d"),
+ "release_date": release_time.astimezone(UTC).strftime("%Y%m%d"),
+
+ "duration": (release_time - airtime).total_seconds(),
+ }
+
+
+
+
+from yt_dlp_plugins.extractor.radiko import RadikoTimeFreeIE, RadikoShareIE, RadikoLiveIE, RadikoShareIE, RadikoPersonIE, RadikoStationButtonIE
+
+RadikoTimeFreeIE._TESTS = []
+
+
+
+# TOKYO MOON - interfm - EVERY FRI 2300
+airtime, release_time = get_latest_airtimes(now, FRI, 23, 0, datetime.timedelta(hours=1))
+RadikoTimeFreeIE._TESTS.append({
+ "url": f"https://radiko.jp/#!/ts/INT/{airtime.timestring()}",
+ "info_dict": {
+ "ext": "m4a",
+ "id": f"INT-{airtime.timestring()}",
+
+ **get_test_timefields(airtime, release_time),
+
+ 'title': 'TOKYO MOON',
+ 'description': 'md5:4185349a530cfc4d0580e6996a511273',
+ 'uploader': 'interfm',
+ 'uploader_id': 'INT',
+ 'uploader_url': 'https://www.interfm.co.jp/',
+ 'channel': 'interfm',
+ 'channel_id': 'INT',
+ 'channel_url': 'https://www.interfm.co.jp/',
+ 'thumbnail': 'https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg',
+ 'chapters': list,
+ 'tags': ['松浦俊夫', 'ジャズの魅力を楽しめる'],
+ 'cast': ['松浦\u3000俊夫'],
+ 'series': 'Tokyo Moon',
+ 'live_status': 'was_live',
+ }
+})
+
+
+# late-night/v. early morning show, to test broadcast day handling
+# this should be monday 27:00 / tuesday 03:00
+airtime, release_time = get_latest_airtimes(now, TUE, 3, 0, datetime.timedelta(hours=2))
+RadikoTimeFreeIE._TESTS.append({
+ "url": f"https://radiko.jp/#!/ts/TBS/{airtime.timestring()}",
+ "info_dict": {
+ "ext": "m4a",
+ "id": f"TBS-{airtime.timestring()}",
+
+ **get_test_timefields(airtime, release_time),
+ 'title': 'CITY CHILL CLUB',
+ 'description': r"re:^目を閉じて…リラックスして[\S\s]+chill@tbs.co.jp$",
+ 'uploader': 'TBSラジオ',
+ 'uploader_id': 'TBS',
+ 'uploader_url': 'https://www.tbsradio.jp/',
+ 'channel': 'TBSラジオ',
+ 'channel_id': 'TBS',
+ 'channel_url': 'https://www.tbsradio.jp/',
+ 'thumbnail': 'https://program-static.cf.radiko.jp/nrf8fowbjo.jpg',
+ 'chapters': list,
+ 'tags': ['CCC905', '音楽との出会いが楽しめる', '人気アーティストトーク', '音楽プロデューサー出演', 'ドライブ中におすすめ', '寝る前におすすめ', '学生におすすめ'],
+ 'cast': list,
+ 'series': 'CITY CHILL CLUB',
+ 'live_status': 'was_live',
+ },
+})
+
+
+# testing 29-hour clock handling
+airtime, release_time = get_latest_airtimes(now, WED, 0, 0, datetime.timedelta(minutes=55))
+share_timestring = (airtime - datetime.timedelta(days=1)).strftime("%Y%m%d") + "240000"
+
+RadikoShareIE._TESTS = [{
+ "url": f"http://radiko.jp/share/?sid=FMT&t={share_timestring}",
+ "info_dict": {
+ "live_status": "was_live",
+ "ext": "m4a",
+ "id": f"FMT-{airtime.timestring()}",
+
+ **get_test_timefields(airtime, release_time),
+
+ "title": "JET STREAM",
+ "series": "JET STREAM",
+ "description": r"re:^JET STREAM・・・作家が描く世界への旅。[\s\S]+https://www.tfm.co.jp/f/jetstream/message$",
+ "chapters": list,
+ "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg",
+
+ "channel": "TOKYO FM",
+ "channel_id": "FMT",
+ "channel_url": "https://www.tfm.co.jp/",
+ "uploader": "TOKYO FM",
+ "uploader_id": "FMT",
+ "uploader_url": "https://www.tfm.co.jp/",
+
+ "cast": ["福山雅治"],
+ "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"],
+ },
+ }]
+
+
+
+IEs = [
+ RadikoTimeFreeIE, RadikoShareIE,
+ RadikoLiveIE, RadikoShareIE, RadikoPersonIE, RadikoStationButtonIE
+]
+
+import test.helper as th
+
+# override to only get testcases from our IEs
+
+def _new_gettestcases(include_onlymatching=False):
+ import yt_dlp.plugins as plugins
+ plugins.load_all_plugins()
+
+ for ie in IEs:
+ yield from ie.get_testcases(include_onlymatching)
+
+def _new_getwebpagetestcases():
+ import yt_dlp.plugins as plugins
+ plugins.load_all_plugins()
+
+ for ie in IEs:
+ for tc in ie.get_webpage_testcases():
+ tc.setdefault('add_ie', []).append('Generic')
+ yield tc
+
+th.gettestcases = _new_gettestcases
+th.getwebpagetestcases = _new_getwebpagetestcases
+
+import test.test_download as td
+
+class TestDownload(td.TestDownload):
+ pass
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/pyproject.toml b/pyproject.toml
index a0bcc03..2a13f3f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,10 +1,15 @@
[project]
name = "yt-dlp-rajiko"
-version = "1.1"
-description = "improved radiko.jp extractor for yt-dlp"
+version = "1.7"
+description = "improved radiko.jp extractor for yt-dlp (fast and areafree)"
+
+readme = "README.md"
+license-files = ["LICENCE"]
+
authors = [
{ name="garret1317" },
]
+
requires-python = ">=3.8"
classifiers = [
"License :: OSI Approved :: Zero-Clause BSD (0BSD)",
@@ -13,6 +18,8 @@ classifiers = [
[project.urls]
Homepage = "https://427738.xyz/yt-dlp-rajiko/"
+"Source Code" = "https://github.com/garret1317/yt-dlp-rajiko/"
+"Release Notes" = "https://427738.xyz/yt-dlp-rajiko/CHANGELOG.xml"
[build-system]
requires = ["setuptools>=61.0"]
diff --git a/yt_dlp_plugins/extractor/radiko.py b/yt_dlp_plugins/extractor/radiko.py
index d12f203..2996290 100644
--- a/yt_dlp_plugins/extractor/radiko.py
+++ b/yt_dlp_plugins/extractor/radiko.py
@@ -6,18 +6,22 @@ import urllib.parse
import pkgutil
from yt_dlp.extractor.common import InfoExtractor
+from yt_dlp.networking.exceptions import HTTPError
from yt_dlp.utils import (
+ ExtractorError,
OnDemandPagedList,
clean_html,
int_or_none,
join_nonempty,
parse_qs,
traverse_obj,
+ urlencode_postdata,
url_or_none,
update_url_query,
)
import yt_dlp_plugins.extractor.radiko_time as rtime
+import yt_dlp_plugins.extractor.radiko_hacks as hacks
class _RadikoBaseIE(InfoExtractor):
@@ -82,7 +86,9 @@ class _RadikoBaseIE(InfoExtractor):
_APP_VERSIONS = ["7.5.0", "7.4.17", "7.4.16", "7.4.15", "7.4.14", "7.4.13", "7.4.12", "7.4.11", "7.4.10", "7.4.9", "7.4.8", "7.4.7", "7.4.6", "7.4.5", "7.4.4", "7.4.3", "7.4.2", "7.4.1", "7.4.0", "7.3.8", "7.3.7", "7.3.6", "7.3.1", "7.3.0", "7.2.11", "7.2.10"]
_DELIVERED_ONDEMAND = ('radiko.jp',)
- _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp')
+ _DOESNT_WORK_WITH_FFMPEG = ('tf-f-rpaa-radiko.smartstream.ne.jp', 'si-f-radiko.smartstream.ne.jp', 'alliance-stream-radiko.smartstream.ne.jp')
+
+ _has_tf30 = None
def _index_regions(self):
region_data = {}
@@ -178,16 +184,21 @@ class _RadikoBaseIE(InfoExtractor):
"X-Radiko-AuthToken": auth_token,
},
"user": auth2_headers["X-Radiko-User"],
+ "has_tf30": self._has_tf30,
}
if not region_mismatch:
self.cache.store("rajiko", station_region, auth_data)
return auth_data
- def _auth(self, station_region):
+ def _auth(self, station_region, need_tf30=False):
cachedata = self.cache.load("rajiko", station_region)
self.write_debug(cachedata)
if cachedata is not None:
+ if need_tf30 and not cachedata.get("has_tf30"):
+ self.write_debug("Cached token doesn't have timefree 30, getting a new one")
+ return self._negotiate_token(station_region)
+
auth_headers = cachedata.get("token")
response = self._download_webpage("https://radiko.jp/v2/api/auth_check", station_region, "Checking cached token",
headers=auth_headers, expected_status=401)
@@ -205,6 +216,17 @@ class _RadikoBaseIE(InfoExtractor):
station = region.find(f'.//station/id[.="{station_id}"]/..') # a <station> with an <id> of our station_id
station_name = station.find("name").text
station_url = url_or_none(station.find("href").text)
+
+ thumbnails = []
+ for logo in station.findall("logo"):
+ thumbnails.append({
+ "url": logo.text,
+ **traverse_obj(logo.attrib, ({
+ "width": ("width", {int_or_none}),
+ "height": ("height", {int_or_none}),
+ }))
+ })
+
meta = {
"id": station_id,
"title": station_name,
@@ -218,7 +240,7 @@ class _RadikoBaseIE(InfoExtractor):
"uploader_id": station_id,
"uploader_url": station_url,
- "thumbnail": url_or_none(station.find("banner").text),
+ "thumbnails": thumbnails,
}
self.cache.store("rajiko", station_id, {
"expiry": (now + datetime.timedelta(days=1)).timestamp(),
@@ -229,8 +251,16 @@ class _RadikoBaseIE(InfoExtractor):
self.to_screen(f"{station_id}: Using cached station metadata")
return cachedata.get("meta")
- def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None):
- device = self._configuration_arg('device', ['aSmartPhone7a'], casesense=True, ie_key="rajiko")[0] # aSmartPhone7a formats = always happy path
+ def _get_station_formats(self, station, timefree, auth_data, start_at=None, end_at=None, use_pc_html5=False):
+ config_device = traverse_obj(self._configuration_arg('device', casesense=True, ie_key="rajiko"), 0)
+
+ if not use_pc_html5:
+ device = config_device or "aSmartPhone7a" # this device only gives us the on-demand one for timefree
+ # that's good imo - we just get the one that works, and don't bother with probing the rest as well
+ else:
+ device = config_device or "pc_html5" # the on-demand one doesnt work with timefree30 stuff sadly
+ # so just use pc_html5 which has everything
+
url_data = self._download_xml(f"https://radiko.jp/v3/station/stream/{device}/{station}.xml",
station, note=f"Downloading {device} stream information")
@@ -238,6 +268,8 @@ class _RadikoBaseIE(InfoExtractor):
formats = []
timefree_int = 1 if timefree else 0
+ do_blacklist_streams = not len(self._configuration_arg("no_stream_blacklist", ie_key="rajiko")) > 0
+ do_as_live_chunks = not len(self._configuration_arg("no_as_live_chunks", ie_key="rajiko")) > 0
for element in url_data.findall(f".//url[@timefree='{timefree_int}'][@areafree='0']/playlist_create_url"):
# find <url>s with matching timefree and no areafree, then get their <playlist_create_url>
url = element.text
@@ -249,7 +281,7 @@ class _RadikoBaseIE(InfoExtractor):
"station_id": station,
"l": "15", # l = length, ie how many seconds in the live m3u8 (max 300)
"lsid": auth_data["user"],
- "type": "b", # it is a mystery
+ "type": "b", # a/b = in-region, c = areafree
})
if timefree:
@@ -268,7 +300,7 @@ class _RadikoBaseIE(InfoExtractor):
preference = -1
entry_protocol = 'm3u8'
- if domain in self._DOESNT_WORK_WITH_FFMPEG:
+ if domain in self._DOESNT_WORK_WITH_FFMPEG and do_blacklist_streams:
self.write_debug(f"skipping {domain} (known not working)")
continue
if domain in self._DELIVERED_ONDEMAND:
@@ -277,10 +309,30 @@ class _RadikoBaseIE(InfoExtractor):
preference = 1
entry_protocol = None
- formats += self._extract_m3u8_formats(
- playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_data["token"],
- live=delivered_live, preference=preference, entry_protocol=entry_protocol,
- note=f"Downloading m3u8 information from {domain}")
+ auth_headers = auth_data["token"]
+
+ if delivered_live and timefree and do_as_live_chunks:
+
+ chunks_playlist = hacks._generate_as_live_playlist(
+ self, playlist_url, start_at, end_at, domain, auth_headers
+ )
+
+ formats.append({
+ "format_id": join_nonempty(domain, "chunked"),
+ "hls_media_playlist_data": chunks_playlist,
+ "preference": preference,
+ "ext": "m4a",
+
+ # fallback to live for ffmpeg etc
+ "url": playlist_url,
+ "http_headers": auth_headers,
+ })
+ else:
+
+ formats += self._extract_m3u8_formats(
+ playlist_url, station, m3u8_id=domain, fatal=False, headers=auth_headers,
+ live=delivered_live, preference=preference, entry_protocol=entry_protocol,
+ note=f"Downloading m3u8 information from {domain}")
return formats
@@ -299,7 +351,7 @@ class RadikoLiveIE(_RadikoBaseIE):
"id": "FMT",
"title": "re:^TOKYO FM.+$",
"alt_title": "TOKYO FM",
- "thumbnail": "https://radiko.jp/res/banner/FMT/20220512162447.jpg",
+ "thumbnail": "https://radiko.jp/v2/static/station/logo/FMT/lrtrim/688x160.png",
"channel": "TOKYO FM",
"channel_id": "FMT",
@@ -319,7 +371,7 @@ class RadikoLiveIE(_RadikoBaseIE):
"id": "NORTHWAVE",
"title": "re:^FM NORTH WAVE.+$",
"alt_title": "FM NORTH WAVE",
- "thumbnail": "https://radiko.jp/res/banner/NORTHWAVE/20150731161543.png",
+ "thumbnail": "https://radiko.jp/v2/static/station/logo/NORTHWAVE/lrtrim/688x160.png",
"uploader": "FM NORTH WAVE",
"uploader_url": "https://www.fmnorth.co.jp/",
@@ -340,7 +392,7 @@ class RadikoLiveIE(_RadikoBaseIE):
"id": "RN1",
"title": "re:^ラジオNIKKEI第1.+$",
"alt_title": "RADIONIKKEI",
- "thumbnail": "https://radiko.jp/res/banner/RN1/20120802154152.png",
+ "thumbnail": "https://radiko.jp/v2/static/station/logo/RN1/lrtrim/688x160.png",
"channel": "ラジオNIKKEI第1",
"channel_url": "http://www.radionikkei.jp/",
@@ -357,7 +409,7 @@ class RadikoLiveIE(_RadikoBaseIE):
region = self._get_station_region(station)
station_meta = self._get_station_meta(region, station)
auth_data = self._auth(region)
- formats = self._get_station_formats(station, False, auth_data)
+ formats = self._get_station_formats(station, False, auth_data, use_pc_html5=True)
return {
"is_live": True,
@@ -368,71 +420,36 @@ class RadikoLiveIE(_RadikoBaseIE):
class RadikoTimeFreeIE(_RadikoBaseIE):
+ _NETRC_MACHINE = "rajiko"
_VALID_URL = r"https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-_]+)/(?P<id>\d+)"
- _TESTS = [{
- "url": "https://radiko.jp/#!/ts/INT/20240809230000",
- "info_dict": {
- "live_status": "was_live",
- "ext": "m4a",
- "id": "INT-20240809230000",
-
- "title": "TOKYO MOON",
- "series": "Tokyo Moon",
- "description": "md5:20e68d2f400a391fa34d4e7c8c702cb8",
- "chapters": "count:14",
- "thumbnail": "https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg",
-
- "upload_date": "20240809",
- "timestamp": 1723212000.0,
- "release_date": "20240809",
- "release_timestamp": 1723215600.0,
- "duration": 3600,
-
- "channel": "interfm",
- "channel_id": "INT",
- "channel_url": "https://www.interfm.co.jp/",
- "uploader": "interfm",
- "uploader_id": "INT",
- "uploader_url": "https://www.interfm.co.jp/",
-
- "cast": ["松浦\u3000俊夫"],
- "tags": ["松浦俊夫"],
- },
- }, {
- # late-night/early-morning show to test broadcast day checking
- "url": "https://radiko.jp/#!/ts/TBS/20240810033000",
- "info_dict": {
- "live_status": "was_live",
- "ext": "m4a",
- "id": "TBS-20240810033000",
-
- "title": "CITY CHILL CLUB",
- "series": "CITY CHILL CLUB",
- "description": "md5:3fba2c1125059bed27247c0be90e58fa",
- "chapters": "count:22",
- "thumbnail": "https://program-static.cf.radiko.jp/ku7t4ztnaq.jpg",
-
- "upload_date": "20240809",
- "timestamp": 1723228200.0,
- "release_date": "20240809",
- "release_timestamp": 1723233600.0,
- "duration": 5400,
-
- "channel": "TBSラジオ",
- "channel_url": "https://www.tbsradio.jp/",
- "channel_id": "TBS",
- "uploader": "TBSラジオ",
- "uploader_url": "https://www.tbsradio.jp/",
- "uploader_id": "TBS",
-
- "tags": ["CCC905", "音楽との出会いが楽しめる", "人気アーティストトーク", "音楽プロデューサー出演", "ドライブ中におすすめ", "寝る前におすすめ", "学生におすすめ"],
- "cast": ["PES"],
- },
- }]
+ # TESTS use a custom-ish script that updates the airdates automatically, see misc/test_extractors.py
+
+ def _perform_login(self, username, password):
+ try:
+ login_info = self._download_json('https://radiko.jp/ap/member/webapi/member/login', None, note='Logging in',
+ data=urlencode_postdata({'mail': username, 'pass': password}))
+ self._has_tf30 = '2' in login_info.get('privileges')
+ # areafree = 1, timefree30 = 2, double plan = both
+ self.write_debug({**login_info, "radiko_session": "PRIVATE", "member_ukey": "PRIVATE"})
+ except ExtractorError as error:
+ if isinstance(error.cause, HTTPError) and error.cause.status == 401:
+ raise ExtractorError('Invalid username and/or password', expected=True)
+ raise
+
+ def _check_tf30(self):
+ if self._has_tf30 is not None:
+ return self._has_tf30
+ if self._get_cookies('https://radiko.jp').get('radiko_session') is None:
+ return
+ account_info = self._download_json('https://radiko.jp/ap/member/webapi/v2/member/login/check',
+ None, note='Checking account status from cookies', expected_status=400)
+ self.write_debug({**account_info, "user_key": "PRIVATE"})
+ self._has_tf30 = account_info.get('timefreeplus') == '1'
+ return self._has_tf30
def _get_programme_meta(self, station_id, url_time):
day = url_time.broadcast_day_string()
- meta = self._download_json(f"https://radiko.jp/v4/program/station/date/{day}/{station_id}.json", station_id,
+ meta = self._download_json(f"https://api.radiko.jp/program/v4/date/{day}/station/{station_id}.json", station_id,
note="Downloading programme data")
programmes = traverse_obj(meta, ("stations", lambda _, v: v["station_id"] == station_id,
"programs", "program"), get_all=False)
@@ -467,10 +484,12 @@ class RadikoTimeFreeIE(_RadikoBaseIE):
"start_time_gte": start.isoformat(),
"end_time_lt": end.isoformat(),
})
- data = self._download_json(api_url, video_id, note="Downloading tracklist").get("data")
+ data_json = self._download_json(
+ api_url, video_id, note="Downloading tracklist", errnote="Downloading tracklist", fatal=False
+ )
chapters = []
- for track in data:
+ for track in traverse_obj(data_json, "data") or []:
artist = traverse_obj(track, ("artist", "name")) or track.get("artist_name")
chapters.append({
"title": join_nonempty(artist, track.get("title"), delim=" - "),
@@ -492,9 +511,13 @@ class RadikoTimeFreeIE(_RadikoBaseIE):
start = times[0]
end = times[1]
now = datetime.datetime.now(tz=rtime.JST)
+ expiry_free, expiry_tf30 = end.expiry()
- if end.expiry(False) < now:
+ if expiry_tf30 < now:
self.raise_no_formats("Programme is no longer available.", video_id=meta["id"], expected=True)
+ need_tf30 = expiry_free < now
+ if need_tf30 and not self._check_tf30():
+ self.raise_login_required("Programme is only available with a Timefree 30 subscription")
elif start > now:
self.raise_no_formats("Programme has not aired yet.", video_id=meta["id"], expected=True)
live_status = "is_upcoming"
@@ -505,12 +528,14 @@ class RadikoTimeFreeIE(_RadikoBaseIE):
region = self._get_station_region(station)
station_meta = self._get_station_meta(region, station)
chapters = self._extract_chapters(station, start, end, video_id=meta["id"])
- auth_data = self._auth(region)
- formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end)
+ auth_data = self._auth(region, need_tf30=need_tf30)
+ formats = self._get_station_formats(station, True, auth_data, start_at=start, end_at=end, use_pc_html5=need_tf30)
return {
**station_meta,
- "alt_title": None,
+ "alt_title": None, # override from station metadata
+ "thumbnails": None,
+
**meta,
"chapters": chapters,
"formats": formats,
@@ -548,22 +573,37 @@ class RadikoSearchIE(InfoExtractor):
}]
def _strip_date(self, date):
+ # lazy way of making a timestring (from eg 2025-05-20 01:00:00)
return date.replace(" ", "").replace("-", "").replace(":", "")
def _pagefunc(self, url, idx):
url = update_url_query(url, {"page_idx": idx})
data = self._download_json(url, None, note=f"Downloading page {idx+1}")
- return [self.url_result("https://radiko.jp/#!/ts/{station}/{time}".format(
- station = i.get("station_id"), time = self._strip_date(i.get("start_time"))))
- for i in data.get("data")]
+ results = []
+ for r in data.get("data"):
+ station = r.get("station_id")
+ timestring = self._strip_date(r.get("start_time"))
+
+ results.append(
+ self.url_result(
+ f"https://radiko.jp/#!/ts/{station}/{timestring}",
+ id=join_nonempty(station, timestring)
+ )
+ )
+ return results
def _real_extract(self, url):
url = url.replace("/#!/", "/!/", 1)
# urllib.parse interprets the path as just one giant fragment because of the #, so we hack it away
queries = parse_qs(url)
- search_url = update_url_query("https://radiko.jp/v3/api/program/search", {
+ if queries.get("cul_area_id"):
+ queries["cur_area_id"] = queries.pop("cul_area_id")
+ # site used to use "cul_area_id" in the search url, now it uses "cur_area_id" (with an r)
+ # and outright rejects the old one with HTTP Error 415: Unsupported Media Type
+
+ search_url = update_url_query("https://api.annex-cf.radiko.jp/v1/programs/legacy/perl/program/search", {
**queries,
"uid": "".join(random.choices("0123456789abcdef", k=32)),
"app_id": "pc",
@@ -588,44 +628,16 @@ class RadikoSearchIE(InfoExtractor):
class RadikoShareIE(InfoExtractor):
_VALID_URL = r"https?://(?:www\.)?radiko\.jp/share/"
- _TESTS = [{
- # 29-hour time -> 24-hour time
- "url": "http://radiko.jp/share/?sid=FMT&t=20240802240000",
- "info_dict": {
- "live_status": "was_live",
- "ext": "m4a",
- "id": "FMT-20240803000000", # the time given (24:00) works out to 00:00 the next day
-
- "title": "JET STREAM",
- "series": "JET STREAM",
- "description": "md5:c1a2172036ebb7a54eeafb47e0a08a50",
- "chapters": "count:9",
- "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg",
-
- "upload_date": "20240802",
- "timestamp": 1722610800.0,
- "release_date": "20240802",
- "release_timestamp": 1722614100.0,
- "duration": 3300,
-
- "channel": "TOKYO FM",
- "channel_id": "FMT",
- "channel_url": "https://www.tfm.co.jp/",
- "uploader": "TOKYO FM",
- "uploader_id": "FMT",
- "uploader_url": "https://www.tfm.co.jp/",
-
- "cast": ["福山雅治"],
- "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"],
- }
- }]
def _real_extract(self, url):
queries = parse_qs(url)
station = traverse_obj(queries, ("sid", 0))
time = traverse_obj(queries, ("t", 0))
time = rtime.RadikoShareTime(time).timestring()
- return self.url_result(f"https://radiko.jp/#!/ts/{station}/{time}", RadikoTimeFreeIE)
+ return self.url_result(
+ f"https://radiko.jp/#!/ts/{station}/{time}", RadikoTimeFreeIE,
+ id=join_nonempty(station, time)
+ )
class RadikoStationButtonIE(InfoExtractor):
@@ -638,19 +650,9 @@ class RadikoStationButtonIE(InfoExtractor):
"info_dict": {
"ext": "m4a",
'live_status': 'is_live',
-
"id": "QRR",
- "title": "re:^文化放送.+$",
- 'alt_title': 'JOQR BUNKA HOSO',
- 'thumbnail': 'https://radiko.jp/res/banner/QRR/20240423144553.png',
- 'channel': '文化放送',
- 'channel_id': 'QRR',
- 'channel_url': 'http://www.joqr.co.jp/',
- 'uploader': '文化放送',
- 'uploader_id': 'QRR',
- 'uploader_url': 'http://www.joqr.co.jp/',
-
- }
+ },
+ 'only_matching': True,
}]
_WEBPAGE_TESTS = [{
@@ -661,7 +663,7 @@ class RadikoStationButtonIE(InfoExtractor):
'id': 'CCL',
"title": "re:^FM COCOLO.+$",
'alt_title': 'FM COCOLO',
- 'thumbnail': 'https://radiko.jp/res/banner/CCL/20161014144826.png',
+ 'thumbnail': 'https://radiko.jp/v2/static/station/logo/CCL/lrtrim/688x160.png',
'channel': 'FM COCOLO',
'channel_id': 'CCL',
@@ -690,7 +692,7 @@ class RadikoPersonIE(InfoExtractor):
},{
"url": "https://radiko.jp/persons/11421",
"params": {'extractor_args': {'rajiko': {'key_station_only': ['']}}},
- "playlist_count": 1,
+ "playlist_mincount": 1,
"info_dict": {
"id": "person-11421",
},
@@ -701,9 +703,9 @@ class RadikoPersonIE(InfoExtractor):
now = rtime.RadikoTime.now(tz=rtime.JST)
- min_start = rtime.earliest_available(False)
- # we set the earliest time as the earliest we can get,
- # so, the start of the broadcast day 1 week ago
+ min_start = (now - datetime.timedelta(days=30)).broadcast_day_start()
+ # we set the earliest time as the earliest we can get (or at least, that it's possible to get),
+ # so, the start of the broadcast day 30 days ago
# that way we can get everything we can actually download, including stuff that aired at eg "26:00"
person_api_url = update_url_query("https://api.radiko.jp/program/api/v1/programs", {
diff --git a/yt_dlp_plugins/extractor/radiko_hacks.py b/yt_dlp_plugins/extractor/radiko_hacks.py
new file mode 100644
index 0000000..6486034
--- /dev/null
+++ b/yt_dlp_plugins/extractor/radiko_hacks.py
@@ -0,0 +1,65 @@
+import datetime
+import re
+
+from yt_dlp.extractor.common import InfoExtractor
+from yt_dlp.utils import (
+ join_nonempty,
+ update_url_query,
+ traverse_obj,
+)
+
+# "hacks" as in great jank/schizo shit that works anyway
+
+def _generate_as_live_playlist(self, playlist_url, start_at, end_at, domain, headers={}):
+ playlist = ""
+ chunk_length = 300 # max the api allows
+
+ duration = int(end_at.timestamp() - start_at.timestamp())
+ cursor = 0
+ chunk_num = 1
+ while cursor < duration:
+ chunk_length = min(chunk_length, duration - cursor)
+
+ chunk_start = start_at + datetime.timedelta(seconds=cursor)
+ chunk_url = update_url_query(playlist_url, {
+ "seek": chunk_start.timestring(),
+ "l": chunk_length,
+ })
+
+ chunk_playlist, real_chunk_length = _get_chunk_playlist(self, chunk_url, domain, chunk_num, headers)
+
+ playlist += chunk_playlist
+ cursor += real_chunk_length
+ chunk_num += 1
+
+ return playlist
+
+def _get_chunk_playlist(self, chunk_url, src_id, chunk_num, headers={}):
+ EXTINF_duration = re.compile(r"^#EXTINF:([\d.]+),", flags=re.MULTILINE)
+
+ playlist = ""
+ chunk_id = join_nonempty(src_id, chunk_num)
+ base_format = self._extract_m3u8_formats(
+ chunk_url, chunk_id, fatal=False, headers=headers,
+ note=f"Preparing {src_id} chunk {chunk_num}"
+ )
+ m3u8_url = traverse_obj(base_format, (..., "url",), get_all=False)
+ playlist = self._download_webpage(m3u8_url, chunk_id, note=f"Getting {src_id} chunk {chunk_num} fragments")
+
+ real_duration = 0
+ for i in EXTINF_duration.findall(playlist):
+ real_duration += float(i)
+ real_duration = round(real_duration)
+
+ # playlists can sometimes be longer than they should
+ # wowza stream does some strange things
+ # it goes along fine with every fragment 5s long as normal
+ # and then during the ad break it does one with a different length (2s here)
+ # i assume so they have a clean split to do ad insertion in? idk
+
+ # but anyway now the chunks aren't always a clean 5mins long
+ # and we get a repeated fragment going into the next chunk
+
+ # so to work around this, we track the real duration from the #EXTINF tags
+
+ return playlist, real_duration
diff --git a/yt_dlp_plugins/extractor/radiko_time.py b/yt_dlp_plugins/extractor/radiko_time.py
index b383098..d2084aa 100755
--- a/yt_dlp_plugins/extractor/radiko_time.py
+++ b/yt_dlp_plugins/extractor/radiko_time.py
@@ -36,16 +36,11 @@ class RadikoTime(datetime.datetime):
dt = datetime.datetime(date.year, date.month, date.day, 5, 0, 0, tzinfo=JST)
return dt
- def expiry(self, tf30):
- available_days = 30 if tf30 else 7
- return self.broadcast_day_end() + datetime.timedelta(days=available_days)
- # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↓↓earliest_available↓↓ AS WELL!
+ def expiry(self):
+ free = self.broadcast_day_end() + datetime.timedelta(days=7)
+ tf30 = self.broadcast_day_end() + datetime.timedelta(days=30)
-
-def earliest_available(tf30):
- available_days = 30 if tf30 else 7
- return (RadikoTime.now(tz=JST) - datetime.timedelta(days=available_days)).broadcast_day_start()
- # IF SOMETHING CHANGES HERE YOU NEED TO UPDATE ↑↑expiry↑↑ AS WELL!
+ return free, tf30
class RadikoSiteTime(RadikoTime):