diff options
Diffstat (limited to 'misc')
| -rwxr-xr-x | misc/generate_html.py | 84 | ||||
| -rw-r--r-- | misc/how to do a release | 42 | ||||
| -rwxr-xr-x | misc/old_generate_changelog.py (renamed from misc/generate_changelog.py) | 0 | ||||
| -rwxr-xr-x | misc/streammon.py | 53 | ||||
| -rwxr-xr-x | misc/test_areas.py (renamed from misc/test-tokens.py) | 0 | ||||
| -rwxr-xr-x | misc/test_extractors.py | 174 |
6 files changed, 329 insertions, 24 deletions
diff --git a/misc/generate_html.py b/misc/generate_html.py new file mode 100755 index 0000000..0e15d6a --- /dev/null +++ b/misc/generate_html.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import os +import hashlib +import re + +pip_index = open("index.html", "w") + +pip_index.write("""<!DOCTYPE HTML> +<html lang="en-GB"> +<head> + <title>yt-dlp-rajiko pip index</title> + <link rel="canonical" href="https://427738.xyz/yt-dlp-rajiko/pip/yt-dlp-rajiko/"> +</head> +<body> + +<ul> +""") + +site_sha256 = [] + +tarballs = [] +wheels = [] + +for item in sorted(os.listdir()):#, key=lambda x: x.name): + if os.path.islink(item): + continue + + if item.endswith(".tar.gz"): + tarballs.append(item) + elif item.endswith(".whl"): + wheels.append(item) + else: + continue + + pip_index.write("\t<li>") + pip_index.write('<a href="') + pip_index.write(item) + + with open(item, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + + pip_index.write("#sha256=") + pip_index.write(checksum) + pip_index.write('">') + pip_index.write(item) + pip_index.write("</a>\n") + + site_string = checksum + " " + '<a href="dl/' + item + '">' + item + "</a><br>" + site_sha256.append(site_string) + +pip_index.write("""</ul> + +</body> +</html>""") + +latest_tarball = tarballs[-1] +latest_wheel = wheels[-1] +print(latest_tarball, latest_wheel) + +os.remove("yt_dlp_rajiko-latest.tar.gz") +os.symlink(latest_tarball, "yt_dlp_rajiko-latest.tar.gz") + +os.remove("yt_dlp_rajiko-latest.whl") +os.symlink(latest_wheel, "yt_dlp_rajiko-latest.whl") + +site_sha256.reverse() + +latest_list = site_sha256[:2] +previous_list = site_sha256[2:] + +latest = "\n".join(["<!-- LATEST SHA256 START -->", "<code>", "\n".join(latest_list), "</code>", "<!-- LATEST SHA256 END -->"]) + +previous = "\n".join(["<!-- PREVIOUS SHA256 START -->", "<code>", "\n".join(previous_list), "</code>", "<!-- PREVIOUS SHA256 END -->"]) + +for i in ["../../index.html", "../../index.ja.html"]: + with open(i, "r+") as f: + page = f.read() + + page = re.sub(r"<!-- LATEST SHA256 START -->.+<!-- LATEST SHA256 END -->", latest, page, flags=re.DOTALL) + page = re.sub(r"<!-- PREVIOUS SHA256 START -->.+<!-- PREVIOUS SHA256 END -->", previous, page, flags=re.DOTALL) + + f.seek(0) + f.truncate(0) + f.write(page) diff --git a/misc/how to do a release b/misc/how to do a release index f5eb85f..6e91e14 100644 --- a/misc/how to do a release +++ b/misc/how to do a release @@ -8,40 +8,34 @@ python3 -m build and then put BOTH items from `dist` into the pip index dir - ~/site2/yt-dlp-rajiko/pip/yt-dlp-rajiko/ because without the .whl pip has to "build" it itself, with all the stuff that needs to be installed for that to work -update the pip index html -update the dl/ "latest" symlinks -## update the changelog file - -~/site2/yt-dlp-rajiko/CHANGELOG +run script to update the pip index html and the dl/ "latest" symlinks +this also updates the sha256s on the site -``` -version number -date (git log v1.0 --pretty --date=rfc2822) -url: whl download link -sha256: sha256 of the whl -brief summary of the release -can span multiple lines +## update the changelog file -bullet points of changes, 1 per line -simple present tense, third person singular - continue "this release...", eg.. -fixes a bug where the computer would explode -makes downloading 5000x faster -``` +write in html, paste into the feed xml like <![CDATA[ +stuff +]]> +make sure to set the link, date +to get date use: +git log --pretty --date=rfc2822 -./generate_changelog.py to make the new rss feed +include the pip instructions, sha256sum etc -## update the website +now push to the server -move the previous release into the "Previous releases" <details> -update the sha256 (just sha256 command in the pip dir) -update the whl link -repeat for japanese version +!!NEW!! +upload to pip proper as well +go to dl/ dir and do +twine upload yt_dlp_rajiko-1.x* -now push to the server ## update github paste the changelog output into a github release, upload the new builds +change link at the bottom to just "below" + +post in the radiko thread on 5ch if i can be bothered and thats probably all diff --git a/misc/generate_changelog.py b/misc/old_generate_changelog.py index 1bce073..1bce073 100755 --- a/misc/generate_changelog.py +++ b/misc/old_generate_changelog.py diff --git a/misc/streammon.py b/misc/streammon.py new file mode 100755 index 0000000..4051833 --- /dev/null +++ b/misc/streammon.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# monitor stream APIs for any changes, so I can check they don't break anything +# run via cronjob every now and then + +import difflib +import os +import sys +from datetime import datetime + +import requests + +s = requests.Session() + +DISCORD_WEBHOOK = "PUT WEBHOOK HERE" +STREAMS_API = "https://radiko.jp/v3/station/stream/{device}/{station}.xml" + +if len(sys.argv) > 1: + PATH = sys.argv[1] +else: + PATH = "" + +devices = ('pc_html5', 'aSmartPhone7a', 'aSmartPhone8') +stations = ('FMT', 'CCL', 'NORTHWAVE', 'TBS') + +for device in devices: + for station in stations: + url = STREAMS_API.format(device=device, station=station) + now = s.get(url).text + + filename = f"{PATH}{station}-{device}.xml" + with open(filename, "a+") as f: + f.seek(0) + past = f.read() + + modtime = datetime.fromtimestamp(os.path.getmtime(filename)) + diff = difflib.unified_diff( + past.splitlines(), now.splitlines(), + fromfile=url, tofile=url, + fromfiledate=str(modtime), tofiledate=str(datetime.now()), + ) + + diff_str = "\n".join(diff) + if diff_str != "": + f.truncate(0) + f.write(now) + + s.post(DISCORD_WEBHOOK, json={ + "embeds": [{ + "type": "rich", + "title": f"Streams changed: {station} {device}", + "description": "\n".join(("```diff", diff_str, "```")) + }] + }) diff --git a/misc/test-tokens.py b/misc/test_areas.py index ba6475f..ba6475f 100755 --- a/misc/test-tokens.py +++ b/misc/test_areas.py diff --git a/misc/test_extractors.py b/misc/test_extractors.py new file mode 100755 index 0000000..18e9783 --- /dev/null +++ b/misc/test_extractors.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# programmes expire, so i have to update the times in the tests every time i run them +# but thats a massive ballache, so i end up just not running them, which leads to cockups +# so, this script has the tests automatically use the latest episode as you run it, by setting dynamically generated time values +# everything else is always the same so it should be fine lol + + +import datetime +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, "/home/g/Downloads/yt-dlp/") # TODO: un-hardcode. has to be the source/git repo because pip doesnt carry the tests + +from yt_dlp_plugins.extractor import radiko_time as rtime + +MON, TUE, WED, THU, FRI, SAT, SUN = range(7) +weekdays = {0: "MON", 1: "TUE", 2: "WED", 3: "THU", 4: "FRI", 5: "SAT", 6: "SUN"} + +now = rtime.RadikoTime.now(tz = rtime.JST) +UTC = datetime.timezone.utc + +def get_latest_airtimes(now, weekday, hour, minute, duration): + days_after_weekday = (7 - (now.weekday() - weekday)) % 7 + latest_airdate = (now + datetime.timedelta(days=days_after_weekday)).replace(hour=hour, minute=minute, second=0, microsecond=0) + if (latest_airdate + duration) > now: + latest_airdate -= datetime.timedelta(days=7) + return latest_airdate, latest_airdate + duration + +def get_test_timefields(airtime, release_time): + return { + "timestamp": airtime.timestamp(), + "release_timestamp": release_time.timestamp(), + "upload_date": airtime.astimezone(UTC).strftime("%Y%m%d"), + "release_date": release_time.astimezone(UTC).strftime("%Y%m%d"), + + "duration": (release_time - airtime).total_seconds(), + } + + + + +from yt_dlp_plugins.extractor.radiko import RadikoTimeFreeIE, RadikoShareIE, RadikoLiveIE, RadikoShareIE, RadikoPersonIE, RadikoStationButtonIE + +RadikoTimeFreeIE._TESTS = [] + + + +# TOKYO MOON - interfm - EVERY FRI 2300 +airtime, release_time = get_latest_airtimes(now, FRI, 23, 0, datetime.timedelta(hours=1)) +RadikoTimeFreeIE._TESTS.append({ + "url": f"https://radiko.jp/#!/ts/INT/{airtime.timestring()}", + "info_dict": { + "ext": "m4a", + "id": f"INT-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + + 'title': 'TOKYO MOON', + 'description': 'md5:4185349a530cfc4d0580e6996a511273', + 'uploader': 'interfm', + 'uploader_id': 'INT', + 'uploader_url': 'https://www.interfm.co.jp/', + 'channel': 'interfm', + 'channel_id': 'INT', + 'channel_url': 'https://www.interfm.co.jp/', + 'thumbnail': 'https://program-static.cf.radiko.jp/ehwtw6mcvy.jpg', + 'chapters': list, + 'tags': ['松浦俊夫', 'ジャズの魅力を楽しめる'], + 'cast': ['松浦\u3000俊夫'], + 'series': 'Tokyo Moon', + 'live_status': 'was_live', + } +}) + + +# late-night/v. early morning show, to test broadcast day handling +# this should be monday 27:00 / tuesday 03:00 +airtime, release_time = get_latest_airtimes(now, TUE, 3, 0, datetime.timedelta(hours=2)) +RadikoTimeFreeIE._TESTS.append({ + "url": f"https://radiko.jp/#!/ts/TBS/{airtime.timestring()}", + "info_dict": { + "ext": "m4a", + "id": f"TBS-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + 'title': 'CITY CHILL CLUB', + 'description': r"re:^目を閉じて…リラックスして[\S\s]+chill@tbs.co.jp$", + 'uploader': 'TBSラジオ', + 'uploader_id': 'TBS', + 'uploader_url': 'https://www.tbsradio.jp/', + 'channel': 'TBSラジオ', + 'channel_id': 'TBS', + 'channel_url': 'https://www.tbsradio.jp/', + 'thumbnail': 'https://program-static.cf.radiko.jp/nrf8fowbjo.jpg', + 'chapters': list, + 'tags': ['CCC905', '音楽との出会いが楽しめる', '人気アーティストトーク', '音楽プロデューサー出演', 'ドライブ中におすすめ', '寝る前におすすめ', '学生におすすめ'], + 'cast': list, + 'series': 'CITY CHILL CLUB', + 'live_status': 'was_live', + }, +}) + + +# testing 29-hour clock handling +airtime, release_time = get_latest_airtimes(now, WED, 0, 0, datetime.timedelta(minutes=55)) +share_timestring = (airtime - datetime.timedelta(days=1)).strftime("%Y%m%d") + "240000" + +RadikoShareIE._TESTS = [{ + "url": f"http://radiko.jp/share/?sid=FMT&t={share_timestring}", + "info_dict": { + "live_status": "was_live", + "ext": "m4a", + "id": f"FMT-{airtime.timestring()}", + + **get_test_timefields(airtime, release_time), + + "title": "JET STREAM", + "series": "JET STREAM", + "description": r"re:^JET STREAM・・・作家が描く世界への旅。[\s\S]+https://www.tfm.co.jp/f/jetstream/message$", + "chapters": list, + "thumbnail": "https://program-static.cf.radiko.jp/greinlrspi.jpg", + + "channel": "TOKYO FM", + "channel_id": "FMT", + "channel_url": "https://www.tfm.co.jp/", + "uploader": "TOKYO FM", + "uploader_id": "FMT", + "uploader_url": "https://www.tfm.co.jp/", + + "cast": ["福山雅治"], + "tags": ["福山雅治", "夜間飛行", "音楽との出会いが楽しめる", "朗読を楽しめる", "寝る前に聴きたい"], + }, + }] + + + +IEs = [ + RadikoTimeFreeIE, RadikoShareIE, + RadikoLiveIE, RadikoShareIE, RadikoPersonIE, RadikoStationButtonIE +] + +import test.helper as th + +# override to only get testcases from our IEs + +def _new_gettestcases(include_onlymatching=False): + import yt_dlp.plugins as plugins + plugins.load_all_plugins() + + for ie in IEs: + yield from ie.get_testcases(include_onlymatching) + +def _new_getwebpagetestcases(): + import yt_dlp.plugins as plugins + plugins.load_all_plugins() + + for ie in IEs: + for tc in ie.get_webpage_testcases(): + tc.setdefault('add_ie', []).append('Generic') + yield tc + +th.gettestcases = _new_gettestcases +th.getwebpagetestcases = _new_getwebpagetestcases + +import test.test_download as td + +class TestDownload(td.TestDownload): + pass + +if __name__ == "__main__": + unittest.main() |