feat: multi-platform URL & playlist support via yt-dlp probe

Generalize URL handling beyond YouTube to any yt-dlp-supported site (SoundCloud, Bandcamp, etc), single tracks and playlists/sets/albums. - probe_url(): one yt-dlp --flat-playlist probe classifies playlist vs track and returns per-entry Hits; YouTube playlists still use ytmusicapi. - _track_url(): YouTube tracks keep the music.youtube album-art URL; other platforms download via their native entry URL (no more videoId reconstruction). - Per-source folders: <root>/<artist>/<extractor>/ (soundcloud/bandcamp/youtube) instead of hardcoded youtube; download_single derives source from metadata. - download_hits() downloads pre-probed Hits; API probes once and passes hits into the job closure. Replaces YouTube-only is_playlist_url/expand_playlist. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-09 00:25:58 -07:00
parent 7309ad3a29
commit f103b6c253
8 changed files with 260 additions and 102 deletions
--- a/tests/test_api_url.py
+++ b/tests/test_api_url.py
@@ -20,10 +20,17 @@ def _wait_done(client, auth, job_id, timeout=2.0):
    raise AssertionError("job never finished")


+def _mk_hit():
+    from server import mf
+    return mf.Hit(source="youtube", kind="track", title="t", artist="a", payload={"videoId": "1"})
+
+
 def test_playlist_url_batch_job(client, auth, monkeypatch):
-    monkeypatch.setattr("server.app.mf.download_playlist",
-                        lambda url, root, quality, dry_run: (2, 3, "My Mix"))
-    r = client.post("/fetch", params={"q": "https://music.youtube.com/playlist?list=PLx"}, headers=auth)
+    monkeypatch.setattr("server.app.mf.probe_url",
+                        lambda url: ("playlist", "My Mix", [_mk_hit(), _mk_hit(), _mk_hit()]))
+    monkeypatch.setattr("server.app.mf.download_hits",
+                        lambda hits, root, quality, dry_run: (2, 3))
+    r = client.post("/fetch", params={"q": "https://soundcloud.com/dj/sets/mix"}, headers=auth)
    assert r.status_code == 200
    body = r.json()
    assert body["status"] == "queued"
@@ -35,17 +42,20 @@ def test_playlist_url_batch_job(client, auth, monkeypatch):


 def test_playlist_zero_success_fails(client, auth, monkeypatch):
-    monkeypatch.setattr("server.app.mf.download_playlist",
-                        lambda url, root, quality, dry_run: (0, 3, "Dead Mix"))
+    monkeypatch.setattr("server.app.mf.probe_url",
+                        lambda url: ("playlist", "Dead Mix", [_mk_hit()]))
+    monkeypatch.setattr("server.app.mf.download_hits",
+                        lambda hits, root, quality, dry_run: (0, 3))
    body = client.post("/fetch", params={"q": "https://www.youtube.com/playlist?list=PLy"}, headers=auth).json()
    done = _wait_done(client, auth, body["job_id"])
    assert done["status"] == "failed"


 def test_single_video_url_download(client, auth, monkeypatch):
+    monkeypatch.setattr("server.app.mf.probe_url", lambda url: ("track", "Song", []))
    monkeypatch.setattr("server.app.mf.download_single",
                        lambda url, root, quality, dry_run: {"title": "Song", "artist": "A", "ok": True})
-    body = client.post("/fetch", params={"q": "https://music.youtube.com/watch?v=abc"}, headers=auth).json()
+    body = client.post("/fetch", params={"q": "https://soundcloud.com/a/song"}, headers=auth).json()
    assert body["hit"]["kind"] == "track"
    done = _wait_done(client, auth, body["job_id"])
    assert done["status"] == "done"
--- a/tests/test_mf_url_exports.py
+++ b/tests/test_mf_url_exports.py
@@ -3,6 +3,6 @@ import server.mf as smf

 def test_url_helpers_reexported():
    assert callable(smf.is_url)
-    assert callable(smf.is_playlist_url)
-    assert callable(smf.download_playlist)
+    assert callable(smf.probe_url)
+    assert callable(smf.download_hits)
    assert callable(smf.download_single)
--- a/tests/test_multiplatform.py
+++ b/tests/test_multiplatform.py
@@ -0,0 +1,79 @@
+import server.mf  # noqa: F401 — loads musicfetch, registers musicfetch_core
+import musicfetch_core as mf
+
+
+# ---- _sanitize_source ----
+def test_sanitize_source():
+    assert mf._sanitize_source("Youtube") == "youtube"
+    assert mf._sanitize_source("Soundcloud") == "soundcloud"
+    assert mf._sanitize_source("") == "downloads"
+
+
+# ---- _entry_to_hit ----
+def test_entry_to_hit_soundcloud_keeps_url_no_videoid():
+    h = mf._entry_to_hit({"id": "t1", "title": "Track", "uploader": "DJ",
+                          "ie_key": "Soundcloud", "url": "https://soundcloud.com/dj/track"})
+    assert h.payload["extractor"] == "soundcloud"
+    assert h.payload["url"] == "https://soundcloud.com/dj/track"
+    assert h.payload["videoId"] is None
+    assert h.artist == "DJ"
+
+
+def test_entry_to_hit_youtube_keeps_videoid():
+    h = mf._entry_to_hit({"id": "vid123", "title": "Song", "channel": "Chan",
+                          "ie_key": "Youtube", "url": "https://youtube.com/watch?v=vid123"})
+    assert h.payload["extractor"] == "youtube"
+    assert h.payload["videoId"] == "vid123"
+
+
+# ---- _track_url ----
+def test_track_url_youtube_prefers_music_youtube():
+    h = mf.Hit(source="youtube", kind="track", title="T", artist="A",
+               payload={"videoId": "vid", "extractor": "youtube", "url": "https://youtube.com/watch?v=vid"})
+    assert mf._track_url(h) == "https://music.youtube.com/watch?v=vid"
+
+
+def test_track_url_soundcloud_uses_native_url():
+    h = mf.Hit(source="youtube", kind="track", title="T", artist="A",
+               payload={"videoId": None, "extractor": "soundcloud", "url": "https://soundcloud.com/a/t"})
+    assert mf._track_url(h) == "https://soundcloud.com/a/t"
+
+
+def test_track_url_ytmusic_search_hit_default_youtube():
+    # ytmusicapi search hits carry only videoId (no extractor) -> music.youtube.
+    h = mf.Hit(source="youtube", kind="track", title="T", artist="A", payload={"videoId": "vid"})
+    assert mf._track_url(h) == "https://music.youtube.com/watch?v=vid"
+
+
+# ---- act_youtube routes to per-source folder ----
+def test_act_youtube_soundcloud_folder(monkeypatch):
+    captured = {}
+    monkeypatch.setattr(mf, "yt_download",
+                        lambda url, target, quality, dry_run, hit=None: captured.update(url=url, target=target) or True)
+    h = mf.Hit(source="youtube", kind="track", title="T", artist="DJ, Other",
+               payload={"videoId": None, "extractor": "soundcloud", "url": "https://soundcloud.com/dj/t"})
+    mf.act_youtube(h, "/media/music", "best", False)
+    assert captured["target"] == "/media/music/DJ/soundcloud"   # first artist only
+    assert captured["url"] == "https://soundcloud.com/dj/t"
+
+
+def test_act_youtube_youtube_folder(monkeypatch):
+    captured = {}
+    monkeypatch.setattr(mf, "yt_download",
+                        lambda url, target, quality, dry_run, hit=None: captured.update(target=target) or True)
+    h = mf.Hit(source="youtube", kind="track", title="T", artist="A",
+               payload={"videoId": "vid", "extractor": "youtube"})
+    mf.act_youtube(h, "/media/music", "best", False)
+    assert captured["target"] == "/media/music/A/youtube"
+
+
+# ---- download_single per-source folder ----
+def test_download_single_bandcamp_folder(monkeypatch):
+    monkeypatch.setattr(mf, "run_yt_dlp_get_metadata",
+                        lambda url: {"title": "Song", "artist": "Band", "extractor": "Bandcamp"})
+    captured = {}
+    monkeypatch.setattr(mf, "yt_download",
+                        lambda url, target, quality, dry_run, hit=None: captured.update(target=target) or True)
+    info = mf.download_single("https://band.bandcamp.com/track/song", "/media/music", "best", False)
+    assert captured["target"] == "/media/music/Band/bandcamp"
+    assert info == {"title": "Song", "artist": "Band", "ok": True}
--- a/tests/test_playlist.py
+++ b/tests/test_playlist.py
@@ -1,74 +1,104 @@
-import server.mf  # noqa: F401
+import json as _json
+
+import server.mf  # noqa: F401 — loads musicfetch, registers musicfetch_core
 import musicfetch_core as mf


-def test_pure_playlist_url_is_playlist():
-    assert mf.is_playlist_url("https://music.youtube.com/playlist?list=PLabc") is True
-    assert mf.is_playlist_url("https://www.youtube.com/playlist?list=PLabc") is True
-
-
-def test_watch_with_list_is_not_playlist():
-    assert mf.is_playlist_url("https://www.youtube.com/watch?v=abc&list=PLx") is False
-
-
-def test_plain_watch_is_not_playlist():
-    assert mf.is_playlist_url("https://www.youtube.com/watch?v=abc") is False
-
-
-def test_non_url_is_not_playlist():
-    assert mf.is_playlist_url("Daft Punk - Discovery") is False
-
-
 class _CP:
    def __init__(self, stdout):
        self.stdout = stdout
        self.returncode = 0


-def test_expand_playlist_ytdlp_fallback(monkeypatch):
-    import json as _json
+# ---- _is_youtube_playlist_url ----
+def test_youtube_playlist_url_true():
+    assert mf._is_youtube_playlist_url("https://music.youtube.com/playlist?list=PLabc") is True
+    assert mf._is_youtube_playlist_url("https://www.youtube.com/playlist?list=PLabc") is True
+
+
+def test_youtube_watch_with_list_is_not_playlist():
+    assert mf._is_youtube_playlist_url("https://www.youtube.com/watch?v=abc&list=PLx") is False
+
+
+def test_non_youtube_url_not_youtube_playlist():
+    # SoundCloud sets are not matched here — probe_url handles them via yt-dlp.
+    assert mf._is_youtube_playlist_url("https://soundcloud.com/user/sets/mix") is False
+
+
+# ---- probe_url ----
+def test_probe_url_youtube_playlist_uses_ytmusic(monkeypatch):
+    h = mf.Hit(source="youtube", kind="track", title="A", artist="X",
+               payload={"videoId": "1", "extractor": "youtube"})
+    monkeypatch.setattr(mf, "_ytmusic_playlist", lambda pid: ("My YT Mix", [h]))
+    monkeypatch.setattr(mf, "YTMusic", object())  # non-None to enter ytmusic branch
+    kind, title, hits = mf.probe_url("https://music.youtube.com/playlist?list=PLx")
+    assert kind == "playlist"
+    assert title == "My YT Mix"
+    assert hits == [h]
+
+
+def test_probe_url_generic_playlist_via_ytdlp(monkeypatch):
    monkeypatch.setattr(mf, "YTMusic", None)
-    payload = {"title": "My Mix", "entries": [
-        {"id": "v1", "title": "Song One", "uploader": "Artist A"},
-        {"id": "v2", "title": "Song Two", "channel": "Artist B"},
-        {"id": None, "title": "skip"},
+    payload = {"title": "SC Set", "_type": "playlist", "entries": [
+        {"id": "t1", "title": "One", "uploader": "DJ", "ie_key": "Soundcloud",
+         "url": "https://soundcloud.com/dj/one"},
+        {"id": None, "url": None, "title": "skip"},
    ]}
    monkeypatch.setattr(mf.subprocess, "run", lambda *a, **k: _CP(_json.dumps(payload)))
-    title, hits = mf.expand_playlist("https://www.youtube.com/playlist?list=PLx")
-    assert title == "My Mix"
-    assert [h.payload["videoId"] for h in hits] == ["v1", "v2"]
-    assert hits[0].artist == "Artist A"
+    kind, title, hits = mf.probe_url("https://soundcloud.com/dj/sets/sc-set")
+    assert kind == "playlist"
+    assert title == "SC Set"
+    assert len(hits) == 1
+    assert hits[0].payload["extractor"] == "soundcloud"
+    assert hits[0].payload["url"] == "https://soundcloud.com/dj/one"


-def test_download_playlist_counts_ok_and_total(monkeypatch):
+def test_probe_url_single_track(monkeypatch):
+    monkeypatch.setattr(mf, "YTMusic", None)
+    payload = {"title": "A Song", "extractor": "soundcloud"}  # no entries -> single
+    monkeypatch.setattr(mf.subprocess, "run", lambda *a, **k: _CP(_json.dumps(payload)))
+    kind, title, hits = mf.probe_url("https://soundcloud.com/dj/one")
+    assert kind == "track"
+    assert title == "A Song"
+    assert hits == []
+
+
+def test_probe_url_failure_returns_track(monkeypatch):
+    monkeypatch.setattr(mf, "YTMusic", None)
+
+    def boom(*a, **k):
+        raise mf.subprocess.CalledProcessError(1, "yt-dlp")
+    monkeypatch.setattr(mf.subprocess, "run", boom)
+    assert mf.probe_url("https://example.com/x") == ("track", "", [])
+
+
+# ---- download_hits ----
+def test_download_hits_counts(monkeypatch):
    h1 = mf.Hit(source="youtube", kind="track", title="A", artist="X", payload={"videoId": "1"})
    h2 = mf.Hit(source="youtube", kind="track", title="B", artist="Y", payload={"videoId": "2"})
    h3 = mf.Hit(source="youtube", kind="track", title="C", artist="Z", payload={"videoId": "3"})
-    monkeypatch.setattr(mf, "expand_playlist", lambda url: ("PL Title", [h1, h2, h3]))
    monkeypatch.setattr(mf, "act_youtube", lambda hit, root, quality, dry_run: hit.title != "B")
-    ok, total, title = mf.download_playlist("u", "/tmp", "best", False)
-    assert (ok, total, title) == (2, 3, "PL Title")
+    assert mf.download_hits([h1, h2, h3], "/tmp", "best", False) == (2, 3)


-def test_download_playlist_track_exception_counts_as_failure(monkeypatch):
+def test_download_hits_track_exception_is_failure(monkeypatch):
    h1 = mf.Hit(source="youtube", kind="track", title="A", artist="X", payload={"videoId": "1"})
    h2 = mf.Hit(source="youtube", kind="track", title="B", artist="Y", payload={"videoId": "2"})
-    monkeypatch.setattr(mf, "expand_playlist", lambda url: ("T", [h1, h2]))

    def fake_act(hit, root, quality, dry_run):
        if hit.title == "B":
            raise RuntimeError("boom")
        return True
    monkeypatch.setattr(mf, "act_youtube", fake_act)
-    ok, total, _ = mf.download_playlist("u", "/tmp", "best", False)
-    assert (ok, total) == (1, 2)
+    assert mf.download_hits([h1, h2], "/tmp", "best", False) == (1, 2)


+# ---- yt_download bool ----
 def test_yt_download_returns_true_on_zero_exit(monkeypatch):
    monkeypatch.setattr(mf.os, "makedirs", lambda *a, **k: None)
    monkeypatch.setattr(mf.subprocess, "run", lambda *a, **k: _CP(""))
    assert mf.yt_download("u", "/tmp/x", "best", False) is True


-def test_yt_download_dry_run_returns_true(monkeypatch):
+def test_yt_download_dry_run_returns_true():
    assert mf.yt_download("u", "/tmp/x", "best", True) is True