feat(server): supply YouTube cookies to the API container via .env

Set YTDLP_COOKIES_HOST in server/.env and compose mounts the host
cookies.txt read-only at /cookies.txt and wires YTDLP_COOKIES
automatically — no compose edits per host. Falls back to /dev/null when
unset so the container still starts. Add server/.env.example, gitignore
server/.env, and document the flow.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.gitignore

@@ -2,4 +2,5 @@ __pycache__/
 *.pyc
 
 server/log.txt
+server/.env
 cookies.txt

README.md

@@ -102,6 +102,7 @@ export LIDARR_API_KEY="your-lidarr-api-key"
 | `--workers N` | Parallel metadata fetches during `--repair` (default 4). |
 | `--cookies FILE` | yt-dlp `cookies.txt` for authenticated YouTube (avoids bot-check / rate limits). |
 | `--cookies-from-browser BROWSER` | Load YouTube cookies from a local browser (e.g. `firefox`). |
+| `--remote-components SPEC` | yt-dlp EJS challenge-solver source (default `ejs:github`; needs `deno`). Fixes "signature/n challenge solving failed". `''` to disable. |
 | `--retag-from-path` | Offline: re-tag artist/title from folder + filename (see below). |
 | `-x`, `--exclude NAME` | Folder under `--root` to skip during `--repair`/`--retag-from-path` (repeatable). |
 | `--debug` | Verbose output. |
@@ -163,6 +164,13 @@ throttling. Cookies also apply to normal fetches/downloads. The same can be set
 container via `$YTDLP_COOKIES` / `$YTDLP_COOKIES_FROM_BROWSER`. If you do get flagged, **stop** —
 retrying extends it; wait ~30-60 min (429) or longer for a bot-check.
 
+**JS challenge solving (not a cookie issue).** Separately from auth, YouTube gates media formats
+behind signature / "n" JS challenges. If yt-dlp warns *"Signature solving failed"* / *"n challenge
+solving failed: Some formats may be missing"*, downloads break even with valid cookies. musicfetch
+passes `--remote-components ejs:github` by default so yt-dlp fetches the [EJS](https://github.com/yt-dlp/yt-dlp/wiki/EJS)
+solver and runs it via a local JS runtime — install **[`deno`](https://deno.land)** and use a recent
+yt-dlp. Override with `--remote-components` / `$YTDLP_REMOTE_COMPONENTS` (empty disables).
+
 #### Getting YouTube cookies
 
 > ⚠️ Use a **throwaway / secondary Google account**, not your main one — bulk automated
@@ -199,15 +207,17 @@ Use a Netscape-format cookie exporter, then point `--cookies` / `$YTDLP_COOKIES`
    ./musicfetch --repair --cookies ~/cookies.txt -o /media/music
    ```
 
-   For the API container, mount it and set the env var (see `server/docker-compose.yml`):
+   For the API container, just point it at the host file via `.env` — no compose
+   edits needed (see `server/.env.example`):
 
-   ```yaml
+   ```bash
-   environment:
+   # server/.env
-     YTDLP_COOKIES: "/cookies.txt"
+   YTDLP_COOKIES_HOST=/host/path/cookies.txt
-   volumes:
-     - /host/path/cookies.txt:/cookies.txt:ro
    ```
 
+   The compose mounts it read-only at `/cookies.txt` and sets `YTDLP_COOKIES`
+   automatically. Leave `YTDLP_COOKIES_HOST` unset to run without cookies.
+
 Cookies expire — if YouTube starts rejecting them, re-export. Treat `cookies.txt` like a
 password (it *is* your logged-in session); keep it out of git (`.gitignore` it).
 

musicfetch

@@ -54,6 +54,21 @@ DEBUG = False
 COOKIES_FILE = os.environ.get("YTDLP_COOKIES", "")
 COOKIES_FROM_BROWSER = os.environ.get("YTDLP_COOKIES_FROM_BROWSER", "")
 
+# Per-request throttle. YouTube rate-limits a session after a burst of metadata
+# fetches ("This content isn't available, try again later"), tripping even
+# single-worker runs after a while. Sleeping between requests (and randomly
+# before each download) keeps the session under the limit. Seconds; 0 disables.
+# Tunable via env or --sleep; default 1s mirrors yt-dlp's own `-t sleep` advice.
+SLEEP_REQUESTS = os.environ.get("YTDLP_SLEEP", "1")
+
+# yt-dlp JS-challenge solver. YouTube now gates formats behind signature / "n"
+# challenges that yt-dlp's bundled solver can't reliably crack; without the EJS
+# remote-component solver script it warns "Signature/n challenge solving failed:
+# Some formats may be missing" and downloads break (looks like a cookie problem,
+# isn't). Enabling ejs:github makes yt-dlp fetch the solver and run it via a local
+# JS runtime (deno). Needs a recent yt-dlp + deno present. Set empty to disable.
+REMOTE_COMPONENTS = os.environ.get("YTDLP_REMOTE_COMPONENTS", "ejs:github")
+
 
 def _cookie_args() -> list:
     """yt-dlp cookie flags (file wins over browser); empty when neither is set."""
@@ -63,6 +78,29 @@ def _cookie_args() -> list:
         return ["--cookies-from-browser", COOKIES_FROM_BROWSER]
     return []
 
+
+def _sleep_args(download: bool = False) -> list:
+    """yt-dlp throttle flags; empty when SLEEP_REQUESTS<=0. Sleeps between
+    extraction HTTP requests; for downloads also adds a randomized pre-download
+    interval (secs..2*secs) to further space out hits to YouTube."""
+    try:
+        secs = float(SLEEP_REQUESTS)
+    except (TypeError, ValueError):
+        secs = 0.0
+    if secs <= 0:
+        return []
+    args = ["--sleep-requests", str(secs)]
+    if download:
+        args += ["--sleep-interval", str(secs), "--max-sleep-interval", str(secs * 2)]
+    return args
+
+
+def _ejs_args() -> list:
+    """yt-dlp EJS challenge-solver flags; empty when REMOTE_COMPONENTS is unset."""
+    if REMOTE_COMPONENTS:
+        return ["--remote-components", REMOTE_COMPONENTS]
+    return []
+
 # Quality choices for --quality.
 QUALITY_CHOICES = ["best", "320", "m4a", "opus", "flac"]
 
@@ -368,7 +406,7 @@ def _ytmusic_search(query: str, limit: int) -> list[Hit]:
 def _ytdlp_search(query: str, limit: int) -> list[Hit]:
     try:
         result = subprocess.run(
-            ["yt-dlp", *_cookie_args(), "--flat-playlist", "-J", f"ytsearch{limit}:{query}"],
+            ["yt-dlp", *_cookie_args(), *_ejs_args(), *_sleep_args(), "--flat-playlist", "-J", f"ytsearch{limit}:{query}"],
             capture_output=True, text=True, check=True,
         )
         data = json.loads(result.stdout)
@@ -640,6 +678,8 @@ def yt_download(url_or_query: str, target_folder: Optional[str], quality: str, d
                 hit: Optional[Hit] = None, outtmpl: Optional[str] = None):
     cmd = ["yt-dlp",
            *_cookie_args(),
+           *_ejs_args(),
+           *_sleep_args(download=True),
            *_quality_args(quality),
            "--embed-metadata",
            "--embed-thumbnail",
@@ -827,7 +867,7 @@ def probe_url(url: str) -> tuple[str, str, list[Hit]]:
             if hits:
                 return "playlist", title, hits
     try:
-        result = subprocess.run(["yt-dlp", *_cookie_args(), "--flat-playlist", "-J", url],
+        result = subprocess.run(["yt-dlp", *_cookie_args(), *_ejs_args(), *_sleep_args(), "--flat-playlist", "-J", url],
                                 capture_output=True, text=True, check=True)
         data = json.loads(result.stdout)
     except (subprocess.CalledProcessError, json.JSONDecodeError) as e:
@@ -866,7 +906,7 @@ def download_single(url: str, root: str, quality: str, dry_run: bool) -> dict:
 
 
 def run_yt_dlp_get_metadata(url: str, extra_args=None) -> Optional[dict]:
-    cmd = ["yt-dlp", *_cookie_args(), "-j", "--no-playlist", *(extra_args or []), url]
+    cmd = ["yt-dlp", *_cookie_args(), *_ejs_args(), *_sleep_args(), "-j", "--no-playlist", *(extra_args or []), url]
     try:
         result = subprocess.run(cmd, capture_output=True, text=True, check=True)
         return json.loads(result.stdout)
@@ -898,18 +938,22 @@ def get_artist_from_metadata(meta: dict) -> str:
 
 def resolve_link_hits(url: str, limit: int) -> tuple[str, list[Hit]]:
     """Resolve a non-YouTube/SoundCloud link via Odesli into a search query plus
-    hits: Lidarr album candidates for "Artist - Title", followed by the EXACT
+    Lidarr-first, YouTube-fallback hits for "Artist - Title". Odesli frequently
-    YouTube track from the shared link (not a fuzzy re-search). Raises OdesliError
+    omits a YouTube link, so we always run a normal youtube_search for the
-    if the link can't be resolved."""
+    fallback; when Odesli DID supply a link, its exact track is inserted as the
+    preferred (first) YouTube hit. Raises OdesliError if the link can't resolve."""
     r = odesli_resolve(url)
     if r is None:
         raise OdesliError(url)
     query = f"{r.artist} - {r.title}".strip(" -")
-    hits = lidarr_search(query, limit)
+    hits = build_combined_hits(query, limit, yt_first=False,
+                               lidarr_only=False, yt_only=False)
     if r.youtube_url:
-        hits = hits + [Hit(source="youtube", kind="track", title=r.title,
+        exact = Hit(source="youtube", kind="track", title=r.title,
                     artist=r.artist, thumbnail=r.thumb,
-                           payload={"url": r.youtube_url})]
+                    payload={"url": r.youtube_url})
+        idx = next((i for i, h in enumerate(hits) if h.source == "youtube"), len(hits))
+        hits = hits[:idx] + [exact] + hits[idx:]
     return query, hits
 
 
@@ -1300,12 +1344,20 @@ def parse_args():
     p.add_argument("--workers", type=int, default=4,
                    help="Parallel yt-dlp metadata fetches during --repair (default 4; "
                         "raise with cookies, lower if YouTube rate-limits).")
+    p.add_argument("--sleep", type=float, metavar="SECS",
+                   help="Seconds to sleep between yt-dlp requests (0 disables). "
+                        "Avoids YouTube rate-limiting on bulk runs. Overrides "
+                        "$YTDLP_SLEEP (default 1).")
     p.add_argument("--cookies", metavar="FILE",
                    help="Path to a yt-dlp cookies.txt (authenticated requests avoid "
                         "YouTube's bot-check / rate limits). Overrides $YTDLP_COOKIES.")
     p.add_argument("--cookies-from-browser", metavar="BROWSER",
                    help="Load YouTube cookies from a local browser, e.g. firefox or "
                         "chrome. Overrides $YTDLP_COOKIES_FROM_BROWSER.")
+    p.add_argument("--remote-components", metavar="SPEC",
+                   help="yt-dlp EJS challenge-solver source (default ejs:github; "
+                        "needs deno). Fixes 'signature/n challenge solving failed'. "
+                        "Pass '' to disable. Overrides $YTDLP_REMOTE_COMPONENTS.")
     p.add_argument("--retag-from-path", action="store_true",
                    help="Offline: re-tag artist/title from folder + filename "
                         "(fixes tags damaged by a prior --repair).")
@@ -1337,13 +1389,17 @@ def _dispatch_chosen(chosen: Hit, hits: list[Hit], root: str, quality: str,
 
 
 def main():
-    global DEBUG, COOKIES_FILE, COOKIES_FROM_BROWSER
+    global DEBUG, COOKIES_FILE, COOKIES_FROM_BROWSER, SLEEP_REQUESTS, REMOTE_COMPONENTS
     args = parse_args()
     DEBUG = args.debug
+    if args.sleep is not None:
+        SLEEP_REQUESTS = args.sleep
     if args.cookies:
         COOKIES_FILE = args.cookies
     if args.cookies_from_browser:
         COOKIES_FROM_BROWSER = args.cookies_from_browser
+    if args.remote_components is not None:  # '' disables; None keeps env default
+        REMOTE_COMPONENTS = args.remote_components
     query = " ".join(args.query).strip()
 
     if args.retag_from_path:

server/.env.example

@@ -0,0 +1,13 @@
+# Copy to server/.env and fill in. docker-compose reads it automatically.
+
+# Lidarr (preferred source)
+LIDARR_API_KEY=
+
+# API key clients must send to musicfetch-api
+MUSICFETCH_API_KEY=
+
+# Optional: authenticated YouTube cookies to dodge the bot-check
+# ("Sign in to confirm you're not a bot") and rate limits.
+# Set to the host path of a Netscape-format cookies.txt (see README
+# "Getting YouTube cookies"). Leave unset to run without cookies.
+# YTDLP_COOKIES_HOST=/host/path/cookies.txt

server/docker-compose.yml

@@ -13,10 +13,14 @@ services:
       MUSICFETCH_API_KEY: "${MUSICFETCH_API_KEY}"
       MUSICFETCH_ROOT: "/media/music"
       MUSICFETCH_PORT: "6769"
-      # Optional: authenticated YouTube cookies to avoid bot-check / rate limits.
+      # Optional: authenticated YouTube cookies to avoid bot-check ("Sign in to
-      # Mount a cookies.txt below and point this at it (in-container path).
+      # confirm you're not a bot") / rate limits. To enable, set in a .env file
-      YTDLP_COOKIES: "${YTDLP_COOKIES:-}"
+      # (see server/.env.example):
+      #   YTDLP_COOKIES_HOST=/host/path/cookies.txt
+      # The host file is mounted read-only at /cookies.txt and used automatically.
+      YTDLP_COOKIES: "${YTDLP_COOKIES_HOST:+/cookies.txt}"
     volumes:
       - /media/music:/media/music
-      # Uncomment and set host path to supply cookies (see YTDLP_COOKIES above):
+      # Cookies bind: maps to the host cookies.txt when YTDLP_COOKIES_HOST is set,
-      # - /path/to/cookies.txt:/cookies.txt:ro
+      # otherwise /dev/null (harmless no-op so the container still starts).
+      - "${YTDLP_COOKIES_HOST:-/dev/null}:/cookies.txt:ro"

tests/test_odesli.py

@@ -107,28 +107,38 @@ def _resolved(yt="https://music.youtube.com/watch?v=YYY"):
                        thumb="https://img/cover.jpg", youtube_url=yt)
 
 
-def test_resolve_link_hits_builds_query_and_exact_yt(monkeypatch):
+def _ytsearch_hit():
+    return mf.Hit(source="youtube", kind="track", title="Bloom (search)",
+                  artist="ODESZA", payload={"videoId": "srch"})
+
+
+def test_resolve_link_hits_lidarr_first_then_exact_then_search(monkeypatch):
+    # Odesli supplies a YouTube link -> exact track is the FIRST youtube hit,
+    # ahead of the fuzzy youtube_search results, and after the Lidarr hits.
     monkeypatch.setattr(mf, "odesli_resolve", lambda url: _resolved())
     lid = mf.Hit(source="lidarr", kind="album", title="A Moment Apart",
                  artist="ODESZA", album="A Moment Apart", year="2017",
                  payload={"album": {"id": 9}})
     monkeypatch.setattr(mf, "lidarr_search", lambda q, limit: [lid])
+    monkeypatch.setattr(mf, "youtube_search", lambda q, limit: [_ytsearch_hit()])
     query, hits = mf.resolve_link_hits("https://open.spotify.com/track/abc", 10)
     assert query == "ODESZA - Bloom"
     assert hits[0].source == "lidarr"
-    yt = hits[-1]
+    yt = [h for h in hits if h.source == "youtube"]
-    assert yt.source == "youtube" and yt.kind == "track"
+    assert yt[0].payload.get("url") == "https://music.youtube.com/watch?v=YYY"  # exact first
-    assert yt.title == "Bloom" and yt.artist == "ODESZA"
+    assert any(h.payload.get("videoId") == "srch" for h in yt)                  # search fallback present
-    assert yt.payload["url"] == "https://music.youtube.com/watch?v=YYY"
 
 
-def test_resolve_link_hits_no_yt_link_lidarr_only(monkeypatch):
+def test_resolve_link_hits_no_odesli_yt_uses_search_fallback(monkeypatch):
+    # Regression: Odesli often omits YouTube links. With no Lidarr match and no
+    # Odesli YouTube link, a normal youtube_search must still yield hits (not empty).
     monkeypatch.setattr(mf, "odesli_resolve", lambda url: _resolved(yt=""))
-    lid = mf.Hit(source="lidarr", kind="album", title="X", artist="ODESZA",
+    monkeypatch.setattr(mf, "lidarr_search", lambda q, limit: [])
-                 payload={"album": {"id": 9}})
+    monkeypatch.setattr(mf, "youtube_search", lambda q, limit: [_ytsearch_hit()])
-    monkeypatch.setattr(mf, "lidarr_search", lambda q, limit: [lid])
     query, hits = mf.resolve_link_hits("https://open.spotify.com/track/abc", 10)
-    assert all(h.source == "lidarr" for h in hits)
+    assert hits, "must not be empty when YouTube search finds the track"
+    assert all(h.source == "youtube" for h in hits)
+    assert not any(h.payload.get("url") for h in hits)  # no exact Odesli hit injected
 
 
 def test_resolve_link_hits_odesli_miss_raises(monkeypatch):

tests/test_repair.py

@@ -401,6 +401,28 @@ def test_metadata_fetch_passes_cookies(monkeypatch):
     assert "/cookies.txt" in captured["cmd"]
 
 
+def test_ejs_args_default(monkeypatch):
+    monkeypatch.setattr(mf, "REMOTE_COMPONENTS", "ejs:github")
+    assert mf._ejs_args() == ["--remote-components", "ejs:github"]
+
+
+def test_ejs_args_disabled(monkeypatch):
+    monkeypatch.setattr(mf, "REMOTE_COMPONENTS", "")
+    assert mf._ejs_args() == []
+
+
+def test_metadata_fetch_passes_ejs(monkeypatch):
+    captured = {}
+
+    class _R:
+        stdout = '{"title": "x"}'
+    monkeypatch.setattr(mf, "REMOTE_COMPONENTS", "ejs:github")
+    monkeypatch.setattr(mf.subprocess, "run", lambda cmd, **k: captured.update(cmd=cmd) or _R())
+    mf.run_yt_dlp_get_metadata("http://u")
+    assert "--remote-components" in captured["cmd"]
+    assert "ejs:github" in captured["cmd"]
+
+
 def test_metadata_fetch_logs_stderr(monkeypatch, capsys):
     def boom(cmd, **k):
         raise mf.subprocess.CalledProcessError(