yt-channel-scraper/tests/test_ratelimit.py

"""Unit tests for the politeness primitives.

Nothing here sleeps for real: `Pacer` takes an injectable clock and sleeper, and
`backoff_delay` returns the delay instead of consuming it. A test suite that
actually waited would be the first thing anyone deleted.
"""

from __future__ import annotations

import pytest

from yt_scraper.ratelimit import (
    Pacer,
    ThrottleGuard,
    backoff_delay,
    is_quota_exhausted,
    is_rate_limited,
    ydl_throttle_opts,
)

# Verbatim from the production database, where 343 of 350 `error` rows carried
# one of these. If the detector stops matching them the circuit breaker becomes
# decorative, so they are pinned here rather than paraphrased.
REAL_THROTTLE_MESSAGES = [
    "ERROR: [youtube] abcdefghijk: Video unavailable. This content isn't available, "
    "try again later. The current session has been rate-limited by YouTube for up to an hour.",
    "ERROR: [youtube] abcdefghijk: This content isn't available, try again later. "
    "The current session has been rate-limited by YouTube for up to an hour. It is recommended t",
    "HTTPError: 429 Client Error: Too Many Requests for url: https://www.youtube.com/api/timedtext",
    "ERROR: [youtube] xyz: Sign in to confirm you're not a bot",
    "HTTP Error 429: Too Many Requests",
]

# Equally verbatim: these are permanent, must NOT trip the breaker, and must
# stay distinguishable from throttling.
REAL_PERMANENT_MESSAGES = [
    "ERROR: [youtube] abcdefghijk: Join this channel to get access to members-only "
    "content like this video, and other exclusive perks.",
    "ERROR: [youtube] abcdefghijk: Private video. Sign in if you've been granted access to this video",
    "ERROR: [youtube] abcdefghijk: This video has been removed by the uploader",
    "no caption tracks published for this video",
    "subtitle downloaded but parsed empty (lang=es, format=json3)",
]


@pytest.mark.parametrize("msg", REAL_THROTTLE_MESSAGES)
def test_detects_real_throttle_messages(msg):
    assert is_rate_limited(msg) is True


@pytest.mark.parametrize("msg", REAL_PERMANENT_MESSAGES)
def test_ignores_permanent_failures(msg):
    assert is_rate_limited(msg) is False


def test_quota_is_not_treated_as_plain_throttling():
    """Google documents quota exhaustion as daily; backing off cannot fix it."""
    assert is_quota_exhausted("403 quotaExceeded") is True
    assert is_quota_exhausted("dailyLimitExceeded") is True
    assert is_quota_exhausted("rate-limited by YouTube") is False


def test_rate_limited_accepts_exception_objects():
    assert is_rate_limited(RuntimeError("HTTP Error 429: Too Many Requests")) is True


# Both spellings occur, and they are different strings. yt-dlp raises
# "HTTP Error 429: ..."; `requests` raises "429 Client Error: ... for url: ...",
# which the project wraps as "HTTPError: 429 ...". Detection used to rely on the
# prose for the second form, so a 429 with no reason phrase — routine over
# HTTP/2 — went unnoticed and the breaker never counted it.
@pytest.mark.parametrize(
    "msg",
    [
        "HTTP Error 429: Too Many Requests",
        "HTTPError: 429 Client Error: Too Many Requests for url: https://youtube.com/api/timedtext",
        "HTTP Error 429: HTTPError: 429 Client Error:  for url: https://youtube.com/api/timedtext",
        "429 Client Error:  for url: https://www.youtube.com/api/timedtext?v=x",
        "HTTP Error 408: Request Timeout",
        "HTTPError: 408 Client Error: Request Timeout for url: https://youtube.com/",
    ],
)
def test_detects_the_status_code_without_relying_on_the_reason_phrase(msg):
    assert is_rate_limited(msg) is True, f"undetected throttle: {msg}"


@pytest.mark.parametrize(
    "msg",
    [
        "HTTP Error 404: Not Found",
        "HTTPError: 403 Client Error: Forbidden for url: https://youtube.com/",
        "HTTP Error 500: Internal Server Error",
        "no caption tracks published for this video",
        # A bare number must not be read as a status code.
        "video 429 seconds long with 408 segments",
    ],
)
def test_does_not_treat_other_statuses_as_throttling(msg):
    assert is_rate_limited(msg) is False, f"false positive: {msg}"


# ---------------------------------------------------------------- backoff


def test_backoff_grows_and_is_capped():
    delays = [backoff_delay(n, base=2.0, cap=60.0) for n in range(8)]
    # Jitter is < 1s so successive doublings still order strictly until the cap.
    assert delays[0] < delays[1] < delays[2] < delays[3]
    assert all(d <= 60.0 for d in delays)
    assert delays[-1] == 60.0


def test_backoff_jitters():
    """Same attempt must not produce the same delay twice, or concurrent
    clients would re-synchronise into waves — the reason Google mandates it."""
    seen = {backoff_delay(2, base=2.0, cap=60.0) for _ in range(30)}
    assert len(seen) > 1


def test_backoff_survives_a_runaway_counter():
    assert backoff_delay(10_000, base=2.0, cap=60.0) == 60.0
    assert backoff_delay(-5, base=2.0, cap=60.0) <= 3.0


# ---------------------------------------------------------------- pacer


class FakeClock:
    def __init__(self):
        self.now = 1000.0
        self.slept: list[float] = []

    def time(self) -> float:
        return self.now

    def sleep(self, seconds: float) -> None:
        self.slept.append(seconds)
        self.now += seconds


def test_pacer_spaces_calls():
    clock = FakeClock()
    pacer = Pacer(2.0, clock=clock.time, sleeper=clock.sleep)
    assert pacer.wait() == 0.0          # first call is free
    assert pacer.wait() == pytest.approx(2.0)
    assert pacer.wait() == pytest.approx(2.0)
    assert clock.slept == [2.0, 2.0]


def test_pacer_does_not_charge_for_time_already_spent():
    """A caller slower than the interval should never wait on top of its own work."""
    clock = FakeClock()
    pacer = Pacer(2.0, clock=clock.time, sleeper=clock.sleep)
    pacer.wait()
    clock.now += 10.0                   # the request itself took 10s
    assert pacer.wait() == 0.0
    assert clock.slept == []


def test_pacer_disabled_by_default_interval():
    clock = FakeClock()
    pacer = Pacer(0.0, clock=clock.time, sleeper=clock.sleep)
    assert [pacer.wait() for _ in range(5)] == [0.0] * 5
    assert clock.slept == []


def test_pacer_charges_for_multi_request_callers():
    """One extract_info is two HTTP requests; billing it as one halves the budget."""
    clock = FakeClock()
    pacer = Pacer(2.0, clock=clock.time, sleeper=clock.sleep)
    pacer.wait(cost=2)                  # first call still free...
    assert pacer.wait() == pytest.approx(4.0)   # ...but it reserved two slots


def test_penalise_pushes_the_next_slot_out():
    clock = FakeClock()
    pacer = Pacer(1.0, clock=clock.time, sleeper=clock.sleep)
    pacer.wait()
    pacer.penalise(30.0)
    assert pacer.wait() == pytest.approx(30.0)


# ---------------------------------------------------------------- guard


def _guard() -> ThrottleGuard:
    return ThrottleGuard(threshold=3, base=0.01, cap=0.05)


def test_guard_trips_after_consecutive_throttling():
    g = _guard()
    assert g.note_failure(REAL_THROTTLE_MESSAGES[0]) > 0
    assert not g.tripped
    g.note_failure(REAL_THROTTLE_MESSAGES[0])
    assert not g.tripped
    g.note_failure(REAL_THROTTLE_MESSAGES[0])
    assert g.tripped
    assert "consecutive" in (g.tripped_reason or "")


def test_success_resets_the_streak():
    """Isolated throttled videos between successes are noise, not a banned session."""
    g = _guard()
    for _ in range(10):
        g.note_failure(REAL_THROTTLE_MESSAGES[0])
        g.note_success()
    assert not g.tripped
    assert g.throttled_total == 10


def test_permanent_failures_never_trip_the_breaker():
    """A channel with a few members-only videos must not look like a ban."""
    g = _guard()
    for msg in REAL_PERMANENT_MESSAGES * 5:
        g.note_failure(msg)
    assert not g.tripped
    assert g.throttled_total == 0


def test_mixed_failures_do_not_accumulate_into_a_trip():
    g = _guard()
    g.note_failure(REAL_THROTTLE_MESSAGES[0])
    g.note_failure("Private video")
    g.note_failure(REAL_THROTTLE_MESSAGES[0])
    g.note_failure("no caption tracks published for this video")
    g.note_failure(REAL_THROTTLE_MESSAGES[0])
    assert not g.tripped


def test_quota_trips_immediately_without_backoff():
    g = _guard()
    assert g.note_failure("403 quotaExceeded") == 0.0
    assert g.tripped
    assert "quota" in (g.tripped_reason or "").lower()


# ---------------------------------------------------------------- ydl opts


def test_throttle_opts_use_names_yt_dlp_actually_reads():
    opts = ydl_throttle_opts(2.5, extractor_retries=4, socket_timeout=15.0)
    assert opts["sleep_interval_requests"] == 2.5
    assert opts["extractor_retries"] == 4
    assert opts["socket_timeout"] == 15.0
    # The bug this whole module exists to prevent.
    assert "sleep_subrequests" not in opts