ComfyUI-Omnivoice/tests/test_web_novel_loader.py

from pathlib import Path
from unittest.mock import patch
from urllib.error import HTTPError

import pytest

from nodes import web_novel_loader as web


URL = "https://novelfull.net/example-book.html"


def index_page(numbers, pages=1):
    links = "".join(
        f'<li><a href="/example-book/chapter-{n}.html">Chapter {n}: Title {n}</a></li>'
        for n in numbers
    )
    return (
        '<a href="/example-book/chapter-999.html">Latest chapter</a>'
        f'<div id="list-chapter" data-total-page="{pages}"><ul>{links}</ul></div>'
    )


def chapter_page(n):
    return (
        f'<h1>Chapter {n}: Title {n}</h1><div id="chapter-content">'
        f'<p>Chapter {n}: Different title</p><p>Translator: Someone</p>'
        '<p>Editor: Someone</p>'
        f'<p>Body {n} begins.</p><p>freёReadNovelFull.com</p>'
        '<p>Body continues with <em>emphasis</em>.</p></div>'
        '<nav>Next Chapter</nav><div>Comments</div>'
    )


def fake_site():
    return {
        URL: index_page([1, 2], pages=2),
        URL + "?page=2": index_page([3], pages=2),
        **{f"https://novelfull.net/example-book/chapter-{n}.html": chapter_page(n)
           for n in range(1, 4)},
    }


@pytest.fixture
def cached_site(tmp_path, monkeypatch):
    monkeypatch.setattr(web, "_CACHE_DIR", tmp_path)
    monkeypatch.setattr(web.time, "sleep", lambda _: None)
    with patch.object(web, "_fetch_html", side_effect=fake_site().__getitem__) as fetch:
        yield fetch


def test_paginated_index_selects_only_requested_chapter(cached_site):
    text, title, chapter_list = web.OmniVoiceWebNovelLoader().load_novel(URL, 3, 3)
    assert text == "Body 3 begins.\n\nBody continues with emphasis."
    assert title == "Chapter 3: Title 3"
    assert chapter_list.splitlines() == [
        "1. Chapter 1: Title 1", "2. Chapter 2: Title 2", "3. Chapter 3: Title 3",
    ]
    assert [call.args[0] for call in cached_site.call_args_list] == [
        URL, URL + "?page=2", "https://novelfull.net/example-book/chapter-3.html",
    ]


def test_cached_chapter_can_be_loaded_offline(cached_site):
    loader = web.OmniVoiceWebNovelLoader()
    expected = loader.load_novel(URL, 2, 2)
    cached_site.reset_mock()
    cached_site.side_effect = RuntimeError("Offline")
    assert web.OmniVoiceWebNovelLoader().load_novel(URL, 2, 2) == expected
    cached_site.assert_not_called()


def test_counter_fetches_next_chapter_without_reloading_index(cached_site):
    loader = web.OmniVoiceWebNovelLoader()
    loader.load_novel(URL, 1, 1)
    cached_site.reset_mock()
    text, title, _ = loader.load_novel(URL, 2, 2)
    assert "Body 2 begins." in text
    assert title == "Chapter 2: Title 2"
    cached_site.assert_called_once_with("https://novelfull.net/example-book/chapter-2.html")


def test_existing_title_join_speaks_title_once(cached_site):
    body, title, _ = web.OmniVoiceWebNovelLoader().load_novel(URL, 1, 1)
    spoken = title + " \n\n" + body.replace("freёReadNovelFull.com\n", "").replace("\\", "")
    assert spoken.count("Chapter 1") == 1
    assert "Translator:" not in spoken
    assert "Editor:" not in spoken
    assert "NovelFull.com" not in spoken
    assert "Next Chapter" not in spoken
    assert "Comments" not in spoken


def test_range_and_end_below_start_match_epub_convention(cached_site):
    loader = web.OmniVoiceWebNovelLoader()
    text, _, _ = loader.load_novel(URL, 2, 999)
    assert "Body 1" not in text
    assert "Body 2" in text and "Body 3" in text
    assert "\n\n---\n\n" in text
    text, _, _ = loader.load_novel(URL, 2, 1)
    assert "Body 2" in text and "Body 3" not in text


def test_counter_beyond_last_chapter_stops_instead_of_repeating(cached_site):
    with pytest.raises(ValueError, match="outside this novel's range"):
        web.OmniVoiceWebNovelLoader().load_novel(URL, 4, 4)
    assert cached_site.call_count == 2  # index pages only


def test_refresh_redownloads_index_and_selected_chapter(cached_site):
    loader = web.OmniVoiceWebNovelLoader()
    loader.load_novel(URL, 1, 1)
    cached_site.reset_mock()
    loader.load_novel(URL, 1, 1, refresh_cache=True)
    assert cached_site.call_count == 3


def test_old_pagination_layout_and_duplicate_links():
    html = '''<div class="list-chapter">
        <a href="/example-book/chapter-1.html">Chapter 1</a>
        <a href="/example-book/chapter-1.html">Chapter 1</a>
        <a href="/other-book/chapter-1.html">Other book</a>
        <a href="https://other.example/example-book/chapter-2.html">Outside</a>
        <a href="/example-book.html?page=3">Last</a></div>'''
    chapters, pages = web._parse_index(html, URL)
    assert pages == 3
    assert chapters == [{"title": "Chapter 1", "url": "https://novelfull.net/example-book/chapter-1.html"}]


def test_repeated_index_page_is_not_cached(tmp_path, monkeypatch):
    monkeypatch.setattr(web.time, "sleep", lambda _: None)
    with patch.object(web, "_fetch_html", return_value=index_page([1], pages=2)):
        with pytest.raises(ValueError, match="index is incomplete"):
            web._load_index(URL, tmp_path, False)
    assert not (tmp_path / "index.json").exists()


@pytest.mark.parametrize("url", [
    "https://example.org/book.html", "file:///tmp/book.html",
    "https://novelfull.net@example.org/book.html", "http://127.0.0.1/book.html",
    "https://novelfull.net:9999/book.html",
    "https://novelfull.net/example-book/chapter-1.html",
])
def test_invalid_urls_are_rejected_before_network_access(url):
    with patch.object(web, "_fetch_html") as fetch:
        with pytest.raises(ValueError):
            web.OmniVoiceWebNovelLoader().load_novel(url, 1, 1)
    fetch.assert_not_called()


def test_pasted_page_url_starts_index_at_first_page(cached_site):
    text, _, _ = web.OmniVoiceWebNovelLoader().load_novel(URL + "?page=2#list", 1, 1)
    assert "Body 1 begins." in text
    assert cached_site.call_args_list[0].args == (URL,)


def test_missing_or_empty_body_does_not_narrate_page_navigation():
    chapter = {"title": "Chapter 1", "url": URL}
    for html in ["<h1>Blocked</h1><nav>Next Chapter</nav>",
                 '<div id="chapter-content"><p>Chapter 1</p><p>Translator: Someone</p></div>']:
        with pytest.raises(ValueError):
            web._parse_chapter(html, chapter)


def test_inline_breaks_ads_and_story_paragraphs_are_preserved():
    html = '''<div id="chapter-content"><p>Chapter 1</p>
        <p>First <strong>story</strong> paragraph.<br/>Second paragraph.</p>
        <iframe>Advertisement</iframe><script>bad()</script><ins>Ad</ins>
        <p>Translator: a character's job, inside the story.</p>
        <p>Before freeReadNovelFull.com after.</p></div>'''
    result = web._parse_chapter(html, {"title": "Chapter 1", "url": URL})
    assert result["text"] == (
        "First story paragraph.\n\nSecond paragraph.\n\n"
        "Translator: a character's job, inside the story.\n\nBefore after."
    )


def test_http_block_is_actionable_and_not_retried():
    with patch.object(web, "urlopen", side_effect=HTTPError(URL, 403, "Forbidden", {}, None)) as request:
        with pytest.raises(RuntimeError, match="HTTP 403"):
            web._fetch_html(URL)
    assert request.call_count == 1


def test_transient_http_error_is_retried(monkeypatch):
    monkeypatch.setattr(web.time, "sleep", lambda _: None)
    with patch.object(web, "urlopen") as request:
        response = request.return_value.__enter__.return_value
        response.geturl.return_value = URL
        response.read.return_value = b"<html>Success</html>"
        request.side_effect = [HTTPError(URL, 503, "Unavailable", {}, None), request.return_value]
        assert web._fetch_html(URL) == "<html>Success</html>"
    assert request.call_count == 2