Improve Serienstream genre loading and bump to 0.1.48

addon/plugins/serienstream_plugin.py

@@ -10,9 +10,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from datetime import datetime
+from html import unescape
+import json
 import hashlib
 import os
 import re
+import time
+import unicodedata
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypeAlias
 
 try:  # pragma: no cover - optional dependency
@@ -68,6 +72,9 @@ HEADERS = {
     "Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
     "Connection": "keep-alive",
 }
+SESSION_CACHE_TTL_SECONDS = 300
+SESSION_CACHE_PREFIX = "viewit.serienstream"
+SESSION_CACHE_MAX_TITLE_URLS = 800
 
 
 @dataclass
@@ -127,6 +134,68 @@ def _absolute_url(href: str) -> str:
     return f"{_get_base_url()}{href}" if href.startswith("/") else href
 
 
+def _session_window() -> Any:
+    if xbmcgui is None:
+        return None
+    try:
+        return xbmcgui.Window(10000)
+    except Exception:
+        return None
+
+
+def _session_cache_key(name: str) -> str:
+    base_hash = hashlib.sha1(_get_base_url().encode("utf-8")).hexdigest()[:12]
+    return f"{SESSION_CACHE_PREFIX}.{base_hash}.{name}"
+
+
+def _session_cache_get(name: str) -> Any:
+    window = _session_window()
+    if window is None:
+        return None
+    raw = ""
+    try:
+        raw = window.getProperty(_session_cache_key(name)) or ""
+    except Exception:
+        return None
+    if not raw:
+        return None
+    try:
+        payload = json.loads(raw)
+    except Exception:
+        return None
+    if not isinstance(payload, dict):
+        return None
+    expires_at = payload.get("expires_at")
+    data = payload.get("data")
+    try:
+        if float(expires_at or 0) <= time.time():
+            return None
+    except Exception:
+        return None
+    return data
+
+
+def _session_cache_set(name: str, data: Any, *, ttl_seconds: int = SESSION_CACHE_TTL_SECONDS) -> None:
+    window = _session_window()
+    if window is None:
+        return
+    payload = {
+        "expires_at": float(time.time() + max(1, int(ttl_seconds))),
+        "data": data,
+    }
+    try:
+        raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+    except Exception:
+        return
+    # Kodi-Properties sind kein Dauer-Storage; begrenzen, damit UI stabil bleibt.
+    if len(raw) > 240_000:
+        return
+    try:
+        window.setProperty(_session_cache_key(name), raw)
+    except Exception:
+        return
+
+
 def _normalize_series_url(identifier: str) -> str:
     if identifier.startswith("http://") or identifier.startswith("https://"):
         return identifier.rstrip("/")
@@ -279,7 +348,7 @@ def _get_soup(url: str, *, session: Optional[RequestsSession] = None) -> Beautif
     return BeautifulSoup(response.text, "html.parser")
 
 
-def _get_soup_simple(url: str) -> BeautifulSoupT:
+def _get_html_simple(url: str) -> str:
     _ensure_requests()
     _log_visit(url)
     sess = get_requests_session("serienstream", headers=HEADERS)
@@ -291,10 +360,36 @@ def _get_soup_simple(url: str) -> BeautifulSoupT:
         raise
     if response.url and response.url != url:
         _log_url(response.url, kind="REDIRECT")
-    _log_response_html(url, response.text)
-    if _looks_like_cloudflare_challenge(response.text):
+    body = response.text
+    _log_response_html(url, body)
+    if _looks_like_cloudflare_challenge(body):
         raise RuntimeError("Cloudflare-Schutz erkannt. requests reicht ggf. nicht aus.")
-    return BeautifulSoup(response.text, "html.parser")
+    return body
+
+
+def _get_soup_simple(url: str) -> BeautifulSoupT:
+    body = _get_html_simple(url)
+    return BeautifulSoup(body, "html.parser")
+
+
+def _extract_genre_names_from_html(body: str) -> List[str]:
+    names: List[str] = []
+    seen: set[str] = set()
+    pattern = re.compile(
+        r"<div[^>]*class=[\"'][^\"']*background-1[^\"']*[\"'][^>]*>.*?<h3[^>]*>(.*?)</h3>",
+        re.IGNORECASE | re.DOTALL,
+    )
+    for match in pattern.finditer(body or ""):
+        text = re.sub(r"<[^>]+>", " ", match.group(1) or "")
+        text = unescape(re.sub(r"\s+", " ", text)).strip()
+        if not text:
+            continue
+        key = text.casefold()
+        if key in seen:
+            continue
+        seen.add(key)
+        names.append(text)
+    return names
 
 
 def search_series(query: str) -> List[SeriesResult]:
@@ -584,10 +679,10 @@ def _extract_latest_episodes(soup: BeautifulSoupT) -> List[LatestEpisode]:
         episode_text = (anchor.select_one(".ep-episode").get_text(strip=True) if anchor.select_one(".ep-episode") else "").strip()
         season_number: Optional[int] = None
         episode_number: Optional[int] = None
-        match = re.search(r"S\\s*(\\d+)", season_text, re.IGNORECASE)
+        match = re.search(r"S\s*(\d+)", season_text, re.IGNORECASE)
         if match:
             season_number = int(match.group(1))
-        match = re.search(r"E\\s*(\\d+)", episode_text, re.IGNORECASE)
+        match = re.search(r"E\s*(\d+)", episode_text, re.IGNORECASE)
         if match:
             episode_number = int(match.group(1))
         if season_number is None or episode_number is None:
@@ -687,10 +782,15 @@ class SerienstreamPlugin(BasisPlugin):
 
     def __init__(self) -> None:
         self._series_results: Dict[str, SeriesResult] = {}
+        self._title_url_cache: Dict[str, str] = self._load_title_url_cache()
+        self._genre_names_cache: Optional[List[str]] = None
         self._season_cache: Dict[str, List[SeasonInfo]] = {}
         self._season_links_cache: Dict[str, List[SeasonInfo]] = {}
         self._episode_label_cache: Dict[Tuple[str, str], Dict[str, EpisodeInfo]] = {}
         self._catalog_cache: Optional[Dict[str, List[SeriesResult]]] = None
+        self._genre_group_cache: Dict[str, Dict[str, List[str]]] = {}
+        self._genre_page_titles_cache: Dict[Tuple[str, int], List[str]] = {}
+        self._genre_page_count_cache: Dict[str, int] = {}
         self._popular_cache: Optional[List[SeriesResult]] = None
         self._requests_available = REQUESTS_AVAILABLE
         self._default_preferred_hosters: List[str] = list(DEFAULT_PREFERRED_HOSTERS)
@@ -713,6 +813,132 @@ class SerienstreamPlugin(BasisPlugin):
                 print(f"Importfehler: {REQUESTS_IMPORT_ERROR}")
             return
 
+    def _load_title_url_cache(self) -> Dict[str, str]:
+        raw = _session_cache_get("title_urls")
+        if not isinstance(raw, dict):
+            return {}
+        result: Dict[str, str] = {}
+        for key, value in raw.items():
+            key_text = str(key or "").strip().casefold()
+            url_text = str(value or "").strip()
+            if not key_text or not url_text:
+                continue
+            result[key_text] = url_text
+        return result
+
+    def _save_title_url_cache(self) -> None:
+        if not self._title_url_cache:
+            return
+        # Begrenzt die Session-Daten auf die jüngsten Einträge.
+        while len(self._title_url_cache) > SESSION_CACHE_MAX_TITLE_URLS:
+            self._title_url_cache.pop(next(iter(self._title_url_cache)))
+        _session_cache_set("title_urls", self._title_url_cache)
+
+    def _remember_series_result(self, title: str, url: str, description: str = "") -> None:
+        title = (title or "").strip()
+        url = (url or "").strip()
+        if not title:
+            return
+        if url:
+            self._series_results[title] = SeriesResult(title=title, description=description, url=url)
+            cache_key = title.casefold()
+            if self._title_url_cache.get(cache_key) != url:
+                self._title_url_cache[cache_key] = url
+                self._save_title_url_cache()
+            return
+        current = self._series_results.get(title)
+        if current is None:
+            self._series_results[title] = SeriesResult(title=title, description=description, url="")
+
+    @staticmethod
+    def _season_links_cache_name(series_url: str) -> str:
+        digest = hashlib.sha1((series_url or "").encode("utf-8")).hexdigest()[:20]
+        return f"season_links.{digest}"
+
+    @staticmethod
+    def _season_episodes_cache_name(season_url: str) -> str:
+        digest = hashlib.sha1((season_url or "").encode("utf-8")).hexdigest()[:20]
+        return f"season_episodes.{digest}"
+
+    def _load_session_season_links(self, series_url: str) -> Optional[List[SeasonInfo]]:
+        raw = _session_cache_get(self._season_links_cache_name(series_url))
+        if not isinstance(raw, list):
+            return None
+        seasons: List[SeasonInfo] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                continue
+            try:
+                number = int(item.get("number"))
+            except Exception:
+                continue
+            url = str(item.get("url") or "").strip()
+            if number <= 0 or not url:
+                continue
+            seasons.append(SeasonInfo(number=number, url=url, episodes=[]))
+        if not seasons:
+            return None
+        seasons.sort(key=lambda s: s.number)
+        return seasons
+
+    def _save_session_season_links(self, series_url: str, seasons: List[SeasonInfo]) -> None:
+        payload = [{"number": int(season.number), "url": season.url} for season in seasons if season.url]
+        if payload:
+            _session_cache_set(self._season_links_cache_name(series_url), payload)
+
+    def _load_session_season_episodes(self, season_url: str) -> Optional[List[EpisodeInfo]]:
+        raw = _session_cache_get(self._season_episodes_cache_name(season_url))
+        if not isinstance(raw, list):
+            return None
+        episodes: List[EpisodeInfo] = []
+        for item in raw:
+            if not isinstance(item, dict):
+                continue
+            try:
+                number = int(item.get("number"))
+            except Exception:
+                continue
+            title = str(item.get("title") or "").strip()
+            original_title = str(item.get("original_title") or "").strip()
+            url = str(item.get("url") or "").strip()
+            season_label = str(item.get("season_label") or "").strip()
+            languages = [str(lang).strip() for lang in list(item.get("languages") or []) if str(lang).strip()]
+            hosters = [str(host).strip() for host in list(item.get("hosters") or []) if str(host).strip()]
+            if number <= 0:
+                continue
+            episodes.append(
+                EpisodeInfo(
+                    number=number,
+                    title=title or f"Episode {number}",
+                    original_title=original_title,
+                    url=url,
+                    season_label=season_label,
+                    languages=languages,
+                    hosters=hosters,
+                )
+            )
+        if not episodes:
+            return None
+        episodes.sort(key=lambda item: item.number)
+        return episodes
+
+    def _save_session_season_episodes(self, season_url: str, episodes: List[EpisodeInfo]) -> None:
+        payload = []
+        for item in episodes:
+            payload.append(
+                {
+                    "number": int(item.number),
+                    "title": item.title,
+                    "original_title": item.original_title,
+                    "url": item.url,
+                    "season_label": item.season_label,
+                    "languages": list(item.languages or []),
+                    "hosters": list(item.hosters or []),
+                }
+            )
+        if payload:
+            _session_cache_set(self._season_episodes_cache_name(season_url), payload)
+
     def _ensure_catalog(self) -> Dict[str, List[SeriesResult]]:
         if self._catalog_cache is not None:
             return self._catalog_cache
@@ -720,14 +946,38 @@ class SerienstreamPlugin(BasisPlugin):
         catalog_url = f"{_get_base_url()}/serien?by=genre"
         soup = _get_soup_simple(catalog_url)
         self._catalog_cache = parse_series_catalog(soup)
+        _session_cache_set("genres", sorted(self._catalog_cache.keys(), key=str.casefold))
         return self._catalog_cache
 
+    def _ensure_genre_names(self) -> List[str]:
+        if self._genre_names_cache is not None:
+            return list(self._genre_names_cache)
+        cached = _session_cache_get("genres")
+        if isinstance(cached, list):
+            genres = [str(value).strip() for value in cached if str(value).strip()]
+            if genres:
+                self._genre_names_cache = sorted(set(genres), key=str.casefold)
+                return list(self._genre_names_cache)
+        catalog_url = f"{_get_base_url()}/serien?by=genre"
+        try:
+            body = _get_html_simple(catalog_url)
+            genres = _extract_genre_names_from_html(body)
+        except Exception:
+            genres = []
+        if not genres:
+            catalog = self._ensure_catalog()
+            genres = sorted(catalog.keys(), key=str.casefold)
+        else:
+            genres = sorted(set(genres), key=str.casefold)
+        self._genre_names_cache = list(genres)
+        _session_cache_set("genres", self._genre_names_cache)
+        return list(self._genre_names_cache)
+
     def genres(self) -> List[str]:
         """Optional: Liefert alle Genres aus dem Serien-Katalog."""
         if not self._requests_available:
             return []
-        catalog = self._ensure_catalog()
-        return sorted(catalog.keys(), key=str.casefold)
+        return self._ensure_genre_names()
 
     def capabilities(self) -> set[str]:
         """Meldet unterstützte Features für Router-Menüs."""
@@ -738,7 +988,8 @@ class SerienstreamPlugin(BasisPlugin):
         if not self._requests_available:
             return []
         entries = self._ensure_popular()
-        self._series_results.update({entry.title: entry for entry in entries if entry.title})
+        for entry in entries:
+            self._remember_series_result(entry.title, entry.url, entry.description)
         return [entry.title for entry in entries if entry.title]
 
     def titles_for_genre(self, genre: str) -> List[str]:
@@ -752,9 +1003,167 @@ class SerienstreamPlugin(BasisPlugin):
             return self.popular_series()
         catalog = self._ensure_catalog()
         entries = catalog.get(genre, [])
-        self._series_results.update({entry.title: entry for entry in entries if entry.title})
+        for entry in entries:
+            self._remember_series_result(entry.title, entry.url, entry.description)
         return [entry.title for entry in entries if entry.title]
 
+    @staticmethod
+    def _title_group_key(title: str) -> str:
+        raw = (title or "").strip()
+        if not raw:
+            return "#"
+        for char in raw:
+            if char.isdigit():
+                return "0-9"
+            if char.isalpha():
+                normalized = char.casefold()
+                if normalized == "ä":
+                    normalized = "a"
+                elif normalized == "ö":
+                    normalized = "o"
+                elif normalized == "ü":
+                    normalized = "u"
+                elif normalized == "ß":
+                    normalized = "s"
+                return normalized.upper()
+        return "#"
+
+    @classmethod
+    def _group_matches(cls, group_code: str, title: str) -> bool:
+        key = cls._title_group_key(title)
+        if group_code == "0-9":
+            return key == "0-9"
+        if key == "0-9" or key == "#":
+            return False
+        if group_code == "A-E":
+            return "A" <= key <= "E"
+        if group_code == "F-J":
+            return "F" <= key <= "J"
+        if group_code == "K-O":
+            return "K" <= key <= "O"
+        if group_code == "P-T":
+            return "P" <= key <= "T"
+        if group_code == "U-Z":
+            return "U" <= key <= "Z"
+        return False
+
+    def _ensure_genre_group_cache(self, genre: str) -> Dict[str, List[str]]:
+        cached = self._genre_group_cache.get(genre)
+        if cached is not None:
+            return cached
+        titles = self.titles_for_genre(genre)
+        grouped: Dict[str, List[str]] = {}
+        for title in titles:
+            for code in ("A-E", "F-J", "K-O", "P-T", "U-Z", "0-9"):
+                if self._group_matches(code, title):
+                    grouped.setdefault(code, []).append(title)
+                    break
+        for code in grouped:
+            grouped[code].sort(key=str.casefold)
+        self._genre_group_cache[genre] = grouped
+        return grouped
+
+    @staticmethod
+    def _genre_slug(genre: str) -> str:
+        value = (genre or "").strip().casefold()
+        value = value.replace("&", " und ")
+        value = unicodedata.normalize("NFKD", value)
+        value = "".join(ch for ch in value if not unicodedata.combining(ch))
+        value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
+        return value
+
+    def _fetch_genre_page_titles(self, genre: str, page: int) -> Tuple[List[str], int]:
+        slug = self._genre_slug(genre)
+        if not slug:
+            return [], 1
+        cache_key = (slug, page)
+        cached = self._genre_page_titles_cache.get(cache_key)
+        cached_pages = self._genre_page_count_cache.get(slug)
+        if cached is not None and cached_pages is not None:
+            return list(cached), int(cached_pages)
+        url = f"{_get_base_url()}/genre/{slug}"
+        if page > 1:
+            url = f"{url}?page={int(page)}"
+        soup = _get_soup_simple(url)
+        titles: List[str] = []
+        seen: set[str] = set()
+        for anchor in soup.select("a.show-card[href]"):
+            href = (anchor.get("href") or "").strip()
+            series_url = _absolute_url(href).split("#", 1)[0].split("?", 1)[0].rstrip("/")
+            if "/serie/" not in series_url:
+                continue
+            img = anchor.select_one("img[alt]")
+            title = ((img.get("alt") if img else "") or "").strip()
+            if not title:
+                continue
+            key = title.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            self._remember_series_result(title, series_url)
+            titles.append(title)
+        max_page = 1
+        for anchor in soup.select("a[href*='?page=']"):
+            href = (anchor.get("href") or "").strip()
+            match = re.search(r"[?&]page=(\d+)", href)
+            if not match:
+                continue
+            try:
+                max_page = max(max_page, int(match.group(1)))
+            except Exception:
+                continue
+        self._genre_page_titles_cache[cache_key] = list(titles)
+        self._genre_page_count_cache[slug] = max_page
+        return list(titles), max_page
+
+    def titles_for_genre_group_page(self, genre: str, group_code: str, page: int = 1, page_size: int = 10) -> List[str]:
+        genre = (genre or "").strip()
+        group_code = (group_code or "").strip()
+        page = max(1, int(page or 1))
+        page_size = max(1, int(page_size or 10))
+        needed = page * page_size + 1
+        matched: List[str] = []
+        try:
+            _, max_pages = self._fetch_genre_page_titles(genre, 1)
+            for page_index in range(1, max_pages + 1):
+                page_titles, _ = self._fetch_genre_page_titles(genre, page_index)
+                for title in page_titles:
+                    if self._group_matches(group_code, title):
+                        matched.append(title)
+                if len(matched) >= needed:
+                    break
+            start = (page - 1) * page_size
+            end = start + page_size
+            return list(matched[start:end])
+        except Exception:
+            grouped = self._ensure_genre_group_cache(genre)
+            titles = grouped.get(group_code, [])
+            start = (page - 1) * page_size
+            end = start + page_size
+            return list(titles[start:end])
+
+    def genre_group_has_more(self, genre: str, group_code: str, page: int = 1, page_size: int = 10) -> bool:
+        genre = (genre or "").strip()
+        group_code = (group_code or "").strip()
+        page = max(1, int(page or 1))
+        page_size = max(1, int(page_size or 10))
+        needed = page * page_size + 1
+        count = 0
+        try:
+            _, max_pages = self._fetch_genre_page_titles(genre, 1)
+            for page_index in range(1, max_pages + 1):
+                page_titles, _ = self._fetch_genre_page_titles(genre, page_index)
+                for title in page_titles:
+                    if self._group_matches(group_code, title):
+                        count += 1
+                        if count >= needed:
+                            return True
+            return False
+        except Exception:
+            grouped = self._ensure_genre_group_cache(genre)
+            titles = grouped.get(group_code, [])
+            return len(titles) > (page * page_size)
+
     def _ensure_popular(self) -> List[SeriesResult]:
         """Laedt und cached die Liste der beliebten Serien aus `/beliebte-serien`."""
         if self._popular_cache is not None:
@@ -784,7 +1193,7 @@ class SerienstreamPlugin(BasisPlugin):
             if not title or title in seen:
                 continue
             url = _absolute_url(href).split("#", 1)[0].split("?", 1)[0].rstrip("/")
-            url = re.sub(r"/staffel-\\d+(?:/.*)?$", "", url).rstrip("/")
+            url = re.sub(r"/staffel-\d+(?:/.*)?$", "", url).rstrip("/")
             if not url:
                 continue
             _log_parsed_url(url)
@@ -835,6 +1244,11 @@ class SerienstreamPlugin(BasisPlugin):
         if cached is not None:
             return list(cached)
         series = self._series_results.get(title)
+        if not series:
+            cached_url = self._title_url_cache.get(title.casefold().strip(), "")
+            if cached_url:
+                series = SeriesResult(title=title, description="", url=cached_url)
+                self._series_results[title] = series
         if not series:
             catalog = self._ensure_catalog()
             lookup_key = title.casefold().strip()
@@ -842,17 +1256,22 @@ class SerienstreamPlugin(BasisPlugin):
                 for entry in entries:
                     if entry.title.casefold().strip() == lookup_key:
                         series = entry
-                        self._series_results[entry.title] = entry
+                        self._remember_series_result(entry.title, entry.url, entry.description)
                         break
                 if series:
                     break
         if not series:
             return []
+        session_links = self._load_session_season_links(series.url)
+        if session_links:
+            self._season_links_cache[title] = list(session_links)
+            return list(session_links)
         try:
             seasons = scrape_series_detail(series.url, load_episodes=False)
         except Exception as exc:  # pragma: no cover - defensive logging
             raise RuntimeError(f"Serienstream-Staffeln konnten nicht geladen werden: {exc}") from exc
         self._season_links_cache[title] = list(seasons)
+        self._save_session_season_links(series.url, seasons)
         return list(seasons)
 
     def remember_series_url(self, title: str, series_url: str) -> None:
@@ -860,7 +1279,7 @@ class SerienstreamPlugin(BasisPlugin):
         series_url = (series_url or "").strip()
         if not title or not series_url:
             return
-        self._series_results[title] = SeriesResult(title=title, description="", url=series_url)
+        self._remember_series_result(title, series_url)
 
     def series_url_for_title(self, title: str) -> str:
         title = (title or "").strip()
@@ -869,6 +1288,9 @@ class SerienstreamPlugin(BasisPlugin):
         direct = self._series_results.get(title)
         if direct and direct.url:
             return direct.url
+        cached_url = self._title_url_cache.get(title.casefold().strip(), "")
+        if cached_url:
+            return cached_url
         lookup_key = title.casefold().strip()
         for entry in self._series_results.values():
             if entry.title.casefold().strip() == lookup_key and entry.url:
@@ -884,6 +1306,14 @@ class SerienstreamPlugin(BasisPlugin):
         target = next((season for season in links if season.number == season_number), None)
         if not target:
             return None
+        cached_episodes = self._load_session_season_episodes(target.url)
+        if cached_episodes:
+            season_info = SeasonInfo(number=target.number, url=target.url, episodes=list(cached_episodes))
+            updated = [season for season in seasons if season.number != season_number]
+            updated.append(season_info)
+            updated.sort(key=lambda item: item.number)
+            self._season_cache[title] = updated
+            return season_info
         try:
             season_soup = _get_soup(target.url, session=get_requests_session("serienstream", headers=HEADERS))
             season_info = SeasonInfo(number=target.number, url=target.url, episodes=_extract_episodes(season_soup))
@@ -894,6 +1324,7 @@ class SerienstreamPlugin(BasisPlugin):
         updated.append(season_info)
         updated.sort(key=lambda item: item.number)
         self._season_cache[title] = updated
+        self._save_session_season_episodes(target.url, season_info.episodes)
         return season_info
 
     def _lookup_episode(self, title: str, season_label: str, episode_label: str) -> Optional[EpisodeInfo]:
@@ -931,7 +1362,9 @@ class SerienstreamPlugin(BasisPlugin):
             self._episode_label_cache.clear()
             self._catalog_cache = None
             raise RuntimeError(f"Serienstream-Suche fehlgeschlagen: {exc}") from exc
-        self._series_results = {result.title: result for result in results}
+        self._series_results = {}
+        for result in results:
+            self._remember_series_result(result.title, result.url, result.description)
         self._season_cache.clear()
         self._season_links_cache.clear()
         self._episode_label_cache.clear()
@@ -960,7 +1393,7 @@ class SerienstreamPlugin(BasisPlugin):
                 for entry in entries:
                     if entry.title.casefold().strip() == lookup_key:
                         series = entry
-                        self._series_results[entry.title] = entry
+                        self._remember_series_result(entry.title, entry.url, entry.description)
                         break
                 if series:
                     break