Nightly: refactor readability, progress callbacks, and resource handling
This commit is contained in:
@@ -11,7 +11,7 @@ from dataclasses import dataclass
|
||||
import re
|
||||
from urllib.parse import quote, urlencode
|
||||
from urllib.parse import urljoin
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
try: # pragma: no cover - optional dependency
|
||||
import requests
|
||||
@@ -53,6 +53,16 @@ SETTING_LOG_URLS = "log_urls_filmpalast"
|
||||
SETTING_DUMP_HTML = "dump_html_filmpalast"
|
||||
SETTING_SHOW_URL_INFO = "show_url_info_filmpalast"
|
||||
SETTING_LOG_ERRORS = "log_errors_filmpalast"
|
||||
ProgressCallback = Optional[Callable[[str, Optional[int]], Any]]
|
||||
|
||||
|
||||
def _emit_progress(callback: ProgressCallback, message: str, percent: Optional[int] = None) -> None:
|
||||
if not callable(callback):
|
||||
return
|
||||
try:
|
||||
callback(str(message or ""), None if percent is None else int(percent))
|
||||
except Exception:
|
||||
return
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Kodi; ViewIt) AppleWebKit/537.36 (KHTML, like Gecko)",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
@@ -206,16 +216,26 @@ def _get_soup(url: str, *, session: Optional[RequestsSession] = None) -> Beautif
|
||||
raise RuntimeError("requests/bs4 sind nicht verfuegbar.")
|
||||
_log_visit(url)
|
||||
sess = session or get_requests_session("filmpalast", headers=HEADERS)
|
||||
response = None
|
||||
try:
|
||||
response = sess.get(url, headers=HEADERS, timeout=DEFAULT_TIMEOUT)
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
_log_error_message(f"GET {url} failed: {exc}")
|
||||
raise
|
||||
if response.url and response.url != url:
|
||||
_log_url_event(response.url, kind="REDIRECT")
|
||||
_log_response_html(url, response.text)
|
||||
return BeautifulSoup(response.text, "html.parser")
|
||||
try:
|
||||
final_url = (response.url or url) if response is not None else url
|
||||
body = (response.text or "") if response is not None else ""
|
||||
if final_url != url:
|
||||
_log_url_event(final_url, kind="REDIRECT")
|
||||
_log_response_html(url, body)
|
||||
return BeautifulSoup(body, "html.parser")
|
||||
finally:
|
||||
if response is not None:
|
||||
try:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class FilmpalastPlugin(BasisPlugin):
|
||||
@@ -352,6 +372,7 @@ class FilmpalastPlugin(BasisPlugin):
|
||||
seen_titles: set[str] = set()
|
||||
seen_urls: set[str] = set()
|
||||
for base_url, params in search_requests:
|
||||
response = None
|
||||
try:
|
||||
request_url = base_url if not params else f"{base_url}?{urlencode(params)}"
|
||||
_log_url_event(request_url, kind="GET")
|
||||
@@ -365,6 +386,12 @@ class FilmpalastPlugin(BasisPlugin):
|
||||
except Exception as exc:
|
||||
_log_error_message(f"search request failed ({base_url}): {exc}")
|
||||
continue
|
||||
finally:
|
||||
if response is not None:
|
||||
try:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
anchors = soup.select("article.liste h2 a[href], article.liste h3 a[href]")
|
||||
if not anchors:
|
||||
@@ -466,9 +493,13 @@ class FilmpalastPlugin(BasisPlugin):
|
||||
titles.sort(key=lambda value: value.casefold())
|
||||
return titles
|
||||
|
||||
async def search_titles(self, query: str) -> List[str]:
|
||||
async def search_titles(self, query: str, progress_callback: ProgressCallback = None) -> List[str]:
|
||||
_emit_progress(progress_callback, "Filmpalast Suche", 15)
|
||||
hits = self._search_hits(query)
|
||||
return self._apply_hits_to_title_index(hits)
|
||||
_emit_progress(progress_callback, f"Treffer verarbeiten ({len(hits)})", 70)
|
||||
titles = self._apply_hits_to_title_index(hits)
|
||||
_emit_progress(progress_callback, f"Fertig: {len(titles)} Treffer", 95)
|
||||
return titles
|
||||
|
||||
def _parse_genres(self, soup: BeautifulSoupT) -> Dict[str, str]:
|
||||
genres: Dict[str, str] = {}
|
||||
@@ -913,6 +944,7 @@ class FilmpalastPlugin(BasisPlugin):
|
||||
|
||||
redirected = link
|
||||
if self._requests_available:
|
||||
response = None
|
||||
try:
|
||||
session = get_requests_session("filmpalast", headers=HEADERS)
|
||||
response = session.get(link, headers=HEADERS, timeout=DEFAULT_TIMEOUT, allow_redirects=True)
|
||||
@@ -920,6 +952,12 @@ class FilmpalastPlugin(BasisPlugin):
|
||||
redirected = (response.url or link).strip() or link
|
||||
except Exception:
|
||||
redirected = link
|
||||
finally:
|
||||
if response is not None:
|
||||
try:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2) Danach optional die Redirect-URL nochmals auflösen.
|
||||
if callable(resolve_with_resolveurl) and redirected and redirected != link:
|
||||
|
||||
Reference in New Issue
Block a user