Merge-build: v2.10.0-beta.3

Author: jasoneri

.gitignore

@@ -176,6 +176,7 @@ custom/start/*
 GUI/src/*
 !GUI/src/preview_format/
 !GUI/src/material_ct.py
+!docs/.vitepress/
 *.db*
 # cgs/build
 *.ico
@@ -206,3 +207,4 @@ deploy/launcher/mac/*.html
 openspec/
 nul
 .omc
+*.*ai

CGS.py

@@ -59,15 +59,7 @@ def handle_current_exception(self, phase):
 
     def handle_exception(self, exc_type, exc_value, exc_traceback, phase):
         if self.ui is not None:
-            try:
-                self.ui.hook_exception(exc_type, exc_value, exc_traceback)
-                return
-            except Exception:
-                trace_text = "".join(traceback.format_exception(*sys.exc_info()))
-                log_path = self._append_fatal_log("hook_exception failed", trace_text)
-                self._write_stderr(f"\n[CGS hook_exception failed] log: {log_path}\n{trace_text}\n")
-                return sys.__excepthook__(*sys.exc_info())
-
+            return self.ui.hook_exception(exc_type, exc_value, exc_traceback)
         trace_text = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback))
         log_path = self._append_fatal_log(phase, trace_text)
         self._write_stderr(f"\n[CGS uncaught] log: {log_path}\n{trace_text}\n")

ComicSpider/pipelines.py

@@ -108,32 +108,23 @@ def file_folder(self, basepath, section, spider, title, item):
         spider.tasks_path[uuid_md5] = path
         return path
 
-    def image_downloaded(self, response, request, info, *, item=None):
-        spider = info.spider
-        try:
-            super(ComicPipeline, self).image_downloaded(response, request, info, item=item)
-            stats = spider.crawler.stats
-            self._sync_item_progress(spider, stats, item, count_download_stat=True)
-        except Exception as e:
-            spider.logger.error(f'traceback: {str(type(e))}:: {str(e)}')
-
     @staticmethod
     def _processed_file_count(stats):
         return (
             stats.get_value('file_status_count/downloaded', default=0) +
             stats.get_value('file_status_count/uptodate', default=0)
         )
 
-    def _sync_item_progress(self, spider, stats, item, *, count_download_stat):
+    def _sync_item_progress(self, spider, stats, item):
         total = getattr(spider, 'total', 0) or 0
         processed = self._processed_file_count(stats)
         percent = int((processed / total) * 100) if total else 0
         spider.emit(BarProgressEvent(job_id=getattr(spider, '_job_id', None), percent=percent))
         task_obj = TaskObj(item.get('uuid_md5'), item.get('page'), item['image_urls'][0])
-        self._record_task_progress(spider, stats, task_obj, count_download_stat=count_download_stat)
+        self._record_task_progress(spider, task_obj)
 
     @staticmethod
-    def _record_task_progress(spider, stats, task_obj, *, count_download_stat=True):
+    def _record_task_progress(spider, task_obj):
         _tasks = spider.tasks[task_obj.taskid]
         _tasks.downloaded.append(task_obj)
         curr_progress = int(len(_tasks.downloaded) / _tasks.tasks_count * 100)
@@ -150,27 +141,16 @@ def _record_task_progress(spider, stats, task_obj, *, count_download_stat=True):
             task_obj=task_obj,
             is_new=False,
         ))
-        if count_download_stat:
-            stats.inc_value('image/downloaded')
-
-    def media_to_download(self, request: Request, info, *, item=None):
-        dfd = maybeDeferred(super().media_to_download, request, info, item=item)
-
-        def _track_uptodate(file_info):
-            if (
-                item is not None and
-                isinstance(file_info, dict) and
-                file_info.get('status') == 'uptodate'
-            ):
-                self._sync_item_progress(info.spider, info.spider.crawler.stats, item, count_download_stat=False)
-            return file_info
-
-        dfd.addCallback(_track_uptodate)
-        return dfd
 
     def item_completed(self, results, item, info):
-        _item = super(ComicPipeline, self).item_completed(results, item, info)
-        return _item
+        completed_item = super(ComicPipeline, self).item_completed(results, item, info)
+        if not any(
+            ok and isinstance(file_info, dict) and file_info.get('status') in {'downloaded', 'uptodate'}
+            for ok, file_info in results
+        ):
+            return completed_item
+        self._sync_item_progress(info.spider, info.spider.crawler.stats, item)
+        return completed_item
 
 
 class WnacgComicPipeline(ComicPipeline):
@@ -255,7 +235,8 @@ def _download_via_curl():
 
             def _handle_curl_result(result):
                 status_code, content = result
-                return self.media_downloaded(
+                return maybeDeferred(
+                    self.media_downloaded,
                     Response(url=request.url,status=status_code,body=content,request=request),
                     request,info,item=item)
 

ComicSpider/runtime/thread_runner.py

@@ -69,6 +69,7 @@ def run(self):
         installed_reactor = f"{reactor.__class__.__module__}.{reactor.__class__.__name__}"
         s.set("TWISTED_REACTOR", installed_reactor, priority="cmdline")
         configure_logging(s)
+        logging.getLogger("PIL.Image").setLevel(logging.WARNING)
         self._runner = CrawlerRunner(s)
         self._settings = s
         self._ready.set()
@@ -113,11 +114,7 @@ def _start_crawl(self, job: SpiderDownloadJob):
         self.state.update(stage="crawling", active_job_id=job.job_id, error=None)
         self.event_q.put(JobAcceptedEvent(job_id=job.job_id))
 
-        d = self._runner.crawl(
-            spider_cls_name,
-            runtime_thread=self,
-            job=job,
-        )
+        d = self._runner.crawl(spider_cls_name, runtime_thread=self, job=job)
         d.addCallback(lambda _: self._on_crawl_finished(job))
         d.addErrback(lambda f: self._on_crawl_error(job, f))
 
@@ -126,7 +123,7 @@ def _on_crawl_finished(self, job):
         error = getattr(job, "runtime_error", None)
         stage = "idle" if success else "error"
         self.state.update(stage=stage, active_job_id=None, progress=0.0, error=error)
-        self.event_q.put(JobFinishedEvent(job_id=job.job_id, success=success))
+        self.event_q.put(JobFinishedEvent(job_id=job.job_id, success=success, error=error))
         if success:
             logger.info(f"Job {job.job_id} finished")
         else:
@@ -136,7 +133,7 @@ def _on_crawl_error(self, job, failure):
         error_msg = str(failure.value) if hasattr(failure, 'value') else str(failure)
         self.state.update(stage="error", active_job_id=None, progress=0.0, error=error_msg)
         self.event_q.put(ErrorEvent(job_id=job.job_id, error=error_msg))
-        self.event_q.put(JobFinishedEvent(job_id=job.job_id, success=False))
+        self.event_q.put(JobFinishedEvent(job_id=job.job_id, success=False, error=error_msg))
         logger.error(f"Job {job.job_id} failed: {error_msg}")
 
     def submit_job(self, job: SpiderDownloadJob):

ComicSpider/spiders/basecomicspider.py

@@ -8,20 +8,22 @@
 
 import scrapy
 
-from variables import *
+from variables import *  # noqa: F403
 from assets import res as ori_res
 from ComicSpider.items import ComicspiderItem
 from ComicSpider.runtime.job_models import create_job_context, iter_download_items
 from GUI.core.font import font_color
 from utils import PresetHtmlEl, temp_p, conf
-from utils.processed_class import TextBrowserState, ProcessState, Url
+from utils.processed_class import TextBrowserState, ProcessState
 
-from utils.protocol import SpiderDownloadJob, JobContext, LogEvent, ProcessStateEvent, TasksObjEvent
+from utils.protocol import SpiderDownloadJob, JobContext, LogEvent, ProcessStateEvent
 from utils.website import (
-    correct_domain,
-    InfoMinix, BookInfo, Episode
+    correct_domain, BookInfo, Episode
+)
+from utils.website.registry import (
+    resolve_provider_descriptor_by_spider,
+    create_spider_site_runtime,
 )
-from utils.website.registry import resolve_spider_adapter
 from utils.website.schema import BodyFormat
 from utils.sql import SqlRecorder, SqlrV
 from utils.meta import MetaRecorder
@@ -71,8 +73,8 @@ class BaseComicSpider(scrapy.Spider):
     text_browser_state = TextBrowserState(text='')
     process_state = ProcessState(process='init')
     say: SayToGui = None
-    adapter = None
-    site = None
+    provider_descriptor = None
+    spider_site_runtime = None
     record_sql: SqlRecorder = None
     rv_sql: SqlrV = None
     ua = {}
@@ -96,6 +98,7 @@ class BaseComicSpider(scrapy.Spider):
     turn_page_search: str = None
     turn_page_info: tuple = None
     _enable_episode_dispatch = False
+    remove_domain_cache_on_finished_miss = True
 
     def preready(self):
         ...
@@ -152,10 +155,7 @@ def _bind_runtime_context(self, job: SpiderDownloadJob):
                     getattr(getattr(item, "from_book", None), "preview_url", None),
                 ])
             elif isinstance(item, BookInfo):
-                candidates.extend([
-                    getattr(item, "url", None),
-                    getattr(item, "preview_url", None),
-                ])
+                candidates.extend([getattr(item, "url", None), getattr(item, "preview_url", None)])
             for candidate in candidates:
                 if origin := self._url_origin(candidate):
                     self._runtime_origin = origin
@@ -275,12 +275,7 @@ def parse_section(self, response):
             if isinstance(url_or_ep, Episode):
                 yield from self._process_episode(url_or_ep)
             elif isinstance(url_or_ep, str):
-                yield scrapy.Request(
-                    url=url_or_ep,
-                    callback=self.parse_fin_page,
-                    meta={'book': book, 'page': page},
-                    dont_filter=True,
-                )
+                yield scrapy.Request(url=url_or_ep, callback=self.parse_fin_page, meta={'book': book, 'page': page}, dont_filter=True)
 
     def need_sec_next_page(self, resp):
         pass
@@ -353,8 +348,8 @@ def from_crawler(cls, crawler, *args, **kwargs):
 
         spider.record_sql = SqlRecorder()
         spider.rv_sql = SqlrV(1 if spider.name in spider.settings.get('SPECIAL') else 0).connect()
-        spider.adapter = resolve_spider_adapter(spider.name)
-        spider.site = spider.adapter.create_session(conf)
+        spider.provider_descriptor = resolve_provider_descriptor_by_spider(spider.name)
+        spider.spider_site_runtime = create_spider_site_runtime(spider.name, conf_state=conf)
         spider.mr = MetaRecorder(conf)
 
         if job:
@@ -373,7 +368,7 @@ def _remove_cache(self):
             os.remove(domain_cache)
 
     def _finish_counters(self, stats):
-        downloaded_count = stats.get_value('image/downloaded', 0)
+        downloaded_count = stats.get_value('file_status_count/downloaded', 0)
         uptodate_count = stats.get_value('file_status_count/uptodate', 0)
         total = self.job_context.total if self.job_context else self.total
         return downloaded_count, uptodate_count, downloaded_count + uptodate_count, total
@@ -412,11 +407,13 @@ def _handle_finished_status(self, stats):
             return
         downloaded_count, uptodate_count, processed_count, total = self._finish_counters(stats)
         exception_count = stats.get_value('process_exception/count', 0)
+        remove_domain_cache = bool(self.remove_domain_cache_on_finished_miss)
         if total and processed_count < total:
             missing_count = total - processed_count
             self.say(font_color(f'miss: new[{downloaded_count}], cache[{uptodate_count}], miss[{missing_count}]<br>',
                 cls='theme-err', size=3))
-            self._remove_cache()
+            if remove_domain_cache:
+                self._remove_cache()
         elif total != 0 and processed_count > 0:
             if downloaded_count:
                 _str = f'{self.res.finished_success % downloaded_count}'
@@ -428,7 +425,8 @@ def _handle_finished_status(self, stats):
             self.say(font_color(
                 f'<br>{self.res.finished_err % last_exception}<br>log path/日志文件地址: [{self.settings.get("LOG_FILE")}]',
                 cls='theme-err', size=3))
-            self._remove_cache()
+            if remove_domain_cache:
+                self._remove_cache()
         else:
             self.say(font_color(f'{self.res.finished_empty}<br>', cls='theme-highlight', size=4))
 

ComicSpider/spiders/ehentai.py

@@ -48,7 +48,7 @@ def frame_book(self, response):
         frame_results = {}
         targets = response.xpath('//table[contains(@class, "itg")]//td[contains(@class, "glcat")]/..')
         with ThreadPoolExecutor() as executor:
-            books = list(executor.map(self.site.parser.parse_search_item, targets))
+            books = list(executor.map(self.spider_site_runtime.parser.parse_search_item, targets))
         for x, book in enumerate(books):
             book.idx = x + 1
             frame_results[book.idx] = book

ComicSpider/spiders/h_comic.py

@@ -26,14 +26,14 @@ def ua(self):
         return HComicUtils.headers
 
     def frame_section(self, response):
-        book = self.site.parser.parse_book(response.text)
+        book = self.spider_site_runtime.parser.parse_book(response.text)
         pages = int(book.pages or 0)
         if pages <= 0:
             self.say(font_color("未解析到页面信息，请稍后重试", cls="theme-err"))
             return {}
         media_id = getattr(book, "media_id", "")
         comic_source = getattr(book, "comic_source", "")
-        image_prefix = self.site.parser.get_image_prefix(comic_source)
+        image_prefix = self.spider_site_runtime.parser.get_image_prefix(comic_source)
         frame_results = {}
         for page in range(1, pages + 1):
             frame_results[page] = f"{image_prefix}/{media_id}/pages/{page}"

ComicSpider/spiders/hitomi.py

@@ -1,17 +1,15 @@
 # -*- coding: utf-8 -*-
-import json
 import asyncio
 from concurrent.futures import ThreadPoolExecutor
 import scrapy
 
 from ComicSpider.runtime.job_models import iter_download_items
 
-from utils import PresetHtmlEl, conf
+from utils import conf
 from utils.website import HitomiUtils, get_loop
-from utils.processed_class import PreviewHtml
 from ComicSpider.items import ComicspiderItem
 
-from .basecomicspider import BaseComicSpider, font_color
+from .basecomicspider import BaseComicSpider
 
 domain = HitomiUtils.index
 
@@ -35,7 +33,7 @@ class HitomiSpider(BaseComicSpider):
     def from_crawler(cls, crawler, *args, **kwargs):
         spider = super(HitomiSpider, cls).from_crawler(crawler, *args, **kwargs)
         try:
-            spider.async_cli = spider.site.get_cli(conf, is_async=True)
+            spider.async_cli = spider.spider_site_runtime.provider.reqer_cls.get_cli(conf, is_async=True)
         except Exception as e:
             if spider.crawler and spider.crawler.engine:
                 spider.crawler.engine.close_spider(spider, reason=f"[error]{str(e)}")
@@ -47,7 +45,8 @@ def from_crawler(cls, crawler, *args, **kwargs):
     def _get_nozomi_sync(self, nozomi_url, page):
         """同步包装的异步nozomi获取方法"""
         async def _async_get():
-            headers = {**HitomiUtils.headers, "Range": self.site.runtime.get_range(page)}
+            provider = self.spider_site_runtime.provider
+            headers = {**provider.headers, "Range": provider.get_range(page)}
             return await self.async_cli.get(nozomi_url, headers=headers)
 
         try:
@@ -65,7 +64,7 @@ def start_requests(self):
     # ==============================================
     def parse(self, response, meta):
         self._emit_process('parse')
-        result = HitomiUtils.parse_nozomi(response.content)
+        result = self.spider_site_runtime.provider.parse_nozomi(response.content)
 
         meta = meta or {}
         meta['results'] = []
@@ -90,10 +89,7 @@ async def fetch_all():
 
         # 整合actual_parse的功能
         for _, resp in sorted(resps, key=lambda x: x[0]):  # 按原始索引排序
-            meta['results'].append({
-                "text": resp.text,
-                "meta": {k: v for k, v in meta.items() if k != 'results'}
-            })
+            meta['results'].append({"text": resp.text, "meta": {k: v for k, v in meta.items() if k != 'results'}})
         yield from self.defer_parse(meta['results'])
 
     def defer_parse(self, rets):
@@ -109,12 +105,17 @@ def parse_section(self, meta):
         this_uuid, this_md5 = book.id_and_md5()
         self._assert_task_not_downloaded(book)
         self.set_task(book)
+        provider = self.spider_site_runtime.provider
+        # Full-image URLs become invalid as soon as Hitomi rotates gg.b, even when the
+        # cached bucket still looks "same-hour" by local heuristic. Refresh once per
+        # download section before materializing image URLs.
+        provider.refresh_gg_if_needed(force=True)
         for index, pic_info in enumerate(book.pics, 1):
             item = ComicspiderItem()
             item['title'] = book.name
             item['page'] = str(index)
             item['section'] = None
-            img_url = self.site.runtime.get_img_url(pic_info['hash'], pic_info['hasavif'])
+            img_url = provider.get_img_url(pic_info['hash'], pic_info['hasavif'])
             item['image_urls'] = [img_url]
             item['uuid'] = this_uuid
             item['uuid_md5'] = this_md5
@@ -138,8 +139,10 @@ def iter_download_requests(self, job):
     def frame_book(self, rets, meta):
         frame_results = {}
         texts = [target['text'] for target in rets]
+        runtime_provider = self.spider_site_runtime.provider
+        parser = runtime_provider.__class__.parser(runtime_provider)
         with ThreadPoolExecutor() as executor:
-            books = list(executor.map(self.site.parser.parse_search_item, texts))
+            books = list(executor.map(parser.parse_search_item, texts))
         for x, book in enumerate(books):
             book.idx = x + 1
             book.preview_url = f"{self.domain}{book.preview_url}"