Change: honor provider chunk settings and expose missing sync tuning controls (#196)

cenodude · Pazzie van den Berg · web-flow · commit 987f1ae41647 · 2026-03-28T03:53:47.000+01:00
* Change: adjust default batch sizes to intended values

* Change: pass SIMKL chunk settings into adapter config

* Change: pass Trakt chunk and paging settings into adapter config

* Change: add missing provider chunk controls to pair config modal

* Fix: run upgrade migration at the correct point in auth flow

---------

Co-authored-by: Pazzie van den Berg &lt;cenodude@outlook.com&gt;
diff --git a/assets/js/modals/pair-config/index.js b/assets/js/modals/pair-config/index.js
@@ -1042,6 +1042,38 @@ function renderFeaturePanel(state){
       `);
     }
 
+    if (hasSimkl(state)) {
+      const sm = (state.cfgRaw?.simkl) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">SIMKL</div>
+        <details id="cx-sm-wl">
+          <summary class="muted" style="margin-bottom:10px;">SIMKL watchlist controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="sm-wl-batch">Batch size</label>
+              <input id="sm-wl-batch" class="input small" type="number" min="1" max="1000" value="${sm.watchlist_batch_size ?? 500}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
+    if (hasMDBList(state)) {
+      const md = (state.cfgRaw?.mdblist) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">MDBList</div>
+        <details id="cx-md-wl">
+          <summary class="muted" style="margin-bottom:10px;">MDBList watchlist controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="md-wl-batch">Batch size</label>
+              <input id="md-wl-batch" class="input small" type="number" min="1" max="1000" value="${md.watchlist_batch_size ?? 500}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
     right.innerHTML = parts.join("");
     applySubDisable("watchlist");
     return;
@@ -1102,6 +1134,46 @@ function renderFeaturePanel(state){
       `);
     }
 
+    if (hasSimkl(state)) {
+      const sm = (state.cfgRaw?.simkl) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">SIMKL</div>
+        <details id="cx-sm-rt">
+          <summary class="muted" style="margin-bottom:10px;">SIMKL ratings controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="sm-rt-chunk">Write chunk size</label>
+              <input id="sm-rt-chunk" class="input small" type="number" min="1" max="1000" value="${sm.ratings_chunk_size ?? 500}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
+    if (hasMDBList(state)) {
+      const md = (state.cfgRaw?.mdblist) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">MDBList</div>
+        <details id="cx-md-rt">
+          <summary class="muted" style="margin-bottom:10px;">MDBList ratings controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row">
+              <label for="md-rt-perpage">Items per page</label>
+              <input id="md-rt-perpage" class="input small" type="number" min="1" max="5000" value="${md.ratings_per_page ?? 200}">
+            </div>
+            <div class="opt-row">
+              <label for="md-rt-maxpages">Max pages</label>
+              <input id="md-rt-maxpages" class="input small" type="number" min="1" max="2000" value="${md.ratings_max_pages ?? 50}">
+            </div>
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="md-rt-chunk">Write chunk size</label>
+              <input id="md-rt-chunk" class="input small" type="number" min="1" max="1000" value="${md.ratings_chunk_size ?? 500}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
     if (hasSimkl(state)) {
       parts.push(`<div class="simkl-alert" role="note" aria-live="polite"><div class="title"><span class="ic">⚠</span> Simkl heads-up for Ratings</div><div class="body"><ul class="bul"><li><b>Movies:</b> Rating auto-marks as <i>Completed</i> on Simkl.</li><li>Can appear under <i>Recently watched</i> and <i>My List</i>.</li></ul><div class="mini">Tip: Prefer small windows when backfilling.</div></div></div>`);
     }
@@ -1231,6 +1303,58 @@ left.innerHTML = `
               <label for="cx-tr-hs-unres">Unresolved Freeze</label>
               <label class="switch"><input id="cx-tr-hs-unres" type="checkbox" ${trCfg.history_unresolved ? "checked" : ""}><span class="slider"></span></label>
             </div>
+            <div class="opt-row">
+              <label for="tr-hs-perpage">Items per page</label>
+              <input id="tr-hs-perpage" class="input small" type="number" min="1" max="500" value="${trCfg.history_per_page ?? 100}">
+            </div>
+            <div class="opt-row">
+              <label for="tr-hs-maxpages">Max pages</label>
+              <input id="tr-hs-maxpages" class="input small" type="number" min="1" max="100000" value="${trCfg.history_max_pages ?? 10000}">
+            </div>
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="tr-hs-chunk">Write chunk size</label>
+              <input id="tr-hs-chunk" class="input small" type="number" min="1" max="1000" value="${trCfg.history_chunk_size ?? 100}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
+    if (hasSimkl(state)) {
+      const sm = (state.cfgRaw?.simkl) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">SIMKL</div>
+        <details id="cx-sm-hs">
+          <summary class="muted" style="margin-bottom:10px;">SIMKL history controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="sm-hs-chunk">Write chunk size</label>
+              <input id="sm-hs-chunk" class="input small" type="number" min="1" max="1000" value="${sm.history_chunk_size ?? 500}">
+            </div>
+          </div>
+        </details>
+      `);
+    }
+
+    if (hasMDBList(state)) {
+      const md = (state.cfgRaw?.mdblist) || {};
+      parts.push(`
+        <div class="panel-title small" style="margin-top:6px">MDBList</div>
+        <details id="cx-md-hs">
+          <summary class="muted" style="margin-bottom:10px;">MDBList history controls</summary>
+          <div class="grid2 compact">
+            <div class="opt-row">
+              <label for="md-hs-perpage">Items per page</label>
+              <input id="md-hs-perpage" class="input small" type="number" min="1" max="5000" value="${md.history_per_page ?? 1000}">
+            </div>
+            <div class="opt-row">
+              <label for="md-hs-maxpages">Max pages</label>
+              <input id="md-hs-maxpages" class="input small" type="number" min="1" max="2000" value="${md.history_max_pages ?? 250}">
+            </div>
+            <div class="opt-row" style="grid-column:1/-1">
+              <label for="md-hs-chunk">Write chunk size</label>
+              <input id="md-hs-chunk" class="input small" type="number" min="1" max="1000" value="${md.history_chunk_size ?? 500}">
+            </div>
           </div>
         </details>
       `);
@@ -1753,6 +1877,18 @@ async function saveConfigBits(state){
       });
     }
 
+    if(ID("sm-wl-batch")){
+      cfg.simkl = Object.assign({}, cfg.simkl||{}, {
+        watchlist_batch_size: Math.max(1, parseInt(ID("sm-wl-batch")?.value||"500",10)||500)
+      });
+    }
+
+    if(ID("md-wl-batch")){
+      cfg.mdblist = Object.assign({}, cfg.mdblist||{}, {
+        watchlist_batch_size: Math.max(1, parseInt(ID("md-wl-batch")?.value||"500",10)||500)
+      });
+    }
+
     const hasJF=String(state.src||"").toUpperCase()==="JELLYFIN"||String(state.dst||"").toUpperCase()==="JELLYFIN";
     if(hasJF){
       const jf=Object.assign({},cfg.jellyfin||{});
@@ -1806,14 +1942,54 @@ async function saveConfigBits(state){
       const perEl   = ID("tr-rt-perpage") || ID("tr-rt-page");
       const pagesEl = ID("tr-rt-maxpages")|| ID("tr-rt-pages");
       const chunkEl = ID("tr-rt-chunk");
+      const hsPerEl = ID("tr-hs-perpage");
+      const hsPagesEl = ID("tr-hs-maxpages");
+      const hsChunkEl = ID("tr-hs-chunk");
 
       if (perEl)   tr.ratings_per_page   = clamp(n(perEl.id), 10, 500);
       if (pagesEl) tr.ratings_max_pages  = clamp(n(pagesEl.id), 1, 1000);
       if (chunkEl) tr.ratings_chunk_size = clamp(n(chunkEl.id), 10, 1000);
+      if (hsPerEl)   tr.history_per_page   = clamp(n(hsPerEl.id), 1, 500);
+      if (hsPagesEl) tr.history_max_pages  = clamp(n(hsPagesEl.id), 1, 100000);
+      if (hsChunkEl) tr.history_chunk_size = clamp(n(hsChunkEl.id), 1, 1000);
 
       cfg.trakt=tr;
     }
 
+    const hasSM = String(state.src||"").toUpperCase()==="SIMKL" || String(state.dst||"").toUpperCase()==="SIMKL";
+    if(hasSM){
+      const sm = Object.assign({}, cfg.simkl||{});
+      const wlBatchEl = ID("sm-wl-batch");
+      const rtChunkEl = ID("sm-rt-chunk");
+      const hsChunkEl = ID("sm-hs-chunk");
+      if (wlBatchEl) sm.watchlist_batch_size = Math.max(1, parseInt(wlBatchEl.value||"500", 10) || 500);
+      if (rtChunkEl) sm.ratings_chunk_size = Math.max(1, parseInt(rtChunkEl.value||"500", 10) || 500);
+      if (hsChunkEl) sm.history_chunk_size = Math.max(1, parseInt(hsChunkEl.value||"500", 10) || 500);
+      cfg.simkl = sm;
+    }
+
+    const hasMD = String(state.src||"").toUpperCase()==="MDBLIST" || String(state.dst||"").toUpperCase()==="MDBLIST";
+    if(hasMD){
+      const md = Object.assign({}, cfg.mdblist||{});
+      const n = x => parseInt((ID(x)?.value||"").trim(), 10);
+      const clamp = (v,min,max)=>Math.min(max,Math.max(min,Number.isFinite(v)?v:min));
+      const wlBatchEl = ID("md-wl-batch");
+      const rtPerEl = ID("md-rt-perpage");
+      const rtPagesEl = ID("md-rt-maxpages");
+      const rtChunkEl = ID("md-rt-chunk");
+      const hsPerEl = ID("md-hs-perpage");
+      const hsPagesEl = ID("md-hs-maxpages");
+      const hsChunkEl = ID("md-hs-chunk");
+      if (wlBatchEl) md.watchlist_batch_size = clamp(n(wlBatchEl.id), 1, 1000);
+      if (rtPerEl) md.ratings_per_page = clamp(n(rtPerEl.id), 1, 5000);
+      if (rtPagesEl) md.ratings_max_pages = clamp(n(rtPagesEl.id), 1, 2000);
+      if (rtChunkEl) md.ratings_chunk_size = clamp(n(rtChunkEl.id), 1, 1000);
+      if (hsPerEl) md.history_per_page = clamp(n(hsPerEl.id), 1, 5000);
+      if (hsPagesEl) md.history_max_pages = clamp(n(hsPagesEl.id), 1, 2000);
+      if (hsChunkEl) md.history_chunk_size = clamp(n(hsChunkEl.id), 1, 1000);
+      cfg.mdblist = md;
+    }
+
     const res=await fetch("/api/config",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(cfg)});
     if(!res.ok)throw new Error("POST /api/config "+res.status);
   }catch(e){console.warn("[cx] saving config bits failed",e)}
diff --git a/assets/js/modals/upgrade-warning/index.js b/assets/js/modals/upgrade-warning/index.js
@@ -163,7 +163,7 @@ export default {
     }
 
     async function ensureAutoSaved() {
-      if (requiresCleanReset || state.step !== "migrate" || state.autoSaveStarted) return;
+      if (requiresCleanReset || state.autoSaveStarted) return;
       state.autoSaveStarted = true;
       state.autoSaveFailed = false;
       state.autoSaveMessage = "Saving the updated config format in the background...";
@@ -213,6 +213,7 @@ export default {
         state.step = "migrate";
         notify("Sign-in saved. CrossWatch is updating the config in the background.");
         render();
+        ensureAutoSaved();
         return;
       } catch (err) {
         state.saving = false;
@@ -412,7 +413,6 @@ export default {
         try { window.cxCloseModal?.(); } catch {}
       });
       ensureNotesLoaded();
-      ensureAutoSaved();
     }
 
     function render() {
@@ -423,6 +423,7 @@ export default {
     }
 
     render();
+    if (!requiresCleanReset && state.authReady) ensureAutoSaved();
   },
 
   unmount() {
diff --git a/cw_platform/config_base.py b/cw_platform/config_base.py
@@ -321,19 +321,19 @@ def _transform_secret_tree(obj: Any, *, decrypt: bool, path: tuple[str, ...] = (
         # Watchlist
         "watchlist_use_etag": True,                     # Use ETag + local shadow to skip unchanged lists
         "watchlist_shadow_ttl_hours": 168,              # Refresh ETag baseline weekly even if 304s keep coming
-        "watchlist_batch_size": 500,                    # Chunk size for add/remove to avoid 429/rate spikes
+        "watchlist_batch_size": 100,                    # Chunk size for add/remove to avoid 429/rate spikes
         "watchlist_log_rate_limits": True,              # Log X-RateLimit-* and Retry-After when present
         "watchlist_freeze_details": True,               # Persist last status & ids in freeze store for debugging
 
         # Ratings
         "ratings_per_page": 100,                        # Items per page when indexing (10–100; clamped to 100)
         "ratings_max_pages": 50,                        # Max pages per type; raise if you have >2k ratings/type
-        "ratings_chunk_size": 500,                      # Batch size for POST/REMOVE
+        "ratings_chunk_size": 100,                      # Batch size for POST/REMOVE
 
         # History
         "history_per_page": 100,                        # Max allowed by Trakt; fastest without spamming
         "history_max_pages": 10000,                     # Safety cap for huge libraries; lower to bound runtime
-        "history_chunk_size": 500,                      # Batch size for history add/remove writes
+        "history_chunk_size": 100,                      # Batch size for history add/remove writes
         "history_unresolved": False,                    # bool, default false (enable the freeze file)
         "history_number_fallback": False,               # episode number fallback (no S/E-based resolution when episode IDs are missing)
 
diff --git a/providers/sync/_mod_SIMKL.py b/providers/sync/_mod_SIMKL.py
@@ -222,6 +222,9 @@ class SIMKLConfig:
     max_retries: int = 3
     rate_get_per_sec: float = 10.0
     rate_post_per_sec: float = 1.0
+    watchlist_batch_size: int = 100
+    ratings_chunk_size: int = 100
+    history_chunk_size: int = 100
 
 
 class SIMKLClient:
@@ -312,6 +315,9 @@ def _rate(key: str, default: float) -> float:
             max_retries=int(simkl_cfg.get("max_retries", cfg.get("max_retries", 3))),
             rate_get_per_sec=rate_get,
             rate_post_per_sec=rate_post,
+            watchlist_batch_size=int(simkl_cfg.get("watchlist_batch_size", 100) or 100),
+            ratings_chunk_size=int(simkl_cfg.get("ratings_chunk_size", 100) or 100),
+            history_chunk_size=int(simkl_cfg.get("history_chunk_size", 100) or 100),
         )
         if not self.cfg.api_key or not self.cfg.access_token:
             raise SIMKLError("SIMKL requires both api_key (or client_id) and access_token")
@@ -321,6 +327,7 @@ def _rate(key: str, default: float) -> float:
 
         self.client = SIMKLClient(self.cfg, simkl_cfg).connect()
         self.raw_cfg = cfg
+        self.config = cfg
         self.progress_factory = (
             lambda feature, total=None, throttle_ms=300: make_snapshot_progress(
                 ctx,
diff --git a/providers/sync/_mod_TRAKT.py b/providers/sync/_mod_TRAKT.py
@@ -171,6 +171,13 @@ class TRAKTConfig:
     max_retries: int = 3
     rate_get_per_sec: float = 3.33
     rate_post_per_sec: float = 1.0
+    watchlist_batch_size: int = 100
+    ratings_per_page: int = 100
+    ratings_max_pages: int = 50
+    ratings_chunk_size: int = 100
+    history_per_page: int = 100
+    history_max_pages: int = 10000
+    history_chunk_size: int = 100
     history_number_fallback: bool = False
     history_collection: bool = False
     history_collection_types: list[str] | None = None
@@ -350,6 +357,13 @@ def _rate(key: str, default: float) -> float:
             max_retries=int(t.get("max_retries", cfg.get("max_retries", 3))),
             rate_get_per_sec=rate_get,
             rate_post_per_sec=rate_post,
+            watchlist_batch_size=int(t.get("watchlist_batch_size", 100) or 100),
+            ratings_per_page=int(t.get("ratings_per_page", 100) or 100),
+            ratings_max_pages=int(t.get("ratings_max_pages", 50) or 50),
+            ratings_chunk_size=int(t.get("ratings_chunk_size", 100) or 100),
+            history_per_page=int(t.get("history_per_page", 100) or 100),
+            history_max_pages=int(t.get("history_max_pages", 10000) or 10000),
+            history_chunk_size=int(t.get("history_chunk_size", 100) or 100),
             history_number_fallback=bool(t.get("history_number_fallback")),
             history_collection=bool(t.get("history_collection")),
             history_collection_types=types or None,
@@ -362,6 +376,7 @@ def _rate(key: str, default: float) -> float:
 
         self.client = TRAKTClient(self.cfg, cfg).connect()
         self.raw_cfg = cfg
+        self.config = cfg
         self.progress_factory = (
             lambda feature, total=None, throttle_ms=300: make_snapshot_progress(
                 ctx,
