Implement dynamic API throttling with LearningDelay an Exponential Backoff

Introduced a LearningDelay class to manage API throttling dynamically. Updated the DeviantartOAuthAPI to utilize the new delay mechanism for improved handling of rate limits.

Recover frame rate limiting with exponential backoff instead of linear.

Author: bbappserver

gallery_dl/extractor/deviantart.py

@@ -1429,6 +1429,44 @@ def items(self):
 ###############################################################################
 # API Interfaces ##############################################################
 
+class LearningDelay():
+    '''
+    Deviantart API is throttled dynamically over time
+    you can learn the ideal ammount to wait
+    '''
+
+    def __init__(self, initial=1, error_margin=0.01):
+        '''
+        Inital is the inital time to wait in seconds,
+        error_margin is added on top of the esitmate but not used in
+        the update. It accounts for fuzzyness in the choice of value
+
+        TODO: You can store an load this calue across gallery-dl execution.
+        '''
+        self._min = 0
+        self._current = initial
+        self._prev = initial + error_margin
+        self._error = error_margin
+        self._convegered = False
+
+    def reject(self):
+        '''The current estimate was too low, increase it'''
+        self._min = self._current
+        self._current = self._prev+self._error
+
+    def update(self):
+        '''Try a smaller estimate'''
+        if self._convegered:
+            return
+        self._current = 0.5*(self._current+self._min)
+        self._convegered = abs(self._current - self._prev) < self._error
+
+    @property
+    def value(self):
+        '''Current guess pluss an engineering margin for rounding errors'''
+        return self._current + self._error
+
+
 class DeviantartOAuthAPI():
     """Interface for the DeviantArt OAuth API
 
@@ -1445,6 +1483,9 @@ def __init__(self, extractor):
 
         self.delay = extractor.config("wait-min", 0)
         self.delay_min = max(2, self.delay)
+        self.exp_backoff_mul = 2
+        self.learned_delay = LearningDelay(initial=self.delay)
+        self.is_throttled = False
 
         self.mature = extractor.config("mature", "true")
         if not isinstance(self.mature, str):
@@ -1746,7 +1787,10 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs):
 
         while True:
             if self.delay:
-                self.extractor.sleep(self.delay, "api")
+                if self.is_throttled:
+                    self.extractor.sleep(self.delay, "api")
+                else:
+                    self.extractor.sleep(self.learned_delay.value, "api")
 
             self.authenticate(None if public else self.refresh_token_key)
             kwargs["headers"] = self.headers
@@ -1760,6 +1804,14 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs):
 
             status = response.status_code
             if 200 <= status < 400:
+                self.learned_delay.update()
+                self.is_throttled = False
+
+                # Exp backoff shouldn't be but could be significant
+                # so nap back to a sane value for linear cooldown
+                if self.delay > 10:
+                    self.delay = 10
+
                 if self.delay > self.delay_min:
                     self.delay -= 1
                 return data
@@ -1775,8 +1827,15 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs):
             self.log.debug(response.text)
             msg = f"API responded with {status} {response.reason}"
             if status == 429:
-                if self.delay < 30:
-                    self.delay += 1
+
+                # When throttled we use exp backoff
+                self.is_throttled = True
+
+                # Exp backoff is a multiplier so enure sane initial delay
+                if self.delay < 1:
+                    self.delay = 1
+                else:
+                    self.delay *= self.exp_backoff_mul
                 self.log.warning("%s. Using %ds delay.", msg, self.delay)
 
                 if self._warn_429 and self.delay >= 3: