Bump datasets version (#1724)

Author: dakinggg

llmfoundry/utils/model_download_utils.py

@@ -142,7 +142,7 @@ def _extract_links_from_html(html: str):
         list[str]: A list of links to download.
     """
     soup = BeautifulSoup(html, 'html.parser')
-    links = [a['href'] for a in soup.find_all('a')]
+    links = [a['href'] for a in soup.find_all('a')]  # type: ignore
     return links
 
 
@@ -208,7 +208,7 @@ def _recursive_download(
         _recursive_download(
             session,
             base_url,
-            urljoin(path, child_link),
+            urljoin(path, child_link),  # type: ignore
             save_dir,
             ignore_cert=ignore_cert,
         )

setup.py

@@ -58,7 +58,7 @@
     'transformers>=4.43.2,<4.47',
     'mosaicml-streaming>=0.11.0,<0.12',
     'torch>=2.5.1,<2.5.2',
-    'datasets>=2.20.0,<3.3',
+    'datasets>=3.3.2,<3.4',
     'fsspec==2023.6.0',  # newer version results in a bug in datasets that duplicates data
     'sentencepiece==0.2.0',
     'einops==0.8.0',