CI: Use PDM to extract requirements from pyproject.toml (#6501)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: gmuloc <gmulocher@arista.com>

Author: 3 people

.gitignore

@@ -37,6 +37,8 @@ dist/
 .python-version
 ## direnv
 .envrc
+## pyenv
+uv.lock
 ## .vscode/*
 .vscode/
 

.pre-commit-config.yaml

@@ -198,6 +198,16 @@ repos:
             docs/plugins/.* | # Excluded auto generated doc for Ansible plugins
           )$
 
+  - repo: https://github.com/pdm-project/pdm
+    rev: "2.26.6"
+    hooks:
+      - id: pdm-export
+        name: Extract requirements from pyavd/pyproject.toml to requirements.txt
+        entry: pdm export -p python-avd -o ansible_collections/arista/avd/requirements.txt --without-hashes -G ansible -G self --pyproject
+        language: python
+        files: python-avd/pyproject.toml
+        pass_filenames: false
+
   - repo: local
     hooks:
 

ansible_collections/arista/avd/plugins/action/verify_requirements.py

@@ -1,16 +1,17 @@
 # Copyright (c) 2023-2026 Arista Networks, Inc.
 # Use of this source code is governed by the Apache License 2.0
 # that can be found in the LICENSE file.
+from __future__ import annotations
 
 import json
 import sys
 import warnings
 from importlib import import_module
-from importlib.metadata import Distribution, PackageNotFoundError, metadata, version
+from importlib.metadata import Distribution, PackageNotFoundError, version
 from logging import getLogger
 from pathlib import Path
 from subprocess import PIPE, Popen
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import yaml
 from ansible import constants as C  # noqa: N812
@@ -20,16 +21,19 @@
 from ansible_collections.arista.avd.plugins import PYTHON_AVD_PATH, RUNNING_FROM_SOURCE
 from ansible_collections.arista.avd.plugins.plugin_utils.utils.avd_action_plugin import AvdActionPlugin, AvdLoggingConfig
 
+if TYPE_CHECKING:
+    # Relying on packaging installed by ansible
+    from packaging.requirements import Requirement
+    from packaging.specifiers import SpecifierSet
+
 try:
     # Relying on packaging installed by ansible
-    from packaging.requirements import InvalidRequirement, Requirement
+    from packaging.requirements import Requirement
     from packaging.specifiers import SpecifierSet
 
     HAS_PACKAGING = True
 except ImportError:
     HAS_PACKAGING = False
-    # Making ansible-test sanity happy
-    Requirement = object
 
 LOGGER = getLogger("ansible_collections.arista.avd")
 DISPLAY = Display()
@@ -81,24 +85,6 @@ def _validate_python_version(info: dict[str, Any]) -> bool:
     return True
 
 
-def _parse_requirements(req_str: str) -> tuple[Requirement, list[str]]:
-    """Parse a requirement string and return the parsed object and a list of extras requirements to parse if any."""
-    try:
-        req = Requirement(req_str)
-    except InvalidRequirement as exc:
-        msg = f"Wrong format for requirement {req_str}"
-        raise ValueError(msg) from exc
-
-    extras = []
-    if req.extras:
-        for subreq_name in metadata(req.name).get_all("Requires-Dist"):
-            subreq = Requirement(subreq_name)
-            if subreq.marker:
-                extras.extend([subreq_name for marker in subreq.marker._markers if str(marker[0]) == "extra" and str(marker[2]) in req.extras])
-
-    return req, extras
-
-
 def _check_requirement(req: Requirement, requirements_dict: dict[str, Any]) -> bool:
     """
     Check one requirement and in-place update requirement_dict.
@@ -195,19 +181,17 @@ def _validate_python_requirements(requirements: list[str], info: dict[str, Any])
     }
 
     # Remove the comments including inline comments
-    requirements = [req.split(" #", maxsplit=1)[0] for req in requirements if req[0] != "#"]
+    requirements = [req.split(" #", maxsplit=1)[0] for req in requirements if req != "" and req[0] != "#"]
     for raw_req in requirements:
-        req, extras = _parse_requirements(raw_req)
+        req = Requirement(raw_req)
         if RUNNING_FROM_SOURCE and req.name == "pyavd":
-            LOGGER.debug("AVD is running from source, *not* checking pyavd version nor any extra.")
+            LOGGER.debug("AVD is running from source, *not* checking pyavd version.")
             requirements_dict["valid"][req.name] = {
                 "installed": "running from source",
                 "required_version": str(req.specifier) if len(req.specifier) > 0 else None,
             }
             continue
 
-        requirements.extend(extras)
-
         valid = valid and _check_requirement(req, requirements_dict)
 
     info["python_requirements"] = requirements_dict

ansible_collections/arista/avd/requirements.txt

@@ -1,2 +1,17 @@
-# PyAVD must follow the exact same version as the Ansible collection.
-pyavd[ansible-collection]==6.0.0.dev8
+# This file is @generated by PDM.
+# Please do not edit it manually.
+
+ansible-core<2.21.0,>=2.16.0
+anta>=1.7.0
+aristaproto>=0.1.1
+cryptography>=43.0.0
+deepmerge>=1.1.0
+distlib>=0.3.9
+grpclib==0.4.9
+jinja2>=3.0
+netaddr>=0.7.19
+pyavd==6.0.0.dev8
+pyavd-utils==0.0.2
+python-socks[asyncio]>=2.7.2
+pyyaml>=6.0.0
+requests>=2.27.0

ansible_collections/arista/avd/tests/unit/action/test_verify_requirements.py

@@ -136,18 +136,16 @@ def test__validate_python_requirements(n_reqs: int, mocked_version: str | None,
 
 
 @pytest.mark.parametrize(
-    ("extras", "running_from_source", "expected_return"),
+    ("running_from_source", "expected_return"),
     [
-        pytest.param(False, False, True, id="pyavd - no extra - not running from source"),
-        pytest.param(True, False, True, id="pyavd - extra - not running from source"),
-        pytest.param(False, True, True, id="pyavd - no extra - running from source"),
-        pytest.param(False, True, True, id="pyavd - extra - running from source"),
+        pytest.param(False, True, id="pyavd - not running from source"),
+        pytest.param(True, True, id="pyavd - running from source"),
     ],
 )
-def test__validate_python_requirements_pyavd(extras: bool, running_from_source: bool, expected_return: bool) -> None:
+def test__validate_python_requirements_pyavd(running_from_source: bool, expected_return: bool) -> None:
     """Testing behavior of the function for pyavd when running from source or not."""
     result = {}
-    req = f"pyavd{'[ansible-collection]' if extras else ''}==5.3.0"
+    req = "pyavd==5.3.0"
 
     requirements = [req]
 
@@ -159,24 +157,14 @@ def test__validate_python_requirements_pyavd(extras: bool, running_from_source:
         ret = _validate_python_requirements(requirements, result)
         assert ret == expected_return
     python_req_result = result["python_requirements"]
+    assert (
+        len(python_req_result["valid"]) + len(python_req_result["mismatched"]) + len(python_req_result["not_found"]) + len(python_req_result["parsing_failed"])
+        == 1
+    )
     if running_from_source:
         assert python_req_result["valid"]["pyavd"]["installed"] == "running from source"
-        # only pyavd is expected for this test when running from source with or without extra
-        assert (
-            len(python_req_result["valid"])
-            + len(python_req_result["mismatched"])
-            + len(python_req_result["not_found"])
-            + len(python_req_result["parsing_failed"])
-            == 1
-        )
-    elif extras:
-        assert (
-            len(python_req_result["valid"])
-            + len(python_req_result["mismatched"])
-            + len(python_req_result["not_found"])
-            + len(python_req_result["parsing_failed"])
-            > 1
-        )
+    else:
+        assert python_req_result["valid"]["pyavd"]["installed"] == "5.3.0"
 
 
 @pytest.mark.parametrize(

pyproject.toml

@@ -292,9 +292,9 @@ replace = "version: {new_version}"
 ### Bumping version using PEP440 style like x.y.z.dev1
 
 [[tool.bumpversion.files]]
-filename = "ansible_collections/arista/avd/requirements.txt"
-search = "pyavd[ansible-collection]=={current_version}"
-replace = "pyavd[ansible-collection]=={new_version}"
+filename = "python-avd/pyproject.toml"
+search = 'pyavd=="{current_version}"'
+replace = 'pyavd=="{new_version}"'
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python-avd/pyproject.toml

@@ -81,6 +81,12 @@ include = ["pyavd*"]
 [tool.setuptools.dynamic]
 version = {attr = "pyavd.__version__"}
 
+[dependency-groups]
+# helping pdm generate the flat list of requirements in pre-commit
+self = [
+  "pyavd==6.0.0.dev8"
+]
+
 [tool.black]
 # Black has been replaced with Ruff.
 # This section should be removed later, but kept here to avoid IDE extensions from messing up the code.