allow uris for member expnames (#305)

Author: braingram

changes/305.feature.rst

@@ -0,0 +1 @@
+Support URIs in member expnames in ModelLibrary.

src/stpipe/library.py

@@ -7,6 +7,7 @@
 from collections.abc import Iterable, MutableMapping
 from pathlib import Path
 from types import MappingProxyType
+from urllib.parse import urlparse
 
 import asdf
 
@@ -279,8 +280,9 @@ def __init__(
 
         for index, member in enumerate(self._members):
             if "group_id" not in member:
-                filename = os.path.join(self._asn_dir, member["expname"])
-                member["group_id"] = self._to_group_id(filename, index)
+                member["group_id"] = self._to_group_id(
+                    self._member_to_filename(member), index
+                )
 
         if not on_disk:
             # if models were provided as input, assign the members here
@@ -548,6 +550,27 @@ def _assign_member_to_model(self, model, member):
         if "asn_pool" in self.asn:
             model.meta.asn["pool_name"] = self.asn["asn_pool"]
 
+    def _member_to_filename(self, member):
+        """
+        Resolve a filename for the provided member.
+
+        Parameters
+        ----------
+        member : dict
+            Member definition (likely from an association).
+
+        Returns
+        -------
+        filename : str
+            Filename for the provided member.
+        """
+        member_filename = member["expname"]
+        if self._asn_dir is None:
+            return member_filename
+        if urlparse(member_filename).scheme:
+            return member_filename
+        return os.path.join(self._asn_dir, member["expname"])
+
     def _load_member(self, index):
         """
         Load a model for the association member at the provided index.
@@ -565,7 +588,7 @@ def _load_member(self, index):
         model : DataModel
         """
         member = self._members[index]
-        filename = os.path.join(self._asn_dir, member["expname"])
+        filename = self._member_to_filename(member)
 
         model = self._datamodels_open(filename, **self._datamodels_open_kwargs)
 

tests/test_library.py

@@ -23,6 +23,7 @@
 _N_GROUPS = len(set(_GROUP_IDS))
 _PRODUCT_NAME = "foo_out"
 _INIT_TYPES = ("filename", "asn", "models")
+_TEST_URI_SCHEME = "file://"
 
 
 def _load_asn(filename):
@@ -84,6 +85,14 @@ def _model_to_group_id(self, model):
         raise NoGroupID(f"{model} missing group_id")
 
 
+class URIModelLibrary(ModelLibrary):
+    def _datamodels_open(self, filename, **kwargs):
+        return super()._datamodels_open(filename.removeprefix(_TEST_URI_SCHEME))
+
+    def _filename_to_group_id(self, filename):
+        return super()._filename_to_group_id(filename.removeprefix(_TEST_URI_SCHEME))
+
+
 def _library_to_models(library):
     """
     A few tests are easier to understand and write when
@@ -145,6 +154,17 @@ def example_asn_path(example_models, tmp_path):
     return asn_filename
 
 
+@pytest.fixture
+def example_uri_asn_path(example_asn_path):
+    asn_dir = example_asn_path.parent
+    asn_data = _load_asn(example_asn_path)
+    for member in asn_data["products"][0]["members"]:
+        member["expname"] = _TEST_URI_SCHEME + str(asn_dir / member["expname"])
+    asn_filename = asn_dir / "uri_asn.json"
+    _write_asn(asn_data, asn_filename)
+    return asn_filename
+
+
 @pytest.fixture
 def example_library(example_asn_path):
     """
@@ -801,3 +821,16 @@ def test_get_filename(table_name, filename):
     if table_name != "MISSING":
         lib._asn["table_name"] = table_name
     assert Step._get_filename(lib) == filename
+
+
+@pytest.mark.parametrize("loaded", [False, True])
+def test_uri_member_paths(example_uri_asn_path, loaded):
+    if loaded:
+        asn = _load_asn(example_uri_asn_path)
+    else:
+        asn = example_uri_asn_path
+    lib = URIModelLibrary(asn)
+    with lib:
+        for i, model in enumerate(lib):
+            assert model
+            lib.shelve(model, i, modify=False)