Support output_class in to_py.

Please note that it changes the logic of traversal if `output_class` is provided: now during previsit we do not create objects instances but rather store their classes and then create them when visiting ojects/lists.

PiperOrigin-RevId: 891851157
Change-Id: Idc03bef7a1a2473ae382e146494c827f92f8bc8d

Author: atorero

py/koladata/base/py_conversions/dataclasses_util.cc

@@ -183,33 +183,6 @@ DataClassesUtil::CreateClassInstanceKwargs(
   return py_instance;
 }
 
-absl::StatusOr<arolla::python::PyObjectPtr>
-DataClassesUtil::CreateClassInstanceArgs(
-    PyObjectPtr absl_nonnull py_class,
-    absl::Span<const arolla::python::PyObjectPtr absl_nonnull> args) {
-  PyObjectPtr py_tuple = PyObjectPtr::Own(PyTuple_New(args.size()));
-  if (py_tuple == nullptr) {
-    return arolla::python::StatusCausedByPyErr(
-        absl::StatusCode::kInternal,
-        absl::StrFormat("could not create a new tuple of size %d",
-                        args.size()));
-  }
-  for (size_t i = 0; i < args.size(); ++i) {
-    PyTuple_SET_ITEM(py_tuple.get(), i, Py_NewRef(args[i].get()));
-  }
-
-  PyObjectPtr py_instance =
-      PyObjectPtr::Own(PyObject_CallOneArg(py_class.get(), py_tuple.get()));
-
-  if (py_instance == nullptr) {
-    return arolla::python::StatusWithRawPyErr(
-        absl::StatusCode::kInvalidArgument,
-        "could not create a new instance of the class");
-  }
-
-  return py_instance;
-}
-
 absl::StatusOr<arolla::python::PyObjectPtr>
 DataClassesUtil::GetSimpleNamespaceClass() {
   RETURN_IF_ERROR(InitFns());

py/koladata/base/py_conversions/dataclasses_util.h

@@ -92,12 +92,6 @@ class DataClassesUtil {
       absl::Span<const std::string> attr_names,
       absl::Span<const arolla::python::PyObjectPtr absl_nonnull> attr_values);
 
-  // Creates a class instance with the given args.
-  // Returned value is owned by DataClassesUtil.
-  absl::StatusOr<arolla::python::PyObjectPtr> CreateClassInstanceArgs(
-      arolla::python::PyObjectPtr absl_nonnull py_class,
-      absl::Span<const arolla::python::PyObjectPtr absl_nonnull> args);
-
   // Returns a new reference to the SimpleNamespace class.
   absl::StatusOr<arolla::python::PyObjectPtr> GetSimpleNamespaceClass();
 

py/koladata/base/py_conversions/dataclasses_util.py

@@ -136,13 +136,25 @@ def get_class_field_type(
     raise ValueError(f"field '{attr_name}' has unsupported type: {t}")
 
   if origin_type := typing.get_origin(py_obj):
-    # `list[Obj]`
+    # `list[Obj] or `tuple[Obj, ...]`
     if (
         attr_name == '__items__'
         and isinstance(origin_type, type)
         and issubclass(origin_type, typing.Sequence)
     ):
-      return typing.get_args(py_obj)[0]
+      args = typing.get_args(py_obj)
+      if len(args) == 0:
+        raise ValueError(
+            'expected list/tuple/Sequence; got instead:'
+            f' {typing.get_origin(py_obj)}'
+        )
+      if origin_type is tuple:
+        if len(args) != 2 or args[1] is not Ellipsis:
+          raise ValueError(
+              'only tuple[T, ...] is supported; got instead:'
+              f' {typing.get_origin(py_obj)}'
+          )
+      return args[0]
 
     if isinstance(origin_type, type) and issubclass(
         origin_type, typing.Mapping
@@ -155,9 +167,13 @@ def get_class_field_type(
         args = typing.get_args(py_obj)
         if len(args) != 2:
           raise ValueError(
-              f'Expected dict; got instead: {typing.get_origin(py_obj)}'
+              f'expected dict; got instead: {typing.get_origin(py_obj)}'
           )
         return args[1]
+    if attr_name == '__keys__' or attr_name == '__values__':
+      raise ValueError(f'expected dict class; got instead: {py_obj}')
+    if attr_name == '__items__':
+      raise ValueError(f'expected list class; got instead: {py_obj}')
     raise ValueError(
         f'unsupported GenericAlias {py_obj} for attribute {attr_name}'
     )
@@ -173,11 +189,30 @@ def has_optional_field(
     attr_name: str,
     type_hints_cache: dict[_Type, dict[str, _Type]],
 ) -> bool:
-  """Returns whether the given attribute exists and is optional."""
+  """Returns whether the given attribute is present and is optional.
+
+  If the attribute is not present, returns False.
+  If the attribute is present but is not optional, raises ValueError.
+
+  Args:
+    py_class: The class to inspect.
+    attr_name: The name of the attribute to inspect.
+    type_hints_cache: A cache of type hints for dataclasses.
+
+  Raises:
+    ValueError: If the attribute is present but is not optional.
+
+  Returns:
+    True if the given attribute exists and is optional, False otherwise.
+  """
   if not dataclasses.is_dataclass(py_class):
     return False
   field = _get_field_type_annotation(py_class, attr_name, type_hints_cache)
-  return _get_underlying_optional_type(field) is not None
+  if field is None:
+    return False
+  if _get_underlying_optional_type(field) is None:
+    raise ValueError(f'field cannot have missing values: {attr_name}')
+  return True
 
 
 _simple_namespace_class = types.SimpleNamespace

py/koladata/base/py_conversions/dataclasses_util_test.py

@@ -109,6 +109,7 @@ class Obj2:
 
     self.assertEqual(util.get_class_field_type(Obj2, 'a', False), Obj1)
     self.assertIsNone(util.get_class_field_type(Obj2, 'b', False))
+    self.assertIsNone(util.get_class_field_type(Obj2, 'b', True))
 
   def test_get_class_field_type_optional(self):
     util = testing_clib.DataClassesUtil()
@@ -121,18 +122,25 @@ class Obj2:
     self.assertEqual(util.get_class_field_type(Obj2, 'a', False), Obj1)
     self.assertEqual(util.get_class_field_type(Obj2, 'b', False), Obj1)
 
-  def test_get_class_field_list(self):
+  def test_get_class_field_list_tuple(self):
     util = testing_clib.DataClassesUtil()
 
     @dataclasses.dataclass
     class Obj2:
       a: list[Obj1]
+      b: tuple[Obj1, ...]
 
     list_type = util.get_class_field_type(Obj2, 'a', False)
+    tuple_type = util.get_class_field_type(Obj2, 'b', False)
     self.assertEqual(list_type, list[Obj1])
+    self.assertEqual(tuple_type, tuple[Obj1, ...])
     self.assertEqual(
         util.get_class_field_type(list_type, '__items__', False), Obj1
     )
+    with self.assertRaisesRegex(ValueError, 'only tuple.T, .... is supported'):
+      _ = util.get_class_field_type(tuple[Obj1, Obj1], '__items__', False)
+    with self.assertRaisesRegex(ValueError, 'only tuple.T, .... is supported'):
+      _ = util.get_class_field_type(tuple[Obj1], '__items__', False)
 
   def test_get_class_field_dict(self):
     util = testing_clib.DataClassesUtil()
@@ -192,6 +200,21 @@ def test_get_class_field_type_errors(self):
     ):
       _ = util.get_class_field_type(int, 'a', False)
 
+    with self.assertRaisesRegex(
+        ValueError, 'expected dict class; got instead: list.int.'
+    ):
+      _ = util.get_class_field_type(list[int], '__keys__', False)
+
+    with self.assertRaisesRegex(
+        ValueError, 'expected dict class; got instead: list.int.'
+    ):
+      _ = util.get_class_field_type(list[int], '__values__', False)
+
+    with self.assertRaisesRegex(
+        ValueError, 'expected list class; got instead: dict.int. int.'
+    ):
+      _ = util.get_class_field_type(dict[int, int], '__items__', False)
+
   def test_has_optional_field(self):
     util = testing_clib.DataClassesUtil()
 
@@ -205,17 +228,36 @@ class Obj2:
       bad_0: int | Any
       bad_1: int | float | None
       bad_2: None | int
+      g: int = 1
 
     self.assertTrue(util.has_optional_field(Obj2, 'a'))
     self.assertFalse(util.has_optional_field(Obj2, 'b'))
     self.assertTrue(util.has_optional_field(Obj2, 'c'))
-    self.assertFalse(util.has_optional_field(Obj2, 'd'))
-    self.assertFalse(util.has_optional_field(Obj2, 'e'))
-    self.assertFalse(util.has_optional_field(Obj2, 'f'))
+    with self.assertRaisesRegex(
+        ValueError,
+        'field cannot have missing values: d',
+    ):
+      _ = util.has_optional_field(Obj2, 'd')
+    with self.assertRaisesRegex(
+        ValueError,
+        'field cannot have missing values: e',
+    ):
+      _ = util.has_optional_field(Obj2, 'e')
+    with self.assertRaisesRegex(
+        ValueError,
+        'field cannot have missing values: f',
+    ):
+      _ = util.has_optional_field(Obj2, 'f')
+    with self.assertRaisesRegex(
+        ValueError,
+        'field cannot have missing values: g',
+    ):
+      _ = util.has_optional_field(Obj2, 'g')
     self.assertFalse(util.has_optional_field(int, 'non_existent_field'))
     with self.assertRaisesRegex(
         ValueError,
-        'only unions `SomeType | None` are supported ; got instead: int | Any',
+        'only unions `SomeType | None` are supported ; got instead: int |'
+        ' typing.Any',
     ):
       _ = util.has_optional_field(Obj2, 'bad_0')
 
@@ -237,13 +279,6 @@ def test_create_class_instance_kwargs(self):
     obj = util.create_class_instance_kwargs(Obj1, ['a', 'b'], [123, 'abc'])
     self.assertEqual(obj, Obj1(a=123, b='abc'))
 
-  def test_create_class_instance_args(self):
-    util = testing_clib.DataClassesUtil()
-    obj = util.create_class_instance_args(list, [1, 2, 3])
-    self.assertEqual(obj, list([1, 2, 3]))
-    obj2 = util.create_class_instance_args(tuple, [1, 2, 3])
-    self.assertEqual(obj2, tuple([1, 2, 3]))
-
   def test_get_simple_namespace_class(self):
     util = testing_clib.DataClassesUtil()
     simple_namespace_class = util.get_simple_namespace_class()

py/koladata/base/py_conversions/testing_clib.cc

@@ -85,21 +85,6 @@ PYBIND11_MODULE(testing_clib, m) {
                      attr_names_vec, attr_values_ptrs));
              return py::reinterpret_steal<py::object>(res.release());
            })
-      .def("create_class_instance_args",
-           [](DataClassesUtil& self, py::handle py_class,
-              absl::Span<const py::handle> attr_values) -> py::object {
-             std::vector<arolla::python::PyObjectPtr> attr_values_ptrs;
-             for (const auto& attr_value : attr_values) {
-               attr_values_ptrs.push_back(
-                   arolla::python::PyObjectPtr::NewRef(attr_value.ptr()));
-             }
-
-             auto res = arolla::python::pybind11_unstatus_or(
-                 self.CreateClassInstanceArgs(
-                     arolla::python::PyObjectPtr::NewRef(py_class.ptr()),
-                     attr_values_ptrs));
-             return py::reinterpret_steal<py::object>(res.release());
-           })
       .def("get_simple_namespace_class",
            [](DataClassesUtil& self) -> py::object {
              arolla::python::PyObjectPtr py_obj =