feat(speckle_ifc): More granular IFC conversion progress

src/speckleifc/__main__.py

@@ -61,4 +61,4 @@ def cmd_line_import() -> None:
 if __name__ == "__main__":
     start = time.time()
     cmd_line_import()
-    print(f"Total time (including cleanup): {(time.time() - start) * 1000}ms")
+    print(f"Total time (including cleanup): {(time.time() - start):.3f}s")

src/speckleifc/importer.py

@@ -22,12 +22,15 @@
 from specklepy.objects.data_objects import DataObject
 from specklepy.objects.models.collections.collection import Collection
 from specklepy.objects.proxies import InstanceProxy
+from specklepy.progress.ingestion_progress import IngestionProgressManager
 
 
 @dataclass
 class ImportJob:
     ifc_file: file
 
+    progress: IngestionProgressManager
+
     _render_material_manager: RenderMaterialProxyManager = field(
         default_factory=lambda: RenderMaterialProxyManager()
     )
@@ -39,6 +42,7 @@ class ImportJob:
     )
     geometries_count: int = 0
     geometries_used: int = 0
+    elements_converted: int = 0
     _current_storey_data_object: DataObject | None = field(default=None, init=False)
 
     _display_value_cache: dict[int, list[Base]] = field(default_factory=dict)
@@ -108,6 +112,12 @@ def _convert_element(
 
         # Restore previous storey context
         self._current_storey_data_object = previous_storey_data_object
+        self.elements_converted += 1
+        if self.progress.should_report_progress():
+            self.progress.report(
+                f"Converted {self.elements_converted:,} elements", None
+            )
+
         return result
 
     def _convert_children(self, step_element: entity_instance) -> list[Base]:
@@ -132,20 +142,27 @@ def _should_convert(step_element: entity_instance) -> bool:
     def convert(self) -> Base:
         start = time.time()
         self.pre_process_geometry()
-        print(f"Geometry conversion complete after {(time.time() - start) * 1000}ms")
+        print(
+            f"Geometry conversion complete after {(time.time() - start):.3f}s"  # noqa: E501
+        )
         print(f"Created {self.geometries_count} geometries")
 
         start = time.time()
         root = self._convert_project_tree()
-        print(f"Object tree conversion complete after {(time.time() - start) * 1000}ms")
+        print(
+            f"Element tree conversion complete after {(time.time() - start):.3f}s"  # noqa: E501
+        )
         print(f"Used {self.geometries_used} geometries")
         return root
 
     def pre_process_geometry(self) -> None:
         iterator = create_geometry_iterator(self.ifc_file)
         if not iterator.initialize():
             raise SpeckleException("Failed to find any geometry in file")
+
+        self.progress.report("Converting geometries", None)
         self.geometries_count = 0
+
         while True:
             shape = cast(TriangulationElement, iterator.get())
             self.geometries_count += 1
@@ -157,6 +174,11 @@ def pre_process_geometry(self) -> None:
                 raise SpeckleException(
                     f"Failed to convert geometry with id: {id}"
                 ) from ex
+
+            if self.progress.should_report_progress():
+                self.progress.report(
+                    f"Converted {self.geometries_count:,} geometries", None
+                )
             if not iterator.next():
                 break
 
@@ -197,6 +219,8 @@ def _convert_project_tree(self) -> Base:
             raise SpeckleException("Expected exactly one IfcProject in file")
         project = projects[0]
 
+        self.progress.report("Converting elements", None)
+
         tree = self.convert_element(project)
         if not isinstance(tree, Collection):
             raise TypeError("Expected root object to convert to a Collection")

src/speckleifc/main.py

@@ -11,19 +11,25 @@
     ModelIngestionFailedInput,
     ModelIngestionStartProcessingInput,
     ModelIngestionSuccessInput,
-    ModelIngestionUpdateInput,
     SourceDataInput,
 )
 from specklepy.core.api.models.current import Project, Version
 from specklepy.core.api.operations import send
 from specklepy.logging import metrics
+from specklepy.progress.ingestion_progress import IngestionProgressManager
+from specklepy.progress.progress_transport import ProgressTransport
 from specklepy.transports.server import ServerTransport
 
+# Since progress messages are currently blocking (no async), we're being extra coarse
+# with progress updates to ensure we're not waisting time sending updates.
+# We could maybe go a little lower, but for now I'm not risking degrading performance
+PROGRESS_INTERVAL_SECONDS = 10
+
 
 def open_and_convert_file(
     file_path: str,
     project: Project,
-    version_message: str,
+    version_message: str | None,
     model_ingestion_id: str,
     client: SpeckleClient,
 ) -> Version:
@@ -33,7 +39,7 @@ def open_and_convert_file(
         path = Path(file_path)
 
         specklepy_version = importlib.metadata.version("specklepy")
-        client.model_ingestion.start_processing(
+        ingestion = client.model_ingestion.start_processing(
             ModelIngestionStartProcessingInput(
                 project_id=project.id,
                 ingestion_id=model_ingestion_id,
@@ -46,39 +52,38 @@ def open_and_convert_file(
                 ),
             )
         )
-
+        progress = IngestionProgressManager(
+            client, ingestion, PROGRESS_INTERVAL_SECONDS
+        )
         account = client.account
         server_url = account.serverInfo.url
         assert server_url
         remote_transport = ServerTransport(project.id, account=account)
+        progress_transport = ProgressTransport(
+            progress,
+        )
 
+        progress.report("Opening file", None)
         ifc_file = open_ifc(file_path)  # pyright: ignore[reportUnknownVariableType]
 
-        client.model_ingestion.update_progress(
-            ModelIngestionUpdateInput(
-                project_id=project.id,
-                ingestion_id=model_ingestion_id,
-                progress_message="Converting file",
-                progress=None,
-            )
-        )
-        import_job = ImportJob(ifc_file)  # pyright: ignore[reportUnknownArgumentType]
+        import_job = ImportJob(ifc_file, progress)  # pyright: ignore[reportUnknownArgumentType]
         data = import_job.convert()
 
-        print(f"File conversion complete after {(time.time() - start) * 1000}ms")
+        print(
+            f"File conversion complete after {(time.time() - start):.3f}s"  # noqa: E501
+        )
 
         start = time.time()
 
-        client.model_ingestion.update_progress(
-            ModelIngestionUpdateInput(
-                project_id=project.id,
-                ingestion_id=model_ingestion_id,
-                progress_message="Uploading objects",
-                progress=None,
-            )
+        progress.report("Uploading objects", None)
+        root_id = send(
+            data,
+            transports=[remote_transport, progress_transport],
+            use_default_cache=False,
+        )
+        print(
+            f"Sending to speckle complete after: {(time.time() - start):.3f}s"  # noqa: E501
         )
-        root_id = send(data, transports=[remote_transport], use_default_cache=False)
-        print(f"Sending to speckle complete after: {(time.time() - start) * 1000}ms")
 
         start = time.time()
 
@@ -95,9 +100,9 @@ def open_and_convert_file(
         version = client.version.get(version_id, project.id)
 
         end = time.time()
-        print(f"Version committed after: {(end - start) * 1000}ms")
+        print(f"Version committed after: {(end - start):.3f}s")
 
-        print(f"Total time (to commit): {(end - very_start) * 1000}ms")
+        print(f"Total time (to commit): {(end - very_start):.3f}s")
         del ifc_file
 
         custom_properties = {"ui": "dui3", "actionSource": "import"}

src/speckleifc/manual_import.py

@@ -0,0 +1,51 @@
+import time
+
+from speckleifc.main import open_and_convert_file
+from specklepy.api.client import SpeckleClient
+from specklepy.core.api.credentials import get_accounts_for_server
+from specklepy.logging import metrics
+
+
+def _manual_import() -> None:
+    from specklepy.core.api.inputs.model_ingestion_inputs import (
+        ModelIngestionCreateInput,
+        SourceDataInput,
+    )
+
+    PROJECT_ID = "412a3c3927"
+    MODEL_ID = "223e61212d"
+    SERVER_URL = "latest.speckle.systems"
+    FILE_PATH = r"C:\Test Files\ifc\AC20-FZK-Haus.ifc"  # noqa: E501
+
+    metrics.set_host_app(
+        "ifc",
+    )
+
+    account = get_accounts_for_server(SERVER_URL)[0]
+    client = SpeckleClient(SERVER_URL, use_ssl=not SERVER_URL.startswith("http://"))
+    client.authenticate_with_account(account)
+
+    ingestion = client.model_ingestion.create(
+        ModelIngestionCreateInput(
+            model_id=MODEL_ID,
+            project_id=PROJECT_ID,
+            progress_message="",
+            source_data=SourceDataInput(
+                source_application_slug="speckleifc",
+                source_application_version="0.0.0",
+                file_name=None,
+                file_size_bytes=None,
+            ),
+            max_idle_timeout_seconds=2700,  # 45mins
+        )
+    )
+    project = client.project.get(PROJECT_ID)
+
+    open_and_convert_file(FILE_PATH, project, None, ingestion.id, client)
+
+
+if __name__ == "__main__":
+    start = time.time()
+
+    _manual_import()
+    print(f"Total time (including cleanup): {(time.time() - start):.3f}s")

src/specklepy/core/api/inputs/model_ingestion_inputs.py

@@ -14,6 +14,7 @@ class ModelIngestionCreateInput(GraphQLBaseModel):
     project_id: str
     progress_message: str
     source_data: SourceDataInput
+    max_idle_timeout_seconds: int | None = None
 
 
 class ModelIngestionStartProcessingInput(GraphQLBaseModel):

src/specklepy/core/api/models/current.py

@@ -254,12 +254,15 @@ class FileUploadUrl(GraphQLBaseModel):
 class ModelIngestionStatusData(GraphQLBaseModel):
     status: ModelIngestionStatus
     progress_message: str | None = None
+    version_id: str | None = None
 
 
 class ModelIngestion(GraphQLBaseModel):
     id: str
     created_at: datetime
     updated_at: datetime
-    cancellation_requested: bool
     model_id: str
+    project_id: str
+    user_id: str
+    cancellation_requested: bool
     status_data: ModelIngestionStatusData

src/specklepy/core/api/resources/current/model_ingestion_resource.py

@@ -50,6 +50,8 @@ def get_ingestion(self, project_id: str, model_ingestion_id: str) -> ModelIngest
                   createdAt
                   updatedAt
                   modelId
+                  projectId
+                  userId
                   cancellationRequested
                   statusData {
                     ... on HasModelIngestionStatus {
@@ -58,6 +60,10 @@ def get_ingestion(self, project_id: str, model_ingestion_id: str) -> ModelIngest
                     ... on HasProgressMessage {
                       progressMessage
                     }
+                    ... on ModelIngestionSuccessStatus
+                    {
+                      versionId
+                    }
                   }
                 }
               }
@@ -87,6 +93,8 @@ def create(self, input: ModelIngestionCreateInput) -> ModelIngestion:
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -124,6 +132,8 @@ def start_processing(
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -159,6 +169,8 @@ def requeue(self, input: ModelIngestionRequeueInput) -> ModelIngestion:
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -196,6 +208,8 @@ def update_progress(self, input: ModelIngestionUpdateInput) -> ModelIngestion:
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -277,6 +291,8 @@ def fail_with_error(self, input: ModelIngestionFailedInput) -> ModelIngestion:
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -320,6 +336,8 @@ def fail_with_cancel(self, input: ModelIngestionCancelledInput) -> ModelIngestio
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {
@@ -370,6 +388,8 @@ def request_cancellation(
                     createdAt
                     updatedAt
                     modelId
+                    projectId
+                    userId
                     cancellationRequested
                     statusData {
                       ... on HasModelIngestionStatus {

src/specklepy/core/api/resources/current/subscription_resource.py

@@ -224,6 +224,8 @@ async def project_model_ingestion_updated(
                   createdAt
                   updatedAt
                   modelId
+                  projectId
+                  userId
                   cancellationRequested
                   statusData {
                     ... on HasModelIngestionStatus {
@@ -232,6 +234,10 @@ async def project_model_ingestion_updated(
                     ... on HasProgressMessage {
                       progressMessage
                     }
+                    ... on ModelIngestionSuccessStatus
+                    {
+                        versionId
+                    }
                   }
                 }
                 type