feat(datahub-actions): bundle datahub-gc and datahub-documents venvs (#16781)

Author: david-leifker

docker/datahub-actions/Dockerfile

@@ -148,7 +148,7 @@ FROM ingestion-base-slim AS bundled-venvs-full
 USER 0
 
 # Set up bundled venv configuration for FULL variant (with PySpark)
-ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file"
+ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file,datahub-gc,datahub-documents"
 ARG BUNDLED_VENV_SLIM_MODE="false"
 ARG BUNDLED_CLI_VERSION
 ENV DATAHUB_BUNDLED_VENV_PATH=/opt/datahub/venvs
@@ -190,7 +190,7 @@ USER 0
 
 # Set up bundled venv configuration for SLIM variant (without PySpark)
 # Venv named s3-bundled but uses s3-slim package internally
-ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file"
+ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file,datahub-gc,datahub-documents"
 ARG BUNDLED_VENV_SLIM_MODE="true"
 ARG BUNDLED_CLI_VERSION
 ENV DATAHUB_BUNDLED_VENV_PATH=/opt/datahub/venvs
@@ -232,7 +232,7 @@ USER 0
 
 # Set up bundled venv configuration for LOCKED variant (without PySpark, network blocked)
 # Same as slim but will have network access disabled in final stage
-ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file"
+ARG BUNDLED_VENV_PLUGINS="s3,demo-data,file,datahub-gc,datahub-documents"
 ARG BUNDLED_VENV_SLIM_MODE="true"
 ARG BUNDLED_CLI_VERSION
 ENV DATAHUB_BUNDLED_VENV_PATH=/opt/datahub/venvs

metadata-ingestion/pyproject.toml

@@ -363,6 +363,8 @@ datahub-documents = [
     "urllib3>=1.26,<3.0",
 ]
 
+datahub-gc = []
+
 datahub-kafka = [
     "confluent_kafka[schemaregistry,avro]>=1.9.0,!= 2.8.1,<3.0.0",
     "fastavro>=1.2.0,<2.0.0",

metadata-ingestion/setup.py

@@ -680,6 +680,7 @@
         "setuptools<82",
     },
     "datahub-debug": {"dnspython==2.7.0", "requests<3.0.0"},
+    "datahub-gc": set(),
     "datahub-documents": unstructured_lib,
     "mode": {"requests<3.0.0", "python-liquid<2", "tenacity>=8.0.1,<9.0.0"}
     | sqlglot_lib