feat: implement python dependency lifecycle and reference counting

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 - #1024: Added flag -export-python-deps to publish command
+- #1025: implement python dependency lifecycle and reference counting
 
 ### Fixed
 - #996: Ensure COS commands execute in exec under a dedicated, isolated context

src/cls/IPM/Lifecycle/Base.cls

@@ -491,13 +491,37 @@ Method %Clean(ByRef pParams) As %Status
             if $$$ISERR(tSC) {
                 quit
             }
+            // validates the python libaraies need to be uninstalled as part of the clean process
+            if $get(pParams("with-py")) {
+                do ..UninstallPythonDeps(.pParams)
+            }
         }
     } catch e {
         set tSC = e.AsStatus()
     }
     quit tSC
 }
 
+Method UninstallPythonDeps(pParams)
+{
+    if '##class(%IPM.Utils.Python).IsPythonEnabled() {
+        quit $$$OK
+    }
+    if '..Module.PythonDependencies.Count() {
+        quit $$$OK
+    }
+    do ..Log("Evaluating Python dependencies for removal...")
+    set key = ""
+    set ^||%IPM.PipCaller = ##class(%IPM.Utils.Python).ResolvePipCaller(.pParams)
+    for {
+        set pyRefOref = ..Module.PythonDependencies.GetNext(.key)
+        quit:key=""
+        set sc = ##class(%IPM.Storage.PythonReference).RemovePythonReference(key, .pParams)
+        $$$ThrowOnError(sc)
+    }
+    //kill ^||%IPM.PipCaller
+}
+
 Method %ExportData(ByRef pParams) As %Status
 {
     quit $$$OK
@@ -746,16 +770,10 @@ Method InstallOrDownloadPythonRequirements(
     }
     set tSC = $$$OK
     try {
-        set target = ##class(%File).NormalizeDirectory("python", $system.Util.ManagerDirectory())
-        if '$system.CLS.IsMthd("%SYS.Python", "Import") {
-            throw ##class(%Exception.General).%New("Embedded Python is not available in this instance.")
-        }
         set processType = ""
-        set tSysModule = ##class(%SYS.Python).Import("sys")
-        set tPyMajor = tSysModule."version_info".major
-        set tPyMinor = tSysModule."version_info".minor
-        set tPyMicro = tSysModule."version_info".micro
-        set tPyVersion = tPyMajor_"."_tPyMinor_"."_tPyMicro
+        set stdout = ""
+        set target = ##class(%IPM.Utils.Python).PythonManagerDir()
+        $$$ThrowOnError(##class(%IPM.Utils.Python).GetPythonVersion(.tPyVersion))
         if download {
             set processType = "Download Python wheels"
             do ..Log(processType _ " START")
@@ -783,6 +801,8 @@ Method InstallOrDownloadPythonRequirements(
             }
             set tSC = ##class(%IPM.Utils.Module).RunCommand(pRoot, command,.stdout)
             $$$ThrowOnError(tSC)
+            set tSC = ..SavePythonDependencies(pythonRequirements)
+            $$$ThrowOnError(tSC)
         }
         do ..Log(processType _ " SUCCESS")
     } catch ex {
@@ -792,6 +812,41 @@ Method InstallOrDownloadPythonRequirements(
     quit tSC
 }
 
+Method SavePythonDependencies(pythonFileLocation As %String = "") As %Status
+{
+    quit:pythonFileLocation="" $$$OK
+    do ..Log("Saving python dependencies from "_pythonFileLocation)
+    set file = ##class(%File).%New(pythonFileLocation)
+    set sc = file.Open("R")
+    if $$$ISERR(sc) {
+        return sc
+    }
+    while 'file.AtEnd {
+        set lib = file.ReadLine()
+        set lib = $zstrip(lib, "<>W")
+        if lib="" continue
+        if (lib [ ".whl") || (lib [ ".wheel") {
+            set whlFile = $piece($translate(lib, "\", "/"), "/", *)
+            set libraryName = $piece(whlFile, "-")
+        } else {
+            set libraryName = $piece(lib,"==")
+        }
+        // check does this Module already have this EXACT library and version?
+        set pyRefObj =..Module.PythonDependencies.GetAt($$$lcase(libraryName))
+        if $isobject(pyRefObj) {
+            // already have this exact library and version, skip to next
+            if pyRefObj.IsVersionMatch(lib) {
+                continue
+            }
+        }
+        set pytonlibObj = ##class(%IPM.Storage.PythonReference).SavePythonDependencies(lib)
+        set sc = ..Module.PythonDependencies.SetAt(pytonlibObj, pytonlibObj.LibraryName)
+    }
+    // Save the python dependencies to the module object before reload.
+    set sc = ..Module.%Save()
+    return sc
+}
+
 ClassMethod ResolvePipCaller(ByRef pParams) As %List
 {
     set tUseStandalonePip = ##class(%IPM.Repo.UniversalSettings).GetValue("UseStandalonePip")

src/cls/IPM/Main.cls

@@ -470,6 +470,13 @@ reinstall -env /path/to/env1.json;/path/to/env2.json example-package
     uninstall HS.JSON
 </example>
 
+<example description="Uninstalls HS.JSON from the current namespace with python dependencies.">
+    uninstall HS.JSON -with-py
+</example>
+
+<example description="Uninstalls HS.JSON from the current namespace with the dependent modules with python dependencies.">
+    uninstall HS.JSON -r -with-py
+</example>
 <!-- Parameters -->
 <parameter name="module" description="Name of module to uninstall" />
 
@@ -481,6 +488,7 @@ reinstall -env /path/to/env1.json;/path/to/env2.json example-package
 <modifier name="quiet" aliases="q" dataAlias="Verbose" dataValue="0" description="Produces minimal output from the command." />
 <modifier name="verbose" aliases="v" dataAlias="Verbose" dataValue="1" description="Produces verbose output from the command." />
 <modifier name="purge" dataAlias="Purge" dataValue="1" description="Purge data from tables during uninstall." />
+<modifier name="with-py" dataAlias="with-py" dataValue="1" description="Remove the associated Python libraries if no longer needed by other modules." />
 
 </command>
 
@@ -972,6 +980,7 @@ ClassMethod ShellInternal(
 			set $$$ZPMCommandToLog = tCommand
 
             if (tCommandInfo = "quit") {
+                do ..OnBeforeShellExit()
                 return
             } elseif (tCommandInfo = "help") {
                 do ..%Help(.tCommandInfo)
@@ -4075,6 +4084,15 @@ ClassMethod GetPythonInstalledLibs(Output list)
 	}
 }
 
+/// Clean up any temporary data or PPG before exiting the shell
+ClassMethod OnBeforeShellExit() [ Internal ]
+{
+    try{
+        kill ^||%IPM.PipCaller
+    }catch ex{
+    }
+}
+
 ClassMethod GetPythonLibrariesList(PythonPath As %String) As %String [ Language = python ]
 {
 import sys

src/cls/IPM/Storage/Module.cls

@@ -87,6 +87,8 @@ Property Deployed As %Boolean(XMLPROJECTION = "Element");
 
 Property UpdatePackage As %Dictionary.Classname;
 
+Property PythonDependencies As array Of %IPM.Storage.PythonReference(XMLPROJECTION = "NONE");
+
 /// Iterates through all of a module's update step classes and tries to either execute or seed those steps.
 /// seedSteps: (Default) 0 = Execute Steps | 1 = Seeds Steps
 Method HandleAllUpdateSteps(
@@ -1923,6 +1925,11 @@ Storage Default
 <Value>UpdatePackage</Value>
 </Value>
 </Data>
+<Data name="PythonDependencies">
+<Attribute>PythonDependencies</Attribute>
+<Structure>subnode</Structure>
+<Subscript>"PythonDependencies"</Subscript>
+</Data>
 <DataLocation>^IPM.Storage.ModuleD</DataLocation>
 <DefaultData>ModuleDefaultData</DefaultData>
 <IdLocation>^IPM.Storage.ModuleD</IdLocation>

src/cls/IPM/Storage/PythonReference.cls

@@ -0,0 +1,176 @@
+/// store the python libraries mentioned in the requirements.txt file
+Class %IPM.Storage.PythonReference Extends (%Persistent, %XML.Adaptor, %IPM.Utils.ComparisonAdaptor)
+{
+
+Parameter DEFAULTGLOBAL = "^IPM.Storage.PythonRefs";
+
+Property LibraryName As %String(MAXLEN = 255) [ Required ];
+
+Property Version As %String;
+
+Property SourceType As %String(VALUELIST = ",pypi,whl") [ InitialExpression = "pypi" ];
+
+/// The physical path to the .whl file on the server
+Property WheelPath As %String(MAXLEN = 1000);
+
+/// The specific platform tag (e.g., 'manylinux_2_17_x86_64' or 'win_amd64')
+Property PlatformTag As %String(MAXLEN = 100);
+
+Property RefCount As %Integer [ InitialExpression = 1 ];
+
+Index LibIdx On LibraryName [ IdKey, Unique ];
+
+ClassMethod SavePythonDependencies(pLibrarySpec As %String) As %IPM.Storage.PythonReference
+{
+    set libraryInfo = ..ParseLibrarySpec(pLibrarySpec)
+    if libraryInfo="" {
+        quit $$$ERROR("Invalid library specification: "_pLibrarySpec)
+    }
+    set libraryName = $listget(libraryInfo)
+    set libararyVersion = $listget(libraryInfo,2)
+    set pyRef = ##class(%IPM.Storage.PythonReference).%OpenId(libraryName)
+    if $isobject(pyRef) {
+        set pyRef.RefCount = pyRef.RefCount + 1
+    } else {
+        set pyRef = ..%New()
+        set pyRef.LibraryName = libraryName
+    }
+    if libararyVersion'="" {
+        set pyRef.Version = libararyVersion
+    }
+    if $listget(libraryInfo,3)="pypi" {
+        set pyRef.SourceType = $listget(libraryInfo,3)
+        set pyRef.PlatformTag = ""
+        set pyRef.WheelPath = ""
+    }
+    else {
+        set pyRef.SourceType = $listget(libraryInfo,3)
+        set pyRef.PlatformTag = $listget(libraryInfo,4)
+        set pyRef.WheelPath = $listget(libraryInfo,5)
+    }
+    $$$ThrowOnError(pyRef.%Save())
+    quit pyRef
+}
+
+ClassMethod ParseLibrarySpec(pLibrarySpec As %String) As %String
+{
+    if pLibrarySpec="" {
+        quit ""
+    }
+    if (pLibrarySpec[".whl")||(pLibrarySpec[".wheel") {
+        set filename = $piece($translate(pLibrarySpec, "\", "/"), "/", *)
+        set libName = $$$lcase($piece(filename, "-", 1))
+        set platformTag = $piece($piece(filename, "-", 5),".")
+        set version = $piece(filename, "-", 2)
+        quit $listbuild(libName, version,"whl",platformTag,pLibrarySpec)
+    }
+    else {
+        set filename = $$$lcase($piece(pLibrarySpec,"=="))
+        set version = $piece(pLibrarySpec,"==",2)
+        quit $listbuild(filename,version,"pypi")
+    }
+}
+
+/// Checks if this specific instance's version matches the libraryInfo
+Method IsVersionMatch(pLibrarySpec As %String) As %Boolean
+{
+    set libraryInfo = ..ParseLibrarySpec(pLibrarySpec)
+    set version = $listget(libraryInfo, 2)
+    if version = "" {
+        quit 1
+    }
+    quit ..Version = version
+}
+
+ClassMethod IsPythonDependencyInstalled(pLibrarySpec As %String) As %Boolean
+{
+    set libraryInfo = ..ParseLibrarySpec(pLibrarySpec)
+    set libraryName = $listget(libraryInfo)
+    set version = $listget(libraryInfo, 2)
+    set pyRef = ##class(%IPM.Storage.PythonReference).%OpenId(libraryName)
+    if '$isobject(pyRef) {
+        quit 0
+    }
+    if version'=""&&(pyRef.Version'=version) {
+        quit 0
+    }
+    quit $$$OK
+}
+
+ClassMethod RemovePythonReference(
+	pLibrarySpec As %String,
+	ByRef pParams) As %Status
+{
+    set libraryInfo = ..ParseLibrarySpec(pLibrarySpec)
+    set libraryName = $listget(libraryInfo)
+    set pyRef = ##class(%IPM.Storage.PythonReference).%OpenId(libraryName)
+    if '$isobject(pyRef) {
+        quit $$$OK
+    }
+    if pyRef.RefCount<=1 {
+        do ..UninstallPythonLibrary(libraryName, .pParams)
+        $$$ThrowOnError(..%DeleteId(libraryName))
+    } else {
+        set pyRef.RefCount = pyRef.RefCount - 1
+        $$$ThrowOnError(pyRef.%Save())
+    }
+    quit $$$OK
+}
+
+ClassMethod UninstallPythonLibrary(
+	pLibraryName As %List,
+	ByRef pParams)
+{
+    set verbose = $get(pParams("Verbose"))
+    set target = ##class(%File).NormalizeDirectory("python", $system.Util.ManagerDirectory())
+    if '$listvalid(pLibraryName) {
+        set pLibraryName = $listfromstring(pLibraryName)
+    }
+    if '$data(^||%IPM.PipCaller, pipCaller) {
+        set pipCaller = ##class(%IPM.Utils.Python).ResolvePipCaller(.pParams)
+    }
+    set command = pipCaller _ $listbuild("uninstall", "-y")_pLibraryName
+    if $$$isUNIX {
+        set command = command _ $listbuild("--break-system-packages")
+    }
+    if verbose {
+        write !, "Running "
+        zwrite command
+    } else {
+        set stdout = ""
+    }
+    set tSC = ##class(%IPM.Utils.Module).RunCommand(target, command,.stdout)
+    $$$ThrowOnError(tSC)
+}
+
+Storage Default
+{
+<Data name="PythonReferenceDefaultData">
+<Value name="1">
+<Value>%%CLASSNAME</Value>
+</Value>
+<Value name="2">
+<Value>Version</Value>
+</Value>
+<Value name="3">
+<Value>SourceType</Value>
+</Value>
+<Value name="4">
+<Value>WheelPath</Value>
+</Value>
+<Value name="5">
+<Value>PlatformTag</Value>
+</Value>
+<Value name="6">
+<Value>RefCount</Value>
+</Value>
+</Data>
+<DataLocation>^IPM.Storage.PythonRefsD</DataLocation>
+<DefaultData>PythonReferenceDefaultData</DefaultData>
+<IdLocation>^IPM.Storage.PythonRefsD</IdLocation>
+<IndexLocation>^IPM.Storage.PythonRefsI</IndexLocation>
+<StreamLocation>^IPM.Storage.PythonRefsS</StreamLocation>
+<Type>%Storage.Persistent</Type>
+}
+
+}