11#!/usr/bin/env python3
22from __future__ import annotations
33
4+ import argparse
45import asyncio
56import json
67import pathlib
1314from ci .logging_config import configure_logging
1415
1516PROJECT_ROOT = pathlib .Path (__file__ ).parent .parent
17+ ODH_MANIFEST_DIR = PROJECT_ROOT / "manifests" / "odh" / "base"
18+ RHOAI_MANIFEST_DIR = PROJECT_ROOT / "manifests" / "rhoai" / "base"
1619
1720log = structlog .get_logger ()
1821
22+ DUMMY_IMAGE = "dummy"
23+
24+
25+ def parse_args () -> argparse .Namespace :
26+ parser = argparse .ArgumentParser (
27+ description = "Fill commit-latest.env with short vcs-ref values from image labels for params-latest.env."
28+ )
29+ parser .add_argument (
30+ "--odh-dir" ,
31+ type = pathlib .Path ,
32+ default = ODH_MANIFEST_DIR ,
33+ help = (
34+ "ODH directory containing params-latest.env (used as URL source of truth when "
35+ f"RHOAI params use '{ DUMMY_IMAGE }
36+ ),
37+ )
38+ parser .add_argument (
39+ "--rhoai-dir" ,
40+ type = pathlib .Path ,
41+ default = RHOAI_MANIFEST_DIR ,
42+ help = "RHOAI manifest directory (default: manifests/rhoai/base under repo root)." ,
43+ )
44+ parser .add_argument (
45+ "--only" ,
46+ choices = ("both" , "odh" , "rhoai" ),
47+ default = "both" ,
48+ help = "Which commit-latest.env file(s) to write (default: both)." ,
49+ )
50+ return parser .parse_args ()
51+
52+
53+ def resolve_manifest_dir (raw : pathlib .Path ) -> pathlib .Path :
54+ return raw if raw .is_absolute () else (PROJECT_ROOT / raw ).resolve ()
55+
56+
57+ def load_params_env (path : pathlib .Path ) -> list [tuple [str , str ]]:
58+ with open (path , "rt" ) as file :
59+ return [
60+ (parts [0 ].strip (), parts [1 ].strip ())
61+ for line in file
62+ if line .strip () and not line .strip ().startswith ("#" )
63+ for parts in [line .strip ().split ("=" , 1 )]
64+ if len (parts ) == 2
65+ ]
66+
67+
68+ def resolve_rhoai_urls (
69+ rhoai_pairs : list [tuple [str , str ]],
70+ odh_by_var : dict [str , str ],
71+ ) -> list [tuple [str , str ]]:
72+ """Replace RHOAI dummy placeholders with the same variable's image URL from ODH params."""
73+ out : list [tuple [str , str ]] = []
74+ for var , raw in rhoai_pairs :
75+ if raw .strip ().lower () == DUMMY_IMAGE :
76+ if var not in odh_by_var :
77+ log .error ("No ODH params-latest entry to resolve RHOAI dummy image" , variable = var )
78+ sys .exit (1 )
79+ url = odh_by_var [var ]
80+ if url .strip ().lower () == DUMMY_IMAGE :
81+ log .error ("ODH params-latest still has dummy for variable" , variable = var )
82+ sys .exit (1 )
83+ out .append ((var , url ))
84+ else :
85+ out .append ((var , raw ))
86+ return out
87+
88+
89+ def write_commit_latest (
90+ var_url_pairs : list [tuple [str , str ]],
91+ vcs_by_url : dict [str , str | None ],
92+ commit_latest_path : pathlib .Path ,
93+ ) -> None :
94+ lines : list [tuple [str , str ]] = []
95+ for var , url in var_url_pairs :
96+ vcs = vcs_by_url [url ]
97+ if vcs is None :
98+ log .error ("Missing vcs-ref for image URL" , url = url , variable = var )
99+ sys .exit (1 )
100+ lines .append ((re .sub (r"-n$" , "-commit-n" , var ), vcs [:7 ]))
101+
102+ with open (commit_latest_path , "wt" ) as file :
103+ for key , short in sorted (lines ):
104+ print (key , short , file = file , sep = "=" )
105+
19106
20107async def get_image_vcs_ref (image_url : str , semaphore : asyncio .Semaphore ) -> tuple [str , str | None ]:
21108 """
@@ -30,10 +117,8 @@ async def get_image_vcs_ref(image_url: str, semaphore: asyncio.Semaphore) -> tup
30117 A tuple containing the original image_url and the value of the 'vcs-ref'
31118 label if found, otherwise None.
32119 """
33- # Using 'docker://' prefix is required for skopeo to identify the transport.
34120 full_image_url = f"docker://{ image_url }
35121
36- # Use 'inspect --config' which is much faster as it only fetches the config blob.
37122 command = ["skopeo" , "inspect" , "--override-os=linux" , "--override-arch=amd64" , "--retry-times=5" , "--config" , full_image_url ]
38123
39124 log .info (f"Starting config inspection for: { image_url } )
@@ -42,17 +127,14 @@ async def get_image_vcs_ref(image_url: str, semaphore: asyncio.Semaphore) -> tup
42127 try :
43128 async with semaphore :
44129 log .info (f"Semaphore acquired, starting skopeo inspect for: { image_url } )
45- # Create an asynchronous subprocess
46130 process = await asyncio .create_subprocess_exec (
47131 * command ,
48132 stdout = asyncio .subprocess .PIPE ,
49- stderr = asyncio .subprocess .PIPE
133+ stderr = asyncio .subprocess .PIPE ,
50134 )
51- # Wait for the command to complete and capture output
52135 stdout , stderr = await process .communicate ()
53136 returncode = process .returncode
54137
55- # Process the results outside the semaphore block
56138 if returncode != 0 :
57139 log .error (f"Skopeo command failed for { image_url } { returncode } )
58140 if stderr :
@@ -63,11 +145,8 @@ async def get_image_vcs_ref(image_url: str, semaphore: asyncio.Semaphore) -> tup
63145 log .error (f"Skopeo command returned success but stdout was empty for { image_url } )
64146 return image_url , None
65147
66- # Decode and parse the JSON output from stdout
67- # The output of 'inspect --config' is the image config JSON directly.
68148 image_config = json .loads (stdout .decode ())
69149
70- # Safely extract the 'vcs-ref' label from the config's 'Labels'
71150 vcs_ref = image_config .get ("config" , {}).get ("Labels" , {}).get ("vcs-ref" )
72151
73152 if vcs_ref :
@@ -81,46 +160,61 @@ async def get_image_vcs_ref(image_url: str, semaphore: asyncio.Semaphore) -> tup
81160 log .error ("The 'skopeo' command was not found. Please ensure it is installed and in your PATH." )
82161 return image_url , None
83162 except json .JSONDecodeError :
84- # This error can now also happen if stdout is None or not valid JSON
85163 log .error (f"Failed to parse skopeo output as JSON for { image_url } )
86164 if stdout :
87165 log .debug (f"Stdout from skopeo for { image_url } { stdout .decode (errors = 'replace' )} )
88166 return image_url , None
89- except Exception as e :
90- log .error ("Unexpected error while processing image" , image_url = image_url , exc_info = True )
167+ except Exception :
168+ log .exception ("Unexpected error while processing image" , image_url = image_url )
91169 return image_url , None
92170
93171
94- async def inspect (images_to_inspect : typing .Iterable [str ]) -> list [tuple [str , str | None ]]:
95- """
96- Main function to orchestrate the concurrent inspection of multiple images.
97- """
98- semaphore = asyncio .Semaphore (22 ) # Limit concurrent skopeo processes
99- tasks = [get_image_vcs_ref (image , semaphore ) for image in images_to_inspect ]
100- return await asyncio .gather (* tasks )
172+ async def inspect_urls (urls : typing .Iterable [str ]) -> dict [str , str | None ]:
173+ unique = list (dict .fromkeys (urls ))
174+ semaphore = asyncio .Semaphore (22 )
175+ tasks = [get_image_vcs_ref (url , semaphore ) for url in unique ]
176+ results = await asyncio .gather (* tasks )
177+ out = dict (results )
178+ if any (v is None for v in out .values ()):
179+ log .error ("Failed to get commit hash for one or more images; aborting." )
180+ sys .exit (1 )
181+ return out
101182
102183
103- async def main ():
104- with open ( PROJECT_ROOT / "manifests/base/params-latest.env" , "rt" ) as file :
105- images_to_inspect : list [ list [ str ]] = [ line . strip (). split ( '=' , 1 ) for line in file . readlines ( )
106- if line . strip () and not line . strip (). startswith ( "#" )]
184+ async def main () -> None :
185+ args = parse_args ()
186+ odh_dir = resolve_manifest_dir ( args . odh_dir )
187+ rhoai_dir = resolve_manifest_dir ( args . rhoai_dir )
107188
108- results = await inspect (value for _ , value in images_to_inspect )
109- if any (commit_hash is None for variable , commit_hash in results ):
110- log .error ("Failed to get commit hash for some images. Quitting, please try again to try again, like." )
111- sys .exit (1 )
189+ odh_params = odh_dir / "params-latest.env"
190+ rhoai_params = rhoai_dir / "params-latest.env"
191+
192+ odh_by_var = dict (load_params_env (odh_params ))
193+
194+ targets : list [tuple [pathlib .Path , list [tuple [str , str ]]]] = []
195+
196+ if args .only in ("both" , "odh" ):
197+ odh_pairs = load_params_env (odh_params )
198+ if any (v .strip ().lower () == DUMMY_IMAGE for _ , v in odh_pairs ):
199+ log .error ("ODH params-latest.env must not use dummy image placeholders." )
200+ raise SystemExit (1 )
201+ targets .append ((odh_dir / "commit-latest.env" , odh_pairs ))
202+
203+ if args .only in ("both" , "rhoai" ):
204+ rhoai_pairs = load_params_env (rhoai_params )
205+ rhoai_resolved = resolve_rhoai_urls (rhoai_pairs , odh_by_var )
206+ targets .append ((rhoai_dir / "commit-latest.env" , rhoai_resolved ))
207+
208+ all_urls : list [str ] = []
209+ for _ , pairs in targets :
210+ all_urls .extend (url for _ , url in pairs )
112211
113- output = []
114- for image , result in zip (images_to_inspect , results , strict = True ):
115- variable , image_digest = image
116- _ , commit_hash = result
117- output .append ((re .sub (r'-n$' , "-commit-n" , variable ), commit_hash [:7 ]))
212+ vcs_by_url = await inspect_urls (all_urls )
118213
119- with open (PROJECT_ROOT / "manifests/base/commit-latest.env" , "wt" ) as file :
120- for line in sorted (output ):
121- print (* line , file = file , sep = "=" , end = "\n " )
214+ for commit_path , pairs in targets :
215+ write_commit_latest (pairs , vcs_by_url , commit_path )
122216
123217
124- if __name__ == ' __main__'
218+ if __name__ == " __main__"
125219 configure_logging ()
126220 asyncio .run (main ())
0 commit comments