Skip to content

Commit a1adc68

Browse files
andrew-anyscaleandrew-anyscale
committed
[CI] Convert release/build.rayci.yml to array + update get_prerequisite_step
Convert all release build steps from Buildkite matrix to rayci array syntax. Build steps use ($) to depend only on their matching python wheel and testdeps. Publish steps use ($) which matches on shared dimensions (python, and platform where aligned by the wanda-platform-env pre-req). Update get_prerequisite_step() to construct array-suffixed Buildkite step keys (e.g. anyscalebuild--platformcpu-python310) by parsing the base_image tag and mapping short platform tags to full strings via a constant lookup table. Topic: array-release Relative: wanda-platform-env Labels: draft Signed-off-by: andrew <[email protected]>
1 parent b1d1bf4 commit a1adc68

File tree

7 files changed

+341
-151
lines changed

7 files changed

+341
-151
lines changed

‎.buildkite/release/build.rayci.yml‎

Lines changed: 133 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,241 +1,255 @@
11
group: release build
22
steps:
33
- name: raycpubaseextra-testdeps
4-
label: "wanda: ray.py{{matrix}}.cpu.base-extra-testdeps"
4+
label: "wanda: ray.py{{array.python}}.cpu.base-extra-testdeps"
55
wanda: docker/base-extra-testdeps/cpu.wanda.yaml
6-
matrix:
7-
- "3.10"
8-
- "3.11"
9-
- "3.12"
10-
- "3.13"
6+
array:
7+
python:
8+
- "3.10"
9+
- "3.11"
10+
- "3.12"
11+
- "3.13"
1112
env:
12-
PYTHON_VERSION: "{{matrix}}"
13+
PYTHON_VERSION: "{{array.python}}"
1314
IMAGE_TYPE: "ray"
1415
depends_on:
1516
- raycpubaseextra(*)
1617

1718
- name: raycudabaseextra-testdeps
18-
label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra-testdeps"
19+
label: "wanda: ray.py{{array.python}}.cu{{array.cuda}}.base-extra-testdeps"
1920
wanda: docker/base-extra-testdeps/cuda.wanda.yaml
20-
matrix:
21-
setup:
22-
python:
23-
- "3.10"
24-
- "3.11"
25-
- "3.12"
26-
- "3.13"
27-
cuda:
28-
- "12.3.2-cudnn9"
21+
array:
22+
python:
23+
- "3.10"
24+
- "3.11"
25+
- "3.12"
26+
- "3.13"
27+
cuda:
28+
- "12.3.2-cudnn9"
2929
adjustments:
3030
- with:
3131
python: "3.12"
3232
cuda: "13.0.0-cudnn"
3333
env:
34-
PYTHON_VERSION: "{{matrix.python}}"
35-
CUDA_VERSION: "{{matrix.cuda}}"
34+
PYTHON_VERSION: "{{array.python}}"
35+
CUDA_VERSION: "{{array.cuda}}"
3636
IMAGE_TYPE: "ray"
3737
depends_on:
3838
- raycudabaseextra(*)
3939

4040
- name: ray-llmbaseextra-testdeps
41-
label: "wanda: ray.py{{matrix.python}}.llm.base-extra-testdeps (cuda {{matrix.cuda}})"
41+
label: "wanda: ray.py{{array.python}}.llm.base-extra-testdeps (cuda {{array.cuda}})"
4242
wanda: docker/base-extra-testdeps/cuda.wanda.yaml
43-
matrix:
44-
setup:
45-
python:
46-
- "3.12"
47-
cuda:
48-
- "13.0.0-cudnn"
43+
array:
44+
python:
45+
- "3.12"
46+
cuda:
47+
- "13.0.0-cudnn"
4948
adjustments:
5049
- with:
5150
python: "3.11"
5251
cuda: "12.8.1-cudnn"
5352
env:
54-
PYTHON_VERSION: "{{matrix.python}}"
55-
CUDA_VERSION: "{{matrix.cuda}}"
53+
PYTHON_VERSION: "{{array.python}}"
54+
CUDA_VERSION: "{{array.cuda}}"
5655
IMAGE_TYPE: "ray-llm"
5756
depends_on:
5857
- ray-llmbaseextra(*)
5958

6059
- name: ray-mlcudabaseextra-testdeps
61-
label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.ml.base-extra-testdeps"
60+
label: "wanda: ray.py{{array.python}}.cu{{array.cuda}}.ml.base-extra-testdeps"
6261
wanda: docker/base-extra-testdeps/cuda.wanda.yaml
63-
matrix:
64-
setup:
65-
python:
66-
- "3.10"
67-
cuda:
68-
- "12.1.1-cudnn8"
62+
array:
63+
python:
64+
- "3.10"
65+
cuda:
66+
- "12.1.1-cudnn8"
6967
env:
70-
PYTHON_VERSION: "{{matrix.python}}"
71-
CUDA_VERSION: "{{matrix.cuda}}"
68+
PYTHON_VERSION: "{{array.python}}"
69+
CUDA_VERSION: "{{array.cuda}}"
7270
IMAGE_TYPE: "ray-ml"
7371
depends_on:
7472
- ray-mlcudabaseextra(*)
7573

7674
- name: ray-anyscale-cpu-build
77-
label: "wanda: ray-anyscale py{{matrix}} cpu"
75+
label: "wanda: ray-anyscale py{{array.python}} cpu"
7876
wanda: ci/docker/ray-anyscale-cpu.wanda.yaml
7977
env_file: rayci.env
80-
matrix:
81-
# This list should be kept in sync with the list of supported Python in
82-
# release test suite
83-
- "3.10"
84-
- "3.11"
85-
- "3.12"
86-
- "3.13"
78+
array:
79+
python:
80+
# This list should be kept in sync with the list of supported Python in
81+
# release test suite
82+
- "3.10"
83+
- "3.11"
84+
- "3.12"
85+
- "3.13"
8786
env:
88-
PYTHON_VERSION: "{{matrix}}"
87+
PYTHON_VERSION: "{{array.python}}"
8988
ARCH_SUFFIX: ""
9089
tags:
9190
- oss
9291
depends_on:
93-
- ray-wheel-build(*)
94-
- raycpubaseextra-testdeps
92+
- ray-wheel-build($)
93+
- raycpubaseextra-testdeps($)
9594

9695
- name: ray-anyscale-cuda-build
97-
label: "wanda: ray-anyscale py{{matrix.python}} {{matrix.platform}}"
96+
label: "wanda: ray-anyscale py{{array.python}} {{array.platform}}"
9897
wanda: ci/docker/ray-anyscale-cuda.wanda.yaml
9998
env_file: rayci.env
100-
matrix:
101-
setup:
102-
python:
103-
# This list should be kept in sync with the list of supported Python in
104-
# release test suite
105-
- "3.10"
106-
- "3.11"
107-
- "3.12"
108-
- "3.13"
109-
platform:
110-
- "cu12.3.2-cudnn9"
99+
array:
100+
platform:
101+
- "cu12.3.2-cudnn9"
102+
python:
103+
# This list should be kept in sync with the list of supported Python in
104+
# release test suite
105+
- "3.10"
106+
- "3.11"
107+
- "3.12"
108+
- "3.13"
111109
adjustments:
112110
- with:
113111
python: "3.12"
114112
platform: "cu13.0.0-cudnn"
115113
env:
116-
PYTHON_VERSION: "{{matrix.python}}"
117-
PLATFORM: "{{matrix.platform}}"
114+
PYTHON_VERSION: "{{array.python}}"
115+
PLATFORM: "{{array.platform}}"
118116
ARCH_SUFFIX: ""
119117
tags:
120118
- oss
121119
depends_on:
122-
- ray-wheel-build(*)
123-
- raycudabaseextra-testdeps
120+
- ray-wheel-build($)
121+
- raycudabaseextra-testdeps($)
124122

125-
- label: ":crane: publish: ray-anyscale py{{matrix.python}} {{matrix.platform}}"
126-
key: anyscalebuild
123+
- label: ":crane: publish: ray-anyscale py{{array.python}} cpu"
124+
key: anyscalecpubuild
127125
instance_type: release-medium
128126
mount_buildkite_agent: true
129127
tags:
130128
- oss
131129
commands:
132130
- bazel run //ci/ray_ci/automation:push_release_test_image --
133-
--python-version {{matrix.python}}
134-
--platform {{matrix.platform}}
131+
--python-version {{array.python}}
132+
--platform cpu
135133
--image-type ray
136134
--upload
137135
depends_on:
138-
- ray-anyscale-cpu-build
139-
- ray-anyscale-cuda-build
140-
matrix:
141-
setup:
142-
python:
143-
- "3.10"
144-
- "3.11"
145-
- "3.12"
146-
- "3.13"
147-
platform:
148-
- cu12.3.2-cudnn9
149-
- cpu
136+
- ray-anyscale-cpu-build($)
137+
array:
138+
python:
139+
- "3.10"
140+
- "3.11"
141+
- "3.12"
142+
- "3.13"
143+
144+
- label: ":crane: publish: ray-anyscale py{{array.python}} {{array.platform}}"
145+
key: anyscalecudabuild
146+
instance_type: release-medium
147+
mount_buildkite_agent: true
148+
tags:
149+
- oss
150+
commands:
151+
- bazel run //ci/ray_ci/automation:push_release_test_image --
152+
--python-version {{array.python}}
153+
--platform {{array.platform}}
154+
--image-type ray
155+
--upload
156+
depends_on:
157+
- ray-anyscale-cuda-build($)
158+
array:
159+
platform:
160+
- cu12.3.2-cudnn9
161+
python:
162+
- "3.10"
163+
- "3.11"
164+
- "3.12"
165+
- "3.13"
150166
adjustments:
151167
- with:
152168
python: "3.12"
153169
platform: cu13.0.0-cudnn
154170

155171
- name: ray-llm-anyscale-cuda-build
156-
label: "wanda: ray-llm-anyscale py{{matrix.python}} {{matrix.platform}}"
172+
label: "wanda: ray-llm-anyscale py{{array.python}} {{array.platform}}"
157173
wanda: ci/docker/ray-llm-anyscale-cuda.wanda.yaml
158174
env_file: rayci.env
159-
matrix:
160-
setup:
161-
python:
162-
- "3.12"
163-
platform:
164-
- "cu13.0.0-cudnn"
175+
array:
176+
platform:
177+
- "cu13.0.0-cudnn"
178+
python:
179+
- "3.12"
165180
adjustments:
166181
- with:
167182
python: "3.11"
168183
platform: "cu12.8.1-cudnn"
169184
env:
170-
PYTHON_VERSION: "{{matrix.python}}"
171-
PLATFORM: "{{matrix.platform}}"
185+
PYTHON_VERSION: "{{array.python}}"
186+
PLATFORM: "{{array.platform}}"
172187
ARCH_SUFFIX: ""
173188
tags:
174189
- oss
175190
depends_on:
176-
- ray-wheel-build(*)
177-
- ray-llmbaseextra-testdeps
191+
- ray-wheel-build($)
192+
- ray-llmbaseextra-testdeps($)
178193

179-
- label: ":crane: publish: ray-llm-anyscale py{{matrix.python}} {{matrix.platform}}"
194+
- label: ":crane: publish: ray-llm-anyscale py{{array.python}} {{array.platform}}"
180195
key: anyscalellmbuild
181196
instance_type: release-medium
182197
mount_buildkite_agent: true
183198
tags:
184199
- oss
185200
commands:
186201
- bazel run //ci/ray_ci/automation:push_release_test_image --
187-
--python-version {{matrix.python}}
188-
--platform {{matrix.platform}}
202+
--python-version {{array.python}}
203+
--platform {{array.platform}}
189204
--image-type ray-llm
190205
--upload
191206
depends_on:
192-
- ray-llm-anyscale-cuda-build
193-
matrix:
194-
setup:
195-
python:
196-
- "3.12"
197-
platform:
198-
- cu13.0.0-cudnn
207+
- ray-llm-anyscale-cuda-build($)
208+
array:
209+
platform:
210+
- cu13.0.0-cudnn
211+
python:
212+
- "3.12"
199213
adjustments:
200214
- with:
201215
python: "3.11"
202216
platform: "cu12.8.1-cudnn"
203217

204218
- name: ray-ml-anyscale-cuda-build
205-
label: "wanda: ray-ml-anyscale py{{matrix.python}} {{matrix.platform}}"
219+
label: "wanda: ray-ml-anyscale py{{array.python}} {{array.platform}}"
206220
wanda: ci/docker/ray-ml-anyscale-cuda.wanda.yaml
207221
env_file: rayci.env
208-
matrix:
209-
setup:
210-
python:
211-
# This list should be kept in sync with the list of supported Python in
212-
# release test suite
213-
- "3.10"
214-
platform:
215-
- "cu12.1.1-cudnn8"
222+
array:
223+
platform:
224+
- "cu12.1.1-cudnn8"
225+
python:
226+
# This list should be kept in sync with the list of supported Python in
227+
# release test suite
228+
- "3.10"
216229
env:
217-
PYTHON_VERSION: "{{matrix.python}}"
218-
PLATFORM: "{{matrix.platform}}"
230+
PYTHON_VERSION: "{{array.python}}"
231+
PLATFORM: "{{array.platform}}"
219232
ARCH_SUFFIX: ""
220233
tags:
221234
- oss
222235
depends_on:
223-
- ray-wheel-build(*)
224-
- ray-mlcudabaseextra-testdeps
236+
- ray-wheel-build($)
237+
- ray-mlcudabaseextra-testdeps($)
225238

226-
- label: ":crane: publish: ray-ml-anyscale py{{matrix}} cu12.1.1-cudnn8"
239+
- label: ":crane: publish: ray-ml-anyscale py{{array.python}} cu12.1.1-cudnn8"
227240
key: anyscalemlbuild
228241
instance_type: release-medium
229242
mount_buildkite_agent: true
230243
tags:
231244
- oss
232245
commands:
233246
- bazel run //ci/ray_ci/automation:push_release_test_image --
234-
--python-version {{matrix}}
247+
--python-version {{array.python}}
235248
--platform cu12.1.1-cudnn8
236249
--image-type ray-ml
237250
--upload
238251
depends_on:
239-
- ray-ml-anyscale-cuda-build
240-
matrix:
241-
- "3.10"
252+
- ray-ml-anyscale-cuda-build($)
253+
array:
254+
python:
255+
- "3.10"

‎ci/ray_ci/oss_config.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ state_machine:
2020
branch:
2121
aws_bucket: ray-ci-results
2222
release_image_step:
23-
ray: anyscalebuild
23+
ray_cpu: anyscalecpubuild
24+
ray_cuda: anyscalecudabuild
2425
ray_ml: anyscalemlbuild
2526
ray_llm: anyscalellmbuild

‎release/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -677,6 +677,7 @@ py_test(
677677
srcs = ["ray_release/tests/test_custom_byod_build_init_helper.py"],
678678
data = [
679679
"ray_release/configs/oss_config.yaml",
680+
"//:ray-images.json",
680681
],
681682
exec_compatible_with = ["//bazel:py3"],
682683
tags = [

0 commit comments

Comments
 (0)