Skip to content

Commit 7e9b063

Browse files
jgusjgus
committed
Support more disks
1 parent 71a3fc9 commit 7e9b063

File tree

6 files changed

+221
-24
lines changed

6 files changed

+221
-24
lines changed

‎example/zfs-multi-raidz3.nix‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
disko.devices = {
3+
disk = {
4+
boot = {
5+
type = "disk";
6+
device = "/dev/vda";
7+
content = {
8+
type = "gpt";
9+
partitions = {
10+
ESP = {
11+
size = "64M";
12+
type = "EF00";
13+
content = {
14+
type = "filesystem";
15+
format = "vfat";
16+
mountpoint = "/boot";
17+
mountOptions = [ "umask=0077" ];
18+
};
19+
};
20+
};
21+
};
22+
};
23+
}
24+
// (builtins.listToAttrs (
25+
builtins.genList (i: {
26+
name = "disk${toString i}";
27+
value = {
28+
type = "disk";
29+
device = "/dev/sd${
30+
if i < 26 then
31+
builtins.substring i 1 "abcdefghijklmnopqrstuvwxyz"
32+
else
33+
"a${builtins.substring (i - 26) 1 "abcdefghijklmnopqrstuvwxyz"}"
34+
}";
35+
content = {
36+
type = "gpt";
37+
partitions = {
38+
zfs = {
39+
size = "100%";
40+
content = {
41+
type = "zfs";
42+
pool = "zroot";
43+
};
44+
};
45+
};
46+
};
47+
};
48+
}) 36
49+
));
50+
zpool = {
51+
zroot = {
52+
type = "zpool";
53+
mode = {
54+
topology = {
55+
type = "topology";
56+
vdev = [
57+
# First raidz3 group: disks 0-10 (11 disks)
58+
{
59+
mode = "raidz3";
60+
members = builtins.genList (i: "disk${toString i}") 11;
61+
}
62+
# Second raidz3 group: disks 11-21 (11 disks)
63+
{
64+
mode = "raidz3";
65+
members = builtins.genList (i: "disk${toString (i + 11)}") 11;
66+
}
67+
# Third raidz3 group: disks 22-32 (11 disks)
68+
{
69+
mode = "raidz3";
70+
members = builtins.genList (i: "disk${toString (i + 22)}") 11;
71+
}
72+
];
73+
# 3 hot spares: disks 33-35
74+
spare = builtins.genList (i: "disk${toString (i + 33)}") 3;
75+
};
76+
};
77+
rootFsOptions = {
78+
compression = "zstd";
79+
"com.sun:auto-snapshot" = "false";
80+
};
81+
mountpoint = "/";
82+
datasets = {
83+
zfs_fs = {
84+
type = "zfs_fs";
85+
mountpoint = "/zfs_fs";
86+
options."com.sun:auto-snapshot" = "true";
87+
};
88+
};
89+
};
90+
};
91+
};
92+
}

‎lib/interactive-vm.nix‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,11 @@ let
3434
deviceExtraOpts.bootindex = "1";
3535
deviceExtraOpts.serial = "root";
3636
};
37-
otherDisks = map (disk: {
37+
otherDisks = lib.imap0 (i: disk: {
3838
name = disk.name;
3939
file = ''"$tmp"/${lib.escapeShellArg disk.imageName}.qcow2'';
4040
driveExtraOpts.werror = "report";
41+
deviceExtraOpts.bus = "bridge${toString (i / 31 + 1)}";
4142
}) (builtins.tail disks);
4243

4344
diskoBasedConfiguration = {
@@ -68,6 +69,19 @@ in
6869
virtualisation.useDefaultFilesystems = false;
6970
virtualisation.diskImage = null;
7071
virtualisation.qemu.drives = [ rootDisk ] ++ otherDisks;
72+
73+
# Using `networkingOptions` instead of `options` here because it's added _before_ the drive options, where `options` is added at the end :-P
74+
virtualisation.qemu.networkingOptions = lib.mkAfter (
75+
let
76+
# A PCI bridge takes one slot and adds 32, so we'll want one for every 31 drives
77+
count = ((builtins.length otherDisks) + 30) / 31;
78+
parentBridge = i: lib.optionalString (i > 0) "bus=bridge${toString i},";
79+
in
80+
(builtins.genList (
81+
i: "-device pci-bridge,id=bridge${toString (i + 1)},${parentBridge i}chassis_nr=${toString (i + 1)}"
82+
) count)
83+
);
84+
7185
boot.zfs.devNodes = "/dev/disk/by-uuid"; # needed because /dev/disk/by-id is empty in qemu-vms
7286
boot.zfs.forceImportAll = true;
7387
boot.zfs.forceImportRoot = lib.mkForce true;

‎lib/make-disk-image.nix‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -172,14 +172,30 @@ let
172172
'';
173173

174174
QEMU_OPTS = lib.concatStringsSep " " (
175+
let
176+
disks = lib.attrValues diskoCfg.devices.disk;
177+
diskCount = builtins.length disks;
178+
bridgeCount = ((diskCount + 30) / 31);
179+
parentBridge = i: lib.optionalString (i > 0) "bus=bridge${toString i},";
180+
bridgeOpt =
181+
i:
182+
"-device pci-bridge,id=bridge${toString (i + 1)},${parentBridge i}chassis_nr=${toString (i + 1)}";
183+
bridgeOpts = builtins.genList bridgeOpt bridgeCount;
184+
diskDeviceOpt =
185+
i: "-device virtio-blk-pci,bus=bridge${toString (i / 31 + 1)},drive=drive${toString (i + 1)}";
186+
diskDeviceOpts = builtins.genList diskDeviceOpt diskCount;
187+
diskDriveOpt =
188+
i: disk:
189+
"-drive file=\"$out\"/${disk.imageName}.${imageFormat},id=drive${toString (i + 1)},if=none,cache=unsafe,werror=report,format=${imageFormat}";
190+
diskDriveOpts = lib.imap0 diskDriveOpt disks;
191+
in
175192
[
176193
"-drive if=pflash,format=raw,unit=0,readonly=on,file=${pkgs.OVMF.firmware}"
177194
"-drive if=pflash,format=raw,unit=1,file=efivars.fd"
178195
]
179-
++ builtins.map (
180-
disk:
181-
"-drive file=\"$out\"/${disk.imageName}.${imageFormat},if=virtio,cache=unsafe,werror=report,format=${imageFormat}"
182-
) (lib.attrValues diskoCfg.devices.disk)
196+
++ bridgeOpts
197+
++ diskDeviceOpts
198+
++ diskDriveOpts
183199
);
184200
in
185201
{

‎lib/tests.nix‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,11 @@ let
5050
};
5151

5252
# list of devices generated inside qemu
53-
devices = [
54-
"/dev/vda"
55-
"/dev/vdb"
56-
"/dev/vdc"
57-
"/dev/vdd"
58-
"/dev/vde"
59-
"/dev/vdf"
60-
"/dev/vdg"
61-
"/dev/vdh"
62-
"/dev/vdi"
63-
"/dev/vdj"
64-
"/dev/vdk"
65-
"/dev/vdl"
66-
"/dev/vdm"
67-
"/dev/vdn"
68-
"/dev/vdo"
69-
];
53+
devices =
54+
let
55+
chars = lib.stringToCharacters "abcdefghijklmnopqrstuvwxyz";
56+
in
57+
(map (c: "/dev/vd${c}") chars) ++ (lib.concatMap (c1: map (c2: "/dev/vd${c1}${c2}") chars) chars);
7058

7159
# This is the test generator for a disko test
7260
makeDiskoTest =
@@ -288,7 +276,22 @@ let
288276
(testConfigInstall ? networking.hostId) && (testConfigInstall.networking.hostId != null)
289277
) testConfigInstall.networking.hostId;
290278

291-
virtualisation.emptyDiskImages = builtins.genList (_: 4096) num-disks;
279+
virtualisation.emptyDiskImages = builtins.genList (i: {
280+
size = 4096;
281+
driveConfig.deviceExtraOpts.bus = "bridge${toString (i / 31 + 1)}";
282+
}) num-disks;
283+
284+
# Using `networkingOptions` instead of `options` here because it's added _before_ the drive options, where `options` is added at the end :-P
285+
virtualisation.qemu.networkingOptions = lib.mkAfter (
286+
let
287+
# A PCI bridge takes one slot and adds 32, so we'll want one for every 31 drives
288+
count = (num-disks + 30) / 31;
289+
parentBridge = i: lib.optionalString (i > 0) "bus=bridge${toString i},";
290+
in
291+
(builtins.genList (
292+
i: "-device pci-bridge,id=bridge${toString (i + 1)},${parentBridge i}chassis_nr=${toString (i + 1)}"
293+
) count)
294+
);
292295

293296
# useful for debugging via repl
294297
system.build.systemToInstall = installed-system-eval;
@@ -301,12 +304,17 @@ let
301304
302305
def disks(oldmachine, num_disks):
303306
disk_flags = []
307+
for i in range((num_disks + 30) // 31):
308+
disk_flags += [
309+
'-device',
310+
f"pci-bridge,id=bridge{i + 1},{f'bus=bridge{i},' if i > 0 else '''}chassis_nr={i + 1}"
311+
]
304312
for i in range(num_disks):
305313
disk_flags += [
306314
'-drive',
307315
f"file={oldmachine.state_dir}/empty{i}.qcow2,id=drive{i + 1},if=none,index={i + 1},werror=report",
308316
'-device',
309-
f"virtio-blk-pci,drive=drive{i + 1}"
317+
f"virtio-blk-pci,bus=bridge{i // 31 + 1},drive=drive{i + 1}"
310318
]
311319
return disk_flags
312320

‎tests/default.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ let
2222
);
2323
incompatibleTests = lib.optionals pkgs.stdenv.buildPlatform.isRiscV64 [
2424
"zfs"
25+
"zfs-multi-raidz3"
2526
"zfs-over-legacy"
2627
"cli"
2728
"module"

‎tests/zfs-multi-raidz3.nix‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
pkgs ? import <nixpkgs> { },
3+
diskoLib ? pkgs.callPackage ../lib { },
4+
}:
5+
diskoLib.testLib.makeDiskoTest {
6+
inherit pkgs;
7+
name = "zfs-multi-raidz3";
8+
disko-config = ../example/zfs-multi-raidz3.nix;
9+
extraInstallerConfig.networking.hostId = "8425e349";
10+
extraSystemConfig = {
11+
networking.hostId = "8425e349";
12+
# It looks like the 60s of NixOS is sometimes not enough for our virtio-based zpool.
13+
# This fixes the flakeiness of the test.
14+
boot.initrd.postResumeCommands = ''
15+
for i in $(seq 1 120); do
16+
if zpool list | grep -q zroot || zpool import -N zroot; then
17+
break
18+
fi
19+
done
20+
'';
21+
};
22+
extraTestScript = ''
23+
def assert_property(ds, property, expected_value):
24+
out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
25+
assert (
26+
out == expected_value
27+
), f"Expected {property}={expected_value} on {ds}, got: {out}"
28+
29+
assert_property("zroot", "compression", "zstd")
30+
assert_property("zroot/zfs_fs", "com.sun:auto-snapshot", "true")
31+
assert_property("zroot/zfs_fs", "compression", "zstd")
32+
machine.succeed("mountpoint /zfs_fs");
33+
34+
# Verify the pool has 36 devices total (33 in raidz3 vdevs + 3 spares)
35+
status_output = machine.succeed("zpool status -P zroot")
36+
37+
# Count the number of disk devices in the pool
38+
device_count = 0
39+
for line in status_output.split("\n"):
40+
if "/dev/disk/by-partlabel/disk-" in line:
41+
device_count += 1
42+
43+
assert device_count == 36, f"Expected 36 devices in pool, found {device_count}"
44+
45+
# Verify we have 3 raidz3 vdevs
46+
raidz3_count = status_output.count("raidz3")
47+
assert raidz3_count == 3, f"Expected 3 raidz3 vdevs, found {raidz3_count}"
48+
49+
# Verify we have 3 spares
50+
spares_count = 0
51+
in_spares_section = False
52+
for line in status_output.split("\n"):
53+
if line.strip().startswith("spares"):
54+
in_spares_section = True
55+
elif in_spares_section and "/dev/disk/by-partlabel/disk-" in line:
56+
spares_count += 1
57+
elif in_spares_section and line.strip() and not line.startswith("\t"):
58+
in_spares_section = False
59+
60+
assert spares_count == 3, f"Expected 3 spare devices, found {spares_count}"
61+
62+
# Verify pool health
63+
pool_state = machine.succeed("zpool list -H -o health zroot").strip()
64+
assert pool_state == "ONLINE", f"Expected pool health ONLINE, got {pool_state}"
65+
'';
66+
}

0 commit comments

Comments
 (0)