Skip to content

Commit 48df1b4

Browse files
dvaerumdvaerum
committed
Add cross-compilation support for disk formatting (ZFS)
When formatting disks for a different target architecture (e.g., creating an aarch64 disk image from an x86_64 host), tools like ZFS that communicate with kernel modules must use host-native binaries. I am guessing there also exist other cases when ZFS comes into play. This is not a problem with tools like `ext4` because they do everything in user space. I have created an example file and used it to build the file system for the archs `aarch64`, `armv7l`, `i686`, `riscv64` and `x86_64`. **Note:** The host system must have the following nixos configs enabled. ```nix # `preferStaticEmulators` is need or the activation script in chroot # cannot be executed because it is a different arch when the host system boot.binfmt.preferStaticEmulators = true; boot.binfmt.emulatedSystems = [ "<TARGET_ARCH>" ]; ```
1 parent 558e846 commit 48df1b4

File tree

10 files changed

+381
-22
lines changed

10 files changed

+381
-22
lines changed

‎default.nix‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ in
3333
cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).destroyNoDeps;
3434

3535
_cliFormat = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).format;
36+
# for cross-compilation: hostPkgs for format, targetPkgs for mount scripts
37+
_cliFormatCross =
38+
cfg: hostPkgs: targetPkgs:
39+
((eval cfg).config.disko.devices._scripts {
40+
pkgs = targetPkgs;
41+
inherit hostPkgs checked;
42+
}).format;
3643
_cliFormatNoDeps =
3744
cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatNoDeps;
3845

‎disko-install‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,45 @@ maybeRun () {
179179
fi
180180
}
181181

182+
checkBinfmtForCrossCompilation() {
183+
local flake=$1
184+
local flakeAttr=$2
185+
186+
# Get the current system (host) from Nix
187+
local host_system
188+
host_system=$(nix-instantiate --eval --expr 'builtins.currentSystem' 2>/dev/null | tr -d '"') || return 0
189+
190+
# Get the target system by evaluating the flake (without building)
191+
local target_system
192+
target_system=$(nix eval --raw "${flake}#nixosConfigurations.${flakeAttr}.pkgs.system" 2>/dev/null) || return 0
193+
194+
# If systems match, no cross-compilation, no binfmt needed
195+
if [[ "$host_system" == "$target_system" ]]; then
196+
return 0
197+
fi
198+
199+
# Cross-compilation detected, check binfmt configuration
200+
local binfmt_file="/proc/sys/fs/binfmt_misc/$target_system"
201+
if [[ ! -f "$binfmt_file" ]]; then
202+
echo "Cross-compiling from $host_system to $target_system requires binfmt_misc configuration." >&2
203+
echo "Add this to your NixOS configuration:" >&2
204+
echo " boot.binfmt.emulatedSystems = [ \"$target_system\" ];" >&2
205+
echo "Then rebuild: nixos-rebuild switch" >&2
206+
exit 1
207+
fi
208+
209+
# Check for F flag (required for chroot)
210+
local flags
211+
flags=$(grep "^flags:" "$binfmt_file" | cut -d: -f2 | tr -d ' ')
212+
if [[ ! "$flags" =~ F ]]; then
213+
echo "Cross-compilation requires the binfmt_misc 'F' flag (current: $flags)" >&2
214+
echo "Add this to your NixOS configuration:" >&2
215+
echo " boot.binfmt.preferStaticEmulators = true;" >&2
216+
echo "Then rebuild: nixos-rebuild switch" >&2
217+
exit 1
218+
fi
219+
}
220+
182221
main() {
183222
parseArgs "$@"
184223

@@ -232,6 +271,9 @@ main() {
232271
trap cleanupMountPoint EXIT
233272
fi
234273

274+
# Check binfmt configuration for cross-compilation before building
275+
checkBinfmtForCrossCompilation "$flake" "$flakeAttr"
276+
235277
if ! outputs=$(nixBuild "${libexec_dir}"/install-cli.nix \
236278
"${nix_args[@]}" \
237279
--no-out-link \

‎example/cross-zfs.nix‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Example ZFS configuration used to test cross-architecture disk formatting.
2+
# When formatting disks for a different architecture (e.g., preparing an
3+
# aarch64 disk from an x86_64 host), disko automatically uses host-native
4+
# tools for ZFS operations that require kernel communication.
5+
{
6+
disko.devices = {
7+
disk = {
8+
main = {
9+
type = "disk";
10+
device = "/dev/disk/by-id/some-disk-id";
11+
content = {
12+
type = "gpt";
13+
partitions = {
14+
ESP = {
15+
size = "512M";
16+
type = "EF00";
17+
content = {
18+
type = "filesystem";
19+
format = "vfat";
20+
mountpoint = "/boot";
21+
mountOptions = [ "umask=0077" ];
22+
};
23+
};
24+
zfs = {
25+
size = "100%";
26+
content = {
27+
type = "zfs";
28+
pool = "zroot";
29+
};
30+
};
31+
};
32+
};
33+
};
34+
};
35+
zpool = {
36+
zroot = {
37+
type = "zpool";
38+
options.cachefile = "none";
39+
rootFsOptions = {
40+
compression = "lz4";
41+
"com.sun:auto-snapshot" = "false";
42+
};
43+
mountpoint = "/";
44+
45+
datasets = {
46+
home = {
47+
type = "zfs_fs";
48+
mountpoint = "/home";
49+
};
50+
nix = {
51+
type = "zfs_fs";
52+
mountpoint = "/nix";
53+
options.atime = "off";
54+
};
55+
};
56+
};
57+
};
58+
};
59+
}

‎install-cli.nix‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,15 @@
55
extraSystemConfig ? "{}",
66
writeEfiBootEntries ? false,
77
rootMountPoint ? "/mnt",
8+
hostSystem ? builtins.currentSystem, # the system running the format script, for cross-compilation
89
}:
910
let
1011
originalSystem = (builtins.getFlake "${flake}").nixosConfigurations."${flakeAttr}";
1112
lib = originalSystem.pkgs.lib;
1213

14+
# Get host pkgs for cross-compilation (format scripts need host-native tools)
15+
hostPkgs = import originalSystem.pkgs.path { system = hostSystem; };
16+
1317
deviceName =
1418
name:
1519
if diskMappings ? ${name} then
@@ -57,11 +61,19 @@ let
5761
)
5862
];
5963
};
64+
# Build scripts with hostPkgs for format/destroy, targetPkgs for mount
65+
# This enables cross-compilation: format scripts use host-native tools
66+
# that can communicate with the running kernel
67+
scripts = diskoSystem.config.disko.devices._scripts {
68+
pkgs = diskoSystem.pkgs; # target pkgs for mount scripts
69+
inherit hostPkgs; # host pkgs for format/destroy scripts
70+
};
6071
in
6172
{
6273
installToplevel = installSystem.config.system.build.toplevel;
6374
closureInfo = installSystem.pkgs.closureInfo {
6475
rootPaths = [ installSystem.config.system.build.toplevel ];
6576
};
66-
inherit (diskoSystem.config.system.build) formatScript mountScript diskoScript;
77+
# Use scripts built with hostPkgs for cross-compilation support
78+
inherit (scripts) formatScript mountScript diskoScript;
6779
}

‎lib/default.nix‎

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,7 @@ let
715715
default =
716716
{
717717
pkgs,
718+
hostPkgs ? pkgs, # for cross-compilation: host tools for format/destroy scripts
718719
checked ? false,
719720
}:
720721
let
@@ -725,7 +726,7 @@ let
725726
else
726727
v;
727728
# @todo Do we need to add bcachefs-tools or not?
728-
destroyDependencies = with pkgs; [
729+
destroyDependencies = with hostPkgs; [
729730
util-linux
730731
e2fsprogs
731732
mdadm
@@ -739,14 +740,26 @@ let
739740
];
740741
in
741742
lib.mapAttrs throwIfNoDisksDetected {
742-
destroy = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy" ''
743-
export PATH=${lib.makeBinPath destroyDependencies}:$PATH
744-
${cfg.config._destroy}
745-
'';
746-
format = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format" ''
747-
export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
748-
${cfg.config._create}
749-
'';
743+
destroy =
744+
(diskoLib.writeCheckedBash {
745+
pkgs = hostPkgs;
746+
inherit checked;
747+
})
748+
"/bin/disko-destroy"
749+
''
750+
export PATH=${lib.makeBinPath destroyDependencies}:$PATH
751+
${cfg.config._destroy}
752+
'';
753+
format =
754+
(diskoLib.writeCheckedBash {
755+
pkgs = hostPkgs;
756+
inherit checked;
757+
})
758+
"/bin/disko-format"
759+
''
760+
export PATH=${lib.makeBinPath (cfg.config._packages hostPkgs)}:$PATH
761+
${cfg.config._create}
762+
'';
750763
mount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-mount" ''
751764
export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
752765
${cfg.config._mount}
@@ -755,15 +768,25 @@ let
755768
export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
756769
${cfg.config._unmount}
757770
'';
758-
formatMount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format-mount" ''
759-
export PATH=${lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ])}:$PATH
760-
${cfg.config._formatMount}
761-
'';
771+
formatMount =
772+
(diskoLib.writeCheckedBash {
773+
pkgs = hostPkgs;
774+
inherit checked;
775+
})
776+
"/bin/disko-format-mount"
777+
''
778+
export PATH=${lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ])}:$PATH
779+
${cfg.config._formatMount}
780+
'';
762781
destroyFormatMount =
763-
(diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy-format-mount"
782+
(diskoLib.writeCheckedBash {
783+
pkgs = hostPkgs;
784+
inherit checked;
785+
})
786+
"/bin/disko-destroy-format-mount"
764787
''
765788
export PATH=${
766-
lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies)
789+
lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ] ++ destroyDependencies)
767790
}:$PATH
768791
${cfg.config._destroyFormatMount}
769792
'';
@@ -844,12 +867,18 @@ let
844867
${cfg.config._mount}
845868
'';
846869

847-
diskoScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko" ''
848-
export PATH=${
849-
lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies)
850-
}:$PATH
851-
${cfg.config._disko}
852-
'';
870+
diskoScript =
871+
(diskoLib.writeCheckedBash {
872+
pkgs = hostPkgs;
873+
inherit checked;
874+
})
875+
"disko"
876+
''
877+
export PATH=${
878+
lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ] ++ destroyDependencies)
879+
}:$PATH
880+
${cfg.config._disko}
881+
'';
853882

854883
# These are useful to skip copying executables uploading a script to an in-memory installer
855884
destroyScriptNoDeps =

‎tests/cross-zfs-aarch64.nix‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
pkgs ? import <nixpkgs> { },
3+
diskoLib ? pkgs.callPackage ../lib { },
4+
}:
5+
let
6+
lib = pkgs.lib;
7+
targetSystem = "aarch64-linux";
8+
crossAttr = "aarch64-multiplatform";
9+
10+
diskoConfig = import ../example/cross-zfs.nix;
11+
testConfig = diskoLib.testLib.prepareDiskoConfig diskoConfig (lib.tail diskoLib.testLib.devices);
12+
13+
tsp-generator = pkgs.callPackage ../. { checked = false; };
14+
crossPkgs = pkgs.pkgsCross.${crossAttr};
15+
hostFormatScript = (tsp-generator._cliFormatCross testConfig) pkgs crossPkgs;
16+
in
17+
diskoLib.testLib.makeDiskoTest {
18+
inherit pkgs;
19+
name = "cross-zfs-${targetSystem}";
20+
disko-config = ../example/cross-zfs.nix;
21+
extraInstallerConfig.networking.hostId = "8425e349";
22+
extraSystemConfig.networking.hostId = "8425e349";
23+
testMode = "direct";
24+
testBoot = false;
25+
extraTestScript = ''
26+
machine.succeed("uname -m | grep -q x86_64")
27+
28+
print("Verifying host-native format script for ${targetSystem} target...")
29+
machine.succeed("test -x ${hostFormatScript}/bin/disko-format")
30+
31+
def assert_property(ds, property, expected_value):
32+
out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
33+
assert (
34+
out == expected_value
35+
), f"Expected {property}={expected_value} on {ds}, got: {out}"
36+
37+
assert_property("zroot", "compression", "lz4")
38+
assert_property("zroot", "com.sun:auto-snapshot", "false")
39+
40+
print("Cross-format test for ${targetSystem} completed successfully!")
41+
'';
42+
}

‎tests/cross-zfs-armv7l.nix‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
pkgs ? import <nixpkgs> { },
3+
diskoLib ? pkgs.callPackage ../lib { },
4+
}:
5+
let
6+
lib = pkgs.lib;
7+
targetSystem = "armv7l-linux";
8+
crossAttr = "armv7l-hf-multiplatform";
9+
10+
diskoConfig = import ../example/cross-zfs.nix;
11+
testConfig = diskoLib.testLib.prepareDiskoConfig diskoConfig (lib.tail diskoLib.testLib.devices);
12+
13+
tsp-generator = pkgs.callPackage ../. { checked = false; };
14+
crossPkgs = pkgs.pkgsCross.${crossAttr};
15+
hostFormatScript = (tsp-generator._cliFormatCross testConfig) pkgs crossPkgs;
16+
in
17+
diskoLib.testLib.makeDiskoTest {
18+
inherit pkgs;
19+
name = "cross-zfs-${targetSystem}";
20+
disko-config = ../example/cross-zfs.nix;
21+
extraInstallerConfig.networking.hostId = "8425e349";
22+
extraSystemConfig.networking.hostId = "8425e349";
23+
testMode = "direct";
24+
testBoot = false;
25+
extraTestScript = ''
26+
machine.succeed("uname -m | grep -q x86_64")
27+
28+
print("Verifying host-native format script for ${targetSystem} target...")
29+
machine.succeed("test -x ${hostFormatScript}/bin/disko-format")
30+
31+
def assert_property(ds, property, expected_value):
32+
out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
33+
assert (
34+
out == expected_value
35+
), f"Expected {property}={expected_value} on {ds}, got: {out}"
36+
37+
assert_property("zroot", "compression", "lz4")
38+
assert_property("zroot", "com.sun:auto-snapshot", "false")
39+
40+
print("Cross-format test for ${targetSystem} completed successfully!")
41+
'';
42+
}

0 commit comments

Comments
 (0)