Add cross-compilation support for disk formatting (ZFS)

When formatting disks for a different target architecture (e.g.,
creating an aarch64 disk image from an x86_64 host), tools like ZFS that
communicate with kernel modules must use host-native binaries.

I am guessing there also exist other cases when ZFS comes into play.

This is not a problem with tools like `ext4` because they do everything
in user space.

I have created an example file and used it to build the file system for
the archs `aarch64`, `armv7l`, `i686`, `riscv64` and `x86_64`.

**Note:** The host system must have the following nixos configs enabled.

```nix
boot.binfmt.preferStaticEmulators = true;
boot.binfmt.emulatedSystems = [ "<TARGET_ARCH>" ];
```

see: disko-install:303-317

Author: dvaerum

default.nix

@@ -33,6 +33,13 @@ in
     cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).destroyNoDeps;
 
   _cliFormat = cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).format;
+  # for cross-compilation: hostPkgs for format, targetPkgs for mount scripts
+  _cliFormatCross =
+    cfg: hostPkgs: targetPkgs:
+    ((eval cfg).config.disko.devices._scripts {
+      pkgs = targetPkgs;
+      inherit hostPkgs checked;
+    }).format;
   _cliFormatNoDeps =
     cfg: pkgs: ((eval cfg).config.disko.devices._scripts { inherit pkgs checked; }).formatNoDeps;
 

disko-install

@@ -179,6 +179,44 @@ maybeRun () {
   fi
 }
 
+checkBinfmtForCrossCompilation() {
+  local nixos_system=$1
+
+  # Get the current system (host) from Nix
+  local host_system
+  host_system=$(nix-instantiate --eval --expr 'builtins.currentSystem' 2>/dev/null | tr -d '"') || return 0
+
+  # Get the target system from the store path
+  local target_system
+  target_system=$(nix-store --query --binding system "$nixos_system" 2>/dev/null) || return 0
+
+  # If systems match, no cross-compilation, no binfmt needed
+  if [[ "$host_system" == "$target_system" ]]; then
+    return 0
+  fi
+
+  # Cross-compilation detected, check binfmt configuration
+  local binfmt_file="/proc/sys/fs/binfmt_misc/$target_system"
+  if [[ ! -f "$binfmt_file" ]]; then
+    echo "Cross-compiling from $host_system to $target_system requires binfmt_misc configuration." >&2
+    echo "Add this to your NixOS configuration:" >&2
+    echo "  boot.binfmt.emulatedSystems = [ \"$target_system\" ];" >&2
+    echo "Then rebuild: nixos-rebuild switch" >&2
+    exit 1
+  fi
+
+  # Check for F flag (required for chroot)
+  local flags
+  flags=$(grep "^flags:" "$binfmt_file" | cut -d: -f2 | tr -d ' ')
+  if [[ ! "$flags" =~ F ]]; then
+    echo "Cross-compilation requires the binfmt_misc 'F' flag (current: $flags)" >&2
+    echo "Add this to your NixOS configuration:" >&2
+    echo "  boot.binfmt.preferStaticEmulators = true;" >&2
+    echo "Then rebuild: nixos-rebuild switch" >&2
+    exit 1
+  fi
+}
+
 main() {
   parseArgs "$@"
 
@@ -262,6 +300,9 @@ main() {
   closure_info=${artifacts[1]}
   disko_script=${artifacts[2]}
 
+  # Check binfmt configuration for cross-compilation before doing any disk operations
+  checkBinfmtForCrossCompilation "$nixos_system"
+
   if [[ -n ${dry_run-} ]]; then
     echo "Would run: $disko_script"
     echo "Would run: nixos-install --system '$nixos_system' --root '$mountPoint'"

example/cross-zfs.nix

@@ -0,0 +1,59 @@
+# Example ZFS configuration used to test cross-architecture disk formatting.
+# When formatting disks for a different architecture (e.g., preparing an
+# aarch64 disk from an x86_64 host), disko automatically uses host-native
+# tools for ZFS operations that require kernel communication.
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/disk/by-id/some-disk-id";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [ "umask=0077" ];
+              };
+            };
+            zfs = {
+              size = "100%";
+              content = {
+                type = "zfs";
+                pool = "zroot";
+              };
+            };
+          };
+        };
+      };
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        options.cachefile = "none";
+        rootFsOptions = {
+          compression = "lz4";
+          "com.sun:auto-snapshot" = "false";
+        };
+        mountpoint = "/";
+
+        datasets = {
+          home = {
+            type = "zfs_fs";
+            mountpoint = "/home";
+          };
+          nix = {
+            type = "zfs_fs";
+            mountpoint = "/nix";
+            options.atime = "off";
+          };
+        };
+      };
+    };
+  };
+}

install-cli.nix

@@ -5,11 +5,15 @@
   extraSystemConfig ? "{}",
   writeEfiBootEntries ? false,
   rootMountPoint ? "/mnt",
+  hostSystem ? builtins.currentSystem, # the system running the format script, for cross-compilation
 }:
 let
   originalSystem = (builtins.getFlake "${flake}").nixosConfigurations."${flakeAttr}";
   lib = originalSystem.pkgs.lib;
 
+  # Get host pkgs for cross-compilation (format scripts need host-native tools)
+  hostPkgs = import originalSystem.pkgs.path { system = hostSystem; };
+
   deviceName =
     name:
     if diskMappings ? ${name} then
@@ -57,11 +61,19 @@ let
       )
     ];
   };
+  # Build scripts with hostPkgs for format/destroy, targetPkgs for mount
+  # This enables cross-compilation: format scripts use host-native tools
+  # that can communicate with the running kernel
+  scripts = diskoSystem.config.disko.devices._scripts {
+    pkgs = diskoSystem.pkgs; # target pkgs for mount scripts
+    inherit hostPkgs; # host pkgs for format/destroy scripts
+  };
 in
 {
   installToplevel = installSystem.config.system.build.toplevel;
   closureInfo = installSystem.pkgs.closureInfo {
     rootPaths = [ installSystem.config.system.build.toplevel ];
   };
-  inherit (diskoSystem.config.system.build) formatScript mountScript diskoScript;
+  # Use scripts built with hostPkgs for cross-compilation support
+  inherit (scripts) formatScript mountScript diskoScript;
 }

lib/default.nix

@@ -715,6 +715,7 @@ let
             default =
               {
                 pkgs,
+                hostPkgs ? pkgs, # for cross-compilation: host tools for format/destroy scripts
                 checked ? false,
               }:
               let
@@ -725,7 +726,7 @@ let
                   else
                     v;
                 # @todo Do we need to add bcachefs-tools or not?
-                destroyDependencies = with pkgs; [
+                destroyDependencies = with hostPkgs; [
                   util-linux
                   e2fsprogs
                   mdadm
@@ -739,14 +740,26 @@ let
                 ];
               in
               lib.mapAttrs throwIfNoDisksDetected {
-                destroy = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy" ''
-                  export PATH=${lib.makeBinPath destroyDependencies}:$PATH
-                  ${cfg.config._destroy}
-                '';
-                format = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format" ''
-                  export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
-                  ${cfg.config._create}
-                '';
+                destroy =
+                  (diskoLib.writeCheckedBash {
+                    pkgs = hostPkgs;
+                    inherit checked;
+                  })
+                    "/bin/disko-destroy"
+                    ''
+                      export PATH=${lib.makeBinPath destroyDependencies}:$PATH
+                      ${cfg.config._destroy}
+                    '';
+                format =
+                  (diskoLib.writeCheckedBash {
+                    pkgs = hostPkgs;
+                    inherit checked;
+                  })
+                    "/bin/disko-format"
+                    ''
+                      export PATH=${lib.makeBinPath (cfg.config._packages hostPkgs)}:$PATH
+                      ${cfg.config._create}
+                    '';
                 mount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-mount" ''
                   export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
                   ${cfg.config._mount}
@@ -755,15 +768,25 @@ let
                   export PATH=${lib.makeBinPath (cfg.config._packages pkgs)}:$PATH
                   ${cfg.config._unmount}
                 '';
-                formatMount = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-format-mount" ''
-                  export PATH=${lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ])}:$PATH
-                  ${cfg.config._formatMount}
-                '';
+                formatMount =
+                  (diskoLib.writeCheckedBash {
+                    pkgs = hostPkgs;
+                    inherit checked;
+                  })
+                    "/bin/disko-format-mount"
+                    ''
+                      export PATH=${lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ])}:$PATH
+                      ${cfg.config._formatMount}
+                    '';
                 destroyFormatMount =
-                  (diskoLib.writeCheckedBash { inherit pkgs checked; }) "/bin/disko-destroy-format-mount"
+                  (diskoLib.writeCheckedBash {
+                    pkgs = hostPkgs;
+                    inherit checked;
+                  })
+                    "/bin/disko-destroy-format-mount"
                     ''
                       export PATH=${
-                        lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies)
+                        lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ] ++ destroyDependencies)
                       }:$PATH
                       ${cfg.config._destroyFormatMount}
                     '';
@@ -844,12 +867,18 @@ let
                   ${cfg.config._mount}
                 '';
 
-                diskoScript = (diskoLib.writeCheckedBash { inherit pkgs checked; }) "disko" ''
-                  export PATH=${
-                    lib.makeBinPath ((cfg.config._packages pkgs) ++ [ pkgs.bash ] ++ destroyDependencies)
-                  }:$PATH
-                  ${cfg.config._disko}
-                '';
+                diskoScript =
+                  (diskoLib.writeCheckedBash {
+                    pkgs = hostPkgs;
+                    inherit checked;
+                  })
+                    "disko"
+                    ''
+                      export PATH=${
+                        lib.makeBinPath ((cfg.config._packages hostPkgs) ++ [ hostPkgs.bash ] ++ destroyDependencies)
+                      }:$PATH
+                      ${cfg.config._disko}
+                    '';
 
                 # These are useful to skip copying executables uploading a script to an in-memory installer
                 destroyScriptNoDeps =

package.nix

@@ -9,7 +9,6 @@
   binlore,
   diskoVersion,
   stdenv,
-  xcp,
 }:
 
 let
@@ -50,7 +49,6 @@ let
               [
                 nix
                 coreutils
-                xcp
               ]
               ++ lib.optional (!stdenv.isDarwin) nixos-install-tools
             )

tests/cross-zfs-aarch64.nix

@@ -0,0 +1,42 @@
+{
+  pkgs ? import <nixpkgs> { },
+  diskoLib ? pkgs.callPackage ../lib { },
+}:
+let
+  lib = pkgs.lib;
+  targetSystem = "aarch64-linux";
+  crossAttr = "aarch64-multiplatform";
+
+  diskoConfig = import ../example/cross-zfs.nix;
+  testConfig = diskoLib.testLib.prepareDiskoConfig diskoConfig (lib.tail diskoLib.testLib.devices);
+
+  tsp-generator = pkgs.callPackage ../. { checked = false; };
+  crossPkgs = pkgs.pkgsCross.${crossAttr};
+  hostFormatScript = (tsp-generator._cliFormatCross testConfig) pkgs crossPkgs;
+in
+diskoLib.testLib.makeDiskoTest {
+  inherit pkgs;
+  name = "cross-zfs-${targetSystem}";
+  disko-config = ../example/cross-zfs.nix;
+  extraInstallerConfig.networking.hostId = "8425e349";
+  extraSystemConfig.networking.hostId = "8425e349";
+  testMode = "direct";
+  testBoot = false;
+  extraTestScript = ''
+    machine.succeed("uname -m | grep -q x86_64")
+
+    print("Verifying host-native format script for ${targetSystem} target...")
+    machine.succeed("test -x ${hostFormatScript}/bin/disko-format")
+
+    def assert_property(ds, property, expected_value):
+        out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
+        assert (
+            out == expected_value
+        ), f"Expected {property}={expected_value} on {ds}, got: {out}"
+
+    assert_property("zroot", "compression", "lz4")
+    assert_property("zroot", "com.sun:auto-snapshot", "false")
+
+    print("Cross-format test for ${targetSystem} completed successfully!")
+  '';
+}

tests/cross-zfs-armv7l.nix

@@ -0,0 +1,42 @@
+{
+  pkgs ? import <nixpkgs> { },
+  diskoLib ? pkgs.callPackage ../lib { },
+}:
+let
+  lib = pkgs.lib;
+  targetSystem = "armv7l-linux";
+  crossAttr = "armv7l-hf-multiplatform";
+
+  diskoConfig = import ../example/cross-zfs.nix;
+  testConfig = diskoLib.testLib.prepareDiskoConfig diskoConfig (lib.tail diskoLib.testLib.devices);
+
+  tsp-generator = pkgs.callPackage ../. { checked = false; };
+  crossPkgs = pkgs.pkgsCross.${crossAttr};
+  hostFormatScript = (tsp-generator._cliFormatCross testConfig) pkgs crossPkgs;
+in
+diskoLib.testLib.makeDiskoTest {
+  inherit pkgs;
+  name = "cross-zfs-${targetSystem}";
+  disko-config = ../example/cross-zfs.nix;
+  extraInstallerConfig.networking.hostId = "8425e349";
+  extraSystemConfig.networking.hostId = "8425e349";
+  testMode = "direct";
+  testBoot = false;
+  extraTestScript = ''
+    machine.succeed("uname -m | grep -q x86_64")
+
+    print("Verifying host-native format script for ${targetSystem} target...")
+    machine.succeed("test -x ${hostFormatScript}/bin/disko-format")
+
+    def assert_property(ds, property, expected_value):
+        out = machine.succeed(f"zfs get -H {property} {ds} -o value").rstrip()
+        assert (
+            out == expected_value
+        ), f"Expected {property}={expected_value} on {ds}, got: {out}"
+
+    assert_property("zroot", "compression", "lz4")
+    assert_property("zroot", "com.sun:auto-snapshot", "false")
+
+    print("Cross-format test for ${targetSystem} completed successfully!")
+  '';
+}