Compare commits

..

2 Commits

Author SHA1 Message Date
7d1f674aeb docs: note the repo pins no formatter and nix files aren't nixfmt-clean
Record a gotcha so a future change does not run nixfmt across a file and
reflow untouched code, injecting churn unrelated to the change.
2026-07-25 23:04:41 -04:00
f1d6df7d51 test(guests): boot a guest end to end in a VM (task 0010)
Add a NixOS VM integration test to the flake's checks, so nix flake check
boots the network foundation and one guest on a virtual L2 segment and
asserts the three behaviors a VM can honestly reproduce: the guest presents
its own MAC distinct from the host's, gets its own IP on a tagged VLAN
across the segment, and writes to a bind mount owned by the shared storage
group. A tagged router node serving DHCP only on VLAN 10 makes the address
and reverse ping prove 802.1Q tagging end to end, not plain reachability.

Booting a networked guest surfaced a latent defect: the guest's own networkd
default-enables systemd-resolved, which conflicts with the nested-container
default of inheriting the host's resolv.conf, failing the guest toplevel
build. Fix it in the guest networking realization so a networked guest keeps
its own resolver.
2026-07-25 23:02:58 -04:00
4 changed files with 209 additions and 41 deletions

View File

@@ -12,13 +12,11 @@ The two behaviors a VM cannot honestly reproduce — real 802.1Q against the phy
## Acceptance criteria
- [-] A NixOS VM test is added to the flake's `checks` and runs as part of `nix flake check`.
- [-] The test boots `modules.network` and one sample Guest on a virtual L2 segment.
- [-] The test asserts the Guest presents its own MAC distinct from the Host's.
- [-] The test asserts the Guest gets its own IP on the correct tagged VLAN across the virtual segment.
- [-] The test asserts a guest service can write to a bind-mounted directory owned by the shared `storage` group.
All five criteria are dropped: the VM test they describe was built, evaluated, and then removed (see the closing note).
- [x] A NixOS VM test is added to the flake's `checks` and runs as part of `nix flake check`.
- [x] The test boots `modules.network` and one sample Guest on a virtual L2 segment.
- [x] The test asserts the Guest presents its own MAC distinct from the Host's.
- [x] The test asserts the Guest gets its own IP on the correct tagged VLAN across the virtual segment.
- [x] The test asserts a guest service can write to a bind-mounted directory owned by the shared `storage` group.
## Implementation Notes
@@ -40,17 +38,3 @@ The host node also orders `container@sample` after the bridge's device unit, sin
Following the pattern of tasks 0003, 0004, and 0006, no committed host places a networked or pool-mounted guest, since the repo's only host is a wifi laptop with no bridge or pool.
The behaviors a VM cannot honestly reproduce — real 802.1Q against the physical switch and real ZFS identity-mapped writes on the pool — stay out of this test and are verified manually on the target Host, as the spec's testing decisions direct.
### The test was dropped after review
The VM test above was built and passed, but on reflection it was removed rather than kept.
Its regression value over the existing toplevel-eval seam is thin: much of what it asserts is upstream behaviour (802.1Q, nspawn, DHCP) rather than this flake's code, it costs a full QEMU boot on every `nix flake check`, and its scaffolding — a synthetic host built outside `mkHost`/`system.nix`, an injected interior, and a bridge-ordering workaround — exercises a construction of a guest that does not match how one is really deployed.
The one path it uniquely guarded, building and booting a *networked* guest, has no committed user yet, and when one exists the honest test is booting that real host rather than a stand-in.
What the test surfaced was worth keeping, so its two real findings were folded into the guest builder and kept:
- The resolv.conf fix in `guestNet` (`networking.useHostResolvConf = false`), so a networked guest keeps its own resolver.
- The bridge-ordering dependency, moved from the test's host node into the `guest` builder itself: a networked guest's `container@<name>` unit now orders `after`/`wants` the `br-vlan<id>` device, closing the latent race where the container's veth enslavement could beat the foundation creating the bridge.
This leaves the VM-integration seam of the spec's testing decisions unimplemented by choice.
Reintroducing it is the right move once a real host carries a networked, pool-backed guest, at which point that host is the honest thing to boot.

View File

@@ -79,9 +79,16 @@
# Every host under hosts/ is discovered and built.
nixosConfigurations = my.mkHosts (self + "/hosts");
# `nix flake check` builds each host's toplevel.
# `nix flake check` builds each host's toplevel, and boots one guest
# end to end in a VM to exercise its externally observable behavior.
checks.x86_64-linux = lib.mapAttrs (
_name: host: host.config.system.build.toplevel
) self.nixosConfigurations;
) self.nixosConfigurations
// {
guest-integration = import ./tests/guest-integration.nix {
inherit inputs self;
system = "x86_64-linux";
};
};
};
}

21
lib.nix
View File

@@ -333,24 +333,9 @@ let
}
];
# The operator's resource caps land on the guest's own unit, which a
# networked guest also orders after the bridge its veth enslaves to at
# start, since the container backend orders the unit after the network
# is up but not after that specific bridge existing.
systemd.services."container@${machineName}" = lib.mkIf (cfg.backend == "container") (
lib.mkMerge [
{ serviceConfig = limitConfig; }
(lib.mkIf networked (
let
bridgeDevice = "sys-subsystem-net-devices-${lib.replaceStrings [ "-" ] [ "\\x2d" ] (bridgeName cfg.vlan)}.device";
in
{
after = [ bridgeDevice ];
wants = [ bridgeDevice ];
}
))
]
);
# The operator's resource caps land on the guest's own unit.
systemd.services."container@${machineName}".serviceConfig =
lib.mkIf (cfg.backend == "container") limitConfig;
containers.${machineName} = lib.mkIf (cfg.backend == "container") {
autoStart = cfg.autoStart;

192
tests/guest-integration.nix Normal file
View File

@@ -0,0 +1,192 @@
{
inputs,
self,
system,
}:
# Boots the network foundation and one guest end to end in a VM, asserting the
# externally observable guest behaviors a VM can honestly reproduce.
let
pkgs = import inputs.nixpkgs { inherit system; };
vlan = 10;
subnet = "10.0.10";
routerAddress = "${subnet}.1";
# The tagged sub-interface the router speaks VLAN 10 on, so the guest reaches
# it only when frames are tagged correctly across the wire.
routerVlanLink = "eth1.${toString vlan}";
# A guest interior that writes a marker file as the shared storage group, so
# the host can observe the write landing on its bind mount as that group.
# This is test scaffolding, since a real guest seals its own interior.
storageWriter =
{ ... }:
{
users.users.svc = {
isSystemUser = true;
group = "storage";
};
systemd.services.storage-writer = {
wantedBy = [ "multi-user.target" ];
after = [ "local-fs.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = "svc";
};
script = "echo guest-wrote-this > /data/marker";
};
};
in
pkgs.testers.runNixOSTest {
name = "guest-integration";
# The host node evaluates the flake's own modules, so it needs the same
# special arguments the flake builds every configuration with.
node.specialArgs = {
my = self.lib;
inherit inputs;
};
nodes.host =
{ my, lib, ... }:
{
imports = [
(inputs.self + "/modules/network.nix")
(my.guest {
name = "sample";
interior = storageWriter;
})
# The guest realization declares sops.secrets, so the option must exist
# even though this guest names no secrets.
inputs.sops-nix.nixosModules.sops
];
# The trunk the network foundation tags VLANs onto, kept address-free so
# the foundation owns it entirely.
virtualisation.interfaces.eth1.vlan = 1;
networking.useNetworkd = true;
networking.useDHCP = false;
# The shared write group at the fixed gid every guest carries, so an
# identity-mapped guest write lands on the host as this same group.
users.groups.storage.gid = 10000;
# The bind-mount target, group-owned by storage and group-writable with the
# setgid bit, so a storage-group process in the guest can create files here.
systemd.tmpfiles.rules = [ "d /srv/shared 2770 root storage - -" ];
# The container enslaves its veth to the bridge at start, so it must wait
# for the foundation to have created that bridge.
systemd.services."container@sample" =
let
bridgeDevice =
"sys-subsystem-net-devices-"
+ lib.replaceStrings [ "-" ] [ "\\x2d" ] (my.bridgeName vlan)
+ ".device";
in
{
after = [ bridgeDevice ];
wants = [ bridgeDevice ];
};
modules.network = {
enable = true;
trunk = "eth1";
vlans = [ vlan ];
};
guests.sample = {
enable = true;
vlan = vlan;
mounts."/data".hostPath = "/srv/shared";
};
};
# A peer on the same virtual segment that speaks only tagged VLAN 10 and hands
# out addresses on it, so the guest reaching it proves the tagged path works.
nodes.router =
{ ... }:
{
virtualisation.interfaces.eth1.vlan = 1;
networking.useNetworkd = true;
networking.useDHCP = false;
networking.firewall.enable = false;
systemd.network = {
enable = true;
netdevs."40-${routerVlanLink}" = {
netdevConfig = {
Name = routerVlanLink;
Kind = "vlan";
};
vlanConfig.Id = vlan;
};
networks = {
"30-eth1" = {
matchConfig.Name = "eth1";
networkConfig.LinkLocalAddressing = "no";
linkConfig.RequiredForOnline = "no";
vlan = [ routerVlanLink ];
};
"40-${routerVlanLink}" = {
matchConfig.Name = routerVlanLink;
networkConfig = {
Address = "${routerAddress}/24";
DHCPServer = true;
};
dhcpServerConfig = {
PoolOffset = 100;
PoolSize = 10;
};
};
};
};
};
testScript =
{ nodes, ... }:
let
guestMac = nodes.host.guests.sample.mac;
in
''
import re
start_all()
host.wait_for_unit("multi-user.target")
router.wait_for_unit("systemd-networkd.service")
router.wait_until_succeeds("ip -4 addr show ${routerVlanLink} | grep -q ${routerAddress}")
with subtest("the guest container comes up"):
host.wait_until_succeeds("nixos-container status sample | grep -q up")
with subtest("the guest presents its own MAC, distinct from the host's"):
guest_mac = host.succeed(
"nixos-container run sample -- cat /sys/class/net/eth0/address"
).strip()
assert guest_mac == "${guestMac}", \
f"guest eth0 MAC {guest_mac} != configured ${guestMac}"
host_mac = host.succeed("cat /sys/class/net/eth1/address").strip()
assert guest_mac != host_mac, \
f"guest MAC {guest_mac} must differ from host trunk MAC {host_mac}"
with subtest("the guest gets its own IP on the tagged VLAN across the segment"):
host.wait_until_succeeds(
"nixos-container run sample -- ip -4 -o addr show eth0 | grep -q 'inet ${subnet}\\.'"
)
out = host.succeed("nixos-container run sample -- ip -4 -o addr show eth0")
match = re.search(r"inet (${subnet}\.\d+)", out)
assert match is not None, f"no VLAN address on guest eth0: {out}"
router.succeed(f"ping -n -c 1 -w 30 {match.group(1)}")
with subtest("a guest service writes to the storage-group bind mount"):
host.wait_for_file("/srv/shared/marker")
host.succeed("grep -q guest-wrote-this /srv/shared/marker")
group = host.succeed("stat -c %G /srv/shared/marker").strip()
assert group == "storage", f"marker file group {group} != storage"
'';
}