test(guests): boot a guest end to end in a VM (task 0010)
Add a NixOS VM integration test to the flake's checks, so nix flake check boots the network foundation and one guest on a virtual L2 segment and asserts the three behaviors a VM can honestly reproduce: the guest presents its own MAC distinct from the host's, gets its own IP on a tagged VLAN across the segment, and writes to a bind mount owned by the shared storage group. A tagged router node serving DHCP only on VLAN 10 makes the address and reverse ping prove 802.1Q tagging end to end, not plain reachability. Booting a networked guest surfaced a latent defect: the guest's own networkd default-enables systemd-resolved, which conflicts with the nested-container default of inheriting the host's resolv.conf, failing the guest toplevel build. Fix it in the guest networking realization so a networked guest keeps its own resolver.
This commit is contained in:
192
tests/guest-integration.nix
Normal file
192
tests/guest-integration.nix
Normal file
@@ -0,0 +1,192 @@
|
||||
{
|
||||
inputs,
|
||||
self,
|
||||
system,
|
||||
}:
|
||||
# Boots the network foundation and one guest end to end in a VM, asserting the
|
||||
# externally observable guest behaviors a VM can honestly reproduce.
|
||||
let
|
||||
pkgs = import inputs.nixpkgs { inherit system; };
|
||||
|
||||
vlan = 10;
|
||||
subnet = "10.0.10";
|
||||
routerAddress = "${subnet}.1";
|
||||
|
||||
# The tagged sub-interface the router speaks VLAN 10 on, so the guest reaches
|
||||
# it only when frames are tagged correctly across the wire.
|
||||
routerVlanLink = "eth1.${toString vlan}";
|
||||
|
||||
# A guest interior that writes a marker file as the shared storage group, so
|
||||
# the host can observe the write landing on its bind mount as that group.
|
||||
# This is test scaffolding, since a real guest seals its own interior.
|
||||
storageWriter =
|
||||
{ ... }:
|
||||
{
|
||||
users.users.svc = {
|
||||
isSystemUser = true;
|
||||
group = "storage";
|
||||
};
|
||||
|
||||
systemd.services.storage-writer = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "local-fs.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
User = "svc";
|
||||
};
|
||||
script = "echo guest-wrote-this > /data/marker";
|
||||
};
|
||||
};
|
||||
in
|
||||
pkgs.testers.runNixOSTest {
|
||||
name = "guest-integration";
|
||||
|
||||
# The host node evaluates the flake's own modules, so it needs the same
|
||||
# special arguments the flake builds every configuration with.
|
||||
node.specialArgs = {
|
||||
my = self.lib;
|
||||
inherit inputs;
|
||||
};
|
||||
|
||||
nodes.host =
|
||||
{ my, lib, ... }:
|
||||
{
|
||||
imports = [
|
||||
(inputs.self + "/modules/network.nix")
|
||||
(my.guest {
|
||||
name = "sample";
|
||||
interior = storageWriter;
|
||||
})
|
||||
# The guest realization declares sops.secrets, so the option must exist
|
||||
# even though this guest names no secrets.
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
];
|
||||
|
||||
# The trunk the network foundation tags VLANs onto, kept address-free so
|
||||
# the foundation owns it entirely.
|
||||
virtualisation.interfaces.eth1.vlan = 1;
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
|
||||
# The shared write group at the fixed gid every guest carries, so an
|
||||
# identity-mapped guest write lands on the host as this same group.
|
||||
users.groups.storage.gid = 10000;
|
||||
|
||||
# The bind-mount target, group-owned by storage and group-writable with the
|
||||
# setgid bit, so a storage-group process in the guest can create files here.
|
||||
systemd.tmpfiles.rules = [ "d /srv/shared 2770 root storage - -" ];
|
||||
|
||||
# The container enslaves its veth to the bridge at start, so it must wait
|
||||
# for the foundation to have created that bridge.
|
||||
systemd.services."container@sample" =
|
||||
let
|
||||
bridgeDevice =
|
||||
"sys-subsystem-net-devices-"
|
||||
+ lib.replaceStrings [ "-" ] [ "\\x2d" ] (my.bridgeName vlan)
|
||||
+ ".device";
|
||||
in
|
||||
{
|
||||
after = [ bridgeDevice ];
|
||||
wants = [ bridgeDevice ];
|
||||
};
|
||||
|
||||
modules.network = {
|
||||
enable = true;
|
||||
trunk = "eth1";
|
||||
vlans = [ vlan ];
|
||||
};
|
||||
|
||||
guests.sample = {
|
||||
enable = true;
|
||||
vlan = vlan;
|
||||
mounts."/data".hostPath = "/srv/shared";
|
||||
};
|
||||
};
|
||||
|
||||
# A peer on the same virtual segment that speaks only tagged VLAN 10 and hands
|
||||
# out addresses on it, so the guest reaching it proves the tagged path works.
|
||||
nodes.router =
|
||||
{ ... }:
|
||||
{
|
||||
virtualisation.interfaces.eth1.vlan = 1;
|
||||
networking.useNetworkd = true;
|
||||
networking.useDHCP = false;
|
||||
networking.firewall.enable = false;
|
||||
|
||||
systemd.network = {
|
||||
enable = true;
|
||||
|
||||
netdevs."40-${routerVlanLink}" = {
|
||||
netdevConfig = {
|
||||
Name = routerVlanLink;
|
||||
Kind = "vlan";
|
||||
};
|
||||
vlanConfig.Id = vlan;
|
||||
};
|
||||
|
||||
networks = {
|
||||
"30-eth1" = {
|
||||
matchConfig.Name = "eth1";
|
||||
networkConfig.LinkLocalAddressing = "no";
|
||||
linkConfig.RequiredForOnline = "no";
|
||||
vlan = [ routerVlanLink ];
|
||||
};
|
||||
"40-${routerVlanLink}" = {
|
||||
matchConfig.Name = routerVlanLink;
|
||||
networkConfig = {
|
||||
Address = "${routerAddress}/24";
|
||||
DHCPServer = true;
|
||||
};
|
||||
dhcpServerConfig = {
|
||||
PoolOffset = 100;
|
||||
PoolSize = 10;
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
let
|
||||
guestMac = nodes.host.guests.sample.mac;
|
||||
in
|
||||
''
|
||||
import re
|
||||
|
||||
start_all()
|
||||
|
||||
host.wait_for_unit("multi-user.target")
|
||||
router.wait_for_unit("systemd-networkd.service")
|
||||
router.wait_until_succeeds("ip -4 addr show ${routerVlanLink} | grep -q ${routerAddress}")
|
||||
|
||||
with subtest("the guest container comes up"):
|
||||
host.wait_until_succeeds("nixos-container status sample | grep -q up")
|
||||
|
||||
with subtest("the guest presents its own MAC, distinct from the host's"):
|
||||
guest_mac = host.succeed(
|
||||
"nixos-container run sample -- cat /sys/class/net/eth0/address"
|
||||
).strip()
|
||||
assert guest_mac == "${guestMac}", \
|
||||
f"guest eth0 MAC {guest_mac} != configured ${guestMac}"
|
||||
host_mac = host.succeed("cat /sys/class/net/eth1/address").strip()
|
||||
assert guest_mac != host_mac, \
|
||||
f"guest MAC {guest_mac} must differ from host trunk MAC {host_mac}"
|
||||
|
||||
with subtest("the guest gets its own IP on the tagged VLAN across the segment"):
|
||||
host.wait_until_succeeds(
|
||||
"nixos-container run sample -- ip -4 -o addr show eth0 | grep -q 'inet ${subnet}\\.'"
|
||||
)
|
||||
out = host.succeed("nixos-container run sample -- ip -4 -o addr show eth0")
|
||||
match = re.search(r"inet (${subnet}\.\d+)", out)
|
||||
assert match is not None, f"no VLAN address on guest eth0: {out}"
|
||||
router.succeed(f"ping -n -c 1 -w 30 {match.group(1)}")
|
||||
|
||||
with subtest("a guest service writes to the storage-group bind mount"):
|
||||
host.wait_for_file("/srv/shared/marker")
|
||||
host.succeed("grep -q guest-wrote-this /srv/shared/marker")
|
||||
group = host.succeed("stat -c %G /srv/shared/marker").strip()
|
||||
assert group == "storage", f"marker file group {group} != storage"
|
||||
'';
|
||||
}
|
||||
Reference in New Issue
Block a user