From dbc3e089611c3b930a32f478e4aa640b0b1a0b1a Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 16 Jun 2022 21:18:04 +0200 Subject: [PATCH] nixosModules.microvm: init, and abstract existing microvms --- flake.nix | 44 ++++++++-- hosts/containers/freifunk/default.nix | 40 +++------ hosts/containers/leon/default.nix | 21 ++--- hosts/containers/oparl/default.nix | 22 ++--- hosts/server10/default.nix | 1 + hosts/server10/microvms.nix | 69 ++++++++++++--- hosts/server10/network.nix | 16 ++-- modules/microvm.nix | 118 ++++++++++++++++++++++++++ 8 files changed, 240 insertions(+), 91 deletions(-) create mode 100644 modules/microvm.nix diff --git a/flake.nix b/flake.nix index c9782a07..f2d934dd 100644 --- a/flake.nix +++ b/flake.nix @@ -217,7 +217,9 @@ host = getHostAddr name; target = ''root@"${host}"''; rebuildArg = "--flake ${self}#${name}"; - profile = self.nixosConfigurations."${name}".config.system.build.toplevel; + hostConfig = self.nixosConfigurations."${name}".config; + profile = hostConfig.system.build.toplevel; + selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config"; in result // { # Generate a small script for copying this flake to the # remote machine and bulding and switching there. @@ -241,6 +243,34 @@ #!${pkgs.runtimeShell} -ex ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise" ''; + } // lib.optionalAttrs (hostConfig ? c3d2.deployment.server) { + "microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" '' + #!${pkgs.runtimeShell} -e + ${hostConfig.system.build.copyToServer} ${self} ${secrets} + + ${hostConfig.system.build.runOnServer} bash -e < flake + systemctl restart microvm@${name}.service + END + ''; + "microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" '' + #!${pkgs.runtimeShell} -e + ${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner} + + ${hostConfig.system.build.runOnServer} bash -e < flake + systemctl restart microvm@${name}.service + END + ''; }) {} (builtins.attrNames self.nixosConfigurations) // builtins.foldl' (result: host: @@ -314,7 +344,7 @@ freifunk = nixosSystem' { modules = [ - microvm.nixosModules.microvm + self.nixosModules.microvm ./hosts/containers/freifunk { nixpkgs.overlays = with secrets.overlays; [ @@ -587,7 +617,7 @@ oparl = nixosSystem' { modules = [ - microvm.nixosModules.microvm + self.nixosModules.microvm ./hosts/containers/oparl { sops.defaultSopsFile = "${secrets}/hosts/oparl/secrets.yaml"; } ]; @@ -595,14 +625,14 @@ leon = nixosSystem' { modules = [ - microvm.nixosModules.microvm + self.nixosModules.microvm ./hosts/containers/leon ]; }; nfsroot = nixosSystem' { modules = [ - microvm.nixosModules.microvm + self.nixosModules.microvm ./hosts/containers/nfsroot ]; }; @@ -630,6 +660,10 @@ c3d2.users = import ./users.nix; c3d2.nncp.neigh = import ./config/nncp-relays.nix; }; + microvm.imports = [ + microvm.nixosModules.microvm + ./modules/microvm.nix + ]; nncp = ./modules/nncp.nix; plume = { imports = [ ./modules/plume.nix ]; diff --git a/hosts/containers/freifunk/default.nix b/hosts/containers/freifunk/default.nix index d7e95689..0ba733bb 100644 --- a/hosts/containers/freifunk/default.nix +++ b/hosts/containers/freifunk/default.nix @@ -31,35 +31,6 @@ in { "${modulesPath}/profiles/minimal.nix" ]; - microvm = { - mem = 512; - hypervisor = "cloud-hypervisor"; - - shares = [ { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "store"; - proto = "virtiofs"; - socket = "store.socket"; - } ] ++ map (dir: { - source = "/var/lib/microvms/${config.networking.hostName}/${dir}"; - mountPoint = "/${dir}"; - tag = dir; - proto = "virtiofs"; - socket = "${dir}.socket"; - }) [ "etc" "home" "var"]; - - interfaces = [ { - type = "tap"; - id = "freifunk-core"; - mac = mac.core; - } { - type = "tap"; - id = "freifunk-bmx"; - mac = mac.bmx; - } ]; - }; - boot.tmpOnTmpfs = true; boot.postBootCommands = '' if [ ! -c /dev/net/tun ]; then @@ -70,6 +41,17 @@ in { c3d2 = { isInHq = false; hq.statistics.enable = true; + deployment = { + server = "server10"; + mounts = [ "etc" "home" "var"]; + interfaces = [ { + net = "core"; + mac = mac.core; + } { + net = "bmx"; + mac = mac.bmx; + } ]; + }; }; services.collectd.plugins.protocols = ""; diff --git a/hosts/containers/leon/default.nix b/hosts/containers/leon/default.nix index d385c8d6..a728ddf4 100644 --- a/hosts/containers/leon/default.nix +++ b/hosts/containers/leon/default.nix @@ -7,24 +7,13 @@ in microvm = { hypervisor = "qemu"; mem = 2048; + }; - shares = [ { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "store"; - proto = "virtiofs"; - socket = "store.socket"; - } ] ++ map (dir: { - source = "/var/lib/microvms/${config.networking.hostName}/${dir}"; - mountPoint = "/${dir}"; - tag = dir; - proto = "virtiofs"; - socket = "${dir}.socket"; - }) [ "etc" "home" "var"]; - + c3d2.deployment = { + server = "server10"; + mounts = [ "etc" "home" "var"]; interfaces = [ { - type = "tap"; - id = "leon"; + net = "serv"; mac = "00:de:8d:c1:9f:45"; } ]; }; diff --git a/hosts/containers/oparl/default.nix b/hosts/containers/oparl/default.nix index 8a99269f..fc2300b1 100644 --- a/hosts/containers/oparl/default.nix +++ b/hosts/containers/oparl/default.nix @@ -13,24 +13,12 @@ in { microvm = { hypervisor = "qemu"; - - shares = [ { - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "store"; - proto = "virtiofs"; - socket = "store.socket"; - } ] ++ map (dir: { - source = "/var/lib/microvms/${config.networking.hostName}/${dir}"; - mountPoint = "/${dir}"; - tag = dir; - proto = "virtiofs"; - socket = "${dir}.socket"; - }) [ "etc" "home" "var"]; - + }; + c3d2.deployment = { + server = "server10"; + mounts = [ "etc" "home" "var"]; interfaces = [ { - type = "tap"; - id = "oparl"; + net = "serv"; mac = "00:de:9a:ce:38:79"; } ]; }; diff --git a/hosts/server10/default.nix b/hosts/server10/default.nix index 52fe74a7..7830d2e7 100644 --- a/hosts/server10/default.nix +++ b/hosts/server10/default.nix @@ -9,6 +9,7 @@ ]; c3d2.hq.statistics.enable = true; + c3d2.deployment.microvmBaseZfsDataset = "server10/vm"; boot= { loader.grub = { diff --git a/hosts/server10/microvms.nix b/hosts/server10/microvms.nix index 490aa3de..ae76e192 100644 --- a/hosts/server10/microvms.nix +++ b/hosts/server10/microvms.nix @@ -1,18 +1,61 @@ +{ config, lib, pkgs, ... }: { - microvm.autostart = [ "oparl" "leon" "nfsroot" ]; - c3d2.server.bridgePorts = { - serv = [ "oparl" "leon" "nfsroot" ]; - core = [ "freifunk-core" ]; - bmx = [ "freifunk-bmx" ]; + options = with lib; { + c3d2.deployment.microvmBaseZfsDataset = mkOption { + type = types.str; + description = "Base ZFS dataset whereunder to create shares for MicroVMs."; + }; }; - nix = { - # fetch github-prebuilt microvm-kernels - binaryCaches = [ - "https://microvm.cachix.org" - ]; - binaryCachePublicKeys = [ - "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" - ]; + config = { + microvm.autostart = [ "oparl" "leon" "nfsroot" "freifunk" ]; + + systemd.services."microvm-virtiofsd@" = { + requires = [ "microvm-zfs-datasets@%i.service" ]; + }; + systemd.services."microvm-zfs-datasets@" = { + description = "Create ZFS datasets for MicroVM '%i'"; + before = [ "microvm-virtiofsd@%i.service" ]; + after = [ "local-fs.target" ]; + partOf = [ "microvm@%i.service" ]; + unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + WorkingDirectory = "/var/lib/microvms/%i"; + SyslogIdentifier = "microvm-zfs-datasets@%i"; + }; + path = with pkgs; [ zfs ]; + scriptArgs = "%i"; + script = '' + zfsExists() { + zfs list $1 >/dev/null 2>/dev/null + } + + NAME="$1" + BASE="${config.c3d2.deployment.microvmBaseZfsDataset}" + zfsExists $BASE || \ + zfs create $BASE + zfsExists $BASE/$NAME || \ + zfs create $BASE/$NAME + for d in current/share/microvm/virtiofs/*; do + SOURCE=$(cat $d/source) + TAG=$(basename $d) + MNT=$(pwd)$SOURCE + zfsExists $BASE/$NAME/$TAG || \ + zfs create -o mountpoint=$MNT $BASE/$NAME/$TAG + done + ''; + }; + + nix = { + # fetch github-prebuilt microvm-kernels + binaryCaches = [ + "https://microvm.cachix.org" + ]; + binaryCachePublicKeys = [ + "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" + ]; + }; }; } diff --git a/hosts/server10/network.nix b/hosts/server10/network.nix index 1729e502..7708d4f3 100644 --- a/hosts/server10/network.nix +++ b/hosts/server10/network.nix @@ -123,17 +123,11 @@ in # Attach eth*/bond0/VLAN to bridge networkConfig.Bridge = net; }; - }) {} zwHostBridges) // - builtins.foldl' (result: net: - builtins.foldl' (result: port: - result // { - "${net}-${port}" = { - matchConfig.Name = port; - networkConfig.Bridge = net; - }; - } - ) result (config.c3d2.server.bridgePorts.${net}) - ) {} (builtins.attrNames config.c3d2.server.bridgePorts); + "ports-${net}" = { + matchConfig.Name = "${net}-*"; + networkConfig.Bridge = net; + }; + }) {} zwHostBridges); }; }; } diff --git a/modules/microvm.nix b/modules/microvm.nix new file mode 100644 index 00000000..ffe6a686 --- /dev/null +++ b/modules/microvm.nix @@ -0,0 +1,118 @@ +{ zentralwerk, options, config, lib, pkgs, ... }: +let + inherit (config.c3d2.deployment) server; + serverFQDN = "${server}.cluster.zentralwerk.org"; + serverConfig = zentralwerk.lib.config.site.hosts.${config.c3d2.deployment.server}; + serverBridges = builtins.attrNames ( + lib.filterAttrs (_: { type, ... }: type == "bridge") + serverConfig.interfaces + ); +in +{ + options.c3d2.deployment = with lib; { + server = mkOption { + type = types.enum [ "server10" ]; + description = "Server that is supposed to host this MicroVM."; + }; + + interfaces = mkOption { + description = "Network interfaces"; + type = types.listOf (types.submodule { + options = { + net = mkOption { + description = "Network to hook up to"; + type = types.enum serverBridges; + }; + mac = mkOption { + description = "Ethernet MAC address"; + type = types.str; + }; + }; + }); + }; + + mounts = mkOption { + description = "Persistent filesystems to create, without leading /."; + type = with types; listOf str; + default = [ "etc" ]; + }; + }; + + config.system.build = with pkgs; { + copyToServer = writeScript "copy-to-${server}" '' + #! ${runtimeShell} -e + + nix copy --to ssh://root@${serverFQDN} $@ + ''; + + runOnServer = writeScript "run-on-${server}" '' + #! ${runtimeShell} -e + + ssh root@${serverFQDN} -- $@ + ''; + }; + + config = { + microvm = { + hypervisor = lib.mkDefault "cloud-hypervisor"; + mem = lib.mkDefault 512; + + interfaces = map ({ net, mac }: { + type = "tap"; + id = builtins.substring 0 15 "${net}-${config.networking.hostName}"; + inherit mac; + }) config.c3d2.deployment.interfaces; + + shares = [ { + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "store"; + proto = "virtiofs"; + socket = "store.socket"; + } ] + ++ map (dir: + if lib.hasPrefix "/" dir + then throw "${dir} starts with a leading /. Just don't!" + else let + tag = builtins.replaceStrings ["/"] ["_"] dir; + in { + source = "/var/lib/microvms/${config.networking.hostName}/${dir}"; + mountPoint = "/${dir}"; + inherit tag; + proto = "virtiofs"; + socket = "${tag}.socket"; + }) config.c3d2.deployment.mounts; + }; + + systemd.services."microvm-zfs-datasets@" = { + description = "Create ZFS datasets for MicroVM '%i'"; + before = [ "microvm-virtiofsd@%i.service" ]; + after = [ "local-fs.target" ]; + partOf = [ "microvm@%i.service" ]; + unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs"; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + WorkingDirectory = "/var/lib/microvms/%i"; + SyslogIdentifier = "microvm-zfs-datasets@%i"; + }; + scriptArgs = "%i"; + script = '' + zfsExists() { + zfs list $1 >/dev/null 2>/dev/null + } + + NAME="$1" + BASE=$(hostname)/vm/$NAME + zfsExists $BASE || \ + zfs create $BASE + for d in current/share/microvm/virtiofs/*; do + SOURCE=$(cat $d/source) + MNT=$(pwd)/$SOURCE + zfsExists $BASE/$d || \ + zfs create -o mountpoint=$MNT $BASE/$d + done + ''; + }; + }; +}