c3d2/nix-config
c3d2
/
nix-config
24
7
Fork 3
Code Issues 12 Pull Requests 2 Releases Wiki Activity

nixosModules.microvm: init, and abstract existing microvms

This commit is contained in:
Astro 2022-06-16 21:18:04 +02:00
parent 9beb323752
commit dbc3e08961
8 changed files with 240 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
flake.nix
View File

@ -217,7 +217,9 @@
host = getHostAddr name;
target = ''root@"${host}"'';
rebuildArg = "--flake ${self}#${name}";
profile = self.nixosConfigurations."${name}".config.system.build.toplevel;
hostConfig = self.nixosConfigurations."${name}".config;
profile = hostConfig.system.build.toplevel;
selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config";
in result // {
# Generate a small script for copying this flake to the
# remote machine and bulding and switching there.
@ -241,6 +243,34 @@
#!${pkgs.runtimeShell} -ex
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
'';
} // lib.optionalAttrs (hostConfig ? c3d2.deployment.server) {
"microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
#!${pkgs.runtimeShell} -e
${hostConfig.system.build.copyToServer} ${self} ${secrets}
${hostConfig.system.build.runOnServer} bash -e <<END
mkdir -p /var/lib/microvms/${name}
cd /var/lib/microvms/${name}
nix build \
-o current \
${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
echo '${selfRef}' > flake
systemctl restart microvm@${name}.service
END
'';
"microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
#!${pkgs.runtimeShell} -e
${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}
${hostConfig.system.build.runOnServer} bash -e <<END
mkdir -p /var/lib/microvms/${name}
cd /var/lib/microvms/${name}
rm -f current
ln -s ${hostConfig.microvm.declaredRunner} current
echo '${selfRef}' > flake
systemctl restart microvm@${name}.service
END
'';
}) {} (builtins.attrNames self.nixosConfigurations) //
builtins.foldl' (result: host:
@ -314,7 +344,7 @@
freifunk = nixosSystem' {
modules = [
microvm.nixosModules.microvm
self.nixosModules.microvm
./hosts/containers/freifunk
{
nixpkgs.overlays = with secrets.overlays; [
@ -587,7 +617,7 @@
oparl = nixosSystem' {
modules = [
microvm.nixosModules.microvm
self.nixosModules.microvm
./hosts/containers/oparl
{ sops.defaultSopsFile = "${secrets}/hosts/oparl/secrets.yaml"; }
];
@ -595,14 +625,14 @@
leon = nixosSystem' {
modules = [
microvm.nixosModules.microvm
self.nixosModules.microvm
./hosts/containers/leon
];
};
nfsroot = nixosSystem' {
modules = [
microvm.nixosModules.microvm
self.nixosModules.microvm
./hosts/containers/nfsroot
];
};
@ -630,6 +660,10 @@
c3d2.users = import ./users.nix;
c3d2.nncp.neigh = import ./config/nncp-relays.nix;
};
microvm.imports = [
microvm.nixosModules.microvm
./modules/microvm.nix
];
nncp = ./modules/nncp.nix;
plume = {
imports = [ ./modules/plume.nix ];

40
hosts/containers/freifunk/default.nix
View File

@ -31,35 +31,6 @@ in {
"${modulesPath}/profiles/minimal.nix"
];
microvm = {
mem = 512;
hypervisor = "cloud-hypervisor";
shares = [ {
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
} ] ++ map (dir: {
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
mountPoint = "/${dir}";
tag = dir;
proto = "virtiofs";
socket = "${dir}.socket";
}) [ "etc" "home" "var"];
interfaces = [ {
type = "tap";
id = "freifunk-core";
mac = mac.core;
} {
type = "tap";
id = "freifunk-bmx";
mac = mac.bmx;
} ];
};
boot.tmpOnTmpfs = true;
boot.postBootCommands = ''
if [ ! -c /dev/net/tun ]; then
@ -70,6 +41,17 @@ in {
c3d2 = {
isInHq = false;
hq.statistics.enable = true;
deployment = {
server = "server10";
mounts = [ "etc" "home" "var"];
interfaces = [ {
net = "core";
mac = mac.core;
} {
net = "bmx";
mac = mac.bmx;
} ];
};
};
services.collectd.plugins.protocols = "";

21
hosts/containers/leon/default.nix
View File

@ -7,24 +7,13 @@ in
microvm = {
hypervisor = "qemu";
mem = 2048;
};
shares = [ {
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
} ] ++ map (dir: {
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
mountPoint = "/${dir}";
tag = dir;
proto = "virtiofs";
socket = "${dir}.socket";
}) [ "etc" "home" "var"];
c3d2.deployment = {
server = "server10";
mounts = [ "etc" "home" "var"];
interfaces = [ {
type = "tap";
id = "leon";
net = "serv";
mac = "00:de:8d:c1:9f:45";
} ];
};

22
hosts/containers/oparl/default.nix
View File

@ -13,24 +13,12 @@ in
{
microvm = {
hypervisor = "qemu";
shares = [ {
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
} ] ++ map (dir: {
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
mountPoint = "/${dir}";
tag = dir;
proto = "virtiofs";
socket = "${dir}.socket";
}) [ "etc" "home" "var"];
};
c3d2.deployment = {
server = "server10";
mounts = [ "etc" "home" "var"];
interfaces = [ {
type = "tap";
id = "oparl";
net = "serv";
mac = "00:de:9a:ce:38:79";
} ];
};

1
hosts/server10/default.nix
View File

@ -9,6 +9,7 @@
];
c3d2.hq.statistics.enable = true;
c3d2.deployment.microvmBaseZfsDataset = "server10/vm";
boot= {
loader.grub = {

69
hosts/server10/microvms.nix
View File

@ -1,18 +1,61 @@
{ config, lib, pkgs, ... }:
{
microvm.autostart = [ "oparl" "leon" "nfsroot" ];
c3d2.server.bridgePorts = {
serv = [ "oparl" "leon" "nfsroot" ];
core = [ "freifunk-core" ];
bmx = [ "freifunk-bmx" ];
options = with lib; {
c3d2.deployment.microvmBaseZfsDataset = mkOption {
type = types.str;
description = "Base ZFS dataset whereunder to create shares for MicroVMs.";
};
};
nix = {
# fetch github-prebuilt microvm-kernels
binaryCaches = [
"https://microvm.cachix.org"
];
binaryCachePublicKeys = [
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
];
config = {
microvm.autostart = [ "oparl" "leon" "nfsroot" "freifunk" ];
systemd.services."microvm-virtiofsd@" = {
requires = [ "microvm-zfs-datasets@%i.service" ];
};
systemd.services."microvm-zfs-datasets@" = {
description = "Create ZFS datasets for MicroVM '%i'";
before = [ "microvm-virtiofsd@%i.service" ];
after = [ "local-fs.target" ];
partOf = [ "microvm@%i.service" ];
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
WorkingDirectory = "/var/lib/microvms/%i";
SyslogIdentifier = "microvm-zfs-datasets@%i";
};
path = with pkgs; [ zfs ];
scriptArgs = "%i";
script = ''
zfsExists() {
zfs list $1 >/dev/null 2>/dev/null
}
NAME="$1"
BASE="${config.c3d2.deployment.microvmBaseZfsDataset}"
zfsExists $BASE || \
zfs create $BASE
zfsExists $BASE/$NAME || \
zfs create $BASE/$NAME
for d in current/share/microvm/virtiofs/*; do
SOURCE=$(cat $d/source)
TAG=$(basename $d)
MNT=$(pwd)$SOURCE
zfsExists $BASE/$NAME/$TAG || \
zfs create -o mountpoint=$MNT $BASE/$NAME/$TAG
done
'';
};
nix = {
# fetch github-prebuilt microvm-kernels
binaryCaches = [
"https://microvm.cachix.org"
];
binaryCachePublicKeys = [
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
];
};
};
}

16
hosts/server10/network.nix
View File

@ -123,17 +123,11 @@ in
# Attach eth*/bond0/VLAN to bridge
networkConfig.Bridge = net;
};
}) {} zwHostBridges) //
builtins.foldl' (result: net:
builtins.foldl' (result: port:
result // {
"${net}-${port}" = {
matchConfig.Name = port;
networkConfig.Bridge = net;
};
}
) result (config.c3d2.server.bridgePorts.${net})
) {} (builtins.attrNames config.c3d2.server.bridgePorts);
"ports-${net}" = {
matchConfig.Name = "${net}-*";
networkConfig.Bridge = net;
};
}) {} zwHostBridges);
};
};
}

118
modules/microvm.nix Normal file
View File

@ -0,0 +1,118 @@
{ zentralwerk, options, config, lib, pkgs, ... }:
let
inherit (config.c3d2.deployment) server;
serverFQDN = "${server}.cluster.zentralwerk.org";
serverConfig = zentralwerk.lib.config.site.hosts.${config.c3d2.deployment.server};
serverBridges = builtins.attrNames (
lib.filterAttrs (_: { type, ... }: type == "bridge")
serverConfig.interfaces
);
in
{
options.c3d2.deployment = with lib; {
server = mkOption {
type = types.enum [ "server10" ];
description = "Server that is supposed to host this MicroVM.";
};
interfaces = mkOption {
description = "Network interfaces";
type = types.listOf (types.submodule {
options = {
net = mkOption {
description = "Network to hook up to";
type = types.enum serverBridges;
};
mac = mkOption {
description = "Ethernet MAC address";
type = types.str;
};
};
});
};
mounts = mkOption {
description = "Persistent filesystems to create, without leading /.";
type = with types; listOf str;
default = [ "etc" ];
};
};
config.system.build = with pkgs; {
copyToServer = writeScript "copy-to-${server}" ''
#! ${runtimeShell} -e
nix copy --to ssh://root@${serverFQDN} $@
'';
runOnServer = writeScript "run-on-${server}" ''
#! ${runtimeShell} -e
ssh root@${serverFQDN} -- $@
'';
};
config = {
microvm = {
hypervisor = lib.mkDefault "cloud-hypervisor";
mem = lib.mkDefault 512;
interfaces = map ({ net, mac }: {
type = "tap";
id = builtins.substring 0 15 "${net}-${config.networking.hostName}";
inherit mac;
}) config.c3d2.deployment.interfaces;
shares = [ {
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "store";
proto = "virtiofs";
socket = "store.socket";
} ]
++ map (dir:
if lib.hasPrefix "/" dir
then throw "${dir} starts with a leading /. Just don't!"
else let
tag = builtins.replaceStrings ["/"] ["_"] dir;
in {
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
mountPoint = "/${dir}";
inherit tag;
proto = "virtiofs";
socket = "${tag}.socket";
}) config.c3d2.deployment.mounts;
};
systemd.services."microvm-zfs-datasets@" = {
description = "Create ZFS datasets for MicroVM '%i'";
before = [ "microvm-virtiofsd@%i.service" ];
after = [ "local-fs.target" ];
partOf = [ "microvm@%i.service" ];
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
WorkingDirectory = "/var/lib/microvms/%i";
SyslogIdentifier = "microvm-zfs-datasets@%i";
};
scriptArgs = "%i";
script = ''
zfsExists() {
zfs list $1 >/dev/null 2>/dev/null
}
NAME="$1"
BASE=$(hostname)/vm/$NAME
zfsExists $BASE || \
zfs create $BASE
for d in current/share/microvm/virtiofs/*; do
SOURCE=$(cat $d/source)
MNT=$(pwd)/$SOURCE
zfsExists $BASE/$d || \
zfs create -o mountpoint=$MNT $BASE/$d
done
'';
};
};
}