forked from c3d2/nix-config
nixosModules.microvm: init, and abstract existing microvms
This commit is contained in:
parent
9beb323752
commit
dbc3e08961
44
flake.nix
44
flake.nix
|
@ -217,7 +217,9 @@
|
|||
host = getHostAddr name;
|
||||
target = ''root@"${host}"'';
|
||||
rebuildArg = "--flake ${self}#${name}";
|
||||
profile = self.nixosConfigurations."${name}".config.system.build.toplevel;
|
||||
hostConfig = self.nixosConfigurations."${name}".config;
|
||||
profile = hostConfig.system.build.toplevel;
|
||||
selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config";
|
||||
in result // {
|
||||
# Generate a small script for copying this flake to the
|
||||
# remote machine and bulding and switching there.
|
||||
|
@ -241,6 +243,34 @@
|
|||
#!${pkgs.runtimeShell} -ex
|
||||
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
|
||||
'';
|
||||
} // lib.optionalAttrs (hostConfig ? c3d2.deployment.server) {
|
||||
"microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
${hostConfig.system.build.copyToServer} ${self} ${secrets}
|
||||
|
||||
${hostConfig.system.build.runOnServer} bash -e <<END
|
||||
mkdir -p /var/lib/microvms/${name}
|
||||
cd /var/lib/microvms/${name}
|
||||
nix build \
|
||||
-o current \
|
||||
${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
|
||||
echo '${selfRef}' > flake
|
||||
systemctl restart microvm@${name}.service
|
||||
END
|
||||
'';
|
||||
"microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}
|
||||
|
||||
${hostConfig.system.build.runOnServer} bash -e <<END
|
||||
mkdir -p /var/lib/microvms/${name}
|
||||
cd /var/lib/microvms/${name}
|
||||
rm -f current
|
||||
ln -s ${hostConfig.microvm.declaredRunner} current
|
||||
echo '${selfRef}' > flake
|
||||
systemctl restart microvm@${name}.service
|
||||
END
|
||||
'';
|
||||
}) {} (builtins.attrNames self.nixosConfigurations) //
|
||||
|
||||
builtins.foldl' (result: host:
|
||||
|
@ -314,7 +344,7 @@
|
|||
|
||||
freifunk = nixosSystem' {
|
||||
modules = [
|
||||
microvm.nixosModules.microvm
|
||||
self.nixosModules.microvm
|
||||
./hosts/containers/freifunk
|
||||
{
|
||||
nixpkgs.overlays = with secrets.overlays; [
|
||||
|
@ -587,7 +617,7 @@
|
|||
|
||||
oparl = nixosSystem' {
|
||||
modules = [
|
||||
microvm.nixosModules.microvm
|
||||
self.nixosModules.microvm
|
||||
./hosts/containers/oparl
|
||||
{ sops.defaultSopsFile = "${secrets}/hosts/oparl/secrets.yaml"; }
|
||||
];
|
||||
|
@ -595,14 +625,14 @@
|
|||
|
||||
leon = nixosSystem' {
|
||||
modules = [
|
||||
microvm.nixosModules.microvm
|
||||
self.nixosModules.microvm
|
||||
./hosts/containers/leon
|
||||
];
|
||||
};
|
||||
|
||||
nfsroot = nixosSystem' {
|
||||
modules = [
|
||||
microvm.nixosModules.microvm
|
||||
self.nixosModules.microvm
|
||||
./hosts/containers/nfsroot
|
||||
];
|
||||
};
|
||||
|
@ -630,6 +660,10 @@
|
|||
c3d2.users = import ./users.nix;
|
||||
c3d2.nncp.neigh = import ./config/nncp-relays.nix;
|
||||
};
|
||||
microvm.imports = [
|
||||
microvm.nixosModules.microvm
|
||||
./modules/microvm.nix
|
||||
];
|
||||
nncp = ./modules/nncp.nix;
|
||||
plume = {
|
||||
imports = [ ./modules/plume.nix ];
|
||||
|
|
|
@ -31,35 +31,6 @@ in {
|
|||
"${modulesPath}/profiles/minimal.nix"
|
||||
];
|
||||
|
||||
microvm = {
|
||||
mem = 512;
|
||||
hypervisor = "cloud-hypervisor";
|
||||
|
||||
shares = [ {
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "store";
|
||||
proto = "virtiofs";
|
||||
socket = "store.socket";
|
||||
} ] ++ map (dir: {
|
||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
||||
mountPoint = "/${dir}";
|
||||
tag = dir;
|
||||
proto = "virtiofs";
|
||||
socket = "${dir}.socket";
|
||||
}) [ "etc" "home" "var"];
|
||||
|
||||
interfaces = [ {
|
||||
type = "tap";
|
||||
id = "freifunk-core";
|
||||
mac = mac.core;
|
||||
} {
|
||||
type = "tap";
|
||||
id = "freifunk-bmx";
|
||||
mac = mac.bmx;
|
||||
} ];
|
||||
};
|
||||
|
||||
boot.tmpOnTmpfs = true;
|
||||
boot.postBootCommands = ''
|
||||
if [ ! -c /dev/net/tun ]; then
|
||||
|
@ -70,6 +41,17 @@ in {
|
|||
c3d2 = {
|
||||
isInHq = false;
|
||||
hq.statistics.enable = true;
|
||||
deployment = {
|
||||
server = "server10";
|
||||
mounts = [ "etc" "home" "var"];
|
||||
interfaces = [ {
|
||||
net = "core";
|
||||
mac = mac.core;
|
||||
} {
|
||||
net = "bmx";
|
||||
mac = mac.bmx;
|
||||
} ];
|
||||
};
|
||||
};
|
||||
services.collectd.plugins.protocols = "";
|
||||
|
||||
|
|
|
@ -7,24 +7,13 @@ in
|
|||
microvm = {
|
||||
hypervisor = "qemu";
|
||||
mem = 2048;
|
||||
};
|
||||
|
||||
shares = [ {
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "store";
|
||||
proto = "virtiofs";
|
||||
socket = "store.socket";
|
||||
} ] ++ map (dir: {
|
||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
||||
mountPoint = "/${dir}";
|
||||
tag = dir;
|
||||
proto = "virtiofs";
|
||||
socket = "${dir}.socket";
|
||||
}) [ "etc" "home" "var"];
|
||||
|
||||
c3d2.deployment = {
|
||||
server = "server10";
|
||||
mounts = [ "etc" "home" "var"];
|
||||
interfaces = [ {
|
||||
type = "tap";
|
||||
id = "leon";
|
||||
net = "serv";
|
||||
mac = "00:de:8d:c1:9f:45";
|
||||
} ];
|
||||
};
|
||||
|
|
|
@ -13,24 +13,12 @@ in
|
|||
{
|
||||
microvm = {
|
||||
hypervisor = "qemu";
|
||||
|
||||
shares = [ {
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "store";
|
||||
proto = "virtiofs";
|
||||
socket = "store.socket";
|
||||
} ] ++ map (dir: {
|
||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
||||
mountPoint = "/${dir}";
|
||||
tag = dir;
|
||||
proto = "virtiofs";
|
||||
socket = "${dir}.socket";
|
||||
}) [ "etc" "home" "var"];
|
||||
|
||||
};
|
||||
c3d2.deployment = {
|
||||
server = "server10";
|
||||
mounts = [ "etc" "home" "var"];
|
||||
interfaces = [ {
|
||||
type = "tap";
|
||||
id = "oparl";
|
||||
net = "serv";
|
||||
mac = "00:de:9a:ce:38:79";
|
||||
} ];
|
||||
};
|
||||
|
|
|
@ -9,6 +9,7 @@
|
|||
];
|
||||
|
||||
c3d2.hq.statistics.enable = true;
|
||||
c3d2.deployment.microvmBaseZfsDataset = "server10/vm";
|
||||
|
||||
boot= {
|
||||
loader.grub = {
|
||||
|
|
|
@ -1,18 +1,61 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
{
|
||||
microvm.autostart = [ "oparl" "leon" "nfsroot" ];
|
||||
c3d2.server.bridgePorts = {
|
||||
serv = [ "oparl" "leon" "nfsroot" ];
|
||||
core = [ "freifunk-core" ];
|
||||
bmx = [ "freifunk-bmx" ];
|
||||
options = with lib; {
|
||||
c3d2.deployment.microvmBaseZfsDataset = mkOption {
|
||||
type = types.str;
|
||||
description = "Base ZFS dataset whereunder to create shares for MicroVMs.";
|
||||
};
|
||||
};
|
||||
|
||||
nix = {
|
||||
# fetch github-prebuilt microvm-kernels
|
||||
binaryCaches = [
|
||||
"https://microvm.cachix.org"
|
||||
];
|
||||
binaryCachePublicKeys = [
|
||||
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
|
||||
];
|
||||
config = {
|
||||
microvm.autostart = [ "oparl" "leon" "nfsroot" "freifunk" ];
|
||||
|
||||
systemd.services."microvm-virtiofsd@" = {
|
||||
requires = [ "microvm-zfs-datasets@%i.service" ];
|
||||
};
|
||||
systemd.services."microvm-zfs-datasets@" = {
|
||||
description = "Create ZFS datasets for MicroVM '%i'";
|
||||
before = [ "microvm-virtiofsd@%i.service" ];
|
||||
after = [ "local-fs.target" ];
|
||||
partOf = [ "microvm@%i.service" ];
|
||||
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
WorkingDirectory = "/var/lib/microvms/%i";
|
||||
SyslogIdentifier = "microvm-zfs-datasets@%i";
|
||||
};
|
||||
path = with pkgs; [ zfs ];
|
||||
scriptArgs = "%i";
|
||||
script = ''
|
||||
zfsExists() {
|
||||
zfs list $1 >/dev/null 2>/dev/null
|
||||
}
|
||||
|
||||
NAME="$1"
|
||||
BASE="${config.c3d2.deployment.microvmBaseZfsDataset}"
|
||||
zfsExists $BASE || \
|
||||
zfs create $BASE
|
||||
zfsExists $BASE/$NAME || \
|
||||
zfs create $BASE/$NAME
|
||||
for d in current/share/microvm/virtiofs/*; do
|
||||
SOURCE=$(cat $d/source)
|
||||
TAG=$(basename $d)
|
||||
MNT=$(pwd)$SOURCE
|
||||
zfsExists $BASE/$NAME/$TAG || \
|
||||
zfs create -o mountpoint=$MNT $BASE/$NAME/$TAG
|
||||
done
|
||||
'';
|
||||
};
|
||||
|
||||
nix = {
|
||||
# fetch github-prebuilt microvm-kernels
|
||||
binaryCaches = [
|
||||
"https://microvm.cachix.org"
|
||||
];
|
||||
binaryCachePublicKeys = [
|
||||
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -123,17 +123,11 @@ in
|
|||
# Attach eth*/bond0/VLAN to bridge
|
||||
networkConfig.Bridge = net;
|
||||
};
|
||||
}) {} zwHostBridges) //
|
||||
builtins.foldl' (result: net:
|
||||
builtins.foldl' (result: port:
|
||||
result // {
|
||||
"${net}-${port}" = {
|
||||
matchConfig.Name = port;
|
||||
networkConfig.Bridge = net;
|
||||
};
|
||||
}
|
||||
) result (config.c3d2.server.bridgePorts.${net})
|
||||
) {} (builtins.attrNames config.c3d2.server.bridgePorts);
|
||||
"ports-${net}" = {
|
||||
matchConfig.Name = "${net}-*";
|
||||
networkConfig.Bridge = net;
|
||||
};
|
||||
}) {} zwHostBridges);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
@ -0,0 +1,118 @@
|
|||
{ zentralwerk, options, config, lib, pkgs, ... }:
|
||||
let
|
||||
inherit (config.c3d2.deployment) server;
|
||||
serverFQDN = "${server}.cluster.zentralwerk.org";
|
||||
serverConfig = zentralwerk.lib.config.site.hosts.${config.c3d2.deployment.server};
|
||||
serverBridges = builtins.attrNames (
|
||||
lib.filterAttrs (_: { type, ... }: type == "bridge")
|
||||
serverConfig.interfaces
|
||||
);
|
||||
in
|
||||
{
|
||||
options.c3d2.deployment = with lib; {
|
||||
server = mkOption {
|
||||
type = types.enum [ "server10" ];
|
||||
description = "Server that is supposed to host this MicroVM.";
|
||||
};
|
||||
|
||||
interfaces = mkOption {
|
||||
description = "Network interfaces";
|
||||
type = types.listOf (types.submodule {
|
||||
options = {
|
||||
net = mkOption {
|
||||
description = "Network to hook up to";
|
||||
type = types.enum serverBridges;
|
||||
};
|
||||
mac = mkOption {
|
||||
description = "Ethernet MAC address";
|
||||
type = types.str;
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
mounts = mkOption {
|
||||
description = "Persistent filesystems to create, without leading /.";
|
||||
type = with types; listOf str;
|
||||
default = [ "etc" ];
|
||||
};
|
||||
};
|
||||
|
||||
config.system.build = with pkgs; {
|
||||
copyToServer = writeScript "copy-to-${server}" ''
|
||||
#! ${runtimeShell} -e
|
||||
|
||||
nix copy --to ssh://root@${serverFQDN} $@
|
||||
'';
|
||||
|
||||
runOnServer = writeScript "run-on-${server}" ''
|
||||
#! ${runtimeShell} -e
|
||||
|
||||
ssh root@${serverFQDN} -- $@
|
||||
'';
|
||||
};
|
||||
|
||||
config = {
|
||||
microvm = {
|
||||
hypervisor = lib.mkDefault "cloud-hypervisor";
|
||||
mem = lib.mkDefault 512;
|
||||
|
||||
interfaces = map ({ net, mac }: {
|
||||
type = "tap";
|
||||
id = builtins.substring 0 15 "${net}-${config.networking.hostName}";
|
||||
inherit mac;
|
||||
}) config.c3d2.deployment.interfaces;
|
||||
|
||||
shares = [ {
|
||||
source = "/nix/store";
|
||||
mountPoint = "/nix/.ro-store";
|
||||
tag = "store";
|
||||
proto = "virtiofs";
|
||||
socket = "store.socket";
|
||||
} ]
|
||||
++ map (dir:
|
||||
if lib.hasPrefix "/" dir
|
||||
then throw "${dir} starts with a leading /. Just don't!"
|
||||
else let
|
||||
tag = builtins.replaceStrings ["/"] ["_"] dir;
|
||||
in {
|
||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
||||
mountPoint = "/${dir}";
|
||||
inherit tag;
|
||||
proto = "virtiofs";
|
||||
socket = "${tag}.socket";
|
||||
}) config.c3d2.deployment.mounts;
|
||||
};
|
||||
|
||||
systemd.services."microvm-zfs-datasets@" = {
|
||||
description = "Create ZFS datasets for MicroVM '%i'";
|
||||
before = [ "microvm-virtiofsd@%i.service" ];
|
||||
after = [ "local-fs.target" ];
|
||||
partOf = [ "microvm@%i.service" ];
|
||||
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
WorkingDirectory = "/var/lib/microvms/%i";
|
||||
SyslogIdentifier = "microvm-zfs-datasets@%i";
|
||||
};
|
||||
scriptArgs = "%i";
|
||||
script = ''
|
||||
zfsExists() {
|
||||
zfs list $1 >/dev/null 2>/dev/null
|
||||
}
|
||||
|
||||
NAME="$1"
|
||||
BASE=$(hostname)/vm/$NAME
|
||||
zfsExists $BASE || \
|
||||
zfs create $BASE
|
||||
for d in current/share/microvm/virtiofs/*; do
|
||||
SOURCE=$(cat $d/source)
|
||||
MNT=$(pwd)/$SOURCE
|
||||
zfsExists $BASE/$d || \
|
||||
zfs create -o mountpoint=$MNT $BASE/$d
|
||||
done
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
Loading…
Reference in New Issue