nixosModules.microvm: init, and abstract existing microvms
This commit is contained in:
parent
9beb323752
commit
dbc3e08961
44
flake.nix
44
flake.nix
|
@ -217,7 +217,9 @@
|
||||||
host = getHostAddr name;
|
host = getHostAddr name;
|
||||||
target = ''root@"${host}"'';
|
target = ''root@"${host}"'';
|
||||||
rebuildArg = "--flake ${self}#${name}";
|
rebuildArg = "--flake ${self}#${name}";
|
||||||
profile = self.nixosConfigurations."${name}".config.system.build.toplevel;
|
hostConfig = self.nixosConfigurations."${name}".config;
|
||||||
|
profile = hostConfig.system.build.toplevel;
|
||||||
|
selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config";
|
||||||
in result // {
|
in result // {
|
||||||
# Generate a small script for copying this flake to the
|
# Generate a small script for copying this flake to the
|
||||||
# remote machine and bulding and switching there.
|
# remote machine and bulding and switching there.
|
||||||
|
@ -241,6 +243,34 @@
|
||||||
#!${pkgs.runtimeShell} -ex
|
#!${pkgs.runtimeShell} -ex
|
||||||
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
|
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
|
||||||
'';
|
'';
|
||||||
|
} // lib.optionalAttrs (hostConfig ? c3d2.deployment.server) {
|
||||||
|
"microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
|
||||||
|
#!${pkgs.runtimeShell} -e
|
||||||
|
${hostConfig.system.build.copyToServer} ${self} ${secrets}
|
||||||
|
|
||||||
|
${hostConfig.system.build.runOnServer} bash -e <<END
|
||||||
|
mkdir -p /var/lib/microvms/${name}
|
||||||
|
cd /var/lib/microvms/${name}
|
||||||
|
nix build \
|
||||||
|
-o current \
|
||||||
|
${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
|
||||||
|
echo '${selfRef}' > flake
|
||||||
|
systemctl restart microvm@${name}.service
|
||||||
|
END
|
||||||
|
'';
|
||||||
|
"microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
|
||||||
|
#!${pkgs.runtimeShell} -e
|
||||||
|
${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}
|
||||||
|
|
||||||
|
${hostConfig.system.build.runOnServer} bash -e <<END
|
||||||
|
mkdir -p /var/lib/microvms/${name}
|
||||||
|
cd /var/lib/microvms/${name}
|
||||||
|
rm -f current
|
||||||
|
ln -s ${hostConfig.microvm.declaredRunner} current
|
||||||
|
echo '${selfRef}' > flake
|
||||||
|
systemctl restart microvm@${name}.service
|
||||||
|
END
|
||||||
|
'';
|
||||||
}) {} (builtins.attrNames self.nixosConfigurations) //
|
}) {} (builtins.attrNames self.nixosConfigurations) //
|
||||||
|
|
||||||
builtins.foldl' (result: host:
|
builtins.foldl' (result: host:
|
||||||
|
@ -314,7 +344,7 @@
|
||||||
|
|
||||||
freifunk = nixosSystem' {
|
freifunk = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
microvm.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/containers/freifunk
|
./hosts/containers/freifunk
|
||||||
{
|
{
|
||||||
nixpkgs.overlays = with secrets.overlays; [
|
nixpkgs.overlays = with secrets.overlays; [
|
||||||
|
@ -587,7 +617,7 @@
|
||||||
|
|
||||||
oparl = nixosSystem' {
|
oparl = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
microvm.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/containers/oparl
|
./hosts/containers/oparl
|
||||||
{ sops.defaultSopsFile = "${secrets}/hosts/oparl/secrets.yaml"; }
|
{ sops.defaultSopsFile = "${secrets}/hosts/oparl/secrets.yaml"; }
|
||||||
];
|
];
|
||||||
|
@ -595,14 +625,14 @@
|
||||||
|
|
||||||
leon = nixosSystem' {
|
leon = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
microvm.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/containers/leon
|
./hosts/containers/leon
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
nfsroot = nixosSystem' {
|
nfsroot = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
microvm.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/containers/nfsroot
|
./hosts/containers/nfsroot
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
@ -630,6 +660,10 @@
|
||||||
c3d2.users = import ./users.nix;
|
c3d2.users = import ./users.nix;
|
||||||
c3d2.nncp.neigh = import ./config/nncp-relays.nix;
|
c3d2.nncp.neigh = import ./config/nncp-relays.nix;
|
||||||
};
|
};
|
||||||
|
microvm.imports = [
|
||||||
|
microvm.nixosModules.microvm
|
||||||
|
./modules/microvm.nix
|
||||||
|
];
|
||||||
nncp = ./modules/nncp.nix;
|
nncp = ./modules/nncp.nix;
|
||||||
plume = {
|
plume = {
|
||||||
imports = [ ./modules/plume.nix ];
|
imports = [ ./modules/plume.nix ];
|
||||||
|
|
|
@ -31,35 +31,6 @@ in {
|
||||||
"${modulesPath}/profiles/minimal.nix"
|
"${modulesPath}/profiles/minimal.nix"
|
||||||
];
|
];
|
||||||
|
|
||||||
microvm = {
|
|
||||||
mem = 512;
|
|
||||||
hypervisor = "cloud-hypervisor";
|
|
||||||
|
|
||||||
shares = [ {
|
|
||||||
source = "/nix/store";
|
|
||||||
mountPoint = "/nix/.ro-store";
|
|
||||||
tag = "store";
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "store.socket";
|
|
||||||
} ] ++ map (dir: {
|
|
||||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
|
||||||
mountPoint = "/${dir}";
|
|
||||||
tag = dir;
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "${dir}.socket";
|
|
||||||
}) [ "etc" "home" "var"];
|
|
||||||
|
|
||||||
interfaces = [ {
|
|
||||||
type = "tap";
|
|
||||||
id = "freifunk-core";
|
|
||||||
mac = mac.core;
|
|
||||||
} {
|
|
||||||
type = "tap";
|
|
||||||
id = "freifunk-bmx";
|
|
||||||
mac = mac.bmx;
|
|
||||||
} ];
|
|
||||||
};
|
|
||||||
|
|
||||||
boot.tmpOnTmpfs = true;
|
boot.tmpOnTmpfs = true;
|
||||||
boot.postBootCommands = ''
|
boot.postBootCommands = ''
|
||||||
if [ ! -c /dev/net/tun ]; then
|
if [ ! -c /dev/net/tun ]; then
|
||||||
|
@ -70,6 +41,17 @@ in {
|
||||||
c3d2 = {
|
c3d2 = {
|
||||||
isInHq = false;
|
isInHq = false;
|
||||||
hq.statistics.enable = true;
|
hq.statistics.enable = true;
|
||||||
|
deployment = {
|
||||||
|
server = "server10";
|
||||||
|
mounts = [ "etc" "home" "var"];
|
||||||
|
interfaces = [ {
|
||||||
|
net = "core";
|
||||||
|
mac = mac.core;
|
||||||
|
} {
|
||||||
|
net = "bmx";
|
||||||
|
mac = mac.bmx;
|
||||||
|
} ];
|
||||||
|
};
|
||||||
};
|
};
|
||||||
services.collectd.plugins.protocols = "";
|
services.collectd.plugins.protocols = "";
|
||||||
|
|
||||||
|
|
|
@ -7,24 +7,13 @@ in
|
||||||
microvm = {
|
microvm = {
|
||||||
hypervisor = "qemu";
|
hypervisor = "qemu";
|
||||||
mem = 2048;
|
mem = 2048;
|
||||||
|
};
|
||||||
|
|
||||||
shares = [ {
|
c3d2.deployment = {
|
||||||
source = "/nix/store";
|
server = "server10";
|
||||||
mountPoint = "/nix/.ro-store";
|
mounts = [ "etc" "home" "var"];
|
||||||
tag = "store";
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "store.socket";
|
|
||||||
} ] ++ map (dir: {
|
|
||||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
|
||||||
mountPoint = "/${dir}";
|
|
||||||
tag = dir;
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "${dir}.socket";
|
|
||||||
}) [ "etc" "home" "var"];
|
|
||||||
|
|
||||||
interfaces = [ {
|
interfaces = [ {
|
||||||
type = "tap";
|
net = "serv";
|
||||||
id = "leon";
|
|
||||||
mac = "00:de:8d:c1:9f:45";
|
mac = "00:de:8d:c1:9f:45";
|
||||||
} ];
|
} ];
|
||||||
};
|
};
|
||||||
|
|
|
@ -13,24 +13,12 @@ in
|
||||||
{
|
{
|
||||||
microvm = {
|
microvm = {
|
||||||
hypervisor = "qemu";
|
hypervisor = "qemu";
|
||||||
|
};
|
||||||
shares = [ {
|
c3d2.deployment = {
|
||||||
source = "/nix/store";
|
server = "server10";
|
||||||
mountPoint = "/nix/.ro-store";
|
mounts = [ "etc" "home" "var"];
|
||||||
tag = "store";
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "store.socket";
|
|
||||||
} ] ++ map (dir: {
|
|
||||||
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
|
||||||
mountPoint = "/${dir}";
|
|
||||||
tag = dir;
|
|
||||||
proto = "virtiofs";
|
|
||||||
socket = "${dir}.socket";
|
|
||||||
}) [ "etc" "home" "var"];
|
|
||||||
|
|
||||||
interfaces = [ {
|
interfaces = [ {
|
||||||
type = "tap";
|
net = "serv";
|
||||||
id = "oparl";
|
|
||||||
mac = "00:de:9a:ce:38:79";
|
mac = "00:de:9a:ce:38:79";
|
||||||
} ];
|
} ];
|
||||||
};
|
};
|
||||||
|
|
|
@ -9,6 +9,7 @@
|
||||||
];
|
];
|
||||||
|
|
||||||
c3d2.hq.statistics.enable = true;
|
c3d2.hq.statistics.enable = true;
|
||||||
|
c3d2.deployment.microvmBaseZfsDataset = "server10/vm";
|
||||||
|
|
||||||
boot= {
|
boot= {
|
||||||
loader.grub = {
|
loader.grub = {
|
||||||
|
|
|
@ -1,18 +1,61 @@
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
{
|
{
|
||||||
microvm.autostart = [ "oparl" "leon" "nfsroot" ];
|
options = with lib; {
|
||||||
c3d2.server.bridgePorts = {
|
c3d2.deployment.microvmBaseZfsDataset = mkOption {
|
||||||
serv = [ "oparl" "leon" "nfsroot" ];
|
type = types.str;
|
||||||
core = [ "freifunk-core" ];
|
description = "Base ZFS dataset whereunder to create shares for MicroVMs.";
|
||||||
bmx = [ "freifunk-bmx" ];
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
nix = {
|
config = {
|
||||||
# fetch github-prebuilt microvm-kernels
|
microvm.autostart = [ "oparl" "leon" "nfsroot" "freifunk" ];
|
||||||
binaryCaches = [
|
|
||||||
"https://microvm.cachix.org"
|
systemd.services."microvm-virtiofsd@" = {
|
||||||
];
|
requires = [ "microvm-zfs-datasets@%i.service" ];
|
||||||
binaryCachePublicKeys = [
|
};
|
||||||
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
|
systemd.services."microvm-zfs-datasets@" = {
|
||||||
];
|
description = "Create ZFS datasets for MicroVM '%i'";
|
||||||
|
before = [ "microvm-virtiofsd@%i.service" ];
|
||||||
|
after = [ "local-fs.target" ];
|
||||||
|
partOf = [ "microvm@%i.service" ];
|
||||||
|
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
WorkingDirectory = "/var/lib/microvms/%i";
|
||||||
|
SyslogIdentifier = "microvm-zfs-datasets@%i";
|
||||||
|
};
|
||||||
|
path = with pkgs; [ zfs ];
|
||||||
|
scriptArgs = "%i";
|
||||||
|
script = ''
|
||||||
|
zfsExists() {
|
||||||
|
zfs list $1 >/dev/null 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
NAME="$1"
|
||||||
|
BASE="${config.c3d2.deployment.microvmBaseZfsDataset}"
|
||||||
|
zfsExists $BASE || \
|
||||||
|
zfs create $BASE
|
||||||
|
zfsExists $BASE/$NAME || \
|
||||||
|
zfs create $BASE/$NAME
|
||||||
|
for d in current/share/microvm/virtiofs/*; do
|
||||||
|
SOURCE=$(cat $d/source)
|
||||||
|
TAG=$(basename $d)
|
||||||
|
MNT=$(pwd)$SOURCE
|
||||||
|
zfsExists $BASE/$NAME/$TAG || \
|
||||||
|
zfs create -o mountpoint=$MNT $BASE/$NAME/$TAG
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
nix = {
|
||||||
|
# fetch github-prebuilt microvm-kernels
|
||||||
|
binaryCaches = [
|
||||||
|
"https://microvm.cachix.org"
|
||||||
|
];
|
||||||
|
binaryCachePublicKeys = [
|
||||||
|
"microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
|
||||||
|
];
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -123,17 +123,11 @@ in
|
||||||
# Attach eth*/bond0/VLAN to bridge
|
# Attach eth*/bond0/VLAN to bridge
|
||||||
networkConfig.Bridge = net;
|
networkConfig.Bridge = net;
|
||||||
};
|
};
|
||||||
}) {} zwHostBridges) //
|
"ports-${net}" = {
|
||||||
builtins.foldl' (result: net:
|
matchConfig.Name = "${net}-*";
|
||||||
builtins.foldl' (result: port:
|
networkConfig.Bridge = net;
|
||||||
result // {
|
};
|
||||||
"${net}-${port}" = {
|
}) {} zwHostBridges);
|
||||||
matchConfig.Name = port;
|
|
||||||
networkConfig.Bridge = net;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
) result (config.c3d2.server.bridgePorts.${net})
|
|
||||||
) {} (builtins.attrNames config.c3d2.server.bridgePorts);
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,118 @@
|
||||||
|
{ zentralwerk, options, config, lib, pkgs, ... }:
|
||||||
|
let
|
||||||
|
inherit (config.c3d2.deployment) server;
|
||||||
|
serverFQDN = "${server}.cluster.zentralwerk.org";
|
||||||
|
serverConfig = zentralwerk.lib.config.site.hosts.${config.c3d2.deployment.server};
|
||||||
|
serverBridges = builtins.attrNames (
|
||||||
|
lib.filterAttrs (_: { type, ... }: type == "bridge")
|
||||||
|
serverConfig.interfaces
|
||||||
|
);
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.c3d2.deployment = with lib; {
|
||||||
|
server = mkOption {
|
||||||
|
type = types.enum [ "server10" ];
|
||||||
|
description = "Server that is supposed to host this MicroVM.";
|
||||||
|
};
|
||||||
|
|
||||||
|
interfaces = mkOption {
|
||||||
|
description = "Network interfaces";
|
||||||
|
type = types.listOf (types.submodule {
|
||||||
|
options = {
|
||||||
|
net = mkOption {
|
||||||
|
description = "Network to hook up to";
|
||||||
|
type = types.enum serverBridges;
|
||||||
|
};
|
||||||
|
mac = mkOption {
|
||||||
|
description = "Ethernet MAC address";
|
||||||
|
type = types.str;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
mounts = mkOption {
|
||||||
|
description = "Persistent filesystems to create, without leading /.";
|
||||||
|
type = with types; listOf str;
|
||||||
|
default = [ "etc" ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config.system.build = with pkgs; {
|
||||||
|
copyToServer = writeScript "copy-to-${server}" ''
|
||||||
|
#! ${runtimeShell} -e
|
||||||
|
|
||||||
|
nix copy --to ssh://root@${serverFQDN} $@
|
||||||
|
'';
|
||||||
|
|
||||||
|
runOnServer = writeScript "run-on-${server}" ''
|
||||||
|
#! ${runtimeShell} -e
|
||||||
|
|
||||||
|
ssh root@${serverFQDN} -- $@
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
config = {
|
||||||
|
microvm = {
|
||||||
|
hypervisor = lib.mkDefault "cloud-hypervisor";
|
||||||
|
mem = lib.mkDefault 512;
|
||||||
|
|
||||||
|
interfaces = map ({ net, mac }: {
|
||||||
|
type = "tap";
|
||||||
|
id = builtins.substring 0 15 "${net}-${config.networking.hostName}";
|
||||||
|
inherit mac;
|
||||||
|
}) config.c3d2.deployment.interfaces;
|
||||||
|
|
||||||
|
shares = [ {
|
||||||
|
source = "/nix/store";
|
||||||
|
mountPoint = "/nix/.ro-store";
|
||||||
|
tag = "store";
|
||||||
|
proto = "virtiofs";
|
||||||
|
socket = "store.socket";
|
||||||
|
} ]
|
||||||
|
++ map (dir:
|
||||||
|
if lib.hasPrefix "/" dir
|
||||||
|
then throw "${dir} starts with a leading /. Just don't!"
|
||||||
|
else let
|
||||||
|
tag = builtins.replaceStrings ["/"] ["_"] dir;
|
||||||
|
in {
|
||||||
|
source = "/var/lib/microvms/${config.networking.hostName}/${dir}";
|
||||||
|
mountPoint = "/${dir}";
|
||||||
|
inherit tag;
|
||||||
|
proto = "virtiofs";
|
||||||
|
socket = "${tag}.socket";
|
||||||
|
}) config.c3d2.deployment.mounts;
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.services."microvm-zfs-datasets@" = {
|
||||||
|
description = "Create ZFS datasets for MicroVM '%i'";
|
||||||
|
before = [ "microvm-virtiofsd@%i.service" ];
|
||||||
|
after = [ "local-fs.target" ];
|
||||||
|
partOf = [ "microvm@%i.service" ];
|
||||||
|
unitConfig.ConditionPathExists = "/var/lib/microvms/%i/current/share/microvm/virtiofs";
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
WorkingDirectory = "/var/lib/microvms/%i";
|
||||||
|
SyslogIdentifier = "microvm-zfs-datasets@%i";
|
||||||
|
};
|
||||||
|
scriptArgs = "%i";
|
||||||
|
script = ''
|
||||||
|
zfsExists() {
|
||||||
|
zfs list $1 >/dev/null 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
NAME="$1"
|
||||||
|
BASE=$(hostname)/vm/$NAME
|
||||||
|
zfsExists $BASE || \
|
||||||
|
zfs create $BASE
|
||||||
|
for d in current/share/microvm/virtiofs/*; do
|
||||||
|
SOURCE=$(cat $d/source)
|
||||||
|
MNT=$(pwd)/$SOURCE
|
||||||
|
zfsExists $BASE/$d || \
|
||||||
|
zfs create -o mountpoint=$MNT $BASE/$d
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
Loading…
Reference in New Issue