network/nix/nixos-module/server/lxc-containers.nix

220 lines
6.3 KiB
Nix

{ hostName, self, config, lib, pkgs, ... }:
let
# Containers that are run on this host
containers =
lib.filterAttrs (_: { role, model, location, ... }:
role == "container" &&
model == "lxc" &&
location == hostName
) config.site.hosts;
enabled = containers != {};
# `lxc.net.*` formatter for lxc.container.conf files
netConfig = ctName: interfaces:
let
config = map (netName:
let
ifData = interfaces.${netName};
in {
type = ifData.type;
name = netName;
flags = "up";
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
then ifData.hwaddr
else "0A:14:48:xx:xx:xx";
} // (lib.optionalAttrs (ifData.type == "veth") {
veth.pair = "${ctName}-${netName}";
veth.mode = "bridge";
link = "${netName}";
}) // (lib.optionalAttrs (ifData.type == "phys") {
link = "ext-${netName}";
})
) (builtins.attrNames interfaces);
serialize = name: x:
if builtins.isString x
then "${name} = ${x}\n"
else if builtins.isAttrs x
then builtins.concatStringsSep "" (
map (n: serialize "${name}.${n}" x.${n}) (builtins.attrNames x)
)
else if builtins.isList x
then
let
enumerate = xs: n:
if xs == []
then []
else [ {
e = builtins.head xs;
i = n;
} ] ++ enumerate (builtins.tail xs) (n + 1);
in
builtins.concatStringsSep "" (
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
)
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
in
serialize "lxc.net" config;
# User-facing script to build/update container NixOS systems
build-script = pkgs.writeScriptBin "build-container" ''
#! ${pkgs.runtimeShell} -e
if [ -z "$1" ]; then
echo "Usage: $0 [containers ...]"
exit 1
fi
mkdir -p /nix/var/nix/gcroots/lxc
for c in $@; do
unset SYSTEM
case "$c" in
${builtins.concatStringsSep "\n" (
map (ctName: ''
${ctName})
echo Using prebuilt system for container $c
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
;;
'') (
builtins.attrNames (
lib.filterAttrs (_: { prebuilt, ... }: prebuilt)
containers
))
)}
*)
echo Building $c
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
;;
esac
echo Installing $c
for d in \
bin dev etc home mnt \
nix/store nix/var \
proc root run sys tmp var usr ; \
do
mkdir -p /var/lib/lxc/$c/rootfs/$d
done
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
set +e
active=$(systemctl is-active lxc@$c)
if [[ "$active" = active ]] ; then
echo Activating $c
systemctl reload lxc@$c || (
echo Reload failed. Restarting $c
systemctl restart lxc@$c
)
else
echo Starting $c
systemctl start lxc@$c
fi
set -e
done
'';
in
{
virtualisation.lxc = lib.mkIf enabled {
enable = true;
# Container configs live in /etc so that they can be created
# through `environment.etc`.
systemConfig = ''
lxc.lxcpath = /etc/lxc/containers
'';
};
environment.systemPackages = [ pkgs.lxc build-script ];
# Create lxc.container.conf files
environment.etc =
builtins.foldl' (etc: ctName: etc // {
"lxc/containers/${ctName}/config" = {
enable = true;
source =
let
inherit (containers.${ctName}) interfaces;
in builtins.toFile "${ctName}.conf" ''
# For lxcfs and sane defaults
lxc.include = /etc/lxc/common.conf
lxc.uts.name = ${ctName}
# Handled by lxc@.service
lxc.start.auto = 0
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
lxc.init.cmd = "/init"
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
lxc.mount.entry = none tmp tmpfs defaults 0 0
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
lxc.autodev = 1
lxc.tty.max = 0
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio sys_time mknod
lxc.apparmor.profile = unchanged
security.privileged = false
lxc.cgroup.memory.limit_in_bytes = 1G
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
# tuntap
lxc.cgroup.devices.allow = c 10:200 rw
${netConfig ctName interfaces}
'';
};
}) {
"lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
} (builtins.attrNames containers);
# Systemd service template for LXC containers
systemd.services."lxc@" = {
description = "LXC container '%i'";
after = [ "network.target" ];
serviceConfig = {
Type = "simple";
ExecStart =
let
script = pkgs.writeScript "start-lxc-container.sh" ''
#! ${pkgs.runtimeShell} -e
if [ ! -e /var/lib/lxc/$1/rootfs ]; then
echo "Container $1 does not exist. Run: build-container $1"
exit 1
fi
exec ${pkgs.lxc}/bin/lxc-start -F -C -n $1
'';
in
"${script} %i";
ExecStop = "${pkgs.lxc}/bin/lxc-stop -n %i";
ExecReload =
let
script = pkgs.writeScript "reload-lxc-container.sh" ''
#! ${pkgs.runtimeShell} -e
SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init))
exec ${pkgs.lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch
'';
in
"${script} %i";
KillMode = "mixed";
OOMPolicy = "kill";
Restart = "always";
RestartSec = "30s";
};
};
# Starts all the containers after boot
systemd.targets.lxc-containers = {
wantedBy = [ "multi-user.target" ];
wants = map (ctName: "lxc@${ctName}.service")
(builtins.attrNames containers);
};
}