network/nix/nixos-module/server/lxc-containers.nix

260 lines
7.6 KiB
Nix
Raw Normal View History

2022-03-22 18:13:17 +01:00
{ self, config, lib, pkgs, ... }:
2021-03-22 23:37:25 +01:00
let
2021-04-10 14:52:13 +02:00
# Containers that are run on this host
2021-03-22 23:37:25 +01:00
containers =
lib.filterAttrs (_: { role, model, ... }:
2021-03-23 00:40:40 +01:00
role == "container" &&
model == "lxc"
) config.site.hosts;
2021-03-23 00:40:40 +01:00
2021-03-22 23:37:25 +01:00
enabled = containers != {};
# linux iface name max length = 15
shortenNetName = name:
if builtins.match "priv(.*)" name != null
then "p" + builtins.substring 4 9 name
else name;
checkIfname = ifname: let
len = builtins.stringLength ifname;
in if len > 15
then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
else ifname;
2021-04-10 14:52:13 +02:00
# `lxc.net.*` formatter for lxc.container.conf files
netConfig = ctName: interfaces:
let
config = map (netName:
let
ifData = interfaces.${netName};
in {
type = ifData.type;
name = checkIfname netName;
flags = "up";
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
then ifData.hwaddr
else "0A:14:48:xx:xx:xx";
} // (lib.optionalAttrs (ifData.type == "veth") {
veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
veth.mode = checkIfname "bridge";
link = checkIfname netName;
}) // (lib.optionalAttrs (ifData.type == "phys") {
link = checkIfname "ext-${netName}";
})
) (builtins.attrNames interfaces);
attrNamesOrdered = attrs:
if attrs ? type
then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
else builtins.attrNames attrs;
serialize = name: x:
if builtins.isString x
then "${name} = ${x}\n"
else if builtins.isAttrs x
then builtins.concatStringsSep "" (
map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
)
else if builtins.isList x
then
let
enumerate = xs: n:
if xs == []
then []
else [ {
e = builtins.head xs;
i = n;
} ] ++ enumerate (builtins.tail xs) (n + 1);
in
builtins.concatStringsSep "" (
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
)
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
in
2021-03-23 00:40:40 +01:00
serialize "lxc.net" config;
2021-04-10 14:52:13 +02:00
# User-facing script to build/update container NixOS systems
build-script = pkgs.writeScriptBin "build-container" ''
#! ${pkgs.runtimeShell} -e
if [ -z "$1" ]; then
echo "Usage: $0 [containers ...]"
exit 1
fi
mkdir -p /nix/var/nix/gcroots/lxc
for c in $@; do
unset SYSTEM
case "$c" in
${builtins.concatStringsSep "\n" (
map (ctName: ''
${ctName})
echo Using prebuilt system for container $c
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
;;
'') (
builtins.attrNames (
lib.filterAttrs (_: { prebuilt, ... }: prebuilt)
containers
))
)}
*)
echo Building $c
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
;;
esac
echo Installing $c
for d in \
bin dev etc home mnt \
nix/store nix/var \
proc root run sys tmp var usr ; \
do
mkdir -p /var/lib/lxc/$c/rootfs/$d
done
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
done
# Activate all the desired container after all of them are
# built
set +e
for c in $@; do
active=$(systemctl is-active lxc@$c)
if [[ "$active" = active ]] ; then
echo Activating $c
systemctl reload lxc@$c || (
echo Reload failed. Restarting $c
systemctl restart lxc@$c
)
else
echo Starting $c
systemctl start lxc@$c
fi
done
set -e
'';
enable-script = pkgs.writeScriptBin "enable-containers" ''
touch /etc/start-containers
systemctl start lxc-containers.target
'';
disable-script = pkgs.writeScriptBin "disable-containers" ''
rm /etc/start-containers
systemctl stop lxc-containers.target lxc@\*.service
'';
2021-03-22 23:37:25 +01:00
in
{
boot.kernel.sysctl = lib.mkIf enabled {
"fs.inotify.max_queued_events" = 1048576;
"fs.inotify.max_user_instances" = 1048576;
"fs.inotify.max_user_watches" = 1048576;
"vm.max_map_count" = 262144;
"kernel.dmesg_restrict" = 1;
"net.ipv4.neigh.default.gc_thresh3" = 8192;
"net.ipv6.neigh.default.gc_thresh3" = 8192;
"kernel.keys.maxkeys" = 2000;
};
2021-03-22 23:37:25 +01:00
virtualisation.lxc = lib.mkIf enabled {
enable = true;
2021-04-10 14:52:13 +02:00
# Container configs live in /etc so that they can be created
# through `environment.etc`.
2021-03-22 23:37:25 +01:00
systemConfig = ''
lxc.lxcpath = /etc/lxc/containers
'';
};
environment.systemPackages = [
# `lxc-attach` et al
pkgs.lxc build-script
# User scripts
enable-script disable-script
];
2021-03-22 23:37:25 +01:00
2021-04-10 14:52:13 +02:00
# Create lxc.container.conf files
2021-03-22 23:37:25 +01:00
environment.etc =
builtins.foldl' (etc: ctName: etc // {
"lxc/containers/${ctName}/config" = {
enable = true;
source =
2021-05-31 00:06:56 +02:00
builtins.toFile "${ctName}.conf" ''
2021-03-22 23:37:25 +01:00
# For lxcfs and sane defaults
lxc.include = /etc/lxc/common.conf
lxc.uts.name = ${ctName}
# Handled by lxc@.service
lxc.start.auto = 0
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
2021-03-22 23:37:25 +01:00
lxc.init.cmd = "/init"
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
lxc.mount.entry = none tmp tmpfs defaults 0 0
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
lxc.autodev = 1
lxc.tty.max = 0
lxc.pty.max = 8
2021-03-22 23:37:25 +01:00
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
2021-03-22 23:37:25 +01:00
security.privileged = false
lxc.apparmor.profile = lxc-container-default-with-mounting
2021-03-22 23:37:25 +01:00
lxc.cgroup.memory.limit_in_bytes = 1G
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
# tuntap
lxc.cgroup.devices.allow = c 10:200 rw
lxc.cgroup2.devices.allow = c 10:200 rw
# ppp
lxc.cgroup.devices.allow = c 108:0 rwm
lxc.cgroup2.devices.allow = c 108:0 rwm
2021-03-22 23:37:25 +01:00
2021-05-31 00:06:56 +02:00
${netConfig ctName containers.${ctName}.physicalInterfaces}
2021-03-22 23:37:25 +01:00
'';
};
}) {
"lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
2021-03-22 23:37:25 +01:00
} (builtins.attrNames containers);
2021-04-10 14:52:13 +02:00
# Systemd service template for LXC containers
systemd.services."lxc@" = {
description = "LXC container '%i'";
after = [ "network.target" ];
unitConfig.ConditionPathExists = [
"/var/lib/lxc/%i/rootfs/init"
"/etc/start-containers"
];
serviceConfig = with pkgs; {
Type = "simple";
ExecStart = "${lxc}/bin/lxc-start -F -C -n %i";
ExecStop = "${lxc}/bin/lxc-stop -n %i";
2021-03-27 03:43:19 +01:00
ExecReload =
let
script = writeScript "reload-lxc-container.sh" ''
#! ${runtimeShell} -e
2021-03-27 03:43:19 +01:00
SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init))
exec ${lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch
2021-03-27 03:43:19 +01:00
'';
in
"${script} %i";
KillMode = "mixed";
OOMPolicy = "kill";
Restart = "always";
RestartSec = "1s";
};
};
2021-03-27 03:43:19 +01:00
2021-04-10 14:52:13 +02:00
# Starts all the containers after boot
systemd.targets.lxc-containers = {
wantedBy = [ "multi-user.target" ];
wants = map (ctName: "lxc@${ctName}.service")
2021-03-27 03:43:19 +01:00
(builtins.attrNames containers);
};
2021-03-22 23:37:25 +01:00
}