You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
260 lines
7.6 KiB
Nix
260 lines
7.6 KiB
Nix
{ self, config, lib, pkgs, ... }:
|
|
|
|
let
|
|
# Containers that are run on this host
|
|
containers =
|
|
lib.filterAttrs (_: { role, model, ... }:
|
|
role == "container" &&
|
|
model == "lxc"
|
|
) config.site.hosts;
|
|
|
|
enabled = containers != {};
|
|
|
|
# linux iface name max length = 15
|
|
shortenNetName = name:
|
|
if builtins.match "priv(.*)" name != null
|
|
then "p" + builtins.substring 4 9 name
|
|
else name;
|
|
|
|
checkIfname = ifname: let
|
|
len = builtins.stringLength ifname;
|
|
in if len > 15
|
|
then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
|
|
else ifname;
|
|
|
|
# `lxc.net.*` formatter for lxc.container.conf files
|
|
netConfig = ctName: interfaces:
|
|
let
|
|
config = map (netName:
|
|
let
|
|
ifData = interfaces.${netName};
|
|
in {
|
|
type = ifData.type;
|
|
name = checkIfname netName;
|
|
flags = "up";
|
|
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
|
|
then ifData.hwaddr
|
|
else "0A:14:48:xx:xx:xx";
|
|
} // (lib.optionalAttrs (ifData.type == "veth") {
|
|
veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
|
|
veth.mode = checkIfname "bridge";
|
|
link = checkIfname netName;
|
|
}) // (lib.optionalAttrs (ifData.type == "phys") {
|
|
link = checkIfname "ext-${netName}";
|
|
})
|
|
) (builtins.attrNames interfaces);
|
|
|
|
attrNamesOrdered = attrs:
|
|
if attrs ? type
|
|
then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
|
|
else builtins.attrNames attrs;
|
|
|
|
serialize = name: x:
|
|
if builtins.isString x
|
|
then "${name} = ${x}\n"
|
|
else if builtins.isAttrs x
|
|
then builtins.concatStringsSep "" (
|
|
map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
|
|
)
|
|
else if builtins.isList x
|
|
then
|
|
let
|
|
enumerate = xs: n:
|
|
if xs == []
|
|
then []
|
|
else [ {
|
|
e = builtins.head xs;
|
|
i = n;
|
|
} ] ++ enumerate (builtins.tail xs) (n + 1);
|
|
in
|
|
builtins.concatStringsSep "" (
|
|
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
|
|
)
|
|
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
|
|
in
|
|
serialize "lxc.net" config;
|
|
|
|
# User-facing script to build/update container NixOS systems
|
|
build-script = pkgs.writeScriptBin "build-container" ''
|
|
#! ${pkgs.runtimeShell} -e
|
|
|
|
if [ -z "$1" ]; then
|
|
echo "Usage: $0 [containers ...]"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p /nix/var/nix/gcroots/lxc
|
|
|
|
for c in $@; do
|
|
unset SYSTEM
|
|
|
|
case "$c" in
|
|
${builtins.concatStringsSep "\n" (
|
|
map (ctName: ''
|
|
${ctName})
|
|
echo Using prebuilt system for container $c
|
|
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
|
|
;;
|
|
'') (
|
|
builtins.attrNames (
|
|
lib.filterAttrs (_: { prebuilt, ... }: prebuilt)
|
|
containers
|
|
))
|
|
)}
|
|
*)
|
|
echo Building $c
|
|
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
|
|
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
|
|
;;
|
|
esac
|
|
|
|
echo Installing $c
|
|
for d in \
|
|
bin dev etc home mnt \
|
|
nix/store nix/var \
|
|
proc root run sys tmp var usr ; \
|
|
do
|
|
mkdir -p /var/lib/lxc/$c/rootfs/$d
|
|
done
|
|
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
|
|
done
|
|
|
|
# Activate all the desired container after all of them are
|
|
# built
|
|
set +e
|
|
for c in $@; do
|
|
active=$(systemctl is-active lxc@$c)
|
|
if [[ "$active" = active ]] ; then
|
|
echo Activating $c
|
|
systemctl reload lxc@$c || (
|
|
echo Reload failed. Restarting $c
|
|
systemctl restart lxc@$c
|
|
)
|
|
else
|
|
echo Starting $c
|
|
systemctl start lxc@$c
|
|
fi
|
|
done
|
|
set -e
|
|
'';
|
|
|
|
enable-script = pkgs.writeScriptBin "enable-containers" ''
|
|
touch /etc/start-containers
|
|
systemctl start lxc-containers.target
|
|
'';
|
|
|
|
disable-script = pkgs.writeScriptBin "disable-containers" ''
|
|
rm /etc/start-containers
|
|
systemctl stop lxc-containers.target lxc@\*.service
|
|
'';
|
|
in
|
|
{
|
|
boot.kernel.sysctl = lib.mkIf enabled {
|
|
"fs.inotify.max_queued_events" = 1048576;
|
|
"fs.inotify.max_user_instances" = 1048576;
|
|
"fs.inotify.max_user_watches" = 1048576;
|
|
"vm.max_map_count" = 262144;
|
|
"kernel.dmesg_restrict" = 1;
|
|
"net.ipv4.neigh.default.gc_thresh3" = 8192;
|
|
"net.ipv6.neigh.default.gc_thresh3" = 8192;
|
|
"kernel.keys.maxkeys" = 2000;
|
|
};
|
|
|
|
virtualisation.lxc = lib.mkIf enabled {
|
|
enable = true;
|
|
# Container configs live in /etc so that they can be created
|
|
# through `environment.etc`.
|
|
systemConfig = ''
|
|
lxc.lxcpath = /etc/lxc/containers
|
|
'';
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
# `lxc-attach` et al
|
|
pkgs.lxc build-script
|
|
# User scripts
|
|
enable-script disable-script
|
|
];
|
|
|
|
# Create lxc.container.conf files
|
|
environment.etc =
|
|
builtins.foldl' (etc: ctName: etc // {
|
|
"lxc/containers/${ctName}/config" = {
|
|
enable = true;
|
|
source =
|
|
builtins.toFile "${ctName}.conf" ''
|
|
# For lxcfs and sane defaults
|
|
lxc.include = /etc/lxc/common.conf
|
|
|
|
lxc.uts.name = ${ctName}
|
|
# Handled by lxc@.service
|
|
lxc.start.auto = 0
|
|
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
|
|
lxc.init.cmd = "/init"
|
|
|
|
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
|
|
lxc.mount.entry = none tmp tmpfs defaults 0 0
|
|
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
|
|
|
|
lxc.autodev = 1
|
|
lxc.tty.max = 0
|
|
lxc.pty.max = 8
|
|
|
|
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
|
|
security.privileged = false
|
|
lxc.apparmor.profile = lxc-container-default-with-mounting
|
|
|
|
lxc.cgroup.memory.limit_in_bytes = 1G
|
|
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
|
|
|
|
# tuntap
|
|
lxc.cgroup.devices.allow = c 10:200 rw
|
|
lxc.cgroup2.devices.allow = c 10:200 rw
|
|
# ppp
|
|
lxc.cgroup.devices.allow = c 108:0 rwm
|
|
lxc.cgroup2.devices.allow = c 108:0 rwm
|
|
|
|
${netConfig ctName containers.${ctName}.physicalInterfaces}
|
|
'';
|
|
};
|
|
}) {
|
|
"lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
|
|
} (builtins.attrNames containers);
|
|
|
|
# Systemd service template for LXC containers
|
|
systemd.services."lxc@" = {
|
|
description = "LXC container '%i'";
|
|
after = [ "network.target" ];
|
|
unitConfig.ConditionPathExists = [
|
|
"/var/lib/lxc/%i/rootfs/init"
|
|
"/etc/start-containers"
|
|
];
|
|
serviceConfig = with pkgs; {
|
|
Type = "simple";
|
|
ExecStart = "${lxc}/bin/lxc-start -F -C -n %i";
|
|
ExecStop = "${lxc}/bin/lxc-stop -n %i";
|
|
ExecReload =
|
|
let
|
|
script = writeScript "reload-lxc-container.sh" ''
|
|
#! ${runtimeShell} -e
|
|
|
|
SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init))
|
|
exec ${lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch
|
|
'';
|
|
in
|
|
"${script} %i";
|
|
KillMode = "mixed";
|
|
OOMPolicy = "kill";
|
|
Restart = "always";
|
|
RestartSec = "1s";
|
|
};
|
|
};
|
|
|
|
# Starts all the containers after boot
|
|
systemd.targets.lxc-containers = {
|
|
wantedBy = [ "multi-user.target" ];
|
|
wants = map (ctName: "lxc@${ctName}.service")
|
|
(builtins.attrNames containers);
|
|
};
|
|
}
|