You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
259 lines
7.6 KiB
259 lines
7.6 KiB
{ self, config, lib, pkgs, ... }: |
|
|
|
let |
|
# Containers that are run on this host |
|
containers = |
|
lib.filterAttrs (_: { role, model, ... }: |
|
role == "container" && |
|
model == "lxc" |
|
) config.site.hosts; |
|
|
|
enabled = containers != {}; |
|
|
|
# linux iface name max length = 15 |
|
shortenNetName = name: |
|
if builtins.match "priv(.*)" name != null |
|
then "p" + builtins.substring 4 9 name |
|
else name; |
|
|
|
checkIfname = ifname: let |
|
len = builtins.stringLength ifname; |
|
in if len > 15 |
|
then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long." |
|
else ifname; |
|
|
|
# `lxc.net.*` formatter for lxc.container.conf files |
|
netConfig = ctName: interfaces: |
|
let |
|
config = map (netName: |
|
let |
|
ifData = interfaces.${netName}; |
|
in { |
|
type = ifData.type; |
|
name = checkIfname netName; |
|
flags = "up"; |
|
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null |
|
then ifData.hwaddr |
|
else "0A:14:48:xx:xx:xx"; |
|
} // (lib.optionalAttrs (ifData.type == "veth") { |
|
veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}"; |
|
veth.mode = checkIfname "bridge"; |
|
link = checkIfname netName; |
|
}) // (lib.optionalAttrs (ifData.type == "phys") { |
|
link = checkIfname "ext-${netName}"; |
|
}) |
|
) (builtins.attrNames interfaces); |
|
|
|
attrNamesOrdered = attrs: |
|
if attrs ? type |
|
then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs) |
|
else builtins.attrNames attrs; |
|
|
|
serialize = name: x: |
|
if builtins.isString x |
|
then "${name} = ${x}\n" |
|
else if builtins.isAttrs x |
|
then builtins.concatStringsSep "" ( |
|
map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x) |
|
) |
|
else if builtins.isList x |
|
then |
|
let |
|
enumerate = xs: n: |
|
if xs == [] |
|
then [] |
|
else [ { |
|
e = builtins.head xs; |
|
i = n; |
|
} ] ++ enumerate (builtins.tail xs) (n + 1); |
|
in |
|
builtins.concatStringsSep "" ( |
|
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0) |
|
) |
|
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}"; |
|
in |
|
serialize "lxc.net" config; |
|
|
|
# User-facing script to build/update container NixOS systems |
|
build-script = pkgs.writeScriptBin "build-container" '' |
|
#! ${pkgs.runtimeShell} -e |
|
|
|
if [ -z "$1" ]; then |
|
echo "Usage: $0 [containers ...]" |
|
exit 1 |
|
fi |
|
|
|
mkdir -p /nix/var/nix/gcroots/lxc |
|
|
|
for c in $@; do |
|
unset SYSTEM |
|
|
|
case "$c" in |
|
${builtins.concatStringsSep "\n" ( |
|
map (ctName: '' |
|
${ctName}) |
|
echo Using prebuilt system for container $c |
|
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"} |
|
;; |
|
'') ( |
|
builtins.attrNames ( |
|
lib.filterAttrs (_: { prebuilt, ... }: prebuilt) |
|
containers |
|
)) |
|
)} |
|
*) |
|
echo Building $c |
|
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs |
|
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c) |
|
;; |
|
esac |
|
|
|
echo Installing $c |
|
for d in \ |
|
bin dev etc home mnt \ |
|
nix/store nix/var \ |
|
proc root run sys tmp var usr ; \ |
|
do |
|
mkdir -p /var/lib/lxc/$c/rootfs/$d |
|
done |
|
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init |
|
done |
|
|
|
# Activate all the desired container after all of them are |
|
# built |
|
set +e |
|
for c in $@; do |
|
active=$(systemctl is-active lxc@$c) |
|
if [[ "$active" = active ]] ; then |
|
echo Activating $c |
|
systemctl reload lxc@$c || ( |
|
echo Reload failed. Restarting $c |
|
systemctl restart lxc@$c |
|
) |
|
else |
|
echo Starting $c |
|
systemctl start lxc@$c |
|
fi |
|
done |
|
set -e |
|
''; |
|
|
|
enable-script = pkgs.writeScriptBin "enable-containers" '' |
|
touch /etc/start-containers |
|
systemctl start lxc-containers.target |
|
''; |
|
|
|
disable-script = pkgs.writeScriptBin "disable-containers" '' |
|
rm /etc/start-containers |
|
systemctl stop lxc-containers.target lxc@\*.service |
|
''; |
|
in |
|
{ |
|
boot.kernel.sysctl = lib.mkIf enabled { |
|
"fs.inotify.max_queued_events" = 1048576; |
|
"fs.inotify.max_user_instances" = 1048576; |
|
"fs.inotify.max_user_watches" = 1048576; |
|
"vm.max_map_count" = 262144; |
|
"kernel.dmesg_restrict" = 1; |
|
"net.ipv4.neigh.default.gc_thresh3" = 8192; |
|
"net.ipv6.neigh.default.gc_thresh3" = 8192; |
|
"kernel.keys.maxkeys" = 2000; |
|
}; |
|
|
|
virtualisation.lxc = lib.mkIf enabled { |
|
enable = true; |
|
# Container configs live in /etc so that they can be created |
|
# through `environment.etc`. |
|
systemConfig = '' |
|
lxc.lxcpath = /etc/lxc/containers |
|
''; |
|
}; |
|
|
|
environment.systemPackages = [ |
|
# `lxc-attach` et al |
|
pkgs.lxc build-script |
|
# User scripts |
|
enable-script disable-script |
|
]; |
|
|
|
# Create lxc.container.conf files |
|
environment.etc = |
|
builtins.foldl' (etc: ctName: etc // { |
|
"lxc/containers/${ctName}/config" = { |
|
enable = true; |
|
source = |
|
builtins.toFile "${ctName}.conf" '' |
|
# For lxcfs and sane defaults |
|
lxc.include = /etc/lxc/common.conf |
|
|
|
lxc.uts.name = ${ctName} |
|
# Handled by lxc@.service |
|
lxc.start.auto = 0 |
|
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs |
|
lxc.init.cmd = "/init" |
|
|
|
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0 |
|
lxc.mount.entry = none tmp tmpfs defaults 0 0 |
|
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed |
|
|
|
lxc.autodev = 1 |
|
lxc.tty.max = 0 |
|
lxc.pty.max = 8 |
|
|
|
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio |
|
security.privileged = false |
|
lxc.apparmor.profile = lxc-container-default-with-mounting |
|
|
|
lxc.cgroup.memory.limit_in_bytes = 1G |
|
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M |
|
|
|
# tuntap |
|
lxc.cgroup.devices.allow = c 10:200 rw |
|
lxc.cgroup2.devices.allow = c 10:200 rw |
|
# ppp |
|
lxc.cgroup.devices.allow = c 108:0 rwm |
|
lxc.cgroup2.devices.allow = c 108:0 rwm |
|
|
|
${netConfig ctName containers.${ctName}.physicalInterfaces} |
|
''; |
|
}; |
|
}) { |
|
"lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf"; |
|
} (builtins.attrNames containers); |
|
|
|
# Systemd service template for LXC containers |
|
systemd.services."lxc@" = { |
|
description = "LXC container '%i'"; |
|
after = [ "network.target" ]; |
|
unitConfig.ConditionPathExists = [ |
|
"/var/lib/lxc/%i/rootfs/init" |
|
"/etc/start-containers" |
|
]; |
|
serviceConfig = with pkgs; { |
|
Type = "simple"; |
|
ExecStart = "${lxc}/bin/lxc-start -F -C -n %i"; |
|
ExecStop = "${lxc}/bin/lxc-stop -n %i"; |
|
ExecReload = |
|
let |
|
script = writeScript "reload-lxc-container.sh" '' |
|
#! ${runtimeShell} -e |
|
|
|
SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init)) |
|
exec ${lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch |
|
''; |
|
in |
|
"${script} %i"; |
|
KillMode = "mixed"; |
|
OOMPolicy = "kill"; |
|
Restart = "always"; |
|
RestartSec = "1s"; |
|
}; |
|
}; |
|
|
|
# Starts all the containers after boot |
|
systemd.targets.lxc-containers = { |
|
wantedBy = [ "multi-user.target" ]; |
|
wants = map (ctName: "lxc@${ctName}.service") |
|
(builtins.attrNames containers); |
|
}; |
|
}
|
|
|