nix/nixos-module/server/lxc-containers: make container config independend of host system
This commit is contained in:
parent
e76c8a9a3a
commit
c41f5c56a6
|
@ -0,0 +1,111 @@
|
||||||
|
{ config, lib, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
|
||||||
|
inherit (config.networking) hostName;
|
||||||
|
|
||||||
|
# linux iface name max length = 15
|
||||||
|
shortenNetName = name:
|
||||||
|
if builtins.match "priv(.*)" name != null
|
||||||
|
then "p" + builtins.substring 4 9 name
|
||||||
|
else if name == "coloradio"
|
||||||
|
then "cr"
|
||||||
|
else if name == "coloradio-gw"
|
||||||
|
then "cr-gw"
|
||||||
|
else name;
|
||||||
|
|
||||||
|
checkIfname = ifname: let
|
||||||
|
len = builtins.stringLength ifname;
|
||||||
|
in if len > 15
|
||||||
|
then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
|
||||||
|
else ifname;
|
||||||
|
|
||||||
|
# `lxc.net.*` formatter for lxc.container.conf files
|
||||||
|
netConfig = ctName: interfaces:
|
||||||
|
let
|
||||||
|
config = map (netName:
|
||||||
|
let
|
||||||
|
ifData = interfaces.${netName};
|
||||||
|
in {
|
||||||
|
type = ifData.type;
|
||||||
|
name = checkIfname netName;
|
||||||
|
flags = "up";
|
||||||
|
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
|
||||||
|
then ifData.hwaddr
|
||||||
|
else "0A:14:48:xx:xx:xx";
|
||||||
|
} // (lib.optionalAttrs (ifData.type == "veth") {
|
||||||
|
veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
|
||||||
|
veth.mode = checkIfname "bridge";
|
||||||
|
link = checkIfname netName;
|
||||||
|
}) // (lib.optionalAttrs (ifData.type == "phys") {
|
||||||
|
link = checkIfname "ext-${netName}";
|
||||||
|
})
|
||||||
|
) (builtins.attrNames interfaces);
|
||||||
|
|
||||||
|
attrNamesOrdered = attrs:
|
||||||
|
if attrs ? type
|
||||||
|
then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
|
||||||
|
else builtins.attrNames attrs;
|
||||||
|
|
||||||
|
serialize = name: x:
|
||||||
|
if builtins.isString x
|
||||||
|
then "${name} = ${x}\n"
|
||||||
|
else if builtins.isAttrs x
|
||||||
|
then builtins.concatStringsSep "" (
|
||||||
|
map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
|
||||||
|
)
|
||||||
|
else if builtins.isList x
|
||||||
|
then
|
||||||
|
let
|
||||||
|
enumerate = xs: n:
|
||||||
|
if xs == []
|
||||||
|
then []
|
||||||
|
else [ {
|
||||||
|
e = builtins.head xs;
|
||||||
|
i = n;
|
||||||
|
} ] ++ enumerate (builtins.tail xs) (n + 1);
|
||||||
|
in
|
||||||
|
builtins.concatStringsSep "" (
|
||||||
|
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
|
||||||
|
)
|
||||||
|
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
|
||||||
|
in
|
||||||
|
serialize "lxc.net" config;
|
||||||
|
|
||||||
|
in
|
||||||
|
{
|
||||||
|
system.build.lxcConfig = builtins.toFile "${hostName}.conf" ''
|
||||||
|
# For lxcfs and sane defaults
|
||||||
|
lxc.include = /etc/lxc/common.conf
|
||||||
|
|
||||||
|
lxc.uts.name = ${hostName}
|
||||||
|
# Handled by lxc@.service
|
||||||
|
lxc.start.auto = 0
|
||||||
|
lxc.rootfs.path = /var/lib/lxc/${hostName}/rootfs
|
||||||
|
lxc.init.cmd = "/init"
|
||||||
|
|
||||||
|
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
|
||||||
|
lxc.mount.entry = none tmp tmpfs defaults 0 0
|
||||||
|
lxc.mount.auto = proc:mixed sys:ro cgroup:mixed
|
||||||
|
|
||||||
|
lxc.autodev = 1
|
||||||
|
lxc.tty.max = 0
|
||||||
|
lxc.pty.max = 8
|
||||||
|
|
||||||
|
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
|
||||||
|
security.privileged = false
|
||||||
|
lxc.apparmor.profile = lxc-container-default-with-mounting
|
||||||
|
|
||||||
|
lxc.cgroup.memory.limit_in_bytes = 1G
|
||||||
|
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
|
||||||
|
|
||||||
|
# tuntap
|
||||||
|
lxc.cgroup.devices.allow = c 10:200 rw
|
||||||
|
lxc.cgroup2.devices.allow = c 10:200 rw
|
||||||
|
# ppp
|
||||||
|
lxc.cgroup.devices.allow = c 108:0 rwm
|
||||||
|
lxc.cgroup2.devices.allow = c 108:0 rwm
|
||||||
|
|
||||||
|
${netConfig hostName config.site.hosts.${hostName}.physicalInterfaces}
|
||||||
|
'';
|
||||||
|
}
|
|
@ -20,6 +20,7 @@ in {
|
||||||
./server/default.nix
|
./server/default.nix
|
||||||
] ++
|
] ++
|
||||||
optionals (hostConfig.role == "container") [
|
optionals (hostConfig.role == "container") [
|
||||||
|
./container/lxc-config.nix
|
||||||
./container/defaults.nix
|
./container/defaults.nix
|
||||||
./container/dhcp-server.nix
|
./container/dhcp-server.nix
|
||||||
./container/wireguard.nix
|
./container/wireguard.nix
|
||||||
|
|
|
@ -10,74 +10,6 @@ let
|
||||||
|
|
||||||
enabled = containers != {};
|
enabled = containers != {};
|
||||||
|
|
||||||
# linux iface name max length = 15
|
|
||||||
shortenNetName = name:
|
|
||||||
if builtins.match "priv(.*)" name != null
|
|
||||||
then "p" + builtins.substring 4 9 name
|
|
||||||
else if name == "coloradio"
|
|
||||||
then "cr"
|
|
||||||
else if name == "coloradio-gw"
|
|
||||||
then "cr-gw"
|
|
||||||
else name;
|
|
||||||
|
|
||||||
checkIfname = ifname: let
|
|
||||||
len = builtins.stringLength ifname;
|
|
||||||
in if len > 15
|
|
||||||
then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
|
|
||||||
else ifname;
|
|
||||||
|
|
||||||
# `lxc.net.*` formatter for lxc.container.conf files
|
|
||||||
netConfig = ctName: interfaces:
|
|
||||||
let
|
|
||||||
config = map (netName:
|
|
||||||
let
|
|
||||||
ifData = interfaces.${netName};
|
|
||||||
in {
|
|
||||||
type = ifData.type;
|
|
||||||
name = checkIfname netName;
|
|
||||||
flags = "up";
|
|
||||||
hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
|
|
||||||
then ifData.hwaddr
|
|
||||||
else "0A:14:48:xx:xx:xx";
|
|
||||||
} // (lib.optionalAttrs (ifData.type == "veth") {
|
|
||||||
veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
|
|
||||||
veth.mode = checkIfname "bridge";
|
|
||||||
link = checkIfname netName;
|
|
||||||
}) // (lib.optionalAttrs (ifData.type == "phys") {
|
|
||||||
link = checkIfname "ext-${netName}";
|
|
||||||
})
|
|
||||||
) (builtins.attrNames interfaces);
|
|
||||||
|
|
||||||
attrNamesOrdered = attrs:
|
|
||||||
if attrs ? type
|
|
||||||
then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
|
|
||||||
else builtins.attrNames attrs;
|
|
||||||
|
|
||||||
serialize = name: x:
|
|
||||||
if builtins.isString x
|
|
||||||
then "${name} = ${x}\n"
|
|
||||||
else if builtins.isAttrs x
|
|
||||||
then builtins.concatStringsSep "" (
|
|
||||||
map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
|
|
||||||
)
|
|
||||||
else if builtins.isList x
|
|
||||||
then
|
|
||||||
let
|
|
||||||
enumerate = xs: n:
|
|
||||||
if xs == []
|
|
||||||
then []
|
|
||||||
else [ {
|
|
||||||
e = builtins.head xs;
|
|
||||||
i = n;
|
|
||||||
} ] ++ enumerate (builtins.tail xs) (n + 1);
|
|
||||||
in
|
|
||||||
builtins.concatStringsSep "" (
|
|
||||||
map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
|
|
||||||
)
|
|
||||||
else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
|
|
||||||
in
|
|
||||||
serialize "lxc.net" config;
|
|
||||||
|
|
||||||
# User-facing script to build/update container NixOS systems
|
# User-facing script to build/update container NixOS systems
|
||||||
build-script = pkgs.writeScriptBin "build-container" ''
|
build-script = pkgs.writeScriptBin "build-container" ''
|
||||||
#! ${pkgs.runtimeShell} -e
|
#! ${pkgs.runtimeShell} -e
|
||||||
|
@ -98,6 +30,7 @@ let
|
||||||
${ctName})
|
${ctName})
|
||||||
echo Using prebuilt system for container $c
|
echo Using prebuilt system for container $c
|
||||||
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
|
SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
|
||||||
|
CONFIG=${self.packages.x86_64-linux."${ctName}-lxc-config"}
|
||||||
;;
|
;;
|
||||||
'') (
|
'') (
|
||||||
builtins.attrNames (
|
builtins.attrNames (
|
||||||
|
@ -109,6 +42,8 @@ let
|
||||||
echo Building $c
|
echo Building $c
|
||||||
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
|
nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
|
||||||
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
|
SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
|
||||||
|
nix build -o /nix/var/nix/gcroots/lxc/$c.config zentralwerk-network#$c-lxc-config
|
||||||
|
CONFIG=$(readlink /nix/var/nix/gcroots/lxc/$c.config)
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
@ -121,6 +56,7 @@ let
|
||||||
mkdir -p /var/lib/lxc/$c/rootfs/$d
|
mkdir -p /var/lib/lxc/$c/rootfs/$d
|
||||||
done
|
done
|
||||||
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
|
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
|
||||||
|
ln -fs $CONFIG /var/lib/lxc/$c/config
|
||||||
done
|
done
|
||||||
|
|
||||||
# Activate all the desired container after all of them are
|
# Activate all the desired container after all of them are
|
||||||
|
@ -166,10 +102,8 @@ in
|
||||||
|
|
||||||
virtualisation.lxc = lib.mkIf enabled {
|
virtualisation.lxc = lib.mkIf enabled {
|
||||||
enable = true;
|
enable = true;
|
||||||
# Container configs live in /etc so that they can be created
|
|
||||||
# through `environment.etc`.
|
|
||||||
systemConfig = ''
|
systemConfig = ''
|
||||||
lxc.lxcpath = /etc/lxc/containers
|
lxc.lxcpath = /var/lib/lxc
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -180,50 +114,7 @@ in
|
||||||
enable-script disable-script
|
enable-script disable-script
|
||||||
];
|
];
|
||||||
|
|
||||||
# Create lxc.container.conf files
|
environment.etc."lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
|
||||||
environment.etc =
|
|
||||||
builtins.foldl' (etc: ctName: etc // {
|
|
||||||
"lxc/containers/${ctName}/config" = {
|
|
||||||
enable = true;
|
|
||||||
source =
|
|
||||||
builtins.toFile "${ctName}.conf" ''
|
|
||||||
# For lxcfs and sane defaults
|
|
||||||
lxc.include = /etc/lxc/common.conf
|
|
||||||
|
|
||||||
lxc.uts.name = ${ctName}
|
|
||||||
# Handled by lxc@.service
|
|
||||||
lxc.start.auto = 0
|
|
||||||
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
|
|
||||||
lxc.init.cmd = "/init"
|
|
||||||
|
|
||||||
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
|
|
||||||
lxc.mount.entry = none tmp tmpfs defaults 0 0
|
|
||||||
lxc.mount.auto = proc:mixed sys:ro cgroup:mixed
|
|
||||||
|
|
||||||
lxc.autodev = 1
|
|
||||||
lxc.tty.max = 0
|
|
||||||
lxc.pty.max = 8
|
|
||||||
|
|
||||||
lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
|
|
||||||
security.privileged = false
|
|
||||||
lxc.apparmor.profile = lxc-container-default-with-mounting
|
|
||||||
|
|
||||||
lxc.cgroup.memory.limit_in_bytes = 1G
|
|
||||||
lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
|
|
||||||
|
|
||||||
# tuntap
|
|
||||||
lxc.cgroup.devices.allow = c 10:200 rw
|
|
||||||
lxc.cgroup2.devices.allow = c 10:200 rw
|
|
||||||
# ppp
|
|
||||||
lxc.cgroup.devices.allow = c 108:0 rwm
|
|
||||||
lxc.cgroup2.devices.allow = c 108:0 rwm
|
|
||||||
|
|
||||||
${netConfig ctName containers.${ctName}.physicalInterfaces}
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
}) {
|
|
||||||
"lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
|
|
||||||
} (builtins.attrNames containers);
|
|
||||||
|
|
||||||
# Systemd service template for LXC containers
|
# Systemd service template for LXC containers
|
||||||
systemd.services."lxc@" = {
|
systemd.services."lxc@" = {
|
||||||
|
|
|
@ -57,6 +57,19 @@ let
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
mkLxcConfig = hostName:
|
||||||
|
self.nixosConfigurations.${hostName}.config.system.build.lxcConfig;
|
||||||
|
|
||||||
|
lxc-configs =
|
||||||
|
builtins.foldl' (rootfs: hostName: rootfs // {
|
||||||
|
"${hostName}-lxc-config" = mkLxcConfig hostName;
|
||||||
|
}) {} (
|
||||||
|
builtins.attrNames (
|
||||||
|
nixpkgs.lib.filterAttrs (_: { role, ... }: role == "container")
|
||||||
|
config.site.hosts
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
vm-packages =
|
vm-packages =
|
||||||
builtins.foldl' (rootfs: hostName: rootfs // {
|
builtins.foldl' (rootfs: hostName: rootfs // {
|
||||||
"${hostName}-vm" = self.nixosConfigurations.${hostName}.config.system.build.vm
|
"${hostName}-vm" = self.nixosConfigurations.${hostName}.config.system.build.vm
|
||||||
|
@ -117,7 +130,7 @@ let
|
||||||
inherit self;
|
inherit self;
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
rootfs-packages // vm-packages // device-templates // openwrt-packages // network-graphs // network-cypher-graphs // starlink // subnetplans // {
|
rootfs-packages // lxc-configs // vm-packages // device-templates // openwrt-packages // network-graphs // network-cypher-graphs // starlink // subnetplans // {
|
||||||
inherit export-openwrt-models export-config dns-slaves
|
inherit export-openwrt-models export-config dns-slaves
|
||||||
encrypt-secrets decrypt-secrets switch-to-production
|
encrypt-secrets decrypt-secrets switch-to-production
|
||||||
homepage gateway-report switch-report vlan-report
|
homepage gateway-report switch-report vlan-report
|
||||||
|
|
Loading…
Reference in New Issue