forked from zentralwerk/network
nixos-module/server/lxc-containers: add to drbd rootfs
This commit is contained in:
parent
87968ddac9
commit
e14eae80fd
|
@ -293,6 +293,11 @@ let
|
||||||
type = with types; nullOr str;
|
type = with types; nullOr str;
|
||||||
default = null;
|
default = null;
|
||||||
};
|
};
|
||||||
|
diskSize = mkOption {
|
||||||
|
type = types.int;
|
||||||
|
default = 64;
|
||||||
|
description = "Root disk size for containers in MB";
|
||||||
|
};
|
||||||
interfaces = mkOption {
|
interfaces = mkOption {
|
||||||
default = {};
|
default = {};
|
||||||
type = with types; attrsOf (submodule interfaceOpts);
|
type = with types; attrsOf (submodule interfaceOpts);
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
imports = [
|
imports = [
|
||||||
./defaults.nix
|
./defaults.nix
|
||||||
./network.nix
|
./network.nix
|
||||||
|
./drbd-utils.nix
|
||||||
./lxc-containers.nix
|
./lxc-containers.nix
|
||||||
./qemu.nix
|
./qemu.nix
|
||||||
./pacemaker.nix
|
./pacemaker.nix
|
||||||
|
|
|
@ -0,0 +1,66 @@
|
||||||
|
{ self, config, lib, pkgs, ... }:
|
||||||
|
with lib;
|
||||||
|
|
||||||
|
let cfg = config.services.drbd-utils; in
|
||||||
|
{
|
||||||
|
###### interface
|
||||||
|
|
||||||
|
options = {
|
||||||
|
|
||||||
|
services.drbd-utils.enable = mkOption {
|
||||||
|
default = false;
|
||||||
|
type = types.bool;
|
||||||
|
description = ''
|
||||||
|
Whether to enable support for DRBD, the Distributed Replicated
|
||||||
|
Block Device.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
services.drbd-utils.config = mkOption {
|
||||||
|
default = "";
|
||||||
|
type = types.lines;
|
||||||
|
description = ''
|
||||||
|
Contents of the <filename>drbd.conf</filename> configuration file.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
###### implementation
|
||||||
|
|
||||||
|
config = mkIf cfg.enable {
|
||||||
|
|
||||||
|
# use the latest drbd-utils from outside nixpkgs
|
||||||
|
nixpkgs.config.packageOverrides = pkgs: {
|
||||||
|
drbd = self.packages.${pkgs.system}.drbd-utils;
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = [ pkgs.drbd ];
|
||||||
|
services.udev.packages = [ pkgs.drbd ];
|
||||||
|
boot.kernelModules = [ "drbd" ];
|
||||||
|
|
||||||
|
boot.extraModprobeConfig =
|
||||||
|
''
|
||||||
|
options drbd usermode_helper=/run/current-system/sw/bin/drbdadm
|
||||||
|
'';
|
||||||
|
|
||||||
|
environment.etc."drbd.conf" =
|
||||||
|
{ source = pkgs.writeText "drbd.conf" cfg.config; };
|
||||||
|
|
||||||
|
systemd.services.drbd = {
|
||||||
|
after = [ "systemd-udev.settle.service" "network-online.target" ];
|
||||||
|
requires = [ "network-online.target" ];
|
||||||
|
wants = [ "systemd-udev.settle.service" ];
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
path = [ pkgs.drbd ];
|
||||||
|
script = ''
|
||||||
|
drbdadm up all
|
||||||
|
'';
|
||||||
|
preStop = ''
|
||||||
|
drbdadm down all
|
||||||
|
'';
|
||||||
|
serviceConfig.RemainAfterExit = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
|
@ -1,6 +1,13 @@
|
||||||
{ hostName, self, config, lib, pkgs, ... }:
|
{ hostName, self, config, lib, pkgs, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
|
drbdPortBase = 17000;
|
||||||
|
|
||||||
|
servers =
|
||||||
|
lib.filterAttrs (_: { role, ... }:
|
||||||
|
role == "server"
|
||||||
|
) config.site.hosts;
|
||||||
|
|
||||||
# Containers that are run on this host
|
# Containers that are run on this host
|
||||||
containers =
|
containers =
|
||||||
lib.filterAttrs (_: { role, model, ... }:
|
lib.filterAttrs (_: { role, model, ... }:
|
||||||
|
@ -109,14 +116,7 @@ let
|
||||||
esac
|
esac
|
||||||
|
|
||||||
echo Installing $c
|
echo Installing $c
|
||||||
for d in \
|
ln -fs $SYSTEM /var/lib/lxc/$c/system
|
||||||
bin dev etc home mnt \
|
|
||||||
nix/store nix/var \
|
|
||||||
proc root run sys tmp var usr ; \
|
|
||||||
do
|
|
||||||
mkdir -p /var/lib/lxc/$c/rootfs/$d
|
|
||||||
done
|
|
||||||
ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# Activate all the desired container after all of them are
|
# Activate all the desired container after all of them are
|
||||||
|
@ -150,6 +150,7 @@ in
|
||||||
"net.ipv6.neigh.default.gc_thresh3" = 8192;
|
"net.ipv6.neigh.default.gc_thresh3" = 8192;
|
||||||
"kernel.keys.maxkeys" = 2000;
|
"kernel.keys.maxkeys" = 2000;
|
||||||
};
|
};
|
||||||
|
boot.kernelModules = [ "loop" ];
|
||||||
|
|
||||||
virtualisation.lxc = lib.mkIf enabled {
|
virtualisation.lxc = lib.mkIf enabled {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
@ -180,8 +181,9 @@ in
|
||||||
# Handled by lxc@.service
|
# Handled by lxc@.service
|
||||||
lxc.start.auto = 0
|
lxc.start.auto = 0
|
||||||
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
|
lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
|
||||||
lxc.init.cmd = "/init"
|
lxc.init.cmd = "/system/init"
|
||||||
|
|
||||||
|
lxc.mount.entry = /var/lib/lxc/${ctName}/system system none bind,ro 0 0
|
||||||
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
|
lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
|
||||||
lxc.mount.entry = none tmp tmpfs defaults 0 0
|
lxc.mount.entry = none tmp tmpfs defaults 0 0
|
||||||
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
|
lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
|
||||||
|
@ -215,8 +217,9 @@ in
|
||||||
systemd.services."lxc@" = {
|
systemd.services."lxc@" = {
|
||||||
description = "LXC container '%i'";
|
description = "LXC container '%i'";
|
||||||
after = [ "network.target" ];
|
after = [ "network.target" ];
|
||||||
|
requires = [ "lxc-rootfs@%i.service" ];
|
||||||
unitConfig.ConditionPathExists = [
|
unitConfig.ConditionPathExists = [
|
||||||
"/var/lib/lxc/%i/rootfs/init"
|
"/var/lib/lxc/%i/system"
|
||||||
];
|
];
|
||||||
serviceConfig = with pkgs; {
|
serviceConfig = with pkgs; {
|
||||||
Type = "simple";
|
Type = "simple";
|
||||||
|
@ -239,6 +242,124 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
systemd.services."lxc-rootfs@" = {
|
||||||
|
wants = [ "drbd.service" ];
|
||||||
|
unitConfig.ConditionPathExists = [
|
||||||
|
"/dev/drbd/by-res/%i"
|
||||||
|
];
|
||||||
|
serviceConfig =
|
||||||
|
let
|
||||||
|
mkScript = name: content:
|
||||||
|
"${pkgs.writeShellScriptBin name ''
|
||||||
|
PATH=$PATH:/run/current-system/sw/bin
|
||||||
|
${content}
|
||||||
|
''}/bin/${name} %i";
|
||||||
|
in {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
ExecStart = mkScript "lxc-rootfs-start" ''
|
||||||
|
drbdadm primary $1
|
||||||
|
ROOT=/var/lib/lxc/$1/rootfs
|
||||||
|
mkdir -p $ROOT
|
||||||
|
mount /dev/drbd/by-res/$1 $ROOT
|
||||||
|
for DIR in \
|
||||||
|
bin dev etc home mnt \
|
||||||
|
nix/store nix/var \
|
||||||
|
proc root run sys \
|
||||||
|
system tmp var usr \
|
||||||
|
; do
|
||||||
|
mkdir -p $ROOT/$DIR
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
ExecStop = mkScript "lxc-rootfs-stop" ''
|
||||||
|
umount /var/lib/lxc/$1/rootfs
|
||||||
|
drbdadm secondary $1
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
# systemd.services.drbd.wants = [ "lxc-volumes.service" ];
|
||||||
|
systemd.services.lxc-volumes = {
|
||||||
|
description = "Create LXC rootfs for drbd";
|
||||||
|
wantedBy = [ "drbd.service" ];
|
||||||
|
path = with pkgs; [ drbd coreutils util-linux e2fsprogs ];
|
||||||
|
script = ''
|
||||||
|
create() {
|
||||||
|
file=$1
|
||||||
|
size=$2
|
||||||
|
[ -e $file ] || \
|
||||||
|
dd if=/dev/zero of=$file bs=1024 seek=$(($size - 1)) count=1
|
||||||
|
}
|
||||||
|
|
||||||
|
${lib.concatStrings (lib.imap0 (i: container: ''
|
||||||
|
mkdir -p /var/lib/lxc/${container}
|
||||||
|
cd /var/lib/lxc/${container}
|
||||||
|
|
||||||
|
if [ ! -e rootfs.img ]; then
|
||||||
|
create rootfs.img ${toString (containers.${container}.diskSize * 1024)}
|
||||||
|
mkfs.ext4 rootfs.img
|
||||||
|
else
|
||||||
|
losetup -d /dev/loop${toString (2 * i)} 2> /dev/null || true
|
||||||
|
fi
|
||||||
|
# TODO: zfsPool
|
||||||
|
losetup /dev/loop${toString (2 * i)} rootfs.img
|
||||||
|
|
||||||
|
if [ ! -e rootfs.meta ]; then
|
||||||
|
create rootfs.meta ${toString (((4095 + 36 + (containers.${container}.diskSize / 32)) / 4096) * 4096)}
|
||||||
|
losetup /dev/loop${toString (2 * i + 1)} rootfs.meta
|
||||||
|
drbdadm create-md ${container}
|
||||||
|
else
|
||||||
|
losetup /dev/loop${toString (2 * i + 1)} rootfs.meta
|
||||||
|
fi
|
||||||
|
'') (builtins.attrNames containers))}
|
||||||
|
'';
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
RemainAfterExit = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services.drbd-utils = {
|
||||||
|
enable = true;
|
||||||
|
config = ''
|
||||||
|
global {
|
||||||
|
# Do not participate in online survey
|
||||||
|
usage-count no;
|
||||||
|
}
|
||||||
|
|
||||||
|
common {
|
||||||
|
# Protocol A: write IO is reported as completed, if it has
|
||||||
|
# reached local disk and local TCP send buffer.
|
||||||
|
protocol A;
|
||||||
|
syncer {
|
||||||
|
rate 60M;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
${lib.concatStrings (lib.imap0 (i: container: ''
|
||||||
|
resource ${container} {
|
||||||
|
net {
|
||||||
|
cram-hmac-alg sha1;
|
||||||
|
shared-secret "TODO";
|
||||||
|
}
|
||||||
|
device minor ${toString i};
|
||||||
|
# /var/lib/lxc/${container}/rootfs.img
|
||||||
|
disk /dev/loop${toString (i * 2)};
|
||||||
|
# /var/lib/lxc/${container}/rootfs.meta
|
||||||
|
meta-disk /dev/loop${toString (i * 2 + 1)};
|
||||||
|
${lib.concatMapStrings (server: ''
|
||||||
|
on ${server} {
|
||||||
|
address ${config.site.net.cluster.hosts4.${server}}:${toString (drbdPortBase + i)};
|
||||||
|
}
|
||||||
|
'') (builtins.attrNames servers)}
|
||||||
|
}
|
||||||
|
'') (builtins.attrNames containers))}
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
networking.firewall.allowedTCPPorts =
|
||||||
|
lib.imap0 (i: _server: drbdPortBase + i)
|
||||||
|
(builtins.attrNames servers);
|
||||||
|
|
||||||
services.pacemaker.services =
|
services.pacemaker.services =
|
||||||
map (ctName: "lxc@${ctName}.service")
|
map (ctName: "lxc@${ctName}.service")
|
||||||
(builtins.attrNames containers);
|
(builtins.attrNames containers);
|
||||||
|
|
|
@ -120,6 +120,6 @@ in
|
||||||
rootfs-packages // vm-packages // device-templates // network-graphs // starlink // subnetplans // {
|
rootfs-packages // vm-packages // device-templates // network-graphs // starlink // subnetplans // {
|
||||||
inherit all-rootfs export-openwrt-models export-config dns-slaves
|
inherit all-rootfs export-openwrt-models export-config dns-slaves
|
||||||
encrypt-secrets decrypt-secrets switch-to-production
|
encrypt-secrets decrypt-secrets switch-to-production
|
||||||
pacemaker
|
drbd-utils pacemaker
|
||||||
;
|
;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue