322 lines
12 KiB
Nix
322 lines
12 KiB
Nix
{ hostRegistry, inputs, lib, microvm, self }:
|
|
|
|
let
|
|
getHostAddr = name:
|
|
let
|
|
hostConf = hostRegistry."${name}" or (throw "Host ${name} has no ip4 or ip6 address");
|
|
in
|
|
hostConf.ip4 or hostConf.ip6;
|
|
|
|
# all the input flakes for `nix copy` to the build machine,
|
|
# to be available for --override-input
|
|
inputPaths = lib.escapeShellArgs (builtins.attrValues inputs);
|
|
overrideInputsArgs = lib.concatStringsSep " " (builtins.attrValues (lib.mapAttrs
|
|
(name: value: "--override-input ${name} ${value}")
|
|
(lib.filterAttrs (name: _value: name != "self") inputs)
|
|
));
|
|
in
|
|
lib.attrsets.mapAttrs
|
|
(system: pkgs:
|
|
let
|
|
overlayPkgs = builtins.intersectAttrs (self.overlays { } { }) pkgs;
|
|
in
|
|
overlayPkgs //
|
|
{
|
|
host-registry = pkgs.runCommand "host-registry" {
|
|
src = builtins.toFile "host-registry.nix" (
|
|
lib.generators.toPretty { } hostRegistry
|
|
);
|
|
} ''
|
|
ln -s $src $out
|
|
'';
|
|
|
|
list-upgradable = pkgs.writeShellScriptBin "list-upgradable" ''
|
|
set -eou pipefail
|
|
|
|
NORMAL="\033[0m"
|
|
RED="\033[0;31m"
|
|
YELLOW="\033[0;33m"
|
|
GREEN="\033[0;32m"
|
|
|
|
${lib.concatMapStringsSep "\n" (name:
|
|
let
|
|
addr = getHostAddr name;
|
|
in lib.optionalString (addr != null) /* bash */ ''
|
|
echo -n -e "${name}: $RED"
|
|
RUNNING=$(ssh -o PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new root@"${addr}" "readlink /run/current-system")
|
|
if [ $? = 0 ] && [ -n "$RUNNING" ]; then
|
|
CURRENT=$(nix eval --raw ".#nixosConfigurations.${name}.config.system.build.toplevel" 2>/dev/null)
|
|
RUNNING_VER=$(basename $RUNNING|rev|cut -d - -f 1|rev)
|
|
RUNNING_DATE=$(echo $RUNNING_VER|cut -d . -f 3)
|
|
CURRENT_VER=$(basename $CURRENT|rev|cut -d - -f 1|rev)
|
|
CURRENT_DATE=$(echo $CURRENT_VER|cut -d . -f 3)
|
|
|
|
if [ "$RUNNING" = "$CURRENT" ]; then
|
|
echo -e "$GREEN"current"$NORMAL $RUNNING_VER"
|
|
elif [ $RUNNING_DATE -gt $CURRENT_DATE ]; then
|
|
echo -e "$GREEN"newer"$NORMAL $RUNNING_VER > $CURRENT_VER"
|
|
elif [ "$RUNNING_VER" = "$CURRENT_VER" ]; then
|
|
echo -e "$YELLOW"modified"$NORMAL $RUNNING_VER"
|
|
elif [ -n "$RUNNING_VER" ]; then
|
|
echo -e "$RED"outdated"$NORMAL $RUNNING_VER < $CURRENT_VER"
|
|
else
|
|
echo -e "$RED"error"$NORMAL $RUNNING_VER"
|
|
fi
|
|
fi
|
|
echo -n -e "$NORMAL"
|
|
'') (builtins.attrNames self.nixosConfigurations)}
|
|
'';
|
|
|
|
prebuild-all = pkgs.runCommand "prebuild-all" {
|
|
preferLocalBuild = true;
|
|
} ''
|
|
mkdir $out
|
|
|
|
${pkgs.lib.concatMapStrings (name: ''
|
|
ln -s ${self.nixosConfigurations."${name}".config.system.build.toplevel} name
|
|
'') (builtins.attrNames self.nixosConfigurations)}
|
|
'';
|
|
|
|
prebuild-all-remote = pkgs.writeShellScriptBin "prebuild-all" ''
|
|
set -eou pipefail
|
|
|
|
nix copy --no-check-sigs --to ssh-ng://$1 ${inputPaths}
|
|
|
|
ssh $1 -- nix build -L --no-link ${lib.concatMapStringsSep " " (name:
|
|
"${self}#nixosConfigurations.${name}.config.system.build.toplevel"
|
|
) (builtins.attrNames self.nixosConfigurations)}
|
|
'';
|
|
|
|
} //
|
|
builtins.foldl'
|
|
(result: name:
|
|
let
|
|
discardStringCtx = builtins.unsafeDiscardStringContext;
|
|
host = getHostAddr name;
|
|
target = "root@${host}";
|
|
rebuildArg = "--flake ${self}#${name} ${overrideInputsArgs} --accept-flake-config";
|
|
hostConfig = self.nixosConfigurations."${name}".config;
|
|
declaredRunnerDrvPath = discardStringCtx hostConfig.microvm.declaredRunner.drvPath;
|
|
declaredRunnerOutPath = discardStringCtx hostConfig.microvm.declaredRunner.outPath;
|
|
toplevelDrvPath = discardStringCtx hostConfig.system.build.toplevel.drvPath;
|
|
toplevelOutPath = discardStringCtx hostConfig.system.build.toplevel.outPath;
|
|
# let /var/lib/microvm/*/flake point to the flake-update branch so that
|
|
# `microvm -u $NAME` updates to what hydra built today.
|
|
selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config?ref=flake-update";
|
|
in
|
|
result // {
|
|
# Generate a small script for copying this flake to the
|
|
# remote machine and bulding and switching there.
|
|
# Can be run with `nix run c3d2#…-nixos-rebuild switch`
|
|
"${name}-nixos-rebuild" = pkgs.writeShellScriptBin "${name}-nixos-rebuild" ''
|
|
set -eou pipefail
|
|
|
|
${lib.optionalString (hostConfig.c3d2.deployment.server or "" != "") ''
|
|
echo "microvms cannot be updated with nixos-rebuild. Use nix run .#microvm-update-${name}"
|
|
exit 2
|
|
''}
|
|
|
|
hostname="$(ssh ${target} cat /etc/hostname)"
|
|
if [[ "$hostname" != ${name} ]]; then
|
|
echo "hostname of ${target} was expected to be ${name} but is $hostname. Aborting to be safe..."
|
|
exit 2
|
|
fi
|
|
nix copy --no-check-sigs --derivation --to ssh-ng://${target} ${toplevelDrvPath}
|
|
|
|
# use nixos-rebuild from target config
|
|
ssh ${target} bash -e <<END
|
|
set -eou pipefail
|
|
|
|
nix build --no-link ${toplevelDrvPath}^*
|
|
${discardStringCtx hostConfig.nix.package}/bin/nix-env -p /nix/var/nix/profiles/system --set ${toplevelOutPath}
|
|
${toplevelOutPath}/bin/switch-to-configuration "''${@:-switch}"
|
|
END
|
|
'';
|
|
|
|
"${name}-nixos-rebuild-hydra" = pkgs.writeShellScriptBin "${name}-nixos-rebuild" /* bash */ ''
|
|
set -eou pipefail
|
|
|
|
echo Copying Flakes
|
|
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${inputPaths}
|
|
|
|
echo Building on Hydra
|
|
ssh root@hydra.serv.zentralwerk.org -- \
|
|
nix build -L -o /tmp/nixos-system-${name} ${toplevelOutPath}
|
|
|
|
echo Built. Obtaining link to data
|
|
TOPLEVEL=$(ssh root@hydra.serv.zentralwerk.org \
|
|
readlink /tmp/nixos-system-${name})
|
|
|
|
echo Checking target ${name}
|
|
ssh ${target} -- bash -e <<EOF
|
|
[[ \$(cat /etc/hostname) == ${name} ]]
|
|
echo Copying data from Hydra to ${name}
|
|
nix copy --from https://nix-cache.hq.c3d2.de \
|
|
$TOPLEVEL
|
|
|
|
echo Activation on ${name}: "$@"
|
|
nix-env -p /nix/var/nix/profiles/system --set $TOPLEVEL
|
|
$TOPLEVEL/bin/switch-to-configuration "$@"
|
|
EOF
|
|
'';
|
|
|
|
"${name}-nixos-rebuild-local" = pkgs.writeShellScriptBin "${name}-nixos-rebuild" /* bash */ ''
|
|
set -eou pipefail
|
|
|
|
if [[ ''${1:-} == build ]]; then
|
|
hostname="$(ssh root@${target} cat /etc/hostname)"
|
|
if [[ "$hostname" != ${name} ]]; then
|
|
echo "hostname of ${target} was expected to be ${name} but is $hostname. Aborting to be safe..."
|
|
exit 2
|
|
fi
|
|
fi
|
|
# don't re-execute, otherwise we run the targetPlatform locally
|
|
_NIXOS_REBUILD_REEXEC=1 ${lib.getExe pkgs.nixos-rebuild} ${rebuildArg} --target-host ${target} --use-remote-sudo "$@"
|
|
'';
|
|
|
|
"${name}-cleanup" = pkgs.writeShellScriptBin "${name}-cleanup" ''
|
|
set -eou pipefail
|
|
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
|
|
'';
|
|
} // (let
|
|
createDirsCopyCurrent = name: /* bash */ ''
|
|
mkdir -p /var/lib/microvms/${name}
|
|
cd /var/lib/microvms/${name}
|
|
chown root:kvm .
|
|
chmod 0775 .
|
|
|
|
rm -f old
|
|
[ -e current ] && cp --no-dereference current old
|
|
'';
|
|
createSymlinks = name: let
|
|
gcrootDir = "/nix/var/nix/gcroots/microvm";
|
|
in /* bash */ ''
|
|
if [[ -e old ]]; then
|
|
echo System package diff:
|
|
nix --extra-experimental-features nix-command store diff-closures ./old ./current || true
|
|
fi
|
|
|
|
mkdir -p ${gcrootDir}
|
|
ln -sfT \$PWD/current ${gcrootDir}/${name}
|
|
ln -sfT \$PWD/booted ${gcrootDir}/booted-${name}
|
|
ln -sfT \$PWD/old ${gcrootDir}/old-${name}
|
|
'';
|
|
in {
|
|
|
|
"microvm-update-${name}" = pkgs.writeShellScriptBin "microvm-update-${name}" (
|
|
if builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]
|
|
then let
|
|
closureInfo = pkgs.closureInfo { rootPaths = [ hostConfig.system.build.toplevel ]; };
|
|
in ''
|
|
set -eou pipefail
|
|
|
|
${hostConfig.system.build.copyToServer} ${declaredRunnerDrvPath} ${discardStringCtx closureInfo.drvPath} ${discardStringCtx hostConfig.nix.package}
|
|
|
|
${hostConfig.system.build.runOnServer} NIXOS_REBUILD="''${NIXOS_REBUILD:-}" bash -e <<END
|
|
${createDirsCopyCurrent name}
|
|
|
|
nix build -L --accept-flake-config -o current ${declaredRunnerDrvPath}^*
|
|
nix build -L --accept-flake-config --no-link ${discardStringCtx closureInfo.drvPath}^*
|
|
echo '${selfRef}' > flake
|
|
|
|
${createSymlinks name}
|
|
|
|
if [[ -z "''${NIXOS_REBUILD:-}" ]]; then
|
|
echo "Restarting MicroVM..."
|
|
systemctl restart microvm@${name}.service
|
|
fi
|
|
END
|
|
|
|
if [[ -n "''${NIXOS_REBUILD:-}" ]]; then
|
|
echo "Switching to configuration..."
|
|
ssh ${target} bash -e <<END
|
|
set -eou pipefail
|
|
|
|
hostname=\$(cat /etc/hostname)
|
|
if [[ "\$hostname" != ${name} ]]; then
|
|
echo "hostname of ${target} was expected to be ${name} but is \$hostname. Aborting to be safe..."
|
|
exit 2
|
|
fi
|
|
|
|
# refresh nix db which is required for nix-env -p ... --set
|
|
${discardStringCtx hostConfig.nix.package}/bin/nix-store --load-db < ${discardStringCtx closureInfo}/registration
|
|
${discardStringCtx hostConfig.nix.package}/bin/nix-env -p /nix/var/nix/profiles/system --set ${toplevelOutPath}
|
|
${toplevelOutPath}/bin/switch-to-configuration "''${@:-switch}"
|
|
END
|
|
# ^ yes, this is required to be like this
|
|
|
|
fi
|
|
''
|
|
else throw "${name} is not configured to run on microvm.nix. Is it a physical host or is it deployed in Skyflake?"
|
|
);
|
|
|
|
"microvm-update-${name}-local" = pkgs.writeShellScriptBin "microvm-update-${name}" ''
|
|
set -eou pipefail
|
|
|
|
${lib.optionalString (!builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) ''
|
|
echo "MicroVM must be configured to proper server" >&2
|
|
exit 2
|
|
''}
|
|
|
|
nix build -L --no-link ${declaredRunnerDrvPath}^*
|
|
${hostConfig.system.build.copyToServer} ${declaredRunnerOutPath}
|
|
|
|
${hostConfig.system.build.runOnServer} bash -e <<END
|
|
set -eou pipefail
|
|
|
|
${createDirsCopyCurrent name}
|
|
ln -sfT ${declaredRunnerOutPath} current
|
|
echo '${selfRef}' > flake
|
|
${createSymlinks name}
|
|
|
|
systemctl restart microvm@${name}.service
|
|
END
|
|
'';
|
|
}))
|
|
{ }
|
|
(builtins.attrNames self.nixosConfigurations) //
|
|
|
|
builtins.foldl'
|
|
(result: host:
|
|
let
|
|
inherit (self.nixosConfigurations.${host}) config;
|
|
in
|
|
result // {
|
|
# boot any machine in a microvm
|
|
"${host}-vm" = (self.nixosConfigurations.${host}.extendModules {
|
|
modules = [{
|
|
microvm = {
|
|
mem = lib.mkForce 2048;
|
|
hypervisor = lib.mkForce "qemu";
|
|
socket = lib.mkForce null;
|
|
shares = lib.mkForce [{
|
|
tag = "ro-store";
|
|
source = "/nix/store";
|
|
mountPoint = "/nix/.ro-store";
|
|
}];
|
|
interfaces = lib.mkForce [{
|
|
type = "user";
|
|
id = "eth0";
|
|
mac = "02:23:de:ad:be:ef";
|
|
}];
|
|
};
|
|
boot.isContainer = lib.mkForce false;
|
|
users.users.root.password = "";
|
|
fileSystems."/".fsType = lib.mkForce "tmpfs";
|
|
services.getty.helpLine = ''
|
|
Log in as "root" with an empty password.
|
|
Use "reboot" to shut qemu down.
|
|
'';
|
|
}] ++ lib.optionals (! config ? microvm) [
|
|
microvm.nixosModules.microvm
|
|
];
|
|
}).config.microvm.declaredRunner;
|
|
|
|
"${host}-tftproot" = config.system.build.tftproot or (lib.trace "No tftproot for ${host}" null);
|
|
}
|
|
)
|
|
{ }
|
|
(builtins.attrNames self.nixosConfigurations)
|
|
)
|
|
self.legacyPackages
|