nix-config/flake.nix

{
  description = "C3D2 NixOS configurations";

  inputs = {
    nixos.url = "github:nixos/nixpkgs/nixos-22.05";
    nixpkgs-mobilizon.url = "github:minijackson/nixpkgs/init-mobilizon";
    nixos-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
    nixos-hardware.url = "github:nixos/nixos-hardware";
    nixos-armv6.url = "github:rnhmjoj/nixpkgs/pr-fix-armv6";

    affection-src = {
      url = "git+https://gitea.nek0.eu/nek0/affection";
      inputs = {
        nixpkgs.follows = "nixos";
        flake-utils.follows = "flake-utils";
      };
    };
    bevy-mandelbrot = {
      # url = "github:matelab/bevy_mandelbrot";
      url = "git+https://gitea.c3d2.de/astro/bevy-mandelbrot.git?ref=main";
      inputs = {
        naersk.follows = "naersk";
        nixpkgs.follows = "nixos";
        rust-overlay.follows = "rust-overlay";
      };
    };
    bevy-julia = {
      # url = "github:matelab/bevy_julia";
      url = "git+https://gitea.c3d2.de/astro/bevy-julia.git?ref=main";
      inputs = {
        nixpkgs.follows = "nixos";
        # breaks the build:
        # naersk.follows = "naersk";
        # naersk.inputs.nixpkgs.follows = "nixpkgs";
        rust-overlay.follows = "rust-overlay";
      };
    };
    # eris = {
    #   url = "git+https://codeberg.org/eris/nix-eris?ref=trunk";
    #   inputs.nixpkgs.follows = "nixos";
    # };
    fenix = {
      url = "github:nix-community/fenix";
      inputs.nixpkgs.follows = "nixos";
    };
    flake-utils.url = "github:numtide/flake-utils";
    harmonia = {
      url = "github:helsinki-systems/harmonia";
      flake = false;
    };
    heliwatch = {
      url = "git+https://gitea.c3d2.de/astro/heliwatch.git";
      inputs = {
        fenix.follows = "fenix";
        nixpkgs.follows = "nixos";
        naersk.follows = "naersk";
        utils.follows = "flake-utils";
      };
    };
    hydra.url = "github:nixos/hydra";
    hydra-ca.url = "github:mlabs-haskell/hydra/aciceri/ca-derivations";
    microvm = {
      url = "github:astro/microvm.nix";
      inputs = {
        nixpkgs.follows = "nixos";
        flake-utils.follows = "flake-utils";
      };
    };
    naersk = {
      url = "github:nix-community/naersk";
      inputs = {
        nixpkgs.follows = "nixos";
      };
    };
    oparl-scraper = {
      url = "github:offenesdresden/ratsinfo-scraper/oparl";
      flake = false;
    };
    openwrt = {
      url = "git+https://git.openwrt.org/openwrt/openwrt.git?ref=openwrt-21.02";
      flake = false;
    };
    openwrt-imagebuilder = {
      url = "github:astro/nix-openwrt-imagebuilder";
      inputs = {
        nixpkgs.follows = "nixos";
      };
    };
    rust-overlay = {
      url = "github:oxalica/rust-overlay";
      inputs = {
        flake-utils.follows = "flake-utils";
        nixpkgs.follows = "nixos";
      };
    };
    scrapers = {
      url = "git+https://gitea.c3d2.de/astro/scrapers.git";
      flake = false;
    };
    sshlogd = {
      url = "git+https://gitea.c3d2.de/astro/sshlogd.git?ref=main";
      inputs = {
        utils.follows = "flake-utils";
        naersk.follows = "naersk";
        nixpkgs.follows = "nixos";
        fenix.follows = "fenix";
      };
    };
    secrets = {
      url = "git+ssh://gitea@gitea.c3d2.de/c3d2-admins/secrets.git";
      inputs = {
        nixpkgs.follows = "nixos";
      };
    };
    sops-nix = {
      url = "github:Mic92/sops-nix";
      inputs = {
        nixpkgs.follows = "nixos";
        nixpkgs-22_05.follows = "nixos";
      };
    };
    spacemsg = {
      url = "github:astro/spacemsg";
      flake = false;
    };
    # syndicate = {
    #   url = "git+https://git.syndicate-lang.org/ehmry/syndicate-flake";
    #   inputs = {
    #     nixpkgs.follows = "nixos";
    #     rust.follows = "rust-overlay";
    #   };
    # };
    ticker = {
      url = "git+https://gitea.c3d2.de/astro/ticker.git";
      inputs = {
        fenix.follows = "fenix";
        # TODO: build regression in https://github.com/nix-community/naersk/commit/6944160c19cb591eb85bbf9b2f2768a935623ed3
        # naersk.follows = "naersk";
        nixpkgs.follows = "nixos";
        utils.follows = "flake-utils";
      };
    };
    tigger = {
      url = "github:astro/tigger";
      flake = false;
    };
    tracer = {
      # url = "git+https://gitea.nek0.eu/nek0/tracer";
      url = "git+https://gitea.c3d2.de/astro/tracer";
      inputs = {
        affection-src.follows = "affection-src";
        nixpkgs.follows = "nixos";
        flake-utils.follows = "flake-utils";
      };
    };
    yammat = {
      url = "git+https://gitea.c3d2.de/C3D2/yammat.git?ref=nix";
      inputs.nixpkgs.follows = "nixos";
    };
    zentralwerk = {
      url = "git+https://gitea.c3d2.de/zentralwerk/network.git";
      inputs = {
        nixpkgs.follows = "nixos";
        openwrt.follows = "openwrt";
        openwrt-imagebuilder.follows = "openwrt-imagebuilder";
      };
    };
  };

  outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
    let
      inherit (nixos) lib;
      forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];

      # all the input flakes for `nix copy` to the build machine,
      # allowing --override-input
      inputPaths = lib.escapeShellArgs (builtins.attrValues inputs);

      extractZwHosts = { hosts4, hosts6, ... }:
        lib.recursiveUpdate (
          builtins.foldl' (result: name:
            lib.recursiveUpdate result {
              "${name}".ip4 = hosts4."${name}";
            }
          ) {} (builtins.attrNames hosts4)
        ) (
          builtins.foldl' (result: ctx:
            builtins.foldl' (result: name:
              lib.recursiveUpdate result {
                "${name}".ip6 = hosts6."${ctx}"."${name}";
              }
            ) result (builtins.attrNames hosts6."${ctx}")
          ) {} (builtins.attrNames hosts6)
        );
      zwHostRegistry = {
        hosts =
          builtins.foldl' (result: net:
            lib.recursiveUpdate result (extractZwHosts zentralwerk.lib.config.site.net."${net}")
          ) {} [ "core" "c3d2" "serv" "flpk" "pub" ];
      };
      extraHostRegistry.hosts = import ./host-registry.nix;
      hostRegistry = lib.recursiveUpdate zwHostRegistry extraHostRegistry;

      getHostAddr = name:
        let
          hostConf = hostRegistry.hosts."${name}";
        in
          if hostConf ? ip4
          then hostConf.ip4
          else if hostConf ? ip6
          then hostConf.ip6
          else throw "Host ${name} has no ip4 or ip6 address";
    in {
      overlay = import ./overlay {
        inherit nixos-unstable;
        inherit (inputs) tracer bevy-mandelbrot bevy-julia;
      };

      legacyPackages = lib.attrsets.mapAttrs (system: pkgs:
        pkgs.appendOverlays [
          fenix.overlay
          naersk.overlay
          # syndicate.overlays.default
          # eris.inputs.nimble.overlay
          # eris.overlays.default
          self.overlay
        ]) nixos.legacyPackages;

      packages = lib.attrsets.mapAttrs (system: pkgs:
        let overlayPkgs = builtins.intersectAttrs (self.overlay {} {}) pkgs;
        in overlayPkgs //
        {
          host-registry = pkgs.runCommand "host-registry" {
            src = builtins.toFile "host-registry.nix" (
              lib.generators.toPretty {} hostRegistry
            );
          } ''
            ln -s $src $out
          '';

          list-upgradable = pkgs.writeScriptBin "list-upgradable" ''
            #! ${pkgs.runtimeShell}

            NORMAL="\033[0m"
            RED="\033[0;31m"
            YELLOW="\033[0;33m"
            GREEN="\033[0;32m"

            ${pkgs.lib.concatMapStringsSep "\n" (name:
              let
                addr = getHostAddr name;
              in nixos.lib.optionalString (addr != null) ''
                echo -n -e "${name}: $RED"
                RUNNING=$(ssh -o PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new root@"${addr}" "readlink /run/current-system")
                if [ $? = 0 ] && [ -n "$RUNNING" ]; then
                  CURRENT=$(nix eval --raw ".#nixosConfigurations.${name}.config.system.build.toplevel" 2>/dev/null)
                  RUNNING_VER=$(basename $RUNNING|rev|cut -d - -f 1|rev)
                  RUNNING_DATE=$(echo $RUNNING_VER|cut -d . -f 3)
                  CURRENT_VER=$(basename $CURRENT|rev|cut -d - -f 1|rev)
                  CURRENT_DATE=$(echo $CURRENT_VER|cut -d . -f 3)

                  if [ "$RUNNING" = "$CURRENT" ]; then
                    echo -e "$GREEN"current"$NORMAL $RUNNING_VER"
                  elif [ $RUNNING_DATE -gt $CURRENT_DATE ]; then
                    echo -e "$GREEN"newer"$NORMAL $RUNNING_VER > $CURRENT_VER"
                  elif [ "$RUNNING_VER" = "$CURRENT_VER" ]; then
                    echo -e "$YELLOW"modified"$NORMAL $RUNNING_VER"
                  elif [ -n "$RUNNING_VER" ]; then
                    echo -e "$RED"outdated"$NORMAL $RUNNING_VER < $CURRENT_VER"
                  else
                    echo -e "$RED"error"$NORMAL $RUNNING_VER"
                  fi
                fi
                echo -n -e "$NORMAL"
              '') (builtins.attrNames self.nixosConfigurations)}
          '';

          prebuild-all = pkgs.runCommand "prebuild-all" {
            preferLocalBuild = true;
          } ''
            mkdir $out

            ${pkgs.lib.concatMapStrings (name: ''
              ln -s ${self.nixosConfigurations."${name}".config.system.build.toplevel} name
            '') (builtins.attrNames self.nixosConfigurations)}
          '';

          prebuild-all-remote = pkgs.writeScriptBin "prebuild-all" ''
            #!${pkgs.runtimeShell} -e

            nix copy --no-check-sigs --to ssh-ng://$1 ${inputPaths}

            set -x
            ssh $1 -- nix build -L --no-link ${
              pkgs.lib.concatMapStringsSep " " (name:
                "${self}#nixosConfigurations.${name}.config.system.build.toplevel"
              ) (builtins.attrNames self.nixosConfigurations)
            }
          '';

        } //

        builtins.foldl' (result: host: result // {
          # TODO: check if the ethernet address is reachable and if not,
          # execute wol on a machine in HQ.
          "${host}-wake" = pkgs.writeScriptBin "${host}-wake" ''
            #!${pkgs.runtimeShell}
            exec ${pkgs.wol}/bin/wol ${hostRegistry.hosts."${host}".ether}
          '';
        }) {} (builtins.attrNames (nixos.lib.filterAttrs (_: { wol ? false, ... }: wol) hostRegistry.hosts)) //

        builtins.foldl' (result: name:
          let
            host = getHostAddr name;
            target = ''root@"${host}"'';
            rebuildArg = "--flake ${self}#${name} --option extra-substituters https://nix-serve.hq.c3d2.de";
            hostConfig = self.nixosConfigurations."${name}".config;
            profile = hostConfig.system.build.toplevel;
            # let /var/lib/microvm/*/flake point to the flake-update branch so that
            # `microvm -u $NAME` updates to what hydra built today.
            selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config?ref=flake-update";
          in result // {
            # Generate a small script for copying this flake to the
            # remote machine and bulding and switching there.
            # Can be run with `nix run c3d2#…-nixos-rebuild switch`
            "${name}-nixos-rebuild" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
              #!${pkgs.runtimeShell} -ex
              [[ $(ssh ${target} cat /etc/hostname) == ${name} ]]
              nix copy --no-check-sigs --to ssh-ng://${target} ${inputPaths}
              ssh ${target} nixos-rebuild ${rebuildArg} "$@"
            '';

            "${name}-nixos-rebuild-hydra" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
              #!${pkgs.runtimeShell} -e
              echo Copying Flakes
              nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${inputPaths}
              echo Building on Hydra
              ssh root@hydra.serv.zentralwerk.org -- \
                nix build -L -o /tmp/nixos-system-${name} \
                ${self}#nixosConfigurations.${name}.config.system.build.toplevel
              echo Built. Obtaining link to data
              TOPLEVEL=$(ssh root@hydra.serv.zentralwerk.org \
                readlink /tmp/nixos-system-${name})
              echo Checking target ${name}
              ssh ${target} -- bash -e <<EOF
              [[ \$(cat /etc/hostname) == ${name} ]]
              echo Copying data from Hydra to ${name}
              nix copy --from https://nix-serve.hq.c3d2.de \
                $TOPLEVEL
              echo Activation on ${name}: "$@"
              nix-env -p /nix/var/nix/profiles/system --set $TOPLEVEL
              $TOPLEVEL/bin/switch-to-configuration "$@"
              EOF
            '';

            "${name}-nixos-rebuild-local" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
               #!${pkgs.runtimeShell} -ex
               [[ $1 == build || $(ssh ${target} cat /etc/hostname) == ${name} ]]
               ${pkgs.nixos-rebuild}/bin/nixos-rebuild ${rebuildArg} --target-host ${target} --use-remote-sudo "$@"
            '';

            "${name}-cleanup" = pkgs.writeScriptBin "${name}-cleanup" ''
              #!${pkgs.runtimeShell} -ex
              ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
            '';
          } // lib.optionalAttrs (builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) {
            "microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
              #!${pkgs.runtimeShell} -e
              ${hostConfig.system.build.copyToServer} ${inputPaths}

              ${hostConfig.system.build.runOnServer} bash -e <<END
              mkdir -p /var/lib/microvms/${name}
              cd /var/lib/microvms/${name}
              chown root:kvm .
              chmod 0775 .

              rm -f old
              [ -e current ] && mv current old

              nix build -L \
                -o current \
                ${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
              echo '${selfRef}' > flake

              [ -e old ] && nix store diff-closures ./old ./current
              ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
              ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
              ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}

              systemctl restart microvm@${name}.service
              END
            '';

            "microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
              #!${pkgs.runtimeShell} -e
              ${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}

              ${hostConfig.system.build.runOnServer} bash -e <<END
              mkdir -p /var/lib/microvms/${name}
              cd /var/lib/microvms/${name}
              chown root:kvm .
              chmod 0775 .

              rm -f old
              [ -e current ] && mv current old

              ln -sfT ${hostConfig.microvm.declaredRunner} current
              echo '${selfRef}' > flake

              [ -e old ] && nix store diff-closures ./old ./current
              ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
              ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
              ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}

              systemctl restart microvm@${name}.service
              END
            '';

          } // lib.optionalAttrs (hostConfig.c3d2.deployment.server or null == "nomad") {
            "nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
              #!${pkgs.runtimeShell} -e

              echo Copying Flakes
              nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}

              echo Building on Hydra
              ssh root@hydra.serv.zentralwerk.org -- \
                nix build -L -o /tmp/microvm-${name}.job \
                ${self}#nixosConfigurations.${name}.config.system.build.nomadJob

              echo -n Built. Obtaining path...
              JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
                readlink /tmp/microvm-${name}.job)
              echo \ $JOB

              for h in server9 server10 ; do
                echo Sharing with $h
                ssh root@$h.cluster.zentralwerk.org -- \
                  bash -e <<EOF &
                nix copy --from https://nix-serve.hq.c3d2.de $JOB
                mkdir -p /glusterfs/fast/microvms/${name}
                chown microvm:kvm /glusterfs/fast/microvms/${name}
                chmod 0775 /glusterfs/fast/microvms/${name}
                mkdir -p /nix/var/nix/gcroots/microvm
                rm -f /nix/var/nix/gcroots/microvm/${name}
                ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
              EOF
              done
              wait

              echo Now starting the job
              ssh root@hydra.serv.zentralwerk.org -- \
                nomad run -detach $JOB
            '';
          }) {} (builtins.attrNames self.nixosConfigurations) //

        builtins.foldl' (result: host:
          let
            inherit (self.nixosConfigurations.${host}) config;
          in
            result //
            nixos.lib.optionalAttrs (config.system.build ? vm) {
              # boot any machine in a microvm
              "${host}-vm" = (self.nixosConfigurations.${host}
                .extendModules {
                  modules = [ {
                    microvm = {
                      mem = nixos.lib.mkForce 2048;
                      hypervisor = nixos.lib.mkForce "qemu";
                      socket = nixos.lib.mkForce null;
                      shares = nixos.lib.mkForce [ {
                        tag = "ro-store";
                        source = "/nix/store";
                        mountPoint = "/nix/.ro-store";
                      } ];
                      interfaces = nixos.lib.mkForce [ {
                        type = "user";
                        id = "eth0";
                        mac = "02:23:de:ad:be:ef";
                      } ];
                    };
                    boot.isContainer = lib.mkForce false;
                    users.users.root.password = "";
                    fileSystems."/".fsType = lib.mkForce "tmpfs";
                    services.getty.helpLine = ''
                        Log in as "root" with an empty password.
                        Use "reboot" to shut qemu down.
                      '';
                  } ] ++ lib.optionals (! config ? microvm) [
                    microvm.nixosModules.microvm
                  ];
                })
                .config.microvm.declaredRunner;
            } //
            nixos.lib.optionalAttrs (config.system.build ? tftproot) {
              "${host}-tftproot" = config.system.build.tftproot;
            }
        ) {} (builtins.attrNames self.nixosConfigurations)
      ) self.legacyPackages;

      nixosConfigurations = let
        nixosSystem' =
          # Our custom NixOS builder
          { nixpkgs ? inputs.nixos, modules, extraArgs ? {}, specialArgs ? { }, system ? "x86_64-linux" }:
          nixpkgs.lib.nixosSystem {
            inherit specialArgs system;

            modules = [
              ({ pkgs, ... }: {
                _module.args = extraArgs // {
                  inherit hostRegistry inputs zentralwerk;
                };
                nixpkgs = {
                  overlays = [ self.overlay ];
                };
              })

              self.nixosModules.c3d2
              ./modules/audio-server.nix
              ./modules/c3d2.nix
              ./modules/stats.nix
              ./modules/pi-sensors.nix
            ] ++ modules;
          };
      in {
        auth = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/auth
          ];
          nixpkgs = inputs.nixos-unstable;
        };

        broker = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/broker
          ];
        };

        factorio = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/factorio
          ];
          nixpkgs = inputs.nixos-unstable;
        };

        freifunk = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/freifunk
            {
              nixpkgs.overlays = with secrets.overlays; [
                freifunk ospf
              ];
            }
          ];
        };

        ftp = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/ftp
          ];
        };

        gitea = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/gitea
          ];
          nixpkgs = inputs.nixos-unstable;
        };

        glotzbert = nixosSystem' {
          modules = [
            ./hosts/glotzbert
            nixos-hardware.nixosModules.common-cpu-intel
            nixos-hardware.nixosModules.common-pc-ssd
            # TODO: migrate away from secrets repo
            secrets.nixosModules.admins
          ];
        };

        hedgedoc = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/hedgedoc
          ];
          nixpkgs = inputs.nixos-unstable;
        };

        pulsebert = nixosSystem' {
          modules = [
            ./hosts/pulsebert
            # build: outputs.nixosConfigurations.pulsebert.config.system.build.sdImage
            # run: unzstd -cd result/sd-image/nixos-sd-image-*-aarch64-linux.img.zst | pv -br | sudo dd bs=4M of=/dev/sdX
            "${inputs.nixos-unstable}/nixos/modules/installer/sd-card/sd-image-aarch64-new-kernel.nix"
            {
              nixpkgs = {
                hostPlatform = "aarch64-linux";
                # buildPlatform = "x86_64-linux";
              };
            }
          ];
          nixpkgs = inputs.nixos-unstable;
        };

        radiobert = nixosSystem' {
          modules = [
            ({ modulesPath, ... }:
            {
              nixpkgs.overlays = [ heliwatch.overlay ];
            })
            ./hosts/radiobert
          ];
          system = "aarch64-linux";
        };

        # nncp = nixosSystem' {
        #   modules = [
        #     self.nixosModules.microvm
        #     self.nixosModules.nncp
        #     syndicate.nixosModules.default
        #     ./hosts/nncp
        #     {
        #       nixpkgs.overlays = [
        #         syndicate.overlays.default
        #         eris.inputs.nimble.overlay
        #         eris.overlays.default
        #       ];
        #     }
        #   ];
        # };

        dacbert = nixosSystem' {
          modules = [
            nixos-hardware.nixosModules.raspberry-pi-4
            self.nixosModules.rpi-netboot
            ./hosts/dacbert
          ];
          system = "aarch64-linux";
        };

        rpi-netboot = nixosSystem' {
          modules = [
            { _module.args = { inherit nixos; }; }
            nixos-hardware.nixosModules.raspberry-pi-4
            self.nixosModules.rpi-netboot
            ./hosts/rpi-netboot
          ];
          system = "aarch64-linux";
        };

        schalter = nixosSystem' {
          nixpkgs = inputs.nixos-armv6;
          modules = [
            "${nixos}/nixos/modules/installer/sd-card/sd-image-raspberrypi.nix"
            ({ lib, ... }: {
              networking.hostName = "schalter";
              hardware.enableRedistributableFirmware = true;
              #networking.wireless.enable = true;
              powerManagement.cpuFreqGovernor = lib.mkDefault "performance";

              boot = {
                loader = {
                  grub.enable = false;
                  raspberryPi = {
                    enable = true;
                    version = 1;
                    uboot.enable = false;
                  };
                  generic-extlinux-compatible.enable = lib.mkForce false;
                };
                # supportedFilesystems = lib.mkForce [ "vfat" "ext4" ];

                tmpOnTmpfs = true;
              };
              nixpkgs.config.packageOverrides = pkgs: {
                makeModulesClosure = x:
                  # prevent kernel install fail due to missing modules
                  pkgs.makeModulesClosure (x // { allowMissing = true; });
              };
              sdImage = {
                compressImage = false;
                imageBaseName = "schalter";
                firmwareSize = 512;
              };

              nixpkgs.crossSystem = nixos.lib.systems.examples.raspberryPi;
            })
          ];
          system = "x86_64-linux";
        };

        matemat = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/matemat
            yammat.nixosModule
            # TODO: migrate away from secrets repo
            secrets.nixosModules.admins
            { nixpkgs.overlays = [ secrets.overlays.matemat ]; }
          ];
        };

        scrape = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/scrape
            {
              nixpkgs.overlays = [ secrets.overlays.scrape ];
              _module.args = { inherit scrapers; };
            }
          ];
        };

        dn42 = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/dn42
            {
              nixpkgs.overlays = [ secrets.overlays.dn42 ];
            }
          ];
        };

        grafana = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/grafana
          ];
        };

        hydra = nixosSystem' {
          modules = [
            inputs.hydra.nixosModules.hydra
            (harmonia + "/module.nix")
            ./hosts/hydra
            {
              _module.args = {
                inherit self;
                inherit (inputs) hydra-ca;
              };
            }
          ];
        };

        mucbot = nixosSystem' {
          modules = [
            { _module.args = { inherit tigger; }; }
            self.nixosModules.microvm
            self.nixosModules.nomad-microvm
            "${tigger}/module.nix"
            { nixpkgs.overlays = [ secrets.overlays.mucbot ]; }
            ./hosts/mucbot
          ];
        };

        public-access-proxy = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/public-access-proxy
          ];
        };

        ticker = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ticker.nixosModules.ticker
            ./hosts/ticker
          ];
        };

        spaceapi = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            "${spacemsg}/spaceapi/module.nix"
            ./hosts/spaceapi
          ];
        };

        stream = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/stream
          ];
        };

        mobilizon = nixosSystem' {
          # TODO: pending https://github.com/NixOS/nixpkgs/pull/119132
          nixpkgs = inputs.nixpkgs-mobilizon;
          modules = [
            self.nixosModules.microvm
            ./hosts/mobilizon
          ];
        };

        c3d2-web = nixosSystem' {
          modules = [
            { _module.args = { inherit nixos; }; }
            self.nixosModules.microvm
            ./hosts/c3d2-web
            { nixpkgs.overlays = [ secrets.overlays.c3d2-web ]; }
          ];
        };

        sdrweb = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            self.nixosModules.nomad-microvm
            { nixpkgs.overlays = [ secrets.overlays.mucbot ]; }
            heliwatch.nixosModules.heliwatch
            ./hosts/sdrweb
          ];
          nixpkgs = nixos-unstable;
        };

        bind = nixosSystem' {
          modules = [
            { nixpkgs.overlays = with secrets.overlays; [ bind ]; }
            self.nixosModules.microvm
            ./hosts/bind
          ];
        };

        jabber = nixosSystem' {
          modules = [
            { nixpkgs.overlays = with secrets.overlays; [ jabber ]; }
            self.nixosModules.microvm
            ./hosts/jabber
          ];
        };

        blogs = nixosSystem' {
          modules = [
            self.nixosModules.plume
            self.nixosModules.microvm
            ./hosts/blogs
          ];
        };

        server9 = nixosSystem' {
          modules = [
            ./hosts/server9
            microvm.nixosModules.host
            self.nixosModules.microvm-host
            self.nixosModules.cluster-network
            self.nixosModules.cluster
            { _module.args = { inherit self; }; }
          ];
        };

        server10 = nixosSystem' {
          modules = [
            ./hosts/server10
            microvm.nixosModules.host
            self.nixosModules.microvm-host
            self.nixosModules.cluster-network
            self.nixosModules.cluster
            { _module.args = { inherit self; }; }
          ];
        };

        oparl = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/oparl
            {
              _module.args = { inherit oparl-scraper; };
            }
          ];
        };

        leon = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/leon
          ];
        };

        leoncloud = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/leoncloud
          ];
        };

        nfsroot = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/nfsroot
            {
              _module.args.tftproots = nixos.lib.filterAttrs (name: _:
                builtins.match ".+-tftproot" name != null
              ) self.packages.x86_64-linux;
            }
          ];
        };

        riscbert = nixosSystem' {
          modules = [
            nixos-hardware.nixosModules.starfive-visionfive-v1
            ./hosts/riscbert
            {
              nixpkgs.crossSystem = {
                config = "riscv64-unknown-linux-gnu";
                system = "riscv64-linux";
              };
            }
          ];
          nixpkgs = nixos-unstable;
          system = "x86_64-linux";
        };

        direkthilfe = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/direkthilfe
          ];
        };

        mediawiki = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/mediawiki
          ];
          nixpkgs = nixos-unstable;
        };

        gnunet = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/gnunet
          ];
          nixpkgs = nixos-unstable;
        };

        zengel = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/zengel
          ];
        };

        sshlog = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            sshlogd.nixosModule
            ./hosts/sshlog
          ];
        };

        network-homepage = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/network-homepage
          ];
        };

      };

      nixosModule = self.nixosModules.c3d2;
      nixosModules = {
        c3d2 = {
          imports = [
            sops-nix.nixosModule
            ./modules/c3d2.nix
            ./modules/nncp.nix
            ./modules/autoupdate.nix
          ];
          c3d2.hosts = hostRegistry.hosts;
          c3d2.users = import ./users.nix;
          c3d2.nncp.neigh = import ./modules/nncp-relays.nix;
        };
        cluster = ./modules/cluster;
        cluster-network = ./modules/cluster-network.nix;
        microvm.imports = [
          microvm.nixosModules.microvm
          ./modules/microvm.nix
        ];
        microvm-host = ./modules/microvm-host.nix;
        nncp = ./modules/nncp.nix;
        nomad-microvm = ./modules/nomad-microvm.nix;
        plume = {
          imports = [ ./modules/plume.nix ];
          nixpkgs.overlays = [ fenix.overlay naersk.overlay ];
        };
        rpi-netboot = ./modules/rpi-netboot.nix;
      };

      # `nix develop`
      devShell = lib.mapAttrs (system: sopsPkgs:
        with nixos.legacyPackages.${system};
        mkShell {
          sopsPGPKeyDirs = [ "./keys" ];
          nativeBuildInputs = [
            sopsPkgs.sops-import-keys-hook
          ];
        }
      ) sops-nix.packages;

      hydraJobs =
        builtins.mapAttrs (_: nixos.lib.hydraJob) (
          builtins.mapAttrs (_: nixosSystem:
            if nixosSystem.config ? microvm.declaredRunner
            then nixosSystem.config.microvm.declaredRunner
            else nixosSystem.config.system.build.toplevel
          ) self.nixosConfigurations
          //
          nixos.lib.filterAttrs (name: _:
            builtins.match ".+-tftproot" name != null
          ) self.packages.aarch64-linux
        );
    };
}