nix-config/hosts/hydra/default.nix

{ config, lib, libS, pkgs, ... }:

{
  imports = [
    ./hardware-configuration.nix
    ./network.nix
    ./updater.nix
  ];

  c3d2 = {
    baremetal = true;
    hq.statistics.enable = true;
  };

  boot = {
    tmp = {
      useTmpfs = true;
      tmpfsSize = "80%";
    };
    kernelModules = [ "kvm-intel" ];
    kernelParams = [ "mitigations=off" "preempt=none" ];
    loader = {
      efi.canTouchEfiVariables = true;
      systemd-boot.enable = true;
    };
    # For cross-building
    binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
  };

  nix = {
    buildMachines = let
      localPlatforms = feature: !(builtins.elem feature [ "x86_64-linux" "i686-linux" ]);
      # strips features that don't make sense on qemu-user
      extraPlatforms = builtins.filter localPlatforms config.nix.settings.extra-platforms;
    in [
      {
        hostName = "localhost";
        maxJobs = config.nix.settings.max-jobs;
        protocol = null;
        speedFactor = 10;
        supportedFeatures = config.nix.settings.system-features;
        systems = [ "x86_64-linux" "i686-linux" ];
      }
      # # local container to have an extra nix daemon for binfmt
      # # NOTE: currently very, very slow and usually builds do not finish in any amount of time
      # {
      #   hostName = "root@192.168.100.3";
      #   maxJobs = 4;
      #   speedFactors = 20;
      #   supportedFeatures = [ "big-parallel" "nixos-test" "benchmark" ];
      #   systems = lib.concatStringsSep "," extraPlatforms;
      # }
      {
        hostName = "client@dacbert.hq.c3d2.de";
        system = lib.concatStringsSep "," [
          "aarch64-linux" # very slow compared to gallium
          "armv6l-linux" "armv7l-linux"
        ];
        speedFactor = 1;
        supportedFeatures = [ "kvm" "nixos-test" ];
        maxJobs = 1;
      }
      # {
      #   hostName = "gallium.s6o.de";
      #   maxJobs = 4;
      #   speedFactor = 10;
      #   sshUser = config.nix.remoteBuilder.name;
      #   # kvm is not supported because /dev/kvm does not exist
      #   supportedFeatures = [ "big-parallel" "nixos-test" "benchmark" ];
      #   system = "aarch64-linux";
      # }
    ];
    daemonCPUSchedPolicy = "idle";
    daemonIOSchedClass = "idle";
    daemonIOSchedPriority = 7;
    optimise = {
      automatic = true;
      dates = [ "05:30" ];
    };
    remoteBuilder = {
      enable = true;
      sshPublicKeys = config.users.users.root.openssh.authorizedKeys.keys;
    };
    settings = {
      allowed-uris = "http:// https:// ssh://";
      builders-use-substitutes = true;
      cores = 20;
      keep-outputs = true;
      max-jobs = 8;
      trusted-users = [ "hydra" "root" "@wheel" ];
      system-features = [
        "kvm" "big-parallel"
        "nixos-test" "benchmark"
      ];
    };
    extraOptions = ''
      !include ${config.sops.secrets."nix/access-tokens".path}
    '';
  };

  networking = {
    hostId = "3f0c4ec4";
    hostName = "hydra";
    nameservers = [ "172.20.73.8" "9.9.9.9" ];
  };

  programs.ssh.knownHosts = lib.mkMerge [
    (libS.mkPubKey "192.168.100.3" "ssh-ed25519" "AAAAC3NzaC1lZDI1NTE5AAAAIBqrnoVELFvO9uc5VlLjiNAXyRTCWUMp5WiTF6o9UorJ")
    (libS.mkPubKey "192.168.100.3" "ssh-rsa" "AAAAB3NzaC1yc2EAAAADAQABAAACAQCwofGcB1HIkIDWR9QNjl/9R39pLusYW2tvmGCZ9p0kfH1ml76OeWHZdXfjpwZJgRM+mk+sbfgKL3xfha+vPiLPJCfMUnKpgoM6zC5i/wi4Ywenh4hPFZG4moVFPBjcMUPmWw7vtED6n5dcW+LOeeuOGwEoBv72UiwhQVg7ULJIT0wu/lj2uduNwiSq8fmxeKZqB+jnJzpc56hGejuWWsGfgpIt2gWirOCqaxNoyRjt/rdGpHsRi8POBIjh5FsvTZVG0zJSgz0ubBsoCivgIr9fGKGxr0dLfDfqqNtrDFwDkkSiymcuo7zRU506pRLeTdrKPhPhvQg3aPOYAQcyvoJKo8xyMim5CbkbIo6TM7os5ubYoNpJ6+WSicYZaI4CG6X7kThkellAKy+yynlwnTTe5Q0DwUJr0znGy4Yi6t/VVE/bFEuAmb0DFbWVf2VqecFAe635hOxmQzzhaf1Zrf4epzcom833o12XdA6abfvuD3dVFSq/9ClzIBFkywNd22LrWhH2Wnh0u38xyHHTdGRQE5z5BKV0TevnmLgni92vMLyoTOdiC4UGhB71ED6tckN0qifzjvAGB2CAr+XT1Zy7ECPXC3SwqBYxcOb10j9pJsdx/gkjg8bovhr4Ve1x5blkzNvLbHA9jCITvfY3ke65JmL/loK1EEoS7odJGrQAbw==")
  ];

  services = {
    fail2ban = {
      enable = true;
      ignoreIP = [
        "2a00:8180:2c00:200::/56"
        "2a0f:5382:acab:1400::/56"
        "fd23:42:c3d2:500::/56"
        "30c:c3d2:b946:76d0::/64"
        "::1/128"
        "172.22.99.0/24"
        "172.20.72.0/21"
        "127.0.0.0/8"
      ];
    };

    gitea-actions = {
      enableRunner = true;
      kvm = true;
      zfsDataset = "hydra/data/podman";
      giteaUrl = "https://gitea.c3d2.de";
    };

    hydra = {
      enable = true;
      buildMachinesFiles = [
        "/etc/nix/machines"
        "/var/lib/hydra/machines"
      ];
      hydraURL = "https://hydra.hq.c3d2.de";
      ldap.enable = true;
      logo = ./c3d2.svg;
      minimumDiskFree = 50;
      minimumDiskFreeEvaluator = 50;
      notificationSender = "hydra@spam.works";
      useSubstitutes = true;
      extraConfig =
        let
          key = config.sops.secrets."nix/signing-key/secretKey".path;
        in
        ''
          binary_cache_secret_key_file = ${key}
          compress_num_threads = 4
          evaluator_workers = 4
          evaluator_max_memory_size = 2048
          max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
          store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
          upload_logs_to_binary_cache = true
        '';
    };

    harmonia = {
      enable = true;
      domain = "nix-cache.hq.c3d2.de";
      port = 5000;
      settings.workers = 20;
      signKeyPath = config.sops.secrets."nix/signing-key/secretKey".path;
    };

    nginx = {
      enable = true;
      virtualHosts = {
        "hydra.hq.c3d2.de" = {
          default = true;
          enableACME = true;
          forceSSL = true;
          locations."/".proxyPass = "http://127.0.0.1:${toString config.services.hydra.port}";
          serverAliases = [
            "hydra.serv.zentralwerk.org"
          ];
        };
        "nix-cache.hq.c3d2.de" = {
          forceSSL = true;
          enableACME = true;
        };
      };
    };

    portunus.addToHosts = true;

    postgresql = {
      package = pkgs.postgresql_16;
      upgrade.stopServices = [ "hydra-evaluator" "hydra-queue-runner" "hydra-server" ];
    };

    resolved.enable = false;

    zfs.trim.enable = true;
  };

  simd.arch = "ivybridge";

  sops = {
    defaultSopsFile = ./secrets.yaml;
    secrets = {
      "ldap/search-user-pw" = {
        mode = "440";
        owner = config.users.users.hydra-queue-runner.name;
        path = "/var/lib/hydra/ldap-password.conf";
      };
      "machine-id" = {
        mode = "444";
        path = "/etc/machine-id";
      };
      "nix/access-tokens" = {
        mode = "444";
      };
      "nix/signing-key/secretKey" = {
        mode = "440";
        owner = config.users.users.hydra-queue-runner.name;
      };
      "ssh-keys/hydra/private" = {
        owner = "hydra";
        path = "/var/lib/hydra/.ssh/id_ed25519";
      };
      "ssh-keys/hydra/public" = {
        owner = "hydra";
        mode = "440";
        path = "/var/lib/hydra/.ssh/id_ed25519.pub";
      };
      "ssh-keys/root/private" = {
        owner = "hydra-queue-runner";
        path = "/var/lib/hydra/queue-runner/.ssh/id_ed25519";
      };
      "ssh-keys/root/public" = {
        owner = "hydra-queue-runner";
        mode = "440";
        path = "/var/lib/hydra/queue-runner/.ssh/id_ed25519.pub";
      };
      "ssh-keys/updater/private" = {
        owner = "updater";
        path = "/var/lib/updater/.ssh/id_ed25519";
      };
      "ssh-keys/updater/public" = {
        owner = "updater";
        mode = "440";
        path = "/var/lib/updater/.ssh/id_ed25519.pub";
      };
    };
  };

  system.stateVersion = "20.09";

  systemd.services = {
    hydra-evaluator.serviceConfig = {
      CPUWeight = 2;
      MemoryHigh = "64G";
      MemoryMax = "64G";
      MemorySwapMax = "64G";
    };

    nix-daemon.serviceConfig = {
      CPUWeight = 5;
      MemoryHigh = "64G";
      MemoryMax = "64G";
      MemorySwapMax = "64G";
    };
  };
}