nix-config/hosts/hydra/default.nix

{ config, lib, pkgs, zentralwerk, ... }:

let
  cachePort = 5000;
in
{
  imports = [
    ./hardware-configuration.nix
    ./network.nix
    ./hydra.nix
    ./updater.nix
    ../../modules/c3d2.nix
  ];

  c3d2 = {
    hq.statistics.enable = true;
    simd.arch = "ivybridge";
  };

  boot = {
    tmpOnTmpfs = true;
    tmpOnTmpfsSize = "80%";
    kernelPackages = pkgs.linuxPackages_latest;
    kernelModules = [ "kvm-intel" ];
    kernelParams = [ "mitigations=off" "preempt=none" ];
    # For cross-building
    binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
  };

  nix = {
    buildMachines = [{
      hostName = "client@dacbert.hq.c3d2.de";
      system = lib.concatStringsSep "," [
        "aarch64-linux" "armv6l-linux" "armv7l-linux"
      ];
      supportedFeatures = [ "kvm" "nixos-test" ];
      maxJobs = 1;
    }];
    daemonCPUSchedPolicy = "idle";
    daemonIOSchedClass = "idle";
    daemonIOSchedPriority = 7;
    settings = {
      allowed-uris = "http:// https:// ssh://";
      auto-optimise-store = true;
      builders-use-substitutes = true;
      cores = 20;
      keep-outputs = true;
      max-jobs = 8;
      trusted-users = [ "hydra" "root" "@wheel" ];
    };
  };

  nixpkgs.config.allowUnfree = true;

  # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
  # containers = {
  #   hydra-ca = {
  #     autoStart = true;
  #     config = { ... }: {
  #       imports = [
  #         hydra-ca.nixosModules.hydra
  #       ];

  #       environment.systemPackages = with pkgs; [ git ];

  #       networking.firewall.allowedTCPPorts = [ 3001 ];

  #       nix = {
  #         settings = {
  #           allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
  #           builders-use-substitutes = true;
  #           experimental-features = "ca-derivations nix-command flakes";
  #           extra-substituters = "https://cache.ngi0.nixos.org/";
  #           extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
  #           substituters = [
  #             "https://cache.ngi0.nixos.org/"
  #           ];
  #           trusted-public-keys = [
  #             "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
  #           ];
  #         };
  #       };

  #       nixpkgs = {
  #         # config.contentAddressedByDefault = true;
  #         overlays = [ self.overlay ];
  #       };

  #       services = {
  #         hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
  #           hydraURL = "https://hydra-ca.hq.c3d2.de";
  #           port = 3001;
  #         };
  #       };

  #       system.stateVersion = "22.05"; # Did you read the comment? No.
  #     };
  #     hostAddress = "192.168.100.1";
  #     localAddress = "192.168.100.2";
  #     privateNetwork = true;
  #   };
  # };

  networking = {
    hosts = with zentralwerk.lib.config.site.net.serv; {
      ${hosts6.up4.auth} = [ "auth.c3d2.de" ];
      ${hosts4.auth} = [ "auth.c3d2.de" ];
    };
    hostName = "hydra";
    firewall.enable = false;
    nameservers = [ "172.20.73.8" "9.9.9.9" ];
    # nat = {
    #   enable = true;
    #   externalInterface = "serv";
    #   internalInterfaces = [ "ve-hydra-ca" ];
    # };
  };

  services = {
    hydra = {
      enable = true;
      buildMachinesFiles = [
        "/etc/nix/machines"
        "/var/lib/hydra/machines"
      ];
      hydraURL = "https://hydra.hq.c3d2.de";
      logo = ./c3d2.svg;
      minimumDiskFree = 50;
      minimumDiskFreeEvaluator = 50;
      notificationSender = "hydra@spam.works";
      useSubstitutes = true;
      extraConfig =
        let
          key = config.sops.secrets."nix-serve/secretKey".path;
        in
        ''
          binary_cache_secret_key_file = ${key}
          evaluator_workers = 4
          evaluator_max_memory_size = 2048
          max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
          store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
          upload_logs_to_binary_cache = true

          # https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional
          <ldap>
            <config>
              <credential>
                class = Password
                password_field = password
                password_type = self_check
              </credential>
              <store>
                class = LDAP
                ldap_server = auth.c3d2.de
                <ldap_server_options>
                  scheme = ldaps
                  timeout = 10
                </ldap_server_options>
                binddn = "uid=search,ou=users,dc=c3d2,dc=de"
                include ldap-password.conf
                start_tls = 0
                <start_tls_options>
                ciphers = TLS_AES_256_GCM_SHA384
                sslversion = tlsv1_3
                # verify = none
                </start_tls_options>
                user_basedn = "ou=users,dc=c3d2,dc=de"
                user_filter = "(&(objectclass=person)(uid=%s))"
                user_scope = one
                user_field = uid
                <user_search_options>
                  deref = always
                </user_search_options>
                # Important for role mappings to work:
                use_roles = 1
                role_basedn = "ou=groups,dc=c3d2,dc=de"
                role_filter = "(&(objectclass=group)(%s))"
                role_scope = one
                role_field = cn
                role_value = dn
                <role_search_options>
                  deref = always
                </role_search_options>
              </store>
            </config>
            <role_mapping>
              # maps directly to user roles
              # Make all users in the hydra-admin group Hydra admins
              hydra-admins = admin
              # Allow all users in the dev group to restart jobs and cancel builds
              #dev = restart-jobs
              #dev = cancel-build
            </role_mapping>
          </ldap>
        '';
    };

    # A rust nix binary cache
    harmonia = {
      enable = true;
      settings = {
        bind = "127.0.0.1:${toString cachePort}";
        workers = 20;
        max_connection_rate = 1024;
        priority = 30;
        sign_key_path = config.sops.secrets."nix-serve/secretKey".path;
      };
    };

    nginx =
      let
        hydraVhost = {
          forceSSL = true;
          enableACME = true;
          locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
        };
      in
      {
        enable = true;
        virtualHosts = {
          "hydra.hq.c3d2.de" = hydraVhost // {
            default = true;
          };
          # "hydra-ca.hq.c3d2.de" = hydraVhost // {
          #   locations."/".proxyPass = "http://192.168.100.2:3001";
          # };
          "hydra.serv.zentralwerk.org" = hydraVhost;
          "nix-serve.hq.c3d2.de" = {
            forceSSL = true;
            enableACME = true;
            locations."/".proxyPass = "http://localhost:${toString cachePort}";
          };
        };
      };

    smartd.enable = true;

    resolved.enable = false;
  };

  sops = {
    defaultSopsFile = ./secrets.yaml;
    secrets."nix-serve/secretKey" = {
      mode = "440";
      owner = config.users.users.hydra-queue-runner.name;
      inherit (config.users.users.hydra-queue-runner) group;
    };
    secrets."ldap/search-user-pw" = {
      mode = "440";
      owner = config.users.users.hydra-queue-runner.name;
      inherit (config.users.users.hydra-queue-runner) group;
      path = "/var/lib/hydra/ldap-password.conf";
    };
  };

  system.stateVersion = "20.09";

  systemd.services = {
    hydra-evaluator.serviceConfig = {
      CPUWeight = 2;
      MemoryHigh = "64G";
      MemoryMax = "64G";
      MemorySwapMax = "64G";
    };

    hydra-init.preStart = let
      makesSenseForQemuUser = feature:
        ! (builtins.elem feature [ "kvm" "benchmark" ]);
      # strips features that don't make sense on qemu-user
      extraPlatformSystemFeatures =
        builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
    in
    # both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
    ''
      cat << EOF > ~/machines
      localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
      hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
      EOF
    '';

    nix-daemon.serviceConfig = {
      CPUWeight = 5;
      MemoryHigh = "64G";
      MemoryMax = "64G";
      MemorySwapMax = "64G";
    };
  };

  # allow reading nix-serve secret
  users.users.harmonia.extraGroups = [ "hydra" ];
}