343 lines
11 KiB
Nix
343 lines
11 KiB
Nix
{ config, lib, libS, pkgs, ... }:
|
|
|
|
let
|
|
cachePort = 5000;
|
|
in
|
|
{
|
|
imports = [
|
|
./hardware-configuration.nix
|
|
./network.nix
|
|
./updater.nix
|
|
];
|
|
|
|
c3d2 = {
|
|
baremetal = true;
|
|
hq.statistics.enable = true;
|
|
};
|
|
|
|
boot = {
|
|
tmpOnTmpfs = true;
|
|
tmpOnTmpfsSize = "80%";
|
|
kernelModules = [ "kvm-intel" ];
|
|
kernelParams = [ "mitigations=off" "preempt=none" ];
|
|
loader = {
|
|
efi.canTouchEfiVariables = true;
|
|
systemd-boot.enable = true;
|
|
};
|
|
# For cross-building
|
|
binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
|
|
};
|
|
|
|
nix = {
|
|
buildMachines = [{
|
|
hostName = "client@dacbert.hq.c3d2.de";
|
|
system = lib.concatStringsSep "," [
|
|
# "aarch64-linux" # very slow compared to gallium
|
|
"armv6l-linux" "armv7l-linux"
|
|
];
|
|
supportedFeatures = [ "kvm" "nixos-test" ];
|
|
maxJobs = 1;
|
|
}];
|
|
daemonCPUSchedPolicy = "idle";
|
|
daemonIOSchedClass = "idle";
|
|
daemonIOSchedPriority = 7;
|
|
optimise = {
|
|
automatic = true;
|
|
dates = [ "05:30" ];
|
|
};
|
|
remoteBuilder = {
|
|
enable = true;
|
|
sshPublicKeys = config.users.users.root.openssh.authorizedKeys.keys ++ [
|
|
/* "..." */
|
|
];
|
|
};
|
|
settings = {
|
|
allowed-uris = "http:// https:// ssh://";
|
|
builders-use-substitutes = true;
|
|
cores = 20;
|
|
keep-outputs = true;
|
|
max-jobs = 8;
|
|
trusted-users = [ "hydra" "root" "@wheel" ];
|
|
system-features = [
|
|
"kvm" "big-parallel"
|
|
"nixos-test" "benchmark"
|
|
];
|
|
};
|
|
extraOptions = ''
|
|
!include ${config.sops.secrets."nix/access-tokens".path}
|
|
'';
|
|
};
|
|
|
|
containers = {
|
|
# hydra-binfmt-builder = {
|
|
# autoStart = true;
|
|
# config = { ... }: {
|
|
# imports = [ (modulesPath + "/profiles/minimal.nix") ];
|
|
|
|
# networking.firewall.allowedTCPPorts = [ 22 ];
|
|
|
|
# nix = {
|
|
# settings = config.nix.settings;
|
|
# extraOptions = config.nix.extraOptions;
|
|
# };
|
|
|
|
# services.openssh.enable = true;
|
|
|
|
# system.stateVersion = "22.11";
|
|
|
|
# users.users."root".openssh.authorizedKeys.keys = [
|
|
# "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBga6vW8lnbFKl+Yd2xBiF71FRyV14eDUnqcMc2AWifI root@hydra"
|
|
# ];
|
|
# };
|
|
# hostAddress = "192.168.100.1";
|
|
# localAddress = "192.168.100.3";
|
|
# privateNetwork = true;
|
|
# };
|
|
|
|
# disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
|
|
# hydra-ca = {
|
|
# autoStart = true;
|
|
# config = { ... }: {
|
|
# imports = [
|
|
# hydra-ca.nixosModules.hydra
|
|
# ];
|
|
|
|
# environment.systemPackages = with pkgs; [ git ];
|
|
|
|
# networking.firewall.allowedTCPPorts = [ 3001 ];
|
|
|
|
# nix = {
|
|
# settings = {
|
|
# allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
|
|
# builders-use-substitutes = true;
|
|
# experimental-features = "ca-derivations nix-command flakes";
|
|
# extra-substituters = "https://cache.ngi0.nixos.org/";
|
|
# extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
|
|
# substituters = [
|
|
# "https://cache.ngi0.nixos.org/"
|
|
# ];
|
|
# trusted-public-keys = [
|
|
# "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
|
|
# ];
|
|
# };
|
|
# };
|
|
|
|
# nixpkgs = {
|
|
# # config.contentAddressedByDefault = true;
|
|
# overlays = [ self.overlay ];
|
|
# };
|
|
|
|
# services = {
|
|
# hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
|
|
# hydraURL = "https://hydra-ca.hq.c3d2.de";
|
|
# port = 3001;
|
|
# };
|
|
# };
|
|
|
|
# system.stateVersion = "22.05"; # Did you read the comment? No.
|
|
# };
|
|
# hostAddress = "192.168.100.1";
|
|
# localAddress = "192.168.100.2";
|
|
# privateNetwork = true;
|
|
# };
|
|
};
|
|
|
|
networking = {
|
|
hostId = "3f0c4ec4";
|
|
hostName = "hydra";
|
|
firewall.enable = false;
|
|
nameservers = [ "172.20.73.8" "9.9.9.9" ];
|
|
# nat = {
|
|
# enable = true;
|
|
# externalInterface = "serv";
|
|
# internalInterfaces = [ "ve-hydra-biLqAU" ];
|
|
# };
|
|
};
|
|
|
|
programs.ssh.knownHosts = lib.mkMerge [
|
|
(libS.mkPubKey "192.168.100.3" "ssh-ed25519" "AAAAC3NzaC1lZDI1NTE5AAAAIBqrnoVELFvO9uc5VlLjiNAXyRTCWUMp5WiTF6o9UorJ")
|
|
(libS.mkPubKey "192.168.100.3" "ssh-rsa" "AAAAB3NzaC1yc2EAAAADAQABAAACAQCwofGcB1HIkIDWR9QNjl/9R39pLusYW2tvmGCZ9p0kfH1ml76OeWHZdXfjpwZJgRM+mk+sbfgKL3xfha+vPiLPJCfMUnKpgoM6zC5i/wi4Ywenh4hPFZG4moVFPBjcMUPmWw7vtED6n5dcW+LOeeuOGwEoBv72UiwhQVg7ULJIT0wu/lj2uduNwiSq8fmxeKZqB+jnJzpc56hGejuWWsGfgpIt2gWirOCqaxNoyRjt/rdGpHsRi8POBIjh5FsvTZVG0zJSgz0ubBsoCivgIr9fGKGxr0dLfDfqqNtrDFwDkkSiymcuo7zRU506pRLeTdrKPhPhvQg3aPOYAQcyvoJKo8xyMim5CbkbIo6TM7os5ubYoNpJ6+WSicYZaI4CG6X7kThkellAKy+yynlwnTTe5Q0DwUJr0znGy4Yi6t/VVE/bFEuAmb0DFbWVf2VqecFAe635hOxmQzzhaf1Zrf4epzcom833o12XdA6abfvuD3dVFSq/9ClzIBFkywNd22LrWhH2Wnh0u38xyHHTdGRQE5z5BKV0TevnmLgni92vMLyoTOdiC4UGhB71ED6tckN0qifzjvAGB2CAr+XT1Zy7ECPXC3SwqBYxcOb10j9pJsdx/gkjg8bovhr4Ve1x5blkzNvLbHA9jCITvfY3ke65JmL/loK1EEoS7odJGrQAbw==")
|
|
];
|
|
|
|
services = {
|
|
hydra = {
|
|
enable = true;
|
|
buildMachinesFiles = [
|
|
"/etc/nix/machines"
|
|
"/var/lib/hydra/machines"
|
|
];
|
|
hydraURL = "https://hydra.hq.c3d2.de";
|
|
ldap = {
|
|
enable = true;
|
|
roleMappings = [
|
|
{ hydra-admins = "admin"; }
|
|
];
|
|
};
|
|
logo = ./c3d2.svg;
|
|
minimumDiskFree = 50;
|
|
minimumDiskFreeEvaluator = 50;
|
|
notificationSender = "hydra@spam.works";
|
|
useSubstitutes = true;
|
|
extraConfig =
|
|
let
|
|
key = config.sops.secrets."nix/signing-key/secretKey".path;
|
|
in
|
|
''
|
|
binary_cache_secret_key_file = ${key}
|
|
compress_num_threads = 4
|
|
evaluator_workers = 4
|
|
evaluator_max_memory_size = 2048
|
|
max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
|
|
store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
|
|
upload_logs_to_binary_cache = true
|
|
'';
|
|
};
|
|
|
|
# A rust nix binary cache
|
|
harmonia = {
|
|
enable = true;
|
|
settings = {
|
|
bind = "[::]:${toString cachePort}";
|
|
workers = 20;
|
|
max_connection_rate = 1024;
|
|
priority = 50;
|
|
};
|
|
signKeyPath = config.sops.secrets."nix/signing-key/secretKey".path;
|
|
};
|
|
|
|
nginx = {
|
|
enable = true;
|
|
virtualHosts = {
|
|
"hydra.hq.c3d2.de" = {
|
|
default = true;
|
|
enableACME = true;
|
|
forceSSL = true;
|
|
locations."/".proxyPass = "http://127.0.0.1:${toString config.services.hydra.port}";
|
|
serverAliases = [
|
|
"hydra-ca.hq.c3d2.de"
|
|
"hydra.serv.zentralwerk.org"
|
|
];
|
|
};
|
|
# "hydra-ca.hq.c3d2.de" = {
|
|
# enableACME = true;
|
|
# forceSSL = true;
|
|
# locations."/".proxyPass = "http://192.168.100.2:3001";
|
|
# };
|
|
"nix-cache.hq.c3d2.de" = {
|
|
forceSSL = true;
|
|
enableACME = true;
|
|
locations."/".proxyPass = "http://127.0.0.1:${toString cachePort}";
|
|
serverAliases = [
|
|
"nix-serve.hq.c3d2.de"
|
|
];
|
|
};
|
|
};
|
|
};
|
|
|
|
portunus.addToHosts = true;
|
|
|
|
postgresql = {
|
|
package = pkgs.postgresql_15;
|
|
upgrade.stopServices = [ "hydra-evaluator" "hydra-queue-runner" "hydra-server" ];
|
|
};
|
|
|
|
resolved.enable = false;
|
|
|
|
zfs.trim.enable = true;
|
|
};
|
|
|
|
simd.arch = "ivybridge";
|
|
|
|
sops = {
|
|
defaultSopsFile = ./secrets.yaml;
|
|
secrets = {
|
|
"ldap/search-user-pw" = {
|
|
mode = "440";
|
|
owner = config.users.users.hydra-queue-runner.name;
|
|
inherit (config.users.users.hydra-queue-runner) group;
|
|
path = "/var/lib/hydra/ldap-password.conf";
|
|
};
|
|
"machine-id" = {
|
|
mode = "444";
|
|
path = "/etc/machine-id";
|
|
};
|
|
"nix/access-tokens" = {
|
|
mode = "444";
|
|
};
|
|
"nix/signing-key/secretKey" = {
|
|
mode = "440";
|
|
owner = config.users.users.hydra-queue-runner.name;
|
|
inherit (config.users.users.hydra-queue-runner) group;
|
|
};
|
|
"restic/password".owner = "root";
|
|
"restic/repository/server8".owner = "root";
|
|
"ssh-keys/hydra/private" = {
|
|
owner = "hydra";
|
|
mode = "400";
|
|
path = "/var/lib/hydra/.ssh/id_ed25519";
|
|
};
|
|
"ssh-keys/hydra/public" = {
|
|
owner = "hydra";
|
|
mode = "440";
|
|
path = "/var/lib/hydra/.ssh/id_ed25519.pub";
|
|
};
|
|
"ssh-keys/root/private" = {
|
|
owner = "hydra-queue-runner";
|
|
mode = "400";
|
|
path = "/var/lib/hydra/queue-runner/.ssh/id_ed25519";
|
|
};
|
|
"ssh-keys/root/public" = {
|
|
owner = "hydra-queue-runner";
|
|
mode = "440";
|
|
path = "/var/lib/hydra/queue-runner/.ssh/id_ed25519.pub";
|
|
};
|
|
"ssh-keys/updater/private" = {
|
|
owner = "updater";
|
|
mode = "400";
|
|
path = "/var/lib/updater/.ssh/id_ed25519";
|
|
};
|
|
"ssh-keys/updater/public" = {
|
|
owner = "updater";
|
|
mode = "440";
|
|
path = "/var/lib/updater/.ssh/id_ed25519.pub";
|
|
};
|
|
};
|
|
};
|
|
|
|
system.stateVersion = "20.09";
|
|
|
|
systemd.services = {
|
|
hydra-evaluator.serviceConfig = {
|
|
CPUWeight = 2;
|
|
MemoryHigh = "64G";
|
|
MemoryMax = "64G";
|
|
MemorySwapMax = "64G";
|
|
};
|
|
|
|
hydra-init.preStart = let
|
|
localPlatforms = feature: !(builtins.elem feature [ "x86_64-linux" "i686-linux" ]);
|
|
# strips features that don't make sense on qemu-user
|
|
extraPlatforms = builtins.filter localPlatforms config.nix.settings.extra-platforms;
|
|
in
|
|
# both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
|
|
# multiple container max-jobs by X because binfmt is very slow especially in configure scripts
|
|
''
|
|
cat << EOF > ~/machines
|
|
localhost x86_64-linux,i686-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
|
|
# local container to have an extra nix daemon for binfmt
|
|
# NOTE: currently very, very slow and usually builds do not finish in any amount of time
|
|
# root@192.168.100.3 ${lib.concatStringsSep "," extraPlatforms} - ${toString (config.nix.settings.max-jobs * 3)} 10 big-parallel,nixos-test -
|
|
# sandro's native aarch64 builder
|
|
${config.nix.remoteBuilder.name}@gallium.supersandro.de aarch64-linux - 4 20 big-parallel,nixos-test,benchmark -
|
|
EOF
|
|
'';
|
|
|
|
nix-daemon.serviceConfig = {
|
|
CPUWeight = 5;
|
|
MemoryHigh = "64G";
|
|
MemoryMax = "64G";
|
|
MemorySwapMax = "64G";
|
|
};
|
|
};
|
|
}
|