From e43ce3b29bf91659523a768f5689dbdb4e443551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sun, 18 Dec 2022 19:03:24 +0100 Subject: [PATCH] hydra: combine default.nix and hydra.nix, remove nix store over ssh --- hosts/hydra/cache-pub.key | 1 - hosts/hydra/default.nix | 285 ++++++++++++++++++++++++++++++++++---- hosts/hydra/hydra.nix | 255 ---------------------------------- hosts/hydra/secrets.yaml | 5 +- modules/c3d2.nix | 2 +- 5 files changed, 264 insertions(+), 284 deletions(-) delete mode 100644 hosts/hydra/cache-pub.key diff --git a/hosts/hydra/cache-pub.key b/hosts/hydra/cache-pub.key deleted file mode 100644 index be3677da..00000000 --- a/hosts/hydra/cache-pub.key +++ /dev/null @@ -1 +0,0 @@ -nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps= \ No newline at end of file diff --git a/hosts/hydra/default.nix b/hosts/hydra/default.nix index 93ffdba1..5353f738 100644 --- a/hosts/hydra/default.nix +++ b/hosts/hydra/default.nix @@ -1,5 +1,8 @@ -{ config, pkgs, ... }: +{ config, lib, pkgs, zentralwerk, ... }: +let + cachePort = 5000; +in { imports = [ ./hardware-configuration.nix @@ -9,28 +12,9 @@ ../../modules/c3d2.nix ]; - nixpkgs.config.allowUnfree = true; - - security.pam.enableSSHAgentAuth = true; - - services.openssh = { - enable = true; - passwordAuthentication = false; - }; - - programs.mosh.enable = true; - - nix = { - sshServe.enable = true; - settings = { - # # restore default overridden by harmonia - # allowed-users = lib.mkForce [ "*" ]; - auto-optimise-store = true; - cores = 20; - keep-outputs = true; - max-jobs = 8; - trusted-users = [ "@wheel" ]; - }; + c3d2 = { + hq.statistics.enable = true; + simd.arch = "ivybridge"; }; boot = { @@ -43,14 +27,265 @@ binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ]; }; + nix = { + buildMachines = [{ + hostName = "client@dacbert.hq.c3d2.de"; + system = lib.concatStringsSep "," [ + "aarch64-linux" "armv6l-linux" "armv7l-linux" + ]; + supportedFeatures = [ "kvm" "nixos-test" ]; + maxJobs = 1; + }]; + daemonCPUSchedPolicy = "idle"; + daemonIOSchedClass = "idle"; + daemonIOSchedPriority = 7; + settings = { + allowed-uris = "http:// https:// ssh://"; + auto-optimise-store = true; + builders-use-substitutes = true; + cores = 20; + keep-outputs = true; + max-jobs = 8; + trusted-users = [ "hydra" "root" "@wheel" ]; + }; + }; + + nixpkgs.config.allowUnfree = true; + + # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store + # containers = { + # hydra-ca = { + # autoStart = true; + # config = { ... }: { + # imports = [ + # hydra-ca.nixosModules.hydra + # ]; + + # environment.systemPackages = with pkgs; [ git ]; + + # networking.firewall.allowedTCPPorts = [ 3001 ]; + + # nix = { + # settings = { + # allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/"; + # builders-use-substitutes = true; + # experimental-features = "ca-derivations nix-command flakes"; + # extra-substituters = "https://cache.ngi0.nixos.org/"; + # extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="; + # substituters = [ + # "https://cache.ngi0.nixos.org/" + # ]; + # trusted-public-keys = [ + # "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=" + # ]; + # }; + # }; + + # nixpkgs = { + # # config.contentAddressedByDefault = true; + # overlays = [ self.overlay ]; + # }; + + # services = { + # hydra-dev = lib.recursiveUpdate config.services.hydra-dev { + # hydraURL = "https://hydra-ca.hq.c3d2.de"; + # port = 3001; + # }; + # }; + + # system.stateVersion = "22.05"; # Did you read the comment? No. + # }; + # hostAddress = "192.168.100.1"; + # localAddress = "192.168.100.2"; + # privateNetwork = true; + # }; + # }; + networking = { + hosts = with zentralwerk.lib.config.site.net.serv; { + ${hosts6.up4.auth} = [ "auth.c3d2.de" ]; + ${hosts4.auth} = [ "auth.c3d2.de" ]; + }; hostName = "hydra"; firewall.enable = false; nameservers = [ "172.20.73.8" "9.9.9.9" ]; + # nat = { + # enable = true; + # externalInterface = "serv"; + # internalInterfaces = [ "ve-hydra-ca" ]; + # }; }; - c3d2.hq.statistics.enable = true; - services.smartd.enable = true; + services = { + hydra = { + enable = true; + buildMachinesFiles = [ + "/etc/nix/machines" + "/var/lib/hydra/machines" + ]; + hydraURL = "https://hydra.hq.c3d2.de"; + logo = ./c3d2.svg; + minimumDiskFree = 50; + minimumDiskFreeEvaluator = 50; + notificationSender = "hydra@spam.works"; + useSubstitutes = true; + extraConfig = + let + key = config.sops.secrets."nix-serve/secretKey".path; + in + '' + binary_cache_secret_key_file = ${key} + evaluator_workers = 4 + evaluator_max_memory_size = 2048 + max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images + store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd + upload_logs_to_binary_cache = true + + # https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional + + + + class = Password + password_field = password + password_type = self_check + + + class = LDAP + ldap_server = auth.c3d2.de + + scheme = ldaps + timeout = 10 + + binddn = "uid=search,ou=users,dc=c3d2,dc=de" + include ldap-password.conf + start_tls = 0 + + ciphers = TLS_AES_256_GCM_SHA384 + sslversion = tlsv1_3 + # verify = none + + user_basedn = "ou=users,dc=c3d2,dc=de" + user_filter = "(&(objectclass=person)(uid=%s))" + user_scope = one + user_field = uid + + deref = always + + # Important for role mappings to work: + use_roles = 1 + role_basedn = "ou=groups,dc=c3d2,dc=de" + role_filter = "(&(objectclass=group)(%s))" + role_scope = one + role_field = cn + role_value = dn + + deref = always + + + + + # maps directly to user roles + # Make all users in the hydra-admin group Hydra admins + hydra-admins = admin + # Allow all users in the dev group to restart jobs and cancel builds + #dev = restart-jobs + #dev = cancel-build + + + ''; + }; + + # A rust nix binary cache + harmonia = { + enable = true; + settings = { + bind = "127.0.0.1:${toString cachePort}"; + workers = 20; + max_connection_rate = 1024; + priority = 30; + sign_key_path = config.sops.secrets."nix-serve/secretKey".path; + }; + }; + + nginx = + let + hydraVhost = { + forceSSL = true; + enableACME = true; + locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}"; + }; + in + { + enable = true; + virtualHosts = { + "hydra.hq.c3d2.de" = hydraVhost // { + default = true; + }; + # "hydra-ca.hq.c3d2.de" = hydraVhost // { + # locations."/".proxyPass = "http://192.168.100.2:3001"; + # }; + "hydra.serv.zentralwerk.org" = hydraVhost; + "nix-serve.hq.c3d2.de" = { + forceSSL = true; + enableACME = true; + locations."/".proxyPass = "http://localhost:${toString cachePort}"; + }; + }; + }; + + smartd.enable = true; + + resolved.enable = false; + }; + + sops = { + defaultSopsFile = ./secrets.yaml; + secrets."nix-serve/secretKey" = { + mode = "440"; + owner = config.users.users.hydra-queue-runner.name; + inherit (config.users.users.hydra-queue-runner) group; + }; + secrets."ldap/search-user-pw" = { + mode = "440"; + owner = config.users.users.hydra-queue-runner.name; + inherit (config.users.users.hydra-queue-runner) group; + path = "/var/lib/hydra/ldap-password.conf"; + }; + }; system.stateVersion = "20.09"; + + systemd.services = { + hydra-evaluator.serviceConfig = { + CPUWeight = 2; + MemoryHigh = "64G"; + MemoryMax = "64G"; + MemorySwapMax = "64G"; + }; + + hydra-init.preStart = let + makesSenseForQemuUser = feature: + ! (builtins.elem feature [ "kvm" "benchmark" ]); + # strips features that don't make sense on qemu-user + extraPlatformSystemFeatures = + builtins.filter makesSenseForQemuUser config.nix.settings.system-features; + in + # both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches + '' + cat << EOF > ~/machines + localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} - + hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} - + EOF + ''; + + nix-daemon.serviceConfig = { + CPUWeight = 5; + MemoryHigh = "64G"; + MemoryMax = "64G"; + MemorySwapMax = "64G"; + }; + }; + + # allow reading nix-serve secret + users.users.harmonia.extraGroups = [ "hydra" ]; } diff --git a/hosts/hydra/hydra.nix b/hosts/hydra/hydra.nix index 9e91e45e..7f69a84e 100644 --- a/hosts/hydra/hydra.nix +++ b/hosts/hydra/hydra.nix @@ -1,257 +1,2 @@ { config, lib, zentralwerk, ... }: -let - cachePort = 5000; - -in -{ - # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store - # containers = { - # hydra-ca = { - # autoStart = true; - # config = { ... }: { - # imports = [ - # hydra-ca.nixosModules.hydra - # ]; - - # environment.systemPackages = with pkgs; [ git ]; - - # networking.firewall.allowedTCPPorts = [ 3001 ]; - - # nix = { - # settings = { - # allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/"; - # builders-use-substitutes = true; - # experimental-features = "ca-derivations nix-command flakes"; - # extra-substituters = "https://cache.ngi0.nixos.org/"; - # extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="; - # substituters = [ - # "https://cache.ngi0.nixos.org/" - # ]; - # trusted-public-keys = [ - # "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=" - # ]; - # }; - # }; - - # nixpkgs = { - # # config.contentAddressedByDefault = true; - # overlays = [ self.overlay ]; - # }; - - # services = { - # hydra-dev = lib.recursiveUpdate config.services.hydra-dev { - # hydraURL = "https://hydra-ca.hq.c3d2.de"; - # port = 3001; - # }; - # }; - - # system.stateVersion = "22.05"; # Did you read the comment? No. - # }; - # hostAddress = "192.168.100.1"; - # localAddress = "192.168.100.2"; - # privateNetwork = true; - # }; - # }; - - networking = { - hosts = with zentralwerk.lib.config.site.net.serv; { - ${hosts6.up4.auth} = [ "auth.c3d2.de" ]; - ${hosts4.auth} = [ "auth.c3d2.de" ]; - }; - # nat = { - # enable = true; - # externalInterface = "serv"; - # internalInterfaces = [ "ve-hydra-ca" ]; - # }; - }; - - nix = { - buildMachines = [{ - hostName = "client@dacbert.hq.c3d2.de"; - system = lib.concatStringsSep "," [ - "aarch64-linux" "armv6l-linux" "armv7l-linux" - ]; - supportedFeatures = [ "kvm" "nixos-test" ]; - maxJobs = 1; - }]; - daemonCPUSchedPolicy = "idle"; - daemonIOSchedClass = "idle"; - daemonIOSchedPriority = 7; - settings = { - allowed-uris = "http:// https:// ssh://"; - builders-use-substitutes = true; - trusted-users = [ "hydra" "root" ]; - }; - }; - - c3d2.simd.arch = "ivybridge"; - - services = { - hydra = { - enable = true; - buildMachinesFiles = [ - "/etc/nix/machines" - "/var/lib/hydra/machines" - ]; - hydraURL = "https://hydra.hq.c3d2.de"; - logo = ./c3d2.svg; - minimumDiskFree = 50; - minimumDiskFreeEvaluator = 50; - notificationSender = "hydra@spam.works"; - useSubstitutes = true; - extraConfig = - let - key = config.sops.secrets."nix-serve/secretKey".path; - in - '' - binary_cache_secret_key_file = ${key} - evaluator_workers = 4 - evaluator_max_memory_size = 2048 - max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images - store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd - upload_logs_to_binary_cache = true - - # https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional - - - - class = Password - password_field = password - password_type = self_check - - - class = LDAP - ldap_server = auth.c3d2.de - - scheme = ldaps - timeout = 10 - - binddn = "uid=search,ou=users,dc=c3d2,dc=de" - include ldap-password.conf - start_tls = 0 - - ciphers = TLS_AES_256_GCM_SHA384 - sslversion = tlsv1_3 - # verify = none - - user_basedn = "ou=users,dc=c3d2,dc=de" - user_filter = "(&(objectclass=person)(uid=%s))" - user_scope = one - user_field = uid - - deref = always - - # Important for role mappings to work: - use_roles = 1 - role_basedn = "ou=groups,dc=c3d2,dc=de" - role_filter = "(&(objectclass=group)(%s))" - role_scope = one - role_field = cn - role_value = dn - - deref = always - - - - - # maps directly to user roles - # Make all users in the hydra-admin group Hydra admins - hydra-admins = admin - # Allow all users in the dev group to restart jobs and cancel builds - #dev = restart-jobs - #dev = cancel-build - - - ''; - }; - - # A rust nix binary cache - harmonia = { - enable = true; - settings = { - bind = "127.0.0.1:${toString cachePort}"; - workers = 20; - max_connection_rate = 1024; - priority = 30; - sign_key_path = config.sops.secrets."nix-serve/secretKey".path; - }; - }; - - nginx = - let - hydraVhost = { - forceSSL = true; - enableACME = true; - locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}"; - }; - in - { - enable = true; - virtualHosts = { - "hydra.hq.c3d2.de" = hydraVhost // { - default = true; - }; - # "hydra-ca.hq.c3d2.de" = hydraVhost // { - # locations."/".proxyPass = "http://192.168.100.2:3001"; - # }; - "hydra.serv.zentralwerk.org" = hydraVhost; - "nix-serve.hq.c3d2.de" = { - forceSSL = true; - enableACME = true; - locations."/".proxyPass = "http://localhost:${toString cachePort}"; - }; - }; - }; - resolved.enable = false; - }; - - sops = { - defaultSopsFile = ./secrets.yaml; - secrets."nix-serve/secretKey" = { - mode = "440"; - owner = config.users.users.hydra-queue-runner.name; - inherit (config.users.users.hydra-queue-runner) group; - }; - secrets."ldap/search-user-pw" = { - mode = "440"; - owner = config.users.users.hydra-queue-runner.name; - inherit (config.users.users.hydra-queue-runner) group; - path = "/var/lib/hydra/ldap-password.conf"; - }; - }; - - systemd.services = { - hydra-evaluator.serviceConfig = { - CPUWeight = 2; - MemoryHigh = "64G"; - MemoryMax = "64G"; - MemorySwapMax = "64G"; - }; - - hydra-init.preStart = let - makesSenseForQemuUser = feature: - ! (builtins.elem feature [ "kvm" "benchmark" ]); - # strips features that don't make sense on qemu-user - extraPlatformSystemFeatures = - builtins.filter makesSenseForQemuUser config.nix.settings.system-features; - in - # both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches - '' - cat << EOF > ~/machines - localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} - - hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} - - EOF - ''; - - nix-daemon.serviceConfig = { - CPUWeight = 5; - MemoryHigh = "64G"; - MemoryMax = "64G"; - MemorySwapMax = "64G"; - }; - }; - - # allow reading nix-serve secret - users.users.harmonia.extraGroups = [ "hydra" ]; -} diff --git a/hosts/hydra/secrets.yaml b/hosts/hydra/secrets.yaml index 0d7a1dcf..bd0179f4 100644 --- a/hosts/hydra/secrets.yaml +++ b/hosts/hydra/secrets.yaml @@ -1,4 +1,5 @@ nix-serve: + publicKey: ENC[AES256_GCM,data:sR5wk7yvH5+lLpSIP0zNqCLvDRRvR8ws4Q8rVcVJx8YkrywwPcvIsJ1h6mVEu3nc6SLoZlQsuxOGCyNGD98CBNY=,iv:fFV2D27hWoxGtqVt3EnS4hMlrqW5LnIZ3LB5k4xmFWs=,tag:g7R1ossy2On6B2nVfKC9iA==,type:str] secretKey: ENC[AES256_GCM,data:cm84sA7E6AnzpVoYuaYepbHGWkRigLdD2RxN21UsXCe7FXQxeTQTxxbzVxJ3G9Lt3kRXuZnODntOo5EQKhs46+wzpO8YLKQxkJXrdluXoGVIWl3/6QFVq66XLJ2i6G4eBK9IH0DYJ+anj8/i8Q==,iv:GEM8Vmx0A8LfJo7QOl0N67Cgk+JqHpp7r+41VivmTg4=,tag:O4Kq4WKgbyt354HSa/7eQQ==,type:str] ldap: search-user-pw: ENC[AES256_GCM,data:tSWin/QPIow2P5Aps/XaT42J+MXb8+a24SEri1QjF1O3bDlCxcR8RHqSX8d4Vg==,iv:P5qMaE2cdKxTaXuKO2nh+LDhKkY3psSlWf+JckmUYt4=,tag:eq8XW7P6FNlkviY5PydkZg==,type:str] @@ -26,8 +27,8 @@ sops: K3Bpb0svZ1YvVm9ha1ArdVBlN3NHM0kKM6CEQ+dStjEsgppQZYjb1zwyzfwAc0FI O5+vi2x8/N/1OH5jeVzLnLjOhXRXrYcR9EDsjT+KDo0ykYh+NjB0DA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2022-12-06T14:25:54Z" - mac: ENC[AES256_GCM,data:4cOG88FIG7UhVb/r8Aq1Nme5+qCpEdpjV+BLOISm1Y6MYgxFTDqCzV2FOdKztpVou5Nly9JUvKfz6eiCWbbIbaO5/DYUObiTKZXv6B1x6blnIW8vMtqcdYWOXH62ycHMV+Sha0D41eXmNp3K1Vs+k3OwYZyHK1HFOqqQ2jpy+Ps=,iv:u0O/A/GBBpDTJVFBfiFzDOIIR5o479YI11fgrv0mR0A=,tag:E9OuAAOhfbzPcnA6Ij6LMA==,type:str] + lastmodified: "2022-12-18T17:49:41Z" + mac: ENC[AES256_GCM,data:oQGsk4gJ4dGCr3zCanOpSsKQyx/OuWiYe0ef8pvWhqFq0+YvwUzoiF3jQmqahLWGLHUl1yb1kb91GEr7q47G83Z9YXVVtTTzJUWpKZQL1WD12f5ERPSlKnT251fvoMtqNqKrRirGA9ao8OxVpeh3/SQCAGTqWAcpz1J7dxvL7E4=,iv:td9t1MKcXH5RYZxsD1wA4oG61AfLLGSiVIEq/sPRZG4=,tag:UqHGkKqXeeBYXQYtvplCdA==,type:str] pgp: - created_at: "2022-07-15T23:31:58Z" enc: | diff --git a/modules/c3d2.nix b/modules/c3d2.nix index bb2b1191..7f989ef7 100644 --- a/modules/c3d2.nix +++ b/modules/c3d2.nix @@ -401,7 +401,7 @@ in fallback = true; # don't self feed hydra trusted-public-keys = lib.mkIf (config.networking.hostName != "hydra") [ - (builtins.readFile ../hosts/hydra/cache-pub.key) + "nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps=" ]; substituters = lib.mkIf (config.networking.hostName != "hydra") ( lib.mkBefore [ "https://nix-serve.hq.c3d2.de" ]