hydra: combine default.nix and hydra.nix, remove nix store over ssh

This commit is contained in:
Sandro - 2022-12-18 19:03:24 +01:00
parent 7d18fad68a
commit e43ce3b29b
Signed by: sandro
GPG Key ID: 3AF5A43A3EECC2E5
5 changed files with 264 additions and 284 deletions

View File

@ -1 +0,0 @@
nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps=

View File

@ -1,5 +1,8 @@
{ config, pkgs, ... }:
{ config, lib, pkgs, zentralwerk, ... }:
let
cachePort = 5000;
in
{
imports = [
./hardware-configuration.nix
@ -9,28 +12,9 @@
../../modules/c3d2.nix
];
nixpkgs.config.allowUnfree = true;
security.pam.enableSSHAgentAuth = true;
services.openssh = {
enable = true;
passwordAuthentication = false;
};
programs.mosh.enable = true;
nix = {
sshServe.enable = true;
settings = {
# # restore default overridden by harmonia
# allowed-users = lib.mkForce [ "*" ];
auto-optimise-store = true;
cores = 20;
keep-outputs = true;
max-jobs = 8;
trusted-users = [ "@wheel" ];
};
c3d2 = {
hq.statistics.enable = true;
simd.arch = "ivybridge";
};
boot = {
@ -43,14 +27,265 @@
binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
};
nix = {
buildMachines = [{
hostName = "client@dacbert.hq.c3d2.de";
system = lib.concatStringsSep "," [
"aarch64-linux" "armv6l-linux" "armv7l-linux"
];
supportedFeatures = [ "kvm" "nixos-test" ];
maxJobs = 1;
}];
daemonCPUSchedPolicy = "idle";
daemonIOSchedClass = "idle";
daemonIOSchedPriority = 7;
settings = {
allowed-uris = "http:// https:// ssh://";
auto-optimise-store = true;
builders-use-substitutes = true;
cores = 20;
keep-outputs = true;
max-jobs = 8;
trusted-users = [ "hydra" "root" "@wheel" ];
};
};
nixpkgs.config.allowUnfree = true;
# disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
# containers = {
# hydra-ca = {
# autoStart = true;
# config = { ... }: {
# imports = [
# hydra-ca.nixosModules.hydra
# ];
# environment.systemPackages = with pkgs; [ git ];
# networking.firewall.allowedTCPPorts = [ 3001 ];
# nix = {
# settings = {
# allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
# builders-use-substitutes = true;
# experimental-features = "ca-derivations nix-command flakes";
# extra-substituters = "https://cache.ngi0.nixos.org/";
# extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
# substituters = [
# "https://cache.ngi0.nixos.org/"
# ];
# trusted-public-keys = [
# "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
# ];
# };
# };
# nixpkgs = {
# # config.contentAddressedByDefault = true;
# overlays = [ self.overlay ];
# };
# services = {
# hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
# hydraURL = "https://hydra-ca.hq.c3d2.de";
# port = 3001;
# };
# };
# system.stateVersion = "22.05"; # Did you read the comment? No.
# };
# hostAddress = "192.168.100.1";
# localAddress = "192.168.100.2";
# privateNetwork = true;
# };
# };
networking = {
hosts = with zentralwerk.lib.config.site.net.serv; {
${hosts6.up4.auth} = [ "auth.c3d2.de" ];
${hosts4.auth} = [ "auth.c3d2.de" ];
};
hostName = "hydra";
firewall.enable = false;
nameservers = [ "172.20.73.8" "9.9.9.9" ];
# nat = {
# enable = true;
# externalInterface = "serv";
# internalInterfaces = [ "ve-hydra-ca" ];
# };
};
c3d2.hq.statistics.enable = true;
services.smartd.enable = true;
services = {
hydra = {
enable = true;
buildMachinesFiles = [
"/etc/nix/machines"
"/var/lib/hydra/machines"
];
hydraURL = "https://hydra.hq.c3d2.de";
logo = ./c3d2.svg;
minimumDiskFree = 50;
minimumDiskFreeEvaluator = 50;
notificationSender = "hydra@spam.works";
useSubstitutes = true;
extraConfig =
let
key = config.sops.secrets."nix-serve/secretKey".path;
in
''
binary_cache_secret_key_file = ${key}
evaluator_workers = 4
evaluator_max_memory_size = 2048
max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
upload_logs_to_binary_cache = true
# https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional
<ldap>
<config>
<credential>
class = Password
password_field = password
password_type = self_check
</credential>
<store>
class = LDAP
ldap_server = auth.c3d2.de
<ldap_server_options>
scheme = ldaps
timeout = 10
</ldap_server_options>
binddn = "uid=search,ou=users,dc=c3d2,dc=de"
include ldap-password.conf
start_tls = 0
<start_tls_options>
ciphers = TLS_AES_256_GCM_SHA384
sslversion = tlsv1_3
# verify = none
</start_tls_options>
user_basedn = "ou=users,dc=c3d2,dc=de"
user_filter = "(&(objectclass=person)(uid=%s))"
user_scope = one
user_field = uid
<user_search_options>
deref = always
</user_search_options>
# Important for role mappings to work:
use_roles = 1
role_basedn = "ou=groups,dc=c3d2,dc=de"
role_filter = "(&(objectclass=group)(%s))"
role_scope = one
role_field = cn
role_value = dn
<role_search_options>
deref = always
</role_search_options>
</store>
</config>
<role_mapping>
# maps directly to user roles
# Make all users in the hydra-admin group Hydra admins
hydra-admins = admin
# Allow all users in the dev group to restart jobs and cancel builds
#dev = restart-jobs
#dev = cancel-build
</role_mapping>
</ldap>
'';
};
# A rust nix binary cache
harmonia = {
enable = true;
settings = {
bind = "127.0.0.1:${toString cachePort}";
workers = 20;
max_connection_rate = 1024;
priority = 30;
sign_key_path = config.sops.secrets."nix-serve/secretKey".path;
};
};
nginx =
let
hydraVhost = {
forceSSL = true;
enableACME = true;
locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
};
in
{
enable = true;
virtualHosts = {
"hydra.hq.c3d2.de" = hydraVhost // {
default = true;
};
# "hydra-ca.hq.c3d2.de" = hydraVhost // {
# locations."/".proxyPass = "http://192.168.100.2:3001";
# };
"hydra.serv.zentralwerk.org" = hydraVhost;
"nix-serve.hq.c3d2.de" = {
forceSSL = true;
enableACME = true;
locations."/".proxyPass = "http://localhost:${toString cachePort}";
};
};
};
smartd.enable = true;
resolved.enable = false;
};
sops = {
defaultSopsFile = ./secrets.yaml;
secrets."nix-serve/secretKey" = {
mode = "440";
owner = config.users.users.hydra-queue-runner.name;
inherit (config.users.users.hydra-queue-runner) group;
};
secrets."ldap/search-user-pw" = {
mode = "440";
owner = config.users.users.hydra-queue-runner.name;
inherit (config.users.users.hydra-queue-runner) group;
path = "/var/lib/hydra/ldap-password.conf";
};
};
system.stateVersion = "20.09";
systemd.services = {
hydra-evaluator.serviceConfig = {
CPUWeight = 2;
MemoryHigh = "64G";
MemoryMax = "64G";
MemorySwapMax = "64G";
};
hydra-init.preStart = let
makesSenseForQemuUser = feature:
! (builtins.elem feature [ "kvm" "benchmark" ]);
# strips features that don't make sense on qemu-user
extraPlatformSystemFeatures =
builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
in
# both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
''
cat << EOF > ~/machines
localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
EOF
'';
nix-daemon.serviceConfig = {
CPUWeight = 5;
MemoryHigh = "64G";
MemoryMax = "64G";
MemorySwapMax = "64G";
};
};
# allow reading nix-serve secret
users.users.harmonia.extraGroups = [ "hydra" ];
}

View File

@ -1,257 +1,2 @@
{ config, lib, zentralwerk, ... }:
let
cachePort = 5000;
in
{
# disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
# containers = {
# hydra-ca = {
# autoStart = true;
# config = { ... }: {
# imports = [
# hydra-ca.nixosModules.hydra
# ];
# environment.systemPackages = with pkgs; [ git ];
# networking.firewall.allowedTCPPorts = [ 3001 ];
# nix = {
# settings = {
# allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
# builders-use-substitutes = true;
# experimental-features = "ca-derivations nix-command flakes";
# extra-substituters = "https://cache.ngi0.nixos.org/";
# extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
# substituters = [
# "https://cache.ngi0.nixos.org/"
# ];
# trusted-public-keys = [
# "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
# ];
# };
# };
# nixpkgs = {
# # config.contentAddressedByDefault = true;
# overlays = [ self.overlay ];
# };
# services = {
# hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
# hydraURL = "https://hydra-ca.hq.c3d2.de";
# port = 3001;
# };
# };
# system.stateVersion = "22.05"; # Did you read the comment? No.
# };
# hostAddress = "192.168.100.1";
# localAddress = "192.168.100.2";
# privateNetwork = true;
# };
# };
networking = {
hosts = with zentralwerk.lib.config.site.net.serv; {
${hosts6.up4.auth} = [ "auth.c3d2.de" ];
${hosts4.auth} = [ "auth.c3d2.de" ];
};
# nat = {
# enable = true;
# externalInterface = "serv";
# internalInterfaces = [ "ve-hydra-ca" ];
# };
};
nix = {
buildMachines = [{
hostName = "client@dacbert.hq.c3d2.de";
system = lib.concatStringsSep "," [
"aarch64-linux" "armv6l-linux" "armv7l-linux"
];
supportedFeatures = [ "kvm" "nixos-test" ];
maxJobs = 1;
}];
daemonCPUSchedPolicy = "idle";
daemonIOSchedClass = "idle";
daemonIOSchedPriority = 7;
settings = {
allowed-uris = "http:// https:// ssh://";
builders-use-substitutes = true;
trusted-users = [ "hydra" "root" ];
};
};
c3d2.simd.arch = "ivybridge";
services = {
hydra = {
enable = true;
buildMachinesFiles = [
"/etc/nix/machines"
"/var/lib/hydra/machines"
];
hydraURL = "https://hydra.hq.c3d2.de";
logo = ./c3d2.svg;
minimumDiskFree = 50;
minimumDiskFreeEvaluator = 50;
notificationSender = "hydra@spam.works";
useSubstitutes = true;
extraConfig =
let
key = config.sops.secrets."nix-serve/secretKey".path;
in
''
binary_cache_secret_key_file = ${key}
evaluator_workers = 4
evaluator_max_memory_size = 2048
max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
upload_logs_to_binary_cache = true
# https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional
<ldap>
<config>
<credential>
class = Password
password_field = password
password_type = self_check
</credential>
<store>
class = LDAP
ldap_server = auth.c3d2.de
<ldap_server_options>
scheme = ldaps
timeout = 10
</ldap_server_options>
binddn = "uid=search,ou=users,dc=c3d2,dc=de"
include ldap-password.conf
start_tls = 0
<start_tls_options>
ciphers = TLS_AES_256_GCM_SHA384
sslversion = tlsv1_3
# verify = none
</start_tls_options>
user_basedn = "ou=users,dc=c3d2,dc=de"
user_filter = "(&(objectclass=person)(uid=%s))"
user_scope = one
user_field = uid
<user_search_options>
deref = always
</user_search_options>
# Important for role mappings to work:
use_roles = 1
role_basedn = "ou=groups,dc=c3d2,dc=de"
role_filter = "(&(objectclass=group)(%s))"
role_scope = one
role_field = cn
role_value = dn
<role_search_options>
deref = always
</role_search_options>
</store>
</config>
<role_mapping>
# maps directly to user roles
# Make all users in the hydra-admin group Hydra admins
hydra-admins = admin
# Allow all users in the dev group to restart jobs and cancel builds
#dev = restart-jobs
#dev = cancel-build
</role_mapping>
</ldap>
'';
};
# A rust nix binary cache
harmonia = {
enable = true;
settings = {
bind = "127.0.0.1:${toString cachePort}";
workers = 20;
max_connection_rate = 1024;
priority = 30;
sign_key_path = config.sops.secrets."nix-serve/secretKey".path;
};
};
nginx =
let
hydraVhost = {
forceSSL = true;
enableACME = true;
locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
};
in
{
enable = true;
virtualHosts = {
"hydra.hq.c3d2.de" = hydraVhost // {
default = true;
};
# "hydra-ca.hq.c3d2.de" = hydraVhost // {
# locations."/".proxyPass = "http://192.168.100.2:3001";
# };
"hydra.serv.zentralwerk.org" = hydraVhost;
"nix-serve.hq.c3d2.de" = {
forceSSL = true;
enableACME = true;
locations."/".proxyPass = "http://localhost:${toString cachePort}";
};
};
};
resolved.enable = false;
};
sops = {
defaultSopsFile = ./secrets.yaml;
secrets."nix-serve/secretKey" = {
mode = "440";
owner = config.users.users.hydra-queue-runner.name;
inherit (config.users.users.hydra-queue-runner) group;
};
secrets."ldap/search-user-pw" = {
mode = "440";
owner = config.users.users.hydra-queue-runner.name;
inherit (config.users.users.hydra-queue-runner) group;
path = "/var/lib/hydra/ldap-password.conf";
};
};
systemd.services = {
hydra-evaluator.serviceConfig = {
CPUWeight = 2;
MemoryHigh = "64G";
MemoryMax = "64G";
MemorySwapMax = "64G";
};
hydra-init.preStart = let
makesSenseForQemuUser = feature:
! (builtins.elem feature [ "kvm" "benchmark" ]);
# strips features that don't make sense on qemu-user
extraPlatformSystemFeatures =
builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
in
# both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
''
cat << EOF > ~/machines
localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
EOF
'';
nix-daemon.serviceConfig = {
CPUWeight = 5;
MemoryHigh = "64G";
MemoryMax = "64G";
MemorySwapMax = "64G";
};
};
# allow reading nix-serve secret
users.users.harmonia.extraGroups = [ "hydra" ];
}

View File

@ -1,4 +1,5 @@
nix-serve:
publicKey: ENC[AES256_GCM,data:sR5wk7yvH5+lLpSIP0zNqCLvDRRvR8ws4Q8rVcVJx8YkrywwPcvIsJ1h6mVEu3nc6SLoZlQsuxOGCyNGD98CBNY=,iv:fFV2D27hWoxGtqVt3EnS4hMlrqW5LnIZ3LB5k4xmFWs=,tag:g7R1ossy2On6B2nVfKC9iA==,type:str]
secretKey: ENC[AES256_GCM,data:cm84sA7E6AnzpVoYuaYepbHGWkRigLdD2RxN21UsXCe7FXQxeTQTxxbzVxJ3G9Lt3kRXuZnODntOo5EQKhs46+wzpO8YLKQxkJXrdluXoGVIWl3/6QFVq66XLJ2i6G4eBK9IH0DYJ+anj8/i8Q==,iv:GEM8Vmx0A8LfJo7QOl0N67Cgk+JqHpp7r+41VivmTg4=,tag:O4Kq4WKgbyt354HSa/7eQQ==,type:str]
ldap:
search-user-pw: ENC[AES256_GCM,data:tSWin/QPIow2P5Aps/XaT42J+MXb8+a24SEri1QjF1O3bDlCxcR8RHqSX8d4Vg==,iv:P5qMaE2cdKxTaXuKO2nh+LDhKkY3psSlWf+JckmUYt4=,tag:eq8XW7P6FNlkviY5PydkZg==,type:str]
@ -26,8 +27,8 @@ sops:
K3Bpb0svZ1YvVm9ha1ArdVBlN3NHM0kKM6CEQ+dStjEsgppQZYjb1zwyzfwAc0FI
O5+vi2x8/N/1OH5jeVzLnLjOhXRXrYcR9EDsjT+KDo0ykYh+NjB0DA==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2022-12-06T14:25:54Z"
mac: ENC[AES256_GCM,data:4cOG88FIG7UhVb/r8Aq1Nme5+qCpEdpjV+BLOISm1Y6MYgxFTDqCzV2FOdKztpVou5Nly9JUvKfz6eiCWbbIbaO5/DYUObiTKZXv6B1x6blnIW8vMtqcdYWOXH62ycHMV+Sha0D41eXmNp3K1Vs+k3OwYZyHK1HFOqqQ2jpy+Ps=,iv:u0O/A/GBBpDTJVFBfiFzDOIIR5o479YI11fgrv0mR0A=,tag:E9OuAAOhfbzPcnA6Ij6LMA==,type:str]
lastmodified: "2022-12-18T17:49:41Z"
mac: ENC[AES256_GCM,data:oQGsk4gJ4dGCr3zCanOpSsKQyx/OuWiYe0ef8pvWhqFq0+YvwUzoiF3jQmqahLWGLHUl1yb1kb91GEr7q47G83Z9YXVVtTTzJUWpKZQL1WD12f5ERPSlKnT251fvoMtqNqKrRirGA9ao8OxVpeh3/SQCAGTqWAcpz1J7dxvL7E4=,iv:td9t1MKcXH5RYZxsD1wA4oG61AfLLGSiVIEq/sPRZG4=,tag:UqHGkKqXeeBYXQYtvplCdA==,type:str]
pgp:
- created_at: "2022-07-15T23:31:58Z"
enc: |

View File

@ -401,7 +401,7 @@ in
fallback = true;
# don't self feed hydra
trusted-public-keys = lib.mkIf (config.networking.hostName != "hydra") [
(builtins.readFile ../hosts/hydra/cache-pub.key)
"nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps="
];
substituters = lib.mkIf (config.networking.hostName != "hydra") (
lib.mkBefore [ "https://nix-serve.hq.c3d2.de" ]