2023-05-19 21:38:16 +02:00
{ config , lib , libS , pkgs , . . . }:
2021-03-12 21:45:12 +01:00
2022-12-18 19:03:24 +01:00
let
cachePort = 5000 ;
in
2021-03-12 21:45:12 +01:00
{
2022-01-09 18:05:13 +01:00
imports = [
2022-05-05 22:34:51 +02:00
./hardware-configuration.nix
2022-06-23 20:10:03 +02:00
./network.nix
2022-01-09 18:05:13 +01:00
./updater.nix
] ;
2021-03-12 21:45:12 +01:00
2023-05-19 21:38:16 +02:00
c3d2 = {
baremetal = true ;
hq . statistics . enable = true ;
} ;
2021-03-12 21:45:12 +01:00
2022-12-18 19:03:24 +01:00
boot = {
tmpOnTmpfs = true ;
tmpOnTmpfsSize = " 8 0 % " ;
kernelModules = [ " k v m - i n t e l " ] ;
kernelParams = [ " m i t i g a t i o n s = o f f " " p r e e m p t = n o n e " ] ;
2023-01-02 05:05:53 +01:00
loader = {
efi . canTouchEfiVariables = true ;
systemd-boot . enable = true ;
} ;
2022-12-18 19:03:24 +01:00
# For cross-building
binfmt . emulatedSystems = [ " a r m v 6 l - l i n u x " " a r m v 7 l - l i n u x " " a a r c h 6 4 - l i n u x " " r i s c v 3 2 - l i n u x " " r i s c v 6 4 - l i n u x " ] ;
} ;
2021-03-12 21:45:12 +01:00
nix = {
2022-12-18 19:03:24 +01:00
buildMachines = [ {
hostName = " c l i e n t @ d a c b e r t . h q . c 3 d 2 . d e " ;
system = lib . concatStringsSep " , " [
2023-01-25 13:25:32 +01:00
# "aarch64-linux" # very slow compared to gallium
" a r m v 6 l - l i n u x " " a r m v 7 l - l i n u x "
2022-12-18 19:03:24 +01:00
] ;
supportedFeatures = [ " k v m " " n i x o s - t e s t " ] ;
maxJobs = 1 ;
} ] ;
daemonCPUSchedPolicy = " i d l e " ;
daemonIOSchedClass = " i d l e " ;
daemonIOSchedPriority = 7 ;
2023-04-30 15:37:58 +02:00
optimise = {
automatic = true ;
dates = [ " 0 5 : 3 0 " ] ;
} ;
2023-04-23 01:17:16 +02:00
remoteBuilder = {
enable = true ;
sshPublicKeys = config . users . users . root . openssh . authorizedKeys . keys ++ [
/* " . . . " */
] ;
} ;
2022-07-16 01:00:06 +02:00
settings = {
2022-12-18 19:03:24 +01:00
allowed-uris = " h t t p : / / h t t p s : / / s s h : / / " ;
builders-use-substitutes = true ;
2022-09-21 21:31:30 +02:00
cores = 20 ;
2022-07-16 01:00:06 +02:00
keep-outputs = true ;
2022-09-21 21:31:30 +02:00
max-jobs = 8 ;
2022-12-18 19:03:24 +01:00
trusted-users = [ " h y d r a " " r o o t " " @ w h e e l " ] ;
2023-01-05 21:49:21 +01:00
system-features = [
" k v m " " b i g - p a r a l l e l "
" n i x o s - t e s t " " b e n c h m a r k "
] ;
2022-07-16 01:00:06 +02:00
} ;
2023-01-07 00:55:15 +01:00
extraOptions = ''
2023-01-07 05:32:16 +01:00
! include $ { config . sops . secrets . " n i x / a c c e s s - t o k e n s " . path }
2023-01-07 00:55:15 +01:00
'' ;
2021-03-12 21:45:12 +01:00
} ;
2023-01-19 00:20:22 +01:00
containers = {
2023-01-21 00:44:41 +01:00
# hydra-binfmt-builder = {
# autoStart = true;
# config = { ... }: {
# imports = [ (modulesPath + "/profiles/minimal.nix") ];
2023-01-19 00:20:22 +01:00
2023-01-21 00:44:41 +01:00
# networking.firewall.allowedTCPPorts = [ 22 ];
2023-01-19 00:20:22 +01:00
2023-01-21 00:44:41 +01:00
# nix = {
# settings = config.nix.settings;
# extraOptions = config.nix.extraOptions;
# };
2023-01-19 00:20:22 +01:00
2023-01-21 00:44:41 +01:00
# services.openssh.enable = true;
2023-01-19 00:20:22 +01:00
2023-01-21 00:44:41 +01:00
# system.stateVersion = "22.11";
2023-01-19 00:20:22 +01:00
2023-01-21 00:44:41 +01:00
# users.users."root".openssh.authorizedKeys.keys = [
# "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBga6vW8lnbFKl+Yd2xBiF71FRyV14eDUnqcMc2AWifI root@hydra"
# ];
# };
# hostAddress = "192.168.100.1";
# localAddress = "192.168.100.3";
# privateNetwork = true;
# };
2023-01-19 00:20:22 +01:00
# disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
2022-12-18 19:03:24 +01:00
# hydra-ca = {
# autoStart = true;
# config = { ... }: {
# imports = [
# hydra-ca.nixosModules.hydra
# ];
# environment.systemPackages = with pkgs; [ git ];
# networking.firewall.allowedTCPPorts = [ 3001 ];
# nix = {
# settings = {
# allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
# builders-use-substitutes = true;
# experimental-features = "ca-derivations nix-command flakes";
# extra-substituters = "https://cache.ngi0.nixos.org/";
# extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
# substituters = [
# "https://cache.ngi0.nixos.org/"
# ];
# trusted-public-keys = [
# "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
# ];
# };
# };
# nixpkgs = {
# # config.contentAddressedByDefault = true;
# overlays = [ self.overlay ];
# };
# services = {
# hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
# hydraURL = "https://hydra-ca.hq.c3d2.de";
# port = 3001;
# };
# };
# system.stateVersion = "22.05"; # Did you read the comment? No.
# };
# hostAddress = "192.168.100.1";
# localAddress = "192.168.100.2";
# privateNetwork = true;
# };
2023-01-19 00:20:22 +01:00
} ;
2021-03-12 21:45:12 +01:00
networking = {
2023-01-02 05:05:53 +01:00
hostId = " 3 f 0 c 4 e c 4 " ;
2021-03-12 21:45:12 +01:00
hostName = " h y d r a " ;
firewall . enable = false ;
2022-05-05 22:34:51 +02:00
nameservers = [ " 1 7 2 . 2 0 . 7 3 . 8 " " 9 . 9 . 9 . 9 " ] ;
2023-01-21 00:44:41 +01:00
# nat = {
# enable = true;
# externalInterface = "serv";
# internalInterfaces = [ "ve-hydra-biLqAU" ];
# };
2022-12-18 19:03:24 +01:00
} ;
2023-01-19 00:20:22 +01:00
programs . ssh . knownHosts = lib . mkMerge [
( libS . mkPubKey " 1 9 2 . 1 6 8 . 1 0 0 . 3 " " s s h - e d 2 5 5 1 9 " " A A A A C 3 N z a C 1 l Z D I 1 N T E 5 A A A A I B q r n o V E L F v O 9 u c 5 V l L j i N A X y R T C W U M p 5 W i T F 6 o 9 U o r J " )
( libS . mkPubKey " 1 9 2 . 1 6 8 . 1 0 0 . 3 " " s s h - r s a " " A A A A B 3 N z a C 1 y c 2 E A A A A D A Q A B A A A C A Q C w o f G c B 1 H I k I D W R 9 Q N j l / 9 R 3 9 p L u s Y W 2 t v m G C Z 9 p 0 k f H 1 m l 7 6 O e W H Z d X f j p w Z J g R M + m k + s b f g K L 3 x f h a + v P i L P J C f M U n K p g o M 6 z C 5 i / w i 4 Y w e n h 4 h P F Z G 4 m o V F P B j c M U P m W w 7 v t E D 6 n 5 d c W + L O e e u O G w E o B v 7 2 U i w h Q V g 7 U L J I T 0 w u / l j 2 u d u N w i S q 8 f m x e K Z q B + j n J z p c 5 6 h G e j u W W s G f g p I t 2 g W i r O C q a x N o y R j t / r d G p H s R i 8 P O B I j h 5 F s v T Z V G 0 z J S g z 0 u b B s o C i v g I r 9 f G K G x r 0 d L f D f q q N t r D F w D k k S i y m c u o 7 z R U 5 0 6 p R L e T d r K P h P h v Q g 3 a P O Y A Q c y v o J K o 8 x y M i m 5 C b k b I o 6 T M 7 o s 5 u b Y o N p J 6 + W S i c Y Z a I 4 C G 6 X 7 k T h k e l l A K y + y y n l w n T T e 5 Q 0 D w U J r 0 z n G y 4 Y i 6 t / V V E / b F E u A m b 0 D F b W V f 2 V q e c F A e 6 3 5 h O x m Q z z h a f 1 Z r f 4 e p z c o m 8 3 3 o 1 2 X d A 6 a b f v u D 3 d V F S q / 9 C l z I B F k y w N d 2 2 L r W h H 2 W n h 0 u 3 8 x y H H T d G R Q E 5 z 5 B K V 0 T e v n m L g n i 9 2 v M L y o T O d i C 4 U G h B 7 1 E D 6 t c k N 0 q i f z j v A G B 2 C A r + X T 1 Z y 7 E C P X C 3 S w q B Y x c O b 1 0 j 9 p J s d x / g k j g 8 b o v h r 4 V e 1 x 5 b l k z N v L b H A 9 j C I T v f Y 3 k e 6 5 J m L / l o K 1 E E o S 7 o d J G r Q A b w = = " )
] ;
2022-12-18 19:03:24 +01:00
services = {
hydra = {
enable = true ;
buildMachinesFiles = [
" / e t c / n i x / m a c h i n e s "
" / v a r / l i b / h y d r a / m a c h i n e s "
] ;
hydraURL = " h t t p s : / / h y d r a . h q . c 3 d 2 . d e " ;
2023-01-17 00:26:49 +01:00
ldap = {
enable = true ;
roleMappings = [
{ hydra-admins = " a d m i n " ; }
] ;
} ;
2022-12-18 19:03:24 +01:00
logo = ./c3d2.svg ;
minimumDiskFree = 50 ;
minimumDiskFreeEvaluator = 50 ;
notificationSender = " h y d r a @ s p a m . w o r k s " ;
useSubstitutes = true ;
extraConfig =
let
2023-01-07 00:55:15 +01:00
key = config . sops . secrets . " n i x / s i g n i n g - k e y / s e c r e t K e y " . path ;
2022-12-18 19:03:24 +01:00
in
''
binary_cache_secret_key_file = $ { key }
2022-12-18 21:19:18 +01:00
compress_num_threads = 4
2022-12-18 19:03:24 +01:00
evaluator_workers = 4
evaluator_max_memory_size = 2048
max_output_size = $ { toString ( 5 * 1024 * 1024 * 1024 ) } # sd card and raw images
store_uri = auto ? secret-key = $ { key } & write-nar-listing = 1 & ls-compression = zstd & log-compression = zstd
upload_logs_to_binary_cache = true
'' ;
} ;
# A rust nix binary cache
harmonia = {
enable = true ;
settings = {
2022-12-20 04:54:52 +01:00
bind = " [ : : ] : ${ toString cachePort } " ;
2022-12-18 19:03:24 +01:00
workers = 20 ;
max_connection_rate = 1024 ;
2022-12-20 04:54:52 +01:00
priority = 50 ;
2022-12-18 19:03:24 +01:00
} ;
2023-05-08 23:32:47 +02:00
signKeyPath = config . sops . secrets . " n i x / s i g n i n g - k e y / s e c r e t K e y " . path ;
2022-12-18 19:03:24 +01:00
} ;
2023-04-03 20:34:04 +02:00
nginx = {
enable = true ;
virtualHosts = {
" h y d r a . h q . c 3 d 2 . d e " = {
default = true ;
2022-12-18 19:03:24 +01:00
enableACME = true ;
2023-04-03 20:34:04 +02:00
forceSSL = true ;
2023-04-11 00:24:32 +02:00
locations . " / " . proxyPass = " h t t p : / / 1 2 7 . 0 . 0 . 1 : ${ toString config . services . hydra . port } " ;
2023-04-03 20:34:04 +02:00
serverAliases = [
" h y d r a - c a . h q . c 3 d 2 . d e "
" h y d r a . s e r v . z e n t r a l w e r k . o r g "
] ;
2022-12-18 19:03:24 +01:00
} ;
2023-04-03 20:34:04 +02:00
# "hydra-ca.hq.c3d2.de" = {
# enableACME = true;
# forceSSL = true;
# locations."/".proxyPass = "http://192.168.100.2:3001";
# };
" n i x - c a c h e . h q . c 3 d 2 . d e " = {
forceSSL = true ;
enableACME = true ;
2023-04-11 00:24:32 +02:00
locations . " / " . proxyPass = " h t t p : / / 1 2 7 . 0 . 0 . 1 : ${ toString cachePort } " ;
2023-04-03 20:34:04 +02:00
serverAliases = [
" n i x - s e r v e . h q . c 3 d 2 . d e "
] ;
2022-12-18 19:03:24 +01:00
} ;
} ;
2023-04-03 20:34:04 +02:00
} ;
2022-12-18 19:03:24 +01:00
2022-12-22 21:25:53 +01:00
portunus . addToHosts = true ;
2022-12-18 19:03:24 +01:00
2023-01-06 23:18:47 +01:00
postgresql = {
package = pkgs . postgresql_15 ;
upgrade . stopServices = [ " h y d r a - e v a l u a t o r " " h y d r a - q u e u e - r u n n e r " " h y d r a - s e r v e r " ] ;
} ;
2022-12-18 19:03:24 +01:00
resolved . enable = false ;
2022-12-22 21:25:53 +01:00
2023-01-02 05:05:53 +01:00
zfs . trim . enable = true ;
2021-03-12 21:45:12 +01:00
} ;
2022-01-10 03:36:31 +01:00
2023-01-02 17:37:29 +01:00
simd . arch = " i v y b r i d g e " ;
2022-12-18 19:03:24 +01:00
sops = {
defaultSopsFile = ./secrets.yaml ;
2023-01-05 00:31:10 +01:00
secrets = {
" l d a p / s e a r c h - u s e r - p w " = {
mode = " 4 4 0 " ;
owner = config . users . users . hydra-queue-runner . name ;
inherit ( config . users . users . hydra-queue-runner ) group ;
path = " / v a r / l i b / h y d r a / l d a p - p a s s w o r d . c o n f " ;
} ;
" m a c h i n e - i d " = {
mode = " 4 4 4 " ;
path = " / e t c / m a c h i n e - i d " ;
} ;
2023-01-07 00:55:15 +01:00
" n i x / a c c e s s - t o k e n s " = {
mode = " 4 4 4 " ;
} ;
" n i x / s i g n i n g - k e y / s e c r e t K e y " = {
2023-01-05 00:31:10 +01:00
mode = " 4 4 0 " ;
owner = config . users . users . hydra-queue-runner . name ;
inherit ( config . users . users . hydra-queue-runner ) group ;
} ;
2023-05-18 01:55:16 +02:00
" r e s t i c / p a s s w o r d " . owner = " r o o t " ;
" r e s t i c / r e p o s i t o r y / s e r v e r 8 " . owner = " r o o t " ;
2023-01-06 23:04:47 +01:00
" s s h - k e y s / h y d r a / p r i v a t e " = {
2023-01-10 22:21:27 +01:00
owner = " h y d r a " ;
2023-01-06 23:04:47 +01:00
mode = " 4 0 0 " ;
path = " / v a r / l i b / h y d r a / . s s h / i d _ e d 2 5 5 1 9 " ;
} ;
" s s h - k e y s / h y d r a / p u b l i c " = {
2023-01-10 22:21:27 +01:00
owner = " h y d r a " ;
2023-01-06 23:04:47 +01:00
mode = " 4 4 0 " ;
path = " / v a r / l i b / h y d r a / . s s h / i d _ e d 2 5 5 1 9 . p u b " ;
} ;
2023-01-05 00:31:10 +01:00
" s s h - k e y s / r o o t / p r i v a t e " = {
2023-01-10 22:21:27 +01:00
owner = " h y d r a - q u e u e - r u n n e r " ;
2023-01-06 23:04:47 +01:00
mode = " 4 0 0 " ;
2023-01-10 22:21:27 +01:00
path = " / v a r / l i b / h y d r a / q u e u e - r u n n e r / . s s h / i d _ e d 2 5 5 1 9 " ;
2023-01-05 00:31:10 +01:00
} ;
" s s h - k e y s / r o o t / p u b l i c " = {
2023-01-10 22:21:27 +01:00
owner = " h y d r a - q u e u e - r u n n e r " ;
2023-01-06 23:04:47 +01:00
mode = " 4 4 0 " ;
2023-01-10 22:21:27 +01:00
path = " / v a r / l i b / h y d r a / q u e u e - r u n n e r / . s s h / i d _ e d 2 5 5 1 9 . p u b " ;
2023-01-05 00:31:10 +01:00
} ;
" s s h - k e y s / u p d a t e r / p r i v a t e " = {
2023-01-09 23:47:13 +01:00
owner = " u p d a t e r " ;
2023-01-06 23:04:47 +01:00
mode = " 4 0 0 " ;
2023-01-05 00:31:10 +01:00
path = " / v a r / l i b / u p d a t e r / . s s h / i d _ e d 2 5 5 1 9 " ;
} ;
" s s h - k e y s / u p d a t e r / p u b l i c " = {
2023-01-09 23:47:13 +01:00
owner = " u p d a t e r " ;
2023-01-06 23:04:47 +01:00
mode = " 4 4 0 " ;
2023-01-05 00:31:10 +01:00
path = " / v a r / l i b / u p d a t e r / . s s h / i d _ e d 2 5 5 1 9 . p u b " ;
} ;
2022-12-18 19:03:24 +01:00
} ;
} ;
2022-05-07 01:34:27 +02:00
2022-12-04 08:53:28 +01:00
system . stateVersion = " 2 0 . 0 9 " ;
2022-12-18 19:03:24 +01:00
systemd . services = {
hydra-evaluator . serviceConfig = {
CPUWeight = 2 ;
MemoryHigh = " 6 4 G " ;
MemoryMax = " 6 4 G " ;
MemorySwapMax = " 6 4 G " ;
} ;
hydra-init . preStart = let
2023-01-19 00:20:22 +01:00
localPlatforms = feature : ! ( builtins . elem feature [ " x 8 6 _ 6 4 - l i n u x " " i 6 8 6 - l i n u x " ] ) ;
# strips features that don't make sense on qemu-user
extraPlatforms = builtins . filter localPlatforms config . nix . settings . extra-platforms ;
2022-12-18 19:03:24 +01:00
in
# both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
2023-01-19 00:20:22 +01:00
# multiple container max-jobs by X because binfmt is very slow especially in configure scripts
2022-12-18 19:03:24 +01:00
''
cat < < EOF > ~/machines
2023-01-19 00:20:22 +01:00
localhost x86_64-linux , i686-linux - $ { toString config . nix . settings . max-jobs } 10 $ { lib . concatStringsSep " , " config . nix . settings . system-features } -
2023-01-21 00:44:41 +01:00
# local container to have an extra nix daemon for binfmt
# NOTE: currently very, very slow and usually builds do not finish in any amount of time
# root@192.168.100.3 ${lib.concatStringsSep "," extraPlatforms} - ${toString (config.nix.settings.max-jobs * 3)} 10 big-parallel,nixos-test -
# sandro's native aarch64 builder
2023-05-04 23:15:05 +02:00
$ { config . nix . remoteBuilder . name } @ gallium . supersandro . de aarch64-linux - 4 20 big-parallel , nixos-test , benchmark -
2022-12-18 19:03:24 +01:00
EOF
'' ;
nix-daemon . serviceConfig = {
CPUWeight = 5 ;
MemoryHigh = " 6 4 G " ;
MemoryMax = " 6 4 G " ;
MemorySwapMax = " 6 4 G " ;
} ;
} ;
2021-03-12 21:45:12 +01:00
}