From 761cc7688839842d76c1b77aba4881118170c16e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Thu, 19 Jan 2023 00:20:22 +0100
Subject: [PATCH] hydra: move slow binfmt builds into container to allow
 running some x86_64 builds along side

---
 flake.lock              |  6 ++---
 hosts/hydra/default.nix | 57 +++++++++++++++++++++++++++++++++--------
 2 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1499b3e1..7cb000e2 100644
--- a/flake.lock
+++ b/flake.lock
@@ -409,11 +409,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1674055737,
-        "narHash": "sha256-M3ELaArrZbqaD6BGcnhNgWZK3RT8IPApM4PHJBxiGXU=",
+        "lastModified": 1674083724,
+        "narHash": "sha256-NF10T+5vcml3zb6eJr5NYn1Y27l7fjo3xzLFhlKvbBc=",
         "owner": "SuperSandro2000",
         "repo": "nixos-modules",
-        "rev": "d3f1a3df8952066af0b48d25d635af1ee8d2b241",
+        "rev": "3190cdda2e2013cafcec24e03b8aab6c9a977554",
         "type": "github"
       },
       "original": {
diff --git a/hosts/hydra/default.nix b/hosts/hydra/default.nix
index 6ffbf331..f1c57981 100644
--- a/hosts/hydra/default.nix
+++ b/hosts/hydra/default.nix
@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, libS, modulesPath, pkgs, ... }:
 
 let
   cachePort = 5000;
@@ -58,8 +58,33 @@ in
 
   nixpkgs.config.allowUnfree = true;
 
-  # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
-  # containers = {
+  containers = {
+    hydra-binfmt-builder = {
+      autoStart = true;
+      config = { ... }: {
+        imports = [ (modulesPath + "/profiles/minimal.nix") ];
+
+        networking.firewall.allowedTCPPorts = [ 22 ];
+
+        nix = {
+          settings = config.nix.settings;
+          extraOptions = config.nix.extraOptions;
+        };
+
+        services.openssh.enable = true;
+
+        system.stateVersion = "22.11";
+
+        users.users."root".openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBga6vW8lnbFKl+Yd2xBiF71FRyV14eDUnqcMc2AWifI root@hydra"
+        ];
+      };
+      hostAddress = "192.168.100.1";
+      localAddress = "192.168.100.3";
+      privateNetwork = true;
+    };
+
+  #   disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
   #   hydra-ca = {
   #     autoStart = true;
   #     config = { ... }: {
@@ -105,20 +130,25 @@ in
   #     localAddress = "192.168.100.2";
   #     privateNetwork = true;
   #   };
-  # };
+  };
 
   networking = {
     hostId = "3f0c4ec4";
     hostName = "hydra";
     firewall.enable = false;
     nameservers = [ "172.20.73.8" "9.9.9.9" ];
-    # nat = {
-    #   enable = true;
-    #   externalInterface = "serv";
-    #   internalInterfaces = [ "ve-hydra-ca" ];
-    # };
+    nat = {
+      enable = true;
+      externalInterface = "serv";
+      internalInterfaces = [ "ve-hydra-biLqAU" ];
+    };
   };
 
+  programs.ssh.knownHosts = lib.mkMerge [
+    (libS.mkPubKey "192.168.100.3" "ssh-ed25519" "AAAAC3NzaC1lZDI1NTE5AAAAIBqrnoVELFvO9uc5VlLjiNAXyRTCWUMp5WiTF6o9UorJ")
+    (libS.mkPubKey "192.168.100.3" "ssh-rsa" "AAAAB3NzaC1yc2EAAAADAQABAAACAQCwofGcB1HIkIDWR9QNjl/9R39pLusYW2tvmGCZ9p0kfH1ml76OeWHZdXfjpwZJgRM+mk+sbfgKL3xfha+vPiLPJCfMUnKpgoM6zC5i/wi4Ywenh4hPFZG4moVFPBjcMUPmWw7vtED6n5dcW+LOeeuOGwEoBv72UiwhQVg7ULJIT0wu/lj2uduNwiSq8fmxeKZqB+jnJzpc56hGejuWWsGfgpIt2gWirOCqaxNoyRjt/rdGpHsRi8POBIjh5FsvTZVG0zJSgz0ubBsoCivgIr9fGKGxr0dLfDfqqNtrDFwDkkSiymcuo7zRU506pRLeTdrKPhPhvQg3aPOYAQcyvoJKo8xyMim5CbkbIo6TM7os5ubYoNpJ6+WSicYZaI4CG6X7kThkellAKy+yynlwnTTe5Q0DwUJr0znGy4Yi6t/VVE/bFEuAmb0DFbWVf2VqecFAe635hOxmQzzhaf1Zrf4epzcom833o12XdA6abfvuD3dVFSq/9ClzIBFkywNd22LrWhH2Wnh0u38xyHHTdGRQE5z5BKV0TevnmLgni92vMLyoTOdiC4UGhB71ED6tckN0qifzjvAGB2CAr+XT1Zy7ECPXC3SwqBYxcOb10j9pJsdx/gkjg8bovhr4Ve1x5blkzNvLbHA9jCITvfY3ke65JmL/loK1EEoS7odJGrQAbw==")
+  ];
+
   services = {
     hydra = {
       enable = true;
@@ -275,12 +305,17 @@ in
       makesSenseForQemuUser = feature: !(builtins.elem feature [ "kvm" "benchmark" ]);
       # strips features that don't make sense on qemu-user
       extraPlatformSystemFeatures = builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
+
+      localPlatforms = feature: !(builtins.elem feature [ "x86_64-linux" "i686-linux" ]);
+      # strips features that don't make sense on qemu-user
+      extraPlatforms = builtins.filter localPlatforms config.nix.settings.extra-platforms;
     in
     # both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
+    # multiple container max-jobs by X because binfmt is very slow especially in configure scripts
     ''
       cat << EOF > ~/machines
-      localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
-      hydra-queue-runner@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
+      localhost x86_64-linux,i686-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
+      root@192.168.100.3 ${lib.concatStringsSep "," extraPlatforms} - ${toString (config.nix.settings.max-jobs * 3)} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
       EOF
     '';