diff --git a/flake.lock b/flake.lock index 5a230996..085d750d 100644 --- a/flake.lock +++ b/flake.lock @@ -610,6 +610,7 @@ "rust-overlay": "rust-overlay", "scrapers": "scrapers", "secrets": "secrets", + "skyflake": "skyflake", "sops-nix": "sops-nix", "spacemsg": "spacemsg", "sshlogd": "sshlogd", @@ -696,6 +697,29 @@ "url": "ssh://gitea@gitea.c3d2.de/c3d2-admins/secrets.git" } }, + "skyflake": { + "inputs": { + "microvm": [ + "microvm" + ], + "nixpkgs": [ + "nixos" + ] + }, + "locked": { + "lastModified": 1667263445, + "narHash": "sha256-dgr0+EWFg5hSTUDBFJG0yPkOXGQLULKrW2GwZM35pF4=", + "owner": "astro", + "repo": "skyflake", + "rev": "dcb5f059baa315f6c24b7c20e9e1a0e8e1a6acb3", + "type": "github" + }, + "original": { + "owner": "astro", + "repo": "skyflake", + "type": "github" + } + }, "sops-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 53e7daf9..c2e4f7ca 100644 --- a/flake.nix +++ b/flake.nix @@ -93,6 +93,13 @@ url = "git+https://gitea.c3d2.de/astro/scrapers.git"; flake = false; }; + skyflake = { + url = "github:astro/skyflake"; + inputs = { + microvm.follows = "microvm"; + nixpkgs.follows = "nixos"; + }; + }; sshlogd = { url = "git+https://gitea.c3d2.de/astro/sshlogd.git?ref=main"; inputs = { @@ -156,7 +163,7 @@ }; }; - outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }: + outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }: let inherit (nixos) lib; forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ]; @@ -731,6 +738,8 @@ hydra = nixosSystem' { modules = [ inputs.hydra.nixosModules.hydra + self.nixosModules.cluster + skyflake.nixosModules.default (harmonia + "/module.nix") ./hosts/hydra { @@ -843,6 +852,7 @@ self.nixosModules.microvm-host self.nixosModules.cluster-network self.nixosModules.cluster + skyflake.nixosModules.default { _module.args = { inherit self; }; } ]; }; @@ -854,6 +864,7 @@ self.nixosModules.microvm-host self.nixosModules.cluster-network self.nixosModules.cluster + skyflake.nixosModules.default { _module.args = { inherit self; }; } ]; }; diff --git a/hosts/hydra/default.nix b/hosts/hydra/default.nix index cee4110d..0e6a954e 100644 --- a/hosts/hydra/default.nix +++ b/hosts/hydra/default.nix @@ -6,7 +6,6 @@ ./network.nix ./hydra.nix ./updater.nix - ./nomad-server.nix ../../modules/c3d2.nix ]; diff --git a/hosts/hydra/nomad-server.nix b/hosts/hydra/nomad-server.nix deleted file mode 100644 index 07b169af..00000000 --- a/hosts/hydra/nomad-server.nix +++ /dev/null @@ -1,29 +0,0 @@ -# Our bare-metal hydra is a server in the nomad cluster, it is not a -# client and therefore does not run cluster jobs -{ zentralwerk, config, pkgs, ... }: -let - inherit (config.networking) hostName; - ipv4Addr = zentralwerk.lib.config.site.net.serv.hosts4.${hostName}; -in -{ - services.nomad = { - enable = true; - # nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2 - package = pkgs.nomad_1_3; - enableDocker = false; - - settings = { - datacenter = "c3d2"; - plugin.raw_exec.config.enabled = true; - server = { - enabled = true; - bootstrap_expect = 3; - server_join = { - retry_join = map (server: - zentralwerk.lib.config.site.net.cluster.hosts4.${server} - ) [ "server9" "server10" ]; - }; - }; - }; - }; -} diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix index c51f8dba..ad1e9063 100644 --- a/modules/cluster/default.nix +++ b/modules/cluster/default.nix @@ -1,10 +1,11 @@ -{ zentralwerk, config, lib, pkgs, ... }: +{ zentralwerk, hostRegistry, config, lib, pkgs, ... }: let inherit (config.networking) hostName; # hydra does *not* use this module because it only runs a nomad # server but no client and no microvms servers = [ "server9" "server10" "hydra" ]; + microvmServers = [ "server9" "server10" ]; serverNet = server: builtins.foldl' (result: net: @@ -16,11 +17,7 @@ let ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName}; in { - imports = [ - ]; - # Open firewall between cluster members - networking.firewall.extraCommands = lib.concatMapStrings (server: let netConfig = zentralwerk.lib.config.site.net.${serverNet server}; @@ -32,57 +29,30 @@ in { '') (builtins.attrValues netConfig.hosts6)} '') servers; - # Storage - - services.glusterfs.enable = true; - - fileSystems."/glusterfs/fast" = { - fsType = "glusterfs"; - device = "${config.networking.hostName}:/fast"; - }; - - # stable uid is useful across glusterfs - users.users.microvm.uid = 997; - - # Nomad - - services.nomad = { - enable = true; - # nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2 - package = pkgs.nomad_1_3; - - enableDocker = false; - dropPrivileges = false; - - settings = rec { + # Cluster configuration + skyflake = { + nodes = builtins.listToAttrs ( + map (name: { + inherit name; + value.address = builtins.trace name hostRegistry.hosts.${name}.ip4; + }) servers + ); + nomad = { datacenter = "c3d2"; - plugin.raw_exec.config.enabled = true; - # no /dev/kvm otherwise. TODO: retry with nomad>1.3.3 - plugin.raw_exec.config.no_cgroups = true; + servers = servers; + # run tasks only on these: + client.enable = builtins.elem hostName microvmServers; + }; + microvmUid = 997; - server = { - enabled = true; - bootstrap_expect = 3; - server_join = { - retry_join = map (server: - zentralwerk.lib.config.site.net.${serverNet server}.hosts4.${server} - ) ( - builtins.filter (server: server != hostName) - servers - ); - }; - }; + users.c3d2.sshKeys = config.users.users.root.openssh.authorizedKeys.keys; - client = { - enabled = true; - network_interface = "cluster"; - inherit (server) server_join; - }; + storage.glusterfs = { + fileSystems = [ { + servers = microvmServers; + mountPoint = "/glusterfs/fast"; + source = "/var/glusterfs-fast"; + } ]; }; }; - - environment.systemPackages = with pkgs; [ - # nomad frontends - damon wander - ]; }