c3d2/nix-config
c3d2
/
nix-config
24
7
Fork 3
Code Issues 12 Pull Requests 2 Releases Wiki Activity

server9, server10, hydra: prepare skyflake

This commit is contained in:
Astro 2022-11-01 02:36:25 +01:00
parent d474949d78
commit 44b8845894
5 changed files with 59 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
flake.lock
View File

@ -610,6 +610,7 @@
"rust-overlay": "rust-overlay",
"scrapers": "scrapers",
"secrets": "secrets",
"skyflake": "skyflake",
"sops-nix": "sops-nix",
"spacemsg": "spacemsg",
"sshlogd": "sshlogd",
@ -696,6 +697,29 @@
"url": "ssh://gitea@gitea.c3d2.de/c3d2-admins/secrets.git"
}
},
"skyflake": {
"inputs": {
"microvm": [
"microvm"
],
"nixpkgs": [
"nixos"
]
},
"locked": {
"lastModified": 1667263445,
"narHash": "sha256-dgr0+EWFg5hSTUDBFJG0yPkOXGQLULKrW2GwZM35pF4=",
"owner": "astro",
"repo": "skyflake",
"rev": "dcb5f059baa315f6c24b7c20e9e1a0e8e1a6acb3",
"type": "github"
},
"original": {
"owner": "astro",
"repo": "skyflake",
"type": "github"
}
},
"sops-nix": {
"inputs": {
"nixpkgs": [

13
flake.nix
View File

@ -93,6 +93,13 @@
url = "git+https://gitea.c3d2.de/astro/scrapers.git";
flake = false;
};
skyflake = {
url = "github:astro/skyflake";
inputs = {
microvm.follows = "microvm";
nixpkgs.follows = "nixos";
};
};
sshlogd = {
url = "git+https://gitea.c3d2.de/astro/sshlogd.git?ref=main";
inputs = {
@ -156,7 +163,7 @@
};
};
outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
let
inherit (nixos) lib;
forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];
@ -731,6 +738,8 @@
hydra = nixosSystem' {
modules = [
inputs.hydra.nixosModules.hydra
self.nixosModules.cluster
skyflake.nixosModules.default
(harmonia + "/module.nix")
./hosts/hydra
{
@ -843,6 +852,7 @@
self.nixosModules.microvm-host
self.nixosModules.cluster-network
self.nixosModules.cluster
skyflake.nixosModules.default
{ _module.args = { inherit self; }; }
];
};
@ -854,6 +864,7 @@
self.nixosModules.microvm-host
self.nixosModules.cluster-network
self.nixosModules.cluster
skyflake.nixosModules.default
{ _module.args = { inherit self; }; }
];
};

1
hosts/hydra/default.nix
View File

@ -6,7 +6,6 @@
./network.nix
./hydra.nix
./updater.nix
./nomad-server.nix
../../modules/c3d2.nix
];

29
hosts/hydra/nomad-server.nix
View File

@ -1,29 +0,0 @@
# Our bare-metal hydra is a server in the nomad cluster, it is not a
# client and therefore does not run cluster jobs
{ zentralwerk, config, pkgs, ... }:
let
inherit (config.networking) hostName;
ipv4Addr = zentralwerk.lib.config.site.net.serv.hosts4.${hostName};
in
{
services.nomad = {
enable = true;
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
package = pkgs.nomad_1_3;
enableDocker = false;
settings = {
datacenter = "c3d2";
plugin.raw_exec.config.enabled = true;
server = {
enabled = true;
bootstrap_expect = 3;
server_join = {
retry_join = map (server:
zentralwerk.lib.config.site.net.cluster.hosts4.${server}
) [ "server9" "server10" ];
};
};
};
};
}

76
modules/cluster/default.nix
View File

@ -1,10 +1,11 @@
{ zentralwerk, config, lib, pkgs, ... }:
{ zentralwerk, hostRegistry, config, lib, pkgs, ... }:
let
inherit (config.networking) hostName;
# hydra does *not* use this module because it only runs a nomad
# server but no client and no microvms
servers = [ "server9" "server10" "hydra" ];
microvmServers = [ "server9" "server10" ];
serverNet = server:
builtins.foldl' (result: net:
@ -16,11 +17,7 @@ let
ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
in {
imports = [
];
# Open firewall between cluster members
networking.firewall.extraCommands = lib.concatMapStrings (server:
let
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
@ -32,57 +29,30 @@ in {
'') (builtins.attrValues netConfig.hosts6)}
'') servers;
# Storage
services.glusterfs.enable = true;
fileSystems."/glusterfs/fast" = {
fsType = "glusterfs";
device = "${config.networking.hostName}:/fast";
};
# stable uid is useful across glusterfs
users.users.microvm.uid = 997;
# Nomad
services.nomad = {
enable = true;
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
package = pkgs.nomad_1_3;
enableDocker = false;
dropPrivileges = false;
settings = rec {
# Cluster configuration
skyflake = {
nodes = builtins.listToAttrs (
map (name: {
inherit name;
value.address = builtins.trace name hostRegistry.hosts.${name}.ip4;
}) servers
);
nomad = {
datacenter = "c3d2";
plugin.raw_exec.config.enabled = true;
# no /dev/kvm otherwise. TODO: retry with nomad>1.3.3
plugin.raw_exec.config.no_cgroups = true;
servers = servers;
# run tasks only on these:
client.enable = builtins.elem hostName microvmServers;
};
microvmUid = 997;
server = {
enabled = true;
bootstrap_expect = 3;
server_join = {
retry_join = map (server:
zentralwerk.lib.config.site.net.${serverNet server}.hosts4.${server}
) (
builtins.filter (server: server != hostName)
servers
);
};
};
users.c3d2.sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
client = {
enabled = true;
network_interface = "cluster";
inherit (server) server_join;
};
storage.glusterfs = {
fileSystems = [ {
servers = microvmServers;
mountPoint = "/glusterfs/fast";
source = "/var/glusterfs-fast";
} ];
};
};
environment.systemPackages = with pkgs; [
# nomad frontends
damon wander
];
}