forked from c3d2/nix-config
server9, server10, hydra: prepare skyflake
This commit is contained in:
parent
d474949d78
commit
44b8845894
24
flake.lock
24
flake.lock
|
@ -610,6 +610,7 @@
|
||||||
"rust-overlay": "rust-overlay",
|
"rust-overlay": "rust-overlay",
|
||||||
"scrapers": "scrapers",
|
"scrapers": "scrapers",
|
||||||
"secrets": "secrets",
|
"secrets": "secrets",
|
||||||
|
"skyflake": "skyflake",
|
||||||
"sops-nix": "sops-nix",
|
"sops-nix": "sops-nix",
|
||||||
"spacemsg": "spacemsg",
|
"spacemsg": "spacemsg",
|
||||||
"sshlogd": "sshlogd",
|
"sshlogd": "sshlogd",
|
||||||
|
@ -696,6 +697,29 @@
|
||||||
"url": "ssh://gitea@gitea.c3d2.de/c3d2-admins/secrets.git"
|
"url": "ssh://gitea@gitea.c3d2.de/c3d2-admins/secrets.git"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"skyflake": {
|
||||||
|
"inputs": {
|
||||||
|
"microvm": [
|
||||||
|
"microvm"
|
||||||
|
],
|
||||||
|
"nixpkgs": [
|
||||||
|
"nixos"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1667263445,
|
||||||
|
"narHash": "sha256-dgr0+EWFg5hSTUDBFJG0yPkOXGQLULKrW2GwZM35pF4=",
|
||||||
|
"owner": "astro",
|
||||||
|
"repo": "skyflake",
|
||||||
|
"rev": "dcb5f059baa315f6c24b7c20e9e1a0e8e1a6acb3",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "astro",
|
||||||
|
"repo": "skyflake",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
"sops-nix": {
|
"sops-nix": {
|
||||||
"inputs": {
|
"inputs": {
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
|
|
13
flake.nix
13
flake.nix
|
@ -93,6 +93,13 @@
|
||||||
url = "git+https://gitea.c3d2.de/astro/scrapers.git";
|
url = "git+https://gitea.c3d2.de/astro/scrapers.git";
|
||||||
flake = false;
|
flake = false;
|
||||||
};
|
};
|
||||||
|
skyflake = {
|
||||||
|
url = "github:astro/skyflake";
|
||||||
|
inputs = {
|
||||||
|
microvm.follows = "microvm";
|
||||||
|
nixpkgs.follows = "nixos";
|
||||||
|
};
|
||||||
|
};
|
||||||
sshlogd = {
|
sshlogd = {
|
||||||
url = "git+https://gitea.c3d2.de/astro/sshlogd.git?ref=main";
|
url = "git+https://gitea.c3d2.de/astro/sshlogd.git?ref=main";
|
||||||
inputs = {
|
inputs = {
|
||||||
|
@ -156,7 +163,7 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
|
outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
|
||||||
let
|
let
|
||||||
inherit (nixos) lib;
|
inherit (nixos) lib;
|
||||||
forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];
|
forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];
|
||||||
|
@ -731,6 +738,8 @@
|
||||||
hydra = nixosSystem' {
|
hydra = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
inputs.hydra.nixosModules.hydra
|
inputs.hydra.nixosModules.hydra
|
||||||
|
self.nixosModules.cluster
|
||||||
|
skyflake.nixosModules.default
|
||||||
(harmonia + "/module.nix")
|
(harmonia + "/module.nix")
|
||||||
./hosts/hydra
|
./hosts/hydra
|
||||||
{
|
{
|
||||||
|
@ -843,6 +852,7 @@
|
||||||
self.nixosModules.microvm-host
|
self.nixosModules.microvm-host
|
||||||
self.nixosModules.cluster-network
|
self.nixosModules.cluster-network
|
||||||
self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
|
skyflake.nixosModules.default
|
||||||
{ _module.args = { inherit self; }; }
|
{ _module.args = { inherit self; }; }
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
@ -854,6 +864,7 @@
|
||||||
self.nixosModules.microvm-host
|
self.nixosModules.microvm-host
|
||||||
self.nixosModules.cluster-network
|
self.nixosModules.cluster-network
|
||||||
self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
|
skyflake.nixosModules.default
|
||||||
{ _module.args = { inherit self; }; }
|
{ _module.args = { inherit self; }; }
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
|
@ -6,7 +6,6 @@
|
||||||
./network.nix
|
./network.nix
|
||||||
./hydra.nix
|
./hydra.nix
|
||||||
./updater.nix
|
./updater.nix
|
||||||
./nomad-server.nix
|
|
||||||
../../modules/c3d2.nix
|
../../modules/c3d2.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
|
@ -1,29 +0,0 @@
|
||||||
# Our bare-metal hydra is a server in the nomad cluster, it is not a
|
|
||||||
# client and therefore does not run cluster jobs
|
|
||||||
{ zentralwerk, config, pkgs, ... }:
|
|
||||||
let
|
|
||||||
inherit (config.networking) hostName;
|
|
||||||
ipv4Addr = zentralwerk.lib.config.site.net.serv.hosts4.${hostName};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
services.nomad = {
|
|
||||||
enable = true;
|
|
||||||
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
|
|
||||||
package = pkgs.nomad_1_3;
|
|
||||||
enableDocker = false;
|
|
||||||
|
|
||||||
settings = {
|
|
||||||
datacenter = "c3d2";
|
|
||||||
plugin.raw_exec.config.enabled = true;
|
|
||||||
server = {
|
|
||||||
enabled = true;
|
|
||||||
bootstrap_expect = 3;
|
|
||||||
server_join = {
|
|
||||||
retry_join = map (server:
|
|
||||||
zentralwerk.lib.config.site.net.cluster.hosts4.${server}
|
|
||||||
) [ "server9" "server10" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -1,10 +1,11 @@
|
||||||
{ zentralwerk, config, lib, pkgs, ... }:
|
{ zentralwerk, hostRegistry, config, lib, pkgs, ... }:
|
||||||
let
|
let
|
||||||
inherit (config.networking) hostName;
|
inherit (config.networking) hostName;
|
||||||
|
|
||||||
# hydra does *not* use this module because it only runs a nomad
|
# hydra does *not* use this module because it only runs a nomad
|
||||||
# server but no client and no microvms
|
# server but no client and no microvms
|
||||||
servers = [ "server9" "server10" "hydra" ];
|
servers = [ "server9" "server10" "hydra" ];
|
||||||
|
microvmServers = [ "server9" "server10" ];
|
||||||
|
|
||||||
serverNet = server:
|
serverNet = server:
|
||||||
builtins.foldl' (result: net:
|
builtins.foldl' (result: net:
|
||||||
|
@ -16,11 +17,7 @@ let
|
||||||
|
|
||||||
ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
|
ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
|
||||||
in {
|
in {
|
||||||
imports = [
|
|
||||||
];
|
|
||||||
|
|
||||||
# Open firewall between cluster members
|
# Open firewall between cluster members
|
||||||
|
|
||||||
networking.firewall.extraCommands = lib.concatMapStrings (server:
|
networking.firewall.extraCommands = lib.concatMapStrings (server:
|
||||||
let
|
let
|
||||||
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
|
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
|
||||||
|
@ -32,57 +29,30 @@ in {
|
||||||
'') (builtins.attrValues netConfig.hosts6)}
|
'') (builtins.attrValues netConfig.hosts6)}
|
||||||
'') servers;
|
'') servers;
|
||||||
|
|
||||||
# Storage
|
# Cluster configuration
|
||||||
|
skyflake = {
|
||||||
services.glusterfs.enable = true;
|
nodes = builtins.listToAttrs (
|
||||||
|
map (name: {
|
||||||
fileSystems."/glusterfs/fast" = {
|
inherit name;
|
||||||
fsType = "glusterfs";
|
value.address = builtins.trace name hostRegistry.hosts.${name}.ip4;
|
||||||
device = "${config.networking.hostName}:/fast";
|
}) servers
|
||||||
};
|
);
|
||||||
|
nomad = {
|
||||||
# stable uid is useful across glusterfs
|
|
||||||
users.users.microvm.uid = 997;
|
|
||||||
|
|
||||||
# Nomad
|
|
||||||
|
|
||||||
services.nomad = {
|
|
||||||
enable = true;
|
|
||||||
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
|
|
||||||
package = pkgs.nomad_1_3;
|
|
||||||
|
|
||||||
enableDocker = false;
|
|
||||||
dropPrivileges = false;
|
|
||||||
|
|
||||||
settings = rec {
|
|
||||||
datacenter = "c3d2";
|
datacenter = "c3d2";
|
||||||
plugin.raw_exec.config.enabled = true;
|
servers = servers;
|
||||||
# no /dev/kvm otherwise. TODO: retry with nomad>1.3.3
|
# run tasks only on these:
|
||||||
plugin.raw_exec.config.no_cgroups = true;
|
client.enable = builtins.elem hostName microvmServers;
|
||||||
|
};
|
||||||
|
microvmUid = 997;
|
||||||
|
|
||||||
server = {
|
users.c3d2.sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
||||||
enabled = true;
|
|
||||||
bootstrap_expect = 3;
|
|
||||||
server_join = {
|
|
||||||
retry_join = map (server:
|
|
||||||
zentralwerk.lib.config.site.net.${serverNet server}.hosts4.${server}
|
|
||||||
) (
|
|
||||||
builtins.filter (server: server != hostName)
|
|
||||||
servers
|
|
||||||
);
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
client = {
|
storage.glusterfs = {
|
||||||
enabled = true;
|
fileSystems = [ {
|
||||||
network_interface = "cluster";
|
servers = microvmServers;
|
||||||
inherit (server) server_join;
|
mountPoint = "/glusterfs/fast";
|
||||||
};
|
source = "/var/glusterfs-fast";
|
||||||
|
} ];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
|
||||||
# nomad frontends
|
|
||||||
damon wander
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user