From 5862adf9ed214575ef3fd634f3dc981436bd3693 Mon Sep 17 00:00:00 2001 From: Astro Date: Thu, 22 Sep 2022 02:03:05 +0200 Subject: [PATCH] modules/microvm: refactor into modules/nomad-microvm --- flake.nix | 3 + hosts/mucbot/default.nix | 2 - hosts/sdrweb/default.nix | 5 - modules/microvm.nix | 135 ---------------------- modules/nomad-microvm.nix | 237 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 142 deletions(-) create mode 100644 modules/nomad-microvm.nix diff --git a/flake.nix b/flake.nix index 0a20e9ac..af0c5d0c 100644 --- a/flake.nix +++ b/flake.nix @@ -706,6 +706,7 @@ modules = [ { _module.args = { inherit tigger; }; } self.nixosModules.microvm + self.nixosModules.nomad-microvm "${tigger}/module.nix" { nixpkgs.overlays = [ secrets.overlays.mucbot ]; } ./hosts/mucbot @@ -763,6 +764,7 @@ sdrweb = nixosSystem' { modules = [ self.nixosModules.microvm + self.nixosModules.nomad-microvm { nixpkgs.overlays = [ secrets.overlays.mucbot ]; } heliwatch.nixosModules.heliwatch ./hosts/sdrweb @@ -921,6 +923,7 @@ ]; microvm-host = ./modules/microvm-host.nix; nncp = ./modules/nncp.nix; + nomad-microvm = ./modules/nomad-microvm.nix; plume = { imports = [ ./modules/plume.nix ]; nixpkgs.overlays = [ fenix.overlay naersk.overlay ]; diff --git a/hosts/mucbot/default.nix b/hosts/mucbot/default.nix index 4f997650..7fb956c1 100644 --- a/hosts/mucbot/default.nix +++ b/hosts/mucbot/default.nix @@ -2,10 +2,8 @@ { c3d2.deployment = { - server = "nomad"; # needs to keep just its ssh key for sops-nix mounts = [ "etc" "var" ]; - mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}"; }; networking.hostName = "mucbot"; diff --git a/hosts/sdrweb/default.nix b/hosts/sdrweb/default.nix index 5329f6b7..1b976f75 100644 --- a/hosts/sdrweb/default.nix +++ b/hosts/sdrweb/default.nix @@ -4,11 +4,6 @@ c3d2 = { isInHq = false; hq.statistics.enable = true; - deployment = { - server = "nomad"; - mounts = [ "etc" "home" "var" ]; - mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}"; - }; }; system.stateVersion = "22.05"; diff --git a/modules/microvm.nix b/modules/microvm.nix index 5db24ade..689b752c 100644 --- a/modules/microvm.nix +++ b/modules/microvm.nix @@ -62,17 +62,6 @@ in }; }; - config.assertions = [ { - assertion = - config.c3d2.deployment.server == "nomad" -> - config.c3d2.deployment.mounts == [] || - lib.hasPrefix "/glusterfs" config.c3d2.deployment.mountBase; - message = '' - Host \"${config.networking.hostName}\" is to be run on nomad - but its mountBase is not located on /glusterfs/...! - ''; - } ]; - config.system.build = with pkgs; { copyToServer = writeScript "copy-to-${server}" '' #! ${runtimeShell} -e @@ -85,130 +74,6 @@ in ssh root@${serverFQDN} -- $@ ''; - - nomadJob = - let - stateDir = "/glusterfs/fast/microvms/${hostName}"; - # only create tuntap if not yet existing - runTuntap = { id, ... }: - pkgs.writeScript "tuntap-${hostName}-${id}" '' - #!${pkgs.runtimeShell} -e - if [ -d /sys/class/net/${id} ]; then - ip tuntap del ${id} mode tap || true - fi - ip tuntap add ${id} mode tap user microvm - ''; - # change working directory before starting virtiofsd - runVirtiofsd = { tag, socket, source, ... }: - pkgs.writeScript "virtiofsd-${hostName}-${tag}" '' - #!${pkgs.runtimeShell} -e - cd ${stateDir} - mkdir -p ${source} - exec ${pkgs.virtiofsd}/bin/virtiofsd \ - --socket-path=${socket} \ - --socket-group=kvm \ - --shared-dir=${source} \ - --sandbox=none - ''; - # change working directory before starting hypervisor, - runMicrovm = - pkgs.writeScript "hypervisor-${hostName}" '' - #!${pkgs.runtimeShell} -e - cd ${stateDir} - # start hypervisor - exec ${config.microvm.declaredRunner}/bin/microvm-run - ''; - stopMicrovm = - pkgs.writeScript "hypervisor-${hostName}-stop" '' - #!${pkgs.runtimeShell} -e - cd ${stateDir} - # stop hypervisor on signal - function handle_signal() { - ${config.microvm.declaredRunner}/bin/microvm-shutdown - exit - } - trap handle_signal TERM - # wait - while true; do - sleep 86400 & - # catch signals: - wait - done - ''; - in pkgs.writeText "${hostName}.job" '' - job "${hostName}" { - datacenters = ["c3d2"] - type = "service" - - group "nixos-${config.system.nixos.label}" { - count = 1 - restart { - attempts = 1 - delay = "2s" - mode = "delay" - interval = "10s" - } - ${lib.concatMapStrings (interface@{ id, ... }: '' - task "interface-${id}" { - lifecycle { - hook = "prestart" - } - driver = "raw_exec" - user = "root" - config { - command = "${runTuntap interface}" - } - } - '') config.microvm.interfaces} - - ${lib.concatMapStrings (share@{ tag, ... }: '' - task "virtiofsd-${tag}" { - lifecycle { - hook = "prestart" - sidecar = true - } - driver = "raw_exec" - user = "root" - config { - command = "${runVirtiofsd share}" - } - kill_signal = "SIGCONT" - kill_timeout = "15s" - - resources { - memory = ${toString (config.microvm.vcpu * 32)} - cpu = ${toString (config.microvm.vcpu * 10)} - } - } - '') config.microvm.shares} - - task "hypervisor" { - driver = "raw_exec" - user = "microvm" - config { - command = "${runMicrovm}" - } - # don't get killed immediately but get shutdown by wait-shutdown - kill_signal = "SIGCONT" - kill_timeout = "15s" - - resources { - memory = ${toString config.microvm.mem} - cpu = ${toString (config.microvm.vcpu * 50)} - } - } - - task "wait-shutdown" { - driver = "raw_exec" - user = "microvm" - config { - command = "${stopMicrovm}" - } - kill_signal = "SIGTERM" - } - } - } - ''; }; config = { diff --git a/modules/nomad-microvm.nix b/modules/nomad-microvm.nix new file mode 100644 index 00000000..0849389e --- /dev/null +++ b/modules/nomad-microvm.nix @@ -0,0 +1,237 @@ +{ hostRegistry, config, lib, pkgs, ... }: +let + inherit (config.networking) hostName; +in +{ + options.c3d2.nomad = with lib; { + services = mkOption { + default = []; + type = types.listOf (types.submodule ({ ... }: { + options = { + provider = mkOption { + type = types.enum [ "consul" "nomad" ]; + default = "nomad"; + }; + name = mkOption { + type = types.str; + }; + tags = mkOption { + type = with types; listOf str; + default = []; + }; + port = mkOption { + type = with types; nullOr int; + default = null; + }; + checks = mkOption { + default = []; + type = types.listOf (types.submodule ({ ... }: { + options = { + type = mkOption { + type = types.enum [ "http" "tcp" ]; + }; + name = mkOption { + type = types.str; + }; + path = mkOption { + type = types.str; + default = "/"; + description = ''For type = "http"''; + }; + interval = mkOption { + type = types.int; + default = 30; + }; + timeout = mkOption { + type = types.int; + default = 10; + }; + }; + })); + }; + }; + })); + }; + }; + + config.assertions = [ { + assertion = + config.c3d2.deployment.server == "nomad" -> + config.c3d2.deployment.mounts == [] || + lib.hasPrefix "/glusterfs" config.c3d2.deployment.mountBase; + message = '' + Host \"${hostName}\" is to be run on nomad + but its mountBase is not located on /glusterfs/...! + ''; + } ]; + + config.c3d2.deployment = { + server = "nomad"; + mounts = lib.mkDefault [ "etc" "home" "var" ]; + mountBase = lib.mkDefault "/glusterfs/fast/microvms/${hostName}"; + }; + + # TODO: enable with nomad_1_4 + # config.c3d2.nomad.services = + # lib.optionals (config.services.nginx.enable) [ { + # name = "${hostName}-nginx"; + # tags = [ "httpd" ]; + # checks = [ { + # type = "http"; + # name = "${hostName}-nginx"; + # } ]; + # } ] ++ + # lib.optionals (config.services.collectd.enable) [ { + # name = "${hostName}-collectd"; + # tags = [ "collectd" ]; + # } ]; + + config.system.build = with pkgs; { + nomadJob = + let + stateDir = "/glusterfs/fast/microvms/${hostName}"; + # only create tuntap if not yet existing + runTuntap = { id, ... }: + pkgs.writeScript "tuntap-${hostName}-${id}" '' + #!${pkgs.runtimeShell} -e + if [ -d /sys/class/net/${id} ]; then + ip tuntap del ${id} mode tap || true + fi + ip tuntap add ${id} mode tap user microvm + ''; + # change working directory before starting virtiofsd + runVirtiofsd = { tag, socket, source, ... }: + pkgs.writeScript "virtiofsd-${hostName}-${tag}" '' + #!${pkgs.runtimeShell} -e + cd ${stateDir} + mkdir -p ${source} + exec ${pkgs.virtiofsd}/bin/virtiofsd \ + --socket-path=${socket} \ + --socket-group=kvm \ + --shared-dir=${source} \ + --sandbox=none + ''; + # change working directory before starting hypervisor, + runMicrovm = + pkgs.writeScript "hypervisor-${hostName}" '' + #!${pkgs.runtimeShell} -e + cd ${stateDir} + # start hypervisor + exec ${config.microvm.declaredRunner}/bin/microvm-run + ''; + stopMicrovm = + pkgs.writeScript "hypervisor-${hostName}-stop" '' + #!${pkgs.runtimeShell} -e + cd ${stateDir} + # stop hypervisor on signal + function handle_signal() { + ${config.microvm.declaredRunner}/bin/microvm-shutdown + exit + } + trap handle_signal TERM + # wait + while true; do + sleep 86400 & + # catch signals: + wait + done + ''; + in pkgs.writeText "${hostName}.job" '' + job "${hostName}" { + datacenters = ["c3d2"] + type = "service" + + group "nixos-${config.system.nixos.label}" { + count = 1 + restart { + attempts = 1 + delay = "2s" + mode = "delay" + interval = "10s" + } + ${lib.concatMapStrings (interface@{ id, ... }: '' + task "interface-${id}" { + lifecycle { + hook = "prestart" + } + driver = "raw_exec" + user = "root" + config { + command = "${runTuntap interface}" + } + } + '') config.microvm.interfaces} + + ${lib.concatMapStrings (share@{ tag, ... }: '' + task "virtiofsd-${tag}" { + lifecycle { + hook = "prestart" + sidecar = true + } + driver = "raw_exec" + user = "root" + config { + command = "${runVirtiofsd share}" + } + kill_signal = "SIGCONT" + kill_timeout = "15s" + + resources { + memory = ${toString (config.microvm.vcpu * 32)} + cpu = ${toString (config.microvm.vcpu * 10)} + } + } + '') config.microvm.shares} + + task "hypervisor" { + driver = "raw_exec" + user = "microvm" + config { + command = "${runMicrovm}" + } + # don't get killed immediately but get shutdown by wait-shutdown + kill_signal = "SIGCONT" + kill_timeout = "15s" + + resources { + memory = ${toString config.microvm.mem} + cpu = ${toString (config.microvm.vcpu * 50)} + } + + ${lib.concatMapStringsSep "\n" (service: '' + service { + provider = "${service.provider}" + name = "${service.name}" + tags = [${lib.concatMapStringsSep ", " (tag: + ''"${tag}"'' + ) service.tags}] + address_mode = "auto" + address = "${hostRegistry.hosts.${hostName}.ip4}" + ${lib.concatMapStrings (check: '' + check { + type = "${check.type}" + ${lib.optionalString (service.port != null) '' + port = ${toString service.port} + ''} + ${lib.optionalString (check.type == "http") '' + path = "${check.path}" + ''} + } + '') service.checks} + } + '') config.c3d2.nomad.services} + } + + task "wait-shutdown" { + driver = "raw_exec" + user = "microvm" + config { + command = "${stopMicrovm}" + } + kill_signal = "SIGTERM" + } + } + } + ''; + }; +}