hydra: redo with full zfs

README.md

@@ -277,3 +277,43 @@ in {
   };
 }
 ```
+
+## Server zfs setup
+
+For the other steps follow https://nixos.org/manual/nixos/unstable/index.html#sec-installation
+
+```
+sgdisk --zap-all /dev/sda
+parted /dev/sda -- mklabel gpt
+parted /dev/sda -- mkpart primary 512MB -40GB
+parted /dev/sda -- mkpart primary linux-swap -40GB 100%
+parted /dev/sda -- mkpart ESP fat32 1MB 512MB
+parted /dev/sda -- set 3 esp on
+mkswap -L swap /dev/sda2
+mkfs.fat -F 32 -n boot /dev/sda3
+pool create \
+  -o ashift=12 \
+  -o autotrim=on \
+  -R /mnt \
+  -O acltype=posixacl \
+  -O canmount=off \
+  -O compression=zstd \
+  -O dnodesize=auto \
+  -O normalization=formD \
+  -O relatime=on \
+  -O xattr=sa \
+  -O mountpoint=/ \
+  hydra /dev/sda1
+ zfs create -o canmount=on -o mountpoint=/ hydra/nixos
+ zfs create -o canmount=on -o mountpoint=/nix hydra/nixos/nix
+ zfs create -o canmount=on -o atime=off -o mountpoint=/nix/store hydra/nixos/nix/store
+ zfs create -o canmount=on -o mountpoint=/nix/var hydra/nixos/nix/var
+ zfs create -o canmount=off -o mountpoint=none hydra/data
+ zfs create -o canmount=on -o mountpoint=/etc hydra/data/etc
+ zfs create -o canmount=on -o mountpoint=/var hydra/data/var
+ zfs create -o canmount=on -o mountpoint=/var/backup hydra/data/var/backup
+ zfs create -o canmount=on -o mountpoint=/var/lib hydra/data/var/lib
+ zfs create -o canmount=on -o mountpoint=/var/log hydra/data/var/log
+ zfs create -o canmount=on -o mountpoint=/home hydra/data/home
+ zfs create -o canmount=off -o mountpoint=none -o refreservation=1G hydra/reserved
+```

config/default.nix

@@ -124,6 +124,11 @@ lib.mkMerge [
       };
     };
 
+    # TODO: move to nixos-modules
+    services = {
+      zfs.autoScrub.enable = true;
+    };
+
     systemd = {
       # Do not break the boot
       enableEmergencyMode = false;

hosts/hydra/default.nix

@@ -19,9 +19,13 @@ in
   boot = {
     tmpOnTmpfs = true;
     tmpOnTmpfsSize = "80%";
-    kernelPackages = pkgs.linuxPackages_latest;
+    kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
     kernelModules = [ "kvm-intel" ];
     kernelParams = [ "mitigations=off" "preempt=none" ];
+    loader = {
+      efi.canTouchEfiVariables = true;
+      systemd-boot.enable = true;
+    };
     # For cross-building
     binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
   };
@@ -101,6 +105,7 @@ in
   # };
 
   networking = {
+    hostId = "3f0c4ec4";
     hostName = "hydra";
     firewall.enable = false;
     nameservers = [ "172.20.73.8" "9.9.9.9" ];
@@ -235,6 +240,8 @@ in
     resolved.enable = false;
 
     smartd.enable = true;
+
+    zfs.trim.enable = true;
   };
 
   sops = {

hosts/hydra/hardware-configuration.nix

@@ -1,43 +1,101 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, modulesPath, ... }:
+{ config, lib, pkgs, modulesPath, ... }:
 
 {
   imports =
     [ (modulesPath + "/installer/scan/not-detected.nix")
     ];
 
-  # Use the GRUB 2 boot loader.
+  boot.initrd.availableKernelModules = [ "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
-  boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
-  # boot.loader.grub.efiSupport = true;
-  # boot.loader.grub.efiInstallAsRemovable = true;
-  # boot.loader.efi.efiSysMountPoint = "/boot/efi";
-  # Define on which hard drive you want to install Grub.
-  boot.loader.grub.device = "/dev/sda"; # or "nodev" for efi only
-
-  boot.initrd.availableKernelModules = [ "uhci_hcd" "ehci_pci" "ata_piix" "usbhid" "usb_storage" "sd_mod" ];
   boot.initrd.kernelModules = [ ];
   boot.kernelModules = [ "kvm-intel" ];
   boot.extraModulePackages = [ ];
 
   fileSystems."/" =
-    { device = "/dev/disk/by-uuid/6259dd9a-582b-41bc-aced-8aff8ed9ff32";
+    { device = "hydra/nixos";
-      fsType = "ext4";
+      fsType = "zfs";
-      options = [ "relatime" "discard" ];
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/nix" =
+    { device = "hydra/nixos/nix";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/nix/store" =
+    { device = "hydra/nixos/nix/store";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/nix/var" =
+    { device = "hydra/nixos/nix/var";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/etc" =
+    { device = "hydra/data/etc";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/var" =
+    { device = "hydra/data/var";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/var/backup" =
+    { device = "hydra/data/var/backup";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/var/lib" =
+    { device = "hydra/data/var/lib";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/var/log" =
+    { device = "hydra/data/var/log";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
+    };
+
+  fileSystems."/home" =
+    { device = "hydra/data/home";
+      fsType = "zfs";
+      options = [ "zfsutil" ];
     };
 
   fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/D8D1-372E";
+    { device = "/dev/disk/by-uuid/93C6-E4BA";
       fsType = "vfat";
     };
 
-  swapDevices = [ {
+  swapDevices =
-    device = "/dev/disk/by-uuid/d5e112a4-fcd3-461a-b2eb-21c6a10ad108";
+    [ { device = "/dev/disk/by-uuid/61ba7849-7815-473e-85f6-d7274eda6ce4"; }
-    discardPolicy = "both";
+    ];
-  } ];
 
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.bond0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.bonding_masters.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp2s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp2s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp6s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp6s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp7s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp7s0f1.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
   hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }
-

hosts/server9/default.nix

@@ -37,7 +37,6 @@ _:
   services = {
     openssh.enable = true;
     smartd.enable = true;
-    zfs.autoScrub.enable = true;
 
     # reserve resources for libvirt VMs
     nomad.settings.client.reserved = {