From 4242ac073e59a9812f7c98a58f4decc08169ee8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Tue, 23 May 2023 20:09:55 +0200 Subject: [PATCH] Use ceph version 16.2 on 23.05 --- flake.nix | 5 +- modules/cluster/default.nix | 7 +- overlays/ceph.nix | 282 ++++++++++++++++++++++++++++++++++++ overlays/default.nix | 2 + 4 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 overlays/ceph.nix diff --git a/flake.nix b/flake.nix index c2d0e479..93078a5d 100644 --- a/flake.nix +++ b/flake.nix @@ -241,9 +241,12 @@ inherit system; modules = [ - (_: { + ({ pkgs, ... }: { _module.args = { inherit hostRegistry libC nixos ssh-public-keys zentralwerk; + + # TODO: drop! + is2305 = (lib.versions.majorMinor pkgs.lib.version) == "23.05"; }; nixpkgs.overlays = overlayList; diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix index 03aaedce..a7d6d55d 100644 --- a/modules/cluster/default.nix +++ b/modules/cluster/default.nix @@ -1,4 +1,4 @@ -{ config, hostRegistry, lib, pkgs, ssh-public-keys, zentralwerk, ... }: +{ config, hostRegistry, is2305, lib, pkgs, ssh-public-keys, zentralwerk, ... }: let inherit (config.networking) hostName; @@ -89,7 +89,10 @@ in { deploy.customizationModule = ./customization; # Ceph storage cluster configuration - storage.ceph = assert lib.versions.majorMinor pkgs.ceph.version == "16.2"; rec { + storage.ceph = rec { + package = let + ceph = if is2305 then pkgs.ceph_16_2 else pkgs.ceph; + in assert (lib.versions.majorMinor ceph.version) == "16.2"; ceph; fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf"; mons = [ "server7" "server8" "server9" "server10" ]; mgrs = mons; diff --git a/overlays/ceph.nix b/overlays/ceph.nix new file mode 100644 index 00000000..15e5071f --- /dev/null +++ b/overlays/ceph.nix @@ -0,0 +1,282 @@ +{ lib, stdenv, runCommand, fetchurl, fetchpatch +, ensureNewerSourcesHook +, cmake, pkg-config +, which, git +, boost175, xz +, libxml2, zlib, lz4 +, openldap, lttng-ust +, babeltrace, gperf +, gtest +, cunit, snappy +, makeWrapper +, leveldb, oath-toolkit +, libnl, libcap_ng +, rdkafka +, nixosTests +, cryptsetup +, sqlite +, lua +, icu +, bzip2 +, doxygen +, graphviz +, fmt +, python39 + +# Optional Dependencies +, yasm ? null, fcgi ? null, expat ? null +, curl ? null, fuse ? null +, libedit ? null, libatomic_ops ? null +, libs3 ? null + +# Mallocs +, jemalloc ? null, gperftools ? null + +# Crypto Dependencies +, cryptopp ? null +, nss ? null, nspr ? null + +# Linux Only Dependencies +, linuxHeaders, util-linux, libuuid, udev, keyutils, rdma-core, rabbitmq-c +, libaio ? null, libxfs ? null, zfs ? null, liburing ? null +, ... +}: + +# We must have one crypto library +assert cryptopp != null || (nss != null && nspr != null); + +let + shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null; + + optYasm = shouldUsePkg yasm; + optFcgi = shouldUsePkg fcgi; + optExpat = shouldUsePkg expat; + optCurl = shouldUsePkg curl; + optFuse = shouldUsePkg fuse; + optLibedit = shouldUsePkg libedit; + optLibatomic_ops = shouldUsePkg libatomic_ops; + optLibs3 = shouldUsePkg libs3; + + optJemalloc = shouldUsePkg jemalloc; + optGperftools = shouldUsePkg gperftools; + + optCryptopp = shouldUsePkg cryptopp; + optNss = shouldUsePkg nss; + optNspr = shouldUsePkg nspr; + + optLibaio = shouldUsePkg libaio; + optLibxfs = shouldUsePkg libxfs; + optZfs = shouldUsePkg zfs; + + hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null; + + + # Malloc implementation (can be jemalloc, tcmalloc or null) + malloc = if optJemalloc != null then optJemalloc else optGperftools; + + # We prefer nss over cryptopp + cryptoStr = if optNss != null && optNspr != null then "nss" else + if optCryptopp != null then "cryptopp" else "none"; + + cryptoLibsMap = { + nss = [ optNss optNspr ]; + cryptopp = [ optCryptopp ]; + none = [ ]; + }; + + getMeta = description: with lib; { + homepage = "https://ceph.io/en/"; + inherit description; + license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ]; + maintainers = with maintainers; [ adev ak johanot krav ]; + platforms = [ "x86_64-linux" "aarch64-linux" ]; + }; + + ceph-common = python.pkgs.buildPythonPackage rec{ + pname = "ceph-common"; + inherit src version; + + sourceRoot = "ceph-${version}/src/python-common"; + + nativeCheckInputs = [ python.pkgs.pytest ]; + propagatedBuildInputs = with python.pkgs; [ pyyaml six ]; + + meta = getMeta "Ceph common module for code shared by manager modules"; + }; + + # Boost 1.75 is not compatible with Python 3.10 + python = python39.override { + packageOverrides = self: super: { + sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec { + version = "1.4.46"; + src = super.fetchPypi { + pname = "SQLAlchemy"; + inherit version; + hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA="; + }; + nativeCheckInputs = oldAttrs.nativeCheckInputs ++ (with super; [ + pytest-xdist + ]); + disabledTestPaths = (oldAttrs.disabledTestPaths or []) ++ [ + "test/aaa_profiling" + "test/ext/mypy" + ]; + }); + }; + }; + + boost = boost175.override { + enablePython = true; + inherit python; + }; + + ceph-python-env = python.withPackages (ps: [ + # Check .requires files below https://github.com/ceph/ceph/tree/main/debian for dependencies + ps.sphinx + ps.flask + ps.routes + ps.cython + ps.setuptools + ps.virtualenv + # Libraries needed by the python tools + ps.mako + ceph-common + ps.cherrypy + ps.cmd2 + ps.colorama + ps.python-dateutil + ps.jsonpatch + ps.pecan + ps.prettytable + ps.pyopenssl + ps.pyjwt + ps.webob + ps.bcrypt + ps.scipy + ps.six + ps.pyyaml + ]); + sitePackages = ceph-python-env.python.sitePackages; + + version = "16.2.10"; + src = fetchurl { + url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz"; + sha256 = "sha256-342+nUV3mCX7QJfZSnKEfnQFCJwJmVQeYnefJwW/AtU="; + }; +in rec { + ceph = stdenv.mkDerivation { + pname = "ceph"; + inherit src version; + + patches = [ + ./0000-fix-SPDK-build-env.patch + # pacific: include/buffer: include + # fixes build with gcc 12 + # https://github.com/ceph/ceph/pull/47295 + (fetchpatch { + url = "https://github.com/ceph/ceph/pull/47295/commits/df88789a38c053513d3b2a9b7d12a952fc0c9042.patch"; + hash = "sha256-je65kBfa5hR0ZKo6ZI10XmD5ZUbKj5rxlGxxI9ZJVfo="; + }) + (fetchpatch { + url = "https://github.com/ceph/ceph/pull/47295/commits/2abcbe4e47705e6e0fcc7d9d9b75625f563199af.patch"; + hash = "sha256-8sWQKoZNHuGuhzX/F+3fY4+kjsrwsfoMdVpfVSj2x5w="; + }) + (fetchpatch { + url = "https://github.com/ceph/ceph/pull/47295/commits/13dc077cf6c65a3b8c4f13d896847b9964b3fcbb.patch"; + hash = "sha256-byfiZh9OJrux/y5m3QCPg0LET6q33ZDXmp/CN+yOSQQ="; + }) + ]; + + nativeBuildInputs = [ + cmake + pkg-config which git python.pkgs.wrapPython makeWrapper + python.pkgs.python # for the toPythonPath function + (ensureNewerSourcesHook { year = "1980"; }) + python + fmt + # for building docs/man-pages presumably + doxygen + graphviz + ]; + + buildInputs = cryptoLibsMap.${cryptoStr} ++ [ + boost xz ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 + malloc zlib openldap lttng-ust babeltrace gperf gtest cunit + snappy lz4 oath-toolkit leveldb libnl libcap_ng rdkafka + cryptsetup sqlite lua icu bzip2 + ] ++ lib.optionals stdenv.isLinux [ + linuxHeaders util-linux libuuid udev keyutils liburing optLibaio optLibxfs optZfs + # ceph 14 + rdma-core rabbitmq-c + ] ++ lib.optionals hasRadosgw [ + optFcgi optExpat optCurl optFuse optLibedit + ]; + + pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ]; + + preConfigure ='' + substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo" + substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe" + substituteInPlace src/common/module.c --replace "/bin/grep" "grep" + + # install target needs to be in PYTHONPATH for "*.pth support" check to succeed + # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others + export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages} + patchShebangs src/script src/spdk src/test src/tools + ''; + + cmakeFlags = [ + "-DWITH_SYSTEM_ROCKSDB=OFF" # breaks Bluestore + "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib" + + "-DWITH_SYSTEM_BOOST=ON" + "-DWITH_SYSTEM_GTEST=ON" + "-DMGR_PYTHON_VERSION=${ceph-python-env.python.pythonVersion}" + "-DWITH_SYSTEMD=OFF" + "-DWITH_TESTS=OFF" + "-DWITH_CEPHFS_SHELL=ON" + # TODO breaks with sandbox, tries to download stuff with npm + "-DWITH_MGR_DASHBOARD_FRONTEND=OFF" + # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now + ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}'' + ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON"; + + postFixup = '' + wrapPythonPrograms + wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})" + + # Test that ceph-volume exists since the build system has a tendency to + # silently drop it with misconfigurations. + test -f $out/bin/ceph-volume + ''; + + outputs = [ "out" "lib" "dev" "doc" "man" ]; + + doCheck = false; # uses pip to install things from the internet + + # Takes 7+h to build with 2 cores. + requiredSystemFeatures = [ "big-parallel" ]; + + meta = getMeta "Distributed storage system"; + + passthru.version = version; + passthru.tests = { inherit (nixosTests) ceph-single-node ceph-multi-node ceph-single-node-bluestore; }; + }; + + ceph-client = runCommand "ceph-client-${version}" { + meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs"; + } '' + mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions} + cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin + cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin + cp -r ${ceph}/bin/rbd-replay* $out/bin + cp -r ${ceph}/sbin/mount.ceph $out/bin + cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin + ln -s bin $out/sbin + cp -r ${ceph}/${sitePackages}/* $out/${sitePackages} + cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions + # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths + substituteInPlace $out/bin/ceph --replace ${ceph} $out + substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out + ''; +} diff --git a/overlays/default.nix b/overlays/default.nix index 1712dcbb..8ee3a3e9 100644 --- a/overlays/default.nix +++ b/overlays/default.nix @@ -13,6 +13,8 @@ with final; { bmxd = callPackage ./bmxd.nix { }; + ceph_16_2 = callPackage ./ceph.nix { }; + dex-oidc = prev.dex-oidc.override { buildGoModule = args: buildGoModule (args // { patches = args.patches or [ ] ++ [