From e3524c4277457eb085c428c11364d09cc793b67b Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 8 Feb 2021 11:37:04 +0100 Subject: [PATCH] erisPatchHook: patch ELF images to load ERIS URNs Add this hook to the Genode stdenv. --- flake.lock | 34 ++++- flake.nix | 33 ++++- overlay/default.nix | 11 ++ overlay/eris-patch-hook/default.nix | 17 +++ overlay/eris-patch-hook/eris_patch.nim | 189 +++++++++++++++++++++++++ overlay/eris-patch-hook/eris_patch.sh | 2 + overlay/llvm-11/default.nix | 2 +- packages/genodelabs/default.nix | 14 +- packages/genodelabs/depot-targets.nix | 3 +- 9 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 overlay/eris-patch-hook/default.nix create mode 100644 overlay/eris-patch-hook/eris_patch.nim create mode 100644 overlay/eris-patch-hook/eris_patch.sh diff --git a/flake.lock b/flake.lock index c70102f..abca65e 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,37 @@ { "nodes": { + "nimble": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1614366386, + "narHash": "sha256-zs91mU2gQUzvcShKckdWfBIV3eTVmjZRZn3j+z8p7eE=", + "owner": "nix-community", + "repo": "flake-nimble", + "rev": "5f7ef9476c394985fee98b71631641970d7ffb07", + "type": "github" + }, + "original": { + "id": "nimble", + "type": "indirect" + } + }, "nixpkgs": { + "locked": { + "lastModified": 1607766819, + "narHash": "sha256-bluEp6ld6wmpeLl5MQPQOpxWMDLnUYyQNEk2rMlAyiU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f448ec33655c48d7306456bee77f3cdabf3757fa", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1613672748, "narHash": "sha256-8f/GGmO8zWSQj2lVjoDyQDURk6Nx9lY0VC7MwkE/Y8U=", @@ -18,7 +49,8 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nimble": "nimble", + "nixpkgs": "nixpkgs_2" } } }, diff --git a/flake.nix b/flake.nix index 2cb0850..39c5624 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ inputs.nixpkgs.url = "github:ehmry/nixpkgs/genodepkgs"; - outputs = { self, nixpkgs }: + outputs = { self, nixpkgs, nimble }: let systems = { localSystem = [ "x86_64-linux" ]; @@ -39,7 +39,7 @@ if localSystem == crossSystem then import nixpkgs { inherit system; - overlays = [ self.overlay ]; + overlays = [ self.overlay nimble.overlay ]; } else import nixpkgs { @@ -49,7 +49,7 @@ useLLVM = true; }; config.allowUnsupportedSystem = true; - overlays = [ self.overlay ]; + overlays = [ self.overlay nimble.overlay ]; }); in rec { @@ -63,8 +63,35 @@ nixpkgs.lib.extend (final: prev: { inherit forAllSystems forAllLocalSystems forAllCrossSystems; + /* For a the name of a derivation output and a derivation, + generate a set of { cap, closure, and path } for a singular + file found within the subdirectory of the output with the + same name as that output. In the case that the derivation + does not have this named output, the subdirectory will be + taken from the default output. This subdirectory must + contain a single file, and the output must contain an + ERIS manifest file. + */ + getEris = output: pkg: + with builtins; + let + pkg' = prev.getOutput output pkg; + erisInfo = + fromJSON (readFile "${pkg'}/nix-support/eris-manifest.json"); + caps = filter + ({ path, ... }: prev.strings.hasPrefix "${pkg'}/${output}" path) + (prev.attrsets.mapAttrsToList (path: + { cap, closure }: { + path = "${pkg'}${ + substring (stringLength pkg') (stringLength path) path + }"; # hack to build a string with context + inherit cap closure; + }) erisInfo); + in assert length caps == 1; head caps; + nixosSystem = { modules, ... }@args: import "${nixpkgs}/nixos/lib/eval-config.nix" (args // { + lib = final; baseModules = # TODO: do not blacklist modules for the Linux guests diff --git a/overlay/default.nix b/overlay/default.nix index 7ef15c8..32b0d4a 100644 --- a/overlay/default.nix +++ b/overlay/default.nix @@ -79,6 +79,12 @@ in nullPkgs // { # keep libposix NEEDED }) coreutils); + erisPatchHook = final.callPackage ./eris-patch-hook { + patchelf = prev.patchelf.overrideAttrs (attrs: { + patches = attrs.patched or [ ] ++ [ ./patchelf/dynstr.patch ]; + }); + }; + gccForLibs = if targetPlatform.isGenode then final.genodePackages.genodeSources.toolchain.cc else @@ -176,6 +182,11 @@ in nullPkgs // { solo5-tools = callPackage ./solo5-tools { }; + stdenv = overrideHost (old: { + extraNativeBuildInputs = old.extraNativeBuildInputs + ++ [ final.buildPackages.erisPatchHook ]; + }) prev.stdenv; + tor = overrideAttrsHost (attrs: { configureFlags = attrs.configureFlags or [ ] ++ [ "--disable-tool-name-check" ]; diff --git a/overlay/eris-patch-hook/default.nix b/overlay/eris-patch-hook/default.nix new file mode 100644 index 0000000..a26bdaf --- /dev/null +++ b/overlay/eris-patch-hook/default.nix @@ -0,0 +1,17 @@ +{ lib, stdenv, patchelf, nimblePackages }: + +stdenv.mkDerivation { + name = "eris_patch"; + nativeBuildInputs = with nimblePackages; [ nim ]; + buildInputs = with nimblePackages; [ eris ]; + inherit patchelf; + dontUnpack = true; + nimFlags = [ "-d:release" ]; + buildPhase = '' + HOME=$TMPDIR + cp ${./eris_patch.nim} eris_patch.nim + nim c $nimFlags eris_patch + ''; + installPhase = "install -Dt $out/bin eris_patch"; + setupHook = ./eris_patch.sh; +} diff --git a/overlay/eris-patch-hook/eris_patch.nim b/overlay/eris-patch-hook/eris_patch.nim new file mode 100644 index 0000000..0dff3ab --- /dev/null +++ b/overlay/eris-patch-hook/eris_patch.nim @@ -0,0 +1,189 @@ +import eris + +import std/asyncdispatch, std/deques, std/json, std/os, std/osproc, std/streams, + std/strutils, std/tables + +if getEnv("dontErisPatch") != "": quit 0 + +let + patchelf = getEnv("ERIS_PATCHELF", "patchelf") + nixStore = getEnv("NIX_STORE", "/nix/store") + manifestSubPath = "nix-support" / "eris-manifest.json" + +const erisBlockSize = 32 shl 10 + # fix the block size for now + +proc isElf(path: string): bool = + var magic: array[4, char] + let file = open(path) + discard readChars(file, magic, 0, 4) + close(file) + magic == [0x7f.char, 'E', 'L', 'F'] + +type PendingFile = ref object + outputRoot, filePath: string + replacements: Table[string, string] + +var + outputManifests = initTable[string, JsonNode]() + pendingFiles = initDeque[PendingFile]() + failed = false +for outputName in getEnv("outputs").splitWhitespace: + let outputRoot = getEnv(outputName) + if fileExists(outputRoot / manifestSubPath): + echo "Not running ERIS patch hook again" + quit 0 + outputManifests[outputRoot] = newJObject() + +let buildInputs = getEnv("buildInputs").splitWhitespace + +proc resolveNeed(rpath: seq[string]; need: string): string = + if need.isAbsolute: + return need + for libdir in rpath: + let absNeed = libdir / need + if fileExists(absNeed): + return absNeed + for outputRoot in outputManifests.keys: + for relPath in [need, "lib" / need]: + let absNeed = outputRoot / relPath + if fileExists(absNeed): + return absNeed + for buildInput in buildInputs: + for relPath in [need, "lib" / need]: + let absNeed = buildInput / relPath + if fileExists(absNeed): + return absNeed + +proc resolveFile(outputRoot, filePath: string): PendingFile = + result = PendingFile( + outputRoot: outputRoot, + filePath: filePath, + replacements: initTable[string, string](8)) + let needs = splitWhitespace(execProcess( + patchelf, args = ["--print-needed", filePath], options = {poUsePath})) + let rpath = splitWhitespace(execProcess( + patchelf, args = ["--print-rpath", filePath], options = {poUsePath})) + for need in needs: + if need == "ld.lib.so" or need.startsWith("urn:"): continue + result.replacements[need] = resolveNeed(rpath, need) + +var capCache = initTable[string, Cap]() + +proc fileUrn(filePath: string; blockSize: Natural): string = + ## Determine the ERIS URN for ``filePath``. + var cap: Cap + if capCache.hasKey(filePath): + cap = capCache[filePath] + else: + try: + let str = newFileStream(filePath) + doAssert(not str.isNil) # yes, that happens + cap = waitFor encode(newDiscardStore(), blockSize, str) + capCache["filePath"] = cap + close(str) + except: + stderr.writeLine("failed to read \"", filePath, "\"") + quit 1 + $cap # & "#" & encodeUrl(extractFilename(filePath), usePlus = false) + +var closureCache = initTable[string, TableRef[string, string]]() + +proc fileClosure(filePath: string): TableRef[string, string] = + ## Recusively find the dependency closure of ``filePath``. + let filePath = expandFilename filePath + if closureCache.hasKey(filePath): + result = closureCache[filePath] + else: + result = newTable[string, string]() + var storePath = filePath + for p in parentDirs(filePath): + # find the top directory of the ``filePath`` derivation + if p == nixStore: break + storePath = p + if storePath.startsWith nixStore: + # read the closure manifest of the dependency + let manifestPath = storePath / manifestSubPath + if fileExists(manifestPath): + let + manifest = parseFile(manifestPath) + entry = manifest[filePath] + for path, cap in entry["closure"].pairs: + result[path] = cap.getStr + let otherClosure = fileClosure(path) + for otherPath, otherCap in otherClosure.pairs: + # merge the closure of the dependency + result[otherPath] = otherCap + closureCache[filePath] = result + +for outputRoot in outputManifests.keys: + let manifestPath = outputRoot / manifestSubPath + if fileExists manifestPath: continue + for filePath in walkDirRec(outputRoot, relative = false): + # Populate the queue of files to patch + if filePath.isElf: + pendingFiles.addLast(resolveFile(outputRoot, filePath)) + +var + prevLen = pendingFiles.len + prevPrevLen = prevLen.succ + # used to detect reference cycles +while pendingFiles.len != 0: + block selfReferenceCheck: + # process the files that have been collected + # taking care not to take a the URN of an + # unprocessed file + let + pendingFile = pendingFiles.popFirst() + filePath = pendingFile.filePath + for need, replacementPath in pendingFile.replacements.pairs: + # search for self-references + if replacementPath == "": + echo need, " not found for ", filePath + failed = true + continue + for outputRoot in outputManifests.keys: + if replacementPath.startsWith(outputRoot): + for other in pendingFiles.items: + echo "compare for self-reference:" + echo '\t', replacementPath + echo '\t', other.filePath + if replacementPath == other.filePath: + echo "defering patch of ", filePath, " with reference to ", other.filePath + pendingFiles.addLast(pendingFile) + break selfReferenceCheck + var + closure = newJObject() + replaceCmd = patchelf & " --set-rpath '' " & filePath + for need, replacementPath in pendingFile.replacements.pairs: + if replacementPath == "": continue + let urn = fileUrn(replacementPath, erisBlockSize) + echo "replace reference to ", need, " with ", urn + replaceCmd.add(" --replace-needed $# $#" % [need, urn]) + closure[replacementPath] = %urn + for path, urn in fileClosure(replacementPath).pairs: + closure[path] = %urn + if pendingFile.replacements.len != 0: + let (msg, exitCode) = execCmdEx(replaceCmd, options = {poUsePath}) + if exitCode != 0: + echo msg + quit exitCode + outputManifests[pendingFile.outputRoot][filePath] = %* { + "cap": fileUrn(filePath, erisBlockSize), + "closure": closure, + } + if pendingFiles.len == prevPrevLen: + failed = true + echo "reference cycle detected in the following:" + for remain in pendingFiles.items: + echo '\t', " ", remain.filePath + break + prevPrevLen = prevLen + prevLen = pendingFiles.len + +if failed: + quit -1 + +for outputRoot, manifest in outputManifests: + createDir(outputRoot / "nix-support") + writeFile(outputRoot / manifestSubPath, $manifest) diff --git a/overlay/eris-patch-hook/eris_patch.sh b/overlay/eris-patch-hook/eris_patch.sh new file mode 100644 index 0000000..29b8b84 --- /dev/null +++ b/overlay/eris-patch-hook/eris_patch.sh @@ -0,0 +1,2 @@ +export ERIS_PATCHELF=@patchelf@/bin/patchelf +postFixupHooks+=('@out@/bin/eris_patch') diff --git a/overlay/llvm-11/default.nix b/overlay/llvm-11/default.nix index 4016774..ac3e98d 100644 --- a/overlay/llvm-11/default.nix +++ b/overlay/llvm-11/default.nix @@ -3,7 +3,7 @@ , buildPackages , buildLlvmTools # tools, but from the previous stage, for cross , targetLlvmLibraries # libraries, but from the next stage, for cross -, genodePackages ? null +, erisPatchHook, genodePackages ? null }: let diff --git a/packages/genodelabs/default.nix b/packages/genodelabs/default.nix index bd81d2f..3e63791 100644 --- a/packages/genodelabs/default.nix +++ b/packages/genodelabs/default.nix @@ -186,6 +186,10 @@ let buildDepot = # Build a Depot target from the Genode sources + # WARNING: buildDepot can produce artifacts with broken linkage + # to their inputs. The Genode depot mechanism links programs and + # libraries to facsimilie stub libraries which are not guaranteed + # to have the same ABI as the current version as the real library. { name, apiOnly ? false, portInputs ? [ ], depotInputs ? [ ] , nativeBuildInputs ? [ ], buildInputs ? [ ], meta ? { }, ... }@extraAttrs: @@ -205,7 +209,11 @@ let enableParallelBuilding = true; nativeBuildInputs = with buildPackages.buildPackages; - [ binutils bison flex stdenv.cc tcl which ] ++ nativeBuildInputs; + [ binutils bison flex stdenv.cc tcl which ] + ++ nativeBuildInputs + ++ lib.optional (!stdenv.hostPlatform.isGenode) erisPatchHook; + + buildInputs = buildInputs ++ depotInputs'; src = genodeSources; # The genode source tree must be copied to the build directory @@ -350,6 +358,7 @@ in makePackages // depotPackages // { install build/bin/core-hw-pc.o $coreObj install build/bin/bootstrap-hw-pc.o $bootstrapObj ''; + dontErisPatch = true; meta.platforms = [ "x86_64-genode" ]; }; @@ -365,6 +374,7 @@ in makePackages // depotPackages // { install build/bin/core-hw-virt_qemu.o $coreObj install build/bin/bootstrap-hw-virt_qemu.o $bootstrapObj ''; + dontErisPatch = true; meta.platforms = [ "aarch64-genode" ]; }; @@ -378,6 +388,7 @@ in makePackages // depotPackages // { mv $out/bin/linux_timer_drv $out/bin/timer_drv ''; HOST_INC_DIR = buildPackages.glibc.dev + "/include"; + dontErisPatch = true; }; base-nova = buildUpstream { @@ -390,5 +401,6 @@ in makePackages // depotPackages // { mv $out/bin/nova_timer_drv $out/bin/timer_drv install build/bin/core-nova.o $coreObj ''; + dontErisPatch = true; }; } diff --git a/packages/genodelabs/depot-targets.nix b/packages/genodelabs/depot-targets.nix index 0ea2cfe..c70e460 100644 --- a/packages/genodelabs/depot-targets.nix +++ b/packages/genodelabs/depot-targets.nix @@ -22,7 +22,7 @@ in { acpi_drv = { }; acpica = { }; ahci_drv = { }; - backdrop = { depotInputs = with self; [ posix libpng ]; }; + backdrop = { depotInputs = with self; [ libpng ]; }; bash-minimal = { enableParallelBuilding = false; nativeBuildInputs = with buildPackages; [ autoconf ]; @@ -205,7 +205,6 @@ in { rump = { portInputs = with ports; [ dde_rump ]; buildInputs = with buildPackages; [ zlib ]; - patches = [ ./patches/rump-libs.patch ]; }; sandbox = { }; sanitizer = { };