erisPatchHook: patch ELF images to load ERIS URNs

Add this hook to the Genode stdenv.
This commit is contained in:
Emery Hemingway 2021-02-08 11:37:04 +01:00
parent 83c36784ff
commit e3524c4277
9 changed files with 297 additions and 8 deletions

View File

@ -1,6 +1,37 @@
{
"nodes": {
"nimble": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1614366386,
"narHash": "sha256-zs91mU2gQUzvcShKckdWfBIV3eTVmjZRZn3j+z8p7eE=",
"owner": "nix-community",
"repo": "flake-nimble",
"rev": "5f7ef9476c394985fee98b71631641970d7ffb07",
"type": "github"
},
"original": {
"id": "nimble",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1607766819,
"narHash": "sha256-bluEp6ld6wmpeLl5MQPQOpxWMDLnUYyQNEk2rMlAyiU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "f448ec33655c48d7306456bee77f3cdabf3757fa",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1613672748,
"narHash": "sha256-8f/GGmO8zWSQj2lVjoDyQDURk6Nx9lY0VC7MwkE/Y8U=",
@ -18,7 +49,8 @@
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
"nimble": "nimble",
"nixpkgs": "nixpkgs_2"
}
}
},

View File

@ -3,7 +3,7 @@
inputs.nixpkgs.url = "github:ehmry/nixpkgs/genodepkgs";
outputs = { self, nixpkgs }:
outputs = { self, nixpkgs, nimble }:
let
systems = {
localSystem = [ "x86_64-linux" ];
@ -39,7 +39,7 @@
if localSystem == crossSystem then
import nixpkgs {
inherit system;
overlays = [ self.overlay ];
overlays = [ self.overlay nimble.overlay ];
}
else
import nixpkgs {
@ -49,7 +49,7 @@
useLLVM = true;
};
config.allowUnsupportedSystem = true;
overlays = [ self.overlay ];
overlays = [ self.overlay nimble.overlay ];
});
in rec {
@ -63,8 +63,35 @@
nixpkgs.lib.extend (final: prev: {
inherit forAllSystems forAllLocalSystems forAllCrossSystems;
/* For a the name of a derivation output and a derivation,
generate a set of { cap, closure, and path } for a singular
file found within the subdirectory of the output with the
same name as that output. In the case that the derivation
does not have this named output, the subdirectory will be
taken from the default output. This subdirectory must
contain a single file, and the output must contain an
ERIS manifest file.
*/
getEris = output: pkg:
with builtins;
let
pkg' = prev.getOutput output pkg;
erisInfo =
fromJSON (readFile "${pkg'}/nix-support/eris-manifest.json");
caps = filter
({ path, ... }: prev.strings.hasPrefix "${pkg'}/${output}" path)
(prev.attrsets.mapAttrsToList (path:
{ cap, closure }: {
path = "${pkg'}${
substring (stringLength pkg') (stringLength path) path
}"; # hack to build a string with context
inherit cap closure;
}) erisInfo);
in assert length caps == 1; head caps;
nixosSystem = { modules, ... }@args:
import "${nixpkgs}/nixos/lib/eval-config.nix" (args // {
lib = final;
baseModules =
# TODO: do not blacklist modules for the Linux guests

View File

@ -79,6 +79,12 @@ in nullPkgs // {
# keep libposix NEEDED
}) coreutils);
erisPatchHook = final.callPackage ./eris-patch-hook {
patchelf = prev.patchelf.overrideAttrs (attrs: {
patches = attrs.patched or [ ] ++ [ ./patchelf/dynstr.patch ];
});
};
gccForLibs = if targetPlatform.isGenode then
final.genodePackages.genodeSources.toolchain.cc
else
@ -176,6 +182,11 @@ in nullPkgs // {
solo5-tools = callPackage ./solo5-tools { };
stdenv = overrideHost (old: {
extraNativeBuildInputs = old.extraNativeBuildInputs
++ [ final.buildPackages.erisPatchHook ];
}) prev.stdenv;
tor = overrideAttrsHost (attrs: {
configureFlags = attrs.configureFlags or [ ]
++ [ "--disable-tool-name-check" ];

View File

@ -0,0 +1,17 @@
{ lib, stdenv, patchelf, nimblePackages }:
stdenv.mkDerivation {
name = "eris_patch";
nativeBuildInputs = with nimblePackages; [ nim ];
buildInputs = with nimblePackages; [ eris ];
inherit patchelf;
dontUnpack = true;
nimFlags = [ "-d:release" ];
buildPhase = ''
HOME=$TMPDIR
cp ${./eris_patch.nim} eris_patch.nim
nim c $nimFlags eris_patch
'';
installPhase = "install -Dt $out/bin eris_patch";
setupHook = ./eris_patch.sh;
}

View File

@ -0,0 +1,189 @@
import eris
import std/asyncdispatch, std/deques, std/json, std/os, std/osproc, std/streams,
std/strutils, std/tables
if getEnv("dontErisPatch") != "": quit 0
let
patchelf = getEnv("ERIS_PATCHELF", "patchelf")
nixStore = getEnv("NIX_STORE", "/nix/store")
manifestSubPath = "nix-support" / "eris-manifest.json"
const erisBlockSize = 32 shl 10
# fix the block size for now
proc isElf(path: string): bool =
var magic: array[4, char]
let file = open(path)
discard readChars(file, magic, 0, 4)
close(file)
magic == [0x7f.char, 'E', 'L', 'F']
type PendingFile = ref object
outputRoot, filePath: string
replacements: Table[string, string]
var
outputManifests = initTable[string, JsonNode]()
pendingFiles = initDeque[PendingFile]()
failed = false
for outputName in getEnv("outputs").splitWhitespace:
let outputRoot = getEnv(outputName)
if fileExists(outputRoot / manifestSubPath):
echo "Not running ERIS patch hook again"
quit 0
outputManifests[outputRoot] = newJObject()
let buildInputs = getEnv("buildInputs").splitWhitespace
proc resolveNeed(rpath: seq[string]; need: string): string =
if need.isAbsolute:
return need
for libdir in rpath:
let absNeed = libdir / need
if fileExists(absNeed):
return absNeed
for outputRoot in outputManifests.keys:
for relPath in [need, "lib" / need]:
let absNeed = outputRoot / relPath
if fileExists(absNeed):
return absNeed
for buildInput in buildInputs:
for relPath in [need, "lib" / need]:
let absNeed = buildInput / relPath
if fileExists(absNeed):
return absNeed
proc resolveFile(outputRoot, filePath: string): PendingFile =
result = PendingFile(
outputRoot: outputRoot,
filePath: filePath,
replacements: initTable[string, string](8))
let needs = splitWhitespace(execProcess(
patchelf, args = ["--print-needed", filePath], options = {poUsePath}))
let rpath = splitWhitespace(execProcess(
patchelf, args = ["--print-rpath", filePath], options = {poUsePath}))
for need in needs:
if need == "ld.lib.so" or need.startsWith("urn:"): continue
result.replacements[need] = resolveNeed(rpath, need)
var capCache = initTable[string, Cap]()
proc fileUrn(filePath: string; blockSize: Natural): string =
## Determine the ERIS URN for ``filePath``.
var cap: Cap
if capCache.hasKey(filePath):
cap = capCache[filePath]
else:
try:
let str = newFileStream(filePath)
doAssert(not str.isNil) # yes, that happens
cap = waitFor encode(newDiscardStore(), blockSize, str)
capCache["filePath"] = cap
close(str)
except:
stderr.writeLine("failed to read \"", filePath, "\"")
quit 1
$cap # & "#" & encodeUrl(extractFilename(filePath), usePlus = false)
var closureCache = initTable[string, TableRef[string, string]]()
proc fileClosure(filePath: string): TableRef[string, string] =
## Recusively find the dependency closure of ``filePath``.
let filePath = expandFilename filePath
if closureCache.hasKey(filePath):
result = closureCache[filePath]
else:
result = newTable[string, string]()
var storePath = filePath
for p in parentDirs(filePath):
# find the top directory of the ``filePath`` derivation
if p == nixStore: break
storePath = p
if storePath.startsWith nixStore:
# read the closure manifest of the dependency
let manifestPath = storePath / manifestSubPath
if fileExists(manifestPath):
let
manifest = parseFile(manifestPath)
entry = manifest[filePath]
for path, cap in entry["closure"].pairs:
result[path] = cap.getStr
let otherClosure = fileClosure(path)
for otherPath, otherCap in otherClosure.pairs:
# merge the closure of the dependency
result[otherPath] = otherCap
closureCache[filePath] = result
for outputRoot in outputManifests.keys:
let manifestPath = outputRoot / manifestSubPath
if fileExists manifestPath: continue
for filePath in walkDirRec(outputRoot, relative = false):
# Populate the queue of files to patch
if filePath.isElf:
pendingFiles.addLast(resolveFile(outputRoot, filePath))
var
prevLen = pendingFiles.len
prevPrevLen = prevLen.succ
# used to detect reference cycles
while pendingFiles.len != 0:
block selfReferenceCheck:
# process the files that have been collected
# taking care not to take a the URN of an
# unprocessed file
let
pendingFile = pendingFiles.popFirst()
filePath = pendingFile.filePath
for need, replacementPath in pendingFile.replacements.pairs:
# search for self-references
if replacementPath == "":
echo need, " not found for ", filePath
failed = true
continue
for outputRoot in outputManifests.keys:
if replacementPath.startsWith(outputRoot):
for other in pendingFiles.items:
echo "compare for self-reference:"
echo '\t', replacementPath
echo '\t', other.filePath
if replacementPath == other.filePath:
echo "defering patch of ", filePath, " with reference to ", other.filePath
pendingFiles.addLast(pendingFile)
break selfReferenceCheck
var
closure = newJObject()
replaceCmd = patchelf & " --set-rpath '' " & filePath
for need, replacementPath in pendingFile.replacements.pairs:
if replacementPath == "": continue
let urn = fileUrn(replacementPath, erisBlockSize)
echo "replace reference to ", need, " with ", urn
replaceCmd.add(" --replace-needed $# $#" % [need, urn])
closure[replacementPath] = %urn
for path, urn in fileClosure(replacementPath).pairs:
closure[path] = %urn
if pendingFile.replacements.len != 0:
let (msg, exitCode) = execCmdEx(replaceCmd, options = {poUsePath})
if exitCode != 0:
echo msg
quit exitCode
outputManifests[pendingFile.outputRoot][filePath] = %* {
"cap": fileUrn(filePath, erisBlockSize),
"closure": closure,
}
if pendingFiles.len == prevPrevLen:
failed = true
echo "reference cycle detected in the following:"
for remain in pendingFiles.items:
echo '\t', " ", remain.filePath
break
prevPrevLen = prevLen
prevLen = pendingFiles.len
if failed:
quit -1
for outputRoot, manifest in outputManifests:
createDir(outputRoot / "nix-support")
writeFile(outputRoot / manifestSubPath, $manifest)

View File

@ -0,0 +1,2 @@
export ERIS_PATCHELF=@patchelf@/bin/patchelf
postFixupHooks+=('@out@/bin/eris_patch')

View File

@ -3,7 +3,7 @@
, buildPackages
, buildLlvmTools # tools, but from the previous stage, for cross
, targetLlvmLibraries # libraries, but from the next stage, for cross
, genodePackages ? null
, erisPatchHook, genodePackages ? null
}:
let

View File

@ -186,6 +186,10 @@ let
buildDepot =
# Build a Depot target from the Genode sources
# WARNING: buildDepot can produce artifacts with broken linkage
# to their inputs. The Genode depot mechanism links programs and
# libraries to facsimilie stub libraries which are not guaranteed
# to have the same ABI as the current version as the real library.
{ name, apiOnly ? false, portInputs ? [ ], depotInputs ? [ ]
, nativeBuildInputs ? [ ], buildInputs ? [ ], meta ? { }, ... }@extraAttrs:
@ -205,7 +209,11 @@ let
enableParallelBuilding = true;
nativeBuildInputs = with buildPackages.buildPackages;
[ binutils bison flex stdenv.cc tcl which ] ++ nativeBuildInputs;
[ binutils bison flex stdenv.cc tcl which ]
++ nativeBuildInputs
++ lib.optional (!stdenv.hostPlatform.isGenode) erisPatchHook;
buildInputs = buildInputs ++ depotInputs';
src = genodeSources;
# The genode source tree must be copied to the build directory
@ -350,6 +358,7 @@ in makePackages // depotPackages // {
install build/bin/core-hw-pc.o $coreObj
install build/bin/bootstrap-hw-pc.o $bootstrapObj
'';
dontErisPatch = true;
meta.platforms = [ "x86_64-genode" ];
};
@ -365,6 +374,7 @@ in makePackages // depotPackages // {
install build/bin/core-hw-virt_qemu.o $coreObj
install build/bin/bootstrap-hw-virt_qemu.o $bootstrapObj
'';
dontErisPatch = true;
meta.platforms = [ "aarch64-genode" ];
};
@ -378,6 +388,7 @@ in makePackages // depotPackages // {
mv $out/bin/linux_timer_drv $out/bin/timer_drv
'';
HOST_INC_DIR = buildPackages.glibc.dev + "/include";
dontErisPatch = true;
};
base-nova = buildUpstream {
@ -390,5 +401,6 @@ in makePackages // depotPackages // {
mv $out/bin/nova_timer_drv $out/bin/timer_drv
install build/bin/core-nova.o $coreObj
'';
dontErisPatch = true;
};
}

View File

@ -22,7 +22,7 @@ in {
acpi_drv = { };
acpica = { };
ahci_drv = { };
backdrop = { depotInputs = with self; [ posix libpng ]; };
backdrop = { depotInputs = with self; [ libpng ]; };
bash-minimal = {
enableParallelBuilding = false;
nativeBuildInputs = with buildPackages; [ autoconf ];
@ -205,7 +205,6 @@ in {
rump = {
portInputs = with ports; [ dde_rump ];
buildInputs = with buildPackages; [ zlib ];
patches = [ ./patches/rump-libs.patch ];
};
sandbox = { };
sanitizer = { };