# SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense import nimblepkg/common, nimblepkg/options, nimblepkg/packageinfo, nimblepkg/packageparser, nimblepkg/version import std/[algorithm, deques, httpclient, json, os, osproc, parseutils, streams, strutils, uri] import preserves const githubPackagesUrl = "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json" type Preserve = preserves.Preserve[void] proc registryCachePath: string = result = getEnv("XDG_CACHE_HOME") if result == "": let home = getEnv("HOME") if home == "": result = getEnv("TMPDIR", "/tmp") else: result = home / ".cache" result.add "/packages.json" proc isGitUrl(uri: Uri): bool = uri.scheme == "git" or uri.scheme.startsWith("git+") or uri.path.endsWith(".git") or uri.hostname.contains("git") proc startProcess(cmd: string; cmdArgs: varargs[string]): Process = # stderr.writeLine(cmd, " ", join(cmdArgs, " ")) startProcess(cmd, args = cmdArgs, options = {poUsePath}) proc gitLsRemote(url: string; withTags: bool): seq[tuple[tag: string, rev: string]] = var line, rev, refer: string var process = if withTags: startProcess("git", "ls-remote", "--tags", url) else: startProcess("git", "ls-remote", url) while process.outputStream.readLine(line): var off = 0 off.inc parseUntil(line, rev, Whitespace, off) off.inc skipWhile(line, Whitespace, off) refer = line[off..line.high] const refsTags = "refs/tags/" headsTags = "refs/heads/" if refer.startsWith(refsTags): refer.removePrefix(refsTags) result.add((refer, rev,)) elif refer.startsWith(headsTags): refer.removePrefix(headsTags) result.add((refer, rev,)) stderr.write(process.errorStream.readAll) close(process) if withTags and result.len == 0: result = gitLsRemote(url, not withTags) proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string] = if wanted.kind == verSpecial: let special = string(wanted.spe) if special.len == 41 and special[0] == '#': result[1] = special[1..39] else: quit("unhandled version " & url & " " & $wanted) else: let withTags = wanted.kind != verAny let pairs = gitLsRemote(url, withTags) var resultVersion: Version for (tag, rev) in pairs: var tagVer = Version(tag) if tagVer.withinRange(wanted) and resultVersion < tagVer: resultVersion = tagVer result = (tag, rev) if result.rev == "" and pairs.len > 0: result = pairs[pairs.high] doAssert result.rev != "", url proc `[]`(dict: Preserve; key: string): Preserve = dict[key.toPreserve] proc `[]=`(dict: var Preserve; key: string; val: Preserve) = dict[key.toPreserve] = val proc `[]=`(dict: var Preserve; key: string; val: string) = dict[key.toPreserve] = val.toPreserve proc collectMetadata(data: var Preserve) = let storePath = data["path"].string var packageNames = newSeq[string]() for (kind, path) in walkDir(storePath): if kind in {pcFile, pcLinkToFile} and path.endsWith(".nimble"): var (_, name, _) = splitFile(path) packageNames.add name if packageNames.len == 0: quit("no .nimble files found in " & storePath) sort(packageNames) data["packages"] = packageNames.toPreserve(void) var nimbleFilePath = findNimbleFile(storePath, true) pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) data["srcDir"] = pkg.srcDir.toPreserve proc prefetchGit(uri: Uri; version: VersionRange): Preserve = var uri = uri subdir = "" uri.scheme.removePrefix("git+") if uri.query != "": if uri.query.startsWith("subdir="): subdir = uri.query[7 .. ^1] uri.query = "" let cloneUrl = $uri let (tag, rev) = matchRev(cloneUrl, version) var archiveUri = uri archiveUri.scheme = "https" archiveUri.path.removeSuffix ".git" archiveUri.path.add "/archive/" archiveUri.path.add rev archiveUri.path.add ".tar.gz" let client = newHttpClient() defer: close(client) let archiveUrl = $archiveUri resp = head(client, archiveUrl) if resp.code in {Http200, Http302}: stderr.writeLine "prefetch ", archiveUrl var lines = execProcess( "nix-prefetch-url", args = @[archiveUrl, "--type", "sha256", "--print-path", "--unpack", "--name", "source"], options = {poUsePath}) var hash, storePath: string off: int off.inc parseUntil(lines, hash, {'\n'}, off).succ off.inc parseUntil(lines, storePath, {'\n'}, off).succ doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines result = initDictionary() result["method"] = "fetchzip" result["path"] = storePath result["rev"] = rev result["sha256"] = hash result["url"] = archiveUrl if subdir != "": result["subdir"] = subdir else: stderr.writeLine "fetch of ", archiveUrl, " returned ", resp.code var args = @["--quiet", "--fetch-submodules", "--url", cloneUrl, "--rev", rev] stderr.writeLine "prefetch ", cloneUrl let dump = execProcess( "nix-prefetch-git", args = args, options = {poUsePath}) try: result = parsePreserves dump except CatchableError: stderr.writeLine "failed to parse output of nix-prefetch-git ", join(args, " ") quit(dump) if subdir != "": result["subdir"] = subdir result["method"] = "git" if tag != "": result["ref"] = tag collectMetadata(result) proc containsPackageUri(lockAttrs: Preserve; pkgUri: string): bool = for e in lockAttrs.items: if e["url".toPreserve].string == pkgUri: return true proc containsPackage(lockAttrs: Preserve; pkgName: string): bool = for e in lockAttrs.items: for other in e["packages"].items: if pkgName == other.string: return true proc collectRequires(pending: var Deque[PkgTuple]; pkgPath: string) = var nimbleFilePath = findNimbleFile(pkgPath, true) pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) for pair in pkg.requires: if pair.name != "nim" and pair.name != "compiler": pending.addLast(pair) var globalRegistry: JsonNode proc getPackgeUri(name: string): tuple[uri: string, meth: string] = if globalRegistry.isNil: let registryPath = registryCachePath() if fileExists(registryPath): globalRegistry = parseFile(registryPath) else: let client = newHttpClient() var raw = client.getContent(githubPackagesUrl) close(client) writeFile(registryPath, raw) globalRegistry = parseJson(raw) var name = name i = 0 while i < globalRegistry.len: var e = globalRegistry[i] if e["name"].getStr == name: if e.hasKey "alias": var alias = e["alias"].getStr doAssert alias != name name = alias i = 0 else: try: return (e["url"].getStr, e["method"].getStr,) except CatchableError: quit("Failed to parse shit JSON " & $e) inc i proc generateLockfile(): Preserve = result = initDictionary() var deps = initSequence() pending: Deque[PkgTuple] collectRequires(pending, getCurrentDir()) while pending.len > 0: let batchLen = pending.len for i in 1..batchLen: var pkgData: Preserve let pkg = pending.popFirst() if pkg.name == "nim" or pkg.name == "compiler": continue var uri = parseUri(pkg.name) if uri.scheme == "": if not deps.containsPackage(pkg.name): pending.addLast(pkg) elif not deps.containsPackageUri(pkg.name): if uri.isGitUrl: pkgData = prefetchGit(uri, pkg.ver) else: quit("unhandled URI " & $uri) collectRequires(pending, pkgData["path"].string) deps.sequence.add pkgData if batchLen == pending.len: var pkgData: Preserve pkg = pending.popFirst() info = getPackgeUri(pkg.name) uri = parseUri info.uri case info.meth of "git": pkgData = prefetchGit(uri, pkg.ver) else: quit("unhandled fetch method " & $info.meth & " for " & info.uri) collectRequires(pending, pkgData["path"].string) deps.sequence.add pkgData sort(deps.sequence) result["depends".toPreserve] = deps proc main = var stream = newFileStream(stdout) lockInfo = generateLockfile() cannonicalize(lockInfo) writeText(stream, lockInfo, textJson) writeLine(stream) close(stream) main()