# SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense import nimblepkg/common, nimblepkg/options, nimblepkg/packageinfo, nimblepkg/packageparser, nimblepkg/version import std/[algorithm, deques, httpclient, json, os, osproc, parseutils, streams, strutils, uri] const githubPackagesUrl = "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json" proc registryCachePath: string = result = getEnv("XDG_CACHE_HOME") if result == "": let home = getEnv("HOME") if home == "": result = getEnv("TMPDIR", "/tmp") else: result = home / ".cache" result.add "/packages.json" func isGitUrl(uri: Uri): bool = uri.path.endsWith(".git") or uri.scheme == "git" or uri.scheme.startsWith("git+") proc isWebGitForgeUrl(uri: Uri): bool = case uri.hostname of "github.com", "git.sr.ht", "codeberg.org": true else: false proc startProcess(cmd: string; cmdArgs: varargs[string]): Process = # stderr.writeLine(cmd, " ", join(cmdArgs, " ")) startProcess(cmd, args = cmdArgs, options = {poUsePath}) proc gitLsRemote(url: string; withTags: bool): seq[tuple[tag: string, rev: string]] = var line, rev, refer: string var process = if withTags: startProcess("git", "ls-remote", "--tags", url) else: startProcess("git", "ls-remote", url) while process.outputStream.readLine(line): var off = 0 off.inc parseUntil(line, rev, Whitespace, off) off.inc skipWhile(line, Whitespace, off) refer = line[off..line.high] const refsTags = "refs/tags/" headsTags = "refs/heads/" if refer.startsWith(refsTags): refer.removePrefix(refsTags) result.add((refer, rev,)) elif refer.startsWith(headsTags): refer.removePrefix(headsTags) result.add((refer, rev,)) stderr.write(process.errorStream.readAll) close(process) if withTags and result.len == 0: result = gitLsRemote(url, not withTags) proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string] = if wanted.kind == verSpecial: let special = string(wanted.spe) if special.len == 41 and special[0] == '#': result[1] = special[1..39] else: quit("unhandled version " & url & " " & $wanted) else: let withTags = wanted.kind != verAny let pairs = gitLsRemote(url, withTags) var resultVersion: Version for (tag, rev) in pairs: var tagVer = Version(tag) if tagVer.withinRange(wanted) and resultVersion < tagVer: resultVersion = tagVer result = (tag, rev) if result.rev == "" and pairs.len > 0: result = pairs[pairs.high] doAssert result.rev != "", url proc collectMetadata(data: JsonNode) = let storePath = data["path"].getStr var packageNames = newSeq[string]() for (kind, path) in walkDir(storePath): if kind in {pcFile, pcLinkToFile} and path.endsWith(".nimble"): var (_, name, _) = splitFile(path) packageNames.add name if packageNames.len == 0: quit("no .nimble files found in " & storePath) sort(packageNames) data["packages"] = %packageNames var nimbleFilePath = findNimbleFile(storePath, true) pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) data["srcDir"] = %pkg.srcDir proc prefetchGit(uri: Uri; version: VersionRange): JsonNode = var uri = uri subdir = "" uri.scheme.removePrefix("git+") if uri.query != "": if uri.query.startsWith("subdir="): subdir = uri.query[7 .. ^1] uri.query = "" let url = $uri let (tag, rev) = matchRev(url, version) var args = @["--quiet", "--fetch-submodules", "--url", url, "--rev", rev] stderr.writeLine "prefetch ", url let dump = execProcess( "nix-prefetch-git", args = args, options = {poUsePath}) try: result = parseJson dump except JsonParsingError: stderr.writeLine "failed to parse output of nix-prefetch-git ", join(args, " ") quit(dump) if subdir != "": result["subdir"] = %* subdir result["method"] = %"git" if tag != "": result["ref"] = %tag collectMetadata(result) proc prefetchGitForge(uri: Uri; version: VersionRange): JsonNode = result = newJObject() var uri = uri subdir = "" uri.scheme.removePrefix("git+") if uri.query != "": if uri.query.startsWith("subdir="): subdir = uri.query[7 .. ^1] uri.query = "" var url = $uri let (tag, rev) = matchRev(url, version) uri.scheme = "https" uri.path.removeSuffix(".git") uri.path.add("/archive/") uri.path.add(rev) uri.path.add(".tar.gz") url = $uri stderr.writeLine "prefetch ", url var lines = execProcess( "nix-prefetch-url", args = @[url, "--type", "sha256", "--print-path", "--unpack", "--name", "source"], options = {poUsePath}) var hash, storePath: string off: int off.inc parseUntil(lines, hash, {'\n'}, off).succ off.inc parseUntil(lines, storePath, {'\n'}, off).succ doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines result["method"] = %"fetchzip" result["path"] = %storePath result["rev"] = %rev result["sha256"] = %hash result["url"] = %url if subdir != "": result["subdir"] = %* subdir if tag != "": result["ref"] = %tag collectMetadata(result) proc containsPackageUri(lockAttrs: JsonNode; pkgUri: string): bool = for e in lockAttrs.items: if e["url"].getStr == pkgUri: return true proc containsPackage(lockAttrs: JsonNode; pkgName: string): bool = for e in lockAttrs.items: for other in e["packages"].items: if pkgName == other.getStr: return true proc collectRequires(pending: var Deque[PkgTuple]; pkgPath: string) = var nimbleFilePath = findNimbleFile(pkgPath, true) pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) for pair in pkg.requires: if pair.name != "nim" and pair.name != "compiler": pending.addLast(pair) var globalRegistry: JsonNode proc getPackgeUri(name: string): tuple[uri: string, meth: string] = if globalRegistry.isNil: let registryPath = registryCachePath() if fileExists(registryPath): globalRegistry = parseFile(registryPath) else: let client = newHttpClient() var raw = client.getContent(githubPackagesUrl) close(client) writeFile(registryPath, raw) globalRegistry = parseJson(raw) var name = name i = 0 while i < globalRegistry.len: var e = globalRegistry[i] if e["name"].getStr == name: if e.hasKey "alias": var alias = e["alias"].getStr doAssert alias != name name = alias i = 0 else: try: return (e["url"].getStr, e["method"].getStr,) except CatchableError: quit("Failed to parse shit JSON " & $e) inc i proc generateLockfile(): JsonNode = result = newJObject() var deps = newJArray() pending: Deque[PkgTuple] collectRequires(pending, getCurrentDir()) while pending.len > 0: let batchLen = pending.len for i in 1..batchLen: var pkgData: JsonNode let pkg = pending.popFirst() if pkg.name == "nim" or pkg.name == "compiler": continue var uri = parseUri(pkg.name) if uri.scheme == "": if not deps.containsPackage(pkg.name): pending.addLast(pkg) elif not deps.containsPackageUri(pkg.name): if uri.isWebGitForgeUrl: pkgData = prefetchGitForge(uri, pkg.ver) elif uri.isGitUrl: pkgData = prefetchGit(uri, pkg.ver) else: quit("unhandled URI " & $uri) collectRequires(pending, pkgData["path"].getStr) deps.add pkgData if batchLen == pending.len: var pkgData: JsonNode pkg = pending.popFirst() info = getPackgeUri(pkg.name) uri = parseUri info.uri if uri.isWebGitForgeUrl: pkgData = prefetchGitForge(uri, pkg.ver) else: case info.meth of "git": pkgData = prefetchGit(uri, pkg.ver) else: quit("unhandled fetch method " & $info.meth & " for " & info.uri) collectRequires(pending, pkgData["path"].getStr) deps.add pkgData sort(deps.elems) result["depends"] = deps proc main = var lockInfo = generateLockfile() stdout.writeLine lockInfo main()