diff --git a/Tupfile b/Tupfile new file mode 100644 index 0000000..67a1a21 --- /dev/null +++ b/Tupfile @@ -0,0 +1,2 @@ +include_rules +: |> !nim_lk |> diff --git a/lock.json b/lock.json new file mode 100644 index 0000000..4cbc05e --- /dev/null +++ b/lock.json @@ -0,0 +1 @@ +{"depends":[]} diff --git a/nim_lk.nimble b/nim_lk.nimble index 667b876..2476729 100644 --- a/nim_lk.nimble +++ b/nim_lk.nimble @@ -3,6 +3,6 @@ bin = @["nim_lk"] description = "Tool for generating Nim lockfiles" license = "BSD-3-Clause" srcDir = "src" -version = "20231004" +version = "20231008" requires "nim >= 2.0.0" diff --git a/src/nim_lk.nim b/src/nim_lk.nim index 9576cae..44602ac 100644 --- a/src/nim_lk.nim +++ b/src/nim_lk.nim @@ -1,11 +1,10 @@ import private/nix import nimblepkg/common, - nimblepkg/download, nimblepkg/packageinfo, - nimblepkg/options, nimblepkg/version, nimblepkg/packageparser, + nimblepkg/options, nimblepkg/cli import std/[algorithm, deques, httpclient, json, monotimes, os, osproc, parseutils, random, sequtils, streams, strutils, tables, times, uri] @@ -28,23 +27,44 @@ func isGitUrl(uri: Uri): bool = uri.scheme == "git" or uri.scheme.startsWith("git+") -proc gitLsRemote(url: string): seq[tuple[tag: string, rev: string]] = - var lines = execProcess( - "git", - args = ["ls-remote", "--tags", url], - options = {poUsePath}, - ) - result.setLen(lines.countLines.pred) - var off = 0 - for i in 0..result.high: - off.inc parseUntil(lines, result[i].rev, {'\x09'}, off) - off.inc skipWhiteSpace(lines, off) - off.inc skipUntil(lines, '/', off).succ - off.inc skipUntil(lines, '/', off).succ - off.inc parseUntil(lines, result[i].tag, {'\x0a'}, off).succ +proc isWebGitForgeUrl(uri: Uri): bool = + result = + case uri.hostname + of "github.com", "git.sr.ht", "codeberg.org": true + else: false + if not result: + stderr.writeLine "not a web forge hostname: '", uri.hostname, "'" + +proc startProcess(cmd: string; cmdArgs: varargs[string]): Process = + startProcess(cmd, args = cmdArgs, options = {poUsePath}) + +proc gitLsRemote(url: string; withTags: bool): seq[tuple[tag: string, rev: string]] = + var line, rev, refer: string + var process = + if withTags: + startProcess("git", "ls-remote", "--tags", url) + else: + startProcess("git", "ls-remote", url) + while process.running: + while process.outputStream.readLine(line): + var off = 0 + off.inc parseUntil(line, rev, Whitespace, off) + off.inc skipWhile(line, Whitespace, off) + refer = line[off..line.high] + const + refsTags = "refs/tags/" + headsTags = "refs/heads/" + if refer.startsWith(refsTags): + refer.removePrefix(refsTags) + elif refer.startsWith(headsTags): + refer.removePrefix(headsTags) + result.add((refer, rev,)) + stderr.write(process.errorStream.readAll) + close(process) proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string] = - let pairs = gitLsRemote(url) + let withTags = wanted.kind != verAny + let pairs = gitLsRemote(url, withTags) var resultVersion: Version for (tag, rev) in pairs: var tagVer = Version(tag) @@ -53,23 +73,25 @@ proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string result = (tag, rev) if result.rev == "" and pairs.len > 0: result = pairs[pairs.high] + doAssert result.rev != "" -proc collectMetadata(data: JsonNode; options: Options) = +proc collectMetadata(data: JsonNode) = let storePath = data["path"].getStr - var packageNames = newJArray() + var packageNames = newSeq[string]() for (kind, path) in walkDir(storePath): if kind in {pcFile, pcLinkToFile} and path.endsWith(".nimble"): var (dir, name, ext) = splitFile(path) - packageNames.add %name + packageNames.add name if packageNames.len == 0: quit("no .nimble files found in " & storePath) - data["packages"] = packageNames + sort(packageNames) + data["packages"] = %packageNames var nimbleFilePath = findNimbleFile(storePath, true) - pkg = readPackageInfo(nimbleFilePath, options) + pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) data["srcDir"] = %pkg.srcDir -proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode = +proc prefetchGit(uri: Uri; version: VersionRange): JsonNode = var uri = uri subdir = "" @@ -80,10 +102,8 @@ proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode = uri.query = "" let url = $uri let (tag, rev) = matchRev(url, version) - var args = @["--quiet", "--fetch-submodules", "--url", url] - if rev != "": - args.add "--rev" - args.add rev + var args = @["--quiet", "--fetch-submodules", "--url", url, "--rev", rev] + stderr.writeLine "prefetch ", url let dump = execProcess( "nix-prefetch-git", args = args, @@ -97,7 +117,50 @@ proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode = result["method"] = %"git" if tag != "": result["ref"] = %tag - collectMetadata(result, options) + collectMetadata(result) + +proc prefetchGitForge(uri: Uri; version: VersionRange): JsonNode = + result = newJObject() + var + uri = uri + subdir = "" + uri.scheme.removePrefix("git+") + if uri.query != "": + if uri.query.startsWith("subdir="): + subdir = uri.query[7 .. ^1] + uri.query = "" + var url = $uri + let (tag, rev) = matchRev(url, version) + + uri.scheme = "https" + uri.path.removeSuffix(".git") + uri.path.add("/archive/") + uri.path.add(rev) + uri.path.add(".tar.gz") + url = $uri + + stderr.writeLine "prefetch ", url + var lines = execProcess( + "nix-prefetch-url", + args = @[url, "--type", "sha256", "--print-path", "--unpack", "--name", "source"], + options = {poUsePath}) + var + hash, storePath: string + off: int + off.inc parseUntil(lines, hash, {'\n'}, off).succ + off.inc parseUntil(lines, storePath, {'\n'}, off).succ + doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines + + result["method"] = %"fetchurl" + result["path"] = %storePath + result["rev"] = %rev + result["sha256"] = %hash + result["url"] = %url + if subdir != "": + result["subdir"] = %* subdir + if tag != "": + result["ref"] = %tag + collectMetadata(result) proc containsPackageUri(lockAttrs: JsonNode; pkgUri: string): bool = for e in lockAttrs.items: @@ -110,10 +173,10 @@ proc containsPackage(lockAttrs: JsonNode; pkgName: string): bool = if pkgName == other.getStr: return true -proc collectRequires(pending: var Deque[PkgTuple]; options: Options; pkgPath: string) = +proc collectRequires(pending: var Deque[PkgTuple]; pkgPath: string) = var nimbleFilePath = findNimbleFile(pkgPath, true) - pkg = readPackageInfo(nimbleFilePath, options) + pkg = readPackageInfo(nimbleFilePath, parseCmdLine()) for pair in pkg.requires: if pair.name != "nim" and pair.name != "compiler": pending.addLast(pair) @@ -149,12 +212,12 @@ proc getPackgeUri(name: string): tuple[uri: string, meth: string] = quit("Failed to parse shit JSON " & $e) inc i -proc generateLockfile(options: Options): JsonNode = +proc generateLockfile(): JsonNode = result = newJObject() var deps = newJArray() pending: Deque[PkgTuple] - collectRequires(pending, options, getCurrentDir()) + collectRequires(pending, getCurrentDir()) while pending.len > 0: let batchLen = pending.len for i in 1..batchLen: @@ -167,34 +230,36 @@ proc generateLockfile(options: Options): JsonNode = if not deps.containsPackage(pkg.name): pending.addLast(pkg) elif not deps.containsPackageUri(pkg.name): - if uri.isGitUrl: - pkgData = prefetchGit(uri, pkg.ver, options) + if uri.isWebGitForgeUrl: + pkgData = prefetchGitForge(uri, pkg.ver) + elif uri.isGitUrl: + pkgData = prefetchGit(uri, pkg.ver) else: quit("unhandled URI " & $uri) - collectRequires(pending, options, pkgData["path"].getStr) + collectRequires(pending, pkgData["path"].getStr) deps.add pkgData if batchLen == pending.len: var + pkgData: JsonNode pkg = pending.popFirst() info = getPackgeUri(pkg.name) - case info.meth - of "git": - stderr.writeLine "prefetch ", info.uri - var pkgData = prefetchGit(parseUri info.uri, pkg.ver, options) - collectRequires(pending, options, pkgData["path"].getStr) - deps.add pkgData + uri = parseUri info.uri + if uri.isWebGitForgeUrl: + pkgData = prefetchGitForge(uri, pkg.ver) else: - quit("unhandled fetch method " & $info.meth & " for " & info.uri) + case info.meth + of "git": + pkgData = prefetchGit(uri, pkg.ver) + else: + quit("unhandled fetch method " & $info.meth & " for " & info.uri) + collectRequires(pending, pkgData["path"].getStr) + deps.add pkgData sort(deps.elems) result["depends"] = deps proc main = - var options = parseCmdLine() - # parse nimble options, not recommended - if options.action.typ != actionCustom: - options.action = Action(typ: actionCustom) - var lockInfo = generateLockfile(options) + var lockInfo = generateLockfile() stdout.writeLine lockInfo main()