ehmry/nim_lk
ehmry
/
nim_lk
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Use fetchurl for web forges

This commit is contained in:
Ehmry - 2023-10-08 14:30:29 +01:00
parent 2e2129e873
commit f978c53a29
4 changed files with 116 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Tupfile Normal file
View File

@ -0,0 +1,2 @@
include_rules
: |> !nim_lk |>

1
lock.json Normal file
View File

@ -0,0 +1 @@
{"depends":[]}

2
nim_lk.nimble
View File

@ -3,6 +3,6 @@ bin = @["nim_lk"]
description = "Tool for generating Nim lockfiles"
license = "BSD-3-Clause"
srcDir = "src"
version = "20231004"
version = "20231008"
requires "nim >= 2.0.0"

159
src/nim_lk.nim
View File

@ -1,11 +1,10 @@
import private/nix
import nimblepkg/common,
nimblepkg/download,
nimblepkg/packageinfo,
nimblepkg/options,
nimblepkg/version,
nimblepkg/packageparser,
nimblepkg/options,
nimblepkg/cli
import std/[algorithm, deques, httpclient, json, monotimes, os, osproc, parseutils, random, sequtils, streams, strutils, tables, times, uri]
@ -28,23 +27,44 @@ func isGitUrl(uri: Uri): bool =
uri.scheme == "git" or
uri.scheme.startsWith("git+")
proc gitLsRemote(url: string): seq[tuple[tag: string, rev: string]] =
var lines = execProcess(
"git",
args = ["ls-remote", "--tags", url],
options = {poUsePath},
)
result.setLen(lines.countLines.pred)
var off = 0
for i in 0..result.high:
off.inc parseUntil(lines, result[i].rev, {'\x09'}, off)
off.inc skipWhiteSpace(lines, off)
off.inc skipUntil(lines, '/', off).succ
off.inc skipUntil(lines, '/', off).succ
off.inc parseUntil(lines, result[i].tag, {'\x0a'}, off).succ
proc isWebGitForgeUrl(uri: Uri): bool =
result =
case uri.hostname
of "github.com", "git.sr.ht", "codeberg.org": true
else: false
if not result:
stderr.writeLine "not a web forge hostname: '", uri.hostname, "'"
proc startProcess(cmd: string; cmdArgs: varargs[string]): Process =
startProcess(cmd, args = cmdArgs, options = {poUsePath})
proc gitLsRemote(url: string; withTags: bool): seq[tuple[tag: string, rev: string]] =
var line, rev, refer: string
var process =
if withTags:
startProcess("git", "ls-remote", "--tags", url)
else:
startProcess("git", "ls-remote", url)
while process.running:
while process.outputStream.readLine(line):
var off = 0
off.inc parseUntil(line, rev, Whitespace, off)
off.inc skipWhile(line, Whitespace, off)
refer = line[off..line.high]
const
refsTags = "refs/tags/"
headsTags = "refs/heads/"
if refer.startsWith(refsTags):
refer.removePrefix(refsTags)
elif refer.startsWith(headsTags):
refer.removePrefix(headsTags)
result.add((refer, rev,))
stderr.write(process.errorStream.readAll)
close(process)
proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string] =
let pairs = gitLsRemote(url)
let withTags = wanted.kind != verAny
let pairs = gitLsRemote(url, withTags)
var resultVersion: Version
for (tag, rev) in pairs:
var tagVer = Version(tag)
@ -53,23 +73,25 @@ proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string
result = (tag, rev)
if result.rev == "" and pairs.len > 0:
result = pairs[pairs.high]
doAssert result.rev != ""
proc collectMetadata(data: JsonNode; options: Options) =
proc collectMetadata(data: JsonNode) =
let storePath = data["path"].getStr
var packageNames = newJArray()
var packageNames = newSeq[string]()
for (kind, path) in walkDir(storePath):
if kind in {pcFile, pcLinkToFile} and path.endsWith(".nimble"):
var (dir, name, ext) = splitFile(path)
packageNames.add %name
packageNames.add name
if packageNames.len == 0:
quit("no .nimble files found in " & storePath)
data["packages"] = packageNames
sort(packageNames)
data["packages"] = %packageNames
var
nimbleFilePath = findNimbleFile(storePath, true)
pkg = readPackageInfo(nimbleFilePath, options)
pkg = readPackageInfo(nimbleFilePath, parseCmdLine())
data["srcDir"] = %pkg.srcDir
proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode =
proc prefetchGit(uri: Uri; version: VersionRange): JsonNode =
var
uri = uri
subdir = ""
@ -80,10 +102,8 @@ proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode =
uri.query = ""
let url = $uri
let (tag, rev) = matchRev(url, version)
var args = @["--quiet", "--fetch-submodules", "--url", url]
if rev != "":
args.add "--rev"
args.add rev
var args = @["--quiet", "--fetch-submodules", "--url", url, "--rev", rev]
stderr.writeLine "prefetch ", url
let dump = execProcess(
"nix-prefetch-git",
args = args,
@ -97,7 +117,50 @@ proc prefetchGit(uri: Uri; version: VersionRange; options: Options): JsonNode =
result["method"] = %"git"
if tag != "":
result["ref"] = %tag
collectMetadata(result, options)
collectMetadata(result)
proc prefetchGitForge(uri: Uri; version: VersionRange): JsonNode =
result = newJObject()
var
uri = uri
subdir = ""
uri.scheme.removePrefix("git+")
if uri.query != "":
if uri.query.startsWith("subdir="):
subdir = uri.query[7 .. ^1]
uri.query = ""
var url = $uri
let (tag, rev) = matchRev(url, version)
uri.scheme = "https"
uri.path.removeSuffix(".git")
uri.path.add("/archive/")
uri.path.add(rev)
uri.path.add(".tar.gz")
url = $uri
stderr.writeLine "prefetch ", url
var lines = execProcess(
"nix-prefetch-url",
args = @[url, "--type", "sha256", "--print-path", "--unpack", "--name", "source"],
options = {poUsePath})
var
hash, storePath: string
off: int
off.inc parseUntil(lines, hash, {'\n'}, off).succ
off.inc parseUntil(lines, storePath, {'\n'}, off).succ
doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines
result["method"] = %"fetchurl"
result["path"] = %storePath
result["rev"] = %rev
result["sha256"] = %hash
result["url"] = %url
if subdir != "":
result["subdir"] = %* subdir
if tag != "":
result["ref"] = %tag
collectMetadata(result)
proc containsPackageUri(lockAttrs: JsonNode; pkgUri: string): bool =
for e in lockAttrs.items:
@ -110,10 +173,10 @@ proc containsPackage(lockAttrs: JsonNode; pkgName: string): bool =
if pkgName == other.getStr:
return true
proc collectRequires(pending: var Deque[PkgTuple]; options: Options; pkgPath: string) =
proc collectRequires(pending: var Deque[PkgTuple]; pkgPath: string) =
var
nimbleFilePath = findNimbleFile(pkgPath, true)
pkg = readPackageInfo(nimbleFilePath, options)
pkg = readPackageInfo(nimbleFilePath, parseCmdLine())
for pair in pkg.requires:
if pair.name != "nim" and pair.name != "compiler":
pending.addLast(pair)
@ -149,12 +212,12 @@ proc getPackgeUri(name: string): tuple[uri: string, meth: string] =
quit("Failed to parse shit JSON " & $e)
inc i
proc generateLockfile(options: Options): JsonNode =
proc generateLockfile(): JsonNode =
result = newJObject()
var
deps = newJArray()
pending: Deque[PkgTuple]
collectRequires(pending, options, getCurrentDir())
collectRequires(pending, getCurrentDir())
while pending.len > 0:
let batchLen = pending.len
for i in 1..batchLen:
@ -167,34 +230,36 @@ proc generateLockfile(options: Options): JsonNode =
if not deps.containsPackage(pkg.name):
pending.addLast(pkg)
elif not deps.containsPackageUri(pkg.name):
if uri.isGitUrl:
pkgData = prefetchGit(uri, pkg.ver, options)
if uri.isWebGitForgeUrl:
pkgData = prefetchGitForge(uri, pkg.ver)
elif uri.isGitUrl:
pkgData = prefetchGit(uri, pkg.ver)
else:
quit("unhandled URI " & $uri)
collectRequires(pending, options, pkgData["path"].getStr)
collectRequires(pending, pkgData["path"].getStr)
deps.add pkgData
if batchLen == pending.len:
var
pkgData: JsonNode
pkg = pending.popFirst()
info = getPackgeUri(pkg.name)
case info.meth
of "git":
stderr.writeLine "prefetch ", info.uri
var pkgData = prefetchGit(parseUri info.uri, pkg.ver, options)
collectRequires(pending, options, pkgData["path"].getStr)
deps.add pkgData
uri = parseUri info.uri
if uri.isWebGitForgeUrl:
pkgData = prefetchGitForge(uri, pkg.ver)
else:
quit("unhandled fetch method " & $info.meth & " for " & info.uri)
case info.meth
of "git":
pkgData = prefetchGit(uri, pkg.ver)
else:
quit("unhandled fetch method " & $info.meth & " for " & info.uri)
collectRequires(pending, pkgData["path"].getStr)
deps.add pkgData
sort(deps.elems)
result["depends"] = deps
proc main =
var options = parseCmdLine()
# parse nimble options, not recommended
if options.action.typ != actionCustom:
options.action = Action(typ: actionCustom)
var lockInfo = generateLockfile(options)
var lockInfo = generateLockfile()
stdout.writeLine lockInfo
main()