Opportunistic ZIP fetching

src/nim_lk.nim

@@ -22,15 +22,11 @@ proc registryCachePath: string =
       result = home / ".cache"
   result.add "/packages.json"
 
-func isGitUrl(uri: Uri): bool =
+proc isGitUrl(uri: Uri): bool =
-  uri.path.endsWith(".git") or
+  uri.scheme == "git" or
-    uri.scheme == "git" or
+    uri.scheme.startsWith("git+") or
-    uri.scheme.startsWith("git+")
+    uri.path.endsWith(".git") or
-
+    uri.hostname.contains("git")
-proc isWebGitForgeUrl(uri: Uri): bool =
-  case uri.hostname
-  of "github.com", "git.sr.ht", "codeberg.org": true
-  else: false
 
 proc startProcess(cmd: string; cmdArgs: varargs[string]): Process =
   # stderr.writeLine(cmd, " ", join(cmdArgs, " "))
@@ -107,64 +103,54 @@ proc prefetchGit(uri: Uri; version: VersionRange): JsonNode =
     if uri.query.startsWith("subdir="):
       subdir = uri.query[7 .. ^1]
     uri.query = ""
-  let url = $uri
+  let cloneUrl = $uri
-  let (tag, rev) = matchRev(url, version)
+  let (tag, rev) = matchRev(cloneUrl, version)
-  var args = @["--quiet", "--fetch-submodules", "--url", url, "--rev", rev]
+  var archiveUri = uri
-  stderr.writeLine "prefetch ", url
+  archiveUri.scheme = "https"
-  let dump = execProcess(
+  archiveUri.path.removeSuffix ".git"
-    "nix-prefetch-git",
+  archiveUri.path.add "/archive/"
-    args = args,
+  archiveUri.path.add rev
-    options = {poUsePath})
+  archiveUri.path.add ".tar.gz"
-  try: result = parseJson dump
+  let client = newHttpClient()
-  except JsonParsingError:
+  defer: close(client)
-    stderr.writeLine "failed to parse output of nix-prefetch-git ", join(args, " ")
+  let
-    quit(dump)
+    archiveUrl = $archiveUri
-  if subdir != "":
+    resp = head(client, archiveUrl)
-    result["subdir"] = %* subdir
+  if resp.code in {Http200, Http302}:
-  result["method"] = %"git"
+    stderr.writeLine "prefetch ", archiveUrl
-  if tag != "":
+    var lines = execProcess(
-    result["ref"] = %tag
+      "nix-prefetch-url",
-  collectMetadata(result)
+      args = @[archiveUrl, "--type", "sha256", "--print-path", "--unpack", "--name", "source"],
-
+      options = {poUsePath})
-proc prefetchGitForge(uri: Uri; version: VersionRange): JsonNode =
+    var
-  result = newJObject()
+      hash, storePath: string
-  var
+      off: int
-    uri = uri
+    off.inc parseUntil(lines, hash, {'\n'}, off).succ
-    subdir = ""
+    off.inc parseUntil(lines, storePath, {'\n'}, off).succ
-  uri.scheme.removePrefix("git+")
+    doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines
-  if uri.query != "":
+    result = newJObject()
-    if uri.query.startsWith("subdir="):
+    result["method"] = %"fetchzip"
-      subdir = uri.query[7 .. ^1]
+    result["path"] = %storePath
-    uri.query = ""
+    result["rev"] = %rev
-  var url = $uri
+    result["sha256"] = %hash
-  let (tag, rev) = matchRev(url, version)
+    result["url"] = %archiveUrl
-
+    if subdir != "":
-  uri.scheme = "https"
+      result["subdir"] = %* subdir
-  uri.path.removeSuffix(".git")
+  else:
-  uri.path.add("/archive/")
+    stderr.writeLine "fetch of ", archiveUrl, " returned ", resp.code
-  uri.path.add(rev)
+    var args = @["--quiet", "--fetch-submodules", "--url", cloneUrl, "--rev", rev]
-  uri.path.add(".tar.gz")
+    stderr.writeLine "prefetch ", cloneUrl
-  url = $uri
+    let dump = execProcess(
-
+      "nix-prefetch-git",
-  stderr.writeLine "prefetch ", url
+      args = args,
-  var lines = execProcess(
+      options = {poUsePath})
-    "nix-prefetch-url",
+    try: result = parseJson dump
-    args = @[url, "--type", "sha256", "--print-path", "--unpack", "--name", "source"],
+    except JsonParsingError:
-    options = {poUsePath})
+      stderr.writeLine "failed to parse output of nix-prefetch-git ", join(args, " ")
-  var
+      quit(dump)
-    hash, storePath: string
+    if subdir != "":
-    off: int
+      result["subdir"] = %* subdir
-  off.inc parseUntil(lines, hash, {'\n'}, off).succ
+    result["method"] = %"git"
-  off.inc parseUntil(lines, storePath, {'\n'}, off).succ
-  doAssert off == lines.len, "unrecognized nix-prefetch-url output:\n" & lines
-
-  result["method"] = %"fetchzip"
-  result["path"] = %storePath
-  result["rev"] = %rev
-  result["sha256"] = %hash
-  result["url"] = %url
-  if subdir != "":
-    result["subdir"] = %* subdir
   if tag != "":
     result["ref"] = %tag
   collectMetadata(result)
@@ -237,9 +223,7 @@ proc generateLockfile(): JsonNode =
         if not deps.containsPackage(pkg.name):
           pending.addLast(pkg)
       elif not deps.containsPackageUri(pkg.name):
-        if uri.isWebGitForgeUrl:
+        if uri.isGitUrl:
-          pkgData = prefetchGitForge(uri, pkg.ver)
-        elif uri.isGitUrl:
           pkgData = prefetchGit(uri, pkg.ver)
         else:
           quit("unhandled URI " & $uri)
@@ -252,14 +236,11 @@ proc generateLockfile(): JsonNode =
         pkg = pending.popFirst()
         info = getPackgeUri(pkg.name)
         uri = parseUri info.uri
-      if uri.isWebGitForgeUrl:
+      case info.meth
-        pkgData = prefetchGitForge(uri, pkg.ver)
+      of "git":
+        pkgData = prefetchGit(uri, pkg.ver)
       else:
-        case info.meth
+        quit("unhandled fetch method " & $info.meth & " for " & info.uri)
-        of "git":
-          pkgData = prefetchGit(uri, pkg.ver)
-        else:
-          quit("unhandled fetch method " & $info.meth & " for " & info.uri)
       collectRequires(pending, pkgData["path"].getStr)
       deps.add pkgData
   sort(deps.elems)