From 092252a727aa07cbfbab08b390692f9ad7d619bf Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sat, 30 Sep 2023 09:47:15 +0100 Subject: [PATCH] Initial commit --- .envrc | 2 + README.md | 1 + Tuprules.tup | 1 + nim_lk.nimble | 8 + shell.nix | 1 + src/Tupfile | 2 + src/nim_lk.nim | 195 +++++ src/nim_lk.nim.cfg | 3 + src/nimblepkg/checksums.nim | 72 ++ src/nimblepkg/cli.nim | 293 +++++++ src/nimblepkg/common.nim | 78 ++ src/nimblepkg/config.nim | 124 +++ src/nimblepkg/deps.nim | 52 ++ src/nimblepkg/developfile.nim | 953 ++++++++++++++++++++++ src/nimblepkg/displaymessages.nim | 172 ++++ src/nimblepkg/download.nim | 324 ++++++++ src/nimblepkg/init.nim | 184 +++++ src/nimblepkg/jsonhelpers.nim | 140 ++++ src/nimblepkg/lockfile.nim | 57 ++ src/nimblepkg/nim.cfg | 2 + src/nimblepkg/nimbledatafile.nim | 66 ++ src/nimblepkg/nimscriptapi.nim | 204 +++++ src/nimblepkg/nimscriptexecutor.nim | 76 ++ src/nimblepkg/nimscriptwrapper.nim | 216 +++++ src/nimblepkg/options.nim | 534 ++++++++++++ src/nimblepkg/packageinfo.nim | 577 +++++++++++++ src/nimblepkg/packageinstaller.nim | 112 +++ src/nimblepkg/packagemetadatafile.nim | 73 ++ src/nimblepkg/packageparser.nim | 511 ++++++++++++ src/nimblepkg/paths.nim | 45 ++ src/nimblepkg/publish.nim | 229 ++++++ src/nimblepkg/reversedeps.nim | 135 ++++ src/nimblepkg/syncfile.nim | 109 +++ src/nimblepkg/tools.nim | 181 +++++ src/nimblepkg/topologicalsort.nim | 169 ++++ src/nimblepkg/vcstools.nim | 1070 +++++++++++++++++++++++++ src/nimblepkg/version.nim | 360 +++++++++ src/private/nix.nim | 120 +++ 38 files changed, 7451 insertions(+) create mode 100644 .envrc create mode 100644 README.md create mode 100644 Tuprules.tup create mode 100644 nim_lk.nimble create mode 100644 shell.nix create mode 100644 src/Tupfile create mode 100644 src/nim_lk.nim create mode 100644 src/nim_lk.nim.cfg create mode 100644 src/nimblepkg/checksums.nim create mode 100644 src/nimblepkg/cli.nim create mode 100644 src/nimblepkg/common.nim create mode 100644 src/nimblepkg/config.nim create mode 100644 src/nimblepkg/deps.nim create mode 100644 src/nimblepkg/developfile.nim create mode 100644 src/nimblepkg/displaymessages.nim create mode 100644 src/nimblepkg/download.nim create mode 100644 src/nimblepkg/init.nim create mode 100644 src/nimblepkg/jsonhelpers.nim create mode 100644 src/nimblepkg/lockfile.nim create mode 100644 src/nimblepkg/nim.cfg create mode 100644 src/nimblepkg/nimbledatafile.nim create mode 100644 src/nimblepkg/nimscriptapi.nim create mode 100644 src/nimblepkg/nimscriptexecutor.nim create mode 100644 src/nimblepkg/nimscriptwrapper.nim create mode 100644 src/nimblepkg/options.nim create mode 100644 src/nimblepkg/packageinfo.nim create mode 100644 src/nimblepkg/packageinstaller.nim create mode 100644 src/nimblepkg/packagemetadatafile.nim create mode 100644 src/nimblepkg/packageparser.nim create mode 100644 src/nimblepkg/paths.nim create mode 100644 src/nimblepkg/publish.nim create mode 100644 src/nimblepkg/reversedeps.nim create mode 100644 src/nimblepkg/syncfile.nim create mode 100644 src/nimblepkg/tools.nim create mode 100644 src/nimblepkg/topologicalsort.nim create mode 100644 src/nimblepkg/vcstools.nim create mode 100644 src/nimblepkg/version.nim create mode 100644 src/private/nix.nim diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..d324c24 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +source_env .. +use nix diff --git a/README.md b/README.md new file mode 100644 index 0000000..4ae46b9 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +Nim lockfile generator diff --git a/Tuprules.tup b/Tuprules.tup new file mode 100644 index 0000000..89e7278 --- /dev/null +++ b/Tuprules.tup @@ -0,0 +1 @@ +NIM_FLAGS += --path:$(TUP_CWD)/../nim/v1.6.8 diff --git a/nim_lk.nimble b/nim_lk.nimble new file mode 100644 index 0000000..198f94b --- /dev/null +++ b/nim_lk.nimble @@ -0,0 +1,8 @@ +author = "Emery Hemingway" +bin = @["nim_lk"] +description = "Tool for generating Nim lockfiles" +license = "BSD-3-Clause" +srcDir = "src" +version = "20231001" + +requires "nim >= 2.0.0" diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..bdd559d --- /dev/null +++ b/shell.nix @@ -0,0 +1 @@ +let pkgs = import { }; in pkgs.nitter diff --git a/src/Tupfile b/src/Tupfile new file mode 100644 index 0000000..beda37a --- /dev/null +++ b/src/Tupfile @@ -0,0 +1,2 @@ +include_rules +: nim_lk.nim |> !nim_bin |> diff --git a/src/nim_lk.nim b/src/nim_lk.nim new file mode 100644 index 0000000..a24fd88 --- /dev/null +++ b/src/nim_lk.nim @@ -0,0 +1,195 @@ +import private/nix + +import nimblepkg/common, + nimblepkg/download, + nimblepkg/packageinfo, + nimblepkg/options, + nimblepkg/version, + nimblepkg/packageparser, + nimblepkg/cli + +import std/[algorithm, deques, httpclient, json, monotimes, os, osproc, parseutils, random, sequtils, streams, strutils, tables, times, uri] + +const githubPackagesUrl = + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json" + +proc registryCachePath: string = + result = getEnv("XDG_CACHE_HOME") + if result == "": + let home = getEnv("HOME") + if home == "": + result = getEnv("TMPDIR", "/tmp") + else: + result = home / ".cache" + result.add "/packages.json" + +func isGitUrl(uri: Uri): bool = + uri.path.endsWith(".git") or + uri.scheme == "git" or + uri.scheme.startsWith("git+") + +proc gitLsRemote(url: string): seq[tuple[tag: string, rev: string]] = + var lines = execProcess( + "git", + args = ["ls-remote", "--tags", url], + options = {poUsePath}, + ) + result.setLen(lines.countLines.pred) + var off = 0 + for i in 0..result.high: + off.inc parseUntil(lines, result[i].rev, {'\x09'}, off) + off.inc skipWhiteSpace(lines, off) + off.inc skipUntil(lines, '/', off).succ + off.inc skipUntil(lines, '/', off).succ + off.inc parseUntil(lines, result[i].tag, {'\x0a'}, off).succ + +proc matchRev(url: string; wanted: VersionRange): tuple[tag: string, rev: string] = + let pairs = gitLsRemote(url) + var resultVersion: Version + for (tag, rev) in pairs: + var tagVer = Version(tag) + if tagVer.withinRange(wanted) and resultVersion < tagVer: + resultVersion = tagVer + result = (tag, rev) + if result.rev == "" and pairs.len > 0: + result = pairs[pairs.high] + +proc collectMetadata(data: JsonNode) = + let storePath = data["path"].getStr + var packageNames = newJArray() + for (kind, path) in walkDir(storePath): + if kind in {pcFile, pcLinkToFile} and path.endsWith(".nimble"): + var (dir, name, ext) = splitFile(path) + packageNames.add %name + if packageNames.len == 0: + quit("no .nimble files found in " & storePath) + data["packages"] = packageNames + +proc prefechtGit(uri: Uri; version: VersionRange): JsonNode = + var + uri = uri + subdir = "" + uri.scheme.removePrefix("git+") + if uri.query != "": + if uri.query.startsWith("subdir="): + subdir = uri.query[7 .. ^1] + uri.query = "" + let url = $uri + let (tag, rev) = matchRev(url, version) + var args = @["--quiet", "--fetch-submodules", "--url", url] + if rev != "": + args.add "--rev" + args.add rev + let dump = execProcess( + "nix-prefetch-git", + args = args, + options = {poUsePath}) + try: result = parseJson dump + except JsonParsingError: + stderr.writeLine "failed to parse output of nix-prefetch-git ", join(args, " ") + quit(dump) + if subdir != "": + result["subdir"] = %* subdir + result["method"] = %"git" + if tag != "": + result["ref"] = %tag + collectMetadata(result) + +proc containsPackageUri(lockAttrs: JsonNode; pkgUri: string): bool = + for e in lockAttrs.items: + if e["url"].getStr == pkgUri: + return true + +proc containsPackage(lockAttrs: JsonNode; pkgName: string): bool = + for e in lockAttrs.items: + for other in e["packages"].items: + if pkgName == other.getStr: + return true + +proc collectRequires(pending: var Deque[PkgTuple]; options: Options; pkgPath: string) = + var + nimbleFilePath = findNimbleFile(pkgPath, true) + pkg = readPackageInfo(nimbleFilePath, options) + for pair in pkg.requires: + if pair.name != "nim" and pair.name != "compiler": + pending.addLast(pair) + +var globalRegistry: JsonNode + +proc getPackgeUri(name: string): tuple[uri: string, meth: string] = + if globalRegistry.isNil: + let registryPath = registryCachePath() + if fileExists(registryPath): + globalRegistry = parseFile(registryPath) + else: + let client = newHttpClient() + var raw = client.getContent(githubPackagesUrl) + close(client) + writeFile(registryPath, raw) + globalRegistry = parseJson(raw) + var + name = name + i = 0 + while i < globalRegistry.len: + var e = globalRegistry[i] + if e["name"].getStr == name: + if e.hasKey "alias": + var alias = e["alias"].getStr + doAssert alias != name + name = alias + i = 0 + else: + try: + return (e["url"].getStr, e["method"].getStr,) + except CatchableError: + quit("Failed to parse shit JSON " & $e) + inc i + +proc generateLockfile(options: Options): JsonNode = + result = newJObject() + var + deps = newJArray() + pending: Deque[PkgTuple] + collectRequires(pending, options, getCurrentDir()) + while pending.len > 0: + let batchLen = pending.len + for i in 1..batchLen: + var pkgData: JsonNode + let pkg = pending.popFirst() + if pkg.name == "nim" or pkg.name == "compiler": + continue + var uri = parseUri(pkg.name) + if uri.scheme == "" and not deps.containsPackage(pkg.name): + pending.addLast(pkg) + elif not deps.containsPackageUri(pkg.name): + if uri.isGitUrl: + pkgData = prefechtGit(uri, pkg.ver) + else: + quit("unhandled URI " & $uri) + collectRequires(pending, options, pkgData["path"].getStr) + deps.add pkgData + + if batchLen == pending.len: + var + pkg = pending.popFirst() + info = getPackgeUri(pkg.name) + case info.meth + of "git": + stderr.writeLine "prefetch ", info.uri + var pkgData = prefechtGit(parseUri info.uri, pkg.ver) + collectRequires(pending, options, pkgData["path"].getStr) + deps.add pkgData + else: + quit("unhandled fetch method " & $info.meth & " for " & info.uri) + sort(deps.elems) + result["depends"] = deps + +proc main = + var options = parseCmdLine() + # parse nimble options, not recommended + if options.action.typ != actionCustom: + options.action = Action(typ: actionCustom) + var lockInfo = generateLockfile(options) + stdout.writeLine lockInfo + +main() diff --git a/src/nim_lk.nim.cfg b/src/nim_lk.nim.cfg new file mode 100644 index 0000000..24360a5 --- /dev/null +++ b/src/nim_lk.nim.cfg @@ -0,0 +1,3 @@ +define:nixbuild +define:ssl +threads:off diff --git a/src/nimblepkg/checksums.nim b/src/nimblepkg/checksums.nim new file mode 100644 index 0000000..5a43452 --- /dev/null +++ b/src/nimblepkg/checksums.nim @@ -0,0 +1,72 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import os, std/sha1, strformat, algorithm +import common, version, sha1hashes, vcstools, paths, cli + +type + ChecksumError* = object of NimbleError + +proc checksumError*(name: string, version: Version, + vcsRevision, checksum, expectedChecksum: Sha1Hash): + ref ChecksumError = + result = newNimbleError[ChecksumError](&""" +Downloaded package checksum does not correspond to that in the lock file: + Package: {name}@v.{version}@r.{vcsRevision} + Checksum: {checksum} + Expected checksum: {expectedChecksum} +""") + +proc updateSha1Checksum(checksum: var Sha1State, fileName, filePath: string) = + if not filePath.fileExists: + # In some cases a file name returned by `git ls-files` or `hg manifest` + # could be an empty directory name and if so trying to open it will result + # in a crash. This happens for example in the case of a git sub module + # directory from which no files are being installed. + return + checksum.update(fileName) + if symlinkExists(filePath): + # Check whether a file is a symbolic link and if so update the checksum with + # the path to the file that the link points to. + var linkPath: string + try: + linkPath = expandSymlink(filePath) + except OSError: + displayWarning(&"Cannot expand symbolic link \"{filePath}\".\n" & + "Skipping it in the calculation of the checksum.") + return + checksum.update(linkPath) + else: + # Otherwise this is an ordinary file and we are adding its content to the + # checksum. + var file: File + try: + file = filePath.open(fmRead) + except IOError: + ## If the file cannot be open for reading do not count its content in the + ## checksum. + displayWarning(&"The file \"{filePath}\" cannot be open for reading.\n" & + "Skipping it in the calculation of the checksum.") + return + defer: close(file) + const bufferSize = 8192 + var buffer = newString(bufferSize) + while true: + var bytesRead = readChars(file, buffer) + if bytesRead == 0: break + checksum.update(buffer.toOpenArray(0, bytesRead - 1)) + +proc calculateDirSha1Checksum*(dir: string): Sha1Hash = + ## Recursively calculates the sha1 checksum of the contents of the directory + ## `dir` and its subdirectories. + ## + ## Raises a `NimbleError` if: + ## - the external command for getting the package file list fails. + ## - the directory does not exist. + + var packageFiles = getPackageFileList(dir.Path) + packageFiles.sort + var checksum = newSha1State() + for file in packageFiles: + updateSha1Checksum(checksum, file, dir / file) + result = initSha1Hash($SecureHash(checksum.finalize())) diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim new file mode 100644 index 0000000..3afda35 --- /dev/null +++ b/src/nimblepkg/cli.nim @@ -0,0 +1,293 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +# +# Rough rules/philosophy for the messages that Nimble displays are the following: +# - Green is only shown when the requested operation is successful. +# - Blue can be used to emphasise certain keywords, for example actions such +# as "Downloading" or "Reading". +# - Red is used when the requested operation fails with an error. +# - Yellow is used for warnings. +# +# - Dim for LowPriority. +# - Bright for HighPriority. +# - Normal for MediumPriority. + +import terminal, sets, strutils +import version + +when not declared(initHashSet): + import common + +type + CLI* = ref object + level: Priority + warnings: HashSet[(string, string)] + suppressionCount: int ## Amount of messages which were not shown. + showColor: bool ## Whether messages should be colored. + suppressMessages: bool ## Whether Warning, Message and Success messages + ## should be suppressed, useful for + ## commands like `dump` whose output should be + ## machine readable. + + Priority* = enum + DebugPriority, LowPriority, MediumPriority, HighPriority + + DisplayType* = enum + Error, Warning, Message, Success + + ForcePrompt* = enum + dontForcePrompt, forcePromptYes, forcePromptNo + +const + longestCategory = len("Downloading") + foregrounds: array[Error .. Success, ForegroundColor] = + [fgRed, fgYellow, fgCyan, fgGreen] + styles: array[DebugPriority .. HighPriority, set[Style]] = + [{styleDim}, {styleDim}, {}, {styleBright}] + + +proc newCLI(): CLI = + result = CLI( + level: HighPriority, + warnings: initHashSet[(string, string)](), + suppressionCount: 0, + showColor: true, + suppressMessages: false + ) + +var globalCLI = newCLI() + + +proc calculateCategoryOffset(category: string): int = + assert category.len <= longestCategory + return longestCategory - category.len + +proc isSuppressed(displayType: DisplayType): bool = + # Don't print any Warning, Message or Success messages when suppression of + # warnings is enabled. That is, unless the user asked for --verbose output. + if globalCLI.suppressMessages and displayType >= Warning and + globalCLI.level == HighPriority: + return true + +proc displayCategory(category: string, displayType: DisplayType, + priority: Priority) = + if isSuppressed(displayType): + return + + # Calculate how much the `category` must be offset to align along a center + # line. + let offset = calculateCategoryOffset(category) + + # Display the category. + let text = "$1$2 " % [spaces(offset), category] + if globalCLI.showColor: + if priority != DebugPriority: + setForegroundColor(stdout, foregrounds[displayType]) + writeStyled(text, styles[priority]) + resetAttributes() + else: + stdout.write(text) + +proc displayLine(category, line: string, displayType: DisplayType, + priority: Priority) = + if isSuppressed(displayType): + return + + displayCategory(category, displayType, priority) + + # Display the message. + echo(line) + +proc display*(category, msg: string, displayType = Message, + priority = MediumPriority) = + # Multiple warnings containing the same messages should not be shown. + let warningPair = (category, msg) + if displayType == Warning: + if warningPair in globalCLI.warnings: + return + else: + globalCLI.warnings.incl(warningPair) + + # Suppress this message if its priority isn't high enough. + # TODO: Per-priority suppression counts? + if priority < globalCLI.level: + if priority != DebugPriority: + globalCLI.suppressionCount.inc + return + + # Display each line in the message. + var i = 0 + for line in msg.splitLines(): + if len(line) == 0: continue + displayLine(if i == 0: category else: "...", line, displayType, priority) + i.inc + +proc displayDebug*(category, msg: string) = + ## Convenience for displaying debug messages. + display(category, msg, priority = DebugPriority) + +proc displayDebug*(msg: string) = + ## Convenience for displaying debug messages with a default category. + displayDebug("Debug:", msg) + +proc displayTip*() = + ## Called just before Nimble exits. Shows some tips for the user, for example + ## the amount of messages that were suppressed and how to show them. + if globalCLI.suppressionCount > 0: + let msg = "$1 messages have been suppressed, use --verbose to show them." % + $globalCLI.suppressionCount + display("Tip:", msg, Warning, HighPriority) + +proc prompt*(forcePrompts: ForcePrompt, question: string): bool = + case forcePrompts + of forcePromptYes: + display("Prompt:", question & " -> [forced yes]", Warning, HighPriority) + return true + of forcePromptNo: + display("Prompt:", question & " -> [forced no]", Warning, HighPriority) + return false + of dontForcePrompt: + displayLine("Prompt:", question & " [y/N]", Warning, HighPriority) + displayCategory("Answer:", Warning, HighPriority) + let yn = stdin.readLine() + case yn.normalize + of "y", "yes": + return true + of "n", "no": + return false + else: + return false + +proc promptCustom*(forcePrompts: ForcePrompt, question, default: string): string = + case forcePrompts: + of forcePromptYes: + display("Prompt:", question & " -> [forced " & default & "]", Warning, + HighPriority) + return default + else: + if default == "": + display("Prompt:", question, Warning, HighPriority) + displayCategory("Answer:", Warning, HighPriority) + let user = stdin.readLine() + if user.len == 0: return promptCustom(forcePrompts, question, default) + else: return user + else: + display("Prompt:", question & " [" & default & "]", Warning, HighPriority) + displayCategory("Answer:", Warning, HighPriority) + let user = stdin.readLine() + if user == "": return default + else: return user + +proc promptCustom*(question, default: string): string = + return promptCustom(dontForcePrompt, question, default) + +proc promptListInteractive(question: string, args: openarray[string]): string = + display("Prompt:", question, Warning, HighPriority) + display("Select", "Cycle with 'Tab', 'Enter' when done", Message, + HighPriority) + displayCategory("Choices:", Warning, HighPriority) + var + current = 0 + selected = false + # Incase the cursor is at the bottom of the terminal + for arg in args: + stdout.write "\n" + # Reset the cursor to the start of the selection prompt + cursorUp(stdout, args.len) + cursorForward(stdout, longestCategory) + hideCursor(stdout) + + # The selection loop + while not selected: + setForegroundColor(fgDefault) + # Loop through the options + for i, arg in args: + # Check if the option is the current + if i == current: + writeStyled("> " & arg & " <", {styleBright}) + else: + writeStyled(" " & arg & " ", {styleDim}) + # Move the cursor back to the start + for s in 0..<(arg.len + 4): + cursorBackward(stdout) + # Move down for the next item + cursorDown(stdout) + # Move the cursor back up to the start of the selection prompt + for i in 0..<(args.len()): + cursorUp(stdout) + resetAttributes(stdout) + + # Begin key input + while true: + case getch(): + of '\t': + current = (current + 1) mod args.len + break + of '\r': + selected = true + break + of '\3': + showCursor(stdout) + raise newException(NimbleError, "Keyboard interrupt") + else: discard + + # Erase all lines of the selection + for i in 0.. [forced " & result & "]", Warning, + HighPriority) + else: + if isatty(stdout): + return promptListInteractive(question, args) + else: + return promptListFallback(question, args) + +proc setVerbosity*(level: Priority) = + globalCLI.level = level + +proc setShowColor*(val: bool) = + globalCLI.showColor = val + +proc setSuppressMessages*(val: bool) = + globalCLI.suppressMessages = val + +when isMainModule: + display("Reading", "config file at /Users/dom/.config/nimble/nimble.ini", + priority = LowPriority) + + display("Reading", "official package list", + priority = LowPriority) + + display("Downloading", "daemonize v0.0.2 using Git", + priority = HighPriority) + + display("Warning", "dashes in package names will be deprecated", Warning, + priority = HighPriority) + + display("Error", """Unable to read package info for /Users/dom/.nimble/pkgs/nimble-0.7.11 +Reading as ini file failed with: + Invalid section: . +Evaluating as NimScript file failed with: + Users/dom/.nimble/pkgs/nimble-0.7.11/nimble.nimble(3, 23) Error: cannot open 'src/nimblepkg/common'. +""", Error, HighPriority) diff --git a/src/nimblepkg/common.nim b/src/nimblepkg/common.nim new file mode 100644 index 0000000..1f5fb40 --- /dev/null +++ b/src/nimblepkg/common.nim @@ -0,0 +1,78 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +# +# Various miscellaneous common types reside here, to avoid problems with +# recursive imports + +when not defined(nimscript): + import sets + + import version + + type + BuildFailed* = object of NimbleError + + PackageInfo* = object + myPath*: string ## The path of this .nimble file + isNimScript*: bool ## Determines if this pkg info was read from a nims file + isMinimal*: bool + isInstalled*: bool ## Determines if the pkg this info belongs to is installed + isLinked*: bool ## Determines if the pkg this info belongs to has been linked via `develop` + postHooks*: HashSet[string] ## Useful to know so that Nimble doesn't execHook unnecessarily + preHooks*: HashSet[string] + name*: string + ## The version specified in the .nimble file.Assuming info is non-minimal, + ## it will always be a non-special version such as '0.1.4'. + ## If in doubt, use `getConcreteVersion` instead. + version*: string + specialVersion*: string ## Either `myVersion` or a special version such as #head. + author*: string + description*: string + license*: string + skipDirs*: seq[string] + skipFiles*: seq[string] + skipExt*: seq[string] + installDirs*: seq[string] + installFiles*: seq[string] + installExt*: seq[string] + requires*: seq[PkgTuple] + bin*: seq[string] + binDir*: string + srcDir*: string + backend*: string + foreignDeps*: seq[string] + + ## Same as quit(QuitSuccess), but allows cleanup. + NimbleQuit* = ref object of CatchableError + + proc raiseNimbleError*(msg: string, hint = "") = + var exc = newException(NimbleError, msg) + exc.hint = hint + raise exc + + proc getOutputInfo*(err: ref NimbleError): (string, string) = + var error = "" + var hint = "" + error = err.msg + when not defined(release): + let stackTrace = getStackTrace(err) + error = stackTrace & "\n\n" & error + if not err.isNil: + hint = err.hint + + return (error, hint) + +const + nimbleVersion* = "0.11.0" + +when not declared(initHashSet): + import sets + + template initHashSet*[A](initialSize = 64): HashSet[A] = + initSet[A](initialSize) + +when not declared(toHashSet): + import sets + + template toHashSet*[A](keys: openArray[A]): HashSet[A] = + toSet(keys) diff --git a/src/nimblepkg/config.nim b/src/nimblepkg/config.nim new file mode 100644 index 0000000..c4a48fc --- /dev/null +++ b/src/nimblepkg/config.nim @@ -0,0 +1,124 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +import parsecfg, streams, strutils, os, tables, uri + +import version, cli + +type + Config* = object + nimbleDir*: string + chcp*: bool # Whether to change the code page in .cmd files on Win. + packageLists*: Table[string, PackageList] ## Names -> packages.json files + cloneUsingHttps*: bool # Whether to replace git:// for https:// + httpProxy*: Uri # Proxy for package list downloads. + nimLibPrefix*: string # Nim stdlib prefix. + + PackageList* = object + name*: string + urls*: seq[string] + path*: string + +proc initConfig(): Config = + result.nimbleDir = getHomeDir() / ".nimble" + + result.httpProxy = initUri() + + result.chcp = true + result.cloneUsingHttps = true + + result.packageLists = initTable[string, PackageList]() + let defaultPkgList = PackageList(name: "Official", urls: @[ + "https://github.com/nim-lang/packages/raw/master/packages.json", + "http://irclogs.nim-lang.org/packages.json", + "http://nim-lang.org/nimble/packages.json" + ]) + result.packageLists["official"] = defaultPkgList + + result.nimLibPrefix = "" + +proc initPackageList(): PackageList = + result.name = "" + result.urls = @[] + result.path = "" + +proc addCurrentPkgList(config: var Config, currentPackageList: PackageList) = + if currentPackageList.name.len > 0: + config.packageLists[currentPackageList.name.normalize] = currentPackageList + +proc parseConfig*(): Config = + result = initConfig() + var confFile = getConfigDir() / "nimble" / "nimble.ini" + + var f = newFileStream(confFile, fmRead) + if f == nil: + # Try the old deprecated babel.ini + # TODO: This can be removed. + confFile = getConfigDir() / "babel" / "babel.ini" + f = newFileStream(confFile, fmRead) + if f != nil: + display("Warning", "Using deprecated config file at " & confFile, + Warning, HighPriority) + if f != nil: + display("Reading", "config file at " & confFile, priority = LowPriority) + var p: CfgParser + open(p, f, confFile) + var currentSection = "" + var currentPackageList = initPackageList() + while true: + var e = next(p) + case e.kind + of cfgEof: + if currentSection.len > 0: + if currentPackageList.urls.len == 0 and currentPackageList.path == "": + raise newException(NimbleError, "Package list '$1' requires either url or path" % currentPackageList.name) + if currentPackageList.urls.len > 0 and currentPackageList.path != "": + raise newException(NimbleError, "Attempted to specify `url` and `path` for the same package list '$1'" % currentPackageList.name) + addCurrentPkgList(result, currentPackageList) + break + of cfgSectionStart: + addCurrentPkgList(result, currentPackageList) + currentSection = e.section + case currentSection.normalize + of "packagelist": + currentPackageList = initPackageList() + else: + raise newException(NimbleError, "Unable to parse config file:" & + " Unknown section: " & e.key) + of cfgKeyValuePair, cfgOption: + case e.key.normalize + of "nimbledir": + # Ensure we don't restore the deprecated nimble dir. + if e.value != getHomeDir() / ".babel": + result.nimbleDir = e.value + of "chcp": + result.chcp = parseBool(e.value) + of "cloneusinghttps": + result.cloneUsingHttps = parseBool(e.value) + of "httpproxy": + result.httpProxy = parseUri(e.value) + of "name": + case currentSection.normalize + of "packagelist": + currentPackageList.name = e.value + else: assert false + of "url": + case currentSection.normalize + of "packagelist": + currentPackageList.urls.add(e.value) + else: assert false + of "path": + case currentSection.normalize + of "packagelist": + if currentPackageList.path != "": + raise newException(NimbleError, "Attempted to specify more than one `path` for the same package list.") + else: + currentPackageList.path = e.value + else: assert false + of "nimlibprefix": + result.nimLibPrefix = e.value + else: + raise newException(NimbleError, "Unable to parse config file:" & + " Unknown key: " & e.key) + of cfgError: + raise newException(NimbleError, "Unable to parse config file: " & e.msg) + close(p) diff --git a/src/nimblepkg/deps.nim b/src/nimblepkg/deps.nim new file mode 100644 index 0000000..14be4a5 --- /dev/null +++ b/src/nimblepkg/deps.nim @@ -0,0 +1,52 @@ +import packageinfotypes, developfile, packageinfo, version, tables, strformat, strutils + +type + DependencyNode = ref object of RootObj + name*: string + version*: string + resolvedTo*: string + error*: string + dependencies*: seq[DependencyNode] + +proc depsRecursive*(pkgInfo: PackageInfo, + dependencies: seq[PackageInfo], + errors: ValidationErrors): seq[DependencyNode] = + result = @[] + + for (name, ver) in pkgInfo.fullRequirements: + var depPkgInfo = initPackageInfo() + let + found = dependencies.findPkg((name, ver), depPkgInfo) + packageName = if found: depPkgInfo.basicInfo.name else: name + + let node = DependencyNode(name: packageName) + + result.add node + node.version = if ver.kind == verAny: "@any" else: $ver + node.resolvedTo = if found: $depPkgInfo.basicInfo.version else: "" + node.error = if errors.contains(packageName): + getValidationErrorMessage(packageName, errors.getOrDefault packageName) + else: "" + + if found: + node.dependencies = depsRecursive(depPkgInfo, dependencies, errors) + +proc printDepsHumanReadable*(pkgInfo: PackageInfo, + dependencies: seq[PackageInfo], + level: int, + errors: ValidationErrors) = + for (name, ver) in pkgInfo.requires: + var depPkgInfo = initPackageInfo() + let + found = dependencies.findPkg((name, ver), depPkgInfo) + packageName = if found: depPkgInfo.basicInfo.name else: name + + echo " ".repeat(level * 2), + packageName, + if ver.kind == verAny: "@any" else: " " & $ver, + if found: fmt "(resolved {depPkgInfo.basicInfo.version})" else: "", + if errors.contains(packageName): + " - error: " & getValidationErrorMessage(packageName, errors.getOrDefault packageName) + else: + "" + if found: printDepsHumanReadable(depPkgInfo, dependencies, level + 1, errors) diff --git a/src/nimblepkg/developfile.nim b/src/nimblepkg/developfile.nim new file mode 100644 index 0000000..c1e9499 --- /dev/null +++ b/src/nimblepkg/developfile.nim @@ -0,0 +1,953 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module implements operations required for working with Nimble develop +## files. + +import sets, json, sequtils, os, strformat, tables, hashes, strutils, math, + std/jsonutils + +import typetraits except distinctBase + +import common, cli, packageinfotypes, packageinfo, packageparser, options, + version, paths, displaymessages, sha1hashes, + tools, vcstools, syncfile, lockfile + +type + DevelopFileJsonData = object + # The raw data read from the JSON develop file. + includes: OrderedSet[Path] + ## Paths to the included in the current one develop files. + dependencies: OrderedSet[Path] + ## Paths to the dependencies directories. + + DevFileNameToPkgs* = Table[Path, HashSet[ref PackageInfo]] + ## Mapping between a develop file name and a set of packages. + + PkgToDevFileNames* = Table[ref PackageInfo, HashSet[Path]] + ## Mapping between a package and a set of develop files. + + DevelopFileData* = object + ## The raw data read from the JSON develop file plus the metadata. + path: Path + ## The full path to the develop file. + jsonData: DevelopFileJsonData + ## The actual content of the develop file. + nameToPkg: Table[string, ref PackageInfo] + ## The list of packages coming from the current develop file or some of + ## its includes, indexed by package name. + pathToPkg: Table[Path, ref PackageInfo] + ## The list of packages coming from the current develop file or some of + ## its includes, indexed by package path. + devFileNameToPkgs: DevFileNameToPkgs + ## For each develop file contains references to the packages coming from + ## it or some of its includes. It is used to keep information for which + ## packages, the reference count must be decreased when a develop file + ## is removed. + pkgToDevFileNames: PkgToDevFileNames + ## For each package contains the set of names of the develop files where + ## the path to its directory is mentioned. Used for colliding names error + ## reporting when packages with same name but different paths are present. + pkgRefCount: CountTable[ref PackageInfo] + ## For each package contains the number of times it is included from + ## different develop files. When the reference count drops to zero the + ## package will be removed from all internal meta data structures. + dependentPkg: PackageInfo + ## The `PackageInfo` of the package in the current directory. + ## It can be missing in the case that this is a develop file intended only + ## for inclusion in other develop files and not related to specific + ## package. + + DevelopFileDataCache = Table[Path, DevelopFileData] + ## A cache for the loaded develop files data used to avoid multiple loads + ## of the same file when its data is queried in the code. + + DevelopFileJsonKeys = enum + ## Develop file JSON objects names. + dfjkVersion = "version" + dfjkIncludes = "includes" + dfjkDependencies = "dependencies" + + NameCollisionRecord = tuple[pkgPath, inclFilePath: Path] + ## Describes the path to a package with a name same as the name of another + ## package, and the path to develop files where it is found. + + CollidingNames = Table[string, HashSet[NameCollisionRecord]] + ## Describes Nimble packages names found more than once in a develop file + ## either directly or via its includes but pointing to different paths. + + InvalidPaths = Table[Path, ref CatchableError] + ## Describes an invalid path to a Nimble package or included develop file. + ## Contains the path as a key and the exact error occurred when we had tried + ## to read the package or the develop file at it. + + ErrorsCollection = object + ## Describes the different errors which are possible to occur on loading of + ## a develop file. + collidingNames: CollidingNames + invalidPackages: InvalidPaths + invalidIncludeFiles: InvalidPaths + +const + developFileName* = "nimble.develop" + ## The default name of a Nimble's develop file. This must always be the name + ## of develop files which are not only for inclusion but associated with a + ## specific package. + developFileVersion* = 1 + ## The version of the develop file's JSON schema. + +proc initDevelopFileData: DevelopFileData = + result = DevelopFileData(dependentPkg: initPackageInfo()) + +proc getNimbleFilePath(pkgInfo: PackageInfo): Path = + ## This is a version of `PackageInfo`'s `getNimbleFileDir` procedure returning + ## `Path` type. + pkgInfo.getNimbleFileDir.Path + +proc assertHasDependentPkg(data: DevelopFileData) = + ## Checks whether there is associated dependent package with the `data`. + assert data.dependentPkg.isLoaded, + "This procedure must be used only with associated with particular " & + "package develop files." + +proc getPkgDevFilePath(pkg: PackageInfo): Path = + ## Returns the path to the develop file associated with the package `pkg`. + pkg.getNimbleFilePath / developFileName + +proc isEmpty*(data: DevelopFileData): bool = + ## Checks whether there is some content (paths to packages directories or + ## includes to other develop files) in the develop file. + data.jsonData.includes.len == 0 and data.jsonData.dependencies.len == 0 + +proc save*(data: DevelopFileData, path: Path, writeEmpty, overwrite: bool) = + ## Saves the `data` to a JSON file with path `path`. If the `data` is empty + ## writes an empty JSON file only if `writeEmpty` is `true`. + ## + ## Raises an `IOError` if: + ## - `overwrite` is `false` and the file with path `path` already exists. + ## - for some reason the writing of the file fails. + + if not writeEmpty and data.isEmpty: + return + + let json = %{ + $dfjkVersion: %developFileVersion, + $dfjkIncludes: %data.jsonData.includes.toSeq, + $dfjkDependencies: %data.jsonData.dependencies.toSeq, + } + + if path.fileExists and not overwrite: + raise nimbleError(fileAlreadyExistsMsg($path)) + + writeFile(path, json.pretty) + displaySuccess(developFileSavedMsg($path), priority = DebugPriority) + +proc developFileExists*(dir: Path): bool = + ## Returns `true` if there is a Nimble develop file with a default name in + ## the directory `dir` or `false` otherwise. + fileExists(dir / developFileName) + +proc developFileExists*(pkg: PackageInfo): bool = + ## Returns `true` if there is a Nimble develop file with a default name in + ## the directory of the package's `pkg` `.nimble` file or `false` otherwise. + pkg.getNimbleFilePath.developFileExists + +proc validatePackage(pkgPath: Path, options: Options): + tuple[pkgInfo: PackageInfo, error: ref CatchableError] = + ## By given file system path `pkgPath`, determines whether it points to a + ## valid Nimble package. + ## + ## Returns a tuple containing: + ## - `pkgInfo` - the package info of the package at `pkgPath` in case + ## `pkgPath` directory contains a valid Nimble package. + ## + ## - `error` - a reference to the exception raised in case `pkgPath` is + ## not a valid package directory. + + try: + result.pkgInfo = getPkgInfo(string(pkgPath), options, true) + except CatchableError as error: + result.error = error + +proc hasErrors(errors: ErrorsCollection): bool = + ## Checks whether there are some errors in the `ErrorsCollection` - `errors`. + errors.collidingNames.len > 0 or errors.invalidPackages.len > 0 or + errors.invalidIncludeFiles.len > 0 + +proc pkgFoundMoreThanOnceMsg*( + pkgName: string, collisions: HashSet[NameCollisionRecord]): string = + result = &"A package with name \"{pkgName}\" is found more than once." + for (pkgPath, inclFilePath) in collisions: + result &= &"\n\"{pkgPath}\" from file \"{inclFilePath}\"" + +proc getErrorsDetails(errors: ErrorsCollection): string = + ## Constructs a message with details about the collected errors. + + for pkgPath, error in errors.invalidPackages: + result &= invalidPkgMsg($pkgPath) + result &= &"\nReason: {error.msg}\n\n" + + for inclFilePath, error in errors.invalidIncludeFiles: + result &= invalidDevFileMsg($inclFilePath) + result &= &"\nReason: {error.msg}\n\n" + + for pkgName, collisions in errors.collidingNames: + result &= pkgFoundMoreThanOnceMsg(pkgName, collisions) + result &= "\n" + +proc add[K, V](t: var Table[K, HashSet[V]], k: K, v: V) = + ## Adds a value `v` to the hash set corresponding to the key `k` of the table + ## `t` by first inserting the key `k` and a new hash set into the table `t`, + ## if they don't already exist. + t.withValue(k, value) do: + value[].incl(v) + do: + t[k] = [v].toHashSet + +proc add[K, V](t: var Table[K, HashSet[V]], k: K, values: HashSet[V]) = + ## Adds all values from the hash set `values` to the hash set corresponding + ## to the key `k` of the table `t` by first inserting the key `k` and a new + ## hash set into the table `t`, if they don't already exist. + for v in values: t.add(k, v) + +proc del[K, V](t: var Table[K, HashSet[V]], k: K, v: V) = + ## Removed a value `v` from the hash set corresponding to the key `k` of the + ## table `t` and removes the key and the corresponding hash set from the + ## table in the case the hash set becomes empty. Does nothing if the key in + ## not present in the table or the value is not present in the hash set. + + t.withValue(k, value) do: + value[].excl(v) + if value[].len == 0: + t.del(k) + +proc assertHasKey[K, V](t: Table[K, V], k: K) = + ## Asserts that the key `k` is present in the table `t`. + assert t.hasKey(k), + &"At this point the key `{k}` should be present in the table {t}." + +proc addPackage(data: var DevelopFileData, pkgInfo: PackageInfo, + comingFrom: Path, actualComingFrom: HashSet[Path], + collidingNames: var CollidingNames) = + ## Adds a package `pkgInfo` to the `data` internal meta data structures. + ## + ## Other parameters: + ## `comingFrom` - the develop file name which loading causes the + ## package to be included. + ## + ## `actualComingFrom` - the set of actual develop files where the package + ## path is mentioned. + ## + ## `collidingNames` - an output parameters where packages with same name + ## but with different paths are registered for error + ## reporting. + + var pkg = data.nameToPkg.getOrDefault(pkgInfo.basicInfo.name) + if pkg == nil: + # If a package with `pkgInfo.name` is missing add it to the + # `DevelopFileData` internal data structures add it. + pkg = pkgInfo.newClone + data.pkgRefCount.inc(pkg) + data.nameToPkg[pkg[].basicInfo.name] = pkg + data.pathToPkg[pkg[].getNimbleFilePath()] = pkg + data.devFileNameToPkgs.add(comingFrom, pkg) + data.pkgToDevFileNames.add(pkg, actualComingFrom) + else: + # If a package with `pkgInfo.name` is already included check whether it has + # the same path as the package we are trying to include. + let + alreadyIncludedPkgPath = pkg[].getNimbleFilePath() + newPkgPath = pkgInfo.getNimbleFilePath() + + if alreadyIncludedPkgPath == newPkgPath: + # If the paths are the same then increase the reference count of the + # package and register the new develop files from where it is coming. + data.pkgRefCount.inc(pkg) + data.devFileNameToPkgs.add(comingFrom, pkg) + data.pkgToDevFileNames.add(pkg, actualComingFrom) + else: + # But if we already have a package with the same name at different path + # register the name collision which to be reported as error. + assertHasKey(data.pkgToDevFileNames, pkg) + for devFileName in data.pkgToDevFileNames[pkg]: + collidingNames.add(pkg[].basicInfo.name, (alreadyIncludedPkgPath, devFileName)) + for devFileName in actualComingFrom: + collidingNames.add(pkg[].basicInfo.name, (newPkgPath, devFileName)) + +proc values[K, V](t: Table[K, V]): seq[V] = + ## Returns a sequence containing table's `t` values. + result.setLen(t.len) + var i: Natural = 0 + for v in t.values: + result[i] = v + inc(i) + +proc addPackages(lhs: var DevelopFileData, pkgs: seq[ref PackageInfo], + rhsPath: Path, rhsPkgToDevFileNames: PkgToDevFileNames, + collidingNames: var CollidingNames) = + ## Adds packages from `pkgs` sequence to the develop file data `lhs`. + for pkgRef in pkgs: + assertHasKey(rhsPkgToDevFileNames, pkgRef) + lhs.addPackage(pkgRef[], rhsPath, rhsPkgToDevFileNames[pkgRef], + collidingNames) + +proc mergeIncludedDevFileData(lhs: var DevelopFileData, rhs: DevelopFileData, + errors: var ErrorsCollection) = + ## Merges develop file data `rhs` coming from some included develop file into + ## `lhs`. + lhs.addPackages(rhs.nameToPkg.values, rhs.path, rhs.pkgToDevFileNames, + errors.collidingNames) + +proc mergeFollowedDevFileData(lhs: var DevelopFileData, rhs: DevelopFileData, + errors: var ErrorsCollection) = + ## Merges develop file data `rhs` coming from some followed package's develop + ## file into `lhs`. + rhs.assertHasDependentPkg + lhs.addPackages(rhs.nameToPkg.values, rhs.path, rhs.pkgToDevFileNames, + errors.collidingNames) + +proc load(path: Path, dependentPkg: PackageInfo, options: Options, + silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool): + DevelopFileData + +template load(dependentPkg: PackageInfo, args: varargs[untyped]): + DevelopFileData = + ## Loads data for the `dependentPkg`'s develop file by searching it in the + ## package's Nimble file directory. Delegates the functionality to the `load` + ## procedure taking path to develop file. + dependentPkg.assertIsLoaded + load(dependentPkg.getPkgDevFilePath, dependentPkg, args) + +proc loadGlobalDependencies(result: var DevelopFileData, + collidingNames: var CollidingNames, + options: Options) = + ## Loads data from the `links` subdirectory in the Nimble cache. The links + ## in the cache are treated as paths in a global develop file used when a + ## local one does not exist. + + for (kind, path) in walkDir(options.getPkgsLinksDir): + if kind != pcDir: + continue + let (pkgName, _, _) = getNameVersionChecksum(path) + let linkFilePath = path / pkgName.getLinkFileName + if not linkFilePath.fileExists: + displayWarning(&"Not found link file in \"{path}\".") + continue + let lines = linkFilePath.readFile.split("\n") + if lines.len != 2: + displayWarning(&"Invalid link file \"{linkFilePath}\".") + continue + let pkgPath = lines[1] + let (pkgInfo, error) = validatePackage(pkgPath, options) + if error == nil: + let path = path.Path + result.addPackage(pkgInfo, path, [path].toHashSet, collidingNames) + else: + displayWarning( + &"Package \"{pkgName}\" at path \"{pkgPath}\" is invalid. Skipping it.") + displayDetails(error.msg) + +proc load(path: Path, dependentPkg: PackageInfo, options: Options, + silentIfFileNotExists, raiseOnValidationErrors, loadGlobalDeps: bool): + DevelopFileData = + ## Loads data from a develop file at path `path`. + ## + ## If `silentIfFileNotExists` then does nothing in the case the develop file + ## does not exists. + ## + ## If `raiseOnValidationErrors` raises a `NimbleError` in the case some of the + ## contents of the develop file are invalid. + ## + ## If `loadGlobalDeps` then load the packages pointed by the link files in the + ## `links` directory in the Nimble cache instead of the once pointed by the + ## local develop file. + ## + ## Raises if the develop file or some of the included develop files: + ## - cannot be read. + ## - has an invalid JSON schema. + ## - contains a path to some invalid package. + ## - contains paths to multiple packages with the same name. + + var cache {.global.}: DevelopFileDataCache + if cache.hasKey(path): + return cache[path] + + result = initDevelopFileData() + result.path = path + result.dependentPkg = dependentPkg + + var + errors {.global.}: ErrorsCollection + visitedFiles {.global.}: HashSet[Path] + visitedPkgs {.global.}: HashSet[Path] + + visitedFiles.incl path + if dependentPkg.isLoaded: + visitedPkgs.incl dependentPkg.getNimbleFileDir + + if loadGlobalDeps: + loadGlobalDependencies(result, errors.collidingNames, options) + else: + if silentIfFileNotExists and not path.fileExists: + return + + try: + fromJson(result.jsonData, parseFile(path), Joptions(allowExtraKeys: true)) + except ValueError as error: + raise nimbleError(notAValidDevFileJsonMsg($path), details = error) + + for depPath in result.jsonData.dependencies: + let depPath = if depPath.isAbsolute: + depPath.normalizedPath else: (path.splitFile.dir / depPath).normalizedPath + let (pkgInfo, error) = validatePackage(depPath, options) + if error == nil: + result.addPackage(pkgInfo, path, [path].toHashSet, errors.collidingNames) + else: + errors.invalidPackages[depPath] = error + + for inclPath in result.jsonData.includes: + let inclPath = inclPath.normalizedPath + if visitedFiles.contains(inclPath): + continue + var inclDevFileData = initDevelopFileData() + try: + inclDevFileData = load( + inclPath, initPackageInfo(), options, false, false, false) + except CatchableError as error: + errors.invalidIncludeFiles[path] = error + continue + result.mergeIncludedDevFileData(inclDevFileData, errors) + + if result.dependentPkg.isLoaded and path.splitPath.tail == developFileName: + # If this is a package develop file, but not a free one, for each of the + # package's develop mode dependencies load its develop file if it is not + # already loaded and merge its data to the current develop file's data. + for path, pkg in result.pathToPkg.dup: + if visitedPkgs.contains(path): + continue + var followedPkgDevFileData = initDevelopFileData() + try: + followedPkgDevFileData = load(pkg[], options, true, false, false) + except: + # The errors will be accumulated in `errors` global variable and + # reported by the `load` call which initiated the recursive process. + discard + result.mergeFollowedDevFileData(followedPkgDevFileData, errors) + + if not errors.hasErrors: + cache[path] = result + return result + elif raiseOnValidationErrors: + raise nimbleError(failedToLoadFileMsg($path), + details = nimbleError(errors.getErrorsDetails)) + +proc addDevelopPackage(data: var DevelopFileData, pkg: PackageInfo): bool = + ## Adds package `pkg`'s path to the develop file. + ## + ## Returns `true` if: + ## - the path is successfully added to the develop file. + ## - the path is already present in the develop file. + ## (Only a warning in printed in this case.) + ## + ## Returns `false` in the case of error when: + ## - a package with the same name but at different path is already present + ## in the develop file or some of its includes. + + let pkgDir = pkg.getNimbleFilePath() + + # Check whether the develop file already contains a package with a name + # `pkg.name` at different path. + if data.nameToPkg.hasKey(pkg.basicInfo.name) and not data.pathToPkg.hasKey(pkgDir): + let otherPath = data.nameToPkg[pkg.basicInfo.name][].getNimbleFilePath() + displayError(pkgAlreadyPresentAtDifferentPathMsg( + pkg.basicInfo.name, $otherPath, $data.path)) + return false + + # Add `pkg` to the develop file model. + let success = not data.jsonData.dependencies.containsOrIncl(pkgDir) + + var collidingNames: CollidingNames + addPackage(data, pkg, data.path, [data.path].toHashSet, collidingNames) + assert collidingNames.len == 0, "Must not have the same package name at " & + "path different than already existing one." + + if success: + displaySuccess(pkgAddedInDevFileMsg( + pkg.getNameAndVersion, $pkgDir, $data.path)) + else: + displayWarning(pkgAlreadyInDevFileMsg( + pkg.getNameAndVersion, $pkgDir, $data.path)) + + return true + +proc addDevelopPackage(data: var DevelopFileData, path: Path, + options: Options): bool = + ## Adds path `path` to some package directory to the develop file. + ## + ## Returns `true` if: + ## - the path is successfully added to the develop file. + ## - the path is already present in . + ## (Only a warning in printed in this case.) + ## + ## Returns `false` in the case of error when: + ## - the path in `path` does not point to a valid Nimble package. + ## - a package with the same name but at different path is already present + ## in the develop file or some of its includes. + + let (pkgInfo, error) = validatePackage(path, options) + if error != nil: + displayError(invalidPkgMsg($path)) + displayDetails(error) + return false + + return addDevelopPackage(data, pkgInfo) + +proc dec[K](t: var CountTable[K], k: K): bool {.discardable.} = + ## Decrements the count of key `k` in table `t`. If the count drops to zero + ## the procedure removes the key from the table. + ## + ## Returns `true` in the case the count for the key `k` drops to zero and the + ## key is removed from the table or `false` otherwise. + ## + ## If the key `k` is missing raises a `KeyError` exception. + if k in t: + t[k] = t[k] - 1 + if t[k] == 0: + t.del(k) + result = true + else: + raise newException(KeyError, &"The key \"{k}\" is not found.") + +proc removePackage(data: var DevelopFileData, pkg: ref PackageInfo, + devFileName: Path) = + ## Decreases the reference count for a package at path `path` and removes the + ## package from the internal meta data structures in case the reference count + ## drops to zero. + + # If the package is found it must be excluded from the develop file mappings + # by using the name of the develop file as result of which manipulation the + # package is being removed. + data.devFileNameToPkgs.del(devFileName, pkg) + data.pkgToDevFileNames.del(pkg, devFileName) + + # Also the reference count of the package should be decreased. + let removed = data.pkgRefCount.dec(pkg) + if not removed: + # If the reference count is not zero no further processing is needed. + return + + # But if the reference count is zero the package should be removed from all + # other meta data structures to free memory for it and its indexes. + data.nameToPkg.del(pkg[].basicInfo.name) + data.pathToPkg.del(pkg[].getNimbleFilePath()) + + # The package `pkg` could already be missing from `pkgToDevFileNames` if it + # is removed with the removal of `devFileName` value, but if it is included + # from some of `devFileName`'s includes it will still be present and we + # should remove it completely to free its memory. + data.pkgToDevFileNames.del(pkg) + +proc removePackage(data: var DevelopFileData, path, devFileName: Path) = + ## Decreases the reference count for a package at path `path` and removes the + ## package from the internal meta data structures in case the reference count + ## drops to zero. + + let pkg = data.pathToPkg.getOrDefault(path) + if pkg == nil: + # If there is no package at path `path` found. + return + + data.removePackage(pkg, devFileName) + +proc removeDevelopPackageByPath(data: var DevelopFileData, path: Path): bool = + ## Removes path `path` to some package directory from the develop file. + ## If the `path` is not present in the develop file prints a warning. + ## + ## Returns `true` if path `path` is successfully removed from the develop file + ## or `false` if there is no such path added in it. + + let success = not data.jsonData.dependencies.missingOrExcl(path) + + if success: + let nameAndVersion = data.pathToPkg[path][].getNameAndVersion() + data.removePackage(path, data.path) + displaySuccess(pkgRemovedFromDevFileMsg(nameAndVersion, $path, $data.path)) + else: + displayWarning(pkgPathNotInDevFileMsg($path, $data.path)) + + return success + +proc removeDevelopPackageByName(data: var DevelopFileData, name: string): bool = + ## Removes path to a package with name `name` from the develop file. + ## If a package with name `name` is not present in the develop file prints a + ## warning. + ## + ## Returns `true` if a package with name `name` is successfully removed from + ## the develop file or `false` if there is no such package added in it. + + let + pkg = data.nameToPkg.getOrDefault(name) + path = if pkg != nil: pkg[].getNimbleFilePath() else: "" + success = not data.jsonData.dependencies.missingOrExcl(path) + + if success: + data.removePackage(path, data.path) + displaySuccess(pkgRemovedFromDevFileMsg( + pkg[].getNameAndVersion, $path, $data.path)) + else: + displayWarning(pkgNameNotInDevFileMsg(name, $data.path)) + + return success + +proc includeDevelopFile(data: var DevelopFileData, path: Path, + options: Options): bool = + ## Includes a develop file at path `path` to the current project's develop + ## file. + ## + ## Returns `true` if the develop file at `path` is: + ## - successfully included in the current project's develop file. + ## - already present in the current project's develop file. + ## (Only a warning in printed in this case.) + ## + ## Returns `false` in the case of error when: + ## - the develop file at `path` could not be loaded. + ## - the inclusion of the develop file at `path` causes a packages names + ## collisions with already added from different place packages with + ## the same name, but with different location. + + var inclFileData = initDevelopFileData() + try: + inclFileData = load(path, initPackageInfo(), options, false, true, false) + except CatchableError as error: + displayError(failedToLoadFileMsg($path)) + displayDetails(error) + return false + + let success = not data.jsonData.includes.containsOrIncl(path) + + if success: + var errors: ErrorsCollection + data.mergeIncludedDevFileData(inclFileData, errors) + if errors.hasErrors: + displayError(failedToInclInDevFileMsg($path, $data.path)) + displayDetails(errors.getErrorsDetails) + # Revert the inclusion in the case of merge errors. + data.jsonData.includes.excl(path) + for pkgPath, _ in inclFileData.pathToPkg: + data.removePackage(pkgPath, path) + return false + + displaySuccess(inclInDevFileMsg($path, $data.path)) + else: + displayWarning(alreadyInclInDevFileMsg($path, $data.path)) + + return true + +proc excludeDevelopFile(data: var DevelopFileData, path: Path): bool = + ## Excludes a develop file at path `path` from the current project's develop + ## file. If there is no such, then only a warning is printed. + ## + ## Returns `true` if a develop file at path `path` is successfully removed + ## from the current project's develop file or `false` if there is no such + ## file included in the current one. + + let success = not data.jsonData.includes.missingOrExcl(path) + + if success: + assertHasKey(data.devFileNameToPkgs, path) + + # Copy the references of the packages which should be deleted, because + # deleting from the same hash set which we iterate will not be correct. + var packages = data.devFileNameToPkgs[path].toSeq + + # Try to remove the packages coming from the develop file at path `path` or + # some of its includes by decreasing their reference count and appropriately + # updating all other internal meta data structures. + for pkg in packages: + data.removePackage(pkg, path) + + displaySuccess(exclFromDevFileMsg($path, $data.path)) + else: + displayWarning(notInclInDevFileMsg($path, $data.path)) + + return success + +proc assertDevelopActionIsSet(options: Options) = + ## Asserts that the currently set action in the `options` object is `develop`. + assert options.action.typ == actionDevelop, + "This procedure must be called only on develop command." + +proc updateDevelopFile*(dependentPkg: PackageInfo, options: Options): bool = + ## Updates a dependent package `dependentPkg`'s develop file with an + ## information from the Nimble's command line. + ## - Adds newly installed develop packages. + ## - Adds packages by path. + ## - Removes packages by path. + ## - Removes packages by name. + ## - Includes other develop files. + ## - Excludes other develop files. + ## + ## Returns `true` if all operations are successful and `false` otherwise. + ## Raises if cannot load an existing develop file. + + options.assertDevelopActionIsSet + + let developFile = options.action.developFile + + var + hasError = false + hasSuccessfulRemoves = false + data = load(developFile, dependentPkg, options, true, true, false) + + defer: + let writeEmpty = hasSuccessfulRemoves or + developFile != developFileName or + not dependentPkg.isLoaded + data.save(developFile, writeEmpty = writeEmpty, overwrite = true) + + for (actionType, argument) in options.action.devActions: + case actionType + of datAdd: + hasError = not data.addDevelopPackage(argument, options) or hasError + of datRemoveByPath: + hasSuccessfulRemoves = data.removeDevelopPackageByPath(argument) or + hasSuccessfulRemoves + of datRemoveByName: + hasSuccessfulRemoves = data.removeDevelopPackageByName(argument) or + hasSuccessfulRemoves + of datInclude: + hasError = not data.includeDevelopFile(argument, options) or hasError + of datExclude: + hasSuccessfulRemoves = data.excludeDevelopFile(argument) or + hasSuccessfulRemoves + + return not hasError + +proc processDevelopDependencies*(dependentPkg: PackageInfo, options: Options): + seq[PackageInfo] = + ## Returns a sequence with the develop mode dependencies of the `dependentPkg` + ## and recursively all of their develop mode dependencies. + + let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists + let data = load(dependentPkg, options, true, true, loadGlobalDeps) + result = newSeqOfCap[PackageInfo](data.nameToPkg.len) + for _, pkg in data.nameToPkg: + result.add pkg[] + +proc getDevelopDependencies*(dependentPkg: PackageInfo, options: Options): + Table[string, ref PackageInfo] = + ## Returns a table with a mapping between names and `PackageInfo`s of develop + ## mode dependencies of package `dependentPkg` and recursively all of their + ## develop mode dependencies. + + let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists + let data = load(dependentPkg, options, true, true, loadGlobalDeps) + return data.nameToPkg + +type + ValidationErrorKind* = enum + ## Types of possible errors when validating the develop file against the + ## lock file with corresponding parts of their error messages. + vekDirIsNotUnderVersionControl = "is not under version control." + vekWorkingCopyIsNotClean = "has not clean working copy." + vekVcsRevisionIsNotPushed = "has not pushed VCS revisions." + vekWorkingCopyNeedsSync = "has not synced working copy." + vekWorkingCopyNeedsLock = "has not locked commits." + vekWorkingCopyNeedsMerge = "has local changes which are in " & + "conflict with the remote changes." + + ValidationErrorFlags = set[ValidationErrorKind] + ## Set containing flags for the already met validation errors. + + ValidationError* = object + ## Contains information for a validation error for some develop mode + ## package. + kind*: ValidationErrorKind + path*: Path + + ValidationErrors* = Table[string, ValidationError] + ## Mapping between package names and their validation errors info. + + NeedsOperation = enum + ## Helper enum for the return type of the procedure determining whether a + ## develop mode dependency working copy needs some operation to resolve the + ## conflict between it and the lock file. + needsNone, needsLock, needsSync, needsMerge + +proc assertHasValidationErrors(errors: ValidationErrors) = + assert errors.len > 0, "Must have validation errors." + +proc getValidationErrorMessage*(name: string, error: ValidationError): string = + ## By given validation error `error` constructs a validation error message for + ## given develop mode dependency package with name `name`. + &"Package \"{name}\" at \"{error.path}\" {error.kind}.\n" + +proc getValidationErrorsMessage*(errors: ValidationErrors): string = + ## Constructs an error message reporting develop mode dependencies validation + ## errors. + + errors.assertHasValidationErrors + result = "Some of package's develop mode dependencies are invalid.\n" + for name, error in errors: + result &= getValidationErrorMessage(name, error) + +proc allAreSet(errorFlags: set[ValidationErrorKind]): bool = + ## Checks whether all possible validation error flags are set. + cast[uint](errorFlags) == uint(2'd ^ ValidationErrorKind.enumLen - 1) + +proc getValidationsErrorsHint(errors: ValidationErrors): string = + ## Constructs a hint message for resolving develop mode dependencies + ## validation errors. + + errors.assertHasValidationErrors + var errorFlags: ValidationErrorFlags + + for _, error in errors: + case error.kind: + of vekDirIsNotUnderVersionControl, vekWorkingCopyIsNotClean, + vekVcsRevisionIsNotPushed: + if error.kind notin errorFlags: + result &= + "When you are using a lock file Nimble requires develop mode " & + "dependencies to be under version control, all local changes to be " & + "committed and pushed on some remote, and lock file to be updated.\n" + of vekWorkingCopyNeedsSync: + if error.kind notin errorFlags: + result &= + "You have to call `nimble sync` to synchronize your develop mode " & + "dependencies working copies with the latest lock file.\n" + of vekWorkingCopyNeedsLock: + if error.kind notin errorFlags: + result &= + "You have to call `nimble lock` to update your lock file with the " & + "latest versions of your develop mode dependencies working copies.\n" + of vekWorkingCopyNeedsMerge: + if error.kind notin errorFlags: + result &= + "You have to merge or rebase working copies of your develop mode " & + "dependencies which have conflicts with remote changes." + + errorFlags.incl error.kind + if errorFlags.allAreSet: break + +proc pkgDirIsNotUnderVersionControl(depPkg: PackageInfo): bool = + ## Checks whether a develop mode dependency package directory is under version + ## control. + depPkg.getNimbleFileDir.getVcsType == vcsTypeNone + +proc workingCopyIsNotClean(depPkg: PackageInfo): bool = + ## Checks whether a working copy directory of a develop mode dependency + ## package is clean. Untracked files are not considered. + not depPkg.getNimbleFileDir.isWorkingCopyClean + +proc vcsRevisionIsNotPushed(depPkg: PackageInfo): bool = + ## Checks whether current VCS revision of the working copy directory of a + ## develop mode dependency package is pushed on some remote. + not depPkg.getNimbleFileDir.isVcsRevisionPresentOnSomeRemote( + depPkg.metaData.vcsRevision) + +proc workingCopyNeeds*(dependencyPkg, dependentPkg: PackageInfo, + options: Options): NeedsOperation = + ## Be getting in consideration the information from the develop mode + ## dependency working copy directory, the lock file and the sync file + ## determines what kind of operation is needed to resolve the conflicts + ## if any. + + let + lockFileVcsRev = dependentPkg.lockedDeps.getOrDefault("").getOrDefault( + dependencyPkg.basicInfo.name, notSetLockFileDep).vcsRevision + syncFile = getSyncFile(dependentPkg) + syncFileVcsRev = syncFile.getDepVcsRevision(dependencyPkg.basicInfo.name) + workingCopyVcsRev = getVcsRevision(dependencyPkg.getNimbleFileDir) + + if lockFileVcsRev == syncFileVcsRev and syncFileVcsRev == workingCopyVcsRev: + # When all revisions are matching nothing have to be done. + return needsNone + + if lockFileVcsRev == syncFileVcsRev and syncFileVcsRev != workingCopyVcsRev: + # When lock file and sync file revisions are matching, but working copy + # revision is different, then most probably there are local changes and + # `nimble lock` is needed. + return needsLock + + if lockFileVcsRev != syncFileVcsRev and syncFileVcsRev == workingCopyVcsRev: + # When lock file revision is different from sync file revision, but sync + # file revision is equal to working copy revision then most probably we have + # `pull` executed but we forgot to call `nimble sync`. + return needsSync + + if lockFileVcsRev == workingCopyVcsRev and + workingCopyVcsRev != syncFileVcsRev: + # When lock file revision is equal to working copy revision, but they are + # different from sync file revision, most probably this is because of + # damaged sync file. Everything is Ok, because the sync file will be + # rewritten on the next `nimble lock` or `nimble sync` command. + return needsNone + + if lockFileVcsRev != syncFileVcsRev and + lockFileVcsRev != workingCopyVcsRev and + syncFileVcsRev != workingCopyVcsRev: + # When all revisions are different from one another this indicates that + # there are local changes which are conflicting with remote changes. The + # user have to resolve them manually by merging or rebasing. + return needsMerge + + assert false, "Here all cases are covered and the program " & + "flow must not reach this assert." + + return needsNone + +template addError(error: ValidationErrorKind) = + errors[depPkg.basicInfo.name] = ValidationError( + path: depPkg.getNimbleFileDir, kind: error) + +proc findValidationErrorsOfDevDepsWithLockFile*( + dependentPkg: PackageInfo, options: Options, + errors: var ValidationErrors) = + ## Collects validation errors for the develop mode dependencies with the + ## content of the lock file by getting in consideration the information from + ## the sync file. In the case of discrepancy, gives a useful advice what have + ## to be done to resolve the conflicts for the not matching packages. + + dependentPkg.assertIsLoaded + + let developDependencies = processDevelopDependencies(dependentPkg, options) + + for depPkg in developDependencies: + if depPkg.pkgDirIsNotUnderVersionControl: + addError(vekDirIsNotUnderVersionControl) + elif depPkg.workingCopyIsNotClean: + addError(vekWorkingCopyIsNotClean) + elif depPkg.vcsRevisionIsNotPushed: + addError(vekVcsRevisionIsNotPushed) + elif depPkg.workingCopyNeeds(dependentPkg, options) == needsSync: + addError(vekWorkingCopyNeedsSync) + elif depPkg.workingCopyNeeds(dependentPkg, options) == needsLock: + addError(vekWorkingCopyNeedsLock) + elif depPkg.workingCopyNeeds(dependentPkg, options) == needsMerge: + addError(vekWorkingCopyNeedsMerge) + +proc validationErrors*(errors: ValidationErrors): ref NimbleError = + result = nimbleError( + msg = errors.getValidationErrorsMessage, + hint = errors.getValidationsErrorsHint) + +proc validateDevelopFileAgainstLockFile( + dependentPkg: PackageInfo, options: Options) = + ## Does validation of the develop file dependencies against the data written + ## in the lock file. + + var errors: ValidationErrors + + findValidationErrorsOfDevDepsWithLockFile(dependentPkg, options, errors) + if errors.len > 0: + raise validationErrors(errors) + +proc validateDevelopFile*(dependentPkg: PackageInfo, options: Options) = + ## The procedure is used in the Nimble's `check` command to transitively + ## validate the contents of the develop files. + + let loadGlobalDeps = not dependentPkg.getPkgDevFilePath.fileExists + discard load(dependentPkg, options, true, true, loadGlobalDeps) + if dependentPkg.areLockedDepsLoaded: + validateDevelopFileAgainstLockFile(dependentPkg, options) diff --git a/src/nimblepkg/displaymessages.nim b/src/nimblepkg/displaymessages.nim new file mode 100644 index 0000000..46e6d27 --- /dev/null +++ b/src/nimblepkg/displaymessages.nim @@ -0,0 +1,172 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module contains procedures producing some of the displayed by Nimble +## error messages in order to facilitate testing by removing the requirement +## the message to be repeated both in Nimble and the testing code. + +import strformat, strutils +import version, packageinfotypes, sha1hashes + +const + validationFailedMsg* = "Validation failed." + + pathGivenButNoPkgsToDownloadMsg* = + "Path option is given but there are no given packages for download." + + developOptionsWithoutDevelopFileMsg* = + "Options 'add', 'remove', 'include' and 'exclude' cannot be given " & + "when no develop file is specified." + + developWithDependenciesWithoutPackagesMsg* = + "Option 'with-dependencies' is given without packages for develop." + + dependencyNotInRangeErrorHint* = + "Update the version of the dependency package in its Nimble file or " & + "update its required version range in the dependent's package Nimble file." + + notADependencyErrorHint* = + "Add the dependency package as a requirement to the Nimble file of the " & + "dependent package." + + multiplePathOptionsGivenMsg* = "Multiple path options are given." + + multipleDevelopFileOptionsGivenMsg* = + "Multiple develop file options are given." + + ignoringCompilationFlagsMsg* = + "Ignoring compilation flags for installed package." + + updatingTheLockFileMsg* = "Updating the lock file..." + generatingTheLockFileMsg* = "Generating the lock file..." + lockFileIsUpdatedMsg* = "The lock file is updated." + lockFileIsGeneratedMsg* = "The lock file is generated." + +proc fileAlreadyExistsMsg*(path: string): string = + &"Cannot create file \"{path}\" because it already exists." + +proc developFileSavedMsg*(path: string): string = + &"The develop file \"{path}\" has been saved." + +proc pkgSetupInDevModeMsg*(pkgName, pkgPath: string): string = + &"\"{pkgName}\" set up in develop mode successfully to \"{pkgPath}\"." + +proc pkgInstalledMsg*(pkgName: string): string = + &"{pkgName} installed successfully." + +proc pkgNotFoundMsg*(pkg: PkgTuple): string = &"Package {pkg} not found." + +proc pkgDepsAlreadySatisfiedMsg*(dep: PkgTuple): string = + &"Dependency on {dep} already satisfied" + +proc invalidPkgMsg*(path: string): string = + &"The package at \"{path}\" is invalid." + +proc invalidDevFileMsg*(path: string): string = + &"The develop file \"{path}\" is invalid." + +proc notAValidDevFileJsonMsg*(devFilePath: string): string = + &"The file \"{devFilePath}\" has not a valid develop file JSON schema." + +proc pkgAlreadyPresentAtDifferentPathMsg*( + pkgName, otherPath, fileName: string): string = + &"A package with a name \"{pkgName}\" at different path \"{otherPath}\" " & + "is already present in the develop file \"{fileName}\"." + +proc pkgAddedInDevFileMsg*(pkg, path, fileName: string): string = + &"The package \"{pkg}\" at path \"{path}\" is added to the develop file " & + &"\"{fileName}\"." + +proc pkgAlreadyInDevFileMsg*(pkg, path, fileName: string): string = + &"The package \"{pkg}\" at path \"{path}\" is already present in the " & + &"develop file \"{fileName}\"." + +proc pkgRemovedFromDevFileMsg*(pkg, path, fileName: string): string = + &"The package \"{pkg}\" at path \"{path}\" is removed from the develop " & + &"file \"{fileName}\"." + +proc pkgPathNotInDevFileMsg*(path, fileName: string): string = + &"The path \"{path}\" is not in the develop file \"{fileName}\"." + +proc pkgNameNotInDevFileMsg*(pkgName, fileName: string): string = + &"A package with name \"{pkgName}\" is not in the develop file " & + &"\"{fileName}\"." + +proc failedToInclInDevFileMsg*(inclFile, devFile: string): string = + &"Failed to include \"{inclFile}\" to the develop file \"{devFile}\"" + +proc inclInDevFileMsg*(path, fileName: string): string = + &"The develop file \"{path}\" is successfully included into the develop " & + &"file \"{fileName}\"" + +proc alreadyInclInDevFileMsg*(path, fileName: string): string = + &"The develop file \"{path}\" is already included in the develop file " & + &"\"{fileName}\"." + +proc exclFromDevFileMsg*(path, fileName: string): string = + &"The develop file \"{path}\" is successfully excluded from the develop " & + &"file \"{fileName}\"." + +proc notInclInDevFileMsg*(path, fileName: string): string = + &"The file \"{path}\" is not included in the develop file \"{fileName}\"." + +proc failedToLoadFileMsg*(path: string): string = + &"Failed to load \"{path}\"." + +proc cannotUninstallPkgMsg*(pkgName: string, pkgVersion: Version, + deps: seq[string]): string = + assert deps.len > 0, "The sequence must have at least one package." + result = &"Cannot uninstall {pkgName} ({pkgVersion}) because\n" + result &= deps.join("\n") + result &= "\ndepend" & (if deps.len == 1: "s" else: "") & " on it" + +proc promptRemovePkgsMsg*(pkgs: seq[string]): string = + assert pkgs.len > 0, "The sequence must have at least one package." + result = "The following packages will be removed:\n" + result &= pkgs.join("\n") + result &= "\nDo you wish to continue?" + +proc pkgWorkingCopyNeedsSyncingMsg*(pkgName, pkgPath: string): string = + &"Package \"{pkgName}\" working copy at path \"{pkgPath}\" needs syncing." + +proc pkgWorkingCopyIsSyncedMsg*(pkgName, pkgPath: string): string = + &"Working copy of package \"{pkgName}\" at \"{pkgPath}\" is synced." + +proc notInRequiredRangeMsg*( + dependencyPkgName, dependencyPkgPath, dependencyPkgVersion, + dependentPkgName, dependentPkgPath, requiredVersionRange: string): string = + &"The version of the package \"{dependencyPkgName}\" at " & + &"\"{dependencyPkgPath}\" is \"{dependencyPkgVersion}\" and it does not " & + &"match the required by the package \"{dependentPkgName}\" at " & + &"\"{dependentPkgPath}\" version \"{requiredVersionRange}\"." + +proc invalidDevelopDependenciesVersionsMsg*(errors: seq[string]): string = + result = "Some of the develop mode dependencies are with versions which " & + "are not in the required by other package's Nimble file range." + for error in errors: + result &= "\n" + result &= error + +proc pkgAlreadyExistsInTheCacheMsg*(name, version, checksum: string): string = + &"A package \"{name}@{version}\" with checksum \"{checksum}\" already " & + "exists the the cache." + +proc pkgAlreadyExistsInTheCacheMsg*(pkgInfo: PackageInfo): string = + pkgAlreadyExistsInTheCacheMsg( + pkgInfo.basicInfo.name, + $pkgInfo.basicInfo.version, + $pkgInfo.basicInfo.checksum) + +proc skipDownloadingInAlreadyExistingDirectoryMsg*(dir, name: string): string = + &"The download directory \"{dir}\" already exists.\n" & + &"Skipping the download of \"{name}\"." + +proc binaryNotDefinedInPkgMsg*(binaryName, pkgName: string): string = + &"Binary '{binaryName}' is not defined in '{pkgName}' package." + +proc notFoundPkgWithNameInPkgDepTree*(pkgName: string): string = + &"Not found package with name '{pkgName}' in the current package's " & + "dependency tree." + +proc pkgLinkFileSavedMsg*(path: string): string = + &"Package link file \"{path}\" is saved." diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim new file mode 100644 index 0000000..cc5d442 --- /dev/null +++ b/src/nimblepkg/download.nim @@ -0,0 +1,324 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import parseutils, os, osproc, strutils, tables, pegs, uri, json + +import packageinfo, packageparser, version, tools, common, options, cli +from algorithm import SortOrder, sorted +from sequtils import toSeq, filterIt, map + +type + DownloadMethod* {.pure.} = enum + git = "git", hg = "hg" + +proc getSpecificDir(meth: DownloadMethod): string {.used.} = + case meth + of DownloadMethod.git: + ".git" + of DownloadMethod.hg: + ".hg" + +proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) = + case meth + of DownloadMethod.git: + cd downloadDir: + # Force is used here because local changes may appear straight after a + # clone has happened. Like in the case of git on Windows where it + # messes up the damn line endings. + doCmd("git checkout --force " & branch) + doCmd("git submodule update --recursive") + of DownloadMethod.hg: + cd downloadDir: + doCmd("hg checkout " & branch) + +proc doPull(meth: DownloadMethod, downloadDir: string) {.used.} = + case meth + of DownloadMethod.git: + doCheckout(meth, downloadDir, "") + cd downloadDir: + doCmd("git pull") + if existsFile(".gitmodules"): + doCmd("git submodule update") + of DownloadMethod.hg: + doCheckout(meth, downloadDir, "default") + cd downloadDir: + doCmd("hg pull") + +proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", + onlyTip = true) = + case meth + of DownloadMethod.git: + let + depthArg = if onlyTip: "--depth 1 " else: "" + branchArg = if branch == "": "" else: "-b " & branch & " " + doCmd("git clone --recursive " & depthArg & branchArg & url & + " " & downloadDir) + of DownloadMethod.hg: + let + tipArg = if onlyTip: "-r tip " else: "" + branchArg = if branch == "": "" else: "-b " & branch & " " + doCmd("hg clone " & tipArg & branchArg & url & " " & downloadDir) + +proc getTagsList(dir: string, meth: DownloadMethod): seq[string] = + cd dir: + var output = execProcess("git tag") + case meth + of DownloadMethod.git: + output = execProcess("git tag") + of DownloadMethod.hg: + output = execProcess("hg tags") + if output.len > 0: + case meth + of DownloadMethod.git: + result = @[] + for i in output.splitLines(): + if i == "": continue + result.add(i) + of DownloadMethod.hg: + result = @[] + for i in output.splitLines(): + if i == "": continue + var tag = "" + discard parseUntil(i, tag, ' ') + if tag != "tip": + result.add(tag) + else: + result = @[] + +proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] = + var + url = url + uri = parseUri url + if uri.query != "": + uri.query = "" + url = $uri + result = @[] + case meth + of DownloadMethod.git: + var (output, exitCode) = doCmdEx("git ls-remote --tags " & url) + if exitCode != QuitSuccess: + raise newException(OSError, "Unable to query remote tags for " & url & + ". Git returned: " & output) + for i in output.splitLines(): + let refStart = i.find("refs/tags/") + # git outputs warnings, empty lines, etc + if refStart == -1: continue + let start = refStart+"refs/tags/".len + let tag = i[start .. i.len-1] + if not tag.endswith("^{}"): result.add(tag) + + of DownloadMethod.hg: + # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository + raise newException(ValueError, "Hg doesn't support remote tag querying.") + +proc getVersionList*(tags: seq[string]): OrderedTable[Version, string] = + ## Return an ordered table of Version -> git tag label. Ordering is + ## in descending order with the most recent version first. + let taggedVers: seq[tuple[ver: Version, tag: string]] = + tags + .filterIt(it != "") + .map(proc(s: string): tuple[ver: Version, tag: string] = + # skip any chars before the version + let i = skipUntil(s, Digits) + # TODO: Better checking, tags can have any + # names. Add warnings and such. + result = (newVersion(s[i .. s.len-1]), s)) + .sorted(proc(a, b: (Version, string)): int = cmp(a[0], b[0]), + SortOrder.Descending) + result = toOrderedTable[Version, string](taggedVers) + +proc getDownloadMethod*(meth: string): DownloadMethod = + case meth + of "git": return DownloadMethod.git + of "hg", "mercurial": return DownloadMethod.hg + else: + raise newException(NimbleError, "Invalid download method: " & meth) + +proc getHeadName*(meth: DownloadMethod): Version = + ## Returns the name of the download method specific head. i.e. for git + ## it's ``head`` for hg it's ``tip``. + case meth + of DownloadMethod.git: newVersion("#head") + of DownloadMethod.hg: newVersion("#tip") + +proc checkUrlType*(url: string): DownloadMethod = + ## Determines the download method based on the URL. + if doCmdEx("git ls-remote " & url).exitCode == QuitSuccess: + return DownloadMethod.git + elif doCmdEx("hg identify " & url).exitCode == QuitSuccess: + return DownloadMethod.hg + else: + raise newException(NimbleError, "Unable to identify url: " & url) + +proc getUrlData*(url: string): (string, Table[string, string]) = + var uri = parseUri(url) + # TODO: use uri.parseQuery once it lands... this code is quick and dirty. + var subdir = "" + if uri.query.startsWith("subdir="): + subdir = uri.query[7 .. ^1] + + uri.query = "" + return ($uri, {"subdir": subdir}.toTable()) + +proc isURL*(name: string): bool = + name.startsWith(peg" @'://' ") + +proc doDownload(url: string, downloadDir: string, verRange: VersionRange, + downMethod: DownloadMethod, + options: Options): Version = + ## Downloads the repository specified by ``url`` using the specified download + ## method. + ## + ## Returns the version of the repository which has been downloaded. + template getLatestByTag(meth: untyped) {.dirty.} = + # Find latest version that fits our ``verRange``. + var latest = findLatest(verRange, versions) + ## Note: HEAD is not used when verRange.kind is verAny. This is + ## intended behaviour, the latest tagged version will be used in this case. + + # If no tagged versions satisfy our range latest.tag will be "". + # We still clone in that scenario because we want to try HEAD in that case. + # https://github.com/nim-lang/nimble/issues/22 + meth + if $latest.ver != "": + result = latest.ver + + removeDir(downloadDir) + if verRange.kind == verSpecial: + # We want a specific commit/branch/tag here. + if verRange.spe == getHeadName(downMethod): + # Grab HEAD. + doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone) + else: + # Grab the full repo. + doClone(downMethod, url, downloadDir, onlyTip = false) + # Then perform a checkout operation to get the specified branch/commit. + # `spe` starts with '#', trim it. + doAssert(($verRange.spe)[0] == '#') + doCheckout(downMethod, downloadDir, substr($verRange.spe, 1)) + result = verRange.spe + else: + case downMethod + of DownloadMethod.git: + # For Git we have to query the repo remotely for its tags. This is + # necessary as cloning with a --depth of 1 removes all tag info. + result = getHeadName(downMethod) + let versions = getTagsListRemote(url, downMethod).getVersionList() + if versions.len > 0: + getLatestByTag: + display("Cloning", "latest tagged version: " & latest.tag, + priority = MediumPriority) + doClone(downMethod, url, downloadDir, latest.tag, + onlyTip = not options.forceFullClone) + else: + # If no commits have been tagged on the repo we just clone HEAD. + doClone(downMethod, url, downloadDir) # Grab HEAD. + of DownloadMethod.hg: + doClone(downMethod, url, downloadDir, onlyTip = not options.forceFullClone) + result = getHeadName(downMethod) + let versions = getTagsList(downloadDir, downMethod).getVersionList() + + if versions.len > 0: + getLatestByTag: + display("Switching", "to latest tagged version: " & latest.tag, + priority = MediumPriority) + doCheckout(downMethod, downloadDir, latest.tag) + +proc downloadPkg*(url: string, verRange: VersionRange, + downMethod: DownloadMethod, + subdir: string, + options: Options, + downloadPath = ""): (string, Version) = + ## Downloads the repository as specified by ``url`` and ``verRange`` using + ## the download method specified. + ## + ## If `downloadPath` isn't specified a location in /tmp/ will be used. + ## + ## Returns the directory where it was downloaded (subdir is appended) and + ## the concrete version which was downloaded. + let downloadDir = + if downloadPath == "": + (getNimbleTempDir() / getDownloadDirName(url, verRange)) + else: + downloadPath + + createDir(downloadDir) + var modUrl = + if url.startsWith("git://") and options.config.cloneUsingHttps: + "https://" & url[6 .. ^1] + else: url + + # Fixes issue #204 + # github + https + trailing url slash causes a + # checkout/ls-remote to fail with Repository not found + if modUrl.contains("github.com") and modUrl.endswith("/"): + modUrl = modUrl[0 .. ^2] + + if subdir.len > 0: + display("Downloading", "$1 using $2 (subdir is '$3')" % + [modUrl, $downMethod, subdir], + priority = HighPriority) + else: + display("Downloading", "$1 using $2" % [modUrl, $downMethod], + priority = HighPriority) + result = ( + downloadDir / subdir, + doDownload(modUrl, downloadDir, verRange, downMethod, options) + ) + + if verRange.kind != verSpecial: + ## Makes sure that the downloaded package's version satisfies the requested + ## version range. + let pkginfo = getPkgInfo(result[0], options) + if pkginfo.version.newVersion notin verRange: + raise newException(NimbleError, + "Downloaded package's version does not satisfy requested version " & + "range: wanted $1 got $2." % + [$verRange, $pkginfo.version]) + +proc echoPackageVersions*(pkg: Package) = + let downMethod = pkg.downloadMethod.getDownloadMethod() + case downMethod + of DownloadMethod.git: + try: + let versions = getTagsListRemote(pkg.url, downMethod).getVersionList() + if versions.len > 0: + let sortedVersions = toSeq(values(versions)) + echo(" versions: " & join(sortedVersions, ", ")) + else: + echo(" versions: (No versions tagged in the remote repository)") + except OSError: + echo(getCurrentExceptionMsg()) + of DownloadMethod.hg: + echo(" versions: (Remote tag retrieval not supported by " & + pkg.downloadMethod & ")") + +proc packageVersionsJson*(pkg: Package): JsonNode = + result = newJArray() + let downMethod = pkg.downloadMethod.getDownloadMethod() + try: + case downMethod + of DownloadMethod.git: + let versions = getTagsListRemote(pkg.url, downMethod).getVersionList() + for v in values(versions): + result.add %v + of DownloadMethod.hg: + discard + except: + result = %* { "exception": getCurrentExceptionMsg() } + +when isMainModule: + # Test version sorting + block: + let data = @["v9.0.0-taeyeon", "v9.0.1-jessica", "v9.2.0-sunny", + "v9.4.0-tiffany", "v9.4.2-hyoyeon"] + let expected = toOrderedTable[Version, string]({ + newVersion("9.4.2-hyoyeon"): "v9.4.2-hyoyeon", + newVersion("9.4.0-tiffany"): "v9.4.0-tiffany", + newVersion("9.2.0-sunny"): "v9.2.0-sunny", + newVersion("9.0.1-jessica"): "v9.0.1-jessica", + newVersion("9.0.0-taeyeon"): "v9.0.0-taeyeon" + }) + doAssert expected == getVersionList(data) + + echo("Everything works!") diff --git a/src/nimblepkg/init.nim b/src/nimblepkg/init.nim new file mode 100644 index 0000000..8e7c33b --- /dev/null +++ b/src/nimblepkg/init.nim @@ -0,0 +1,184 @@ +import os, strutils + +import ./cli, ./tools + +type + PkgInitInfo* = tuple + pkgName: string + pkgVersion: string + pkgAuthor: string + pkgDesc: string + pkgLicense: string + pkgBackend: string + pkgSrcDir: string + pkgNimDep: string + pkgType: string + +proc writeExampleIfNonExistent(file: string, content: string) = + if not existsFile(file): + writeFile(file, content) + else: + display("Info:", "File " & file & " already exists, did not write " & + "example code", priority = HighPriority) + +proc createPkgStructure*(info: PkgInitInfo, pkgRoot: string) = + # Create source directory + createDirD(pkgRoot / info.pkgSrcDir) + + # Initialise the source code directories and create some example code. + var nimbleFileOptions = "" + case info.pkgType + of "binary": + let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim") + writeExampleIfNonExistent(mainFile, +""" +# This is just an example to get you started. A typical binary package +# uses this file as the main entry point of the application. + +when isMainModule: + echo("Hello, World!") +""" + ) + nimbleFileOptions.add("bin = @[\"$1\"]\n" % info.pkgName) + of "library": + let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim") + writeExampleIfNonExistent(mainFile, +""" +# This is just an example to get you started. A typical library package +# exports the main API in this file. Note that you cannot rename this file +# but you can remove it if you wish. + +proc add*(x, y: int): int = + ## Adds two files together. + return x + y +""" + ) + + createDirD(pkgRoot / info.pkgSrcDir / info.pkgName) + let submodule = pkgRoot / info.pkgSrcDir / info.pkgName / + "submodule".addFileExt("nim") + writeExampleIfNonExistent(submodule, +""" +# This is just an example to get you started. Users of your library will +# import this file by writing ``import $1/submodule``. Feel free to rename or +# remove this file altogether. You may create additional modules alongside +# this file as required. + +type + Submodule* = object + name*: string + +proc initSubmodule*(): Submodule = + ## Initialises a new ``Submodule`` object. + Submodule(name: "Anonymous") +""" % info.pkgName + ) + of "hybrid": + let mainFile = pkgRoot / info.pkgSrcDir / info.pkgName.changeFileExt("nim") + writeExampleIfNonExistent(mainFile, +""" +# This is just an example to get you started. A typical hybrid package +# uses this file as the main entry point of the application. + +import $1pkg/submodule + +when isMainModule: + echo(getWelcomeMessage()) +""" % info.pkgName + ) + + let pkgSubDir = pkgRoot / info.pkgSrcDir / info.pkgName & "pkg" + createDirD(pkgSubDir) + let submodule = pkgSubDir / "submodule".addFileExt("nim") + writeExampleIfNonExistent(submodule, +""" +# This is just an example to get you started. Users of your hybrid library will +# import this file by writing ``import $1pkg/submodule``. Feel free to rename or +# remove this file altogether. You may create additional modules alongside +# this file as required. + +proc getWelcomeMessage*(): string = "Hello, World!" +""" % info.pkgName + ) + nimbleFileOptions.add("installExt = @[\"nim\"]\n") + nimbleFileOptions.add("bin = @[\"$1\"]\n" % info.pkgName) + else: + assert false, "Invalid package type specified." + + let pkgTestDir = "tests" + # Create test directory + case info.pkgType + of "binary": + discard + of "hybrid", "library": + let pkgTestPath = pkgRoot / pkgTestDir + createDirD(pkgTestPath) + + writeFile(pkgTestPath / "config".addFileExt("nims"), + "switch(\"path\", \"$$projectDir/../$#\")" % info.pkgSrcDir + ) + + if info.pkgType == "library": + writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"), +""" +# This is just an example to get you started. You may wish to put all of your +# tests into a single file, or separate them into multiple `test1`, `test2` +# etc. files (better names are recommended, just make sure the name starts with +# the letter 't'). +# +# To run these tests, simply execute `nimble test`. + +import unittest + +import $1 +test "can add": + check add(5, 5) == 10 +""" % info.pkgName + ) + else: + writeExampleIfNonExistent(pkgTestPath / "test1".addFileExt("nim"), +""" +# This is just an example to get you started. You may wish to put all of your +# tests into a single file, or separate them into multiple `test1`, `test2` +# etc. files (better names are recommended, just make sure the name starts with +# the letter 't'). +# +# To run these tests, simply execute `nimble test`. + +import unittest + +import $1pkg/submodule +test "correct welcome": + check getWelcomeMessage() == "Hello, World!" +""" % info.pkgName + ) + else: + assert false, "Invalid package type specified." + + # Write the nimble file + let nimbleFile = pkgRoot / info.pkgName.changeFileExt("nimble") + # Only write backend if it isn't "c" + var pkgBackend = "" + if (info.pkgBackend != "c"): + pkgBackend = "backend = " & info.pkgbackend.escape() + writeFile(nimbleFile, """# Package + +version = $# +author = "$#" +description = "$#" +license = $# +srcDir = $# +$# +$# + +# Dependencies + +requires "nim >= $#" +""" % [ + info.pkgVersion.escape(), info.pkgAuthor.replace("\"", "\\\""), info.pkgDesc.replace("\"", "\\\""), + info.pkgLicense.escape(), info.pkgSrcDir.escape(), nimbleFileOptions, + pkgBackend, info.pkgNimDep + ] + ) + + display("Info:", "Nimble file created successfully", priority=MediumPriority) diff --git a/src/nimblepkg/jsonhelpers.nim b/src/nimblepkg/jsonhelpers.nim new file mode 100644 index 0000000..03c97b4 --- /dev/null +++ b/src/nimblepkg/jsonhelpers.nim @@ -0,0 +1,140 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import json + +proc newJObjectIfKeyNotExists(obj: JsonNode, key: string): JsonNode = + assert obj.kind == JObject + if not obj.hasKey(key): + let newObj = newJObject() + obj.add(key, newObj) + return newObj + else: + return obj[key] + +proc addIfNotExist*(obj: JsonNode, keys: varargs[string], + val: JsonNode): JsonNode = + # If the path in the `obj` json tree described by `keys` does not exist create + # it, add the node `val` to it and return the added node, otherwise return the + # value of the existing object at the end of the path. + + assert obj.kind == JObject + var obj = obj + for i in 0 ..< keys.len() - 1: + obj = obj.newJObjectIfKeyNotExists(keys[i]) + if not obj.hasKey(keys[^1]): + obj.add(keys[^1], val) + return val + else: + return obj[keys[^1]] + +proc cleanUpEmptyObjects*(obj: JsonNode): JsonNode = + if obj.kind == JObject: + result = newJObject() + for key, value in obj: + var newValue = cleanUpEmptyObjects(value) + if newValue.kind notin {JObject, JArray} or newValue.len != 0: + result.add(key, newValue) + elif obj.kind == JArray: + result = newJArray() + for value in obj: + var newValue = cleanUpEmptyObjects(value) + if newValue.kind notin {JObject, JArray} or newValue.len != 0: + result.add(newValue) + else: + result = obj + +when isMainModule: + import unittest + + test "bewJObjectIfKeyNotExists": + proc testProc(testedJson, key, expectedResult: string) = + let testedJson = parseJson(testedJson) + let expectedResult = parseJson(expectedResult) + let actualResult = newJObjectIfKeyNotExists(testedJson, key) + check actualResult == expectedResult + + testProc("{}", "key", "{}") + testProc("{ \"key1\": \"value1\", \"key2\": {} }", "key3", "{}") + testProc("{ \"key1\": \"value1\", \"key2\": {} }", "key1", "\"value1\"") + testProc("{ \"key1\": \"value1\", \"key2\": { \"key3\": [ 2, 3, 5] } }", + "key2", "{ \"key3\": [ 2, 3, 5] }") + + test "addIfNotExist": + proc testProc(testedJson: string, keys: varargs[string], + jsonToAdd, expectedResult, expectedEndObject: string) = + let expectedResult = parseJson(expectedResult) + let jsonToAdd = parseJson(jsonToAdd) + let actualResult = parseJson(testedJson) + let expectedEndObject = parseJson(expectedEndObject) + let addedOrOldNode =actualResult.addIfNotExist(keys, jsonToAdd) + check actualResult == expectedResult + check addedOrOldNode == expectedEndObject + + testProc("{}", "key", "[]", "{ \"key\": [] }", "[]") + testProc("{}", "key1", "key2", "{}", "{ \"key1\": { \"key2\": {} } }", "{}") + testProc("{ \"key\": {} }", "key", "[]", "{ \"key\": {} }", "{}") + testProc("{ \"key1\": { \"key2\": {} } }", "key1", "key2", "[1, 2, 3]", + "{ \"key1\": { \"key2\": {} } }", "{}") + testProc("{ \"key1\": {}, \"key2\": {} }", "key2", "key3", + "{ \"key4\": [1] }", + "{ \"key1\": {}, \"key2\": { \"key3\": { \"key4\": [1] } } }", + "{ \"key4\": [1] }") + + test "cleanUpEmptyObjects": + proc testProc(testedJson, expectedJson: string) = + let testedJsonNode = parseJson(testedJson) + let expectedResult = parseJson(expectedJson) + let actualResult = cleanUpEmptyObjects(testedJsonNode) + check actualResult == expectedResult + + testProc("{}", "{}") + testProc("[]", "[]") + testProc("{ \"key\": \"value\" }", "{ \"key\": \"value\" }") + testProc("[ 3, 1415 ]", "[ 3, 1415 ]") + + testProc("{ \"key\": [ \"value1\", \"value2\" ] }", + "{ \"key\": [ \"value1\", \"value2\" ] }") + + testProc("{ \"key\": {} }", "{}") + testProc("[ [], [] ]", "[]") + testProc("[ { \"key1\": [ { \"key1.1\": [] } ] }, { \"key2\": [] } ]", "[]") + + testProc(""" { + "key1": { + "key1.1": "value1.1", + "key1.2": "value1.2" + }, + "key2": {}, + "key3": [ + { + "key3.1": "value3.1", + "key3.2": "value3.2" + }, + {}, + { + "key3.3": "value3.3" + }, + {} + ], + "key4": { + "key4.1": {}, + "key4.2": [] + }, + "key5": 5 + }""", """ { + "key1": { + "key1.1": "value1.1", + "key1.2": "value1.2" + }, + "key3": [ + { + "key3.1": "value3.1", + "key3.2": "value3.2" + }, + { + "key3.3": "value3.3" + }, + ], + "key5": 5 + }""") diff --git a/src/nimblepkg/lockfile.nim b/src/nimblepkg/lockfile.nim new file mode 100644 index 0000000..40f6125 --- /dev/null +++ b/src/nimblepkg/lockfile.nim @@ -0,0 +1,57 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import tables, os, json +import version, sha1hashes, packageinfotypes + +type + LockFileJsonKeys* = enum + lfjkVersion = "version" + lfjkPackages = "packages" + lfjkPkgVcsRevision = "vcsRevision" + lfjkTasks = "tasks" + +const + lockFileVersion = 2 + +proc initLockFileDep*: LockFileDep = + result = LockFileDep( + version: notSetVersion, + vcsRevision: notSetSha1Hash, + checksums: Checksums(sha1: notSetSha1Hash)) + +const + notSetLockFileDep* = initLockFileDep() + +proc writeLockFile*(fileName: string, packages: AllLockFileDeps) = + ## Saves lock file on the disk in topologically sorted order of the + ## dependencies. + + let mainJsonNode = %{ + $lfjkVersion: %lockFileVersion, + $lfjkPackages: %packages[noTask] + } + # Store task graph seperate + mainJsonNode[$lfjkTasks] = newJObject() + for task, deps in packages: + if task != noTask: + mainJsonNode[$lfjkTasks][task] = %deps + + var s = mainJsonNode.pretty + s.add '\n' + writeFile(fileName, s) + +proc readLockFile*(filePath: string): AllLockFileDeps = + {.warning[UnsafeDefault]: off.} + {.warning[ProveInit]: off.} + let data = parseFile(filePath) + result[noTask] = data[$lfjkPackages].to(LockFileDeps) + if $lfjkTasks in data: + for task, deps in data[$lfjkTasks]: + result[task] = deps.to(LockFileDeps) + {.warning[ProveInit]: on.} + {.warning[UnsafeDefault]: on.} + +proc getLockedDependencies*(lockFile: string): AllLockFileDeps = + if lockFile.fileExists: + result = lockFile.readLockFile diff --git a/src/nimblepkg/nim.cfg b/src/nimblepkg/nim.cfg new file mode 100644 index 0000000..d7edcd3 --- /dev/null +++ b/src/nimblepkg/nim.cfg @@ -0,0 +1,2 @@ +--path:"$nim/" +--path:"$lib/packages/docutils" \ No newline at end of file diff --git a/src/nimblepkg/nimbledatafile.nim b/src/nimblepkg/nimbledatafile.nim new file mode 100644 index 0000000..08181ff --- /dev/null +++ b/src/nimblepkg/nimbledatafile.nim @@ -0,0 +1,66 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import json, os, strformat +import common, options, jsonhelpers, version, cli + +type + NimbleDataJsonKeys* = enum + ndjkVersion = "version" + ndjkRevDep = "reverseDeps" + ndjkRevDepName = "name" + ndjkRevDepVersion = "version" + ndjkRevDepChecksum = "checksum" + ndjkRevDepPath = "path" + +const + nimbleDataFileName* = "nimbledata2.json" + nimbleDataFileVersion = 1 + +var isNimbleDataFileLoaded = false + +proc saveNimbleData(filePath: string, nimbleData: JsonNode) = + # TODO: This file should probably be locked. + if isNimbleDataFileLoaded: + writeFile(filePath, nimbleData.pretty) + displayInfo(&"Nimble data file \"{filePath}\" has been saved.", LowPriority) + +proc saveNimbleDataToDir(nimbleDir: string, nimbleData: JsonNode) = + saveNimbleData(nimbleDir / nimbleDataFileName, nimbleData) + +proc saveNimbleData*(options: Options) = + saveNimbleDataToDir(options.getNimbleDir(), options.nimbleData) + +proc newNimbleDataNode*(): JsonNode = + %{ $ndjkVersion: %nimbleDataFileVersion, $ndjkRevDep: newJObject() } + +proc removeDeadDevelopReverseDeps*(options: var Options) = + template revDeps: var JsonNode = options.nimbleData[$ndjkRevDep] + var hasDeleted = false + for name, versions in revDeps: + for version, hashSums in versions: + for hashSum, dependencies in hashSums: + for dep in dependencies: + if dep.hasKey($ndjkRevDepPath) and + not dep[$ndjkRevDepPath].str.dirExists: + dep.delete($ndjkRevDepPath) + hasDeleted = true + if hasDeleted: + options.nimbleData[$ndjkRevDep] = cleanUpEmptyObjects(revDeps) + +proc loadNimbleData*(options: var Options) = + let + nimbleDir = options.getNimbleDir() + fileName = nimbleDir / nimbleDataFileName + + if fileExists(fileName): + options.nimbleData = parseFile(fileName) + removeDeadDevelopReverseDeps(options) + displayInfo(&"Nimble data file \"{fileName}\" has been loaded.", + LowPriority) + else: + displayWarning(&"Nimble data file \"{fileName}\" is not found.", + LowPriority) + options.nimbleData = newNimbleDataNode() + + isNimbleDataFileLoaded = true diff --git a/src/nimblepkg/nimscriptapi.nim b/src/nimblepkg/nimscriptapi.nim new file mode 100644 index 0000000..e7e1bee --- /dev/null +++ b/src/nimblepkg/nimscriptapi.nim @@ -0,0 +1,204 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module is implicitly imported in NimScript .nimble files. + +import system except getCommand, setCommand, switch, `--` +import strformat, strutils, tables + +when not defined(nimscript): + import os + +var + packageName* = "" ## Set this to the package name. It + ## is usually not required to do that, nims' filename is + ## the default. + version*: string ## The package's version. + author*: string ## The package's author. + description*: string ## The package's description. + license*: string ## The package's license. + srcdir*: string ## The package's source directory. + binDir*: string ## The package's binary directory. + backend*: string ## The package's backend. + + skipDirs*, skipFiles*, skipExt*, installDirs*, installFiles*, + installExt*, bin*: seq[string] = @[] ## Nimble metadata. + requiresData*: seq[string] = @[] ## The package's dependencies. + + foreignDeps*: seq[string] = @[] ## The foreign dependencies. Only + ## exported for 'distros.nim'. + + beforeHooks: seq[string] = @[] + afterHooks: seq[string] = @[] + commandLineParams: seq[string] = @[] + flags: TableRef[string, seq[string]] + + command = "e" + project = "" + success = false + retVal = true + projectFile = "" + outFile = "" + +proc requires*(deps: varargs[string]) = + ## Call this to set the list of requirements of your Nimble + ## package. + for d in deps: requiresData.add(d) + +proc foreignDep*(deps: varargs[string]) = + ## Call this to set the list of external dependencies of your Nimble + ## package. + for d in deps: foreignDeps.add(d) + +proc getParams() = + # Called by nimscriptwrapper.nim:execNimscript() + # nim e --flags /full/path/to/file.nims /full/path/to/file.out action + for i in 2 .. paramCount(): + let + param = paramStr(i) + if param[0] != '-': + if projectFile.len == 0: + projectFile = param + elif outFile.len == 0: + outFile = param + else: + commandLineParams.add param.normalize + +proc getCommand*(): string = + return command + +proc setCommand*(cmd: string, prj = "") = + command = cmd + if prj.len != 0: + project = prj + +proc switch*(key: string, value="") = + if flags.isNil: + flags = newTable[string, seq[string]]() + + if flags.hasKey(key): + flags[key].add(value) + else: + flags[key] = @[value] + +template `--`*(key, val: untyped) = + switch(astToStr(key), strip astToStr(val)) + +template `--`*(key: untyped) = + switch(astToStr(key), "") + +template printIfLen(varName) = + if varName.len != 0: + result &= astToStr(varName) & ": \"\"\"" & varName & "\"\"\"\n" + +template printSeqIfLen(varName) = + if varName.len != 0: + result &= astToStr(varName) & ": \"" & varName.join(", ") & "\"\n" + +proc printPkgInfo(): string = + if backend.len == 0: + backend = "c" + + result = "[Package]\n" + if packageName.len != 0: + result &= "name: \"" & packageName & "\"\n" + printIfLen version + printIfLen author + printIfLen description + printIfLen license + printIfLen srcdir + printIfLen binDir + printIfLen backend + + printSeqIfLen skipDirs + printSeqIfLen skipFiles + printSeqIfLen skipExt + printSeqIfLen installDirs + printSeqIfLen installFiles + printSeqIfLen installExt + printSeqIfLen bin + printSeqIfLen beforeHooks + printSeqIfLen afterHooks + + if requiresData.len != 0 or foreignDeps.len != 0: + result &= "\n[Deps]\n" + if requiresData.len != 0: + result &= &"requires: \"{requiresData.join(\", \")}\"\n" + if foreignDeps.len != 0: + result &= &"foreignDeps: \"{foreignDeps.join(\", \")}\"\n" + +proc onExit*() = + if "printPkgInfo".normalize in commandLineParams: + if outFile.len != 0: + writeFile(outFile, printPkgInfo()) + else: + var + output = "" + output &= "\"success\": " & $success & ", " + output &= "\"command\": \"" & command & "\", " + if project.len != 0: + output &= "\"project\": \"" & project & "\", " + if not flags.isNil and flags.len != 0: + output &= "\"flags\": {" + for key, val in flags.pairs: + output &= "\"" & key & "\": [" + for v in val: + let v = if v.len > 0 and v[0] == '"': strutils.unescape(v) + else: v + output &= v.escape & ", " + output = output[0 .. ^3] & "], " + output = output[0 .. ^3] & "}, " + + output &= "\"retVal\": " & $retVal + + if outFile.len != 0: + writeFile(outFile, "{" & output & "}") + +# TODO: New release of Nim will move this `task` template under a +# `when not defined(nimble)`. This will allow us to override it in the future. +template task*(name: untyped; description: string; body: untyped): untyped = + ## Defines a task. Hidden tasks are supported via an empty description. + ## Example: + ## + ## .. code-block:: nim + ## task build, "default build is via the C backend": + ## setCommand "c" + proc `name Task`*() = body + + if commandLineParams.len == 0 or "help" in commandLineParams: + success = true + echo(astToStr(name), " ", description) + elif astToStr(name).normalize in commandLineParams: + success = true + `name Task`() + +template before*(action: untyped, body: untyped): untyped = + ## Defines a block of code which is evaluated before ``action`` is executed. + proc `action Before`*(): bool = + result = true + body + + beforeHooks.add astToStr(action) + + if (astToStr(action) & "Before").normalize in commandLineParams: + success = true + retVal = `action Before`() + +template after*(action: untyped, body: untyped): untyped = + ## Defines a block of code which is evaluated after ``action`` is executed. + proc `action After`*(): bool = + result = true + body + + afterHooks.add astToStr(action) + + if (astToStr(action) & "After").normalize in commandLineParams: + success = true + retVal = `action After`() + +proc getPkgDir*(): string = + ## Returns the package directory containing the .nimble file currently + ## being evaluated. + result = projectFile.rsplit(seps={'/', '\\', ':'}, maxsplit=1)[0] + +getParams() diff --git a/src/nimblepkg/nimscriptexecutor.nim b/src/nimblepkg/nimscriptexecutor.nim new file mode 100644 index 0000000..122c41c --- /dev/null +++ b/src/nimblepkg/nimscriptexecutor.nim @@ -0,0 +1,76 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import os, strutils, sets + +import packageparser, common, packageinfo, options, nimscriptwrapper, cli, + version + +proc execHook*(options: Options, hookAction: ActionType, before: bool): bool = + ## Returns whether to continue. + result = true + + # For certain commands hooks should not be evaluated. + if hookAction in noHookActions: + return + + var nimbleFile = "" + try: + nimbleFile = findNimbleFile(getCurrentDir(), true) + except NimbleError: return true + # PackageInfos are cached so we can read them as many times as we want. + let pkgInfo = getPkgInfoFromFile(nimbleFile, options) + let actionName = + if hookAction == actionCustom: options.action.command + else: ($hookAction)[6 .. ^1] + let hookExists = + if before: actionName.normalize in pkgInfo.preHooks + else: actionName.normalize in pkgInfo.postHooks + if pkgInfo.isNimScript and hookExists: + let res = execHook(nimbleFile, actionName, before, options) + if res.success: + result = res.retVal + +proc execCustom*(options: Options, + execResult: var ExecutionResult[bool], + failFast = true): bool = + ## Executes the custom command using the nimscript backend. + ## + ## If failFast is true then exceptions will be raised when something is wrong. + ## Otherwise this function will just return false. + + # Custom command. Attempt to call a NimScript task. + let nimbleFile = findNimbleFile(getCurrentDir(), true) + if not nimbleFile.isNimScript(options) and failFast: + writeHelp() + + execResult = execTask(nimbleFile, options.action.command, options) + if not execResult.success: + if not failFast: + return + raiseNimbleError(msg = "Could not find task $1 in $2" % + [options.action.command, nimbleFile], + hint = "Run `nimble --help` and/or `nimble tasks` for" & + " a list of possible commands.") + + if execResult.command.normalize == "nop": + display("Warning:", "Using `setCommand 'nop'` is not necessary.", Warning, + HighPriority) + return + + if not execHook(options, actionCustom, false): + return + + return true + +proc getOptionsForCommand*(execResult: ExecutionResult, + options: Options): Options = + ## Creates an Options object for the requested command. + var newOptions = options.briefClone() + parseCommand(execResult.command, newOptions) + for arg in execResult.arguments: + parseArgument(arg, newOptions) + for flag, vals in execResult.flags: + for val in vals: + parseFlag(flag, val, newOptions) + return newOptions diff --git a/src/nimblepkg/nimscriptwrapper.nim b/src/nimblepkg/nimscriptwrapper.nim new file mode 100644 index 0000000..805f7de --- /dev/null +++ b/src/nimblepkg/nimscriptwrapper.nim @@ -0,0 +1,216 @@ +# Copyright (C) Andreas Rumpf. All rights reserved. +# BSD License. Look at license.txt for more info. + +## Implements the new configuration system for Nimble. Uses Nim as a +## scripting language. + +import hashes, json, os, strutils, tables, times, osproc, strtabs + +import version, options, cli, tools + +type + Flags = TableRef[string, seq[string]] + ExecutionResult*[T] = object + success*: bool + command*: string + arguments*: seq[string] + flags*: Flags + retVal*: T + stdout*: string + +const + internalCmd = "e" + nimscriptApi = staticRead("nimscriptapi.nim") + printPkgInfo = "printPkgInfo" + +proc isCustomTask(actionName: string, options: Options): bool = + options.action.typ == actionCustom and actionName != printPkgInfo + +proc needsLiveOutput(actionName: string, options: Options, isHook: bool): bool = + let isCustomTask = isCustomTask(actionName, options) + return isCustomTask or isHook or actionName == "" + +proc writeExecutionOutput(data: string) = + # TODO: in the future we will likely want this to be live, users will + # undoubtedly be doing loops and other crazy things in their top-level + # Nimble files. + display("Info", data) + +proc execNimscript( + nimsFile, projectDir, actionName: string, options: Options, isHook: bool +): tuple[output: string, exitCode: int, stdout: string] = + let + nimsFileCopied = getTempDir() / nimsFile.splitFile().name & "_" & getProcessId() & ".nims" + outFile = getNimbleTempDir() & ".out" + + let + isScriptResultCopied = + nimsFileCopied.fileExists() and + nimsFileCopied.getLastModificationTime() >= nimsFile.getLastModificationTime() + + if not isScriptResultCopied: + nimsFile.copyFile(nimsFileCopied) + + defer: + # Only if copied in this invocation, allows recursive calls of nimble + if not isScriptResultCopied and options.shouldRemoveTmp(nimsFileCopied): + nimsFileCopied.removeFile() + + var cmd = ( + "nim e $# -p:$# $# $# $#" % [ + "--hints:off --verbosity:0", + (getTempDir() / "nimblecache").quoteShell, + nimsFileCopied.quoteShell, + outFile.quoteShell, + actionName + ] + ).strip() + + let isCustomTask = isCustomTask(actionName, options) + if isCustomTask: + for i in options.action.arguments: + cmd &= " " & i.quoteShell() + for key, val in options.action.flags.pairs(): + cmd &= " $#$#" % [if key.len == 1: "-" else: "--", key] + if val.len != 0: + cmd &= ":" & val.quoteShell() + + displayDebug("Executing " & cmd) + + if needsLiveOutput(actionName, options, isHook): + result.exitCode = execCmd(cmd) + else: + # We want to capture any possible errors when parsing a .nimble + # file's metadata. See #710. + (result.stdout, result.exitCode) = execCmdEx(cmd) + if outFile.fileExists(): + result.output = outFile.readFile() + if options.shouldRemoveTmp(outFile): + discard outFile.tryRemoveFile() + +proc getNimsFile(scriptName: string, options: Options): string = + let + cacheDir = getTempDir() / "nimblecache" + shash = $scriptName.parentDir().hash().abs() + prjCacheDir = cacheDir / scriptName.splitFile().name & "_" & shash + nimscriptApiFile = cacheDir / "nimscriptapi.nim" + + result = prjCacheDir / scriptName.extractFilename().changeFileExt ".nims" + + let + iniFile = result.changeFileExt(".ini") + + isNimscriptApiCached = + nimscriptApiFile.fileExists() and nimscriptApiFile.getLastModificationTime() > + getAppFilename().getLastModificationTime() + + isScriptResultCached = + isNimscriptApiCached and result.fileExists() and result.getLastModificationTime() > + scriptName.getLastModificationTime() + + if not isNimscriptApiCached: + createDir(cacheDir) + writeFile(nimscriptApiFile, nimscriptApi) + + if not isScriptResultCached: + createDir(result.parentDir()) + writeFile(result, """ +import system except getCommand, setCommand, switch, `--`, + packageName, version, author, description, license, srcDir, binDir, backend, + skipDirs, skipFiles, skipExt, installDirs, installFiles, installExt, bin, foreignDeps, + requires, task, packageName +""" & + "import nimscriptapi, strutils\n" & scriptName.readFile() & "\nonExit()\n") + discard tryRemoveFile(iniFile) + +proc getIniFile*(scriptName: string, options: Options): string = + let + nimsFile = getNimsFile(scriptName, options) + + result = nimsFile.changeFileExt(".ini") + + let + isIniResultCached = + result.fileExists() and result.getLastModificationTime() > + scriptName.getLastModificationTime() + + if not isIniResultCached: + let (output, exitCode, stdout) = execNimscript( + nimsFile, scriptName.parentDir(), printPkgInfo, options, isHook=false + ) + + if exitCode == 0 and output.len != 0: + result.writeFile(output) + stdout.writeExecutionOutput() + else: + raise newException(NimbleError, stdout & "\nprintPkgInfo() failed") + +proc execScript( + scriptName, actionName: string, options: Options, isHook: bool +): ExecutionResult[bool] = + let nimsFile = getNimsFile(scriptName, options) + + let (output, exitCode, stdout) = + execNimscript( + nimsFile, scriptName.parentDir(), actionName, options, isHook + ) + + if exitCode != 0: + let errMsg = + if stdout.len != 0: + stdout + else: + "Exception raised during nimble script execution" + raise newException(NimbleError, errMsg) + + let + j = + if output.len != 0: + parseJson(output) + else: + parseJson("{}") + + result.flags = newTable[string, seq[string]]() + result.success = j{"success"}.getBool() + result.command = j{"command"}.getStr() + if "project" in j: + result.arguments.add j["project"].getStr() + if "flags" in j: + for flag, vals in j["flags"].pairs: + result.flags[flag] = @[] + for val in vals.items(): + result.flags[flag].add val.getStr() + result.retVal = j{"retVal"}.getBool() + + stdout.writeExecutionOutput() + +proc execTask*(scriptName, taskName: string, + options: Options): ExecutionResult[bool] = + ## Executes the specified task in the specified script. + ## + ## `scriptName` should be a filename pointing to the nimscript file. + display("Executing", "task $# in $#" % [taskName, scriptName], + priority = HighPriority) + + result = execScript(scriptName, taskName, options, isHook=false) + +proc execHook*(scriptName, actionName: string, before: bool, + options: Options): ExecutionResult[bool] = + ## Executes the specified action's hook. Depending on ``before``, either + ## the "before" or the "after" hook. + ## + ## `scriptName` should be a filename pointing to the nimscript file. + let hookName = + if before: actionName.toLowerAscii & "Before" + else: actionName.toLowerAscii & "After" + display("Attempting", "to execute hook $# in $#" % [hookName, scriptName], + priority = MediumPriority) + + result = execScript(scriptName, hookName, options, isHook=true) + +proc hasTaskRequestedCommand*(execResult: ExecutionResult): bool = + ## Determines whether the last executed task used ``setCommand`` + return execResult.command != internalCmd + +proc listTasks*(scriptName: string, options: Options) = + discard execScript(scriptName, "", options, isHook=false) diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim new file mode 100644 index 0000000..2c5671e --- /dev/null +++ b/src/nimblepkg/options.nim @@ -0,0 +1,534 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import json, strutils, os, parseopt, strtabs, uri, tables, terminal +import sequtils, sugar +import std/options as std_opt +from httpclient import Proxy, newProxy + +import config, version, common, cli + +type + Options* = object + forcePrompts*: ForcePrompt + depsOnly*: bool + uninstallRevDeps*: bool + queryVersions*: bool + queryInstalled*: bool + jsonOutput*:bool + nimbleDir*: string + verbosity*: cli.Priority + action*: Action + config*: Config + nimbleData*: JsonNode ## Nimbledata.json + pkgInfoCache*: TableRef[string, PackageInfo] + showHelp*: bool + showVersion*: bool + noColor*: bool + disableValidation*: bool + continueTestsOnFailure*: bool + ## Whether packages' repos should always be downloaded with their history. + forceFullClone*: bool + # Temporary storage of flags that have not been captured by any specific Action. + unknownFlags*: seq[(CmdLineKind, string, string)] + + ActionType* = enum + actionNil, actionRefresh, actionInit, actionDump, actionPublish, + actionInstall, actionSearch, + actionList, actionBuild, actionPath, actionUninstall, actionCompile, + actionDoc, actionCustom, actionTasks, actionDevelop, actionCheck, + actionRun + + Action* = object + case typ*: ActionType + of actionNil, actionList, actionPublish, actionTasks, actionCheck: nil + of actionRefresh: + optionalURL*: string # Overrides default package list. + of actionInstall, actionPath, actionUninstall, actionDevelop: + packages*: seq[PkgTuple] # Optional only for actionInstall + # and actionDevelop. + passNimFlags*: seq[string] + of actionSearch: + search*: seq[string] # Search string. + of actionInit, actionDump: + projName*: string + of actionCompile, actionDoc, actionBuild: + file*: string + backend*: string + compileOptions: seq[string] + of actionRun: + runFile: string + compileFlags: seq[string] + runFlags*: seq[string] + of actionCustom: + command*: string + arguments*: seq[string] + flags*: StringTableRef + +const + help* = """ +Usage: nimble COMMAND [opts] + +Commands: + install [pkgname, ...] Installs a list of packages. + [-d, --depsOnly] Install only dependencies. + [-p, --passNim] Forward specified flag to compiler. + develop [pkgname, ...] Clones a list of packages for development. + Symlinks the cloned packages or any package + in the current working directory. + check Verifies the validity of a package in the + current working directory. + init [pkgname] Initializes a new Nimble project in the + current directory or if a name is provided a + new directory of the same name. + publish Publishes a package on nim-lang/packages. + The current working directory needs to be the + toplevel directory of the Nimble package. + uninstall [pkgname, ...] Uninstalls a list of packages. + [-i, --inclDeps] Uninstall package and dependent package(s). + build [opts, ...] [bin] Builds a package. + run [opts, ...] bin Builds and runs a package. + A binary name needs + to be specified after any compilation options, + any flags after the binary name are passed to + the binary when it is run. + c, cc, js [opts, ...] f.nim Builds a file inside a package. Passes options + to the Nim compiler. + test Compiles and executes tests + [-c, --continue] Don't stop execution on a failed test. + doc, doc2 [opts, ...] f.nim Builds documentation for a file inside a + package. Passes options to the Nim compiler. + refresh [url] Refreshes the package list. A package list URL + can be optionally specified. + search pkg/tag Searches for a specified package. Search is + performed by tag and by name. + [--json] Format output as JSON. + [--ver] Query remote server for package version. + list Lists all packages. + [--json] Format output as JSON. + [--ver] Query remote server for package version. + [-i, --installed] Lists all installed packages. + tasks Lists the tasks specified in the Nimble + package's Nimble file. + path pkgname ... Shows absolute path to the installed packages + specified. + dump [pkgname] Outputs Nimble package information for + external tools. The argument can be a + .nimble file, a project directory or + the name of an installed package. + + +Options: + -h, --help Print this help message. + -v, --version Print version information. + -y, --accept Accept all interactive prompts. + -n, --reject Reject all interactive prompts. + --json Produce JSON formatted output when + searching or listing packages + --ver Query remote server for package version + information when searching or listing packages + --nimbleDir:dirname Set the Nimble directory. + --verbose Show all non-debug output. + --debug Show all output including debug messages. + --noColor Don't colorise output. + +For more information read the Github readme: + https://github.com/nim-lang/nimble#readme +""" + +const noHookActions* = {actionCheck} + +proc writeHelp*(quit=true) = + echo(help) + if quit: + raise NimbleQuit(msg: "") + +proc writeVersion*() = + echo("nimble v$# compiled at $# $#" % + [nimbleVersion, CompileDate, CompileTime]) + const execResult = gorgeEx("git rev-parse HEAD") + when execResult[0].len > 0 and execResult[1] == QuitSuccess: + echo "git hash: ", execResult[0] + else: + {.warning: "Couldn't determine GIT hash: " & execResult[0].} + echo "git hash: couldn't determine git hash" + raise NimbleQuit(msg: "") + +proc parseActionType*(action: string): ActionType = + case action.normalize() + of "install": + result = actionInstall + of "path": + result = actionPath + of "build": + result = actionBuild + of "run": + result = actionRun + of "c", "compile", "js", "cpp", "cc": + result = actionCompile + of "doc", "doc2": + result = actionDoc + of "init": + result = actionInit + of "dump": + result = actionDump + of "update", "refresh": + result = actionRefresh + of "search": + result = actionSearch + of "list": + result = actionList + of "uninstall", "remove", "delete", "del", "rm": + result = actionUninstall + of "publish": + result = actionPublish + of "tasks": + result = actionTasks + of "develop": + result = actionDevelop + of "check": + result = actionCheck + else: + result = actionCustom + +proc initAction*(options: var Options, key: string) = + ## Intialises `options.actions` fields based on `options.actions.typ` and + ## `key`. + let keyNorm = key.normalize() + case options.action.typ + of actionInstall, actionPath, actionDevelop, actionUninstall: + options.action.packages = @[] + options.action.passNimFlags = @[] + of actionCompile, actionDoc, actionBuild: + options.action.compileOptions = @[] + options.action.file = "" + if keyNorm == "c" or keyNorm == "compile": options.action.backend = "" + else: options.action.backend = keyNorm + of actionInit: + options.action.projName = "" + of actionDump: + options.action.projName = "" + options.forcePrompts = forcePromptYes + of actionRefresh: + options.action.optionalURL = "" + of actionSearch: + options.action.search = @[] + of actionCustom: + options.action.command = key + options.action.arguments = @[] + options.action.flags = newStringTable() + of actionPublish, actionList, actionTasks, actionCheck, actionRun, + actionNil: discard + +proc prompt*(options: Options, question: string): bool = + ## Asks an interactive question and returns the result. + ## + ## The proc will return immediately without asking the user if the global + ## forcePrompts has a value different than dontForcePrompt. + return prompt(options.forcePrompts, question) + +proc promptCustom*(options: Options, question, default: string): string = + ## Asks an interactive question and returns the result. + ## + ## The proc will return "default" without asking the user if the global + ## forcePrompts is forcePromptYes. + return promptCustom(options.forcePrompts, question, default) + +proc promptList*(options: Options, question: string, args: openarray[string]): string = + ## Asks an interactive question and returns the result. + ## + ## The proc will return one of the provided args. If not prompting the first + ## options is selected. + return promptList(options.forcePrompts, question, args) + +proc getNimbleDir*(options: Options): string = + result = options.config.nimbleDir + if options.nimbleDir.len != 0: + # --nimbleDir: takes priority... + result = options.nimbleDir + else: + # ...followed by the environment variable. + let env = getEnv("NIMBLE_DIR") + if env.len != 0: + display("Warning:", "Using the environment variable: NIMBLE_DIR='" & + env & "'", Warning) + result = env + + return expandTilde(result) + +proc getPkgsDir*(options: Options): string = + options.getNimbleDir() / "pkgs" + +proc getBinDir*(options: Options): string = + options.getNimbleDir() / "bin" + +proc parseCommand*(key: string, result: var Options) = + result.action = Action(typ: parseActionType(key)) + initAction(result, key) + +proc parseArgument*(key: string, result: var Options) = + case result.action.typ + of actionNil: + assert false + of actionInstall, actionPath, actionDevelop, actionUninstall: + # Parse pkg@verRange + if '@' in key: + let i = find(key, '@') + let (pkgName, pkgVer) = (key[0 .. i-1], key[i+1 .. key.len-1]) + if pkgVer.len == 0: + raise newException(NimbleError, "Version range expected after '@'.") + result.action.packages.add((pkgName, pkgVer.parseVersionRange())) + else: + result.action.packages.add((key, VersionRange(kind: verAny))) + of actionRefresh: + result.action.optionalURL = key + of actionSearch: + result.action.search.add(key) + of actionInit, actionDump: + if result.action.projName != "": + raise newException( + NimbleError, "Can only perform this action on one package at a time." + ) + result.action.projName = key + of actionCompile, actionDoc: + result.action.file = key + of actionList, actionPublish: + result.showHelp = true + of actionBuild: + result.action.file = key + of actionRun: + if result.action.runFile.len == 0: + result.action.runFile = key + else: + result.action.runFlags.add(key) + of actionCustom: + result.action.arguments.add(key) + else: + discard + +proc getFlagString(kind: CmdLineKind, flag, val: string): string = + let prefix = + case kind + of cmdShortOption: "-" + of cmdLongOption: "--" + else: "" + if val == "": + return prefix & flag + else: + return prefix & flag & ":" & val + +proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = + + let f = flag.normalize() + + # Global flags. + var isGlobalFlag = true + case f + of "help", "h": result.showHelp = true + of "version", "v": result.showVersion = true + of "accept", "y": result.forcePrompts = forcePromptYes + of "reject", "n": result.forcePrompts = forcePromptNo + of "nimbledir": result.nimbleDir = val + of "verbose": result.verbosity = LowPriority + of "debug": result.verbosity = DebugPriority + of "nocolor": result.noColor = true + of "disablevalidation": result.disableValidation = true + else: isGlobalFlag = false + + var wasFlagHandled = true + # Action-specific flags. + case result.action.typ + of actionSearch, actionList: + case f + of "installed", "i": + result.queryInstalled = true + of "ver": + result.queryVersions = true + of "json": + result.jsonOutput = true + else: + wasFlagHandled = false + of actionInstall: + case f + of "depsonly", "d": + result.depsOnly = true + of "passnim", "p": + result.action.passNimFlags.add(val) + else: + wasFlagHandled = false + of actionUninstall: + case f + of "incldeps", "i": + result.uninstallRevDeps = true + else: + wasFlagHandled = false + of actionCompile, actionDoc, actionBuild: + if not isGlobalFlag: + result.action.compileOptions.add(getFlagString(kind, flag, val)) + of actionRun: + result.action.runFlags.add(getFlagString(kind, flag, val)) + of actionCustom: + if result.action.command.normalize == "test": + if f == "continue" or f == "c": + result.continueTestsOnFailure = true + result.action.flags[flag] = val + else: + wasFlagHandled = false + + if not wasFlagHandled and not isGlobalFlag: + result.unknownFlags.add((kind, flag, val)) + +proc initOptions(): Options = + Options( + action: Action(typ: actionNil), + pkgInfoCache: newTable[string, PackageInfo](), + verbosity: HighPriority, + noColor: not isatty(stdout) + ) + +proc parseMisc(options: var Options) = + # Load nimbledata.json + let nimbledataFilename = options.getNimbleDir() / "nimbledata.json" + + if fileExists(nimbledataFilename): + try: + options.nimbleData = parseFile(nimbledataFilename) + except: + raise newException(NimbleError, "Couldn't parse nimbledata.json file " & + "located at " & nimbledataFilename) + else: + options.nimbleData = %{"reverseDeps": newJObject()} + +proc handleUnknownFlags(options: var Options) = + if options.action.typ == actionRun: + # ActionRun uses flags that come before the command as compilation flags + # and flags that come after as run flags. + options.action.compileFlags = + map(options.unknownFlags, x => getFlagString(x[0], x[1], x[2])) + options.unknownFlags = @[] + else: + # For everything else, handle the flags that came before the command + # normally. + let unknownFlags = options.unknownFlags + options.unknownFlags = @[] + for flag in unknownFlags: + parseFlag(flag[1], flag[2], options, flag[0]) + + # Any unhandled flags? + if options.unknownFlags.len > 0: + let flag = options.unknownFlags[0] + raise newException( + NimbleError, + "Unknown option: " & getFlagString(flag[0], flag[1], flag[2]) + ) + +proc parseCmdLine*(): Options = + result = initOptions() + + # Parse command line params first. A simple `--version` shouldn't require + # a config to be parsed. + for kind, key, val in getOpt(): + case kind + of cmdArgument: + if result.action.typ == actionNil: + parseCommand(key, result) + else: + parseArgument(key, result) + of cmdLongOption, cmdShortOption: + parseFlag(key, val, result, kind) + of cmdEnd: assert(false) # cannot happen + + handleUnknownFlags(result) + + # Set verbosity level. + setVerbosity(result.verbosity) + + # Set whether color should be shown. + setShowColor(not result.noColor) + + # Parse config. + result.config = parseConfig() + + # Parse other things, for example the nimbledata.json file. + parseMisc(result) + + if result.action.typ == actionNil and not result.showVersion: + result.showHelp = true + + if result.action.typ != actionNil and result.showVersion: + # We've got another command that should be handled. For example: + # nimble run foobar -v + result.showVersion = false + +proc getProxy*(options: Options): Proxy = + ## Returns ``nil`` if no proxy is specified. + var url = "" + if ($options.config.httpProxy).len > 0: + url = $options.config.httpProxy + else: + try: + if existsEnv("http_proxy"): + url = getEnv("http_proxy") + elif existsEnv("https_proxy"): + url = getEnv("https_proxy") + elif existsEnv("HTTP_PROXY"): + url = getEnv("HTTP_PROXY") + elif existsEnv("HTTPS_PROXY"): + url = getEnv("HTTPS_PROXY") + except ValueError: + display("Warning:", "Unable to parse proxy from environment: " & + getCurrentExceptionMsg(), Warning, HighPriority) + + if url.len > 0: + var parsed = parseUri(url) + if parsed.scheme.len == 0 or parsed.hostname.len == 0: + parsed = parseUri("http://" & url) + let auth = + if parsed.username.len > 0: parsed.username & ":" & parsed.password + else: "" + return newProxy($parsed, auth) + else: + return nil + +proc briefClone*(options: Options): Options = + ## Clones the few important fields and creates a new Options object. + var newOptions = initOptions() + newOptions.config = options.config + newOptions.nimbleData = options.nimbleData + newOptions.nimbleDir = options.nimbleDir + newOptions.forcePrompts = options.forcePrompts + newOptions.pkgInfoCache = options.pkgInfoCache + return newOptions + +proc shouldRemoveTmp*(options: Options, file: string): bool = + result = true + if options.verbosity <= DebugPriority: + let msg = "Not removing temporary path because of debug verbosity: " & file + display("Warning:", msg, Warning, MediumPriority) + return false + +proc getCompilationFlags*(options: var Options): var seq[string] = + case options.action.typ + of actionBuild, actionDoc, actionCompile: + return options.action.compileOptions + of actionRun: + return options.action.compileFlags + else: + assert false + +proc getCompilationFlags*(options: Options): seq[string] = + var opt = options + return opt.getCompilationFlags() + +proc getCompilationBinary*(options: Options): Option[string] = + case options.action.typ + of actionBuild, actionDoc, actionCompile: + let file = options.action.file.changeFileExt("") + if file.len > 0: + return some(file) + of actionRun: + let runFile = options.action.runFile.changeFileExt(ExeExt) + if runFile.len > 0: + return some(runFile) + else: + discard \ No newline at end of file diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim new file mode 100644 index 0000000..e12c83e --- /dev/null +++ b/src/nimblepkg/packageinfo.nim @@ -0,0 +1,577 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +# Stdlib imports +import system except TResult +import hashes, json, strutils, os, sets, tables, httpclient + +# Local imports +import version, tools, common, options, cli, config + +type + Package* = object ## Definition of package from packages.json. + # Required fields in a package. + name*: string + url*: string # Download location. + license*: string + downloadMethod*: string + description*: string + tags*: seq[string] # Even if empty, always a valid non nil seq. \ + # From here on, optional fields set to the empty string if not available. + version*: string + dvcsTag*: string + web*: string # Info url for humans. + alias*: string ## A name of another package, that this package aliases. + + MetaData* = object + url*: string + + NimbleLink* = object + nimbleFilePath*: string + packageDir*: string + +proc initPackageInfo*(path: string): PackageInfo = + result.myPath = path + result.specialVersion = "" + result.preHooks.init() + result.postHooks.init() + # reasonable default: + result.name = path.splitFile.name + result.version = "" + result.author = "" + result.description = "" + result.license = "" + result.skipDirs = @[] + result.skipFiles = @[] + result.skipExt = @[] + result.installDirs = @[] + result.installFiles = @[] + result.installExt = @[] + result.requires = @[] + result.foreignDeps = @[] + result.bin = @[] + result.srcDir = "" + result.binDir = "" + result.backend = "c" + +proc toValidPackageName*(name: string): string = + result = "" + for c in name: + case c + of '_', '-': + if result[^1] != '_': result.add('_') + of AllChars - IdentChars - {'-'}: discard + else: result.add(c) + +proc getNameVersion*(pkgpath: string): tuple[name, version: string] = + ## Splits ``pkgpath`` in the format ``/home/user/.nimble/pkgs/package-0.1`` + ## into ``(packagea, 0.1)`` + ## + ## Also works for file paths like: + ## ``/home/user/.nimble/pkgs/package-0.1/package.nimble`` + if pkgPath.splitFile.ext in [".nimble", ".nimble-link", ".babel"]: + return getNameVersion(pkgPath.splitPath.head) + + result.name = "" + result.version = "" + let tail = pkgpath.splitPath.tail + + const specialSeparator = "-#" + var sepIdx = tail.find(specialSeparator) + if sepIdx == -1: + sepIdx = tail.rfind('-') + + if sepIdx == -1: + result.name = tail + return + + result.name = tail[0 .. sepIdx - 1] + result.version = tail.substr(sepIdx + 1) + +proc optionalField(obj: JsonNode, name: string, default = ""): string = + ## Queries ``obj`` for the optional ``name`` string. + ## + ## Returns the value of ``name`` if it is a valid string, or aborts execution + ## if the field exists but is not of string type. If ``name`` is not present, + ## returns ``default``. + if hasKey(obj, name): + if obj[name].kind == JString: + return obj[name].str + else: + raise newException(NimbleError, "Corrupted packages.json file. " & name & + " field is of unexpected type.") + else: return default + +proc requiredField(obj: JsonNode, name: string): string = + ## Queries ``obj`` for the required ``name`` string. + ## + ## Aborts execution if the field does not exist or is of invalid json type. + result = optionalField(obj, name) + if result.len == 0: + raise newException(NimbleError, + "Package in packages.json file does not contain a " & name & " field.") + +proc fromJson(obj: JSonNode): Package = + ## Constructs a Package object from a JSON node. + ## + ## Aborts execution if the JSON node doesn't contain the required fields. + result.name = obj.requiredField("name") + if obj.hasKey("alias"): + result.alias = obj.requiredField("alias") + else: + result.alias = "" + result.version = obj.optionalField("version") + result.url = obj.optionalField("url") + result.downloadMethod = obj.optionalField("method") + result.dvcsTag = obj.optionalField("dvcs-tag") + result.license = obj.requiredField("license") + result.tags = @[] + for t in obj["tags"]: + result.tags.add(t.str) + result.description = obj.requiredField("description") + result.web = obj.optionalField("web") + +proc readMetaData*(path: string): MetaData = + ## Reads the metadata present in ``~/.nimble/pkgs/pkg-0.1/nimblemeta.json`` + var bmeta = path / "nimblemeta.json" + if not existsFile(bmeta): + result.url = "" + display("Warning:", "No nimblemeta.json file found in " & path, + Warning, HighPriority) + return + # TODO: Make this an error. + let cont = readFile(bmeta) + let jsonmeta = parseJson(cont) + result.url = jsonmeta["url"].str + +proc readNimbleLink*(nimbleLinkPath: string): NimbleLink = + let s = readFile(nimbleLinkPath).splitLines() + result.nimbleFilePath = s[0] + result.packageDir = s[1] + +proc writeNimbleLink*(nimbleLinkPath: string, contents: NimbleLink) = + let c = contents.nimbleFilePath & "\n" & contents.packageDir + writeFile(nimbleLinkPath, c) + +proc needsRefresh*(options: Options): bool = + ## Determines whether a ``nimble refresh`` is needed. + ## + ## In the future this will check a stored time stamp to determine how long + ## ago the package list was refreshed. + result = true + for name, list in options.config.packageLists: + if fileExists(options.getNimbleDir() / "packages_" & name & ".json"): + result = false + +proc validatePackagesList(path: string): bool = + ## Determines whether package list at ``path`` is valid. + try: + let pkgList = parseFile(path) + if pkgList.kind == JArray: + if pkgList.len == 0: + display("Warning:", path & " contains no packages.", Warning, + HighPriority) + return true + except ValueError, JsonParsingError: + return false + +proc fetchList*(list: PackageList, options: Options) = + ## Downloads or copies the specified package list and saves it in $nimbleDir. + let verb = if list.urls.len > 0: "Downloading" else: "Copying" + display(verb, list.name & " package list", priority = HighPriority) + + var + lastError = "" + copyFromPath = "" + if list.urls.len > 0: + for i in 0 ..< list.urls.len: + let url = list.urls[i] + display("Trying", url) + let tempPath = options.getNimbleDir() / "packages_temp.json" + + # Grab the proxy + let proxy = getProxy(options) + if not proxy.isNil: + var maskedUrl = proxy.url + if maskedUrl.password.len > 0: maskedUrl.password = "***" + display("Connecting", "to proxy at " & $maskedUrl, + priority = LowPriority) + + try: + let client = newHttpClient(proxy = proxy) + client.downloadFile(url, tempPath) + except: + let message = "Could not download: " & getCurrentExceptionMsg() + display("Warning:", message, Warning) + lastError = message + continue + + if not validatePackagesList(tempPath): + lastError = "Downloaded packages.json file is invalid" + display("Warning:", lastError & ", discarding.", Warning) + continue + + copyFromPath = tempPath + display("Success", "Package list downloaded.", Success, HighPriority) + lastError = "" + break + + elif list.path != "": + if not validatePackagesList(list.path): + lastError = "Copied packages.json file is invalid" + display("Warning:", lastError & ", discarding.", Warning) + else: + copyFromPath = list.path + display("Success", "Package list copied.", Success, HighPriority) + + if lastError.len != 0: + raise newException(NimbleError, "Refresh failed\n" & lastError) + + if copyFromPath.len > 0: + copyFile(copyFromPath, + options.getNimbleDir() / "packages_$1.json" % list.name.toLowerAscii()) + +proc readPackageList(name: string, options: Options): JsonNode = + # If packages.json is not present ask the user if they want to download it. + if needsRefresh(options): + if options.prompt("No local packages.json found, download it from " & + "internet?"): + for name, list in options.config.packageLists: + fetchList(list, options) + else: + # The user might not need a package list for now. So let's try + # going further. + return newJArray() + return parseFile(options.getNimbleDir() / "packages_" & + name.toLowerAscii() & ".json") + +proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool +proc resolveAlias(pkg: Package, options: Options): Package = + result = pkg + # Resolve alias. + if pkg.alias.len > 0: + display("Warning:", "The $1 package has been renamed to $2" % + [pkg.name, pkg.alias], Warning, HighPriority) + if not getPackage(pkg.alias, options, result): + raise newException(NimbleError, "Alias for package not found: " & + pkg.alias) + +proc getPackage*(pkg: string, options: Options, resPkg: var Package): bool = + ## Searches any packages.json files defined in ``options.config.packageLists`` + ## Saves the found package into ``resPkg``. + ## + ## Pass in ``pkg`` the name of the package you are searching for. As + ## convenience the proc returns a boolean specifying if the ``resPkg`` was + ## successfully filled with good data. + ## + ## Aliases are handled and resolved. + for name, list in options.config.packageLists: + display("Reading", "$1 package list" % name, priority = LowPriority) + let packages = readPackageList(name, options) + for p in packages: + if normalize(p["name"].str) == normalize(pkg): + resPkg = p.fromJson() + resPkg = resolveAlias(resPkg, options) + return true + +proc getPackageList*(options: Options): seq[Package] = + ## Returns the list of packages found in the downloaded packages.json files. + result = @[] + var namesAdded = initHashSet[string]() + for name, list in options.config.packageLists: + let packages = readPackageList(name, options) + for p in packages: + let pkg: Package = p.fromJson() + if pkg.name notin namesAdded: + result.add(pkg) + namesAdded.incl(pkg.name) + +proc findNimbleFile*(dir: string; error: bool): string = + result = "" + var hits = 0 + for kind, path in walkDir(dir): + if kind in {pcFile, pcLinkToFile}: + let ext = path.splitFile.ext + case ext + of ".babel", ".nimble", ".nimble-link": + result = path + inc hits + else: discard + if hits >= 2: + raise newException(NimbleError, + "Only one .nimble file should be present in " & dir) + elif hits == 0: + if error: + raise newException(NimbleError, + "Specified directory ($1) does not contain a .nimble file." % dir) + else: + display("Warning:", "No .nimble or .nimble-link file found for " & + dir, Warning, HighPriority) + + if result.splitFile.ext == ".nimble-link": + # Return the path of the real .nimble file. + result = readNimbleLink(result).nimbleFilePath + if not fileExists(result): + let msg = "The .nimble-link file is pointing to a missing file: " & result + let hintMsg = + "Remove '$1' or restore the file it points to." % dir + display("Warning:", msg, Warning, HighPriority) + display("Hint:", hintMsg, Warning, HighPriority) + +proc getInstalledPkgsMin*(libsDir: string, options: Options): + seq[tuple[pkginfo: PackageInfo, meta: MetaData]] = + ## Gets a list of installed packages. The resulting package info is + ## minimal. This has the advantage that it does not depend on the + ## ``packageparser`` module, and so can be used by ``nimscriptwrapper``. + ## + ## ``libsDir`` is in most cases: ~/.nimble/pkgs/ (options.getPkgsDir) + result = @[] + for kind, path in walkDir(libsDir): + if kind == pcDir: + let nimbleFile = findNimbleFile(path, false) + if nimbleFile != "": + let meta = readMetaData(path) + let (name, version) = getNameVersion(path) + var pkg = initPackageInfo(nimbleFile) + pkg.name = name + pkg.version = version + pkg.specialVersion = version + pkg.isMinimal = true + pkg.isInstalled = true + let nimbleFileDir = nimbleFile.splitFile().dir + pkg.isLinked = cmpPaths(nimbleFileDir, path) != 0 + + # Read the package's 'srcDir' (this is stored in the .nimble-link so + # we can easily grab it) + if pkg.isLinked: + let nimbleLinkPath = path / name.addFileExt("nimble-link") + let realSrcPath = readNimbleLink(nimbleLinkPath).packageDir + assert realSrcPath.startsWith(nimbleFileDir) + pkg.srcDir = realSrcPath.replace(nimbleFileDir) + pkg.srcDir.removePrefix(DirSep) + result.add((pkg, meta)) + +proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool = + ## Determines whether the specified package's version is within the + ## specified range. The check works with ordinary versions as well as + ## special ones. + return withinRange(newVersion(pkgInfo.version), verRange) or + withinRange(newVersion(pkgInfo.specialVersion), verRange) + +proc resolveAlias*(dep: PkgTuple, options: Options): PkgTuple = + ## Looks up the specified ``dep.name`` in the packages.json files to resolve + ## a potential alias into the package's real name. + result = dep + var pkg: Package + # TODO: This needs better caching. + if getPackage(dep.name, options, pkg): + # The resulting ``pkg`` will contain the resolved name or the original if + # no alias is present. + result.name = pkg.name + +proc findPkg*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]], + dep: PkgTuple, + r: var PackageInfo): bool = + ## Searches ``pkglist`` for a package of which version is within the range + ## of ``dep.ver``. ``True`` is returned if a package is found. If multiple + ## packages are found the newest one is returned (the one with the highest + ## version number) + ## + ## **Note**: dep.name here could be a URL, hence the need for pkglist.meta. + for pkg in pkglist: + if cmpIgnoreStyle(pkg.pkginfo.name, dep.name) != 0 and + cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue + if withinRange(pkg.pkgInfo, dep.ver): + let isNewer = newVersion(r.version) < newVersion(pkg.pkginfo.version) + if not result or isNewer: + r = pkg.pkginfo + result = true + +proc findAllPkgs*(pkglist: seq[tuple[pkgInfo: PackageInfo, meta: MetaData]], + dep: PkgTuple): seq[PackageInfo] = + ## Searches ``pkglist`` for packages of which version is within the range + ## of ``dep.ver``. This is similar to ``findPkg`` but returns multiple + ## packages if multiple are found. + result = @[] + for pkg in pkglist: + if cmpIgnoreStyle(pkg.pkgInfo.name, dep.name) != 0 and + cmpIgnoreStyle(pkg.meta.url, dep.name) != 0: continue + if withinRange(pkg.pkgInfo, dep.ver): + result.add pkg.pkginfo + +proc getRealDir*(pkgInfo: PackageInfo): string = + ## Returns the directory containing the package source files. + if pkgInfo.srcDir != "" and (not pkgInfo.isInstalled or pkgInfo.isLinked): + result = pkgInfo.mypath.splitFile.dir / pkgInfo.srcDir + else: + result = pkgInfo.mypath.splitFile.dir + +proc getOutputDir*(pkgInfo: PackageInfo, bin: string): string = + ## Returns a binary output dir for the package. + if pkgInfo.binDir != "": + result = pkgInfo.mypath.splitFile.dir / pkgInfo.binDir / bin + else: + result = pkgInfo.mypath.splitFile.dir / bin + +proc echoPackage*(pkg: Package) = + echo(pkg.name & ":") + if pkg.alias.len > 0: + echo(" Alias for ", pkg.alias) + else: + echo(" url: " & pkg.url & " (" & pkg.downloadMethod & ")") + echo(" tags: " & pkg.tags.join(", ")) + echo(" description: " & pkg.description) + echo(" license: " & pkg.license) + if pkg.web.len > 0: + echo(" website: " & pkg.web) + +func toJson*(pkg: Package, queryVersions = false): JsonNode = + result = %*{ + "name": %pkg.name, + "url": %pkg.url, + "method": %pkg.downloadMethod, + "tags": %pkg.tags, + "description": %pkg.description, + "license": %pkg.license, + } + if pkg.web.len > 0: + result["web"] = %pkg.web + +proc getDownloadDirName*(pkg: Package, verRange: VersionRange): string = + result = pkg.name + let verSimple = getSimpleString(verRange) + if verSimple != "": + result.add "_" + result.add verSimple + +proc checkInstallFile(pkgInfo: PackageInfo, + origDir, file: string): bool = + ## Checks whether ``file`` should be installed. + ## ``True`` means file should be skipped. + + for ignoreFile in pkgInfo.skipFiles: + if ignoreFile.endswith("nimble"): + raise newException(NimbleError, ignoreFile & " must be installed.") + if samePaths(file, origDir / ignoreFile): + result = true + break + + for ignoreExt in pkgInfo.skipExt: + if file.splitFile.ext == ('.' & ignoreExt): + result = true + break + + if file.splitFile().name[0] == '.': result = true + +proc checkInstallDir(pkgInfo: PackageInfo, + origDir, dir: string): bool = + ## Determines whether ``dir`` should be installed. + ## ``True`` means dir should be skipped. + for ignoreDir in pkgInfo.skipDirs: + if samePaths(dir, origDir / ignoreDir): + result = true + break + + let thisDir = splitPath(dir).tail + assert thisDir != "" + if thisDir[0] == '.': result = true + if thisDir == "nimcache": result = true + +proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, + action: proc (f: string)) = + ## Runs `action` for each filename of the files that have a whitelisted + ## file extension. + for kind, path in walkDir(dir): + if kind == pcDir: + iterFilesWithExt(path, pkgInfo, action) + else: + if path.splitFile.ext.substr(1) in pkgInfo.installExt: + action(path) + +proc iterFilesInDir(dir: string, action: proc (f: string)) = + ## Runs `action` for each file in ``dir`` and any + ## subdirectories that are in it. + for kind, path in walkDir(dir): + if kind == pcDir: + iterFilesInDir(path, action) + else: + action(path) + +proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo, + options: Options, action: proc (f: string)) = + ## Runs `action` for each file within the ``realDir`` that should be + ## installed. + let whitelistMode = + pkgInfo.installDirs.len != 0 or + pkgInfo.installFiles.len != 0 or + pkgInfo.installExt.len != 0 + if whitelistMode: + for file in pkgInfo.installFiles: + let src = realDir / file + if not src.existsFile(): + if options.prompt("Missing file " & src & ". Continue?"): + continue + else: + raise NimbleQuit(msg: "") + + action(src) + + for dir in pkgInfo.installDirs: + # TODO: Allow skipping files inside dirs? + let src = realDir / dir + if not src.existsDir(): + if options.prompt("Missing directory " & src & ". Continue?"): + continue + else: + raise NimbleQuit(msg: "") + + iterFilesInDir(src, action) + + iterFilesWithExt(realDir, pkgInfo, action) + else: + for kind, file in walkDir(realDir): + if kind == pcDir: + let skip = pkgInfo.checkInstallDir(realDir, file) + if skip: continue + # we also have to stop recursing if we reach an in-place nimbleDir + if file == options.getNimbleDir().expandFilename(): continue + + iterInstallFiles(file, pkgInfo, options, action) + else: + let skip = pkgInfo.checkInstallFile(realDir, file) + if skip: continue + + action(file) + +proc getPkgDest*(pkgInfo: PackageInfo, options: Options): string = + let versionStr = '-' & pkgInfo.specialVersion + let pkgDestDir = options.getPkgsDir() / (pkgInfo.name & versionStr) + return pkgDestDir + +proc `==`*(pkg1: PackageInfo, pkg2: PackageInfo): bool = + if pkg1.name == pkg2.name and pkg1.myPath == pkg2.myPath: + return true + +proc hash*(x: PackageInfo): Hash = + var h: Hash = 0 + h = h !& hash(x.myPath) + result = !$h + +when isMainModule: + doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") == + ("packagea", "0.1") + doAssert getNameVersion("/home/user/.nimble/libs/package-a-0.1") == + ("package-a", "0.1") + doAssert getNameVersion("/home/user/.nimble/libs/package-a-0.1/package.nimble") == + ("package-a", "0.1") + doAssert getNameVersion("/home/user/.nimble/libs/package-#head") == + ("package", "#head") + doAssert getNameVersion("/home/user/.nimble/libs/package-#branch-with-dashes") == + ("package", "#branch-with-dashes") + # readPackageInfo (and possibly more) depends on this not raising. + doAssert getNameVersion("/home/user/.nimble/libs/package") == ("package", "") + + doAssert toValidPackageName("foo__bar") == "foo_bar" + doAssert toValidPackageName("jhbasdh!£$@%#^_&*_()qwe") == "jhbasdh_qwe" + + echo("All tests passed!") diff --git a/src/nimblepkg/packageinstaller.nim b/src/nimblepkg/packageinstaller.nim new file mode 100644 index 0000000..4bdc921 --- /dev/null +++ b/src/nimblepkg/packageinstaller.nim @@ -0,0 +1,112 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +import os, strutils, sets, json + +# Local imports +import cli, options, tools + +when defined(windows): + import version + +when not declared(initHashSet) or not declared(toHashSet): + import common + +when defined(windows): + # This is just for Win XP support. + # TODO: Drop XP support? + from winlean import WINBOOL, DWORD + type + OSVERSIONINFO* {.final, pure.} = object + dwOSVersionInfoSize*: DWORD + dwMajorVersion*: DWORD + dwMinorVersion*: DWORD + dwBuildNumber*: DWORD + dwPlatformId*: DWORD + szCSDVersion*: array[0..127, char] + + proc GetVersionExA*(VersionInformation: var OSVERSIONINFO): WINBOOL{.stdcall, + dynlib: "kernel32", importc: "GetVersionExA".} + +proc setupBinSymlink*(symlinkDest, symlinkFilename: string, + options: Options): seq[string] = + result = @[] + let currentPerms = getFilePermissions(symlinkDest) + setFilePermissions(symlinkDest, currentPerms + {fpUserExec}) + when defined(unix): + display("Creating", "symlink: $1 -> $2" % + [symlinkDest, symlinkFilename], priority = MediumPriority) + if existsFile(symlinkFilename): + let msg = "Symlink already exists in $1. Replacing." % symlinkFilename + display("Warning:", msg, Warning, HighPriority) + removeFile(symlinkFilename) + + createSymlink(symlinkDest, symlinkFilename) + result.add symlinkFilename.extractFilename + elif defined(windows): + # There is a bug on XP, described here: + # http://stackoverflow.com/questions/2182568/batch-script-is-not-executed-if-chcp-was-called + # But this workaround brakes code page on newer systems, so we need to detect OS version + var osver = OSVERSIONINFO() + osver.dwOSVersionInfoSize = cast[DWORD](sizeof(OSVERSIONINFO)) + if GetVersionExA(osver) == WINBOOL(0): + raise newException(NimbleError, + "Can't detect OS version: GetVersionExA call failed") + let fixChcp = osver.dwMajorVersion <= 5 + + # Create cmd.exe/powershell stub. + let dest = symlinkFilename.changeFileExt("cmd") + display("Creating", "stub: $1 -> $2" % [symlinkDest, dest], + priority = MediumPriority) + var contents = "@" + if options.config.chcp: + if fixChcp: + contents.add "chcp 65001 > nul && " + else: contents.add "chcp 65001 > nul\n@" + contents.add "\"" & symlinkDest & "\" %*\n" + writeFile(dest, contents) + result.add dest.extractFilename + # For bash on Windows (Cygwin/Git bash). + let bashDest = dest.changeFileExt("") + display("Creating", "Cygwin stub: $1 -> $2" % + [symlinkDest, bashDest], priority = MediumPriority) + writeFile(bashDest, "\"" & symlinkDest & "\" \"$@\"\n") + result.add bashDest.extractFilename + else: + {.error: "Sorry, your platform is not supported.".} + +proc saveNimbleMeta*(pkgDestDir, url, vcsRevision: string, + filesInstalled, bins: HashSet[string], + isLink: bool = false) = + ## Saves the specified data into a ``nimblemeta.json`` file inside + ## ``pkgDestDir``. + ## + ## filesInstalled - A list of absolute paths to files which have been + ## installed. + ## bins - A list of binary filenames which have been installed for this + ## package. + ## + ## isLink - Determines whether the installed package is a .nimble-link. + var nimblemeta = %{"url": %url} + if vcsRevision.len > 0: + nimblemeta["vcsRevision"] = %vcsRevision + let files = newJArray() + nimblemeta["files"] = files + for file in filesInstalled: + files.add(%changeRoot(pkgDestDir, "", file)) + let binaries = newJArray() + nimblemeta["binaries"] = binaries + for bin in bins: + binaries.add(%bin) + nimblemeta["isLink"] = %isLink + writeFile(pkgDestDir / "nimblemeta.json", $nimblemeta) + +proc saveNimbleMeta*(pkgDestDir, pkgDir, vcsRevision, nimbleLinkPath: string) = + ## Overload of saveNimbleMeta for linked (.nimble-link) packages. + ## + ## pkgDestDir - The directory where the package has been installed. + ## For example: ~/.nimble/pkgs/jester-#head/ + ## + ## pkgDir - The directory where the original package files are. + ## For example: ~/projects/jester/ + saveNimbleMeta(pkgDestDir, "file://" & pkgDir, vcsRevision, + toHashSet[string]([nimbleLinkPath]), initHashSet[string](), true) \ No newline at end of file diff --git a/src/nimblepkg/packagemetadatafile.nim b/src/nimblepkg/packagemetadatafile.nim new file mode 100644 index 0000000..6d28ab8 --- /dev/null +++ b/src/nimblepkg/packagemetadatafile.nim @@ -0,0 +1,73 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import json, os, strformat, sets, sequtils +import common, version, packageinfotypes, cli, tools, sha1hashes + +type + MetaDataError* = object of NimbleError + + PackageMetaDataJsonKeys = enum + pmdjkVersion = "version" + pmdjkMetaData = "metaData" + +const + packageMetaDataFileName* = "nimblemeta.json" + packageMetaDataFileVersion = 1 + +proc initPackageMetaData*(): PackageMetaData = + result = PackageMetaData( + vcsRevision: notSetSha1Hash) + +proc metaDataError(msg: string): ref MetaDataError = + newNimbleError[MetaDataError](msg) + +proc `%`(specialVersions: HashSet[Version]): JsonNode = + %specialVersions.toSeq + +proc initFromJson(specialVersions: var HashSet[Version], jsonNode: JsonNode, + jsonPath: var string) = + case jsonNode.kind + of JArray: + let originalJsonPathLen = jsonPath.len + for i in 0 ..< jsonNode.len: + jsonPath.add '[' + jsonPath.addInt i + jsonPath.add ']' + var version = newVersion("") + initFromJson(version, jsonNode[i], jsonPath) + specialVersions.incl version + jsonPath.setLen originalJsonPathLen + else: + assert false, "The `jsonNode` must be of kind JArray." + +proc saveMetaData*(metaData: PackageMetaData, dirName: string, + changeRoots = true) = + ## Saves some important data to file in the package installation directory. + var metaDataWithChangedPaths = metaData + if changeRoots: + for i, file in metaData.files: + metaDataWithChangedPaths.files[i] = changeRoot(dirName, "", file) + let json = %{ + $pmdjkVersion: %packageMetaDataFileVersion, + $pmdjkMetaData: %metaDataWithChangedPaths} + writeFile(dirName / packageMetaDataFileName, json.pretty) + +proc loadMetaData*(dirName: string, raiseIfNotFound: bool): PackageMetaData = + ## Returns package meta data read from file in package installation directory + result = initPackageMetaData() + let fileName = dirName / packageMetaDataFileName + if fileExists(fileName): + {.warning[ProveInit]: off.} + {.warning[UnsafeSetLen]: off.} + result = parseFile(fileName)[$pmdjkMetaData].to(PackageMetaData) + {.warning[UnsafeSetLen]: on.} + {.warning[ProveInit]: on.} + elif raiseIfNotFound: + raise metaDataError(&"No {packageMetaDataFileName} file found in {dirName}") + else: + displayWarning(&"No {packageMetaDataFileName} file found in {dirName}") + +proc fillMetaData*(packageInfo: var PackageInfo, dirName: string, + raiseIfNotFound: bool) = + packageInfo.metaData = loadMetaData(dirName, raiseIfNotFound) diff --git a/src/nimblepkg/packageparser.nim b/src/nimblepkg/packageparser.nim new file mode 100644 index 0000000..abacc3d --- /dev/null +++ b/src/nimblepkg/packageparser.nim @@ -0,0 +1,511 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +import parsecfg, sets, streams, strutils, os, tables, sugar +from sequtils import apply, map + +import version, tools, common, nimscriptwrapper, options, packageinfo, cli + +## Contains procedures for parsing .nimble files. Moved here from ``packageinfo`` +## because it depends on ``nimscriptwrapper`` (``nimscriptwrapper`` also +## depends on other procedures in ``packageinfo``. + +type + NimbleFile* = string + + ValidationError* = object of NimbleError + warnInstalled*: bool # Determines whether to show a warning for installed pkgs + warnAll*: bool + +const reservedNames = [ + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", +] + +proc newValidationError(msg: string, warnInstalled: bool, + hint: string, warnAll: bool): ref ValidationError = + result = newException(ValidationError, msg) + result.warnInstalled = warnInstalled + result.warnAll = warnAll + result.hint = hint + +proc raiseNewValidationError(msg: string, warnInstalled: bool, + hint: string = "", warnAll = false) = + stderr.writeLine "nimble: ", msg + +proc validatePackageName*(name: string) = + ## Raises an error if specified package name contains invalid characters. + ## + ## A valid package name is one which is a valid nim module name. So only + ## underscores, letters and numbers allowed. + if name.len == 0: return + + if name[0] in {'0'..'9'}: + raiseNewValidationError(name & + "\"$1\" is an invalid package name: cannot begin with $2" % + [name, $name[0]], true) + + var prevWasUnderscore = false + for c in name: + case c + of '_': + if prevWasUnderscore: + raiseNewValidationError( + "$1 is an invalid package name: cannot contain \"__\"" % name, true) + prevWasUnderscore = true + of AllChars - IdentChars: + raiseNewValidationError( + "$1 is an invalid package name: cannot contain '$2'" % [name, $c], + true) + else: + prevWasUnderscore = false + + if name.endsWith("pkg"): + raiseNewValidationError("\"$1\" is an invalid package name: cannot end" & + " with \"pkg\"" % name, false) + if name.toUpperAscii() in reservedNames: + raiseNewValidationError( + "\"$1\" is an invalid package name: reserved name" % name, false) + +proc validateVersion*(ver: string) = + for c in ver: + if c notin ({'.'} + Digits): + raiseNewValidationError( + "Version may only consist of numbers and the '.' character " & + "but found '" & c & "'.", false) + +proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) = + ## This ensures that a package's source code does not leak into + ## another package's namespace. + ## https://github.com/nim-lang/nimble/issues/144 + let + realDir = pkgInfo.getRealDir() + normalizedBinNames = pkgInfo.bin.map( + (x) => x.changeFileExt("").toLowerAscii() + ) + correctDir = + if pkgInfo.name.toLowerAscii() in normalizedBinNames: + pkgInfo.name & "pkg" + else: + pkgInfo.name + + proc onFile(path: string) = + # Remove the root to leave only the package subdirectories. + # ~/package-0.1/package/utils.nim -> package/utils.nim. + var trailPath = changeRoot(realDir, "", path) + if trailPath.startsWith(DirSep): trailPath = trailPath[1 .. ^1] + let (dir, file, ext) = trailPath.splitFile + # We're only interested in nim files, because only they can pollute our + # namespace. + if ext != (ExtSep & "nim"): + return + + if dir.len == 0: + if file != pkgInfo.name: + # A source file was found in the top level of srcDir that doesn't share + # a name with the package. + let + msg = ("Package '$1' has an incorrect structure. " & + "The top level of the package source directory " & + "should contain at most one module, " & + "named '$2', but a file named '$3' was found. This " & + "will be an error in the future.") % + [pkgInfo.name, pkgInfo.name & ext, file & ext] + hint = ("If this is the primary source file in the package, " & + "rename it to '$1'. If it's a source file required by " & + "the main module, or if it is one of several " & + "modules exposed by '$4', then move it into a '$2' subdirectory. " & + "If it's a test file or otherwise not required " & + "to build the the package '$1', prevent its installation " & + "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " & + "https://github.com/nim-lang/nimble#libraries for more info.") % + [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name] + raiseNewValidationError(msg, true, hint, true) + else: + assert(not pkgInfo.isMinimal) + # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize. + if not (dir.startsWith(correctDir & DirSep) or dir == correctDir): + let + msg = ("Package '$2' has an incorrect structure. " & + "It should contain a single directory hierarchy " & + "for source files, named '$3', but file '$1' " & + "is in a directory named '$4' instead. " & + "This will be an error in the future.") % + [file & ext, pkgInfo.name, correctDir, dir] + hint = ("If '$1' contains source files for building '$2', rename it " & + "to '$3'. Otherwise, prevent its installation " & + "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") % + [dir, pkgInfo.name, correctDir] + raiseNewValidationError(msg, true, hint, true) + + iterInstallFiles(realDir, pkgInfo, options, onFile) + +proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) = + let path = pkgInfo.myPath + if pkgInfo.name == "": + raiseNewValidationError("Incorrect .nimble file: " & path & + " does not contain a name field.", false) + + if pkgInfo.name.normalize != path.splitFile.name.normalize: + raiseNewValidationError( + "The .nimble file name must match name specified inside " & path, true) + + if pkgInfo.version == "": + raiseNewValidationError("Incorrect .nimble file: " & path & + " does not contain a version field.", false) + + if not pkgInfo.isMinimal: + if pkgInfo.author == "": + raiseNewValidationError("Incorrect .nimble file: " & path & + " does not contain an author field.", false) + if pkgInfo.description == "": + raiseNewValidationError("Incorrect .nimble file: " & path & + " does not contain a description field.", false) + if pkgInfo.license == "": + raiseNewValidationError("Incorrect .nimble file: " & path & + " does not contain a license field.", false) + if pkgInfo.backend notin ["c", "cc", "objc", "cpp", "js"]: + raiseNewValidationError("'" & pkgInfo.backend & + "' is an invalid backend.", false) + + validatePackageStructure(pkginfo, options) + + +proc nimScriptHint*(pkgInfo: PackageInfo) = + if not pkgInfo.isNimScript: + display("Warning:", "The .nimble file for this project could make use of " & + "additional features, if converted into the new NimScript format." & + "\nFor more details see:" & + "https://github.com/nim-lang/nimble#creating-packages", + Warning, HighPriority) + +proc multiSplit(s: string): seq[string] = + ## Returns ``s`` split by newline and comma characters. + ## + ## Before returning, all individual entries are stripped of whitespace and + ## also empty entries are purged from the list. If after all the cleanups are + ## done no entries are found in the list, the proc returns a sequence with + ## the original string as the only entry. + result = split(s, {char(0x0A), char(0x0D), ','}) + apply(result, proc(x: var string) = x = x.strip()) + for i in countdown(result.len()-1, 0): + if len(result[i]) < 1: + result.del(i) + # Huh, nothing to return? Return given input. + if len(result) < 1: + if s.strip().len != 0: + return @[s] + else: + return @[] + +proc readPackageInfoFromNimble(path: string; result: var PackageInfo) = + var fs = newFileStream(path, fmRead) + if fs != nil: + var p: CfgParser + open(p, fs, path) + defer: close(p) + var currentSection = "" + while true: + var ev = next(p) + case ev.kind + of cfgEof: + break + of cfgSectionStart: + currentSection = ev.section + of cfgKeyValuePair: + case currentSection.normalize + of "package": + case ev.key.normalize + of "name": result.name = ev.value + of "version": result.version = ev.value + of "author": result.author = ev.value + of "description": result.description = ev.value + of "license": result.license = ev.value + of "srcdir": result.srcDir = ev.value + of "bindir": result.binDir = ev.value + of "skipdirs": + result.skipDirs.add(ev.value.multiSplit) + of "skipfiles": + result.skipFiles.add(ev.value.multiSplit) + of "skipext": + result.skipExt.add(ev.value.multiSplit) + of "installdirs": + result.installDirs.add(ev.value.multiSplit) + of "installfiles": + result.installFiles.add(ev.value.multiSplit) + of "installext": + result.installExt.add(ev.value.multiSplit) + of "bin": + for i in ev.value.multiSplit: + if i.splitFile().ext == ".nim": + raise newException(NimbleError, "`bin` entry should not be a source file: " & i) + result.bin.add(i.addFileExt(ExeExt)) + of "backend": + result.backend = ev.value.toLowerAscii() + case result.backend.normalize + of "javascript": result.backend = "js" + else: discard + of "beforehooks": + for i in ev.value.multiSplit: + result.preHooks.incl(i.normalize) + of "afterhooks": + for i in ev.value.multiSplit: + result.postHooks.incl(i.normalize) + else: + raise newException(NimbleError, "Invalid field: " & ev.key) + of "deps", "dependencies": + case ev.key.normalize + of "requires": + for v in ev.value.multiSplit: + result.requires.add(parseRequires(v.strip)) + of "foreigndeps": + for v in ev.value.multiSplit: + result.foreignDeps.add(v.strip) + else: + raise newException(NimbleError, "Invalid field: " & ev.key) + else: raise newException(NimbleError, + "Invalid section: " & currentSection) + of cfgOption: raise newException(NimbleError, + "Invalid package info, should not contain --" & ev.value) + of cfgError: + raise newException(NimbleError, "Error parsing .nimble file: " & ev.msg) + else: + raise newException(ValueError, "Cannot open package info: " & path) + +proc readPackageInfoFromNims(scriptName: string, options: Options, + result: var PackageInfo) = + let + iniFile = getIniFile(scriptName, options) + + if iniFile.fileExists(): + readPackageInfoFromNimble(iniFile, result) + +proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) = + # Binary packages shouldn't install .nim files by default. + # (As long as the package info doesn't explicitly specify what should be + # installed.) + let installInstructions = + pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len + if installInstructions == 0 and pkgInfo.bin.len > 0: + pkgInfo.skipExt.add("nim") + + # When a package doesn't specify a `srcDir` it's fair to assume that + # the .nim files are in the root of the package. So we can explicitly select + # them and prevent the installation of anything else. The user can always + # override this with `installFiles`. + if pkgInfo.srcDir == "": + if dirExists(pkgInfo.getRealDir() / pkgInfo.name): + pkgInfo.installDirs.add(pkgInfo.name) + if fileExists(pkgInfo.getRealDir() / pkgInfo.name.addFileExt("nim")): + pkgInfo.installFiles.add(pkgInfo.name.addFileExt("nim")) + +proc readPackageInfo*(nf: NimbleFile, options: Options, + onlyMinimalInfo=false): PackageInfo = + ## Reads package info from the specified Nimble file. + ## + ## Attempts to read it using the "old" Nimble ini format first, if that + ## fails attempts to evaluate it as a nimscript file. + ## + ## If both fail then returns an error. + ## + ## When ``onlyMinimalInfo`` is true, only the `name` and `version` fields are + ## populated. The ``isNimScript`` field can also be relied on. + ## + ## This version uses a cache stored in ``options``, so calling it multiple + ## times on the same ``nf`` shouldn't require re-evaluation of the Nimble + ## file. + + assert fileExists(nf) + + # Check the cache. + if options.pkgInfoCache.hasKey(nf): + return options.pkgInfoCache[nf] + + result = initPackageInfo(nf) + let minimalInfo = getNameVersion(nf) + + validatePackageName(nf.splitFile.name) + + var success = false + var iniError: ref NimbleError + # Attempt ini-format first. + try: + readPackageInfoFromNimble(nf, result) + success = true + result.isNimScript = false + except NimbleError: + iniError = (ref NimbleError)(getCurrentException()) + + if not success: + if onlyMinimalInfo: + result.name = minimalInfo.name + result.version = minimalInfo.version + result.isNimScript = true + result.isMinimal = true + + # It's possible this proc will receive a .nimble-link file eventually, + # I added this assert to hopefully make this error clear for everyone. + let msg = "No version detected. Received nimble-link?" + assert result.version.len > 0, msg + else: + try: + readPackageInfoFromNims(nf, options, result) + result.isNimScript = true + except NimbleError as exc: + if exc.hint.len > 0: + raise + let msg = "Could not read package info file in " & nf & ";\n" & + " Reading as ini file failed with: \n" & + " " & iniError.msg & ".\n" & + " Evaluating as NimScript file failed with: \n" & + " " & exc.msg & "." + raise newException(NimbleError, msg) + + # By default specialVersion is the same as version. + result.specialVersion = result.version + + # Only attempt to read a special version if `nf` is inside the $nimbleDir. + if nf.startsWith(options.getNimbleDir()): + # The package directory name may include a "special" version + # (example #head). If so, it is given higher priority and therefore + # overwrites the .nimble file's version. + let version = parseVersionRange(minimalInfo.version) + if version.kind == verSpecial: + result.specialVersion = minimalInfo.version + + # Apply rules to infer which files should/shouldn't be installed. See #469. + inferInstallRules(result, options) + + if not result.isMinimal: + options.pkgInfoCache[nf] = result + + # Validate the rest of the package info last. + if not options.disableValidation: + validateVersion(result.version) + validatePackageInfo(result, options) + +proc validate*(file: NimbleFile, options: Options, + error: var ValidationError, pkgInfo: var PackageInfo): bool = + try: + pkgInfo = readPackageInfo(file, options) + except ValidationError as exc: + error = exc[] + return false + + return true + +proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo = + ## Reads the specified .nimble file and returns its data as a PackageInfo + ## object. Any validation errors are handled and displayed as warnings. + try: + result = readPackageInfo(file, options) + except ValidationError: + let exc = (ref ValidationError)(getCurrentException()) + if exc.warnAll: + display("Warning:", exc.msg, Warning, HighPriority) + display("Hint:", exc.hint, Warning, HighPriority) + else: + raise + +proc getPkgInfo*(dir: string, options: Options): PackageInfo = + ## Find the .nimble file in ``dir`` and parses it, returning a PackageInfo. + let nimbleFile = findNimbleFile(dir, true) + return getPkgInfoFromFile(nimbleFile, options) + +proc getInstalledPkgs*(libsDir: string, options: Options): + seq[tuple[pkginfo: PackageInfo, meta: MetaData]] = + ## Gets a list of installed packages. + ## + ## ``libsDir`` is in most cases: ~/.nimble/pkgs/ + const + readErrorMsg = "Installed package '$1@$2' is outdated or corrupt." + validationErrorMsg = readErrorMsg & "\nPackage did not pass validation: $3" + hintMsg = "The corrupted package will need to be removed manually. To fix" & + " this error message, remove $1." + + proc createErrorMsg(tmplt, path, msg: string): string = + let (name, version) = getNameVersion(path) + return tmplt % [name, version, msg] + + display("Loading", "list of installed packages", priority = MediumPriority) + + result = @[] + for kind, path in walkDir(libsDir): + if kind == pcDir: + let nimbleFile = findNimbleFile(path, false) + if nimbleFile != "": + let meta = readMetaData(path) + var pkg: PackageInfo + try: + pkg = readPackageInfo(nimbleFile, options, onlyMinimalInfo=false) + except ValidationError: + let exc = (ref ValidationError)(getCurrentException()) + exc.msg = createErrorMsg(validationErrorMsg, path, exc.msg) + exc.hint = hintMsg % path + if exc.warnInstalled or exc.warnAll: + display("Warning:", exc.msg, Warning, HighPriority) + # Don't show hints here because they are only useful for package + # owners. + else: + raise exc + except: + let tmplt = readErrorMsg & "\nMore info: $3" + let msg = createErrorMsg(tmplt, path, getCurrentException().msg) + var exc = newException(NimbleError, msg) + exc.hint = hintMsg % path + raise exc + + pkg.isInstalled = true + pkg.isLinked = + cmpPaths(nimbleFile.splitFile().dir, path) != 0 + result.add((pkg, meta)) + +proc isNimScript*(nf: string, options: Options): bool = + result = readPackageInfo(nf, options).isNimScript + +proc toFullInfo*(pkg: PackageInfo, options: Options): PackageInfo = + if pkg.isMinimal: + result = getPkgInfoFromFile(pkg.mypath, options) + result.isInstalled = pkg.isInstalled + result.isLinked = pkg.isLinked + else: + return pkg + +proc getConcreteVersion*(pkgInfo: PackageInfo, options: Options): string = + ## Returns a non-special version from the specified ``pkgInfo``. If the + ## ``pkgInfo`` is minimal it looks it up and retrieves the concrete version. + result = pkgInfo.version + if pkgInfo.isMinimal: + let pkgInfo = pkgInfo.toFullInfo(options) + result = pkgInfo.version + assert(not newVersion(result).isSpecial) + +when isMainModule: + validatePackageName("foo_bar") + validatePackageName("f_oo_b_a_r") + try: + validatePackageName("foo__bar") + assert false + except NimbleError: + assert true + + echo("Everything passed!") diff --git a/src/nimblepkg/paths.nim b/src/nimblepkg/paths.nim new file mode 100644 index 0000000..9b773ba --- /dev/null +++ b/src/nimblepkg/paths.nim @@ -0,0 +1,45 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module implements operations with file system paths in a way independent +## of weather the path is absolute or relative to the current directory. + +import os, json, hashes + +type Path* = distinct string + +converter toPath*(path: string): Path = Path(path) + +proc `%`*(path: Path): JsonNode {.borrow.} +proc `$`*(path: Path): string {.borrow.} + +proc isAbsolute*(path: Path): bool {.borrow.} +proc splitFile*(path: Path): tuple[dir, name, ext: Path] {.borrow.} +proc splitPath*(path: Path): tuple[head, tail: Path] {.borrow.} +proc normalizedPath*(path: Path): Path {.borrow.} +proc dirExists*(dirname: Path): bool {.borrow.} +proc fileExists*(filename: Path): bool {.borrow.} +proc parseFile*(filename: Path): JsonNode {.borrow.} +proc `/`*(head, tail: Path): Path {.borrow.} +proc writeFile*(filename: Path, content: string) {.borrow.} +proc len*(path: Path): int {.borrow.} +proc isRootDir*(path: Path): bool {.borrow.} +proc parentDir*(path: Path): Path {.borrow.} +proc quoteShell*(s: Path): Path {.borrow.} + +proc hash*(path: Path): Hash = hash(absolutePath(string(path))) + +proc `==`*(lhs, rhs: Path): bool = + absolutePath(string(lhs)) == absolutePath(string(rhs)) + +when isMainModule: + import unittest + + const testDir: Path = "some/relative/path/" + let absolutePathToTestDir: Path = getCurrentDir() / testDir + + test "hashing": + check hash(testDir) == hash(absolutePathToTestDir) + + test "equals": + check testDir == absolutePathToTestDir diff --git a/src/nimblepkg/publish.nim b/src/nimblepkg/publish.nim new file mode 100644 index 0000000..f11b979 --- /dev/null +++ b/src/nimblepkg/publish.nim @@ -0,0 +1,229 @@ +# Copyright (C) Andreas Rumpf. All rights reserved. +# BSD License. Look at license.txt for more info. + +## Implements 'nimble publish' to create a pull request against +## nim-lang/packages automatically. + +import system except TResult +import httpclient, strutils, json, os, browsers, times, uri +import version, tools, common, cli, config, options + +type + Auth = object + user: string + token: string ## Github access token + http: HttpClient ## http client for doing API requests + +const + ApiKeyFile = "github_api_token" + ApiTokenEnvironmentVariable = "NIMBLE_GITHUB_API_TOKEN" + ReposUrl = "https://api.github.com/repos/" + +proc userAborted() = + raise newException(NimbleError, "User aborted the process.") + +proc createHeaders(a: Auth) = + a.http.headers = newHttpHeaders({ + "Authorization": "token $1" % a.token, + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "*/*" + }) + +proc requestNewToken(cfg: Config): string = + display("Info:", "Please create a new personal access token on Github in" & + " order to allow Nimble to fork the packages repository.", + priority = HighPriority) + display("Hint:", "Make sure to give the access token access to public repos" & + " (public_repo scope)!", Warning, HighPriority) + sleep(5000) + display("Info:", "Your default browser should open with the following URL: " & + "https://github.com/settings/tokens/new", priority = HighPriority) + sleep(3000) + openDefaultBrowser("https://github.com/settings/tokens/new") + let token = promptCustom("Personal access token?", "").strip() + # inform the user that their token will be written to disk + let tokenWritePath = cfg.nimbleDir / ApiKeyFile + display("Info:", "Writing access token to file:" & tokenWritePath, + priority = HighPriority) + writeFile(tokenWritePath, token) + sleep(3000) + return token + +proc getGithubAuth(o: Options): Auth = + let cfg = o.config + result.http = newHttpClient(proxy = getProxy(o)) + # always prefer the environment variable to asking for a new one + if existsEnv(ApiTokenEnvironmentVariable): + result.token = getEnv(ApiTokenEnvironmentVariable) + display("Info:", "Using the '" & ApiTokenEnvironmentVariable & + "' environment variable for the GitHub API Token.", + priority = HighPriority) + else: + # try to read from disk, if it cannot be found write a new one + try: + let apiTokenFilePath = cfg.nimbleDir / ApiKeyFile + result.token = readFile(apiTokenFilePath).strip() + display("Info:", "Using GitHub API Token in file: " & apiTokenFilePath, + priority = HighPriority) + except IOError: + result.token = requestNewToken(cfg) + createHeaders(result) + let resp = result.http.getContent("https://api.github.com/user").parseJson() + + result.user = resp["login"].str + display("Success:", "Verified as " & result.user, Success, HighPriority) + +proc isCorrectFork(j: JsonNode): bool = + # Check whether this is a fork of the nimble packages repo. + result = false + if j{"fork"}.getBool(): + result = j{"parent"}{"full_name"}.getStr() == "nim-lang/packages" + +proc forkExists(a: Auth): bool = + try: + let x = a.http.getContent(ReposUrl & a.user & "/packages") + let j = parseJson(x) + result = isCorrectFork(j) + except JsonParsingError, IOError: + result = false + +proc createFork(a: Auth) = + try: + discard a.http.postContent(ReposUrl & "nim-lang/packages/forks") + except HttpRequestError: + raise newException(NimbleError, "Unable to create fork. Access token" & + " might not have enough permissions.") + +proc createPullRequest(a: Auth, packageName, branch: string): string = + display("Info", "Creating PR", priority = HighPriority) + var body = a.http.postContent(ReposUrl & "nim-lang/packages/pulls", + body="""{"title": "Add package $1", "head": "$2:$3", + "base": "master"}""" % [packageName, a.user, branch]) + var pr = parseJson(body) + return pr{"html_url"}.getStr() + +proc `%`(s: openArray[string]): JsonNode = + result = newJArray() + for x in s: result.add(%x) + +proc cleanupWhitespace(s: string): string = + ## Removes trailing whitespace and normalizes line endings to LF. + result = newStringOfCap(s.len) + var i = 0 + while i < s.len: + if s[i] == ' ': + var j = i+1 + while s[j] == ' ': inc j + if s[j] == '\c': + inc j + if s[j] == '\L': inc j + result.add '\L' + i = j + elif s[j] == '\L': + result.add '\L' + i = j+1 + else: + result.add ' ' + inc i + elif s[i] == '\c': + inc i + if s[i] == '\L': inc i + result.add '\L' + elif s[i] == '\L': + result.add '\L' + inc i + else: + result.add s[i] + inc i + if result[^1] != '\L': + result.add '\L' + +proc editJson(p: PackageInfo; url, tags, downloadMethod: string) = + var contents = parseFile("packages.json") + doAssert contents.kind == JArray + contents.add(%*{ + "name": p.name, + "url": url, + "method": downloadMethod, + "tags": tags.split(), + "description": p.description, + "license": p.license, + "web": url + }) + writeFile("packages.json", contents.pretty.cleanupWhitespace) + +proc publish*(p: PackageInfo, o: Options) = + ## Publishes the package p. + let auth = getGithubAuth(o) + var pkgsDir = getNimbleUserTempDir() / "nimble-packages-fork" + if not forkExists(auth): + createFork(auth) + display("Info:", "Waiting 10s to let Github create a fork", + priority = HighPriority) + os.sleep(10_000) + + display("Info:", "Finished waiting", priority = LowPriority) + if dirExists(pkgsDir): + display("Removing", "old packages fork git directory.", + priority = LowPriority) + removeDir(pkgsDir) + createDir(pkgsDir) + cd pkgsDir: + # Avoid git clone to prevent token from being stored in repo + # https://github.com/blog/1270-easier-builds-and-deployments-using-git-over-https-and-oauth + display("Copying", "packages fork into: " & pkgsDir, priority = HighPriority) + doCmd("git init") + doCmd("git pull https://github.com/" & auth.user & "/packages") + # Make sure to update the fork + display("Updating", "the fork", priority = HighPriority) + doCmd("git pull https://github.com/nim-lang/packages.git master") + doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages master") + + if not dirExists(pkgsDir): + raise newException(NimbleError, + "Cannot find nimble-packages-fork git repository. Cloning failed.") + + if not fileExists(pkgsDir / "packages.json"): + raise newException(NimbleError, + "No packages file found in cloned fork.") + + # We need to do this **before** the cd: + # Determine what type of repo this is. + var url = "" + var downloadMethod = "" + if dirExists(os.getCurrentDir() / ".git"): + let (output, exitCode) = doCmdEx("git ls-remote --get-url") + if exitCode == 0: + url = output.string.strip + if url.endsWith(".git"): url.setLen(url.len - 4) + downloadMethod = "git" + let parsed = parseUri(url) + if parsed.scheme == "": + # Assuming that we got an ssh write/read URL. + let sshUrl = parseUri("ssh://" & url) + url = "https://" & sshUrl.hostname & "/" & sshUrl.port & sshUrl.path + elif dirExists(os.getCurrentDir() / ".hg"): + downloadMethod = "hg" + # TODO: Retrieve URL from hg. + else: + raise newException(NimbleError, + "No .git nor .hg directory found. Stopping.") + + if url.len == 0: + url = promptCustom("Github URL of " & p.name & "?", "") + if url.len == 0: userAborted() + + let tags = promptCustom( + "Whitespace separated list of tags? (For example: web library wrapper)", + "" + ) + + cd pkgsDir: + editJson(p, url, tags, downloadMethod) + let branchName = "add-" & p.name & getTime().utc.format("HHmm") + doCmd("git checkout -B " & branchName) + doCmd("git commit packages.json -m \"Added package " & p.name & "\"") + display("Pushing", "to remote of fork.", priority = HighPriority) + doCmd("git push https://" & auth.token & "@github.com/" & auth.user & "/packages " & branchName) + let prUrl = createPullRequest(auth, p.name, branchName) + display("Success:", "Pull request successful, check at " & prUrl , Success, HighPriority) diff --git a/src/nimblepkg/reversedeps.nim b/src/nimblepkg/reversedeps.nim new file mode 100644 index 0000000..45d9940 --- /dev/null +++ b/src/nimblepkg/reversedeps.nim @@ -0,0 +1,135 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import os, json, sets + +import options, common, version, download, packageinfo + +proc saveNimbleData*(options: Options) = + # TODO: This file should probably be locked. + writeFile(options.getNimbleDir() / "nimbledata.json", + pretty(options.nimbleData)) + +proc addRevDep*(nimbleData: JsonNode, dep: tuple[name, version: string], + pkg: PackageInfo) = + # Add a record which specifies that `pkg` has a dependency on `dep`, i.e. + # the reverse dependency of `dep` is `pkg`. + if not nimbleData["reverseDeps"].hasKey(dep.name): + nimbleData["reverseDeps"][dep.name] = newJObject() + if not nimbleData["reverseDeps"][dep.name].hasKey(dep.version): + nimbleData["reverseDeps"][dep.name][dep.version] = newJArray() + let revDep = %{ "name": %pkg.name, "version": %pkg.specialVersion} + let thisDep = nimbleData["reverseDeps"][dep.name][dep.version] + if revDep notin thisDep: + thisDep.add revDep + +proc removeRevDep*(nimbleData: JsonNode, pkg: PackageInfo) = + ## Removes ``pkg`` from the reverse dependencies of every package. + assert(not pkg.isMinimal) + proc remove(pkg: PackageInfo, depTup: PkgTuple, thisDep: JsonNode) = + for ver, val in thisDep: + if ver.newVersion in depTup.ver: + var newVal = newJArray() + for revDep in val: + if not (revDep["name"].str == pkg.name and + revDep["version"].str == pkg.specialVersion): + newVal.add revDep + thisDep[ver] = newVal + + for depTup in pkg.requires: + if depTup.name.isURL(): + # We sadly must go through everything in this case... + for key, val in nimbleData["reverseDeps"]: + remove(pkg, depTup, val) + else: + let thisDep = nimbleData{"reverseDeps", depTup.name} + if thisDep.isNil: continue + remove(pkg, depTup, thisDep) + + # Clean up empty objects/arrays + var newData = newJObject() + for key, val in nimbleData["reverseDeps"]: + if val.len != 0: + var newVal = newJObject() + for ver, elem in val: + if elem.len != 0: + newVal[ver] = elem + if newVal.len != 0: + newData[key] = newVal + nimbleData["reverseDeps"] = newData + +proc getRevDepTups*(options: Options, pkg: PackageInfo): seq[PkgTuple] = + ## Returns a list of *currently installed* reverse dependencies for `pkg`. + result = @[] + let thisPkgsDep = + options.nimbleData["reverseDeps"]{pkg.name}{pkg.specialVersion} + if not thisPkgsDep.isNil: + let pkgList = getInstalledPkgsMin(options.getPkgsDir(), options) + for pkg in thisPkgsDep: + let pkgTup = ( + name: pkg["name"].getStr(), + ver: parseVersionRange(pkg["version"].getStr()) + ) + var pkgInfo: PackageInfo + if not findPkg(pkgList, pkgTup, pkgInfo): + continue + + result.add(pkgTup) + +proc getRevDeps*(options: Options, pkg: PackageInfo): HashSet[PackageInfo] = + result.init() + let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options) + for rdepTup in getRevDepTups(options, pkg): + for rdepInfo in findAllPkgs(installedPkgs, rdepTup): + result.incl rdepInfo + +proc getAllRevDeps*(options: Options, pkg: PackageInfo, result: var HashSet[PackageInfo]) = + if pkg in result: + return + + let installedPkgs = getInstalledPkgsMin(options.getPkgsDir(), options) + for rdepTup in getRevDepTups(options, pkg): + for rdepInfo in findAllPkgs(installedPkgs, rdepTup): + if rdepInfo in result: + continue + + getAllRevDeps(options, rdepInfo, result) + result.incl pkg + +when isMainModule: + var nimbleData = %{"reverseDeps": newJObject()} + + let nimforum1 = PackageInfo( + isMinimal: false, + name: "nimforum", + specialVersion: "0.1.0", + requires: @[("jester", parseVersionRange("0.1.0")), + ("captcha", parseVersionRange("1.0.0")), + ("auth", parseVersionRange("#head"))] + ) + let nimforum2 = PackageInfo(isMinimal: false, name: "nimforum", specialVersion: "0.2.0") + let play = PackageInfo(isMinimal: false, name: "play", specialVersion: "#head") + + nimbleData.addRevDep(("jester", "0.1.0"), nimforum1) + nimbleData.addRevDep(("jester", "0.1.0"), play) + nimbleData.addRevDep(("captcha", "1.0.0"), nimforum1) + nimbleData.addRevDep(("auth", "#head"), nimforum1) + nimbleData.addRevDep(("captcha", "1.0.0"), nimforum2) + nimbleData.addRevDep(("auth", "#head"), nimforum2) + + doAssert nimbleData["reverseDeps"]["jester"]["0.1.0"].len == 2 + doAssert nimbleData["reverseDeps"]["captcha"]["1.0.0"].len == 2 + doAssert nimbleData["reverseDeps"]["auth"]["#head"].len == 2 + + block: + nimbleData.removeRevDep(nimforum1) + let jester = nimbleData["reverseDeps"]["jester"]["0.1.0"][0] + doAssert jester["name"].getStr() == play.name + doAssert jester["version"].getStr() == play.specialVersion + + let captcha = nimbleData["reverseDeps"]["captcha"]["1.0.0"][0] + doAssert captcha["name"].getStr() == nimforum2.name + doAssert captcha["version"].getStr() == nimforum2.specialVersion + + echo("Everything works!") + diff --git a/src/nimblepkg/syncfile.nim b/src/nimblepkg/syncfile.nim new file mode 100644 index 0000000..f28c94d --- /dev/null +++ b/src/nimblepkg/syncfile.nim @@ -0,0 +1,109 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module implement operations on a special `sync` file which is being kept +## in the hidden special VCS directory. It is used to keep the revisions of the +## package's develop mode dependencies at the time when the last `lock` or +## `sync` operation had been performed. The file is used to determine whether a +## new `lock` or `sync` command or a VCS `merge` or `rebase` command is needed +## when there is a conflict between the data written in it and the data from the +## lock file and from the working copy. + +import tables, json, os +import common, sha1hashes, paths, vcstools, packageinfotypes + +type + SyncFileData = Table[string, Sha1Hash] + # Maps develop mode dependency name to the VCS revision it has in the time + # of the last `lock` or `sync` operation or when it is added as a develop + # mode dependency if there is no such operations after that moment. + + SyncFile = object + path: Path + data: SyncFileData + + SyncFileJsonKeys = enum + ## Represents the keys for the `sync` file Json objects. + lsfjkVersion = "version" + lsfjkData = "data" + +const + syncFileExt = ".nimble.sync" + syncFileVersion = 1 + +proc getPkgDir(pkgInfo: PackageInfo): string = + pkgInfo.myPath.splitFile.dir + +proc getSyncFilePath(pkgInfo: PackageInfo): Path = + ## Returns a path to the sync file for package `pkgInfo`. + + let (vcsType, vcsSpecialDirPath) = + # Do not use `pkgInfo.getNimbleFileDir` in order to avoid circular + # dependencies. + getVcsTypeAndSpecialDirPath(pkgInfo.getPkgDir) + + if vcsType == vcsTypeNone: + # The directory is not under version control, and we have not a place where + # to hide the sync file. + raise nimbleError( + msg = "Sync file require current working directory to be under some " & + "supported type of version control.", + hint = "Put package's working directory under version control.") + + return vcsSpecialDirPath / (pkgInfo.basicInfo.name & syncFileExt).Path + +proc load(syncFile: ref SyncFile, path: Path) = + ## Loads a sync file. + + syncFile.path = path + if not path.fileExists: + return + + {.warning[UnsafeDefault]: off.} + {.warning[ProveInit]: off.} + syncFile.data = parseFile(path)[$lsfjkData].to(SyncFileData) + {.warning[ProveInit]: on.} + {.warning[UnsafeDefault]: on.} + +proc getSyncFile*(pkgInfo: PackageInfo): ref SyncFile = + # Returns a reference to the sync file data of the current working directory + # package `pkgInfo`. + + assert pkgInfo.getPkgDir == getCurrentDir(): + "The package `pkgInfo` must be the current working directory package." + + var syncFile {.global.}: ref SyncFile + once: + syncFile.new + let path = getSyncFilePath(pkgInfo) + syncFile.load(path) + return syncFile + +proc save*(syncFile: ref SyncFile) = + ## Saves a sync file. + + let jsonNode = %{ + $lsfjkVersion: %syncFileVersion, + $lsfjkData: %syncFile.data, + } + + writeFile(syncFile.path, jsonNode.pretty) + +proc getDepVcsRevision*(syncFile: ref SyncFile, depName: string): Sha1Hash = + ## Returns the revision written in the sync file for develop mode dependency + ## `depName`. + syncFile.data.getOrDefault(depName, notSetSha1Hash) + +proc setDepVcsRevision*(syncFile: ref SyncFile, depName: string, + vcsRevision: Sha1Hash) = + ## Sets the revision in the sync file for the develop mode dependency + ## `depName` to be equal to `vcsRevision`. + + syncFile.data[depName] = vcsRevision + +proc clear*(syncFile: ref SyncFile) = + ## Clears all the data from the sync file. + + {.warning[UnsafeDefault]: off.} + syncFile.data.clear + {.warning[UnsafeDefault]: on.} diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim new file mode 100644 index 0000000..b39fea7 --- /dev/null +++ b/src/nimblepkg/tools.nim @@ -0,0 +1,181 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. +# +# Various miscellaneous utility functions reside here. +import osproc, pegs, strutils, os, uri, sets, json, parseutils +import version, cli + +proc extractBin(cmd: string): string = + if cmd[0] == '"': + return cmd.captureBetween('"') + else: + return cmd.split(' ')[0] + +proc doCmd*(cmd: string, showOutput = false) = + let bin = extractBin(cmd) + if findExe(bin) == "": + raise newException(NimbleError, "'" & bin & "' not in PATH.") + + # To keep output in sequence + stdout.flushFile() + stderr.flushFile() + + displayDebug("Executing", cmd) + if showOutput: + let exitCode = execCmd(cmd) + displayDebug("Finished", "with exit code " & $exitCode) + if exitCode != QuitSuccess: + raise newException(NimbleError, + "Execution failed with exit code $1\nCommand: $2" % + [$exitCode, cmd]) + else: + let (output, exitCode) = execCmdEx(cmd) + displayDebug("Finished", "with exit code " & $exitCode) + displayDebug("Output", output) + + if exitCode != QuitSuccess: + raise newException(NimbleError, + "Execution failed with exit code $1\nCommand: $2\nOutput: $3" % + [$exitCode, cmd, output]) + +proc doCmdEx*(cmd: string): tuple[output: TaintedString, exitCode: int] = + let bin = extractBin(cmd) + if findExe(bin) == "": + raise newException(NimbleError, "'" & bin & "' not in PATH.") + return execCmdEx(cmd) + +template cd*(dir: string, body: untyped) = + ## Sets the current dir to ``dir``, executes ``body`` and restores the + ## previous working dir. + let lastDir = getCurrentDir() + setCurrentDir(dir) + body + setCurrentDir(lastDir) + +proc getNimBin*: string = + result = "nim" + if findExe("nim") != "": result = findExe("nim") + elif findExe("nimrod") != "": result = findExe("nimrod") + +proc getNimrodVersion*: Version = + let nimBin = getNimBin() + let vOutput = doCmdEx('"' & nimBin & "\" -v").output + var matches: array[0..MaxSubpatterns, string] + if vOutput.find(peg"'Version'\s{(\d+\.)+\d}", matches) == -1: + raise newException(NimbleError, "Couldn't find Nim version.") + newVersion(matches[0]) + +proc samePaths*(p1, p2: string): bool = + ## Normalizes path (by adding a trailing slash) and compares. + var cp1 = if not p1.endsWith("/"): p1 & "/" else: p1 + var cp2 = if not p2.endsWith("/"): p2 & "/" else: p2 + cp1 = cp1.replace('/', DirSep).replace('\\', DirSep) + cp2 = cp2.replace('/', DirSep).replace('\\', DirSep) + + return cmpPaths(cp1, cp2) == 0 + +proc changeRoot*(origRoot, newRoot, path: string): string = + ## origRoot: /home/dom/ + ## newRoot: /home/test/ + ## path: /home/dom/bar/blah/2/foo.txt + ## Return value -> /home/test/bar/blah/2/foo.txt + + ## The additional check of `path.samePaths(origRoot)` is necessary to prevent + ## a regression, where by ending the `srcDir` defintion in a nimble file in a + ## trailing separator would cause the `path.startsWith(origRoot)` evaluation to + ## fail because of the value of `origRoot` would be longer than `path` due to + ## the trailing separator. This would cause this method to throw during package + ## installation. + if path.startsWith(origRoot) or path.samePaths(origRoot): + return newRoot / path.substr(origRoot.len, path.len-1) + else: + raise newException(ValueError, + "Cannot change root of path: Path does not begin with original root.") + +proc copyFileD*(fro, to: string): string = + ## Returns the destination (``to``). + display("Copying", "file $# to $#" % [fro, to], priority = LowPriority) + copyFileWithPermissions(fro, to) + result = to + +proc copyDirD*(fro, to: string): seq[string] = + ## Returns the filenames of the files in the directory that were copied. + result = @[] + display("Copying", "directory $# to $#" % [fro, to], priority = LowPriority) + for path in walkDirRec(fro): + createDir(changeRoot(fro, to, path.splitFile.dir)) + result.add copyFileD(path, changeRoot(fro, to, path)) + +proc createDirD*(dir: string) = + display("Creating", "directory $#" % dir, priority = LowPriority) + createDir(dir) + +proc getDownloadDirName*(uri: string, verRange: VersionRange): string = + ## Creates a directory name based on the specified ``uri`` (url) + result = "" + let puri = parseUri(uri) + for i in puri.hostname: + case i + of strutils.Letters, strutils.Digits: + result.add i + else: discard + result.add "_" + for i in puri.path: + case i + of strutils.Letters, strutils.Digits: + result.add i + else: discard + + let verSimple = getSimpleString(verRange) + if verSimple != "": + result.add "_" + result.add verSimple + +proc incl*(s: var HashSet[string], v: seq[string] | HashSet[string]) = + for i in v: + s.incl i + +when not declared(json.contains): + proc contains*(j: JsonNode, elem: JsonNode): bool = + for i in j: + if i == elem: + return true + +proc contains*(j: JsonNode, elem: tuple[key: string, val: JsonNode]): bool = + for key, val in pairs(j): + if key == elem.key and val == elem.val: + return true + +when not defined(windows): + from posix import getpid + +proc getProcessId*(): string = + when defined(windows): + proc GetCurrentProcessId(): int32 {.stdcall, dynlib: "kernel32", + importc: "GetCurrentProcessId".} + result = $GetCurrentProcessId() + else: + result = $getpid() + +proc getNimbleTempDir*(): string = + ## Returns a path to a temporary directory. + ## + ## The returned path will be the same for the duration of the process but + ## different for different runs of it. You have to make sure to create it + ## first. In release builds the directory will be removed when nimble finishes + ## its work. + result = getTempDir() / "nimble_" & getProcessId() + +proc getNimbleUserTempDir*(): string = + ## Returns a path to a temporary directory. + ## + ## The returned path will be the same for the duration of the process but + ## different for different runs of it. You have to make sure to create it + ## first. In release builds the directory will be removed when nimble finishes + ## its work. + var tmpdir: string + if existsEnv("TMPDIR") and existsEnv("USER"): + tmpdir = joinPath(getEnv("TMPDIR"), getEnv("USER")) + else: + tmpdir = getTempDir() + return tmpdir diff --git a/src/nimblepkg/topologicalsort.nim b/src/nimblepkg/topologicalsort.nim new file mode 100644 index 0000000..8e0c1fe --- /dev/null +++ b/src/nimblepkg/topologicalsort.nim @@ -0,0 +1,169 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +import sequtils, tables, strformat, algorithm, sets +import common, packageinfotypes, packageinfo, options, cli + +proc getDependencies(packages: seq[PackageInfo], package: PackageInfo, + options: Options): + seq[string] = + ## Returns the names of the packages which are dependencies of a given + ## package. It is needed because some of the names of the packages in the + ## `requires` clause of a package could be URLs. + for dep in package.requires: + if dep.name.isNim: + continue + var depPkgInfo = initPackageInfo() + var found = findPkg(packages, dep, depPkgInfo) + if not found: + let resolvedDep = dep.resolveAlias(options) + found = findPkg(packages, resolvedDep, depPkgInfo) + if not found: + raise nimbleError( + "Cannot build the dependency graph.\n" & + &"Missing package \"{dep.name}\".") + result.add depPkgInfo.basicInfo.name + +proc buildDependencyGraph*(packages: seq[PackageInfo], options: Options): + LockFileDeps = + ## Creates records which will be saved to the lock file. + for pkgInfo in packages: + result[pkgInfo.basicInfo.name] = LockFileDep( + version: pkgInfo.basicInfo.version, + vcsRevision: pkgInfo.metaData.vcsRevision, + url: pkgInfo.metaData.url, + downloadMethod: pkgInfo.metaData.downloadMethod, + dependencies: getDependencies(packages, pkgInfo, options), + checksums: Checksums(sha1: pkgInfo.basicInfo.checksum)) + +proc topologicalSort*(graph: LockFileDeps): + tuple[order: seq[string], cycles: seq[seq[string]]] = + ## Topologically sorts dependency graph which will be saved to the lock file. + ## + ## Returns tuple containing sequence with the package names in the + ## topologically sorted order and another sequence with detected cyclic + ## dependencies if any (should not be such). Only cycles which don't have + ## edges part of another cycle are being detected (in the order of the + ## visiting). + + type + NodeMark = enum + nmNotMarked + nmTemporary + nmPermanent + + NodeInfo = tuple[mark: NodeMark, cameFrom: string] + NodesInfo = OrderedTable[string, NodeInfo] + + var + order = newSeqOfCap[string](graph.len) + cycles: seq[seq[string]] + nodesInfo: NodesInfo + + proc getCycle(finalNode: string): seq[string] = + var + path = newSeqOfCap[string](graph.len) + previousNode = nodesInfo[finalNode].cameFrom + + path.add finalNode + while previousNode != finalNode: + path.add previousNode + previousNode = nodesInfo[previousNode].cameFrom + + path.add previousNode + path.reverse() + + return path + + proc printNotADagWarning() = + let message = cycles.foldl( + a & "\nCycle detected: " & b.foldl(&"{a} -> {b}"), + "The dependency graph is not a DAG.") + display("Warning", message, Warning, HighPriority) + + for node, _ in graph: + nodesInfo[node] = (mark: nmNotMarked, cameFrom: "") + + proc visit(node: string) = + template nodeInfo: untyped = nodesInfo[node] + + if nodeInfo.mark == nmPermanent: + return + + if nodeInfo.mark == nmTemporary: + cycles.add getCycle(node) + return + + nodeInfo.mark = nmTemporary + + let neighbors = graph[node].dependencies + for node2 in neighbors: + nodesInfo[node2].cameFrom = node + visit(node2) + + nodeInfo.mark = nmPermanent + order.add node + + for node, nodeInfo in nodesInfo: + if nodeInfo.mark != nmPermanent: + visit(node) + + if cycles.len > 0: + printNotADagWarning() + + return (order, cycles) + +when isMainModule: + import unittest + from version import notSetVersion + from sha1hashes import notSetSha1Hash + + proc initLockFileDep(deps: seq[string] = @[]): LockFileDep = + result = LockFileDep( + version: notSetVersion, + vcsRevision: notSetSha1Hash, + dependencies: deps, + checksums: Checksums(sha1: notSetSha1Hash)) + + suite "topological sort": + + test "graph without cycles": + let + graph = { + "json_serialization": initLockFileDep( + @["serialization", "stew"]), + "faststreams": initLockFileDep(@["stew"]), + "testutils": initLockFileDep(), + "stew": initLockFileDep(), + "serialization": initLockFileDep(@["faststreams", "stew"]), + "chronicles": initLockFileDep( + @["json_serialization", "testutils"]) + }.toOrderedTable + + expectedTopologicallySortedOrder = @[ + "stew", "faststreams", "serialization", "json_serialization", + "testutils", "chronicles"] + expectedCycles: seq[seq[string]] = @[] + + (actualTopologicallySortedOrder, actualCycles) = topologicalSort(graph) + + check actualTopologicallySortedOrder == expectedTopologicallySortedOrder + check actualCycles == expectedCycles + + test "graph with cycles": + let + graph = { + "A": initLockFileDep(@["B", "E"]), + "B": initLockFileDep(@["A", "C"]), + "C": initLockFileDep(@["D"]), + "D": initLockFileDep(@["B"]), + "E": initLockFileDep(@["D", "E"]) + }.toOrderedTable + + expectedTopologicallySortedOrder = @["D", "C", "B", "E", "A"] + expectedCycles = @[@["A", "B", "A"], @["B", "C", "D", "B"], @["E", "E"]] + + (actualTopologicallySortedOrder, actualCycles) = topologicalSort(graph) + + check actualTopologicallySortedOrder == expectedTopologicallySortedOrder + check actualCycles == expectedCycles diff --git a/src/nimblepkg/vcstools.nim b/src/nimblepkg/vcstools.nim new file mode 100644 index 0000000..104bfd2 --- /dev/null +++ b/src/nimblepkg/vcstools.nim @@ -0,0 +1,1070 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## This module implements some operations which use version control system +## tools like Git and Mercurial on which Nimble depends. + +import tables, strutils, strformat, os, sets +import common, paths, tools, sha1hashes + +type + VcsType* = enum + ## Represents a marker for the type of VCS under which is some file system + ## directory. + vcsTypeNone = "none" + vcsTypeGit = "git" + vcsTypeHg = "hg" + + VcsTypeAndSpecialDirPath = tuple[vcsType: VcsType, path: Path] + ## Represents a cache entry for the directory VCS type and VCS special + ## directory path used by `getVcsTypeAndSpecialDirPath` procedure. + + BranchType* = enum + ## Determines the branch type which to be queried. + btLocal, btRemoteTracking, btBoth + + RemoteUrlType {.pure.} = enum + ## Represents the type of URL of some VCS remote repository. Fetch URLs are + ## for downloading data from the repository and push URLs are for uploading + ## data to it. + fetch, push + +const + noVcsSpecialDir = "" + gitSpecialDir = ".git" + hgSpecialDir = ".hg" + + noVcsDefaultBranch = "" + gitDefaultBranch = "master" + hgDefaultBranch = "default" + + noVcsDefaultRemote = "" + gitDefaultRemote = "origin" + hgDefaultRemote = "default" + +proc getVcsSpecialDir*(vcsType: VcsType): string = + ## Returns a special dir for given VCS type or an empty string for + ## `vcsTypeNone`. + return case vcsType + of vcsTypeNone: noVcsSpecialDir + of vcsTypeGit: gitSpecialDir + of vcsTypeHg: hgSpecialDir + +proc getVcsDefaultBranchName*(vcsType: VcsType): string = + ## Returns the name of the default branch for given VCS. + return case vcsType + of vcsTypeNone: noVcsDefaultBranch + of vcsTypeGit: gitDefaultBranch + of vcsTypeHg: hgDefaultBranch + +proc getVcsDefaultRemoteName*(vcsType: VcsType): string = + return case vcsType + of vcsTypeNone: noVcsDefaultRemote + of vcsTypeGit: gitDefaultRemote + of vcsTypeHg: hgDefaultRemote + +proc dirDoesNotExistErrorMsg(dir: Path): string = + &"The directory \"{dir}\" does not exist." + +proc hasVcsSubDir*(dir: Path): VcsType = + ## Checks whether a directory has a special subdirectory for some supported + ## kind of VCS. + if (dir / gitSpecialDir.Path).dirExists: + result = vcsTypeGit + elif (dir / hgSpecialDir.Path).dirExists: + result = vcsTypeHg + else: + result = vcsTypeNone + +proc getVcsTypeAndSpecialDirPath*(dir: Path): VcsTypeAndSpecialDirPath = + ## By given directory `dir` gets the type of VCS under which is it by + ## traversing the parent directories until some specific directory like + ## `.git`, `.hg` or the root of the file system is found. Additionally it + ## returns the path to the VCS special directory if the directory `dir is + ## under some supported VCS type. + ## + ## The procedure uses a in memory cache to bypass multiple checks for the same + ## directory in single run of Nimble. + ## + ## Raises a `NimbleError` in the case the directory `dir` does not exist. + + var cache {.global.}: Table[Path, VcsTypeAndSpecialDirPath] + if cache.hasKey(dir): + return cache[dir] + + if not dir.dirExists: + raise nimbleError(dirDoesNotExistErrorMsg(dir)) + + var + dirIter = dir + vcsType = vcsTypeNone + + while not dirIter.isRootDir: + vcsType = hasVcsSubDir(dirIter) + if vcsType != vcsTypeNone: + break + dirIter = dirIter.parentDir + + if vcsType == vcsTypeNone: + vcsType = hasVcsSubDir(dirIter) + else: + dirIter = dirIter / vcsType.getVcsSpecialDir.Path + + result = (vcsType, dirIter) + cache[dir] = result + +proc getVcsType*(dir: Path): VcsType = + ## Returns VCS type of the given directory. + ## Raises a `NimbleError` in the case the directory `dir` does not exist. + dir.getVcsTypeAndSpecialDirPath.vcsType + +proc git(path: Path): string = + ## Returns string for Git call at specific path `path`. + &"git -C {path.quoteShell}" + +proc hg(path: Path): string = + ## Returns string for Mercurial call at specific path `path`. + &"hg --cwd {path.quoteShell}" + +proc dirInNotUnderSourceControlErrorMsg*(dir: Path): string = + &"The directory \"{dir}\" is not under source control." + +template doVcsCmdImpl(dir: Path, gitCmd, hgCmd: string, + doCmd, noVcsAction: untyped): untyped = + ## This is a helper template for executing Git or Mercurial external command + ## `gitCmd` or `hgCmd` in the directory `dir` according to the type of version + ## control applied to the directory via some procedure `doCmd` or executing + ## some other action `noVcsAction` in the case it is not under some of the + ## supported VCS types. + + case getVcsType(dir) + of vcsTypeGit: + `doCmd`(git(dir) & " " & gitCmd) + of vcsTypeHg: + `doCmd`(hg(dir) & " " & hgCmd) + of vcsTypeNone: + `noVcsAction` + +template doVcsCmdImpl(dir: Path, gitCmd, hgCmd: string, + doCmd: untyped): untyped = + ## This is a helper template for executing Git or Mercurial external command + ## `gitCmd` or `hgCmd` in the directory `dir` according to the type of version + ## control applied to the directory via some procedure `doCmd` or raising a + ## `NimbleError` in the case it is not under some of the supported VCS types. + + doVcsCmdImpl(dir, gitCmd, hgCmd, doCmd): + raise nimbleError(dirInNotUnderSourceControlErrorMsg(dir)) + +template doVcsCmd(dir: Path, gitCmd, hgCmd: string): untyped = + ## This is a helper template for executing Git or Mercurial external command + ## `gitCmd` or `hgCmd` in the directory `dir` according to the type of version + ## control applied to the directory via `doCmdEx` procedure or raising a + ## `NimbleError` in the case it is not under some of the supported VCS types. + + doVcsCmdImpl(dir, gitCmd, hgCmd): doCmdEx + +template tryDoVcsCmd(dir: Path, gitCmd, hgCmd: string, + noVcsAction: untyped): untyped = + ## This is a helper template for executing Git or Mercurial external command + ## `gitCmd` or `hgCmd` in the directory `dir` according to the type of version + ## control applied to the directory via `tryDoCmdEx` procedure or executing + ## some other action `noVcsAction` in the case it is not under some of the + ## supported VCS types. + + doVcsCmdImpl(dir, gitCmd, hgCmd): + tryDoCmdEx + do: + `noVcsAction` + +template tryDoVcsCmd(dir: Path, gitCmd, hgCmd: string): untyped = + ## This is a helper template for executing Git or Mercurial external command + ## `gitCmd` or `hgCmd` in the directory `dir` according to the type of version + ## control applied to the directory via `tryDoCmdEx` procedure or raising a + ## `NimbleError` in the case it is not under some of the supported VCS types. + + doVcsCmdImpl(dir, gitCmd, hgCmd): tryDoCmdEx + +proc getVcsRevision*(dir: Path): Sha1Hash = + ## Returns current revision number if the directory `dir` is under version + ## control, or an invalid Sha1 checksum otherwise. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - there is no vcsRevisions in the repository. + + let vcsRevision = tryDoVcsCmd(dir, + gitCmd = "rev-parse HEAD", + hgCmd = "id -i --debug", + noVcsAction = $notSetSha1Hash) + + return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) + +proc getPackageFileListWithoutVcs(dir: Path): seq[string] = + ## Recursively walks the directory `dir` and returns a list of files in it and + ## its subdirectories. + for file in walkDirRec($dir, yieldFilter = {pcFile, pcLinkToFile}, + relative = true): + when defined(windows): + # On windows relative paths to files which are included in the calculation + # of the package checksum must be the same as on POSIX systems. + let file = file.replace('\\', '/') + result.add file + +proc getPackageFileList*(dir: Path): seq[string] = + ## Retrieves a sequence of file names from the directory `dir` and its + ## sub-directories by trying to get it from Git, Mercurial or directly from + ## the file system if both fail. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + + const noVcsOutput = "/" + + let output = tryDoVcsCmd(dir, + gitCmd = "ls-files", + hgCmd = "manifest", + noVcsAction = noVcsOutput) + + return + if output != noVcsOutput: + output.strip.splitLines + else: + dir.getPackageFileListWithoutVcs + +proc isWorkingCopyClean*(path: Path): bool = + ## Checks whether a repository at path `path` has a clean working copy. Do + ## not consider untracked and ignored files. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + let output = tryDoVcsCmd(path, + gitCmd = "status --untracked-files=no --porcelain", + hgCmd = "status -q --color=off") + return output.strip.len == 0 + +proc getRemotesNames*(path: Path): seq[string] = + ## Retrieves a sequence with the names of the set remotes for the repository + ## at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + let output = tryDoVcsCmd(path, + gitCmd = "remote", + hgCmd = "paths -q").strip + + if output.len > 0: + result = output.splitLines + +proc getRemoteUrl(path: Path, remoteName: string, + urlType: RemoteUrlType): string = + ## Retrieves a fetch or push URL for the remote with name `remoteName` set in + ## repository at path `repositoryPath`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + let fetchOrPush = case urlType + of RemoteUrlType.fetch: "" + of RemoteUrlType.push: "--push" + + result = tryDoVcsCmd(path, + gitCmd = &"remote get-url {fetchOrPush} {remoteName}", + hgCmd = &"paths {remoteName}") + + return result.strip + +proc getRemoteFetchUrl*(path: Path, remoteName: string): string = + ## Retrieves a fetch URL for the remote with name `remoteName` set in + ## repository at path `repositoryPath`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + getRemoteUrl(path, remoteName, RemoteUrlType.fetch) + +proc getRemotePushUrl*(path: Path, remoteName: string): string = + ## Retrieves a push URL for the remote with name `remoteName` set in + ## repository at path `repositoryPath`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + getRemoteUrl(path, remoteName, RemoteUrlType.push) + +proc getRemotesPushUrls*(path: Path): seq[string] = + ## Retrieves a sequence with the push URLs of the set remotes for the + ## repository at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + let remotesNames = path.getRemotesNames + result = newSeqOfCap[string](remotesNames.len) + for remote in remotesNames: + result.add getRemotePushUrl(path, remote) + +proc isVcsRevisionPresentOnSomeRemote*( + path: Path, vcsRevision: Sha1Hash): bool = + ## Checks whether a VCS revision `vcsRevision` is present on some of the + ## remotes of the repository at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external source control tool is not found. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + # Note: When `--depth=1` is missing the git command returns success even the + # revision is present only locally, but when it is present dispute being + # `--dry-run` the code below for some reason corrupts the working copy of the + # Git repository living it in a grafted state and for this reason another + # solution must be found. It seems like a bug in Git. + + # for remotePushUrl in path.getRemotesPushUrls: + # let + # remotePushUrl = remotePushUrl.quoteShell + # (_, exitCode) = doVcsCmd(path, + # gitCmd = &"fetch {remotePushUrl} {vcsRevision} -q --dry-run", + # hgCmd = &"pull {remotePushUrl} -r {vcsRevision} -q") + # if exitCode == QuitSuccess: + # return true + + let vcsType = path.getVcsType + if vcsType == vcsTypeGit: + for remotePushUrl in path.getRemotesPushUrls: + let + remotePushUrl = remotePushUrl.quoteShell + (_, fetchCmdExitCode) = doCmdEx(&"{git(path)} fetch {remotePushUrl}") + if fetchCmdExitCode == QuitFailure: + continue + + let (branchCmdOutput, branchCmdExitCode) = doCmdEx( + &"{git(path)} branch -r --contains {vcsRevision}") + if branchCmdExitCode == QuitSuccess and branchCmdOutput.len > 0: + return true + elif vcsType == vcsTypeHg: + for remotePushUrl in path.getRemotesPushUrls: + let + remotePushUrl = remotePushUrl.quoteShell + (_, exitCode) = doCmdEx( + &"{hg(path)} pull {remotePushUrl} -r {vcsRevision} -q") + if exitCode == QuitSuccess: + return true + else: + raise nimbleError(dirInNotUnderSourceControlErrorMsg(path)) + +proc getCurrentBranch*(path: Path): string = + ## Get the name of the current branch for the VCS repository at path `path`. + ## Returns an empty string in the case the repository is in a detached state. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + result = tryDoVcsCmd(path, + gitCmd = "branch --show-current", + hgCmd = "branch") + + return result.strip + +proc getBranchesOnWhichVcsRevisionIsPresent*( + path: Path, vcsRevision: Sha1Hash, branchType = btBoth): HashSet[string] = + ## Returns a set of the names of all branches which contain revision + ## `vcsRevision` for a repository at path `path`. If the VCS system is Git + ## `branchType` determines which branches to be returned: local branches, + ## remote tracking branches or both. The parameter has no effect for + ## Mercurial repositories. + ## + ## Note: In Mercurial a revision is present always only on a single branch. + ## For this reason we are searching for all branches where the revision is + ## found as an ancestor of some revision of the branch. + ## + ## Raises a `NimbleError` if: + ## - the external source control tool is not found. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + let + branchTypeParam = case branchType + of btLocal: "" + of btRemoteTracking: "-r" + of btBoth: "-a" + (output, errorCode) = doVcsCmd(path, + gitCmd = &"branch {branchTypeParam} --no-color --contains {vcsRevision}", + hgCmd = &"log -r {vcsRevision}:: -T '{{branch}}\\n'") + + if errorCode != QuitSuccess or output.len == 0: + # If the VCS revision is not found in any local branch Git exits with + # failure, but Mercurial exits with success and an empty output. In both + # cases we are returning an empty set. + return + + let vcsType = path.getVcsType + for line in output.strip.splitLines: + var line = line.strip(chars = Whitespace + {'*', '\''}) + if vcsType == vcsTypeGit and branchType == btBoth: + # For "git branch -a" remote branches are starting with "remotes" which + # have to be removed for uniformity with "git branch -r". + const prefix = "remotes/" + if line.startsWith(prefix): + line = line[prefix.len .. ^1] + if line.len > 0: + result.incl line + +proc isVcsRevisionPresentOnSomeBranch*(path: Path, vcsRevision: Sha1Hash): + bool = + ## Checks whether a given VCS revision `vcsRevision` is found on any local + ## branch of the repository at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external source control tool is not found. + ## - the directory does not exist. + ## - the directory is not under supported VCS type + + getBranchesOnWhichVcsRevisionIsPresent(path, vcsRevision).len > 0 + +proc isVcsRevisionPresentOnBranch*( + path: Path, vcsRevision: Sha1Hash, branchName: string): bool = + ## Checks whether a given VCS revision `vcsRevision` is present on a branch + ## with name `branchName` in a repository at path `path`. Returns `true` if + ## so or `false` if either the branch don't exist or it does not contain the + ## revision. + ## + ## Raises a `NimbleError` if: + ## - the external source control tool is not found. + ## - the directory does not exist. + ## - the directory is not under supported VCS type + + let branches = getBranchesOnWhichVcsRevisionIsPresent(path, vcsRevision) + return branches.contains(branchName) + +proc retrieveRemoteChangeSets*(path: Path, remoteName, branchName: string) = + ## Retrieve remote `remoteName` and branch `branchName` change sets for the + ## repository at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + discard tryDoVcsCmd(path, + gitCmd = &"fetch {remoteName} {branchName}", + hgCmd = &"pull {remoteName} -b {branchName}") + +proc retrieveRemoteChangeSets*(path: Path, remoteName: string) = + ## Retrieve remote `remoteName` change sets for the repository at path `path`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + discard tryDoVcsCmd(path, + gitCmd = &"fetch {remoteName}", + hgCmd = &"pull {remoteName}") + +proc retrieveRemoteChangeSets*(path: Path) = + ## Retrieves all change sets for the repository at path `path` from every + ## remote and every branch. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + for remote in getRemotesNames(path): + retrieveRemoteChangeSets(path, remote) + +proc setWorkingCopyToVcsRevision*(path: Path, vcsRevision: Sha1Hash) = + ## Sets working copy of a repository at path `path` to have active a + ## particular VCS revision `vcsRevision`. + ## + ## Note: This is a detached state in the case of Git or a revision's branch + ## in the case of Mercurial. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + discard tryDoVcsCmd(path, + gitCmd = &"checkout {vcsRevision}", + hgCmd = &"update {vcsRevision}") + +proc setCurrentBranchToVcsRevision*(path: Path, vcsRevision: Sha1Hash) = + ## Changes the current VCS revision for repository at path `path`. + ## + ## - For Git sets a current branch HEAD to point to the given VCS revision. + ## + ## - For Mercurial just updates the working copy to the given VCS revision, + ## because in Mercurial the branch is part of the commit meta data. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + discard tryDoVcsCmd(path, + gitCmd = &"reset --hard {vcsRevision}", + hgCmd = &"update {vcsRevision}") + +proc switchBranch*(path: Path, branchName: string) = + ## Switches the current working copy at path `path` branch to `branchName`. + ## + ## Raises a `NimbleError` if: + ## - the external command fails. + ## - the directory does not exist. + ## - the directory is not under supported VCS type. + + discard tryDoVcsCmd(path, + gitCmd = &"checkout {branchName}", + hgCmd = &"update {branchName}") + +proc getCorrespondingRemoteAndBranch*(path: Path): + tuple[remote, branch: string] = + ## Gets the name of the remote and the remote branch which current branch of + ## repository at path `path` tracks. If there is no such returns the default + ## remote and current branch names. + ## + ## Note: For Mercurial there is no such thing like remote tracing branch and + ## this procedure always returns the default remote and current branch name. + ## + ## Raises a `NimbleError` if: + ## - the external source control tool is not found. + ## - the directory does not exist. + ## - the directory is not under supported VCS type + + var + output: string + exitCode: int + + let vcsType = path.getVcsType + case vcsType + of vcsTypeGit: + (output, exitCode) = doCmdEx(git(path) & + " rev-parse --abbrev-ref --symbolic-full-name @{u}") + of vcsTypeHg: + (output, exitCode) = ("", QuitFailure) + of vcsTypeNone: + raise nimbleError(dirInNotUnderSourceControlErrorMsg(path)) + + if exitCode == QuitSuccess: + # Separate the remote name from the branch name. + let remotes = path.getRemotesNames + let output = output.strip + for remote in remotes: + if output.startsWith(remote): + return (remote, output[remote.len + 1 .. ^1]) + else: + return (vcsType.getVcsDefaultRemoteName, path.getCurrentBranch) + +proc hasCorrespondingRemoteBranch*(path: Path, remoteBranches: HashSet[string]): + tuple[hasBranch: bool, branchName: string] = + # If the directory at path `path` is a Git repository and its current branch + # has corresponding remote tracking branch in the provided set + # `remoteBranches` returns `true` and the name of the branch or `false` and an + # empty string otherwise. + + if path.getVcsType != vcsTypeGit: + return (false, "") + var (output, exitCode) = doCmdEx(git(path) & + " rev-parse --abbrev-ref --symbolic-full-name @{u}") + output = output.strip + result.hasBranch = exitCode == QuitSuccess and output in remoteBranches + if result.hasBranch: + result.branchName = output + +proc assertIsGitRepository(path: Path) = + assert path.getVcsType == vcsTypeGit, + "This procedure makes sense only for a Git repositories." + +proc getLocalBranchesTrackingRemoteBranch*(path: Path, remoteBranch: string): + seq[string] = + ## By given path to a Git repository and a remote tracking branch name + ## returns a sequence with all local branches which track the remote branch. + path.assertIsGitRepository + let output = tryDoCmdEx(git(path) & + &" for-each-ref --format=\"%(if:equals={remoteBranch})%(upstream:short)%" & + "(then)%(refname:short)%(end)\" refs/heads").strip + if output.len > 0: + output.split('\n') + else: + @[] + +proc getLocalBranchName*(path: Path, remoteBranch: string): string = + ## By given path to a Git repository and name of a remote branch returns a new + ## name which to be used for a local branch name which consists of the name + ## of the remote branch without a remote name prefix. For example: + ## + ## * "origin/master" -> "master" + ## * "upstream/feature/lock-file" -> "feature/lock-file" + + path.assertIsGitRepository + let remotes = path.getRemotesNames + for remote in remotes: + if remoteBranch.startsWith(remote): + return remoteBranch[remote.len + 1 .. ^1] + +proc fastForwardMerge*(path: Path, remoteBranch, localBranch: string) = + ## Tries to fast forward merge a remote branch `remoteBranch` to a local + ## branch `localBranch` in a Git repository at path `path`. + path.assertIsGitRepository + let currentBranch = path.getCurrentBranch + tryDoCmdEx(&"{git(path)} checkout --detach") + tryDoCmdEx(&"{git(path)} fetch . {remoteBranch}:{localBranch}") + if currentBranch.len > 0: + tryDoCmdEx(&"{git(path)} checkout {currentBranch}") + +when isMainModule: + import unittest, std/sha1, sequtils + + type + NameToVcsRevision = OrderedTable[string, Sha1Hash] + ## Maps some user supplied string id to VCS commit revision id. + + const + tempDir = getTempDir() + testGitDir = tempDir / "testGitDir" + testHgDir = tempDir / "testHgDir" + testNoVcsDir = tempDir / "testNoVcsDir" + testSubDir = "testSubDir" + testFile = "test.txt" + testFile2 = "test2.txt" + testFileContent = "This is a test file.\n" + testSubDirFile = &"{testSubDir}/{testFile}" + testRemotes: seq[tuple[name, url: string]] = @[ + ("origin", "testRemote1Dir"), + ("other", "testRemote2Dir"), + ("upstream", "testRemote3Dir")] + noSuchVcsRevisionSha1 = initSha1Hash( + "ffffffffffffffffffffffffffffffffffffffff") + newBranchName = "new-branch" + remoteNewBranch = &"{testRemotes[1].name}/{newBranchName}" + newBranchFileName = "test2.txt" + newBranchFileContent = "This is a new branch file content." + testRemoteCommitFile = "remote.txt" + + var nameToVcsRevision: NameToVcsRevision + + proc getMercurialRcFileContent(): string = + result = """ +[ui] +username = John Doe +[paths] +""" + for remote in testRemotes: + result &= &"{remote.name} = {testHgDir / remote.url}\n" + + proc initRepo(vcsType: VcsType, url = ".") = + tryDoCmdEx(&"{vcsType} init {url}") + if vcsType == vcsTypeGit: + tryDoCmdEx(&"git -C {url} config user.name \"John Doe\"") + tryDoCmdEx(&"git -C {url} config user.email \"john.doe@example.com\"") + + proc collectFiles(files: varargs[string]): string = + for file in files: result &= file & " " + + proc addFiles(vcsType: VcsType, files: varargs[string]) = + tryDoCmdEx(&"{vcsType} add {collectFiles(files)}") + + proc revertAddFiles(vcsType: VcsType, files: varargs[string]) = + let files = collectFiles(files) + case vcsType + of vcsTypeGit: + tryDoCmdEx(&"git reset HEAD -- {files}") + of vcsTypeHg: + tryDoCmdEx(&"hg revert {files}") + of vcsTypeNone: + assert false, "Must not enter here." + + proc commit(vcsType: VcsType, name: string) = + # Use user supplied name for the commit as commit message. + tryDoCmdEx(&"{vcsType} commit -m {name}") + nameToVcsRevision[name] = getVcsRevision(".") + + proc addRemotes(vcsType: VcsType) = + case vcsType + of vcsTypeGit: + for remote in testRemotes: + tryDoCmdEx(&"git remote add {remote.name} {remote.url}") + of vcsTypeHg: + writeFile(".hg/hgrc", getMercurialRcFileContent()) + of vcsTypeNone: + assert false, "VCS type must not be 'vcsTypeNone'." + + proc setupRemoteRepo(vcsType: VcsType, remoteUrl: string) = + createDir remoteUrl + initRepo(vcsType, remoteUrl) + if vcsType == vcsTypeGit: + cd remoteUrl: + tryDoCmdEx("git config receive.denyCurrentBranch ignore") + + proc setupRemoteRepos(vcsType: VcsType) = + for remote in testRemotes: + setupRemoteRepo(vcsType, remote.url) + + proc switchBranch(vcsType: VcsType, branchName: string) = + let command = case vcsType + of vcsTypeGit: "checkout" + of vcsTypeHg: "update" + of vcsTypeNone: + assert false, "Must not enter here."; "" + + tryDoCmdEx(&"{vcsType} {command} {branchName}") + + proc createCommitOnRemote(vcsType: VcsType, remoteName, remoteUrl: string) = + cd remoteUrl: + writeFile(testRemoteCommitFile, "") + addFiles(vcsType, testRemoteCommitFile) + commit(vcsType, remoteName) + + proc createCommitOnTestRemotes(vcsType: VcsType) = + for remote in testRemotes: + createCommitOnRemote(vcsType, remote.name, remote.url) + + proc setupNewBranch(vcsType: VcsType, branchName: string) = + tryDoCmdEx(&"{vcsType} branch {branchName}") + + proc pushToRemote(vcsType: VcsType, remoteName: string) = + if vcsType == vcsTypeGit and remoteName == gitDefaultRemote: + let branchName = getCurrentBranch(".") + tryDoCmdEx(&"git push --set-upstream {remoteName} {branchName}") + tryDoCmdEx(&"{vcsType} push {remoteName}") + + proc pushToTestRemotes(vcsType: VcsType) = + for remote in testRemotes: + pushToRemote(vcsType, remote.name) + + proc createTestFiles() = + writeFile(testFile, testFileContent) + createDir testSubDir + writeFile(testSubDirFile, "") + + proc commitInTheNewBranch(vcsType: VcsType, name: string) = + writeFile(newBranchFileName, newBranchFileContent) + addFiles(vcsType, newBranchFileName) + commit(vcsType, name) + + proc getExpectedLocalBranchesForVcsType(vcsType: VcsType): HashSet[string] = + let defaultBranchName = vcsType.getVcsDefaultBranchName + result = [defaultBranchName, newBranchName].toHashSet + + proc getExpectedRemoteTrackingBranchesForVcsType(vcsType: VcsType): + HashSet[string] = + if vcsType == vcsTypeGit: + for remote in testRemotes: + result.incl &"{remote.name}/{vcsType.getVcsDefaultBranchName}" + else: + result = vcsType.getExpectedLocalBranchesForVcsType + + proc getExpectedBranchesForVcsType(vcsType: VcsType): HashSet[string] = + result = vcsType.getExpectedLocalBranchesForVcsType + result = result + vcsType.getExpectedRemoteTrackingBranchesForVcsType + + proc createNewBranchOnTestRemotes(vcsType: VcsType) = + for remote in testRemotes: + cd remote.url: + setupNewBranch(vcsType, newBranchName) + + proc setupSuite(vcsType: VcsType, vcsTestDir: string) = + cdNewDir vcsTestDir: + initRepo(vcsType) + createTestFiles() + addFiles(vcsType, testFile, testSubDirFile) + commit(vcsType, vcsType.getVcsDefaultBranchName) + addRemotes(vcsType) + setupRemoteRepos(vcsType) + pushToTestRemotes(vcsType) + createCommitOnTestRemotes(vcsType) + createNewBranchOnTestRemotes(vcsType) + setupNewBranch(vcsType, newBranchName) + if vcsType == vcsTypeHg: + # Mercurial requires to have a commit for the new branch before + # switching to it. + commitInTheNewBranch(vcsType, newBranchName) + switchBranch(vcsType, newBranchName) + defer: + # Restore the main branch on scope exit. + switchBranch(vcsType, getVcsDefaultBranchName(vcsType)) + if vcsType != vcsTypeHg: + # In the case of Mercurial at this point the commit is already done. + commitInTheNewBranch(vcsType, newBranchName) + + template installRemoteTrackingBranch(testDir: string): untyped {.dirty.} = + # A hack for `testDir` to be available in `&` macro. + let td = testDir + # Fetch remote branch + tryDoCmdEx(&"git -C {td} fetch {testRemotes[1].name} {newBranchName}") + defer: + # Delete the newly fetched remote branch on scope exit to clean the + # state of the repo. + tryDoCmdEx( + &"git -C {td} branch -dr {testRemotes[1].name}/{newBranchName}") + + # Tell the current branch to track it + tryDoCmdEx( + &"git -C {td} branch -u {testRemotes[1].name}/{newBranchName}") + + proc setupNoVcsSuite() = + cdNewDir testNoVcsDir: + createTestFiles() + + proc tearDownSuite(dir: string) = + removeDir dir + + template suiteTestCode(vcsType: VcsType, testDir: string, + remoteUrlPath: untyped) {.dirty.} = + assert vcsType != vcsTypeNone, + "The type of the VCS must not be 'vcsTypeNone'" + + setupSuite(vcsType, testDir) + + test "getVcsTypeAndSpecialDirPath": + let expectedVcsSpecialDirPath = testDir.Path / getVcsSpecialDir(vcsType) + check getVcsTypeAndSpecialDirPath(testDir) == + (vcsType, expectedVcsSpecialDirPath) + check getVcsTypeAndSpecialDirPath(testDir / testSubDir) == + (vcsType, expectedVcsSpecialDirPath) + + test "getVcsRevision": + check isValidSha1Hash($getVcsRevision(testDir)) + + test "getPackageFileList": + check getPackageFileList(testDir) == @[testFile, testSubDirFile] + + test "isWorkingCopyClean": + check isWorkingCopyClean(testDir) + cd testDir: + # Make working copy state not clean. + writeFile(testFile2, "") + addFiles(vcsType, testFile2) + defer: + # Restore previous state on scope exit. + cd testDir: + revertAddFiles(vcsType, testFile2) + removeFile testFile2 + check not isWorkingCopyClean(testDir) + + test "getRemotesNames": + check getRemotesNames(testDir) == testRemotes.mapIt(it.name) + for remote in testRemotes: + # Test for empty list when there are not set remotes. + check getRemotesNames(testDir/remote.url) == newSeq[string]() + + test "getRemotePushUrl": + for remote in testRemotes: + check getRemotePushUrl(testDir, remote.name) == remoteUrlPath + + test "getRemotesPushUrls": + var remoteUrls: seq[string] + for remote in testRemotes: + # Test for empty list when there are not set remotes. + check getRemotesPushUrls(testDir/remote.url) == newSeq[string]() + remoteUrls.add remoteUrlPath + check getRemotesPushUrls(testDir) == remoteUrls + + test "isVcsRevisionPresentOnSomeRemote": + let vcsRevision = getVcsRevision(testDir) + check isVcsRevisionPresentOnSomeRemote(testDir, vcsRevision) + check not isVcsRevisionPresentOnSomeRemote(testDir, noSuchVcsRevisionSha1) + + test "getCurrentBranch": + let vcsDefaultBranchName = getVcsDefaultBranchName(vcsType) + check getCurrentBranch(testDir) == vcsDefaultBranchName + cd testDir: + switchBranch(vcsType, newBranchName) + defer: switchBranch(vcsType, vcsDefaultBranchName) + check getCurrentBranch(".") == newBranchName + check getCurrentBranch(testDir) == vcsDefaultBranchName + + test "getBranchesOnWhichVcsRevisionIsPresent": + let vcsRevision = getVcsRevision(testDir) + + check getBranchesOnWhichVcsRevisionIsPresent( + testDir, vcsRevision, btBoth) == + vcsType.getExpectedBranchesForVcsType + + check getBranchesOnWhichVcsRevisionIsPresent( + testDir, vcsRevision, btLocal) == + vcsType.getExpectedLocalBranchesForVcsType + + check getBranchesOnWhichVcsRevisionIsPresent( + testDir, vcsRevision, btRemoteTracking) == + vcsType.getExpectedRemoteTrackingBranchesForVcsType + + check getBranchesOnWhichVcsRevisionIsPresent( + testDir, noSuchVcsRevisionSha1) == HashSet[string]() + + test "isVcsRevisionPresentOnSomeBranch": + check isVcsRevisionPresentOnSomeBranch( + testDir, getVcsRevision(testDir)) + check not isVcsRevisionPresentOnSomeBranch( + testDir, noSuchVcsRevisionSha1) + + test "isVcsRevisionPresentOnBranch": + let vcsRevision = getVcsRevision(testDir) + let branchName = getCurrentBranch(testDir) + check isVcsRevisionPresentOnBranch(testDir, vcsRevision, branchName) + check not isVcsRevisionPresentOnBranch( + testDir, noSuchVcsRevisionSha1, branchName) + check not isVcsRevisionPresentOnBranch( + testDir, vcsRevision, "not-existing-branch") + + test "retrieveRemoteChangeSets (for single remote and branch)": + let remoteName = testRemotes[2].name + let remoteVcsRevision = nameToVcsRevision[remoteName] + check not isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + retrieveRemoteChangeSets(testDir, remoteName, + vcsType.getVcsDefaultBranchName) + check isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + + test "retrieveRemoteChangeSets (for single remote)": + let remoteName = testRemotes[0].name + let remoteVcsRevision = nameToVcsRevision[remoteName] + check not isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + retrieveRemoteChangeSets(testDir, remoteName) + check isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + + test "retrieveRemoteChangeSets (for all remotes and branches)": + let remoteVcsRevision = nameToVcsRevision[testRemotes[1].name] + check not isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + retrieveRemoteChangeSets(testDir) + check isVcsRevisionPresentOnSomeBranch(testDir, remoteVcsRevision) + + test "setWorkingCopyToVcsRevision": + let oldRevision = getVcsRevision(testDir) + let changeRevision = nameToVcsRevision[newBranchName] + setWorkingCopyToVcsRevision(testDir, changeRevision) + defer: + # Restore the repository state at scope exit. + cd testDir: switchBranch(vcsType, getVcsDefaultBranchName(vcsType)) + let newRevision = getVcsRevision(testDir) + check newRevision != oldRevision + check newRevision == changeRevision + + test "setCurrentBranchToVcsRevision": + let oldRevision = getVcsRevision(testDir) + let changeRevision = nameToVcsRevision[newBranchName] + let branchName = getCurrentBranch(testDir) + setCurrentBranchToVcsRevision(testDir, changeRevision) + defer: + # Restore the repository state at scope exit. + setCurrentBranchToVcsRevision(testDir, oldRevision) + check getVcsRevision(testDir) == oldRevision + let newRevision = getVcsRevision(testDir) + # Check that the revision is actually changed, + check newRevision != oldRevision + # to the intended one. + check newRevision == changeRevision + case vcsType + of vcsTypeGit: + # In the case of Git test that the branch is not changed, + check getCurrentBranch(testDir) == branchName + of vcsTypeHg: + # but for Mercurial the branch name is part of the commit meta data + # and it will be changed. + check getCurrentBranch(testDir) == newBranchName + of vcsTypeNone: + assert false, "Must not enter here." + + test "switchBranch": + switchBranch(testDir, newBranchName) + defer: + # Restore the repository state at scope exit. + cd testDir: switchBranch(vcsType, getVcsDefaultBranchName(vcsType)) + check getCurrentBranch(testDir) == newBranchName + expect NimbleError: switchBranch(testDir, "not-existing-branch") + + test "getCorrespondingRemoteAndBranch": + let (remote, branch) = testDir.getCorrespondingRemoteAndBranch + # There is no setup remote tracking branch and the default remote name for + # the VCS type and current branch name are returned. + check remote == vcsType.getVcsDefaultRemoteName + check branch == testDir.getCurrentBranch + + if vcsType == vcsTypeGit: + testDir.installRemoteTrackingBranch + let (remote, branch) = testDir.getCorrespondingRemoteAndBranch + check remote == testRemotes[1].name + check branch == newBranchName + + test "hasCorrespondingRemoteBranch": + if vcsType == vcsTypeGit: + testDir.installRemoteTrackingBranch + let remoteTrackingBranches = getBranchesOnWhichVcsRevisionIsPresent( + testDir, testDir.getVcsRevision, btRemoteTracking) + check testDir.hasCorrespondingRemoteBranch(remoteTrackingBranches) == + (true, remoteNewBranch) + else: + check testDir.hasCorrespondingRemoteBranch(HashSet[string]()) == + (false, "") + + test "getLocalBranchesTrackingRemoteBranch": + if vcsType == vcsTypeGit: + testDir.installRemoteTrackingBranch + check testDir.getLocalBranchesTrackingRemoteBranch("not-existing") == + newSeqOfCap[string](0) + check testDir.getLocalBranchesTrackingRemoteBranch(remoteNewBranch) == + @[vcsType.getVcsDefaultBranchName] + else: + skip() + + test "getLocalBranchName": + if vcsType == vcsTypeGit: + check testDir.getLocalBranchName(remoteNewBranch) == newBranchName + else: + skip() + + test "fastForwardMerge": + if vcsType == vcsTypeGit: + const testBranchName = "test-branch" + cd testDir: + vcsType.setupNewBranch(testBranchName) + vcsType.switchBranch(testBranchName) + testDir.installRemoteTrackingBranch + cd testDir: vcsType.switchBranch(vcsType.getVcsDefaultBranchName) + testDir.fastForwardMerge(remoteNewBranch, testBranchName) + expect NimbleError: + testDir.fastForwardMerge(remoteNewBranch, newBranchName) + else: + skip() + + tearDownSuite(testDir) + + suite "Git": + suiteTestCode(vcsTypeGit, testGitDir): remote.url + + suite "Mercurial": + suiteTestCode(vcsTypeHg, testHgDir): + testHgDir / remote.url + + suite "no version control": + setupNoVcsSuite() + + test "getVcsTypeAndSpecialDirPath": + const rootDir = when defined(windows): ':' else: '/' + let (vcsType, specialDirPath) = getVcsTypeAndSpecialDirPath(testNoVcsDir) + check vcsType == vcsTypeNone + check ($specialDirPath)[^1] == rootDir + + test "getVcsRevision": + check not isValidSha1Hash($getVcsRevision(testNoVcsDir)) + + test "getPackageFileList": + check getPackageFileList(testNoVcsDir) == @[testFile, testSubDirFile] + + tearDownSuite(testNoVcsDir) diff --git a/src/nimblepkg/version.nim b/src/nimblepkg/version.nim new file mode 100644 index 0000000..94d5a82 --- /dev/null +++ b/src/nimblepkg/version.nim @@ -0,0 +1,360 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +## Module for handling versions and version ranges such as ``>= 1.0 & <= 1.5`` +import strutils, tables, hashes, parseutils +type + Version* = distinct string + + VersionRangeEnum* = enum + verLater, # > V + verEarlier, # < V + verEqLater, # >= V -- Equal or later + verEqEarlier, # <= V -- Equal or earlier + verIntersect, # > V & < V + verEq, # V + verAny, # * + verSpecial # #head + + VersionRange* = ref VersionRangeObj + VersionRangeObj = object + case kind*: VersionRangeEnum + of verLater, verEarlier, verEqLater, verEqEarlier, verEq: + ver*: Version + of verSpecial: + spe*: Version + of verIntersect: + verILeft, verIRight: VersionRange + of verAny: + nil + + ## Tuple containing package name and version range. + PkgTuple* = tuple[name: string, ver: VersionRange] + + ParseVersionError* = object of ValueError + NimbleError* = object of CatchableError + hint*: string + +proc `$`*(ver: Version): string {.borrow.} + +proc hash*(ver: Version): Hash {.borrow.} + +proc newVersion*(ver: string): Version = + doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits, + "Wrong version: " & ver) + return Version(ver) + +proc isSpecial*(ver: Version): bool = + return ($ver).len > 0 and ($ver)[0] == '#' + +proc `<`*(ver: Version, ver2: Version): bool = + # Handling for special versions such as "#head" or "#branch". + if ver.isSpecial or ver2.isSpecial: + # TODO: This may need to be reverted. See #311. + if ver2.isSpecial and ($ver2).normalize == "#head": + return ($ver).normalize != "#head" + + if not ver2.isSpecial: + # `#aa111 < 1.1` + return ($ver).normalize != "#head" + + # Handling for normal versions such as "0.1.0" or "1.0". + var sVer = string(ver).split('.') + var sVer2 = string(ver2).split('.') + for i in 0..max(sVer.len, sVer2.len)-1: + var sVerI = 0 + if i < sVer.len: + discard parseInt(sVer[i], sVerI) + var sVerI2 = 0 + if i < sVer2.len: + discard parseInt(sVer2[i], sVerI2) + if sVerI < sVerI2: + return true + elif sVerI == sVerI2: + discard + else: + return false + +proc `==`*(ver: Version, ver2: Version): bool = + if ver.isSpecial or ver2.isSpecial: + return ($ver).toLowerAscii() == ($ver2).toLowerAscii() + + var sVer = string(ver).split('.') + var sVer2 = string(ver2).split('.') + for i in 0..max(sVer.len, sVer2.len)-1: + var sVerI = 0 + if i < sVer.len: + discard parseInt(sVer[i], sVerI) + var sVerI2 = 0 + if i < sVer2.len: + discard parseInt(sVer2[i], sVerI2) + if sVerI == sVerI2: + result = true + else: + return false + +proc cmp*(a, b: Version): int = + if a < b: -1 + elif a > b: 1 + else: 0 + +proc `<=`*(ver: Version, ver2: Version): bool = + return (ver == ver2) or (ver < ver2) + +proc `==`*(range1: VersionRange, range2: VersionRange): bool = + if range1.kind != range2.kind : return false + result = case range1.kind + of verLater, verEarlier, verEqLater, verEqEarlier, verEq: + range1.ver == range2.ver + of verSpecial: + range1.spe == range2.spe + of verIntersect: + range1.verILeft == range2.verILeft and range1.verIRight == range2.verIRight + of verAny: true + +proc withinRange*(ver: Version, ran: VersionRange): bool = + case ran.kind + of verLater: + return ver > ran.ver + of verEarlier: + return ver < ran.ver + of verEqLater: + return ver >= ran.ver + of verEqEarlier: + return ver <= ran.ver + of verEq: + return ver == ran.ver + of verSpecial: + return ver == ran.spe + of verIntersect: + return withinRange(ver, ran.verILeft) and withinRange(ver, ran.verIRight) + of verAny: + return true + +proc contains*(ran: VersionRange, ver: Version): bool = + return withinRange(ver, ran) + +proc makeRange*(version: string, op: string): VersionRange = + if version == "": + raise newException(ParseVersionError, + "A version needs to accompany the operator.") + case op + of ">": + result = VersionRange(kind: verLater) + of "<": + result = VersionRange(kind: verEarlier) + of ">=": + result = VersionRange(kind: verEqLater) + of "<=": + result = VersionRange(kind: verEqEarlier) + of "": + result = VersionRange(kind: verEq) + else: + raise newException(ParseVersionError, "Invalid operator: " & op) + result.ver = Version(version) + +proc parseVersionRange*(s: string): VersionRange = + # >= 1.5 & <= 1.8 + if s.len == 0: + result = VersionRange(kind: verAny) + return + + if s[0] == '#': + result = VersionRange(kind: verSpecial) + result.spe = s.Version + return + + var i = 0 + var op = "" + var version = "" + while i < s.len: + case s[i] + of '>', '<', '=': + op.add(s[i]) + of '&': + result = VersionRange(kind: verIntersect) + result.verILeft = makeRange(version, op) + + # Parse everything after & + # Recursion <3 + result.verIRight = parseVersionRange(substr(s, i + 1)) + + # Disallow more than one verIntersect. It's pointless and could lead to + # major unpredictable mistakes. + if result.verIRight.kind == verIntersect: + raise newException(ParseVersionError, + "Having more than one `&` in a version range is pointless") + + return + + of '0'..'9', '.': + version.add(s[i]) + + of ' ': + # Make sure '0.9 8.03' is not allowed. + if version != "" and i < s.len - 1: + if s[i+1] in {'0'..'9', '.'}: + raise newException(ParseVersionError, + "Whitespace is not allowed in a version literal.") + + else: + raise newException(ParseVersionError, + "Unexpected char in version range '" & s & "': " & s[i]) + inc(i) + result = makeRange(version, op) + +proc toVersionRange*(ver: Version): VersionRange = + ## Converts a version to either a verEq or verSpecial VersionRange. + new(result) + if ver.isSpecial: + result = VersionRange(kind: verSpecial) + result.spe = ver + else: + result = VersionRange(kind: verEq) + result.ver = ver + +proc parseRequires*(req: string): PkgTuple = + try: + if ' ' in req: + var i = skipUntil(req, Whitespace) + result.name = req[0 .. i].strip + result.ver = parseVersionRange(req[i .. req.len-1]) + elif '#' in req: + var i = skipUntil(req, {'#'}) + result.name = req[0 .. i-1] + result.ver = parseVersionRange(req[i .. req.len-1]) + else: + result.name = req.strip + result.ver = VersionRange(kind: verAny) + except ParseVersionError: + raise newException(NimbleError, + "Unable to parse dependency version range: " & getCurrentExceptionMsg()) + +proc `$`*(verRange: VersionRange): string = + case verRange.kind + of verLater: + result = "> " + of verEarlier: + result = "< " + of verEqLater: + result = ">= " + of verEqEarlier: + result = "<= " + of verEq: + result = "" + of verSpecial: + return $verRange.spe + of verIntersect: + return $verRange.verILeft & " & " & $verRange.verIRight + of verAny: + return "any version" + + result.add(string(verRange.ver)) + +proc getSimpleString*(verRange: VersionRange): string = + ## Gets a string with no special symbols and spaces. Used for dir name + ## creation in tools.nim + case verRange.kind + of verSpecial: + result = $verRange.spe + of verLater, verEarlier, verEqLater, verEqEarlier, verEq: + result = $verRange.ver + of verIntersect: + result = getSimpleString(verRange.verILeft) & "_" & + getSimpleString(verRange.verIRight) + of verAny: + result = "" + +proc newVRAny*(): VersionRange = + result = VersionRange(kind: verAny) + +proc newVREarlier*(ver: string): VersionRange = + result = VersionRange(kind: verEarlier) + result.ver = newVersion(ver) + +proc newVREq*(ver: string): VersionRange = + result = VersionRange(kind: verEq) + result.ver = newVersion(ver) + +proc findLatest*(verRange: VersionRange, + versions: OrderedTable[Version, string]): tuple[ver: Version, tag: string] = + result = (newVersion(""), "") + for ver, tag in versions: + if not withinRange(ver, verRange): continue + if ver > result.ver: + result = (ver, tag) + +proc `$`*(dep: PkgTuple): string = + return dep.name & "@" & $dep.ver + +when isMainModule: + doAssert(newVersion("1.0") < newVersion("1.4")) + doAssert(newVersion("1.0.1") > newVersion("1.0")) + doAssert(newVersion("1.0.6") <= newVersion("1.0.6")) + doAssert(not withinRange(newVersion("0.1.0"), parseVersionRange("> 0.1"))) + doAssert(not (newVersion("0.1.0") < newVersion("0.1"))) + doAssert(not (newVersion("0.1.0") > newVersion("0.1"))) + doAssert(newVersion("0.1.0") < newVersion("0.1.0.0.1")) + doAssert(newVersion("0.1.0") <= newVersion("0.1")) + + var inter1 = parseVersionRange(">= 1.0 & <= 1.5") + doAssert inter1.kind == verIntersect + var inter2 = parseVersionRange("1.0") + doAssert(inter2.kind == verEq) + + doAssert(not withinRange(newVersion("1.5.1"), inter1)) + doAssert(withinRange(newVersion("1.0.2.3.4.5.6.7.8.9.10.11.12"), inter1)) + + doAssert(newVersion("1") == newVersion("1")) + doAssert(newVersion("1.0.2.4.6.1.2.123") == newVersion("1.0.2.4.6.1.2.123")) + doAssert(newVersion("1.0.2") != newVersion("1.0.2.4.6.1.2.123")) + doAssert(newVersion("1.0.3") != newVersion("1.0.2")) + + doAssert(not (newVersion("") < newVersion("0.0.0"))) + doAssert(newVersion("") < newVersion("1.0.0")) + doAssert(newVersion("") < newVersion("0.1.0")) + + var versions = toOrderedTable[Version, string]({ + newVersion("0.1.1"): "v0.1.1", + newVersion("0.2.3"): "v0.2.3", + newVersion("0.5"): "v0.5" + }) + doAssert findLatest(parseVersionRange(">= 0.1 & <= 0.4"), versions) == + (newVersion("0.2.3"), "v0.2.3") + + # TODO: Allow these in later versions? + #doAssert newVersion("0.1-rc1") < newVersion("0.2") + #doAssert newVersion("0.1-rc1") < newVersion("0.1") + + # Special tests + doAssert newVersion("#ab26sgdt362") != newVersion("#qwersaggdt362") + doAssert newVersion("#ab26saggdt362") == newVersion("#ab26saggdt362") + doAssert newVersion("#head") == newVersion("#HEAD") + doAssert newVersion("#head") == newVersion("#head") + + var sp = parseVersionRange("#ab26sgdt362") + doAssert newVersion("#ab26sgdt362") in sp + doAssert newVersion("#ab26saggdt362") notin sp + + doAssert newVersion("#head") in parseVersionRange("#head") + + # We assume that #head > 0.1.0, in practice this shouldn't be a problem. + doAssert(newVersion("#head") > newVersion("0.1.0")) + doAssert(not(newVersion("#head") > newVersion("#head"))) + doAssert(withinRange(newVersion("#head"), parseVersionRange(">= 0.5.0"))) + doAssert newVersion("#a111") < newVersion("#head") + # We assume that all other special versions are not higher than a normal + # version. + doAssert newVersion("#a111") < newVersion("1.1") + + # An empty version range should give verAny + doAssert parseVersionRange("").kind == verAny + + # toVersionRange tests + doAssert toVersionRange(newVersion("#head")).kind == verSpecial + doAssert toVersionRange(newVersion("0.2.0")).kind == verEq + + # Something raised on IRC + doAssert newVersion("1") == newVersion("1.0") + + echo("Everything works!") diff --git a/src/private/nix.nim b/src/private/nix.nim new file mode 100644 index 0000000..03aeae2 --- /dev/null +++ b/src/private/nix.nim @@ -0,0 +1,120 @@ +import std/sequtils, std/strutils, std/tables + +type + ValueKind* = enum + tInt, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, + tExternal, + tFloat + Value* = ref object + case kind: ValueKind + of tInt: + num*: BiggestInt + of tBool: + boolean*: bool + of tString: + str*: string + of tPath: + path*: string + of tNull: + discard + of tAttrs: + attrs*: Table[string, Value] + of tList: + list*: seq[Value] + of tThunk: discard + of tApp: discard + of tLambda: discard + of tBlackhole: discard + of tPrimOp: discard + of tPrimOpApp: discard + of tExternal: discard + of tFloat: + fnum*: float + +func `$`*(v: Value): string = + case v.kind: + of tInt: + result = $v.num + of tBool: + result = if v.boolean: "True" else: "False" + of tString: + result = "\"$1\"" % (v.str.replace("\"", "\\\"")) + of tPath: + result = v.path + of tNull: + result = "null" + of tAttrs: + result = "{" + for key, val in v.attrs.pairs: + let key = if key.validIdentifier: key else: key.escape + result.add("$1=$2;" % [key, $val]) + result.add "}" + of tList: + result = "[ " + for e in v.list: + result.add $e + result.add " " + result.add "]" + of tFloat: + result = $v.fnum + else: + result = $v.kind + +func toNix*(x: Value): Value = x + +func toNix*(x: SomeInteger): Value = + Value(kind: tInt, num: x) + +func toNix*(x: bool): Value = + Value(kind: tBool, boolean: x) + +func toNix*(x: string): Value = + Value(kind: tString, str: x) + +func toPath*(x: string): Value = + Value(kind: tPath, path: x) + +template toNix*(pairs: openArray[(string, Value)]): Value = + Value(kind: tAttrs, attrs: toTable pairs) + +func toNix*(x: seq[Value]): Value = + Value(kind: tList, list: x) + +template toNix*(x: seq[untyped]): Value = + map(x, toNix).toNix + +func toNix*(x: openarray[Value]): Value = + Value(kind: tList, list: x[x.low .. x.high]) + +func toNix*(x: float): Value = + Value(kind: tFloat, fnum: x) + +template `[]=`*(result: Value; key: string; val: untyped) = + result.attrs[key] = val.toNix + +proc `[]`*(attrs: Value; key: string): Value = + attrs.attrs[key] + +proc newNull*(): Value = + Value(kind: tNull) + +proc newAttrs*(): Value = + Value(kind: tAttrs, attrs: initTable[string, Value]()) + +func newList*(): Value = + Value(kind: tList) + +proc add*(x, y: Value) = + x.list.add y