Builds pkgs binaries in parallel for vnext

.gitmodules

@@ -10,3 +10,9 @@
 [submodule "vendor/chronos"]
 	path = vendor/chronos
 	url = https://github.com/status-im/nim-chronos.git
+[submodule "vendor/results"]
+	path = vendor/results
+	url = https://github.com/arnetheduck/nim-results.git
+[submodule "vendor/stew"]
+	path = vendor/stew
+	url = https://github.com/status-im/nim-stew.git

config.nims

@@ -6,5 +6,8 @@ switch("define", "ssl")
 switch("path", "vendor" / "zippy" / "src")
 switch("path", "vendor" / "sat" / "src")
 switch("path", "vendor" / "checksums" / "src")
+switch("path", "vendor" / "chronos")
+switch("path", "vendor" / "results")
+switch("path", "vendor" / "stew")
 switch("define", "zippyNoSimd")
 

src/nimble.nim

@@ -5,7 +5,7 @@ import os, tables, strtabs, json, browsers, algorithm, sets, uri, sugar, sequtil
        strformat
 
 import std/options as std_opt
-
+import chronos
 import strutils except toLower
 from unicode import toLower
 import sat/sat
@@ -660,10 +660,13 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     # Copy this package's files based on the preferences specified in PkgInfo.
     var filesInstalled: HashSet[string]
     iterInstallFiles(realDir, pkgInfo, options,
-      proc (file: string) =
-        createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir))
-        let dest = changeRoot(realDir, pkgDestDir, file)
-        filesInstalled.incl copyFileD(file, dest)
+      proc (file: string) {.raises: [].} =
+        try:
+          createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir))
+          let dest = changeRoot(realDir, pkgDestDir, file)
+          filesInstalled.incl copyFileD(file, dest)
+        except Exception:
+          discard
     )
 
     # Copy the .nimble file.
@@ -2566,9 +2569,22 @@ proc run(options: Options, nimBin: string) =
     # In vnext path, build develop mode packages (similar to old code path)
     if pkgInfo.isLink:
       # Use vnext buildPkg for develop mode packages
-      let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and 
+      let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and
         options.satResult.rootPackage.basicInfo.name == pkgInfo.basicInfo.name
-      buildPkg(nimBin, pkgInfo, isInRootDir, options)
+      let buildTasks = waitFor buildPkg(nimBin, pkgInfo, isInRootDir, options)
+      # Execute the build tasks immediately for run command
+      for task in buildTasks:
+        let future = task.startBuild()
+        let response = waitFor future
+        if response.stdOutput.len > 0:
+          display("Output", response.stdOutput, priority = HighPriority)
+        if response.stdError.len > 0:
+          displayWarning(response.stdError)
+        if response.status != 0:
+          raise buildFailed(&"Build failed for binary: {task.bin}", details = nil)
+      # Create symlinks after build completes
+      if buildTasks.len > 0:
+        vnext.createBinSymlink(pkgInfo, options)
 
     if options.getCompilationFlags.len > 0:
       displayWarning(ignoringCompilationFlagsMsg)

src/nimblepkg/asyncfileops.nim

@@ -0,0 +1,39 @@
+# Async file operations using chronos async processes
+# Similar to Node.js - uses external commands for file I/O
+
+import std/os
+import chronos except Duration
+import chronos/asyncproc
+
+export chronos except Duration
+export asyncproc
+
+proc copyFileAsync*(source, dest: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} =
+  ## Async file copy using chronos async processes
+  when defined(windows):
+    # Windows: use xcopy for better handling
+    let cmd = "xcopy /Y /Q " & quoteShell(source) & " " & quoteShell(dest) & "*"
+  else:
+    # Unix: use cp command with preserve permissions and recursive for dirs
+    let cmd = "cp -f -p -r " & quoteShell(source) & " " & quoteShell(dest)
+
+  let exitCode = await execCommand(cmd)
+  if exitCode != 0:
+    raise newException(IOError, "Failed to copy file from " & source & " to " & dest & " (exit code: " & $exitCode & ")")
+
+proc copyDirAsync*(sourceDir, destDir: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} =
+  ## Async directory copy using chronos async processes - copies entire directory tree
+  when defined(windows):
+    # Windows: use robocopy for robust directory copying
+    # /E = copy subdirs including empty, /NFL = no file list, /NDL = no dir list, /NJH = no job header, /NJS = no job summary, /NC = no class, /NS = no size, /NP = no progress
+    let cmd = "robocopy " & quoteShell(sourceDir) & " " & quoteShell(destDir) & " /E /NFL /NDL /NJH /NJS /NC /NS /NP"
+    let exitCode = await execCommand(cmd)
+    # robocopy exit codes: 0-7 are success (0=no files, 1=files copied, 2=extra files, etc.)
+    if exitCode > 7:
+      raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")")
+  else:
+    # Unix: use cp -r to copy entire directory recursively
+    let cmd = "cp -r -p " & quoteShell(sourceDir) & "/. " & quoteShell(destDir)
+    let exitCode = await execCommand(cmd)
+    if exitCode != 0:
+      raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")")

src/nimblepkg/cli.nim

@@ -76,22 +76,23 @@ proc displayInfoLine*(field, msg: string) =
 
 proc displayCategory(category: string, displayType: DisplayType,
                      priority: Priority) =
-  if isSuppressed(displayType):
-    return
+  {.cast(gcsafe).}:
+    if isSuppressed(displayType):
+      return
 
-  # Calculate how much the `category` must be offset to align along a center
-  # line.
-  let offset = calculateCategoryOffset(category)
-
-  # Display the category.
-  let text = "$1$2 " % [spaces(offset), category]
-  if globalCLI.showColor:
-    if priority != DebugPriority:
-      setForegroundColor(stdout, foregrounds[displayType])
-    writeStyled(text, styles[priority])
-    resetAttributes()
-  else:
-    stdout.write(text)
+    # Calculate how much the `category` must be offset to align along a center
+    # line.
+    let offset = calculateCategoryOffset(category)
+
+    # Display the category.
+    let text = "$1$2 " % [spaces(offset), category]
+    if globalCLI.showColor:
+      if priority != DebugPriority:
+        setForegroundColor(stdout, foregrounds[displayType])
+      writeStyled(text, styles[priority])
+      resetAttributes()
+    else:
+      stdout.write(text)
 
 
 proc displayLine(category, line: string, displayType: DisplayType,
@@ -106,27 +107,28 @@ proc displayLine(category, line: string, displayType: DisplayType,
 
 proc display*(category, msg: string, displayType = Message,
               priority = MediumPriority) =
-  # Multiple warnings containing the same messages should not be shown.
-  let warningPair = (category, msg)
-  if displayType == Warning:
-    if warningPair in globalCLI.warnings:
-      return
-    else:
-      globalCLI.warnings.incl(warningPair)
+  {.cast(gcsafe).}:
+    # Multiple warnings containing the same messages should not be shown.
+    let warningPair = (category, msg)
+    if displayType == Warning:
+      if warningPair in globalCLI.warnings:
+        return
+      else:
+        globalCLI.warnings.incl(warningPair)
 
-  # Suppress this message if its priority isn't high enough.
-  # TODO: Per-priority suppression counts?
-  if priority < globalCLI.level:
-    if priority != DebugPriority:
-      globalCLI.suppressionCount.inc
-    return
+    # Suppress this message if its priority isn't high enough.
+    # TODO: Per-priority suppression counts?
+    if priority < globalCLI.level:
+      if priority != DebugPriority:
+        globalCLI.suppressionCount.inc
+      return
 
-  # Display each line in the message.
-  var i = 0
-  for line in msg.splitLines():
-    if len(line) == 0: continue
-    displayLine(if i == 0: category else: "...", line, displayType, priority)
-    i.inc
+    # Display each line in the message.
+    var i = 0
+    for line in msg.splitLines():
+      if len(line) == 0: continue
+      displayLine(if i == 0: category else: "...", line, displayType, priority)
+      i.inc
 
 proc displayWarning*(message: string, priority = HighPriority) =
   display("Warning: ", message, Warning, priority)

src/nimblepkg/nimscriptexecutor.nim

@@ -5,30 +5,32 @@ import os, strutils, sets
 
 import packageparser, common, options, nimscriptwrapper, cli
 
-proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool =
+proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool {. raises: [].} =
   ## Returns whether to continue.
   result = true
+  {.cast(gcsafe).}:
+    # For certain commands hooks should not be evaluated.
+    if hookAction in noHookActions:
+      return
 
-  # For certain commands hooks should not be evaluated.
-  if hookAction in noHookActions:
-    return
+    var nimbleFile = ""
+    try:
+      nimbleFile = findNimbleFile(getCurrentDir(), true, options)
 
-  var nimbleFile = ""
-  try:
-    nimbleFile = findNimbleFile(getCurrentDir(), true, options)
-  except NimbleError: return true
-  # PackageInfos are cached so we can read them as many times as we want.
-  let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options)
-  let actionName =
-    if hookAction == actionCustom: options.action.command
-    else: ($hookAction)[6 .. ^1]
-  let hookExists =
-    if before: actionName.normalize in pkgInfo.preHooks
-    else: actionName.normalize in pkgInfo.postHooks
-  if pkgInfo.isNimScript and hookExists:
-    let res = execHook(nimBin, nimbleFile, actionName, before, options)
-    if res.success:
-      result = res.retVal
+    # PackageInfos are cached so we can read them as many times as we want.
+      let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options)
+      let actionName =
+        if hookAction == actionCustom: options.action.command
+        else: ($hookAction)[6 .. ^1]
+      let hookExists =
+        if before: actionName.normalize in pkgInfo.preHooks
+        else: actionName.normalize in pkgInfo.postHooks
+      if pkgInfo.isNimScript and hookExists:
+        let res = execHook(nimBin, nimbleFile, actionName, before, options)
+        if res.success:
+          result = res.retVal
+    except NimbleError: return true
+    except Exception: return false #TODO fix the propagation of Exception 
 
 proc execCustom*(nimBin: string, nimbleFile: string, options: Options,
                  execResult: var ExecutionResult[bool]): bool =

src/nimblepkg/options.nim

@@ -79,6 +79,7 @@ type
     filePathPkgs*: seq[PackageInfo] #Packages loaded from file:// requires. Top level is always included.
     isFilePathDiscovering*: bool # Whether we are discovering file:// requires to fill up filePathPkgs. If true, it wont validate file:// requires.
     visitedHooks*: seq[VisitedHook] # Whether we are executing hooks.
+    jobs*: int # Number of build jobs to run in parallel. 0 means unlimited.
 
   ActionType* = enum
     actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade
@@ -293,6 +294,8 @@ Nimble Options:
       --features                  Activate features. Only used when using the declarative parser.
       --ignoreSubmodules          Ignore submodules when cloning a repository.
       --legacy                    Use the legacy code path (pre nimble 1.0.0)
+      --jobs                      Number of build jobs to run in parallel. 0 means unlimited. Default is 1.
+
 For more information read the GitHub readme:
   https://github.com/nim-lang/nimble#readme
 """
@@ -784,6 +787,13 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) =
     result.features = val.split(";").mapIt(it.strip)
   of "ignoresubmodules":
     result.ignoreSubmodules = true
+  of "jobs":
+    try:
+      result.jobs = parseInt(val)
+    except ValueError:
+      raise nimbleError(&"{val} is not a valid value")
+    if result.jobs < 0:
+      raise nimbleError("Number of jobs must be greater than or equal to 0")
   else: isGlobalFlag = false
 
   var wasFlagHandled = true
@@ -918,7 +928,8 @@ proc initOptions*(): Options =
     useDeclarativeParser: false,
     legacy: false, #default to legacy code path for nimble < 1.0.0
     satResult: SatResult(),
-    localDeps: true
+    localDeps: true,
+    jobs: 1
   )
 
   # Load visited hooks from environment variable to prevent recursive hook execution

src/nimblepkg/packageinfo.nim

@@ -444,7 +444,7 @@ proc checkInstallDir(pkgInfo: PackageInfo,
   if thisDir == "nimcache": result = true
 
 proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo,
-                      action: proc (f: string)) =
+                      action: proc (f: string): void {.raises: [].}) =
   ## Runs `action` for each filename of the files that have a whitelisted
   ## file extension.
   for kind, path in walkDir(dir):
@@ -454,7 +454,7 @@ proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo,
       if path.splitFile.ext.substr(1) in pkgInfo.installExt:
         action(path)
 
-proc iterFilesInDir(dir: string, action: proc (f: string)) =
+proc iterFilesInDir(dir: string, action: proc (f: string): void {.raises: [].}) =
   ## Runs `action` for each file in ``dir`` and any
   ## subdirectories that are in it.
   for kind, path in walkDir(dir):
@@ -520,7 +520,7 @@ proc iterInstallFilesSimple*(realDir: string, pkgInfo: PackageInfo,
         action(file)
 
 proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo,
-                       options: Options, action: proc (f: string)) =
+                       options: Options, action: proc (f: string): void {.raises: [].}) =
   ## Runs `action` for each file within the ``realDir`` that should be
   ## installed.
   # Get the package root directory for skipDirs comparison
@@ -670,9 +670,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti
       var rebuild = false
       iterFilesWithExt(dir, pkgInfo,
         proc (file: string) =
-          let srcTimestamp = getFileInfo(file).lastWriteTime
-          if binTimestamp < srcTimestamp:
-            rebuild = true
+          try:
+            let srcTimestamp = getFileInfo(file).lastWriteTime
+            if binTimestamp < srcTimestamp:
+              rebuild = true
+          except OSError:
+            discard
       )
       return rebuild
     else:
@@ -685,9 +688,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti
     var rebuild = false
     iterFilesWithExt(dir, pkgInfo,
       proc (file: string) =
-        let srcTimestamp = getFileInfo(file).lastWriteTime
-        if binTimestamp < srcTimestamp:
-          rebuild = true
+        try:
+          let srcTimestamp = getFileInfo(file).lastWriteTime
+          if binTimestamp < srcTimestamp:
+            rebuild = true
+        except OSError:
+          discard
     )
     return rebuild
 

src/nimblepkg/packageinfotypes.nim

@@ -169,9 +169,10 @@ proc appendGloballyActiveFeatures*(pkgName: string, features: seq[string]) =
 
 proc getGloballyActiveFeatures*(): seq[string] = 
   #returns features.{pkgName}.{feature}
-  for pkgName, features in globallyActiveFeatures:
-    for feature in features:
-      result.add(&"features.{pkgName}.{feature}")
+  {.cast(gcsafe).}:
+    for pkgName, features in globallyActiveFeatures:
+      for feature in features:
+        result.add(&"features.{pkgName}.{feature}")
 
 proc initSATResult*(pass: SATPass): SATResult =
   SATResult(pkgsToInstall: @[], solvedPkgs: @[], output: "", pkgs: initHashSet[PackageInfo](), 

src/nimblepkg/vcstools.nim

@@ -192,13 +192,13 @@ proc getVcsRevision*(dir: Path): Sha1Hash =
   ##   - the external command fails.
   ##   - the directory does not exist.
   ##   - there is no vcsRevisions in the repository.
+  {.cast(gcsafe).}:
+    let vcsRevision = tryDoVcsCmd(dir,
+      gitCmd = "rev-parse HEAD",
+      hgCmd  = "id -i --debug",
+      noVcsAction = $notSetSha1Hash)
 
-  let vcsRevision = tryDoVcsCmd(dir,
-    gitCmd = "rev-parse HEAD",
-    hgCmd  = "id -i --debug",
-    noVcsAction = $notSetSha1Hash)
-
-  return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'}))
+    return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'}))
 
 proc getVcsRevisions*(dir: Path): Sha1Hash =
   ## Returns current revision number if the directory `dir` is under version