Parallel builds v2

.gitmodules

@@ -10,3 +10,9 @@
 [submodule "vendor/chronos"]
 	path = vendor/chronos
 	url = https://github.com/status-im/nim-chronos.git
+[submodule "vendor/results"]
+	path = vendor/results
+	url = https://github.com/arnetheduck/nim-results.git
+[submodule "vendor/stew"]
+	path = vendor/stew
+	url = https://github.com/status-im/nim-stew.git

config.nims

@@ -6,5 +6,8 @@ switch("define", "ssl")
 switch("path", "vendor" / "zippy" / "src")
 switch("path", "vendor" / "sat" / "src")
 switch("path", "vendor" / "checksums" / "src")
+switch("path", "vendor" / "chronos")
+switch("path", "vendor" / "results")
+switch("path", "vendor" / "stew")
 switch("define", "zippyNoSimd")
 

src/nimble.nim

@@ -5,7 +5,7 @@ import os, tables, strtabs, json, browsers, algorithm, sets, uri, sugar, sequtil
        strformat
 
 import std/options as std_opt
-
+import chronos
 import strutils except toLower
 from unicode import toLower
 import sat/sat
@@ -660,10 +660,13 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     # Copy this package's files based on the preferences specified in PkgInfo.
     var filesInstalled: HashSet[string]
     iterInstallFiles(realDir, pkgInfo, options,
-      proc (file: string) =
-        createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir))
-        let dest = changeRoot(realDir, pkgDestDir, file)
-        filesInstalled.incl copyFileD(file, dest)
+      proc (file: string) {.raises: [].} =
+        try:
+          createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir))
+          let dest = changeRoot(realDir, pkgDestDir, file)
+          filesInstalled.incl copyFileD(file, dest)
+        except Exception:
+          discard
     )
 
     # Copy the .nimble file.
@@ -2568,7 +2571,7 @@ proc run(options: Options, nimBin: string) =
       # Use vnext buildPkg for develop mode packages
       let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and 
         options.satResult.rootPackage.basicInfo.name == pkgInfo.basicInfo.name
-      buildPkg(nimBin, pkgInfo, isInRootDir, options)
+      waitFor buildPkg(nimBin, pkgInfo, isInRootDir, options)
 
     if options.getCompilationFlags.len > 0:
       displayWarning(ignoringCompilationFlagsMsg)

src/nimblepkg/asyncfileops.nim

@@ -0,0 +1,39 @@
+# Async file operations using chronos async processes
+# Similar to Node.js - uses external commands for file I/O
+
+import std/os
+import chronos except Duration
+import chronos/asyncproc
+
+export chronos except Duration
+export asyncproc
+
+proc copyFileAsync*(source, dest: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} =
+  ## Async file copy using chronos async processes
+  when defined(windows):
+    # Windows: use xcopy for better handling
+    let cmd = "xcopy /Y /Q " & quoteShell(source) & " " & quoteShell(dest) & "*"
+  else:
+    # Unix: use cp command with preserve permissions and recursive for dirs
+    let cmd = "cp -f -p -r " & quoteShell(source) & " " & quoteShell(dest)
+
+  let exitCode = await execCommand(cmd)
+  if exitCode != 0:
+    raise newException(IOError, "Failed to copy file from " & source & " to " & dest & " (exit code: " & $exitCode & ")")
+
+proc copyDirAsync*(sourceDir, destDir: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} =
+  ## Async directory copy using chronos async processes - copies entire directory tree
+  when defined(windows):
+    # Windows: use robocopy for robust directory copying
+    # /E = copy subdirs including empty, /NFL = no file list, /NDL = no dir list, /NJH = no job header, /NJS = no job summary, /NC = no class, /NS = no size, /NP = no progress
+    let cmd = "robocopy " & quoteShell(sourceDir) & " " & quoteShell(destDir) & " /E /NFL /NDL /NJH /NJS /NC /NS /NP"
+    let exitCode = await execCommand(cmd)
+    # robocopy exit codes: 0-7 are success (0=no files, 1=files copied, 2=extra files, etc.)
+    if exitCode > 7:
+      raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")")
+  else:
+    # Unix: use cp -r to copy entire directory recursively
+    let cmd = "cp -r -p " & quoteShell(sourceDir) & "/. " & quoteShell(destDir)
+    let exitCode = await execCommand(cmd)
+    if exitCode != 0:
+      raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")")

src/nimblepkg/cli.nim

@@ -48,7 +48,13 @@ proc newCLI(): CLI =
     showColor: true,
   )
 
-var globalCLI = newCLI()
+var globalCLI {.threadvar.}: CLI
+
+proc getGlobalCLI(): CLI =
+  if globalCLI == nil:
+    globalCLI = newCLI()
+  return globalCLI
+
 
 proc calculateCategoryOffset(category: string): int =
   assert category.len <= longestCategory
@@ -57,14 +63,14 @@ proc calculateCategoryOffset(category: string): int =
 proc isSuppressed(displayType: DisplayType): bool =
   # Don't print any Warning, Message or Success messages when suppression of
   # warnings is enabled. That is, unless the user asked for --verbose output.
-  if globalCLI.suppressMessages and displayType >= Warning and
-     globalCLI.level == HighPriority:
+  if getGlobalCLI().suppressMessages and displayType >= Warning and
+     getGlobalCLI().level == HighPriority:
     return true
 
 proc displayFormatted*(displayType: DisplayType, msgs: varargs[string]) =
   ## for styling outputs lines using the DisplayTypes
   for msg in msgs:
-    if globalCLI.showColor:
+    if getGlobalCLI().showColor:
       stdout.styledWrite(foregrounds[displayType], msg)
     else:
       stdout.write(msg)
@@ -85,7 +91,7 @@ proc displayCategory(category: string, displayType: DisplayType,
 
   # Display the category.
   let text = "$1$2 " % [spaces(offset), category]
-  if globalCLI.showColor:
+  if getGlobalCLI().showColor:
     if priority != DebugPriority:
       setForegroundColor(stdout, foregrounds[displayType])
     writeStyled(text, styles[priority])
@@ -109,16 +115,16 @@ proc display*(category, msg: string, displayType = Message,
   # Multiple warnings containing the same messages should not be shown.
   let warningPair = (category, msg)
   if displayType == Warning:
-    if warningPair in globalCLI.warnings:
+    if warningPair in getGlobalCLI().warnings:
       return
     else:
-      globalCLI.warnings.incl(warningPair)
+      getGlobalCLI().warnings.incl(warningPair)
 
   # Suppress this message if its priority isn't high enough.
   # TODO: Per-priority suppression counts?
-  if priority < globalCLI.level:
+  if priority < getGlobalCLI().level:
     if priority != DebugPriority:
-      globalCLI.suppressionCount.inc
+      getGlobalCLI().suppressionCount.inc
     return
 
   # Display each line in the message.
@@ -173,7 +179,7 @@ proc displayDebug*(msg: string) =
 proc displayTip*() =
   ## Called just before Nimble exits. Shows some tips for the user, for example
   ## the amount of messages that were suppressed and how to show them.
-  if globalCLI.suppressionCount > 0:
+  if getGlobalCLI().suppressionCount > 0:
     let msg = "$1 messages have been suppressed, use --verbose to show them." %
              $globalCLI.suppressionCount
     display("Tip:", msg, Warning, HighPriority)
@@ -187,7 +193,7 @@ proc prompt*(forcePrompts: ForcePrompt, question: string): bool =
     display("Prompt:", question & " -> [forced no]", Warning, HighPriority)
     return false
   of dontForcePrompt:
-    if globalCLI.level > SilentPriority:
+    if getGlobalCLI().level > SilentPriority:
       display("Prompt:", question & " [y/N]", Warning, HighPriority)
       displayCategory("Answer:", Warning, HighPriority)
       let yn = stdin.readLine()
@@ -321,10 +327,10 @@ proc promptList*(forcePrompts: ForcePrompt, question: string, args: openarray[st
       return promptListFallback(question, args)
 
 proc setVerbosity*(level: Priority) =
-  globalCLI.level = level
+  getGlobalCLI().level = level
 
 proc setShowColor*(val: bool) =
-  globalCLI.showColor = val
+  getGlobalCLI().showColor = val
 
 proc setSuppressMessages*(val: bool) =
-  globalCLI.suppressMessages = val
+  getGlobalCLI().suppressMessages = val

src/nimblepkg/declarativeparser.nim

@@ -210,24 +210,25 @@ proc getNimCompilationPath*(nimbleFile: string): string =
   let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
   var parser: Parser
   var includePath = ""
-  if setupParser(parser, fileIdx, newIdentCache(), conf):
-    let ast = parseAll(parser)
-    proc findIncludePath(n: PNode) =
-      case n.kind
-      of nkStmtList, nkStmtListExpr:
-        for child in n:
-          findIncludePath(child)
-      of nkIncludeStmt:
-        # Found an include statement
-        if n.len > 0 and n[0].kind in {nkStrLit..nkTripleStrLit}:
-          includePath = n[0].strVal
-          # echo "Found include: ", includePath
-      else:
-        for i in 0..<n.safeLen:
-          findIncludePath(n[i])
-
-    findIncludePath(ast)
-    closeParser(parser)
+  {.cast(gcsafe).}:
+    if setupParser(parser, fileIdx, newIdentCache(), conf):
+      let ast = parseAll(parser)
+      proc findIncludePath(n: PNode) =
+        case n.kind
+        of nkStmtList, nkStmtListExpr:
+          for child in n:
+            findIncludePath(child)
+        of nkIncludeStmt:
+          # Found an include statement
+          if n.len > 0 and n[0].kind in {nkStrLit..nkTripleStrLit}:
+            includePath = n[0].strVal
+            # echo "Found include: ", includePath
+        else:
+          for i in 0..<n.safeLen:
+            findIncludePath(n[i])
+
+      findIncludePath(ast)
+      closeParser(parser)
 
   if includePath.len > 0:
     if includePath.contains("compilation.nim"):
@@ -251,53 +252,53 @@ proc extractNimVersion*(nimbleFile: string): string =
 
   let compFileIdx = fileInfoIdx(conf, AbsoluteFile compilationPath)
   var parser: Parser
-
-  if setupParser(parser, compFileIdx, newIdentCache(), conf):
-    let ast = parseAll(parser)
-
-    # Process AST to find NimMajor, NimMinor, NimPatch definitions
-    proc processNode(n: PNode) =
-      case n.kind
-      of nkStmtList, nkStmtListExpr:
-        for child in n:
-          processNode(child)
-      of nkConstSection:
-        for child in n:
-          if child.kind == nkConstDef:
-            var identName = ""
-            case child[0].kind
-            of nkPostfix:
-              if child[0][1].kind == nkIdent:
-                identName = child[0][1].ident.s
-            of nkIdent:
-              identName = child[0].ident.s
-            of nkPragmaExpr:
-              # Handle pragma expression (like NimMajor* {.intdefine.})
-              if child[0][0].kind == nkIdent:
-                identName = child[0][0].ident.s
-              elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent:
-                identName = child[0][0][1].ident.s
-            else: discard
-              # echo "Unhandled node kind for const name: ", child[0].kind
-            # Extract value
-            if child.len > 2:
-              case child[2].kind
-              of nkIntLit:
-                let value = child[2].intVal.int
-                case identName
-                of "NimMajor": major = value
-                of "NimMinor": minor = value
-                of "NimPatch": patch = value
-                else: discard
-              else:
-                discard
-      else:
-        discard
-
-    processNode(ast)
-    closeParser(parser)
-  # echo "Extracted version: ", major, ".", minor, ".", patch
-  return &"{major}.{minor}.{patch}"
+  {.cast(gcsafe).}:
+    if setupParser(parser, compFileIdx, newIdentCache(), conf):
+      let ast = parseAll(parser)
+      
+      # Process AST to find NimMajor, NimMinor, NimPatch definitions
+      proc processNode(n: PNode) =
+        case n.kind
+        of nkStmtList, nkStmtListExpr:
+          for child in n:
+            processNode(child)
+        of nkConstSection:
+          for child in n:
+            if child.kind == nkConstDef:
+              var identName = ""
+              case child[0].kind
+              of nkPostfix:
+                if child[0][1].kind == nkIdent:
+                  identName = child[0][1].ident.s
+              of nkIdent:
+                identName = child[0].ident.s
+              of nkPragmaExpr:
+                # Handle pragma expression (like NimMajor* {.intdefine.})
+                if child[0][0].kind == nkIdent:
+                  identName = child[0][0].ident.s
+                elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent:
+                  identName = child[0][0][1].ident.s
+              else: discard
+                # echo "Unhandled node kind for const name: ", child[0].kind
+              # Extract value
+              if child.len > 2:
+                case child[2].kind
+                of nkIntLit:
+                  let value = child[2].intVal.int
+                  case identName
+                  of "NimMajor": major = value
+                  of "NimMinor": minor = value
+                  of "NimPatch": patch = value
+                  else: discard
+                else:
+                  discard
+        else:
+          discard
+      
+      processNode(ast)
+      closeParser(parser)
+    # echo "Extracted version: ", major, ".", minor, ".", patch
+    return &"{major}.{minor}.{patch}"
 
 proc parseRequiresFile*(requiresFile: string): seq[string] =
   ## Parse a plain text requires file.
@@ -340,12 +341,13 @@ proc extractRequiresInfo*(nimbleFile: string, options: Options): NimbleFileInfo
     localError(config, info, warnUser, msg)
 
   let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
-  var parser: Parser
-  if setupParser(parser, fileIdx, newIdentCache(), conf):
-    let ast = parseAll(parser)
-    extract(ast, conf, result, options)
-    closeParser(parser)
-  result.hasErrors = result.hasErrors or conf.errorCounter > 0
+  {.cast(gcsafe).}:
+    var parser: Parser
+    if setupParser(parser, fileIdx, newIdentCache(), conf):
+      let ast = parseAll(parser)
+      extract(ast, conf, result, options)
+      closeParser(parser)
+    result.hasErrors = result.hasErrors or conf.errorCounter > 0
 
   # Add requires from external requires file
   let nimbleDir = nimbleFile.splitFile.dir