add registries benchmarks

.github/workflows/benchmark.yaml

@@ -8,7 +8,7 @@ on:
         default: '["next", "astro", "vue", "svelte"]'
       variations:
         description: 'The benchmark variations to run'
-        default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules",  "lockfile", "lockfile+node_modules"]'
+        default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules",  "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]'
       binaries:
         description: 'The binaries to run the benchmarks on'
         default: '"npm,yarn,berry,pnpm,vlt,bun,deno,nx,turbo,node"'
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         fixture: ${{ fromJson(inputs.fixtures || '["next", "astro", "vue", "svelte"]') }}
-        variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules",  "lockfile", "lockfile+node_modules"]') }}
+        variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules",  "lockfile", "lockfile+node_modules", "registry-local-cache", "registry-proxied"]') }}
         include:
           - variation: "run"
             fixture: "run"
@@ -115,7 +115,9 @@ jobs:
               "lockfile",
               "lockfile+node_modules",
               "node_modules",
-              "run"
+              "run",
+              "registry-local-cache",
+              "registry-proxied"
             ];
 
             // Helper functions for statistical calculations

registry-configs/verdaccio/config.yaml

@@ -0,0 +1,24 @@
+storage: ./storage
+auth:
+  htpasswd:
+    file: ./htpasswd
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+packages:
+  '@*/*':
+    access: $all
+    publish: $authenticated
+    proxy: npmjs
+  '**':
+    access: $all
+    publish: $authenticated
+    proxy: npmjs
+server:
+  keepAliveTimeout: 60
+  bodyParser:
+    limit: 100mb
+logs:
+  - type: stdout
+    format: pretty
+    level: warn

registry-configs/vsr/vlt.json

@@ -0,0 +1,7 @@
+{
+  "registry": {
+    "upstream": "https://registry.npmjs.org/",
+    "cache": "./cache",
+    "port": 1337
+  }
+}

scripts/benchmark.sh

@@ -47,6 +47,12 @@ case "$2" in
     run)
         bash ../../scripts/variations/run.sh "../../scripts" "../../results" "run" "run"
         ;;
+    registry-local-cache)
+        bash ../../scripts/variations/registry-local-cache.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
+    registry-proxied)
+        bash ../../scripts/variations/registry-proxied.sh "../../scripts" "../../results" "$1" "$2"
+        ;;
     *)
         echo "Error: Unknown install variation '$2'"
         exit 1

scripts/process-results.sh

@@ -53,7 +53,7 @@ echo "Processing results..."
 
 # Process variations results
 for fixture in next astro svelte vue; do
-    for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run; do
+    for variation in cache cache+lockfile cache+lockfile+node_modules cache+node_modules clean lockfile lockfile+node_modules node_modules run registry-local-cache registry-proxied; do
         if [ -f "results/results-$fixture-$variation/benchmarks.json" ]; then
             print_summary "results/results-$fixture-$variation/benchmarks.json" "$fixture" "$variation"
             cp "results/results-$fixture-$variation/benchmarks.json" "results/$DATE/$fixture-$variation.json"

scripts/registry-helpers.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+# Registry helper functions
+
+# Registry ports
+VSR_PORT=1337
+VERDACCIO_PORT=4873
+
+# Start registry function
+start_registry() {
+  local registry_type=$1
+  local output_folder=$2
+
+  # Kill any existing registry processes
+  stop_registry "$registry_type"
+  sleep 2
+
+  if [ "$registry_type" = "vsr" ]; then
+    echo "Starting VSR registry on port $VSR_PORT..."
+    cd "$(dirname "$0")/../registry-configs/vsr"
+    npx @vltpkg/vsr --port $VSR_PORT --config ./vlt.json > "$output_folder/vsr.log" 2>&1 &
+    echo $! > /tmp/vsr.pid
+    cd - > /dev/null
+
+    # Wait for VSR to start
+    local retries=0
+    while ! curl -s "http://localhost:$VSR_PORT/-/ping" > /dev/null 2>&1; do
+      sleep 1
+      retries=$((retries + 1))
+      if [ $retries -gt 10 ]; then
+        echo "Error: VSR failed to start"
+        return 1
+      fi
+    done
+
+    echo "http://localhost:$VSR_PORT"
+  else
+    echo "Starting Verdaccio registry on port $VERDACCIO_PORT..."
+    cd "$(dirname "$0")/../registry-configs/verdaccio"
+    verdaccio --config ./config.yaml > "$output_folder/verdaccio.log" 2>&1 &
+    echo $! > /tmp/verdaccio.pid
+    cd - > /dev/null
+
+    # Wait for Verdaccio to start
+    local retries=0
+    while ! curl -s "http://localhost:$VERDACCIO_PORT/-/ping" > /dev/null 2>&1; do
+      sleep 1
+      retries=$((retries + 1))
+      if [ $retries -gt 10 ]; then
+        echo "Error: Verdaccio failed to start"
+        return 1
+      fi
+    done
+
+    echo "http://localhost:$VERDACCIO_PORT"
+  fi
+}
+
+# Stop registry function
+stop_registry() {
+  local registry_type=$1
+
+  if [ "$registry_type" = "vsr" ]; then
+    if [ -f /tmp/vsr.pid ]; then
+      kill $(cat /tmp/vsr.pid) 2>/dev/null || true
+      rm -f /tmp/vsr.pid
+    fi
+    pkill -f "vsr --port $VSR_PORT" || true
+  else
+    if [ -f /tmp/verdaccio.pid ]; then
+      kill $(cat /tmp/verdaccio.pid) 2>/dev/null || true
+      rm -f /tmp/verdaccio.pid
+    fi
+    pkill -f "verdaccio --config" || true
+  fi
+}
+
+# Clean registry cache function
+clean_registry_cache() {
+  local registry_type=$1
+
+  if [ "$registry_type" = "vsr" ]; then
+    rm -rf "$(dirname "$0")/../registry-configs/vsr/cache" || true
+  else
+    rm -rf "$(dirname "$0")/../registry-configs/verdaccio/storage" || true
+  fi
+}
+
+# Stop all registries
+stop_all_registries() {
+  stop_registry "vsr"
+  stop_registry "verdaccio"
+}
+
+# Main execution logic
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <command> [args]"
+  echo "Commands:"
+  echo "  start <vsr|verdaccio> <output_folder> - Start a registry"
+  echo "  stop <vsr|verdaccio> - Stop a registry"
+  echo "  stop_all - Stop all registries"
+  echo "  clean <vsr|verdaccio> - Clean registry cache"
+  exit 1
+fi
+
+case "$1" in
+  start)
+    if [ $# -lt 3 ]; then
+      echo "Usage: $0 start <vsr|verdaccio> <output_folder>"
+      exit 1
+    fi
+    start_registry "$2" "$3"
+    ;;
+  stop)
+    if [ $# -lt 2 ]; then
+      echo "Usage: $0 stop <vsr|verdaccio>"
+      exit 1
+    fi
+    stop_registry "$2"
+    ;;
+  stop_all)
+    stop_all_registries
+    ;;
+  clean)
+    if [ $# -lt 2 ]; then
+      echo "Usage: $0 clean <vsr|verdaccio>"
+      exit 1
+    fi
+    clean_registry_cache "$2"
+    ;;
+  *)
+    echo "Unknown command: $1"
+    exit 1
+    ;;
+esac

scripts/setup.sh

@@ -32,7 +32,7 @@ echo "hyperfine: $HYPERFINE_VERSION"
 
 # Install Node.js package managers and tools
 echo "Installing package managers and tools..."
-npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest
+npm install -g npm@latest corepack@latest vlt@latest bun@latest deno@latest nx@latest turbo@latest @vltpkg/vsr@latest verdaccio@latest
 
 # Configure Package Managers
 echo "Configuring package managers..."
@@ -83,4 +83,49 @@ echo "{
   \"node\": \"$NODE_VERSION\"
 }" > ./results/versions.json
 
+# Create registry configuration directories
+echo "Creating registry configuration directories..."
+mkdir -p ./registry-configs/vsr
+mkdir -p ./registry-configs/verdaccio
+
+# Create Verdaccio configuration
+cat > ./registry-configs/verdaccio/config.yaml << 'EOF'
+storage: ./storage
+auth:
+  htpasswd:
+    file: ./htpasswd
+uplinks:
+  npmjs:
+    url: https://registry.npmjs.org/
+packages:
+  '@*/*':
+    access: $all
+    publish: $authenticated
+    proxy: npmjs
+  '**':
+    access: $all
+    publish: $authenticated
+    proxy: npmjs
+server:
+  keepAliveTimeout: 60
+  bodyParser:
+    limit: 100mb
+logs:
+  - type: stdout
+    format: pretty
+    level: warn
+EOF
+
+# Create VSR configuration
+cat > ./registry-configs/vsr/vlt.json << 'EOF'
+{
+  "registry": {
+    "upstream": "https://registry.npmjs.org/",
+    "cache": "./cache",
+    "port": 1337
+  }
+}
+EOF
+
+echo "Registry configurations created successfully!"
 echo "Setup completed successfully!"

scripts/variations/registry-local-cache.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Exit on error
+set -Eeuxo pipefail
+
+# Load common variables
+source "$1/variations/common.sh"
+
+# Update command strings to use registry configuration
+update_commands_for_registry() {
+  BENCH_COMMAND_NPM="npm install --registry=$BENCH_REGISTRY_URL --no-audit --no-fund --silent >> $BENCH_OUTPUT_FOLDER/npm-output-\${HYPERFINE_ITERATION}.log 2>&1"
+  BENCH_COMMAND_YARN="corepack yarn@1 install --registry $BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/yarn-output-\${HYPERFINE_ITERATION}.log 2>&1"
+  BENCH_COMMAND_BERRY="echo \"$BENCH_COMMAND_BERRY_PRE\" > .yarnrc.yml; echo \"npmRegistryServer: $BENCH_REGISTRY_URL\" >> .yarnrc.yml; corepack yarn@latest install > $BENCH_OUTPUT_FOLDER/berry-output-\${HYPERFINE_ITERATION}.log 2>&1"
+  BENCH_COMMAND_PNPM="corepack pnpm@latest install --registry=$BENCH_REGISTRY_URL --silent > $BENCH_OUTPUT_FOLDER/pnpm-output-\${HYPERFINE_ITERATION}.log 2>&1"
+  BENCH_COMMAND_VLT="vlt install --registry=$BENCH_REGISTRY_URL --view=silent > $BENCH_OUTPUT_FOLDER/vlt-output-\${HYPERFINE_ITERATION}.log 2>&1"
+  # Bun and Deno don't support custom registries in the same way
+  BENCH_COMMAND_BUN=""
+  BENCH_COMMAND_DENO=""
+  BENCH_INCLUDE_BUN=""
+  BENCH_INCLUDE_DENO=""
+}
+
+# Pre-warm registry function
+warm_registry() {
+  local registry_type=$1
+
+  echo "Pre-warming $registry_type registry..."
+
+  # Save current directory
+  local current_dir=$(pwd)
+
+  # Use npm to install packages and warm the cache
+  npm install --registry="$BENCH_REGISTRY_URL" --silent > "$BENCH_OUTPUT_FOLDER/warm-$registry_type.log" 2>&1 || true
+
+  # Clean up after warming
+  bash "$BENCH_SCRIPTS/clean-helpers.sh" clean_node_modules clean_lockfiles
+
+  cd "$current_dir"
+}
+
+# Function to run benchmarks for a specific registry
+run_registry_benchmark() {
+  local registry_type=$1
+
+  # Stop any existing registries
+  bash "$BENCH_SCRIPTS/registry-helpers.sh" stop_all
+
+  # Clean the registry cache first
+  bash "$BENCH_SCRIPTS/registry-helpers.sh" clean "$registry_type"
+
+  # Start the registry
+  BENCH_REGISTRY_URL=$(bash "$BENCH_SCRIPTS/registry-helpers.sh" start "$registry_type" "$BENCH_OUTPUT_FOLDER")
+  export BENCH_REGISTRY_URL
+
+  echo "Registry started at: $BENCH_REGISTRY_URL"
+
+  # Update commands with registry URL
+  update_commands_for_registry
+
+  # Pre-warm the registry
+  warm_registry "$registry_type"
+
+  # Run the benchmark suite
+  echo "Running benchmarks with $registry_type registry (pre-warmed)..."
+  hyperfine --ignore-failure \
+    --time-unit=millisecond \
+    --export-json="$BENCH_OUTPUT_FOLDER/benchmarks-$registry_type.json" \
+    --warmup="$BENCH_WARMUP" \
+    --runs="$BENCH_RUNS" \
+    --prepare="sleep 1; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
+    --conclude="sleep 1; bash $BENCH_SCRIPTS/package-count.sh $BENCH_OUTPUT_FOLDER; bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
+    --cleanup="bash $BENCH_SCRIPTS/clean-helpers.sh clean_all" \
+    ${BENCH_INCLUDE_NPM:+--command-name="npm-$registry_type" "$BENCH_COMMAND_NPM"} \
+    ${BENCH_INCLUDE_YARN:+--command-name="yarn-$registry_type" "$BENCH_COMMAND_YARN"} \
+    ${BENCH_INCLUDE_BERRY:+--command-name="berry-$registry_type" "$BENCH_COMMAND_BERRY"} \
+    ${BENCH_INCLUDE_PNPM:+--command-name="pnpm-$registry_type" "$BENCH_COMMAND_PNPM"} \
+    ${BENCH_INCLUDE_VLT:+--command-name="vlt-$registry_type" "$BENCH_COMMAND_VLT"}
+
+  # Stop the registry
+  bash "$BENCH_SCRIPTS/registry-helpers.sh" stop "$registry_type"
+}
+
+# Run benchmarks for both registries
+run_registry_benchmark "vsr"
+run_registry_benchmark "verdaccio"
+
+# Merge results
+echo "Merging benchmark results..."
+jq -s '.[0] * {results: (.[0].results + .[1].results)}' \
+  "$BENCH_OUTPUT_FOLDER/benchmarks-vsr.json" \
+  "$BENCH_OUTPUT_FOLDER/benchmarks-verdaccio.json" \
+  > "$BENCH_OUTPUT_FOLDER/benchmarks.json"
+
+collect_package_count