Merge pull request #452 from cnpm/sync-upstream-package

support sync upstream first. fixed #451

Author: dead-horse

config/index.js

@@ -133,7 +133,6 @@ var config = {
   // registry url name
   registryHost: 'r.cnpmjs.org',
 
-
   /**
    * registry mode config
    */
@@ -170,8 +169,10 @@ var config = {
   disturl: 'http://nodejs.org/dist',
   syncDist: false,
 
-  // sync source
+  // sync source, upstream registry
   sourceNpmRegistry: 'http://registry.npm.taobao.org',
+  // upstream registry is base on cnpm/cnpmjs.org or not
+  sourceNpmRegistryIsCNpm: false,
 
   // if install return 404, try to sync from source registry
   syncByInstall: true,

package.json

@@ -13,13 +13,14 @@
     "agentkeepalive": "~1.2.0",
     "bluebird": "~2.3.2",
     "bytes": "~1.0.0",
-    "cfork": "~1.1.0",
+    "cfork": "~1.1.1",
     "cheerio": "~0.17.0",
     "co": "~3.1.0",
-    "co-defer": "~0.1.0",
+    "co-defer": "~0.1.1",
     "co-gather": "~0.0.1",
     "co-read": "~0.1.0",
     "co-redis": "~1.1.0",
+    "co-sleep": "~0.0.1",
     "co-write": "~0.3.0",
     "copy-to": "~1.0.1",
     "debug": "~2.0.0",
@@ -30,25 +31,25 @@
     "graceful": "~0.1.0",
     "gravatar": "~1.0.6",
     "humanize-number": "~0.0.2",
-    "koa": "~0.10.0",
+    "koa": "~0.12.1",
     "koa-limit": "~1.0.2",
     "koa-markdown": "~0.0.3",
-    "koa-middlewares": "~1.2.0",
+    "koa-middlewares": "~1.4.0",
     "marked": "~0.3.2",
     "mime": "~1.2.11",
     "mini-logger": "~0.3.0",
     "mkdirp": "~0.5.0",
-    "moment": "~2.8.2",
+    "moment": "~2.8.3",
     "ms": "~0.6.2",
     "multiline": "~1.0.0",
-    "mysql": "~2.4.3",
+    "mysql": "~2.5.1",
     "nodemailer": "0.7.1",
     "ready": "~0.1.1",
     "redis": "~0.12.1",
-    "semver": "~3.0.1",
+    "semver": "~4.0.0",
     "thunkify-wrap": "~1.0.2",
-    "urllib": "~1.4.1",
-    "utility": "~1.1.0"
+    "urllib": "~1.5.2",
+    "utility": "~1.2.0"
   },
   "devDependencies": {
     "autod": "~0.2.0",

proxy/sync_module_worker.js

@@ -25,6 +25,7 @@ var util = require('util');
 var fs = require('fs');
 var path = require('path');
 var crypto = require('crypto');
+var sleep = require('co-sleep');
 var urllib = require('../common/urllib');
 var utility = require('utility');
 var ms = require('ms');
@@ -140,6 +141,64 @@ SyncModuleWorker.prototype._doneOne = function* (concurrencyId, name, success) {
   });
 };
 
+SyncModuleWorker.prototype.syncUpstream = function* (name) {
+  var url = config.sourceNpmRegistry + '/' + name + '/sync';
+  var r = yield urllib.request(url, {
+    method: 'put',
+    timeout: 20000,
+    headers: {
+      'content-length': 0
+    },
+    dataType: 'json'
+  });
+
+  if (r.status !== 201 || !r.data.ok) {
+    return this.log('sync upstream %s error, status: %s, response: %j',
+      url, r.status, r.data);
+  }
+
+  var logURL =  url + '/log/' + r.data.logId;
+  var offset = 0;
+  this.log('Syncing upstream %s', logURL);
+
+  var count = 0;
+  while (true) {
+    count++;
+    var synclogURL = logURL + '?offset=' + offset;
+    var rs = yield urllib.request(synclogURL, {
+      timeout: 20000,
+      dataType: 'json'
+    });
+
+    if (rs.status !== 200 || !rs.data.ok) {
+      this.log('sync upstream %s error, status: %s, response: %j',
+        synclogURL, rs.status, rs.data);
+      break;
+    }
+
+    var data = rs.data;
+    var syncDone = false;
+    if (data.log.indexOf('[done] Sync') >= 0) {
+      syncDone = true;
+      data.log = data.log.replace('[done] Sync', '[upstream sync done]') +
+        '\n-------------------------------------------------------------\n';
+    }
+    this.log(data.log);
+
+    if (syncDone) {
+      break;
+    }
+    if (count >= 30) {
+      this.log('sync upstream %s fail, give up %s',
+        logURL, '\n-------------------------------------------------------------\n');
+      break;
+    }
+
+    offset += data.log.split('\n').length;
+    yield sleep(2000);
+  }
+};
+
 SyncModuleWorker.prototype.next = function *(concurrencyId) {
   var name = this.names.shift();
   if (!name) {
@@ -150,6 +209,12 @@ SyncModuleWorker.prototype.next = function *(concurrencyId) {
   that.syncingNames[name] = true;
   var pkg = null;
   var status = 0;
+
+  // sync upstream
+  if (config.sourceNpmRegistryIsCNpm) {
+    yield* this.syncUpstream(name);
+  }
+
   // get from npm
   try {
     var result = yield npm.request('/' + name);

test/proxy/sync_module_worker.test.js

@@ -15,11 +15,15 @@
  */
 
 var should = require('should');
+var mm = require('mm');
 var SyncModuleWorker = require('../../proxy/sync_module_worker');
 var mysql = require('../../common/mysql');
 var Log = require('../../proxy/module_log');
+var config = require('../../config');
 
 describe('proxy/sync_module_worker.test.js', function () {
+  afterEach(mm.restore);
+
   it('should start a sync worker', function (done) {
     Log.create({
       name: 'mk2testmodule',
@@ -38,6 +42,25 @@ describe('proxy/sync_module_worker.test.js', function () {
     });
   });
 
+  it('should sync upstream first', function (done) {
+    mm(config, 'sourceNpmRegistryIsCNpm', true);
+    Log.create({
+      name: 'mk2testmodule',
+      username: 'fengmk2',
+    }, function (err, result) {
+      should.not.exist(err);
+      result.id.should.above(0);
+      var worker = new SyncModuleWorker({
+        logId: result.id,
+        name: 'mk2testmodule',
+        username: 'fengmk2'
+      });
+
+      worker.start();
+      worker.on('end', done);
+    });
+  });
+
   it('should start a sync worker with names and noDep', function (done) {
     var worker = new SyncModuleWorker({
       name: ['mk2testmodule'],
@@ -90,4 +113,17 @@ describe('proxy/sync_module_worker.test.js', function () {
       done();
     });
   });
+
+  describe('syncUpstream()', function () {
+    it('should sync upstream work', function* () {
+      var worker = new SyncModuleWorker({
+        name: ['tnpm'],
+        username: 'fengmk2'
+      });
+      yield [
+        worker.syncUpstream('tnpm'),
+        worker.syncUpstream('pedding'),
+      ];
+    });
+  });
 });

view/web/sync.html

@@ -50,8 +50,7 @@ <h2>Log</h2>
 
     var offset = 0;
     var logs = '';
-    var successFlag = false;
-    var failFlag = false;
+    var syncDone = false;
     var hasFail = false;
     function getSyncLog(id) {
       $.ajax({
@@ -65,17 +64,16 @@ <h2>Log</h2>
           offset += data.log.split('\n').length;
           logs = logs + '\n' + data.log;
 
-          if (data.log.indexOf('Success: [') >= 0) {
-            successFlag = true;
+          if (data.log.indexOf('[done] Sync') >= 0) {
+            syncDone = true;
           }
           if (data.log.indexOf('Fail: [') >= 0) {
-            failFlag = true;
             var failInfo = data.log.match(/Fail: \[ (.*?) \]/);
             if (failInfo && failInfo[1]) {
               hasFail = true;
             }
           }
-          if (successFlag && failFlag) {
+          if (syncDone) {
             logs += '\nSync package ' + name + ' complete!';
             if (hasFail) {
               logs += ' But some packages sync failed, you can refresh to sync again.';