diff --git a/.gitignore b/.gitignore index f124bd2..28c9018 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ results node_modules npm-debug.log +.editorconfig diff --git a/.travis.yml b/.travis.yml index b2c1c5e..498da6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,6 @@ sudo: false language: node_js node_js: - - '0.10' - - '0.11' - - '0.12' - - '4' - - '6' - - '7' + - '8' + - '10' script: npm run test-travis diff --git a/lib/formstream.js b/lib/formstream.js index e7f39ce..d8dfedd 100644 --- a/lib/formstream.js +++ b/lib/formstream.js @@ -1,293 +1,279 @@ -/** - * Data format: - * - ---FormStreamBoundary1349886663601\r\n -Content-Disposition: form-data; name="foo"\r\n -\r\n -\r\n ---FormStreamBoundary1349886663601\r\n -Content-Disposition: form-data; name="file"; filename="formstream.test.js"\r\n -Content-Type: application/javascript\r\n -\r\n -\r\n ---FormStreamBoundary1349886663601\r\n -Content-Disposition: form-data; name="pic"; filename="fawave.png"\r\n -Content-Type: image/png\r\n -\r\n -\r\n ---FormStreamBoundary1349886663601-- - - * - */ - 'use strict'; -var Stream = require('stream'); -var parseStream = require('pause-stream'); -var util = require('util'); -var mime = require('mime'); -var path = require('path'); -var fs = require('fs'); -var destroy = require('destroy'); - -var PADDING = '--'; -var NEW_LINE = '\r\n'; -var NEW_LINE_BUFFER = new Buffer(NEW_LINE); - -function FormStream() { - if (!(this instanceof FormStream)) { - return new FormStream(); - } +const fs = require('fs'); +const path = require('path'); +const { Readable } = require('stream'); - FormStream.super_.call(this); +const debug = require('debug')('formstream'); +const destroy = require('destroy'); +const mime = require('mime'); - this._boundary = this._generateBoundary(); - this._streams = []; - this._buffers = []; - this._endData = new Buffer(PADDING + this._boundary + PADDING + NEW_LINE); - this._contentLength = 0; - this._isAllStreamSizeKnown = true; - this._knownStreamSize = 0; -} +const PADDING = '--'; +const NEW_LINE = '\r\n'; +const NEW_LINE_BUFFER = Buffer.from(NEW_LINE); -util.inherits(FormStream, Stream); -module.exports = FormStream; +class FormStream extends Readable { + constructor() { + super(); -FormStream.prototype._generateBoundary = function() { - // https://github.com/felixge/node-form-data/blob/master/lib/form_data.js#L162 - // This generates a 50 character boundary similar to those used by Firefox. - // They are optimized for boyer-moore parsing. - var boundary = '--------------------------'; - for (var i = 0; i < 24; i++) { - boundary += Math.floor(Math.random() * 10).toString(16); - } + this._boundary = this._generateBoundary(); + this._streams = []; + this._buffers = []; + this._contentLength = 0; + this._isAllStreamSizeKnown = true; + this._knownStreamSize = 0; - return boundary; -}; + this._eachEnd = null; -FormStream.prototype.setTotalStreamSize = function (size) { - // this method should not make any sense if the length of each stream is known. - if (this._isAllStreamSizeKnown) { - return this; - } + this._endData = Buffer.from(`${PADDING}${this._boundary}${PADDING}${NEW_LINE}`); - size = size || 0; - - for (var i = 0; i < this._streams.length; i++) { - size += this._streams[i][0].length; - size += NEW_LINE_BUFFER.length; // stream field end pedding size + this._currentStream = null; + this._readingOneStream = false; } - this._knownStreamSize = size; - this._isAllStreamSizeKnown = true; - - return this; -}; + _read() { + if (this._readingOneStream) { + debug('> still reading...'); + return; + } -FormStream.prototype.headers = function (options) { - var headers = { - 'Content-Type': 'multipart/form-data; boundary=' + this._boundary - }; + if (this._buffers && this._buffers.length) { + debug('pushing buffers...'); + for (const buf of this._buffers) { + this.push(buf[0]); + this.push(buf[1]); + this.push(NEW_LINE_BUFFER); + } + this._buffers = []; + } - // calculate total stream size - this._contentLength += this._knownStreamSize; + this._tryNextStream(); + } - // calculate length of end padding - this._contentLength += this._endData.length; + _tryNextStream() { + if (this._readingOneStream) return; + this._readingOneStream = true; + + debug('start read...'); + const item = this._streams.shift(); + if (!item) { + debug('no more stream'); + this.push(this._endData); + this.push(null); + this._readingOneStream = false; + return; + } - if (this._isAllStreamSizeKnown) { - headers['Content-Length'] = String(this._contentLength); + debug('push leading...'); + this.push(item[0]); + + const stream = item[1]; + this._currentStream = stream; + + stream.on('end', () => { + debug('end chunk'); + this._currentStream = null; + this.push(NEW_LINE_BUFFER); + debug('set reading = false'); + this._readingOneStream = false; + if (stream.destroy) stream.destroy(); + this._tryNextStream(); + }); + + stream.on('data', data => { + this.push(data); + debug('data chunk >', data.length); + stream.resume(); + }); + + if (stream.resume) { + stream.resume(); + debug('stream resume'); + } } - if (options) { - for (var k in options) { - headers[k] = options[k]; + _generateBoundary() { + // https://github.com/felixge/node-form-data/blob/master/lib/form_data.js#L162 + // This generates a 50 character boundary similar to those used by Firefox. + // They are optimized for boyer-moore parsing. + let boundary = '--------------------------'; + for (let i = 0; i < 24; i++) { + boundary += Math.floor(Math.random() * 10).toString(16); } + return boundary; } - return headers; -}; - -FormStream.prototype.file = function (name, filepath, filename, filesize) { - var mimeType = mime.lookup(filepath); + setTotalStreamSize(size) { + // this method should not make any sense if the length of each stream is known. + if (this._isAllStreamSizeKnown) { + return this; + } - if (typeof filename === 'number' && !filesize) { - filesize = filename; - filename = path.basename(filepath); - } else if (!filename) { - filename = path.basename(filepath); - } + size = size || 0; - var stream = fs.createReadStream(filepath); - - return this.stream(name, stream, filename, mimeType, filesize); -}; - -/** - * Add a form field - * @param {String} name field name - * @param {String|Buffer} value field value - * @return {this} - */ -FormStream.prototype.field = function (name, value) { - if (!Buffer.isBuffer(value)) { - // field(String, Number) - // https://github.com/qiniu/nodejs-sdk/issues/123 - if (typeof value === 'number') { - value = String(value); + for (let i = 0; i < this._streams.length; i++) { + size += this._streams[i][0].length; + size += NEW_LINE_BUFFER.length; // stream field end pedding size } - value = new Buffer(value); - } - return this.buffer(name, value); -}; - -FormStream.prototype.stream = function (name, stream, filename, mimeType, size) { - if (typeof mimeType === 'number' && !size) { - size = mimeType; - mimeType = mime.lookup(filename); - } else if (!mimeType) { - mimeType = mime.lookup(filename); + + this._knownStreamSize = size; + this._isAllStreamSizeKnown = true; + + return this; } - stream.once('error', this.emit.bind(this, 'error')); - // if form stream destroy, also destroy the source stream - this.once('destroy', function () { - destroy(stream); - }); + headers(options) { + const headers = { + 'Content-Type': `multipart/form-data; boundary=${this._boundary}`, + }; + + // calculate total stream size + this._contentLength += this._knownStreamSize; - var leading = this._leading({ name: name, filename: filename }, mimeType); + // calculate length of end padding + this._contentLength += this._endData.length; - var ps = parseStream().pause(); - stream.pipe(ps); + if (this._isAllStreamSizeKnown) { + headers['Content-Length'] = this._contentLength.toString(); + } - this._streams.push([leading, ps]); + if (options) { + for (const k in options) { + if (!options.hasOwnProperty(k)) continue; + headers[k] = options[k]; + } + } - // if the size of this stream is known, plus the total content-length; - // otherwise, content-length is unknown. - if (typeof size === 'number') { - this._knownStreamSize += leading.length; - this._knownStreamSize += size; - this._knownStreamSize += NEW_LINE_BUFFER.length; - } else { - this._isAllStreamSizeKnown = false; + return headers; } - process.nextTick(this.resume.bind(this)); + file(name, filepath, filename, filesize) { + const mimeType = mime.lookup(filepath); - return this; -}; + if (typeof filename === 'number' && !filesize) { + filesize = filename; + filename = path.basename(filepath); + } else if (!filename) { + filename = path.basename(filepath); + } -FormStream.prototype.buffer = function (name, buffer, filename, mimeType) { - if (filename && !mimeType) { - mimeType = mime.lookup(filename); - } + const stream = fs.createReadStream(filepath); - var disposition = { name: name }; - if (filename) { - disposition.filename = filename; + return this.stream(name, stream, filename, mimeType, filesize); } - var leading = this._leading(disposition, mimeType); + stream(name, stream, filename, mimeType, size) { + if (typeof mimeType === 'number' && !size) { + size = mimeType; + mimeType = mime.lookup(filename); + } else if (!mimeType) { + mimeType = mime.lookup(filename); + } - this._buffers.push([leading, buffer]); + if (stream.pause) stream.pause(); - // plus buffer length to total content-length - this._contentLength += leading.length; - this._contentLength += buffer.length; - this._contentLength += NEW_LINE_BUFFER.length; + // if form stream destroy, also destroy the source stream + stream.once('error', this.emit.bind(this, 'error')); - process.nextTick(this.resume.bind(this)); + const leading = this._leading({ name, filename }, mimeType); + this._streams.push([ leading, stream ]); - return this; -}; + // if the size of this stream is known, plus the total content-length; + // otherwise, content-length is unknown. + if (typeof size === 'number') { + this._knownStreamSize += leading.length; + this._knownStreamSize += size; + this._knownStreamSize += NEW_LINE_BUFFER.length; + } else { + this._isAllStreamSizeKnown = false; + } -FormStream.prototype._leading = function (disposition, type) { - var leading = [PADDING + this._boundary]; + return this; + } - var disps = []; + pause() { + if (this._currentStream && this._currentStream.pause) { + this._currentStream.pause(); + } + super.pause(); + } - if (disposition) { - for (var k in disposition) { - disps.push(k + '="' + disposition[k] + '"'); + resume() { + if (this._currentStream && this._currentStream.resume) { + this._currentStream.resume(); } + super.resume(); } - leading.push('Content-Disposition: form-data; ' + disps.join('; ')); + buffer(name, buffer, filename, mimeType) { + if (filename && !mimeType) { + mimeType = mime.lookup(filename); + } - if (type) { - leading.push('Content-Type: ' + type); - } + const disposition = { name: name }; + if (filename) { + disposition.filename = filename; + } - leading.push(''); - leading.push(''); + const leading = this._leading(disposition, mimeType); - return new Buffer(leading.join(NEW_LINE)); -}; + this._buffers.push([ leading, buffer ]); -FormStream.prototype._emitBuffers = function () { - if (!this._buffers.length) { - return; - } + // plus buffer length to total content-length + this._contentLength += leading.length; + this._contentLength += buffer.length; + this._contentLength += NEW_LINE_BUFFER.length; - for (var i = 0; i < this._buffers.length; i++) { - var item = this._buffers[i]; - this.emit('data', item[0]); // part leading - this.emit('data', item[1]); // part content - this.emit('data', NEW_LINE_BUFFER); + return this; } - this._buffers = []; -}; - -FormStream.prototype._emitStream = function (item) { - var self = this; - // item: [ fieldData, stream ] - self.emit('data', item[0]); - - var stream = item[1]; - stream.on('data', function (data) { - self.emit('data', data); - }); - stream.on('end', function () { - self.emit('data', NEW_LINE_BUFFER); - return process.nextTick(self.drain.bind(self)); - }); - stream.resume(); -}; - -FormStream.prototype._emitEnd = function () { - // ending format: - // - // --{boundary}--\r\n - this.emit('data', this._endData); - this.emit('end'); -}; - -FormStream.prototype.drain = function () { - this._emitBuffers(); - - var item = this._streams.shift(); - if (item) { - this._emitStream(item); - } else { - this._emitEnd(); - } + _leading(disposition, type) { + const leading = [ `${PADDING}${this._boundary}` ]; + + const disps = []; - return this; -}; + if (disposition) { + for (const k in disposition) { + if (!disposition.hasOwnProperty(k)) continue; + disps.push(`${k}="${disposition[k]}"`); + } + } + + leading.push(`Content-Disposition: form-data; ${disps.join('; ')}`); + + if (type) { + leading.push(`Content-Type: ${type}`); + } -FormStream.prototype.resume = function () { - this.paused = false; + leading.push(''); + leading.push(''); - if (!this._draining) { - this._draining = true; - this.drain(); + return Buffer.from(leading.join(NEW_LINE)); } - return this; -}; + field(name, value) { + if (!Buffer.isBuffer(value)) { + // field(String, Number) + // https://github.com/qiniu/nodejs-sdk/issues/123 + if (typeof value === 'number') { + value = value.toString(); + } + value = Buffer.from(value); + } + + return this.buffer(name, value); + } -FormStream.prototype.close = FormStream.prototype.destroy = function () { - this.emit('destroy'); -}; + _destroy() { + for (const stream of this._streams) { + stream[1].destroy(); + } + this._streams = []; + if (this._currentStream) { + this._currentStream.destroy(); + this._currentStream = null; + } + } +} + +module.exports = FormStream; diff --git a/package.json b/package.json index 16d9d8f..c914562 100644 --- a/package.json +++ b/package.json @@ -27,16 +27,15 @@ "request" ], "dependencies": { - "destroy": "^1.0.4", - "mime": "^1.3.4", - "pause-stream": "~0.0.11" + "debug": "^4.0.1", + "mime": "^1.3.4" }, "devDependencies": { "autod": "*", "connect-multiparty": "1", "express": "4", "istanbul": "*", - "mocha": "*", + "mocha": "3", "pedding": "1", "should": "4", "urllib": "2" diff --git a/test/formstream.test.js b/test/formstream.test.js index 1d40120..6d17f7a 100644 --- a/test/formstream.test.js +++ b/test/formstream.test.js @@ -7,23 +7,25 @@ var fs = require('fs'); var path = require('path'); var should = require('should'); var urllib = require('urllib'); -var formstream = require('../'); +var FormStream = require('../'); var root = path.join(__dirname, 'fixtures'); var app = require('./fixtures/server'); function cunterStream(name, count) { - var s = new Stream(); + var s = new Stream.Readable({ + read: () => {} + }); s.size = 0; var timer = setInterval(function () { var data = name + ' counter stream' + count + '\r\n'; s.size += data.length; - s.emit('data', data); + s.push(data); count--; if (count <= 0) { clearInterval(timer); process.nextTick(function () { - s.emit('end'); + s.push(null); }); } }, 100); @@ -76,11 +78,11 @@ describe('formstream.test.js', function () { it('should post fields only with content-length', function (done) { done = pedding(2, done); - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); - form.on('destroy', done); + form.on('end', done); post(port, '/post', form, function (err, data) { should.not.exist(err); data.body.should.eql({ @@ -98,7 +100,7 @@ describe('formstream.test.js', function () { it.skip('should upload a stream without size, use "Transfer-Encoding: chunked"', function (done) { var ChunkedStream = require('../../chunked'); - var form = formstream(); + var form = new FormStream(); form.stream('file', fs.createReadStream(path.join(__dirname, 'fixtures', 'foo.txt')), 'foo.txt'); form.field('name', '哈哈'); var headers = form.headers(); @@ -127,13 +129,14 @@ describe('formstream.test.js', function () { it('should post fields and file', function (done) { done = pedding(2, done); var now = Date.now(); - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); form.field('now', now); form.file('file', __filename); - form.on('destroy', done); + form.on('end', done); + form.on('error', console.log); post(port, '/post', form, function (err, data) { data.body.should.eql({ foo: 'bar', @@ -155,7 +158,7 @@ describe('formstream.test.js', function () { it('should post fields and file with content-length', function (done) { fs.stat(__filename, function (err, stat) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -181,7 +184,7 @@ describe('formstream.test.js', function () { }); it('should post fields and file with wrong stream size will return error', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -194,7 +197,7 @@ describe('formstream.test.js', function () { }); it('should post fields and file with wrong size will return error', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -208,7 +211,7 @@ describe('formstream.test.js', function () { if (process.version.indexOf('v0.8.') !== 0) { // node 0.8, createSteram not exists file will throw error it('should post not exist file return error ENOENT', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -224,7 +227,7 @@ describe('formstream.test.js', function () { } it('should post fields and stream', function (done) { - var form = formstream(); + var form = new FormStream(); var s1 = cunterStream('no1', 5); form.stream('stream1', s1, 'stream1中文名.txt', 'text/html'); var s2 = cunterStream('no2', 3); @@ -263,7 +266,7 @@ describe('formstream.test.js', function () { }); it('should post fields, 2 file', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -289,7 +292,7 @@ describe('formstream.test.js', function () { it('should post fields, 2 file with content-length', function (done) { var size = 0; var ready = function () { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -327,7 +330,7 @@ describe('formstream.test.js', function () { describe('buffer()', function () { it('should post file content buffer', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -364,7 +367,7 @@ describe('formstream.test.js', function () { }); it('should post file content buffer with content-length', function (done) { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); form.field('name', '中文名字'); form.field('pwd', '哈哈pwd'); @@ -397,7 +400,7 @@ describe('formstream.test.js', function () { describe('headers()', function () { it('should get headers contains correct content-length', function () { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar'); var headers = form.headers({ 'X-Test': 'hello' }); headers.should.have.keys('Content-Type', 'Content-Length', 'X-Test'); @@ -407,17 +410,17 @@ describe('formstream.test.js', function () { }); it('should trust .setTotalStreamSize() only if has any stream/file without size specified', function () { - var headers1 = formstream() + var headers1 = new FormStream() .field('field', 'plain') - .file('file', './logo.png', 'file') + .file('file', './test/fixtures/logo.png', 'file') .buffer('buffer', new Buffer(20), 'buffer') .stream('stream', cunterStream('stream', 5), 'stream') .setTotalStreamSize(10) .headers(); - var headers2 = formstream() + var headers2 = new FormStream() .field('field', 'plain') - .file('file', './logo.png', 'file', 10) + .file('file', './test/fixtures/logo.png', 'file', 10) .buffer('buffer', new Buffer(20), 'buffer') .stream('stream', cunterStream('stream', 5), 'stream', 30) .headers(); @@ -434,27 +437,27 @@ describe('formstream.test.js', function () { describe('chaining', function () { it('should do chaining calls with .field()', function () { - var form = formstream(); + var form = new FormStream(); form.field('foo', 'bar').should.equal(form); }); it('should do chaining calls with .file()', function () { - var form = formstream(); - form.file('foo', './logo.png', 'bar').should.equal(form); + var form = new FormStream(); + form.file('foo', './test/fixtures/logo.png', 'bar').should.equal(form); }); it('should do chaining calls with .buffer()', function () { - var form = formstream(); + var form = new FormStream(); form.buffer('foo', new Buffer('foo content'), 'bar').should.equal(form); }); it('should do chaining calls with .stream()', function () { - var form = formstream(); + var form = new FormStream(); form.stream('foo', cunterStream('stream', 5), 'bar').should.equal(form); }); it('should do chaining calls with .setTotalStreamSize()', function () { - var form = formstream(); + var form = new FormStream(); form.setTotalStreamSize(10).should.equal(form); }); });