From 9a8bd9085518609a7398b48be7d0a50173285da8 Mon Sep 17 00:00:00 2001 From: oleg Date: Fri, 26 Feb 2016 23:43:38 +0300 Subject: [PATCH 01/13] move universal executor methods to base from local --- lib/executor/base.js | 128 ++++++++++++++++++++++++++++++++++++++- lib/executor/local.js | 136 +----------------------------------------- 2 files changed, 128 insertions(+), 136 deletions(-) diff --git a/lib/executor/base.js b/lib/executor/base.js index 7411a1f..4e4fcc4 100644 --- a/lib/executor/base.js +++ b/lib/executor/base.js @@ -4,6 +4,7 @@ var Steppy = require('twostep').Steppy, path = require('path'), _ = require('underscore'), EventEmitter = require('events').EventEmitter, + fs = require('fs'), inherits = require('util').inherits; function Executor(params) { @@ -20,15 +21,138 @@ Executor.prototype.throttledEmit = _(function() { }).throttle(500); Executor.prototype._getSources = function(params, callback) { + var self = this, + scm; + Steppy( + function() { + self._getChanges(params, this.slot()); + }, + function(err, data) { + scm = data.scm; + this.pass(data.changes); + scm.update(data.rev, this.slot()); + }, + function(err, changes) { + scm.getCurrent(this.slot()); + this.pass(changes); + scm.getRev(params.rev, this.slot()); + }, + function(err, currentRev, changes, latestRev) { + this.pass({ + rev: currentRev, + changes: changes, + isLatest: currentRev.id === latestRev.id + }); + }, + callback + ); }; -Executor.prototype._runStep = function(params, callback) { +Executor.prototype._runStep = function(step, callback) { + var self = this, + params = _(step).clone(); + Steppy( + function() { + if (params.type !== 'shell') { + throw new Error('Unknown step type: ' + params.type); + } + // set command cwd to executor cwd + params.cwd = self.cwd; + var command = self._createCommand( + _({ + emitIn: true, + emitOut: true, + emitErr: true, + attachStderr: true + }).extend(params) + ); + + command.on('stdin', function(data) { + self.emit('data', '> ' + String(data)); + }); + + command.on('stdout', function(data) { + self.emit('data', String(data)); + }); + + command.on('stderr', function(data) { + self.emit('data', 'stderr: ' + String(data)); + }); + + // TODO: should be fixed properly, currently it's quick fix for + // NODE_ENV which affects npm install/prune calls + params.options = params.options || {}; + params.options.env = params.options.env || process.env; + delete params.options.env.NODE_ENV; + + command.run(params, this.slot()) + }, + callback + ); +}; + +Executor.prototype._getChanges = function(params, callback) { + var self = this, + scm, isFirstRun, oldRev; + Steppy( + function() { + var slot = this.slot(); + fs.exists(self.cwd, function(exists) { + slot(null, exists); + }); + }, + function(err, exists) { + var scmParams = {type: params.type}; + if (exists) { + scmParams.cwd = self.cwd; + isFirstRun = false; + } else { + scmParams.repository = params.repository; + isFirstRun = true; + } + scm = self._createScm(scmParams); + + scm.on('stdin', function(data) { + self.emit('data', '> ' + String(data)); + }); + + if (isFirstRun) { + this.pass(null); + } else { + scm.getCurrent(this.slot()); + } + }, + function(err, id) { + oldRev = id; + + if (isFirstRun) { + scm.clone(self.cwd, params.rev, this.slot()); + } else { + scm.pull(params.rev, this.slot()) + } + }, + function() { + scm.getChanges(oldRev && oldRev.id, params.rev, this.slot()); + }, + function(err, changes) { + var target = self._getTarget(params.rev, changes); + this.pass({ + scm: scm, + oldRev: oldRev, + rev: target.rev, + changes: target.changes + }); + }, + callback + ); }; // Does current project scm has new changes to build Executor.prototype.hasScmChanges = function(callback) { - + this._getChanges(this.project.scm, function(err, data) { + callback(err, !err && data.changes.length > 0); + }); }; Executor.prototype.run = function(params, callback) { diff --git a/lib/executor/local.js b/lib/executor/local.js index 6043cd1..6c25f04 100644 --- a/lib/executor/local.js +++ b/lib/executor/local.js @@ -5,7 +5,6 @@ var Steppy = require('twostep').Steppy, ParentExecutor = require('./base').Executor, createScm = require('../scm').createScm, createCommand = require('../command').createCommand, - fs = require('fs'), _ = require('underscore'); function Executor(params) { @@ -20,137 +19,6 @@ Executor.prototype._createScm = function(params) { return createScm(params); }; -Executor.prototype._getChanges = function(params, callback) { - var self = this, - scm, isFirstRun, oldRev; - Steppy( - function() { - var slot = this.slot(); - fs.exists(self.cwd, function(exists) { - slot(null, exists); - }); - }, - function(err, exists) { - var scmParams = {type: params.type}; - if (exists) { - scmParams.cwd = self.cwd; - isFirstRun = false; - } else { - scmParams.repository = params.repository; - isFirstRun = true; - } - scm = self._createScm(scmParams); - - scm.on('stdin', function(data) { - self.emit('data', '> ' + String(data)); - }); - - if (isFirstRun) { - this.pass(null); - } else { - scm.getCurrent(this.slot()); - } - }, - function(err, id) { - oldRev = id; - - if (isFirstRun) { - scm.clone(self.cwd, params.rev, this.slot()); - } else { - scm.pull(params.rev, this.slot()) - } - }, - function() { - scm.getChanges(oldRev && oldRev.id, params.rev, this.slot()); - }, - function(err, changes) { - var target = self._getTarget(params.rev, changes); - this.pass({ - scm: scm, - oldRev: oldRev, - rev: target.rev, - changes: target.changes - }); - }, - callback - ); +Executor.prototype._createCommand = function(params) { + return createCommand(params); }; - -Executor.prototype.hasScmChanges = function(callback) { - this._getChanges(this.project.scm, function(err, data) { - callback(err, !err && data.changes.length > 0); - }); -}; - -Executor.prototype._getSources = function(params, callback) { - var self = this, - scm; - Steppy( - function() { - self._getChanges(params, this.slot()); - }, - function(err, data) { - scm = data.scm; - this.pass(data.changes); - scm.update(data.rev, this.slot()); - }, - function(err, changes) { - scm.getCurrent(this.slot()); - this.pass(changes); - scm.getRev(params.rev, this.slot()); - }, - function(err, currentRev, changes, latestRev) { - this.pass({ - rev: currentRev, - changes: changes, - isLatest: currentRev.id === latestRev.id - }); - }, - callback - ); -}; - -Executor.prototype._runStep = function(step, callback) { - var self = this, - params = _(step).clone(); - - Steppy( - function() { - if (params.type !== 'shell') { - throw new Error('Unknown step type: ' + params.type); - } - // set command cwd to executor cwd - params.cwd = self.cwd; - var command = createCommand( - _({ - emitIn: true, - emitOut: true, - emitErr: true, - attachStderr: true - }).extend(params) - ); - - command.on('stdin', function(data) { - self.emit('data', '> ' + String(data)); - }); - - command.on('stdout', function(data) { - self.emit('data', String(data)); - }); - - command.on('stderr', function(data) { - self.emit('data', 'stderr: ' + String(data)); - }); - - // TODO: should be fixed properly, currently it's quick fix for - // NODE_ENV which affects npm install/prune calls - params.options = params.options || {}; - params.options.env = params.options.env || process.env; - delete params.options.env.NODE_ENV; - - command.run(params, this.slot()) - }, - callback - ); -}; - From 5a6a440d1148b5c0ab355b6945f9a8ef15fefdd3 Mon Sep 17 00:00:00 2001 From: oleg Date: Fri, 26 Feb 2016 23:58:46 +0300 Subject: [PATCH 02/13] extract is cloned executor method --- lib/executor/base.js | 10 +++------- lib/executor/local.js | 7 +++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/executor/base.js b/lib/executor/base.js index 4e4fcc4..d46d5ed 100644 --- a/lib/executor/base.js +++ b/lib/executor/base.js @@ -4,7 +4,6 @@ var Steppy = require('twostep').Steppy, path = require('path'), _ = require('underscore'), EventEmitter = require('events').EventEmitter, - fs = require('fs'), inherits = require('util').inherits; function Executor(params) { @@ -97,14 +96,11 @@ Executor.prototype._getChanges = function(params, callback) { scm, isFirstRun, oldRev; Steppy( function() { - var slot = this.slot(); - fs.exists(self.cwd, function(exists) { - slot(null, exists); - }); + self._isCloned(this.slot()); }, - function(err, exists) { + function(err, cloned) { var scmParams = {type: params.type}; - if (exists) { + if (cloned) { scmParams.cwd = self.cwd; isFirstRun = false; } else { diff --git a/lib/executor/local.js b/lib/executor/local.js index 6c25f04..2b21cee 100644 --- a/lib/executor/local.js +++ b/lib/executor/local.js @@ -5,6 +5,7 @@ var Steppy = require('twostep').Steppy, ParentExecutor = require('./base').Executor, createScm = require('../scm').createScm, createCommand = require('../command').createCommand, + fs = require('fs'), _ = require('underscore'); function Executor(params) { @@ -22,3 +23,9 @@ Executor.prototype._createScm = function(params) { Executor.prototype._createCommand = function(params) { return createCommand(params); }; + +Executor.prototype._isCloned = function(callback) { + fs.exists(this.cwd, function(exists) { + callback(null, exists); + }); +}; From a7840fc219230c3278af3013dba69439ed634db7 Mon Sep 17 00:00:00 2001 From: oleg Date: Sat, 27 Feb 2016 23:29:18 +0300 Subject: [PATCH 03/13] scm uses command (but not command itself) --- lib/command/base.js | 16 +-------------- lib/command/spawn.js | 13 ++++++++++--- lib/scm/base.js | 46 +++++++++++++++++++++++++++++++++----------- lib/scm/index.js | 5 +++++ 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/lib/command/base.js b/lib/command/base.js index 6b7b3e2..449940b 100644 --- a/lib/command/base.js +++ b/lib/command/base.js @@ -4,23 +4,9 @@ var EventEmitter = require('events').EventEmitter, inherits = require('util').inherits; function Command(params) { - params = params || {}; - this.emitIn = params.emitIn; - this.emitOut = params.emitOut; - this.emitErr = params.emitErr; - this.attachStderr = params.attachStderr; + this.setParams(params); } exports.Command = Command; inherits(Command, EventEmitter); - -Command.prototype.enableEmitter = function() { - this.emitOut = true; - return this; -}; - -Command.prototype.disableEmitter = function() { - this.emitOut = false; - return this; -}; diff --git a/lib/command/spawn.js b/lib/command/spawn.js index 40f614e..6ff463b 100644 --- a/lib/command/spawn.js +++ b/lib/command/spawn.js @@ -6,15 +6,22 @@ var spawn = require('child_process').spawn, _ = require('underscore'); function Command(params) { - params = params || {}; - ParentCommand.call(this, params); - this.cwd = params.cwd; + ParentCommand.call(this, params || {}); } exports.Command = Command; inherits(Command, ParentCommand); +Command.prototype.setParams = function(params) { + if (params.cwd) this.cwd = params.cwd; + if (params.emitIn) this.emitIn = params.emitIn; + if (params.emitOut) this.emitOut = params.emitOut; + if (params.emitErr) this.emitErr = params.emitErr; + if (params.attachStderr) this.attachStderr = params.attachStderr; + if (params.collectOut) this.collectOut = params.collectOut; +}; + /** * Executes `params.cmd` with `params.args` and `params.options` */ diff --git a/lib/scm/base.js b/lib/scm/base.js index 222182a..5190d58 100644 --- a/lib/scm/base.js +++ b/lib/scm/base.js @@ -1,22 +1,46 @@ 'use strict'; -var ParentCommand = require('../command/spawn').Command, - inherits = require('util').inherits; +var EventEmitter = require('events').EventEmitter, + inherits = require('util').inherits, + _ = require('underscore'); function Scm(params) { - ParentCommand.call(this, params); - this.repository = params.repository; - if (!this.repository && !this.cwd) throw new Error( - '`repository` or `cwd` must be set' - ); - this.collectOut = true; - this.emitIn = true; - this.attachStderr = true; + var self = this; + + EventEmitter.call(self); + + self.repository = params.repository; + self.cwd = params.cwd; + + if (!self.repository && !self.cwd) { + throw new Error('`repository` or `cwd` must be set'); + } + + self.command = params.command; + + self.command.setParams({ + collectOut: true, + emitIn: true, + attachStderr: true + }); + + self.command.on('stdin', function(data) { + self.emit('stdin', data); + }); } exports.Scm = Scm; -inherits(Scm, ParentCommand); +inherits(Scm, EventEmitter); + +Scm.prototype.run = function(params, callback) { + if (this.cwd) { + params.options = params.options || {}; + params.options.cwd = this.cwd; + } + + this.command.run(params, callback); +}; /** * Clone repository to the `dst` update to `rev` and set `this.cwd` to `dst` diff --git a/lib/scm/index.js b/lib/scm/index.js index 89d0ddc..7b74c75 100644 --- a/lib/scm/index.js +++ b/lib/scm/index.js @@ -1,6 +1,11 @@ 'use strict'; +var SpawnCommand = require('../command/spawn').Command; + exports.createScm = function(params) { var Constructor = require('./' + params.type).Scm; + + params.command = params.command || new SpawnCommand(); + return new Constructor(params); }; From f721de039d63ada4d6e4731f33e14b3d17a1a714 Mon Sep 17 00:00:00 2001 From: oleg Date: Sat, 27 Feb 2016 23:47:36 +0300 Subject: [PATCH 04/13] scm run -> _run --- lib/scm/base.js | 2 +- lib/scm/git.js | 12 ++++++------ lib/scm/mercurial.js | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/scm/base.js b/lib/scm/base.js index 5190d58..e012136 100644 --- a/lib/scm/base.js +++ b/lib/scm/base.js @@ -33,7 +33,7 @@ exports.Scm = Scm; inherits(Scm, EventEmitter); -Scm.prototype.run = function(params, callback) { +Scm.prototype._run = function(params, callback) { if (this.cwd) { params.options = params.options || {}; params.options.cwd = this.cwd; diff --git a/lib/scm/git.js b/lib/scm/git.js index 8de94ad..1747724 100644 --- a/lib/scm/git.js +++ b/lib/scm/git.js @@ -60,14 +60,14 @@ Scm.prototype.clone = function(dst, rev, callback) { function() { // git can't clearly clone specified rev but can clone branch // possible solution to change clone params to (dst, branch, callback) - self.run({ + self._run({ cmd: 'git', args: ['clone', '--recursive', self.repository, dst] }, this.slot()); self.cwd = dst; }, function() { - self.run({cmd: 'git', args: ['checkout', '-f', rev]}, this.slot()); + self._run({cmd: 'git', args: ['checkout', '-f', rev]}, this.slot()); }, callback ); @@ -82,7 +82,7 @@ Scm.prototype.pull = function(rev, callback) { }, function(err, currentRev) { this.pass(currentRev); - self.run({cmd: 'git', args: ['pull']}, this.slot()); + self._run({cmd: 'git', args: ['pull']}, this.slot()); }, function(err, currentRev) { self.update(currentRev.id, this.slot()); @@ -104,7 +104,7 @@ Scm.prototype.getRev = function(rev, callback) { var self = this; Steppy( function() { - self.run({cmd: 'git', args: [ + self._run({cmd: 'git', args: [ 'show', rev, '--pretty=' + self._revTemplate ]}, this.slot()); @@ -146,7 +146,7 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) { function(err, currentRev) { this.pass(currentRev); - self.run({cmd: 'git', args: [ + self._run({cmd: 'git', args: [ 'log', rev1 ? rev1 + '..' + rev2 : rev2, '--pretty=' + self._revTemplate + self._linesSeparator ]}, this.slot()); @@ -172,5 +172,5 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) { }; Scm.prototype.update = function(rev, callback) { - this.run({cmd: 'git', args: ['checkout', '-f', rev]}, callback); + this._run({cmd: 'git', args: ['checkout', '-f', rev]}, callback); }; diff --git a/lib/scm/mercurial.js b/lib/scm/mercurial.js index c69a4e2..01c6b7e 100644 --- a/lib/scm/mercurial.js +++ b/lib/scm/mercurial.js @@ -55,7 +55,7 @@ Scm.prototype.clone = function(dst, rev, callback) { var self = this; Steppy( function() { - self.run({ + self._run({ cmd: 'hg', args: ['clone', '--rev', rev, self.repository, dst] }, this.slot()); @@ -66,14 +66,14 @@ Scm.prototype.clone = function(dst, rev, callback) { }; Scm.prototype.pull = function(rev, callback) { - this.run({cmd: 'hg', args: ['pull', '--rev', rev]}, callback); + this._run({cmd: 'hg', args: ['pull', '--rev', rev]}, callback); }; Scm.prototype.getCurrent = function(callback) { var self = this; Steppy( function() { - self.run({cmd: 'hg', args: [ + self._run({cmd: 'hg', args: [ 'parent', '--template', self._revTemplate ]}, this.slot()); }, @@ -88,7 +88,7 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) { var self = this; Steppy( function() { - self.run({cmd: 'hg', args: [ + self._run({cmd: 'hg', args: [ 'log', '--rev', rev2 + ':' + rev1, '--template', self._revTemplate + self._linesSeparator ]}, this.slot()); @@ -111,5 +111,5 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) { }; Scm.prototype.update = function(rev, callback) { - this.run({cmd: 'hg', args: ['up', '-C', rev]}, callback); + this._run({cmd: 'hg', args: ['up', '-C', rev]}, callback); }; From baceedd70e9de18aa81d5f234c473af5a18a834b Mon Sep 17 00:00:00 2001 From: oleg Date: Sun, 28 Feb 2016 22:13:24 +0300 Subject: [PATCH 05/13] do not set default command for scm --- lib/executor/local.js | 4 +++- lib/scm/base.js | 2 ++ lib/scm/index.js | 5 ----- test/scm.js | 9 +++++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/executor/local.js b/lib/executor/local.js index 2b21cee..b22a4df 100644 --- a/lib/executor/local.js +++ b/lib/executor/local.js @@ -6,7 +6,7 @@ var Steppy = require('twostep').Steppy, createScm = require('../scm').createScm, createCommand = require('../command').createCommand, fs = require('fs'), - _ = require('underscore'); + SpawnCommand = require('../command/spawn').Command; function Executor(params) { ParentExecutor.call(this, params); @@ -17,6 +17,8 @@ inherits(Executor, ParentExecutor); exports.Executor = Executor; Executor.prototype._createScm = function(params) { + params.command = new SpawnCommand(); + return createScm(params); }; diff --git a/lib/scm/base.js b/lib/scm/base.js index e012136..a9ab156 100644 --- a/lib/scm/base.js +++ b/lib/scm/base.js @@ -18,6 +18,8 @@ function Scm(params) { self.command = params.command; + if (!self.command) throw new Error('`command` is required'); + self.command.setParams({ collectOut: true, emitIn: true, diff --git a/lib/scm/index.js b/lib/scm/index.js index 7b74c75..89d0ddc 100644 --- a/lib/scm/index.js +++ b/lib/scm/index.js @@ -1,11 +1,6 @@ 'use strict'; -var SpawnCommand = require('../command/spawn').Command; - exports.createScm = function(params) { var Constructor = require('./' + params.type).Scm; - - params.command = params.command || new SpawnCommand(); - return new Constructor(params); }; diff --git a/test/scm.js b/test/scm.js index 214da67..3e9717a 100644 --- a/test/scm.js +++ b/test/scm.js @@ -42,7 +42,8 @@ var expect = require('expect.js'), it('create scm instance attached to new repository without errors', function() { scm = createScm({ type: type, - repository: originalRepositoryPath + repository: originalRepositoryPath, + command: new SpawnCommand() }); }); @@ -129,7 +130,11 @@ var expect = require('expect.js'), }); it('create scm instance attached to existing `cwd` without errors', function() { - scm = createScm({type: type, cwd: repositoryPath}); + scm = createScm({ + type: type, + cwd: repositoryPath, + command: new SpawnCommand() + }); }); it('expect repository log from rev0 to default revision equals to ' + From 2a7d6dfb983dbc9eb6033b2a47d30c231df10948 Mon Sep 17 00:00:00 2001 From: oleg Date: Sun, 28 Feb 2016 23:21:08 +0300 Subject: [PATCH 06/13] local to base node, add register, create functions --- lib/distributor.js | 6 +++--- lib/{node.js => node/base.js} | 10 +--------- lib/node/index.js | 18 ++++++++++++++++++ lib/node/local.js | 20 ++++++++++++++++++++ test/distributor/helpers.js | 8 ++++---- test/node.js | 4 ++-- 6 files changed, 48 insertions(+), 18 deletions(-) rename lib/{node.js => node/base.js} (91%) create mode 100644 lib/node/index.js create mode 100644 lib/node/local.js diff --git a/lib/distributor.js b/lib/distributor.js index 8f70277..7bc7b92 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -2,7 +2,7 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), - Node = require('./node').Node, + createNode = require('./node').createNode, EventEmitter = require('events').EventEmitter, inherits = require('util').inherits, logger = require('./logger')('distributor'); @@ -36,8 +36,8 @@ inherits(Distributor, EventEmitter); exports.Distributor = Distributor; -Distributor.prototype._createNode = function(nodeParams) { - return new Node(nodeParams); +Distributor.prototype._createNode = function(params) { + return createNode(params); }; Distributor.prototype._runNext = function(callback) { diff --git a/lib/node.js b/lib/node/base.js similarity index 91% rename from lib/node.js rename to lib/node/base.js index 0238375..81d03a7 100644 --- a/lib/node.js +++ b/lib/node/base.js @@ -1,7 +1,6 @@ 'use strict'; -var _ = require('underscore'), - createExecutor = require('./executor').createExecutor; +var _ = require('underscore'); function Node(params) { @@ -76,13 +75,6 @@ Node.prototype.getFreeExecutorsCount = function() { return this.maxExecutorsCount - _(this.executors).size(); }; -Node.prototype._createExecutor = function(project) { - return createExecutor({ - type: this.type, - project: project - }); -}; - Node.prototype.hasScmChanges = function(project, callback) { this._createExecutor(project).hasScmChanges(callback); }; diff --git a/lib/node/index.js b/lib/node/index.js new file mode 100644 index 0000000..8db4ad7 --- /dev/null +++ b/lib/node/index.js @@ -0,0 +1,18 @@ +'use strict'; + +var constructors = { + local: require('./local').Node +}; + +exports.register = function(type, constructor) { + constructor[type] = constructor; +}; + +exports.createNode = function(params) { + if (params.type in constructors === false) { + throw new Error('Unknown node type: ', params.type); + } + + var Constructor = constructors[params.type]; + return new Constructor(params); +}; diff --git a/lib/node/local.js b/lib/node/local.js new file mode 100644 index 0000000..efe9fbd --- /dev/null +++ b/lib/node/local.js @@ -0,0 +1,20 @@ +'user strict'; + +var inherits = require('util').inherits, + ParentNode = require('./base').Node, + LocalExecutor = require('../executor/local').Executor; + +function Node(params) { + ParentNode.call(this, params); +} + +inherits(Node, ParentNode); + +exports.Node = Node; + +Node.prototype._createExecutor = function(project) { + return new LocalExecutor({ + type: this.type, + project: project + }); +}; diff --git a/test/distributor/helpers.js b/test/distributor/helpers.js index 2da42c6..3fcc3be 100644 --- a/test/distributor/helpers.js +++ b/test/distributor/helpers.js @@ -2,16 +2,16 @@ var _ = require('underscore'), sinon = require('sinon'), - Node = require('../../lib/node').Node, + createNode = require('../../lib/node').createNode, EventEmitter = require('events').EventEmitter, ProjectsCollection = require('../../lib/project').ProjectsCollection, Distributor = require('../../lib/distributor').Distributor, Notifier = require('../../lib/notifier').Notifier; -var createNode = function(executorRun) { +var createMockedNode = function(executorRun) { return function(params) { - var node = new Node(params); + var node = createNode(params); node._createExecutor = function(project) { var executor = new EventEmitter(); executor.project = project; @@ -38,7 +38,7 @@ exports.createDistributor = function(params) { distributorParams.executorRun || sinon.stub().callsArgAsync(1) ); // patch method which will be called at constructor - sinon.stub(Distributor.prototype, '_createNode', createNode( + sinon.stub(Distributor.prototype, '_createNode', createMockedNode( executorRun )); delete distributorParams.executorRun; diff --git a/test/node.js b/test/node.js index add084d..9438c26 100644 --- a/test/node.js +++ b/test/node.js @@ -1,6 +1,6 @@ 'use strict'; -var Node = require('../lib/node').Node, +var createNode = require('../lib/node').createNode, expect = require('expect.js'); @@ -19,7 +19,7 @@ describe('Node', function() { describe('basic', function() { it('instance should be created without errors', function() { - node = new Node({ + node = createNode({ type: 'local', maxExecutorsCount: 1 }); From 4304cd009e4a0037fa7b0440240a33aff0e082a5 Mon Sep 17 00:00:00 2001 From: oleg Date: Mon, 29 Feb 2016 23:53:42 +0300 Subject: [PATCH 07/13] expose command, executor, node and scm to api --- app.js | 8 ++++++++ lib/command/index.js | 4 ++++ lib/executor/index.js | 4 ++++ lib/node/index.js | 8 ++++++-- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index 3ac9439..cdca103 100644 --- a/app.js +++ b/app.js @@ -16,6 +16,10 @@ var env = process.env.NODE_ENV || 'development', ProjectsCollection = require('./lib/project').ProjectsCollection, BuildsCollection = require('./lib/build').BuildsCollection, libLogger = require('./lib/logger'), + libNode = require('./lib/node'), + libCommand = require('./lib/command'), + libExecutor = require('./lib/executor'), + libScm = require('./lib/scm'), EventEmitter = require('events').EventEmitter, validateConfig = require('./lib/validateConfig'), utils = require('./lib/utils'); @@ -64,6 +68,10 @@ app.lib = {}; app.lib.BaseReaderLoader = BaseReaderLoader; app.lib.BaseNotifierTransport = BaseNotifierTransport; app.lib.logger = libLogger; +app.lib.command = libCommand; +app.lib.executor = libExecutor; +app.lib.scm = libScm; +app.lib.node = libNode; var configDefaults = { notify: {}, diff --git a/lib/command/index.js b/lib/command/index.js index 4bf01c2..886f3ca 100644 --- a/lib/command/index.js +++ b/lib/command/index.js @@ -1,5 +1,9 @@ 'use strict'; +var SpawnCommand = require('./spawn').Command; + +exports.SpawnCommand = SpawnCommand; + exports.createCommand = function(params) { var Constructor = require('./' + params.type).Command; return new Constructor(params); diff --git a/lib/executor/index.js b/lib/executor/index.js index c17c42d..ba8c43b 100644 --- a/lib/executor/index.js +++ b/lib/executor/index.js @@ -1,5 +1,9 @@ 'use strict'; +var BaseExecutor = require('./base').Executor; + +exports.BaseExecutor = BaseExecutor; + exports.createExecutor = function(params) { var Constructor = require('./' + params.type).Executor; return new Constructor(params); diff --git a/lib/node/index.js b/lib/node/index.js index 8db4ad7..d62fd7e 100644 --- a/lib/node/index.js +++ b/lib/node/index.js @@ -1,16 +1,20 @@ 'use strict'; +var BaseNode = require('./base').Node; + +exports.BaseNode = BaseNode; + var constructors = { local: require('./local').Node }; exports.register = function(type, constructor) { - constructor[type] = constructor; + constructors[type] = constructor; }; exports.createNode = function(params) { if (params.type in constructors === false) { - throw new Error('Unknown node type: ', params.type); + throw new Error('Unknown node type: ' + params.type); } var Constructor = constructors[params.type]; From 1bf07741abf2741e9877cbf418701474f23d485d Mon Sep 17 00:00:00 2001 From: oleg Date: Tue, 1 Mar 2016 00:11:58 +0300 Subject: [PATCH 08/13] create distributor node after plugins load --- app.js | 4 +++- distributor.js | 2 +- lib/distributor.js | 32 +++++++++++++++++++------------- test/distributor/helpers.js | 1 + 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/app.js b/app.js index cdca103..250547b 100644 --- a/app.js +++ b/app.js @@ -230,7 +230,7 @@ Steppy( completeUncompletedBuilds(this.slot()); }, function() { - require('./distributor').init(app, this.slot()); + require('./distributor').create(app, this.slot()); }, function(err, distributor) { app.builds = new BuildsCollection({ @@ -244,6 +244,8 @@ Steppy( require(plugin).register(app); }); + distributor.init(); + app.notifier.init(app.config.notify, this.slot()); }, function() { diff --git a/distributor.js b/distributor.js index e7e4a75..f523c68 100644 --- a/distributor.js +++ b/distributor.js @@ -7,7 +7,7 @@ var Steppy = require('twostep').Steppy, logger = require('./lib/logger')('distributor'); -exports.init = function(app, callback) { +exports.create = function(app, callback) { var distributor = new Distributor({ nodes: app.config.nodes, projects: app.projects, diff --git a/lib/distributor.js b/lib/distributor.js index 7bc7b92..f35e21a 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -9,33 +9,39 @@ var Steppy = require('twostep').Steppy, function Distributor(params) { - var self = this; - // nodes to execute builds - self.nodes = _(params.nodes).map(function(nodeParams) { - return self._createNode(nodeParams); - }); - // queued projects to build - self.queue = []; + this.nodes = params.nodes; - self.saveBuild = params.saveBuild || function(build, callback) { + // queued projects to build + this.queue = []; + + this.saveBuild = params.saveBuild || function(build, callback) { callback(null, build); }; - self.removeBuild = params.removeBuild || function(build, callback) { + this.removeBuild = params.removeBuild || function(build, callback) { callback(); }; - self.projects = params.projects; - self.notifier = params.notifier; + this.projects = params.projects; + this.notifier = params.notifier; - self.buildLogLineNumbersHash = {}, - self.lastLinesHash = {}; + this.buildLogLineNumbersHash = {}, + this.lastLinesHash = {}; } inherits(Distributor, EventEmitter); exports.Distributor = Distributor; +// do deferred initialization (e.g. create nodes after all plugins load) +Distributor.prototype.init = function() { + var self = this; + // nodes to execute builds + self.nodes = _(self.nodes).map(function(nodeParams) { + return self._createNode(nodeParams); + }); +}; + Distributor.prototype._createNode = function(params) { return createNode(params); }; diff --git a/test/distributor/helpers.js b/test/distributor/helpers.js index 3fcc3be..a66da3b 100644 --- a/test/distributor/helpers.js +++ b/test/distributor/helpers.js @@ -52,6 +52,7 @@ exports.createDistributor = function(params) { }); var distributor = new Distributor(distributorParams); + distributor.init(); if (mockNode) { Distributor.prototype._createNode.restore(); From e52f03711be69ef84fc81e5097ae1d7b5402a8da Mon Sep 17 00:00:00 2001 From: oleg Date: Tue, 1 Mar 2016 23:59:23 +0300 Subject: [PATCH 09/13] set cwd at local executor instead of base --- lib/executor/base.js | 2 -- lib/executor/local.js | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/executor/base.js b/lib/executor/base.js index d46d5ed..f5144ae 100644 --- a/lib/executor/base.js +++ b/lib/executor/base.js @@ -1,14 +1,12 @@ 'use strict'; var Steppy = require('twostep').Steppy, - path = require('path'), _ = require('underscore'), EventEmitter = require('events').EventEmitter, inherits = require('util').inherits; function Executor(params) { this.project = params.project; - this.cwd = path.join(this.project.dir, 'workspace'); } exports.Executor = Executor; diff --git a/lib/executor/local.js b/lib/executor/local.js index b22a4df..d701531 100644 --- a/lib/executor/local.js +++ b/lib/executor/local.js @@ -6,10 +6,12 @@ var Steppy = require('twostep').Steppy, createScm = require('../scm').createScm, createCommand = require('../command').createCommand, fs = require('fs'), + path = require('path'), SpawnCommand = require('../command/spawn').Command; function Executor(params) { ParentExecutor.call(this, params); + this.cwd = path.join(this.project.dir, 'workspace'); } inherits(Executor, ParentExecutor); From 16d436774187242208ab6f6daf635e1608506d35 Mon Sep 17 00:00:00 2001 From: oleg Date: Wed, 9 Mar 2016 21:51:34 +0300 Subject: [PATCH 10/13] allow name, options for node + custom node types --- lib/validateConfig.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/validateConfig.js b/lib/validateConfig.js index e144773..c29a682 100644 --- a/lib/validateConfig.js +++ b/lib/validateConfig.js @@ -19,8 +19,10 @@ module.exports = function(config, callback) { items: { type: 'object', properties: { - type: {type: 'string', enum: ['local']}, - maxExecutorsCount: {type: 'integer'} + name: {type: 'string'}, + type: {type: 'string'}, + maxExecutorsCount: {type: 'integer'}, + options: {type: 'object'} } }, minItems: 1 From 84747099b6114854bb8468890e53414c25acc693 Mon Sep 17 00:00:00 2001 From: oleg Date: Thu, 10 Mar 2016 23:34:42 +0300 Subject: [PATCH 11/13] add usage srategies for node --- lib/distributor.js | 16 ++++- lib/node/base.js | 27 +++++++-- lib/validateConfig.js | 1 + test/distributor/blocking.js | 2 +- test/node.js | 111 ++++++++++++++++++++++++++++++++++- 5 files changed, 148 insertions(+), 9 deletions(-) diff --git a/lib/distributor.js b/lib/distributor.js index f35e21a..7aad53d 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -35,9 +35,19 @@ exports.Distributor = Distributor; // do deferred initialization (e.g. create nodes after all plugins load) Distributor.prototype.init = function() { - var self = this; - // nodes to execute builds + var self = this, + namesHash = {}; + self.nodes = _(self.nodes).map(function(nodeParams) { + if (!nodeParams.name) { + nodeParams.name = nodeParams.type; + } + + if (namesHash[nodeParams.name]) { + throw new Error('Node name `' + nodeParams.name + ' already used'); + } + namesHash[nodeParams.name] = 1; + return self._createNode(nodeParams); }); }; @@ -195,7 +205,7 @@ Distributor.prototype._updateWaitReasons = function() { }); var waitReason = _(waitReasons).compact().join(', '); - // set only non-empty reasons + // set only non-empty and new reasons if (waitReason && waitReason !== item.build.waitReason) { self._updateBuild(item.build, {waitReason: waitReason}); } diff --git a/lib/node/base.js b/lib/node/base.js index 81d03a7..5843a65 100644 --- a/lib/node/base.js +++ b/lib/node/base.js @@ -6,11 +6,20 @@ var _ = require('underscore'); function Node(params) { this.type = params.type; this.maxExecutorsCount = params.maxExecutorsCount; + this.name = params.name; + this.usageStrategy = params.usageStrategy || 'maximum'; + + if (!this.usageStrategiesHash[this.usageStrategy]) { + throw new Error('Unknown usage strategy: ' + this.usageStrategy); + } + this.executors = {}; } exports.Node = Node; +Node.prototype.usageStrategiesHash = {maximum: 1, specificProject: 1}; + Node.prototype._getBlockerExecutor = function(getBlockers, getTarget) { return _(this.executors).find(function(executor) { var target = getTarget(executor); @@ -27,10 +36,20 @@ Node.prototype._getBlockerExecutor = function(getBlockers, getTarget) { Node.prototype.getExecutorWaitReason = function(project) { var waitReason; - if (_(this.executors).size() >= this.maxExecutorsCount) { - waitReason = 'All executors are busy'; + var targetNodeNames = project.node && project.node.target; + + if (targetNodeNames && !_(targetNodeNames).isArray()) { + targetNodeNames = [targetNodeNames]; + } + + if (targetNodeNames && !_(targetNodeNames).contains(this.name)) { + waitReason = this.name + ': not a target node'; + } else if (this.usageStrategy === 'specificProject' && !targetNodeNames) { + waitReason = this.name + ': only for specific projects'; + } else if (_(this.executors).size() >= this.maxExecutorsCount) { + waitReason = this.name + ': all executors are busy'; } else if (project.name in this.executors) { - waitReason = 'Project already running on node'; + waitReason = this.name + ': project already running on node'; } else { var blockerExecutor; @@ -58,7 +77,7 @@ Node.prototype.getExecutorWaitReason = function(project) { if (blockerExecutor) { waitReason = ( - 'Blocked by currently running "' + + this.name + ': blocked by currently running "' + blockerExecutor.project.name + '"' ); } diff --git a/lib/validateConfig.js b/lib/validateConfig.js index c29a682..89c007c 100644 --- a/lib/validateConfig.js +++ b/lib/validateConfig.js @@ -21,6 +21,7 @@ module.exports = function(config, callback) { properties: { name: {type: 'string'}, type: {type: 'string'}, + usageStrategy: {type: 'string'}, maxExecutorsCount: {type: 'integer'}, options: {type: 'object'} } diff --git a/test/distributor/blocking.js b/test/distributor/blocking.js index 3b009f0..e199fd1 100644 --- a/test/distributor/blocking.js +++ b/test/distributor/blocking.js @@ -93,7 +93,7 @@ describe('Distributor blocking with max 2 executors count', function() { var spy = updateBuildSpy; expect(spy.getCall(3).args[0].project.name).equal('project2'); expect(spy.getCall(3).args[1].waitReason).equal( - 'Blocked by currently running "project1"' + 'local: blocked by currently running "project1"' ); }); diff --git a/test/node.js b/test/node.js index 9438c26..f932969 100644 --- a/test/node.js +++ b/test/node.js @@ -1,7 +1,8 @@ 'use strict'; var createNode = require('../lib/node').createNode, - expect = require('expect.js'); + expect = require('expect.js'), + _ = require('underscore'); describe('Node', function() { @@ -9,6 +10,114 @@ describe('Node', function() { project1 = {name: 'project1'}, project2 = {name: 'project2'}; + var createNodeMock = function(params) { + params = params || {}; + + var node = createNode(_({ + name: 'executor1', + type: 'local', + maxExecutorsCount: 1, + usageStrategy: 'maximum' + }).extend(params)); + + // only for testing + if (params.executors) { + node.executors = params.executors; + } + return node; + }; + + describe('wait reason', function() { + + it('should be not a target node when node target is not match', function() { + var waitReason = createNodeMock({ + name: 'executor1' + }).getExecutorWaitReason({ + name: 'project1', + node: {target: 'other executor'} + }); + expect(waitReason).eql('executor1: not a target node'); + }); + + it('should be falsy when node target match', function() { + var waitReason = createNodeMock({ + name: 'executor1' + }).getExecutorWaitReason({ + name: 'project1', + node: {target: 'executor1'} + }); + expect(waitReason).not.ok(); + }); + + it('should be falsy when node target (array) match', function() { + var waitReason = createNodeMock({ + name: 'executor1' + }).getExecutorWaitReason({ + name: 'project1', + node: {target: ['executor1']} + }); + expect(waitReason).not.ok(); + }); + + it('should be only for specific projects when target is not set', function() { + var waitReason = createNodeMock({ + usageStrategy: 'specificProject' + }).getExecutorWaitReason({ + name: 'project1' + }); + expect(waitReason).eql('executor1: only for specific projects'); + }); + + it('should be all executors are busy when true', function() { + var waitReason = createNodeMock({ + maxExecutorsCount: 1, + executors: {project2: 1} + }).getExecutorWaitReason({ + name: 'project1' + }); + expect(waitReason).eql('executor1: all executors are busy'); + }); + + it('should be project already running on node when true', function() { + var waitReason = createNodeMock({ + maxExecutorsCount: 2, + executors: {project1: 1} + }).getExecutorWaitReason({ + name: 'project1' + }); + expect(waitReason).eql('executor1: project already running on node'); + }); + + it('should be blocked by project when blocked by executing', function() { + var waitReason = createNodeMock({ + maxExecutorsCount: 2, + executors: {project2: {project: {name: 'project2'}}} + }).getExecutorWaitReason({ + name: 'project1', + blockedBy: ['project2'] + }); + expect(waitReason).eql( + 'executor1: blocked by currently running "project2"' + ); + }); + + it('should be blocked by project when executing blocks it', function() { + var waitReason = createNodeMock({ + maxExecutorsCount: 2, + executors: {project2: {project: { + name: 'project2', + blocks: ['project1'] + }}} + }).getExecutorWaitReason({ + name: 'project1' + }); + expect(waitReason).eql( + 'executor1: blocked by currently running "project2"' + ); + }); + + }); + var expectNodeHasFreeExecutor = function(project, value) { it('should' + (value ? ' ' : ' not ') + 'has free executors for ' + project.name, function() { From 42b90a1587e67322b16de73c6616fe383d5e0943 Mon Sep 17 00:00:00 2001 From: oleg Date: Sat, 12 Mar 2016 22:57:18 +0300 Subject: [PATCH 12/13] update node name already used error message --- lib/distributor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/distributor.js b/lib/distributor.js index 7aad53d..df5d8ec 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -44,7 +44,7 @@ Distributor.prototype.init = function() { } if (namesHash[nodeParams.name]) { - throw new Error('Node name `' + nodeParams.name + ' already used'); + throw new Error('Node name `' + nodeParams.name + '` already used'); } namesHash[nodeParams.name] = 1; From dd4c0c633a103c13b144d6d62c450f36a1784540 Mon Sep 17 00:00:00 2001 From: oleg Date: Mon, 14 Mar 2016 21:29:57 +0300 Subject: [PATCH 13/13] expose scm --- lib/scm/index.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/scm/index.js b/lib/scm/index.js index 89d0ddc..f68bb6e 100644 --- a/lib/scm/index.js +++ b/lib/scm/index.js @@ -1,6 +1,23 @@ 'use strict'; +var BaseScm = require('./base').Scm; + +exports.BaseScm = BaseScm; + +var constructors = { + git: require('./git').Scm, + mercurial: require('./mercurial').Scm +}; + +exports.register = function(type, constructor) { + constructors[type] = constructor; +}; + exports.createScm = function(params) { - var Constructor = require('./' + params.type).Scm; + if (params.type in constructors === false) { + throw new Error('Unknown scm type: ' + params.type); + } + + var Constructor = constructors[params.type]; return new Constructor(params); };