martin/nci
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/nci.git synced 2025-01-11 03:45:08 +00:00
Code Issues Projects Releases Wiki Activity

merge with master (package.json and terminal index merged manually)

Browse Source
This commit is contained in:
oleg 2016-01-08 20:29:14 +03:00
commit c70fab9c79
31 changed files with 1008 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -6,7 +6,7 @@ work in progress...
[![Build Status](https://travis-ci.org/node-ci/nci.svg?branch=master)](https://travis-ci.org/node-ci/nci)
## TODO for release 0.9
## TODO for release 0.5
* ~~Dashboard (builds list, projects autocomlete)~~
* ~~Build page (build info(dates, changes, etc), console)~~
@ -33,15 +33,25 @@ work in progress...
* ~~git checkout before reset~~
* slow move out from build page (with lot of output) to main page - several sec
at ff (ff very slow on remove/replace terminal element)
* ~~when long line appear console output row numbers not on the same line with
content~~
* some "undefined" comments in scm changes
* ~~some "undefined" comments in scm changes~~
* projects list scroll
* ~~Error during send: TypeError: Cannot read property 'changes' of undefined~~
* ~~Builds loss~~
* ~~error on git after change branch: fatal: ambiguous argument '18a8ea4..branch':
unknown revision or path not in the working tree.~~
* "Uncaught TypeError: Cannot read property 'name' of undefined" at item.js (jade)
* strange git with merge commits changes detection, e.g. whem update from
"0.3.7" commit to master "new build timeline style, sime layout fixes" and
"add some responsive styles to build timeline, revert in-progress pulsate
animation" appear but should not.
* include fonts and other external static (if any)
* build console doesn't stick to the bottom at ff
* ~~more strict server and project configs valifation~~
* ui browser tests needed
* use one from: jquery or native browser methods
## Feature requests
@ -54,6 +64,8 @@ unknown revision or path not in the working tree.~~
down to the output which could be very long)~~
* speed up build points animation at ff (maybe borrow something from animate.css?)
* current successfully streak icons at project page
* cancell in progress build + buld/step timeout
* rev hash link to repo web ui
## Roadmap

42
app.js
View File

@ -10,9 +10,11 @@ var env = process.env.NODE_ENV || 'development',
_ = require('underscore'),
reader = require('./lib/reader'),
notifier = require('./lib/notifier'),
project = require('./lib/project'),
ProjectsCollection = require('./lib/project').ProjectsCollection,
BuildsCollection = require('./lib/build').BuildsCollection,
libLogger = require('./lib/logger'),
EventEmitter = require('events').EventEmitter;
EventEmitter = require('events').EventEmitter,
validateConfig = require('./lib/validateConfig');
var app = new EventEmitter(),
logger = libLogger('app'),
@ -61,7 +63,6 @@ app.lib = {};
app.lib.reader = reader;
app.lib.notifier = notifier;
app.lib.logger = libLogger;
app.lib.project = project;
var configDefaults = {
notify: {},
@ -182,6 +183,11 @@ Steppy(
reader.load(app.config.paths.data, 'config', this.slot());
},
function(err, mkdirResult, config) {
this.pass(mkdirResult);
validateConfig(config, this.slot());
},
function(err, mkdirResult, config) {
_(app.config).defaults(config);
_(app.config).defaults(configDefaults);
@ -200,20 +206,22 @@ Steppy(
db.init(app.config.paths.db, {db: dbBackend}, this.slot());
},
function() {
// load all projects for the first time
project.loadAll(app.config.paths.projects, this.slot());
app.projects = new ProjectsCollection({
db: db,
reader: reader,
baseDir: app.config.paths.projects
});
completeUncompletedBuilds(this.slot());
},
function(err, projects) {
// note that `app.projects` is live variable
app.projects = projects;
logger.log('Loaded projects: ', _(app.projects).pluck('name'));
function(err) {
require('./distributor').init(app, this.slot());
},
function(err, distributor) {
app.distributor = distributor;
app.builds = new BuildsCollection({
db: db,
distributor: distributor
});
// register other plugins
require('./lib/notifier/console').register(app);
@ -230,15 +238,17 @@ Steppy(
require('./scheduler').init(app, this.slot());
// notify about first project loading
_(app.projects).each(function(project) {
app.emit('projectLoaded', project);
});
// init resources
require('./resources')(app);
},
function() {
// load projects after all plugins to provide ability for plugins to
// handle `projectLoaded` event
app.projects.loadAll(this.slot());
},
function(err) {
logger.log('Loaded projects: ', _(app.projects.getAll()).pluck('name'));
var host = app.config.http.host,
port = app.config.http.port;
logger.log('Start http server on %s:%s', host, port);

2
app/components/builds/view/index.jade
View File

@ -14,7 +14,7 @@ mixin statusBadge(build)
.row
if this.state.build
.col-sm-3.hidden-xs
BuildSidebar(projectName=this.state.build.project.name)
BuildSidebar(projectName=this.state.build.project.name, currentBuild=this.state.build)
.col-sm-9
h1.page-header

4
app/components/builds/view/sidebar/index.jade
View File

@ -1,6 +1,8 @@
.builds.builds__timeline.builds__timeline-small
each item in this.state.items
.builds_item(key=item.id, class="builds_item__#{item.status}")
- var buildItemClasses = ['builds_item__' + item.status];
- if (item.id === this.props.currentBuild.id) buildItemClasses.push('builds_item__current');
.builds_item(key=item.id, class=buildItemClasses)
.builds_inner
.row
.builds_header

2
app/components/projects/view/index.jade
View File

@ -32,7 +32,7 @@ div
p Current successfully streak:
if lastDoneBuild
span= this.state.project.doneBuildsStreak
span= this.state.project.doneBuildsStreak.buildsCount
else
| -

23
app/components/terminal/index.js
View File

@ -4,6 +4,7 @@ var _ = require('underscore'),
React = require('react'),
Reflux = require('reflux'),
terminalStore = require('../../stores/terminal'),
buildStore = require('../../stores/build'),
ansiUp = require('ansi_up'),
template = require('./index.jade');
@ -18,9 +19,28 @@ var Component = React.createClass({
this.listenTo(terminalStore, this.updateItems);
var node = document.getElementsByClassName('terminal')[0];
this.initialScrollPosition = node.getBoundingClientRect().top;
if (this.props.showPreloader) {
this.getTerminal().insertAdjacentHTML('afterend',
'<img src="/images/preloader.gif" class="terminal_preloader"/>'
);
this.listenTo(buildStore, function(build) {
if (build.completed) {
this.removePreloader();
}
});
}
window.onscroll = this.onScroll;
},
removePreloader: function() {
var preloader = document.getElementsByClassName(
'terminal_preloader'
)[0];
if (preloader) {
preloader.parentNode.removeChild(preloader);
}
},
componentWillUnmount: function() {
window.onscroll = null;
},
@ -90,6 +110,9 @@ var Component = React.createClass({
this.data = build.data;
this.renderBuffer();
}
if (this.props.showPreloader && build.buildCompleted) {
this.removePreloader();
}
},
shouldComponentUpdate: function() {
return false;

15
app/styles/components/builds.less
View File

@ -177,6 +177,21 @@
border-right-color: darken(@well-bg, 10%);
top: 13px;
}
&__current {
&:before {
left: 10px;
border-right-color: @component-active-bg;
top: 13px;
}
.builds {
&_inner {
border-left: 6px solid @component-active-bg;
}
}
}
}
&_header {

7
data/config.yaml
View File

@ -1,7 +1,7 @@
plugins:
# - nci-mail-notification
# - nci-jabber-notification
# plugins:
# - nci-mail-notification
# - nci-jabber-notification
nodes:
- type: local
@ -14,7 +14,6 @@ http:
storage:
backend: memdown
# backend: leveldown
notify:
mail:

130
distributor.js
View File

@ -3,9 +3,6 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
Distributor = require('./lib/distributor').Distributor,
getAvgProjectBuildDuration = (
require('./lib/project').getAvgProjectBuildDuration
),
db = require('./db'),
logger = require('./lib/logger')('distributor');
@ -18,13 +15,21 @@ exports.init = function(app, callback) {
Steppy(
function() {
if (_(build.project).has('avgBuildDuration')) {
this.pass(build.project.avgBuildDuration);
this.pass(null);
} else {
getAvgProjectBuildDuration(build.project.name, this.slot());
app.builds.getRecent({
projectName: build.project.name,
status: 'done',
limit: 10
}, this.slot());
}
},
function(err, avgBuildDuration) {
build.project.avgBuildDuration = avgBuildDuration;
function(err, doneBuilds) {
if (doneBuilds) {
build.project.avgBuildDuration = (
app.builds.getAvgBuildDuration(doneBuilds)
);
}
db.builds.put(build, this.slot());
},
@ -44,116 +49,7 @@ exports.init = function(app, callback) {
}
});
var buildDataResourcesHash = {};
// create resource for build data
var createBuildDataResource = function(buildId) {
if (buildId in buildDataResourcesHash) {
return;
}
var buildDataResource = app.dataio.resource('build' + buildId);
buildDataResource.on('connection', function(client) {
var callback = this.async();
Steppy(
function() {
db.logLines.find({
start: {buildId: buildId},
}, this.slot());
},
function(err, lines) {
client.emit('sync', 'data', {lines: lines});
this.pass(true);
},
function(err) {
if (err) {
logger.error(
'error during read log for "' + buildId + '":',
err.stack || err
);
}
callback();
}
);
});
buildDataResourcesHash[buildId] = buildDataResource;
};
exports.createBuildDataResource = createBuildDataResource;
var buildsResource = app.dataio.resource('builds');
distributor.on('buildUpdate', function(build, changes) {
if (build.status === 'queued') {
createBuildDataResource(build.id);
}
// notify about build's project change, coz building affects project
// related stat (last build date, avg build time, etc)
if (changes.completed) {
var projectsResource = app.dataio.resource('projects');
projectsResource.clientEmitSyncChange({name: build.project.name});
}
buildsResource.clientEmitSync('change', {
buildId: build.id, changes: changes
});
});
distributor.on('buildCancel', function(build) {
buildsResource.clientEmitSync('cancel', {buildId: build.id});
});
var buildLogLineNumbersHash = {},
lastLinesHash = {};
distributor.on('buildData', function(build, data) {
var cleanupText = function(text) {
return text.replace('\r', '');
};
var splittedData = data.split('\n'),
logLineNumber = buildLogLineNumbersHash[build.id] || 0;
lastLinesHash[build.id] = lastLinesHash[build.id] || '';
// if we don't have last line, so we start new line
if (!lastLinesHash[build.id]) {
logLineNumber++;
}
lastLinesHash[build.id] += _(splittedData).first();
var lines = [{
text: cleanupText(lastLinesHash[build.id]),
buildId: build.id,
number: logLineNumber
}];
if (splittedData.length > 1) {
// if we have last '' we have to take all except last
// this shown that string ends with eol
if (_(splittedData).last() === '') {
lastLinesHash[build.id] = '';
splittedData = _(splittedData.slice(1)).initial();
} else {
lastLinesHash[build.id] = _(splittedData).last();
splittedData = _(splittedData).tail();
}
lines = lines.concat(_(splittedData).map(function(line) {
return {
text: cleanupText(line),
buildId: build.id,
number: ++logLineNumber
};
}));
}
buildLogLineNumbersHash[build.id] = logLineNumber;
app.dataio.resource('build' + build.id).clientEmitSync(
'data',
{lines: lines}
);
distributor.on('buildLogLines', function(build, lines) {
// write build logs to db
db.logLines.put(lines, function(err) {
if (err) {

50
docs/developing-plugins/builds-collection.md Normal file
View File

@ -0,0 +1,50 @@
## BuildsCollection()
Facade entity which accumulates operations with currently running and
db saved builds.
## BuildsCollection.create(params:Object, [callback(err)]:Function)
Create build by running given project.
- `params.projectName` - project to build
- `params.withScmChangesOnly` - if true then build will be started only if
there is scm changes for project
- `params.queueQueued` - if true then currently queued project can be queued
again
- `params.initiator` - contains information about initiator of the build,
must contain `type` property e.g. when one build triggers another:
initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
## BuildsCollection.cancel(id:Number, [callback(err)]:Function)
Cancel build by id.
Note that only queued build can be canceled currently.
## BuildsCollection.get(id:Number, callback(err,build):Function)
Get build by id.
## BuildsCollection.getLogLines(params:Object, callback(err,logLinesData):Function)
Get log lines for the given build.
- `params.buildId` - target build
- `params.from` - if set then lines from that number will be returned
- `params.to` - if set then lines to that number will be returned
## BuildsCollection.getAvgBuildDuration(builds:Array.<Object>)
Calculate average build duration for the given builds.
## BuildsCollection.getRecent(params:Object, callback(err,builds):Function)
Get builds sorted by date in descending order.
- `params.projectName` - optional project filter
- `params.status` - optional status filter, can be used only when
`params.projectName` is set. When used builds in the result will contain
only following fields: id, number, startDate, endDate
## BuildsCollection.getDoneStreak(params:Object, callback(err,doneStreak):Function)
Get info about current done builds streak.
- `params.projectName` - optional project filter

55
docs/developing-plugins/projects-collection.md Normal file
View File

@ -0,0 +1,55 @@
## ProjectsCollection()
Projects collection contains all currently loaded projects and provides
operations for manipulating with them.
All projects stored on disk in `baseDir` and loaded to memory so
they can be received (by `get`, `getAll` and other methods) in a sync way.
Note that id for the particular project is a `name` of that project.
## ProjectsCollection.validateConfig(config:Object, callback(err,config):Function)
Validate and return given config.
## ProjectsCollection.load(name:String, [callback(err)]:Function)
Load project to collection.
`projectLoaded` event with loaded config as argument will be emitted after
load.
## ProjectsCollection.loadAll([callback(err)]:Function)
Load all projects (from `this.baseDir`).
Calls `load` for every project in a base dir.
## ProjectsCollection.unload(name:String, [callback(err)]:Function)
Unload project from collection
`projectUnloaded` event with unloaded config as argument will be emitted
after unload.
## ProjectsCollection.get(name:String)
Get project config by name.
Returns config object or undefined if project is not found.
## ProjectsCollection.getAll()
Get configs for all currently loaded projects.
Returns array of config objects.
## ProjectsCollection.filter(predicate:Function)
Get project configs which match to predicate.
Returns array of config objects or empty array if there is no matched
project.
## ProjectsCollection.remove(name:String, [callback(err)]:Function)
Remove project by name.
Calls `unload`, removes project from disk and db.
## ProjectsCollection.rename(name:String, [callback(err)]:Function)
Rename project.
Renames project on disk and db, also changes name for loaded project.

47
httpApi.js
View File

@ -48,22 +48,25 @@ router.getRoute = function(req) {
};
module.exports = function(app) {
var logger = app.lib.logger('http api');
var logger = app.lib.logger('http api'),
accessToken = (Math.random() * Math.random()).toString(36).substring(2);
logger.log('access token is: %s', accessToken);
// run building of a project
router.post('/api/0.1/builds', function(req, res, next) {
Steppy(
function() {
var projectName = req.body.project,
project = _(app.projects).findWhere({name: projectName});
project = app.projects.get(projectName);
if (project) {
res.statusCode = 204;
logger.log('Run project "%s"', projectName);
app.distributor.run({
app.builds.create({
projectName: projectName,
withScmChangesOnly: req.body.withScmChangesOnly,
skipQueued: req.body.skipQueued,
queueQueued: req.body.queueQueued,
initiator: {type: 'httpApi'}
});
} else {
@ -76,16 +79,19 @@ module.exports = function(app) {
);
});
// TODO: restrict access with some sort of token
router.del('/api/0.1/projects/:name', function(req, res, next) {
var projectName = req.params.name;
var token = req.body.token,
projectName = req.params.name;
Steppy(
function() {
logger.log('Cleaning up project "%s"', projectName);
app.lib.project.remove({
baseDir: app.config.paths.projects,
name: projectName
}, this.slot());
if (token !== accessToken) {
throw new Error('Access token doesn`t match');
}
app.projects.remove(projectName, this.slot());
},
function() {
logger.log('Project "%s" cleaned up', projectName);
@ -97,7 +103,8 @@ module.exports = function(app) {
});
router.patch('/api/0.1/projects/:name', function(req, res, next) {
var projectName = req.params.name,
var token = req.body.token,
projectName = req.params.name,
newProjectName = req.body.name;
Steppy(
@ -106,30 +113,28 @@ module.exports = function(app) {
'Rename project "%s" to "%s"', projectName, newProjectName
);
if (token !== accessToken) {
throw new Error('Access token doesn`t match');
}
if (!newProjectName) throw new Error('new project name is not set');
var curProject = _(app.projects).findWhere({name: projectName});
var curProject = app.projects.get(projectName);
if (!curProject) {
throw new Error('Project "' + projectName + '" not found');
}
this.pass(curProject);
var newProject = _(app.projects).findWhere({name: newProjectName});
var newProject = app.projects.get(newProjectName);
if (newProject) {
throw new Error(
'Project name "' + newProjectName + '" already used'
);
}
app.lib.project.rename({
baseDir: app.config.paths.projects,
name: projectName,
newName: newProjectName
}, this.slot());
app.projects.rename(projectName, newProjectName, this.slot());
},
function(err, curProject) {
curProject.name = newProjectName;
function(err) {
res.statusCode = 204;
res.end();
},

222
lib/build.js Normal file
View File

@ -0,0 +1,222 @@
'use strict';
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
/**
* Facade entity which accumulates operations with currently running and
* db saved builds.
*/
function BuildsCollection(params) {
this.db = params.db;
this.distributor = params.distributor;
this._proxyDistributorEvent('buildUpdate', 'buildUpdated');
this._proxyDistributorEvent('buildCancel', 'buildCanceled');
this._proxyDistributorEvent('buildLogLines', 'buildLogLines');
}
exports.BuildsCollection = BuildsCollection;
inherits(BuildsCollection, EventEmitter);
BuildsCollection.prototype._proxyDistributorEvent = function(source, dest) {
var self = this;
self.distributor.on(source, function() {
self.emit.apply(self, [dest].concat(_(arguments).toArray()));
});
};
/**
* Create build by running given project.
* - `params.projectName` - project to build
* - `params.withScmChangesOnly` - if true then build will be started only if
* there is scm changes for project
* - `params.queueQueued` - if true then currently queued project can be queued
* again
* - `params.initiator` - contains information about initiator of the build,
* must contain `type` property e.g. when one build triggers another:
* initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
*
* @param {Object} params
* @param {Function} [callback(err)]
*/
BuildsCollection.prototype.create = function(params, callback) {
this.distributor.run(params, callback);
};
/**
* Cancel build by id.
* Note that only queued build can be canceled currently.
*
* @param {Number} id
* @param {Function} [callback(err)]
*/
BuildsCollection.prototype.cancel = function(id, callback) {
this.distributor.cancel(id, callback);
};
/**
* Get build by id.
*
* @param {Number} id
* @param {Function} callback(err,build)
*/
BuildsCollection.prototype.get = function(id, callback) {
this.db.builds.find({start: {id: id}}, function(err, builds) {
callback(err, builds && builds[0]);
});
};
/**
* Get log lines for the given build.
* - `params.buildId` - target build
* - `params.from` - if set then lines from that number will be returned
* - `params.to` - if set then lines to that number will be returned
*
* @param {Object} params
* @param {Function} callback(err,logLinesData)
*/
BuildsCollection.prototype.getLogLines = function(params, callback) {
var self = this;
Steppy(
function() {
var findParams = {
start: {buildId: params.buildId},
end: {buildId: params.buildId}
};
if (params.from) findParams.start.number = params.from;
if (params.to) findParams.end.number = params.to;
var count = params.from && params.to ? params.to - params.from + 1: 0;
self.db.logLines.find(findParams, this.slot());
this.pass(count);
},
function(err, logLines, count) {
this.pass({
lines: logLines,
isLast: count ? logLines.length < count : true
});
},
callback
);
};
BuildsCollection.prototype.getLogLinesTail = function(params, callback) {
var self = this;
Steppy(
function() {
var findParams = {
reverse: true,
start: {buildId: params.buildId},
limit: params.limit
};
self.db.logLines.find(findParams, this.slot());
},
function(err, logLines) {
var lines = logLines.reverse(),
total = logLines.length ? logLines[logLines.length - 1].number : 0;
this.pass({lines: lines, total: total});
},
callback
);
};
/**
* Calculate average build duration for the given builds.
*
* @param {Object[]} builds
*/
BuildsCollection.prototype.getAvgBuildDuration = function(builds) {
var durationsSum = _(builds).reduce(function(sum, build) {
return sum + (build.endDate - build.startDate);
}, 0);
return Math.round(durationsSum / builds.length);
};
/**
* Get builds sorted by date in descending order.
* - `params.projectName` - optional project filter
* - `params.status` - optional status filter, can be used only when
* `params.projectName` is set. When used builds in the result will contain
* only following fields: id, number, startDate, endDate
*
* @param {Object} params
* @param {Function} callback(err,builds)
*/
BuildsCollection.prototype.getRecent = function(params, callback) {
params.limit = params.limit || 20;
var self = this;
Steppy(
function() {
var findParams = {start: {}, limit: params.limit};
// such condition for match one of projections:
// projectName, descCreateDate
// projectName, status, descCreateDate
// or just descCreateDate projection if project name is not set
if (params.projectName) {
findParams.start.projectName = params.projectName;
if (params.status) findParams.start.status = params.status;
}
findParams.start.descCreateDate = '';
self.db.builds.find(findParams, this.slot());
},
callback
);
};
/**
* Get info about current done builds streak.
* - `params.projectName` - optional project filter
*
* @param {Object} params
* @param {Function} callback(err,doneStreak)
*/
BuildsCollection.prototype.getDoneStreak = function(params, callback) {
var self = this;
Steppy(
function() {
var start = {};
if (params.projectName) start.projectName = params.projectName;
start.descCreateDate = '';
// tricky but effective streak counting inside filter goes below
var doneBuildsStreakCallback = _(this.slot()).once(),
doneBuildsStreak = {buildsCount: 0};
self.db.builds.find({
start: start,
filter: function(build) {
// error exits streak
if (build.status === 'error') {
doneBuildsStreakCallback(null, doneBuildsStreak);
return true;
}
if (build.status === 'done') {
doneBuildsStreak.buildsCount++;
}
},
limit: 1
}, function(err) {
doneBuildsStreakCallback(err, doneBuildsStreak);
});
},
callback
);
};

86
lib/distributor.js
View File

@ -27,6 +27,9 @@ function Distributor(params) {
};
self.projects = params.projects;
self.buildLogLineNumbersHash = {},
self.lastLinesHash = {};
}
inherits(Distributor, EventEmitter);
@ -101,7 +104,7 @@ Distributor.prototype._runNext = function(callback) {
});
executor.on('data', function(data) {
self.emit('buildData', build, data);
self._onBuildData(build, data);
});
executor.once('scmData', function(scmData) {
@ -115,7 +118,7 @@ Distributor.prototype._runNext = function(callback) {
id: build.id,
number: build.number,
project: {name: build.project.name}
},
}
});
}
});
@ -127,6 +130,57 @@ Distributor.prototype._runNext = function(callback) {
);
};
Distributor.prototype._onBuildData = function(build, data) {
var self = this;
self.emit('buildData', build, data);
var cleanupText = function(text) {
return text.replace('\r', '');
};
var splittedData = data.split('\n'),
logLineNumber = self.buildLogLineNumbersHash[build.id] || 0;
self.lastLinesHash[build.id] = self.lastLinesHash[build.id] || '';
// if we don't have last line, so we start new line
if (!self.lastLinesHash[build.id]) {
logLineNumber++;
}
self.lastLinesHash[build.id] += _(splittedData).first();
var lines = [{
text: cleanupText(self.lastLinesHash[build.id]),
buildId: build.id,
number: logLineNumber
}];
if (splittedData.length > 1) {
// if we have last '' we have to take all except last
// this shown that string ends with eol
if (_(splittedData).last() === '') {
self.lastLinesHash[build.id] = '';
splittedData = _(splittedData.slice(1)).initial();
} else {
self.lastLinesHash[build.id] = _(splittedData).last();
splittedData = _(splittedData).tail();
}
lines = lines.concat(_(splittedData).map(function(line) {
return {
text: cleanupText(line),
buildId: build.id,
number: ++logLineNumber
};
}));
}
self.buildLogLineNumbersHash[build.id] = logLineNumber;
self.emit('buildLogLines', build, lines);
};
Distributor.prototype._updateWaitReasons = function() {
var self = this;
_(self.queue).each(function(item) {
@ -164,7 +218,7 @@ Distributor.prototype._onBuildComplete = function(build, callback) {
id: build.id,
number: build.number,
project: {name: build.project.name}
},
}
}, triggerAfterGroup.slot());
}
});
@ -232,17 +286,21 @@ Distributor.prototype._updateBuild = function(build, changes, callback) {
);
};
Distributor.prototype.cancel = function(params, callback) {
Distributor.prototype.cancel = function(id, callback) {
callback = callback || function(err) {
if (err) logger.error('Error during cancel: ', err.stack || err);
};
var self = this;
Steppy(
function() {
var queueItemIndex = _(self.queue).findIndex(function(item) {
return item.build.id === params.buildId;
return item.build.id === id;
});
if (queueItemIndex === -1) {
throw new Error(
'Build with id "' + params.buildId + '" not found for cancel'
'Build with id "' + id + '" not found for cancel'
);
}
@ -263,19 +321,15 @@ Distributor.prototype.cancel = function(params, callback) {
};
Distributor.prototype.run = function(params, callback) {
callback = callback || function(err) {
if (err) logger.error('Error during run: ', err.stack || err);
};
var self = this,
project;
callback = callback || function(err) {
if (err) {
logger.error('Error during run: ', err.stack || err);
}
};
Steppy(
function() {
project = _(self.projects).chain()
.findWhere({name: params.projectName})
.clone()
.value();
project = _(self.projects.get(params.projectName)).clone();
if (params.withScmChangesOnly) {
self.nodes[0].hasScmChanges(project, this.slot());
@ -292,7 +346,7 @@ Distributor.prototype.run = function(params, callback) {
return callback();
}
if (params.skipQueued) {
if (!params.queueQueued) {
var queuedItem = _(self.queue).find(function(item) {
return item.project.name === project.name;
});

2
lib/executor/base.js
View File

@ -17,7 +17,7 @@ inherits(Executor, EventEmitter);
Executor.prototype.throttledEmit = _(function() {
this.emit.apply(this, arguments);
}).throttle(1500);
}).throttle(500);
Executor.prototype._getSources = function(params, callback) {
};

302
lib/project.js
View File

@ -4,65 +4,92 @@ var Steppy = require('twostep').Steppy,
fs = require('fs'),
path = require('path'),
_ = require('underscore'),
reader = require('./reader'),
db = require('../db'),
utils = require('./utils'),
SpawnCommand = require('./command/spawn').Command;
SpawnCommand = require('./command/spawn').Command,
validateParams = require('./validateParams'),
EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
/**
* Validates and returns given `config` to the `callback`(err, config)
* Projects collection contains all currently loaded projects and provides
* operations for manipulating with them.
* All projects stored on disk in `baseDir` and loaded to memory so
* they can be received (by `get`, `getAll` and other methods) in a sync way.
* Note that id for the particular project is a `name` of that project.
*/
exports.validateConfig = function(config, callback) {
callback(null, config);
};
function ProjectsCollection(params) {
this.db = params.db;
this.reader = params.reader;
this.baseDir = params.baseDir;
this.configs = [];
}
exports.ProjectsCollection = ProjectsCollection;
inherits(ProjectsCollection, EventEmitter);
/**
* Loads and returns project
* Validate and return given config.
*
* @param {Object} config
* @param {Function} callback(err,config)
*/
exports.load = function(baseDir, name, callback) {
var dir = path.join(baseDir, name);
ProjectsCollection.prototype.validateConfig = function(config, callback) {
Steppy(
function() {
fs.readdir(dir, this.slot());
},
function(err, dirContent) {
exports.loadConfig(dir, this.slot());
},
function(err, config) {
exports.validateConfig(config, this.slot());
},
function(err, config) {
config.name = name;
config.dir = dir;
this.pass(config);
},
callback
);
};
/**
* Loads all projects from `baseDir` and returns array of projects
*/
exports.loadAll = function(baseDir, callback) {
Steppy(
function() {
fs.readdir(baseDir, this.slot());
},
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
exports.load(baseDir, dir, loadGroup.slot());
validateParams(config, {
type: 'object',
properties: {
scm: {
type: 'object',
required: true,
properties: {
type: {enum: ['git', 'mercurial'], required: true},
repository: {type: 'string', required: true},
rev: {type: 'string', required: true}
}
},
steps: {
type: 'array',
required: true,
items: {
type: 'object',
properties: {
cmd: {type: 'string', required: true},
name: {type: 'string'},
type: {enum: ['shell']},
shell: {type: 'string'}
}
}
}
},
additionalProperties: true
});
this.pass(null);
},
callback
function(err) {
if (err) {
err.message = (
'Error during validation of project "' + config.name +
'": ' + err.message
);
}
callback(err, config);
}
);
};
exports.loadConfig = function(dir, callback) {
ProjectsCollection.prototype._getProjectPath = function(name) {
return path.join(this.baseDir, name);
};
ProjectsCollection.prototype._loadConfig = function(dir, callback) {
var self = this;
Steppy(
function() {
reader.load(dir, 'config', this.slot());
self.reader.load(dir, 'config', this.slot());
},
function(err, config) {
// convert steps object to array
@ -82,7 +109,7 @@ exports.loadConfig = function(dir, callback) {
// apply defaults
_(config.steps).each(function(step) {
if (!step.type) step.type = 'shell';
if (!step.name) step.name = utils.prune(step.cmd, 40);
if (!step.name && step.cmd) step.name = utils.prune(step.cmd, 40);
});
this.pass(config);
@ -91,73 +118,158 @@ exports.loadConfig = function(dir, callback) {
);
};
exports.saveConfig = function(config, dir, callback) {
fs.writeFile(
path.join(dir, 'config.json'),
JSON.stringify(config, null, 4),
callback
);
};
/**
* Load project to collection.
* `projectLoaded` event with loaded config as argument will be emitted after
* load.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.load = function(name, callback) {
callback = callback || _.noop;
var self = this,
dir = self._getProjectPath(name);
exports.create = function(baseDir, config, callback) {
var dir;
Steppy(
function() {
dir = path.join(baseDir, config.name);
fs.mkdir(dir, this.slot());
if (self.get(name)) {
throw new Error(
'Can`t load already loaded project "' + name + '"'
);
}
self._loadConfig(dir, this.slot());
},
function(err) {
exports.saveConfig(config, baseDir, this.slot());
function(err, config) {
config.name = name;
config.dir = dir;
self.validateConfig(config, this.slot());
},
function(err) {
exports.load(dir, this.slot());
function(err, config) {
self.configs.push(config);
self.emit('projectLoaded', config);
this.pass(null);
},
callback
);
};
exports.getAvgProjectBuildDuration = function(projectName, callback) {
/**
* Load all projects (from `this.baseDir`).
* Calls `load` for every project in a base dir.
*
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.loadAll = function(callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
// get last done builds to calc avg build time
db.builds.find({
start: {
projectName: projectName,
status: 'done',
descCreateDate: ''
},
limit: 10
}, this.slot());
fs.readdir(self.baseDir, this.slot());
},
function(err, doneBuilds) {
var durationsSum = _(doneBuilds).reduce(function(memo, build) {
return memo + (build.endDate - build.startDate);
}, 0);
this.pass(Math.round(durationsSum / doneBuilds.length));
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
self.load(dir, loadGroup.slot());
});
},
callback
);
};
exports.remove = function(params, callback) {
/**
* Unload project from collection
* `projectUnloaded` event with unloaded config as argument will be emitted
* after unload.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.unload = function(name, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
db.builds.find({
start: {projectName: params.name, descCreateDate: ''}
var index = _(self.configs).findIndex(function(config) {
return config.name === name;
});
if (index === -1) {
throw new Error('Can`t unload not loaded project: "' + name + '"');
}
var unloadedConfig = self.configs.splice(index, 1)[0];
self.emit('projectUnloaded', unloadedConfig);
this.pass(null);
},
callback
);
};
/**
* Get project config by name.
* Returns config object or undefined if project is not found.
*
* @param {String} name
*/
ProjectsCollection.prototype.get = function(name) {
return _(this.configs).findWhere({name: name});
};
/**
* Get configs for all currently loaded projects.
* Returns array of config objects.
*/
ProjectsCollection.prototype.getAll = function() {
return this.configs;
};
/**
* Get project configs which match to predicate.
* Returns array of config objects or empty array if there is no matched
* project.
*
* @param {Function} predicate
*/
ProjectsCollection.prototype.filter = function(predicate) {
return _(this.configs).filter(predicate);
};
/**
* Remove project by name.
* Calls `unload`, removes project from disk and db.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.remove = function(name, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
self.db.builds.find({
start: {projectName: name, descCreateDate: ''}
}, this.slot());
new SpawnCommand().run({cmd: 'rm', args: [
'-Rf', path.join(params.baseDir, params.name)
'-Rf', self._getProjectPath(name)
]}, this.slot());
self.unload(name, this.slot());
},
function(err, builds) {
if (builds.length) {
db.builds.del(builds, this.slot());
self.db.builds.del(builds, this.slot());
var logLinesRemoveGroup = this.makeGroup();
_(builds).each(function(build) {
db.logLines.remove({
self.db.logLines.remove({
start: {buildId: build.id}
}, logLinesRemoveGroup.slot());
});
@ -169,24 +281,40 @@ exports.remove = function(params, callback) {
);
};
exports.rename = function(params, callback) {
/**
* Rename project.
* Renames project on disk and db, also changes name for loaded project.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.rename = function(name, newName, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
fs.rename(
path.join(params.baseDir, params.name),
path.join(params.baseDir, params.newName),
self._getProjectPath(name),
self._getProjectPath(newName),
this.slot()
);
db.builds.multiUpdate(
{start: {projectName: params.name, descCreateDate: ''}},
self.db.builds.multiUpdate(
{start: {projectName: name, descCreateDate: ''}},
function(build) {
build.project.name = params.newName;
build.project.name = newName;
return build;
},
this.slot()
);
},
function() {
// just update currently loaded project name by link
self.get(name).name = newName;
this.pass(null);
},
callback
);
};

12
lib/scm/git.js
View File

@ -15,9 +15,8 @@ inherits(Scm, ParentScm);
Scm.prototype.defaultRev = 'master';
// use 2 invisible separators as fields separator
Scm.prototype._fieldsSeparator = String.fromCharCode(2063);
Scm.prototype._fieldsSeparator += Scm.prototype._fieldsSeparator;
Scm.prototype._fieldsSeparator = '\u2063' + '\u2063';
Scm.prototype._linesSeparator = '\u2028' + '\u2028';
Scm.prototype._revTemplate = [
'%h', '%cn', '%cd', '%s', '%d'
@ -149,15 +148,16 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) {
self.run({cmd: 'git', args: [
'log', rev1 ? rev1 + '..' + rev2 : rev2,
'--pretty=' + self._revTemplate
'--pretty=' + self._revTemplate + self._linesSeparator
]}, this.slot());
},
function(err, currentRev, stdout) {
// always skip last line - it's empty
var rows = stdout.split('\n').slice(0, -1);
var rows = stdout.split(self._linesSeparator).slice(0, -1);
var changes = _(rows).map(function(str) {
return self._parseRev(str);
// remove line break which git log add between commits
return self._parseRev(str.replace(/^\n/, ''));
});
this.pass(currentRev, changes);

53
lib/validateConfig.js Normal file
View File

@ -0,0 +1,53 @@
'use strict';
var Steppy = require('twostep').Steppy,
validateParams = require('./validateParams');
module.exports = function(config, callback) {
Steppy(
function() {
validateParams(config, {
type: 'object',
properties: {
plugins: {
type: 'array',
items: {type: 'string'}
},
nodes: {
type: 'array',
required: true,
items: {
type: 'object',
properties: {
type: {type: 'string', enum: ['local']},
maxExecutorsCount: {type: 'integer'}
}
},
minItems: 1
},
storage: {
type: 'object',
required: true,
properties: {
backend: {type: 'string', required: true}
}
},
notify: {
type: 'object'
}
},
additionalProperties: true
});
this.pass(null);
},
function(err) {
if (err) {
err.message = (
'Error during validation server config: "' + err.message
);
}
callback(err, config);
}
);
};

19
lib/validateParams.js Normal file
View File

@ -0,0 +1,19 @@
'use strict';
var conform = require('conform'),
_ = require('underscore');
module.exports = function(params, schema, validateOptions) {
var defaultValidateOptions = {
additionalProperties: false,
failOnFirstError: true,
cast: true,
castSource: true,
applyDefaultValue: true
};
conform.validate(params, schema, _({}).extend(
defaultValidateOptions, validateOptions
));
return params;
};

9
package.json
View File

@ -1,13 +1,13 @@
{
"name": "nci",
"version": "0.4.1",
"version": "0.4.2",
"description": "Continuous integration server written in node.js",
"bin": {
"nci": "bin/nci"
},
"scripts": {
"makeTestRepos": "rm -rf test/repos/{mercurial,git}; cd test/repos/ && tar -xf mercurial.tar.gz && tar -xf git.tar.gz",
"test": "npm run makeTestRepos && mocha --bail --reporter=spec --timeout 4000",
"test": "npm run makeTestRepos && mocha --bail --reporter=spec --timeout 10000",
"build-less": "lessc app/styles/index.less > static/css/index.css",
"build-js": "browserify app/app.js -t ./transforms/jade.js | uglifyjs -mc > static/js/app.build.js",
"watch-less": "chokidar --initial --silent app/styles/**/*.less app/styles/index.less -c 'npm run build-less'",
@ -15,6 +15,9 @@
"watch-server": "nodemon --ignore app --ignore static --ignore node_modules --ignore data app.js",
"dev": "parallelshell 'npm run build-fonts' 'npm run watch-less' 'npm run watch-js' 'npm run watch-server'",
"sync": "npm install && npm prune",
"docProjectsCollection": "dox --api --skipSingleStar < lib/project.js | sed '/^ - \\[ProjectsCollection/ d' > docs/developing-plugins/projects-collection.md",
"docBuildsCollection": "dox --api --skipSingleStar < lib/build.js | sed '/^ - \\[BuildsCollection/ d' > docs/developing-plugins/builds-collection.md",
"doc": "nrun docProjectsCollection && nrun docBuildsCollection",
"build-fonts": "cp ./node_modules/bootstrap/fonts/* ./static/fonts/ & cp ./node_modules/font-awesome/fonts/* ./static/fonts/",
"build-clean": "rm static/index.html",
"build-html": "jade views/index.jade --obj '{\"env\": \"production\"}' -o static/",
@ -52,6 +55,7 @@
"browserify": "12.0.1",
"chokidar": "1.0.3",
"colors": "1.1.2",
"conform": "0.2.12",
"cron": "1.0.9",
"data.io": "0.3.0",
"font-awesome": "4.5.0",
@ -71,6 +75,7 @@
},
"devDependencies": {
"chokidar-cli": "1.2.0",
"dox": "0.8.0",
"expect.js": "0.3.1",
"jade": "1.11.0",
"jshint": "2.9.1-rc1",

30
projectsWatcher.js
View File

@ -2,45 +2,37 @@
var _ = require('underscore'),
path = require('path'),
chokidar = require('chokidar'),
project = require('./lib/project'),
logger = require('./lib/logger')('projects watcher');
chokidar = require('chokidar');
exports.init = function(app, callback) {
var logger = app.lib.logger('projects watcher');
// start file watcher for reloading projects on change
var syncProject = function(filename, fileInfo) {
var baseDir = app.config.paths.projects,
projectName = path.relative(
baseDir,
path.dirname(filename)
);
var projectName = path.relative(
app.config.paths.projects,
path.dirname(filename)
);
var projectIndex = _(app.projects).findIndex(function(project) {
return project.name === projectName;
});
if (projectIndex !== -1) {
if (app.projects.get(projectName)) {
logger.log('Unload project: "' + projectName + '"');
var unloadedProject = app.projects.splice(projectIndex, 1)[0];
app.emit('projectUnloaded', unloadedProject);
app.projects.unload(projectName);
}
// on add or change (info is falsy on unlink)
if (fileInfo) {
logger.log('Load project "' + projectName + '" on change');
project.load(baseDir, projectName, function(err, project) {
app.projects.load(projectName, function(err) {
if (err) {
return logger.error(
'Error during load project "' + projectName + '": ',
err.stack || err
);
}
app.projects.push(project);
logger.log(
'Project "' + projectName + '" loaded:',
JSON.stringify(project, null, 4)
JSON.stringify(app.projects.get(projectName), null, 4)
);
app.emit('projectLoaded', project);
});
}
};

65
resources/builds.js
View File

@ -2,31 +2,22 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
db = require('../db'),
utils = require('../lib/utils'),
logger = require('../lib/logger')('builds resource');
module.exports = function(app) {
var resource = app.dataio.resource('builds'),
distributor = app.distributor;
var resource = app.dataio.resource('builds');
resource.use('readAll', function(req, res, next) {
Steppy(
function() {
var data = req.data || {};
var data = req.data || {},
getParams = {limit: data.limit || 20};
var start = {};
if (data.projectName) {
start.projectName = data.projectName;
getParams.projectName = data.projectName;
}
start.descCreateDate = data.descCreateDate || '';
var findParams = _(data).pick('offset', 'limit');
findParams.start = start;
findParams.limit = findParams.limit || 20;
db.builds.find(findParams, this.slot());
app.builds.getRecent(getParams, this.slot());
},
function(err, builds) {
// omit big fields not needed for list
@ -49,12 +40,10 @@ module.exports = function(app) {
resource.use('read', function(req, res, next) {
Steppy(
function() {
var findParams = {};
findParams.start = _(req.data).pick('id');
db.builds.find(findParams, this.slot());
app.builds.get(req.data.id, this.slot());
},
function(err, build) {
res.send(build[0]);
res.send(build);
},
next
);
@ -63,19 +52,13 @@ module.exports = function(app) {
resource.use('getBuildLogTail', function(req, res, next) {
Steppy(
function() {
var findParams = {
reverse: true,
start: {buildId: req.data.buildId},
app.builds.getLogLinesTail({
buildId: req.data.buildId,
limit: req.data.length
};
db.logLines.find(findParams, this.slot());
}, this.slot());
},
function(err, logLines) {
var lines = logLines.reverse(),
total = logLines.length ? logLines[0].number : 0;
res.send({lines: lines, total: total});
function(err, tail) {
res.send(tail);
},
next
);
@ -84,23 +67,13 @@ module.exports = function(app) {
resource.use('getBuildLogLines', function(req, res, next) {
Steppy(
function() {
var buildId = req.data.buildId,
from = req.data.from,
to = req.data.to,
count = to - from;
db.logLines.find({
start: {buildId: buildId, number: from},
end: {buildId: buildId, number: to}
}, this.slot());
this.pass(count);
app.builds.getLogLines(
_(req.data).pick('buildId', 'from', 'to'),
this.slot()
);
},
function(err, logLines, count) {
res.send({
lines: logLines,
isLast: logLines.length < count
});
function(err, logLinesData) {
res.send(logLinesData);
},
next
);
@ -111,7 +84,7 @@ module.exports = function(app) {
function() {
var buildId = req.data.buildId;
logger.log('Cancel build: "%s"', buildId);
distributor.cancel({buildId: buildId}, this.slot());
app.builds.cancel(buildId, this.slot());
},
function() {
res.send();

37
resources/helpers.js Normal file
View File

@ -0,0 +1,37 @@
'use strict';
var Steppy = require('twostep').Steppy,
logger = require('../lib/logger')('create build resource');
var buildDataResourcesHash = {};
// create resource for build data
exports.createBuildDataResource = function(app, buildId) {
if (buildId in buildDataResourcesHash) {
return;
}
var buildDataResource = app.dataio.resource('build' + buildId);
buildDataResource.on('connection', function(client) {
var callback = this.async();
Steppy(
function() {
app.builds.getLogLines({buildId: buildId}, this.slot());
},
function(err, logLinesData) {
client.emit('sync', 'data', {lines: logLinesData.lines});
this.pass(null);
},
function(err) {
if (err) {
logger.error(
'error during read log for "' + buildId + '":',
err.stack || err
);
}
callback();
}
);
});
buildDataResourcesHash[buildId] = buildDataResource;
};

33
resources/index.js
View File

@ -1,11 +1,42 @@
'use strict';
var _ = require('underscore'),
errorHandler = require('./errorHandler');
errorHandler = require('./errorHandler'),
helpers = require('./helpers');
module.exports = function(app) {
_(['builds', 'projects']).each(function(resource) {
var resource = require('./' + resource)(app);
resource.use(errorHandler);
});
var buildsResource = app.dataio.resource('builds');
app.builds.on('buildUpdated', function(build, changes) {
if (build.status === 'queued') {
helpers.createBuildDataResource(app, build.id);
}
// notify about build's project change, coz building affects project
// related stat (last build date, avg build time, etc)
if (changes.completed) {
var projectsResource = app.dataio.resource('projects');
projectsResource.clientEmitSyncChange(build.project.name);
}
buildsResource.clientEmitSync('change', {
buildId: build.id, changes: changes
});
});
app.builds.on('buildCanceled', function(build) {
buildsResource.clientEmitSync('cancel', {buildId: build.id});
});
app.builds.on('buildLogLines', function(build, lines) {
app.dataio.resource('build' + build.id).clientEmitSync(
'data',
{lines: lines}
);
});
};

81
resources/projects.js
View File

@ -2,80 +2,50 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
getAvgProjectBuildDuration =
require('../lib/project').getAvgProjectBuildDuration,
createBuildDataResource = require('../distributor').createBuildDataResource,
logger = require('../lib/logger')('projects resource'),
db = require('../db');
helpers = require('./helpers'),
logger = require('../lib/logger')('projects resource');
module.exports = function(app) {
var resource = app.dataio.resource('projects'),
distributor = app.distributor;
var resource = app.dataio.resource('projects');
resource.use('createBuildDataResource', function(req, res) {
createBuildDataResource(req.data.buildId);
helpers.createBuildDataResource(app, req.data.buildId);
res.send();
});
resource.use('readAll', function(req, res) {
var filteredProjects = app.projects,
var filteredProjects = app.projects.getAll(),
nameQuery = req.data && req.data.nameQuery;
if (nameQuery) {
filteredProjects = _(filteredProjects).filter(function(project) {
filteredProjects = app.projects.filter(function(project) {
return project.name.indexOf(nameQuery) !== -1;
});
}
filteredProjects = _(filteredProjects).sortBy('name');
res.send(filteredProjects);
});
var getProject = function(params, callback) {
// get project with additional fields
var getProject = function(name, callback) {
var project;
Steppy(
function() {
project = _(app.projects).findWhere(params.condition);
project = _(app.projects.get(name)).clone();
getAvgProjectBuildDuration(project.name, this.slot());
// get last done build
db.builds.find({
start: {
projectName: project.name,
status: 'done',
descCreateDate: ''
},
limit: 1
app.builds.getRecent({
projectName: project.name,
status: 'done',
limit: 10
}, this.slot());
// tricky but effective streak counting inside filter goes below
var doneBuildsStreakCallback = _(this.slot()).once(),
doneBuildsStreak = 0;
db.builds.find({
start: {
projectName: project.name,
descCreateDate: ''
},
filter: function(build) {
// error exits streak
if (build.status === 'error') {
doneBuildsStreakCallback(null, doneBuildsStreak);
return true;
}
if (build.status === 'done') {
doneBuildsStreak++;
}
},
limit: 1
}, function(err) {
doneBuildsStreakCallback(err, doneBuildsStreak);
});
app.builds.getDoneStreak({projectName: project.name}, this.slot());
},
function(err, avgProjectBuildDuration, lastDoneBuilds, doneBuildsStreak) {
project.lastDoneBuild = lastDoneBuilds[0];
project.avgBuildDuration = avgProjectBuildDuration;
function(err, doneBuilds, doneBuildsStreak) {
project.avgBuildDuration = app.builds.getAvgBuildDuration(doneBuilds);
project.lastDoneBuild = doneBuilds[0];
project.doneBuildsStreak = doneBuildsStreak;
this.pass(project);
@ -84,12 +54,12 @@ module.exports = function(app) {
);
};
// resource custom method which finds project by condition
// resource custom method which finds project by name
// and emits event about it change to clients
resource.clientEmitSyncChange = function(condition) {
resource.clientEmitSyncChange = function(name) {
Steppy(
function() {
getProject({condition: condition}, this.slot());
getProject(name, this.slot());
},
function(err, project) {
resource.clientEmitSync('change', {project: project});
@ -106,7 +76,7 @@ module.exports = function(app) {
resource.use('read', function(req, res) {
Steppy(
function() {
getProject({condition: req.data}, this.slot());
getProject(req.data.name, this.slot());
},
function(err, project) {
res.send(project);
@ -117,9 +87,10 @@ module.exports = function(app) {
resource.use('run', function(req, res) {
var projectName = req.data.projectName;
logger.log('Run the project: "%s"', projectName);
distributor.run({
app.builds.create({
projectName: projectName,
initiator: {type: 'user'}
initiator: {type: 'user'},
queueQueued: true
});
res.send();
});

15
scheduler.js
View File

@ -1,24 +1,27 @@
'use strict';
var _ = require('underscore'),
logger = require('./lib/logger')('scheduler'),
CronJob = require('cron').CronJob;
exports.init = function(app, callback) {
var distributor = app.distributor,
var logger = app.lib.logger('scheduler'),
projectJobs = {};
app.on('projectLoaded', function(project) {
app.projects.on('projectLoaded', function(project) {
var time = project.buildEvery && project.buildEvery.time;
if (time) {
logger.log('Start job for loaded project "%s"', project.name);
logger.log(
'Start job for loaded project "%s" by schedule "%s"',
project.name,
time
);
projectJobs[project.name] = {};
projectJobs[project.name].job = new CronJob({
cronTime: time,
onTick: function() {
logger.log('Run project "%s"', project.name);
distributor.run({
app.builds.create({
projectName: project.name,
withScmChangesOnly: project.buildEvery.withScmChangesOnly,
initiator: {type: 'scheduler'}
@ -29,7 +32,7 @@ exports.init = function(app, callback) {
}
});
app.on('projectUnloaded', function(project) {
app.projects.on('projectUnloaded', function(project) {
if (project.name in projectJobs) {
logger.log('Stop job for unloaded project "%s"', project.name);
projectJobs[project.name].job.stop();

17
test/distributor/blocking.js
View File

@ -4,6 +4,7 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock,
Steppy = require('twostep').Steppy;
@ -119,11 +120,11 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run 2 non-blocking projects in parallel', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2'
}];
}]);
});
itRunParallelProjects();
@ -131,12 +132,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 2 blocked by 1', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2',
blockedBy: ['project1']
}];
}]);
});
itRunSequentialProjects();
@ -144,12 +145,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 1 blocks 2', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project2']
}, {
name: 'project2'
}];
}]);
});
itRunSequentialProjects();
@ -159,7 +160,7 @@ describe('Distributor blocking with max 2 executors count', function() {
'should run 1, 2 in parallel, when 1 block 3, 2 blocked by 3',
function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project3']
}, {
@ -167,7 +168,7 @@ describe('Distributor blocking with max 2 executors count', function() {
blockedBy: ['project3']
}, {
name: 'project3'
}];
}]);
});
itRunParallelProjects();

8
test/distributor/helpers.js
View File

@ -1,7 +1,8 @@
'use strict';
var Node = require('../../lib/node').Node,
EventEmitter = require('events').EventEmitter;
EventEmitter = require('events').EventEmitter,
ProjectsCollection = require('../../lib/project').ProjectsCollection;
exports.createNodeMock = function(executorRun) {
@ -17,3 +18,8 @@ exports.createNodeMock = function(executorRun) {
};
};
exports.createProjectsMock = function(configs) {
var projects = new ProjectsCollection({});
projects.configs = configs;
return projects;
};

9
test/distributor/main.js
View File

@ -3,12 +3,13 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor main', function() {
var distributor,
projects = [{name: 'project1'}];
projects = createProjectsMock([{name: 'project1'}]);
var expectUpdateBuild = function(distributor, build, number, conditionsHash) {
var conditions = conditionsHash[number];
@ -154,7 +155,7 @@ describe('Distributor main', function() {
var originalRunNext = distributor._runNext;
distributor._runNext = function() {
distributor.cancel({buildId: 1}, function(err) {
distributor.cancel(1, function(err) {
cancelError = err;
});
originalRunNext.apply(distributor, arguments);
@ -197,7 +198,7 @@ describe('Distributor main', function() {
var originalRunNext = distributor._runNext;
distributor._runNext = function() {
distributor.cancel({buildId: 2}, function(err) {
distributor.cancel(2, function(err) {
cancelError = err;
});
originalRunNext.apply(distributor, arguments);

6
test/distributor/runSelfAfterCatchRev.js
View File

@ -2,9 +2,9 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
helpers = require('../helpers'),
createProjectsMock = require('./helpers').createProjectsMock,
path = require('path');
describe('Distributor run self after catch', function() {
var distributor, executorRunSpy, scmDataSpy;
@ -17,7 +17,7 @@ describe('Distributor run self after catch', function() {
helpers.removeDirIfExists(workspacePath, done);
distributor = new Distributor({
projects: [{
projects: createProjectsMock([{
name: 'project1',
dir: __dirname,
scm: helpers.repository.scm,
@ -25,7 +25,7 @@ describe('Distributor run self after catch', function() {
{type: 'shell', cmd: 'echo 1'}
],
catchRev: {comment: /.*/}
}],
}]),
nodes: nodes
});

39
test/distributor/triggerAfter.js
View File

@ -3,7 +3,8 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor trigger after', function() {
@ -13,14 +14,14 @@ describe('Distributor trigger after', function() {
describe('done when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{status: 'done', project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -39,11 +40,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -77,7 +82,9 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run totally 1 time', function() {
@ -91,14 +98,14 @@ describe('Distributor trigger after', function() {
describe('status is not set when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -117,11 +124,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -155,11 +166,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {