martin/nci
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/nci.git synced 2025-01-11 05:05:08 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #5 from node-ci/feature_lib_project_refactoring

Browse Source 
Feature lib project refactoring
This commit is contained in:
Oleg Korobenko 2016-01-08 00:01:36 +03:00
commit 3f84e7c94e
22 changed files with 801 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
app.js
View File

@ -10,7 +10,8 @@ var env = process.env.NODE_ENV || 'development',
_ = require('underscore'),
reader = require('./lib/reader'),
notifier = require('./lib/notifier'),
project = require('./lib/project'),
ProjectsCollection = require('./lib/project').ProjectsCollection,
BuildsCollection = require('./lib/build').BuildsCollection,
libLogger = require('./lib/logger'),
EventEmitter = require('events').EventEmitter,
validateConfig = require('./lib/validateConfig');
@ -205,20 +206,22 @@ Steppy(
db.init(app.config.paths.db, {db: dbBackend}, this.slot());
},
function() {
// load all projects for the first time
project.loadAll(app.config.paths.projects, this.slot());
app.projects = new ProjectsCollection({
db: db,
reader: reader,
baseDir: app.config.paths.projects
});
completeUncompletedBuilds(this.slot());
},
function(err, projects) {
// note that `app.projects` is live variable
app.projects = projects;
logger.log('Loaded projects: ', _(app.projects).pluck('name'));
function(err) {
require('./distributor').init(app, this.slot());
},
function(err, distributor) {
app.distributor = distributor;
app.builds = new BuildsCollection({
db: db,
distributor: distributor
});
// register other plugins
require('./lib/notifier/console').register(app);
@ -235,15 +238,17 @@ Steppy(
require('./scheduler').init(app, this.slot());
// notify about first project loading
_(app.projects).each(function(project) {
app.emit('projectLoaded', project);
});
// init resources
require('./resources')(app);
},
function() {
// load projects after all plugins to provide ability for plugins to
// handle `projectLoaded` event
app.projects.loadAll(this.slot());
},
function(err) {
logger.log('Loaded projects: ', _(app.projects.getAll()).pluck('name'));
var host = app.config.http.host,
port = app.config.http.port;
logger.log('Start http server on %s:%s', host, port);

130
distributor.js
View File

@ -3,9 +3,6 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
Distributor = require('./lib/distributor').Distributor,
getAvgProjectBuildDuration = (
require('./lib/project').getAvgProjectBuildDuration
),
db = require('./db'),
logger = require('./lib/logger')('distributor');
@ -18,13 +15,21 @@ exports.init = function(app, callback) {
Steppy(
function() {
if (_(build.project).has('avgBuildDuration')) {
this.pass(build.project.avgBuildDuration);
this.pass(null);
} else {
getAvgProjectBuildDuration(build.project.name, this.slot());
app.builds.getRecent({
projectName: build.project.name,
status: 'done',
limit: 10
}, this.slot());
}
},
function(err, avgBuildDuration) {
build.project.avgBuildDuration = avgBuildDuration;
function(err, doneBuilds) {
if (doneBuilds) {
build.project.avgBuildDuration = (
app.builds.getAvgBuildDuration(doneBuilds)
);
}
db.builds.put(build, this.slot());
},
@ -44,116 +49,7 @@ exports.init = function(app, callback) {
}
});
var buildDataResourcesHash = {};
// create resource for build data
var createBuildDataResource = function(buildId) {
if (buildId in buildDataResourcesHash) {
return;
}
var buildDataResource = app.dataio.resource('build' + buildId);
buildDataResource.on('connection', function(client) {
var callback = this.async();
Steppy(
function() {
db.logLines.find({
start: {buildId: buildId},
}, this.slot());
},
function(err, lines) {
client.emit('sync', 'data', {lines: lines});
this.pass(true);
},
function(err) {
if (err) {
logger.error(
'error during read log for "' + buildId + '":',
err.stack || err
);
}
callback();
}
);
});
buildDataResourcesHash[buildId] = buildDataResource;
};
exports.createBuildDataResource = createBuildDataResource;
var buildsResource = app.dataio.resource('builds');
distributor.on('buildUpdate', function(build, changes) {
if (build.status === 'queued') {
createBuildDataResource(build.id);
}
// notify about build's project change, coz building affects project
// related stat (last build date, avg build time, etc)
if (changes.completed) {
var projectsResource = app.dataio.resource('projects');
projectsResource.clientEmitSyncChange({name: build.project.name});
}
buildsResource.clientEmitSync('change', {
buildId: build.id, changes: changes
});
});
distributor.on('buildCancel', function(build) {
buildsResource.clientEmitSync('cancel', {buildId: build.id});
});
var buildLogLineNumbersHash = {},
lastLinesHash = {};
distributor.on('buildData', function(build, data) {
var cleanupText = function(text) {
return text.replace('\r', '');
};
var splittedData = data.split('\n'),
logLineNumber = buildLogLineNumbersHash[build.id] || 0;
lastLinesHash[build.id] = lastLinesHash[build.id] || '';
// if we don't have last line, so we start new line
if (!lastLinesHash[build.id]) {
logLineNumber++;
}
lastLinesHash[build.id] += _(splittedData).first();
var lines = [{
text: cleanupText(lastLinesHash[build.id]),
buildId: build.id,
number: logLineNumber
}];
if (splittedData.length > 1) {
// if we have last '' we have to take all except last
// this shown that string ends with eol
if (_(splittedData).last() === '') {
lastLinesHash[build.id] = '';
splittedData = _(splittedData.slice(1)).initial();
} else {
lastLinesHash[build.id] = _(splittedData).last();
splittedData = _(splittedData).tail();
}
lines = lines.concat(_(splittedData).map(function(line) {
return {
text: cleanupText(line),
buildId: build.id,
number: ++logLineNumber
};
}));
}
buildLogLineNumbersHash[build.id] = logLineNumber;
app.dataio.resource('build' + build.id).clientEmitSync(
'data',
{lines: lines}
);
distributor.on('buildLogLines', function(build, lines) {
// write build logs to db
db.logLines.put(lines, function(err) {
if (err) {

50
docs/developing-plugins/builds-collection.md Normal file
View File

@ -0,0 +1,50 @@
## BuildsCollection()
Facade entity which accumulates operations with currently running and
db saved builds.
## BuildsCollection.create(params:Object, [callback(err)]:Function)
Create build by running given project.
- `params.projectName` - project to build
- `params.withScmChangesOnly` - if true then build will be started only if
there is scm changes for project
- `params.queueQueued` - if true then currently queued project can be queued
again
- `params.initiator` - contains information about initiator of the build,
must contain `type` property e.g. when one build triggers another:
initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
## BuildsCollection.cancel(id:Number, [callback(err)]:Function)
Cancel build by id.
Note that only queued build can be canceled currently.
## BuildsCollection.get(id:Number, callback(err,build):Function)
Get build by id.
## BuildsCollection.getLogLines(params:Object, callback(err,logLinesData):Function)
Get log lines for the given build.
- `params.buildId` - target build
- `params.from` - if set then lines from that number will be returned
- `params.to` - if set then lines to that number will be returned
## BuildsCollection.getAvgBuildDuration(builds:Array.<Object>)
Calculate average build duration for the given builds.
## BuildsCollection.getRecent(params:Object, callback(err,builds):Function)
Get builds sorted by date in descending order.
- `params.projectName` - optional project filter
- `params.status` - optional status filter, can be used only when
`params.projectName` is set. When used builds in the result will contain
only following fields: id, number, startDate, endDate
## BuildsCollection.getDoneStreak(params:Object, callback(err,doneStreak):Function)
Get info about current done builds streak.
- `params.projectName` - optional project filter

55
docs/developing-plugins/projects-collection.md Normal file
View File

@ -0,0 +1,55 @@
## ProjectsCollection()
Projects collection contains all currently loaded projects and provides
operations for manipulating with them.
All projects stored on disk in `baseDir` and loaded to memory so
they can be received (by `get`, `getAll` and other methods) in a sync way.
Note that id for the particular project is a `name` of that project.
## ProjectsCollection.validateConfig(config:Object, callback(err,config):Function)
Validate and return given config.
## ProjectsCollection.load(name:String, [callback(err)]:Function)
Load project to collection.
`projectLoaded` event with loaded config as argument will be emitted after
load.
## ProjectsCollection.loadAll([callback(err)]:Function)
Load all projects (from `this.baseDir`).
Calls `load` for every project in a base dir.
## ProjectsCollection.unload(name:String, [callback(err)]:Function)
Unload project from collection
`projectUnloaded` event with unloaded config as argument will be emitted
after unload.
## ProjectsCollection.get(name:String)
Get project config by name.
Returns config object or undefined if project is not found.
## ProjectsCollection.getAll()
Get configs for all currently loaded projects.
Returns array of config objects.
## ProjectsCollection.filter(predicate:Function)
Get project configs which match to predicate.
Returns array of config objects or empty array if there is no matched
project.
## ProjectsCollection.remove(name:String, [callback(err)]:Function)
Remove project by name.
Calls `unload`, removes project from disk and db.
## ProjectsCollection.rename(name:String, [callback(err)]:Function)
Rename project.
Renames project on disk and db, also changes name for loaded project.

26
httpApi.js
View File

@ -2,8 +2,7 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
querystring = require('querystring'),
libProject = require('./lib/project');
querystring = require('querystring');
/*
* Pure rest api on pure nodejs follows below
*/
@ -59,12 +58,12 @@ module.exports = function(app) {
Steppy(
function() {
var projectName = req.body.project,
project = _(app.projects).findWhere({name: projectName});
project = app.projects.get(projectName);
if (project) {
res.statusCode = 204;
logger.log('Run project "%s"', projectName);
app.distributor.run({
app.builds.create({
projectName: projectName,
withScmChangesOnly: req.body.withScmChangesOnly,
queueQueued: req.body.queueQueued,
@ -92,10 +91,7 @@ module.exports = function(app) {
throw new Error('Access token doesn`t match');
}
libProject.remove({
baseDir: app.config.paths.projects,
name: projectName
}, this.slot());
app.projects.remove(projectName, this.slot());
},
function() {
logger.log('Project "%s" cleaned up', projectName);
@ -123,28 +119,22 @@ module.exports = function(app) {
if (!newProjectName) throw new Error('new project name is not set');
var curProject = _(app.projects).findWhere({name: projectName});
var curProject = app.projects.get(projectName);
if (!curProject) {
throw new Error('Project "' + projectName + '" not found');
}
this.pass(curProject);
var newProject = _(app.projects).findWhere({name: newProjectName});
var newProject = app.projects.get(newProjectName);
if (newProject) {
throw new Error(
'Project name "' + newProjectName + '" already used'
);
}
libProject.rename({
baseDir: app.config.paths.projects,
name: projectName,
newName: newProjectName
}, this.slot());
app.projects.rename(projectName, newProjectName, this.slot());
},
function(err, curProject) {
curProject.name = newProjectName;
function(err) {
res.statusCode = 204;
res.end();
},

222
lib/build.js Normal file
View File

@ -0,0 +1,222 @@
'use strict';
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
/**
* Facade entity which accumulates operations with currently running and
* db saved builds.
*/
function BuildsCollection(params) {
this.db = params.db;
this.distributor = params.distributor;
this._proxyDistributorEvent('buildUpdate', 'buildUpdated');
this._proxyDistributorEvent('buildCancel', 'buildCanceled');
this._proxyDistributorEvent('buildLogLines', 'buildLogLines');
}
exports.BuildsCollection = BuildsCollection;
inherits(BuildsCollection, EventEmitter);
BuildsCollection.prototype._proxyDistributorEvent = function(source, dest) {
var self = this;
self.distributor.on(source, function() {
self.emit.apply(self, [dest].concat(_(arguments).toArray()));
});
};
/**
* Create build by running given project.
* - `params.projectName` - project to build
* - `params.withScmChangesOnly` - if true then build will be started only if
* there is scm changes for project
* - `params.queueQueued` - if true then currently queued project can be queued
* again
* - `params.initiator` - contains information about initiator of the build,
* must contain `type` property e.g. when one build triggers another:
* initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
*
* @param {Object} params
* @param {Function} [callback(err)]
*/
BuildsCollection.prototype.create = function(params, callback) {
this.distributor.run(params, callback);
};
/**
* Cancel build by id.
* Note that only queued build can be canceled currently.
*
* @param {Number} id
* @param {Function} [callback(err)]
*/
BuildsCollection.prototype.cancel = function(id, callback) {
this.distributor.cancel(id, callback);
};
/**
* Get build by id.
*
* @param {Number} id
* @param {Function} callback(err,build)
*/
BuildsCollection.prototype.get = function(id, callback) {
this.db.builds.find({start: {id: id}}, function(err, builds) {
callback(err, builds && builds[0]);
});
};
/**
* Get log lines for the given build.
* - `params.buildId` - target build
* - `params.from` - if set then lines from that number will be returned
* - `params.to` - if set then lines to that number will be returned
*
* @param {Object} params
* @param {Function} callback(err,logLinesData)
*/
BuildsCollection.prototype.getLogLines = function(params, callback) {
var self = this;
Steppy(
function() {
var findParams = {
start: {buildId: params.buildId},
end: {buildId: params.buildId}
};
if (params.from) findParams.start.number = params.from;
if (params.to) findParams.end.number = params.to;
var count = params.from && params.to ? params.to - params.from + 1: 0;
self.db.logLines.find(findParams, this.slot());
this.pass(count);
},
function(err, logLines, count) {
this.pass({
lines: logLines,
isLast: count ? logLines.length < count : true
});
},
callback
);
};
BuildsCollection.prototype.getLogLinesTail = function(params, callback) {
var self = this;
Steppy(
function() {
var findParams = {
reverse: true,
start: {buildId: params.buildId},
limit: params.limit
};
self.db.logLines.find(findParams, this.slot());
},
function(err, logLines) {
var lines = logLines.reverse(),
total = logLines.length ? logLines[logLines.length - 1].number : 0;
this.pass({lines: lines, total: total});
},
callback
);
};
/**
* Calculate average build duration for the given builds.
*
* @param {Object[]} builds
*/
BuildsCollection.prototype.getAvgBuildDuration = function(builds) {
var durationsSum = _(builds).reduce(function(sum, build) {
return sum + (build.endDate - build.startDate);
}, 0);
return Math.round(durationsSum / builds.length);
};
/**
* Get builds sorted by date in descending order.
* - `params.projectName` - optional project filter
* - `params.status` - optional status filter, can be used only when
* `params.projectName` is set. When used builds in the result will contain
* only following fields: id, number, startDate, endDate
*
* @param {Object} params
* @param {Function} callback(err,builds)
*/
BuildsCollection.prototype.getRecent = function(params, callback) {
params.limit = params.limit || 20;
var self = this;
Steppy(
function() {
var findParams = {start: {}, limit: params.limit};
// such condition for match one of projections:
// projectName, descCreateDate
// projectName, status, descCreateDate
// or just descCreateDate projection if project name is not set
if (params.projectName) {
findParams.start.projectName = params.projectName;
if (params.status) findParams.start.status = params.status;
}
findParams.start.descCreateDate = '';
self.db.builds.find(findParams, this.slot());
},
callback
);
};
/**
* Get info about current done builds streak.
* - `params.projectName` - optional project filter
*
* @param {Object} params
* @param {Function} callback(err,doneStreak)
*/
BuildsCollection.prototype.getDoneStreak = function(params, callback) {
var self = this;
Steppy(
function() {
var start = {};
if (params.projectName) start.projectName = params.projectName;
start.descCreateDate = '';
// tricky but effective streak counting inside filter goes below
var doneBuildsStreakCallback = _(this.slot()).once(),
doneBuildsStreak = {buildsCount: 0};
self.db.builds.find({
start: start,
filter: function(build) {
// error exits streak
if (build.status === 'error') {
doneBuildsStreakCallback(null, doneBuildsStreak);
return true;
}
if (build.status === 'done') {
doneBuildsStreak.buildsCount++;
}
},
limit: 1
}, function(err) {
doneBuildsStreakCallback(err, doneBuildsStreak);
});
},
callback
);
};

80
lib/distributor.js
View File

@ -27,6 +27,9 @@ function Distributor(params) {
};
self.projects = params.projects;
self.buildLogLineNumbersHash = {},
self.lastLinesHash = {};
}
inherits(Distributor, EventEmitter);
@ -101,7 +104,7 @@ Distributor.prototype._runNext = function(callback) {
});
executor.on('data', function(data) {
self.emit('buildData', build, data);
self._onBuildData(build, data);
});
executor.once('scmData', function(scmData) {
@ -127,6 +130,57 @@ Distributor.prototype._runNext = function(callback) {
);
};
Distributor.prototype._onBuildData = function(build, data) {
var self = this;
self.emit('buildData', build, data);
var cleanupText = function(text) {
return text.replace('\r', '');
};
var splittedData = data.split('\n'),
logLineNumber = self.buildLogLineNumbersHash[build.id] || 0;
self.lastLinesHash[build.id] = self.lastLinesHash[build.id] || '';
// if we don't have last line, so we start new line
if (!self.lastLinesHash[build.id]) {
logLineNumber++;
}
self.lastLinesHash[build.id] += _(splittedData).first();
var lines = [{
text: cleanupText(self.lastLinesHash[build.id]),
buildId: build.id,
number: logLineNumber
}];
if (splittedData.length > 1) {
// if we have last '' we have to take all except last
// this shown that string ends with eol
if (_(splittedData).last() === '') {
self.lastLinesHash[build.id] = '';
splittedData = _(splittedData.slice(1)).initial();
} else {
self.lastLinesHash[build.id] = _(splittedData).last();
splittedData = _(splittedData).tail();
}
lines = lines.concat(_(splittedData).map(function(line) {
return {
text: cleanupText(line),
buildId: build.id,
number: ++logLineNumber
};
}));
}
self.buildLogLineNumbersHash[build.id] = logLineNumber;
self.emit('buildLogLines', build, lines);
};
Distributor.prototype._updateWaitReasons = function() {
var self = this;
_(self.queue).each(function(item) {
@ -232,17 +286,21 @@ Distributor.prototype._updateBuild = function(build, changes, callback) {
);
};
Distributor.prototype.cancel = function(params, callback) {
Distributor.prototype.cancel = function(id, callback) {
callback = callback || function(err) {
if (err) logger.error('Error during cancel: ', err.stack || err);
};
var self = this;
Steppy(
function() {
var queueItemIndex = _(self.queue).findIndex(function(item) {
return item.build.id === params.buildId;
return item.build.id === id;
});
if (queueItemIndex === -1) {
throw new Error(
'Build with id "' + params.buildId + '" not found for cancel'
'Build with id "' + id + '" not found for cancel'
);
}
@ -263,19 +321,15 @@ Distributor.prototype.cancel = function(params, callback) {
};
Distributor.prototype.run = function(params, callback) {
callback = callback || function(err) {
if (err) logger.error('Error during run: ', err.stack || err);
};
var self = this,
project;
callback = callback || function(err) {
if (err) {
logger.error('Error during run: ', err.stack || err);
}
};
Steppy(
function() {
project = _(self.projects).chain()
.findWhere({name: params.projectName})
.clone()
.value();
project = _(self.projects.get(params.projectName)).clone();
if (params.withScmChangesOnly) {
self.nodes[0].hasScmChanges(project, this.slot());

265
lib/project.js
View File

@ -4,17 +4,37 @@ var Steppy = require('twostep').Steppy,
fs = require('fs'),
path = require('path'),
_ = require('underscore'),
reader = require('./reader'),
db = require('../db'),
utils = require('./utils'),
SpawnCommand = require('./command/spawn').Command,
validateParams = require('./validateParams');
validateParams = require('./validateParams'),
EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
/**
* Validates and returns given `config` to the `callback`(err, config)
* Projects collection contains all currently loaded projects and provides
* operations for manipulating with them.
* All projects stored on disk in `baseDir` and loaded to memory so
* they can be received (by `get`, `getAll` and other methods) in a sync way.
* Note that id for the particular project is a `name` of that project.
*/
exports.validateConfig = function(config, callback) {
function ProjectsCollection(params) {
this.db = params.db;
this.reader = params.reader;
this.baseDir = params.baseDir;
this.configs = [];
}
exports.ProjectsCollection = ProjectsCollection;
inherits(ProjectsCollection, EventEmitter);
/**
* Validate and return given config.
*
* @param {Object} config
* @param {Function} callback(err,config)
*/
ProjectsCollection.prototype.validateConfig = function(config, callback) {
Steppy(
function() {
validateParams(config, {
@ -60,50 +80,16 @@ exports.validateConfig = function(config, callback) {
);
};
/**
* Loads and returns project
*/
exports.load = function(baseDir, name, callback) {
var dir = path.join(baseDir, name);
Steppy(
function() {
fs.readdir(dir, this.slot());
},
function(err, dirContent) {
exports.loadConfig(dir, this.slot());
},
function(err, config) {
config.name = name;
config.dir = dir;
exports.validateConfig(config, this.slot());
},
callback
);
ProjectsCollection.prototype._getProjectPath = function(name) {
return path.join(this.baseDir, name);
};
/**
* Loads all projects from `baseDir` and returns array of projects
*/
exports.loadAll = function(baseDir, callback) {
Steppy(
function() {
fs.readdir(baseDir, this.slot());
},
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
exports.load(baseDir, dir, loadGroup.slot());
});
},
callback
);
};
ProjectsCollection.prototype._loadConfig = function(dir, callback) {
var self = this;
exports.loadConfig = function(dir, callback) {
Steppy(
function() {
reader.load(dir, 'config', this.slot());
self.reader.load(dir, 'config', this.slot());
},
function(err, config) {
// convert steps object to array
@ -120,7 +106,7 @@ exports.loadConfig = function(dir, callback) {
});
}
// apply defaults to not yet validated config
// apply defaults
_(config.steps).each(function(step) {
if (!step.type) step.type = 'shell';
if (!step.name && step.cmd) step.name = utils.prune(step.cmd, 40);
@ -132,73 +118,158 @@ exports.loadConfig = function(dir, callback) {
);
};
exports.saveConfig = function(config, dir, callback) {
fs.writeFile(
path.join(dir, 'config.json'),
JSON.stringify(config, null, 4),
callback
);
};
/**
* Load project to collection.
* `projectLoaded` event with loaded config as argument will be emitted after
* load.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.load = function(name, callback) {
callback = callback || _.noop;
var self = this,
dir = self._getProjectPath(name);
exports.create = function(baseDir, config, callback) {
var dir;
Steppy(
function() {
dir = path.join(baseDir, config.name);
fs.mkdir(dir, this.slot());
if (self.get(name)) {
throw new Error(
'Can`t load already loaded project "' + name + '"'
);
}
self._loadConfig(dir, this.slot());
},
function(err) {
exports.saveConfig(config, baseDir, this.slot());
function(err, config) {
config.name = name;
config.dir = dir;
self.validateConfig(config, this.slot());
},
function(err) {
exports.load(dir, this.slot());
function(err, config) {
self.configs.push(config);
self.emit('projectLoaded', config);
this.pass(null);
},
callback
);
};
exports.getAvgProjectBuildDuration = function(projectName, callback) {
/**
* Load all projects (from `this.baseDir`).
* Calls `load` for every project in a base dir.
*
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.loadAll = function(callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
// get last done builds to calc avg build time
db.builds.find({
start: {
projectName: projectName,
status: 'done',
descCreateDate: ''
},
limit: 10
}, this.slot());
fs.readdir(self.baseDir, this.slot());
},
function(err, doneBuilds) {
var durationsSum = _(doneBuilds).reduce(function(memo, build) {
return memo + (build.endDate - build.startDate);
}, 0);
this.pass(Math.round(durationsSum / doneBuilds.length));
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
self.load(dir, loadGroup.slot());
});
},
callback
);
};
exports.remove = function(params, callback) {
/**
* Unload project from collection
* `projectUnloaded` event with unloaded config as argument will be emitted
* after unload.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.unload = function(name, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
db.builds.find({
start: {projectName: params.name, descCreateDate: ''}
var index = _(self.configs).findIndex(function(config) {
return config.name === name;
});
if (index === -1) {
throw new Error('Can`t unload not loaded project: "' + name + '"');
}
var unloadedConfig = self.configs.splice(index, 1)[0];
self.emit('projectUnloaded', unloadedConfig);
this.pass(null);
},
callback
);
};
/**
* Get project config by name.
* Returns config object or undefined if project is not found.
*
* @param {String} name
*/
ProjectsCollection.prototype.get = function(name) {
return _(this.configs).findWhere({name: name});
};
/**
* Get configs for all currently loaded projects.
* Returns array of config objects.
*/
ProjectsCollection.prototype.getAll = function() {
return this.configs;
};
/**
* Get project configs which match to predicate.
* Returns array of config objects or empty array if there is no matched
* project.
*
* @param {Function} predicate
*/
ProjectsCollection.prototype.filter = function(predicate) {
return _(this.configs).filter(predicate);
};
/**
* Remove project by name.
* Calls `unload`, removes project from disk and db.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.remove = function(name, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
self.db.builds.find({
start: {projectName: name, descCreateDate: ''}
}, this.slot());
new SpawnCommand().run({cmd: 'rm', args: [
'-Rf', path.join(params.baseDir, params.name)
'-Rf', self._getProjectPath(name)
]}, this.slot());
self.unload(name, this.slot());
},
function(err, builds) {
if (builds.length) {
db.builds.del(builds, this.slot());
self.db.builds.del(builds, this.slot());
var logLinesRemoveGroup = this.makeGroup();
_(builds).each(function(build) {
db.logLines.remove({
self.db.logLines.remove({
start: {buildId: build.id}
}, logLinesRemoveGroup.slot());
});
@ -210,24 +281,40 @@ exports.remove = function(params, callback) {
);
};
exports.rename = function(params, callback) {
/**
* Rename project.
* Renames project on disk and db, also changes name for loaded project.
*
* @param {String} name
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.rename = function(name, newName, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
fs.rename(
path.join(params.baseDir, params.name),
path.join(params.baseDir, params.newName),
self._getProjectPath(name),
self._getProjectPath(newName),
this.slot()
);
db.builds.multiUpdate(
{start: {projectName: params.name, descCreateDate: ''}},
self.db.builds.multiUpdate(
{start: {projectName: name, descCreateDate: ''}},
function(build) {
build.project.name = params.newName;
build.project.name = newName;
return build;
},
this.slot()
);
},
function() {
// just update currently loaded project name by link
self.get(name).name = newName;
this.pass(null);
},
callback
);
};

4
package.json
View File

@ -10,6 +10,9 @@
"test": "npm run makeTestRepos && mocha --bail --reporter=spec --timeout 10000",
"dev": "gulp",
"sync": "npm install && npm prune && bower install && bower prune",
"docProjectsCollection": "dox --api --skipSingleStar < lib/project.js | sed '/^ - \\[ProjectsCollection/ d' > docs/developing-plugins/projects-collection.md",
"docBuildsCollection": "dox --api --skipSingleStar < lib/build.js | sed '/^ - \\[BuildsCollection/ d' > docs/developing-plugins/builds-collection.md",
"doc": "nrun docProjectsCollection && nrun docBuildsCollection",
"buildJs": "r.js -o static/js/requirejs/buid.js",
"buildClean": "rm static/index.html",
"buildHtml": "jade views/index.jade --obj '{\"env\": \"production\"}' -o static/",
@ -56,6 +59,7 @@
},
"devDependencies": {
"bower": "1.4.1",
"dox": "0.8.0",
"expect.js": "0.3.1",
"gulp": "3.8.11",
"gulp-less": "3.0.3",

30
projectsWatcher.js
View File

@ -2,45 +2,37 @@
var _ = require('underscore'),
path = require('path'),
chokidar = require('chokidar'),
project = require('./lib/project'),
logger = require('./lib/logger')('projects watcher');
chokidar = require('chokidar');
exports.init = function(app, callback) {
var logger = app.lib.logger('projects watcher');
// start file watcher for reloading projects on change
var syncProject = function(filename, fileInfo) {
var baseDir = app.config.paths.projects,
projectName = path.relative(
baseDir,
path.dirname(filename)
);
var projectName = path.relative(
app.config.paths.projects,
path.dirname(filename)
);
var projectIndex = _(app.projects).findIndex(function(project) {
return project.name === projectName;
});
if (projectIndex !== -1) {
if (app.projects.get(projectName)) {
logger.log('Unload project: "' + projectName + '"');
var unloadedProject = app.projects.splice(projectIndex, 1)[0];
app.emit('projectUnloaded', unloadedProject);
app.projects.unload(projectName);
}
// on add or change (info is falsy on unlink)
if (fileInfo) {
logger.log('Load project "' + projectName + '" on change');
project.load(baseDir, projectName, function(err, project) {
app.projects.load(projectName, function(err) {
if (err) {
return logger.error(
'Error during load project "' + projectName + '": ',
err.stack || err
);
}
app.projects.push(project);
logger.log(
'Project "' + projectName + '" loaded:',
JSON.stringify(project, null, 4)
JSON.stringify(app.projects.get(projectName), null, 4)
);
app.emit('projectLoaded', project);
});
}
};

65
resources/builds.js
View File

@ -2,31 +2,22 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
db = require('../db'),
utils = require('../lib/utils'),
logger = require('../lib/logger')('builds resource');
module.exports = function(app) {
var resource = app.dataio.resource('builds'),
distributor = app.distributor;
var resource = app.dataio.resource('builds');
resource.use('readAll', function(req, res, next) {
Steppy(
function() {
var data = req.data || {};
var data = req.data || {},
getParams = {limit: data.limit || 20};
var start = {};
if (data.projectName) {
start.projectName = data.projectName;
getParams.projectName = data.projectName;
}
start.descCreateDate = data.descCreateDate || '';
var findParams = _(data).pick('offset', 'limit');
findParams.start = start;
findParams.limit = findParams.limit || 20;
db.builds.find(findParams, this.slot());
app.builds.getRecent(getParams, this.slot());
},
function(err, builds) {
// omit big fields not needed for list
@ -49,12 +40,10 @@ module.exports = function(app) {
resource.use('read', function(req, res, next) {
Steppy(
function() {
var findParams = {};
findParams.start = _(req.data).pick('id');
db.builds.find(findParams, this.slot());
app.builds.get(req.data.id, this.slot());
},
function(err, build) {
res.send(build[0]);
res.send(build);
},
next
);
@ -63,19 +52,13 @@ module.exports = function(app) {
resource.use('getBuildLogTail', function(req, res, next) {
Steppy(
function() {
var findParams = {
reverse: true,
start: {buildId: req.data.buildId},
app.builds.getLogLinesTail({
buildId: req.data.buildId,
limit: req.data.length
};
db.logLines.find(findParams, this.slot());
}, this.slot());
},
function(err, logLines) {
var lines = logLines.reverse(),
total = logLines.length ? logLines[0].number : 0;
res.send({lines: lines, total: total});
function(err, tail) {
res.send(tail);
},
next
);
@ -84,23 +67,13 @@ module.exports = function(app) {
resource.use('getBuildLogLines', function(req, res, next) {
Steppy(
function() {
var buildId = req.data.buildId,
from = req.data.from,
to = req.data.to,
count = to - from;
db.logLines.find({
start: {buildId: buildId, number: from},
end: {buildId: buildId, number: to}
}, this.slot());
this.pass(count);
app.builds.getLogLines(
_(req.data).pick('buildId', 'from', 'to'),
this.slot()
);
},
function(err, logLines, count) {
res.send({
lines: logLines,
isLast: logLines.length < count
});
function(err, logLinesData) {
res.send(logLinesData);
},
next
);
@ -111,7 +84,7 @@ module.exports = function(app) {
function() {
var buildId = req.data.buildId;
logger.log('Cancel build: "%s"', buildId);
distributor.cancel({buildId: buildId}, this.slot());
app.builds.cancel(buildId, this.slot());
},
function() {
res.send();

37
resources/helpers.js Normal file
View File

@ -0,0 +1,37 @@
'use strict';
var Steppy = require('twostep').Steppy,
logger = require('../lib/logger')('create build resource');
var buildDataResourcesHash = {};
// create resource for build data
exports.createBuildDataResource = function(app, buildId) {
if (buildId in buildDataResourcesHash) {
return;
}
var buildDataResource = app.dataio.resource('build' + buildId);
buildDataResource.on('connection', function(client) {
var callback = this.async();
Steppy(
function() {
app.builds.getLogLines({buildId: buildId}, this.slot());
},
function(err, logLinesData) {
client.emit('sync', 'data', {lines: logLinesData.lines});
this.pass(null);
},
function(err) {
if (err) {
logger.error(
'error during read log for "' + buildId + '":',
err.stack || err
);
}
callback();
}
);
});
buildDataResourcesHash[buildId] = buildDataResource;
};

33
resources/index.js
View File

@ -1,11 +1,42 @@
'use strict';
var _ = require('underscore'),
errorHandler = require('./errorHandler');
errorHandler = require('./errorHandler'),
helpers = require('./helpers');
module.exports = function(app) {
_(['builds', 'projects']).each(function(resource) {
var resource = require('./' + resource)(app);
resource.use(errorHandler);
});
var buildsResource = app.dataio.resource('builds');
app.builds.on('buildUpdated', function(build, changes) {
if (build.status === 'queued') {
helpers.createBuildDataResource(app, build.id);
}
// notify about build's project change, coz building affects project
// related stat (last build date, avg build time, etc)
if (changes.completed) {
var projectsResource = app.dataio.resource('projects');
projectsResource.clientEmitSyncChange(build.project.name);
}
buildsResource.clientEmitSync('change', {
buildId: build.id, changes: changes
});
});
app.builds.on('buildCanceled', function(build) {
buildsResource.clientEmitSync('cancel', {buildId: build.id});
});
app.builds.on('buildLogLines', function(build, lines) {
app.dataio.resource('build' + build.id).clientEmitSync(
'data',
{lines: lines}
);
});
};

78
resources/projects.js
View File

@ -2,80 +2,50 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
getAvgProjectBuildDuration =
require('../lib/project').getAvgProjectBuildDuration,
createBuildDataResource = require('../distributor').createBuildDataResource,
logger = require('../lib/logger')('projects resource'),
db = require('../db');
helpers = require('./helpers'),
logger = require('../lib/logger')('projects resource');
module.exports = function(app) {
var resource = app.dataio.resource('projects'),
distributor = app.distributor;
var resource = app.dataio.resource('projects');
resource.use('createBuildDataResource', function(req, res) {
createBuildDataResource(req.data.buildId);
helpers.createBuildDataResource(app, req.data.buildId);
res.send();
});
resource.use('readAll', function(req, res) {
var filteredProjects = app.projects,
var filteredProjects = app.projects.getAll(),
nameQuery = req.data && req.data.nameQuery;
if (nameQuery) {
filteredProjects = _(filteredProjects).filter(function(project) {
filteredProjects = app.projects.filter(function(project) {
return project.name.indexOf(nameQuery) !== -1;
});
}
filteredProjects = _(filteredProjects).sortBy('name');
res.send(filteredProjects);
});
var getProject = function(params, callback) {
// get project with additional fields
var getProject = function(name, callback) {
var project;
Steppy(
function() {
project = _(app.projects).findWhere(params.condition);
project = _(app.projects.get(name)).clone();
getAvgProjectBuildDuration(project.name, this.slot());
// get last done build
db.builds.find({
start: {
projectName: project.name,
status: 'done',
descCreateDate: ''
},
limit: 1
app.builds.getRecent({
projectName: project.name,
status: 'done',
limit: 10
}, this.slot());
// tricky but effective streak counting inside filter goes below
var doneBuildsStreakCallback = _(this.slot()).once(),
doneBuildsStreak = 0;
db.builds.find({
start: {
projectName: project.name,
descCreateDate: ''
},
filter: function(build) {
// error exits streak
if (build.status === 'error') {
doneBuildsStreakCallback(null, doneBuildsStreak);
return true;
}
if (build.status === 'done') {
doneBuildsStreak++;
}
},
limit: 1
}, function(err) {
doneBuildsStreakCallback(err, doneBuildsStreak);
});
app.builds.getDoneStreak({projectName: project.name}, this.slot());
},
function(err, avgProjectBuildDuration, lastDoneBuilds, doneBuildsStreak) {
project.lastDoneBuild = lastDoneBuilds[0];
project.avgBuildDuration = avgProjectBuildDuration;
function(err, doneBuilds, doneBuildsStreak) {
project.avgBuildDuration = app.builds.getAvgBuildDuration(doneBuilds);
project.lastDoneBuild = doneBuilds[0];
project.doneBuildsStreak = doneBuildsStreak;
this.pass(project);
@ -84,12 +54,12 @@ module.exports = function(app) {
);
};
// resource custom method which finds project by condition
// resource custom method which finds project by name
// and emits event about it change to clients
resource.clientEmitSyncChange = function(condition) {
resource.clientEmitSyncChange = function(name) {
Steppy(
function() {
getProject({condition: condition}, this.slot());
getProject(name, this.slot());
},
function(err, project) {
resource.clientEmitSync('change', {project: project});
@ -106,7 +76,7 @@ module.exports = function(app) {
resource.use('read', function(req, res) {
Steppy(
function() {
getProject({condition: req.data}, this.slot());
getProject(req.data.name, this.slot());
},
function(err, project) {
res.send(project);
@ -117,7 +87,7 @@ module.exports = function(app) {
resource.use('run', function(req, res) {
var projectName = req.data.projectName;
logger.log('Run the project: "%s"', projectName);
distributor.run({
app.builds.create({
projectName: projectName,
initiator: {type: 'user'},
queueQueued: true

15
scheduler.js
View File

@ -1,24 +1,27 @@
'use strict';
var _ = require('underscore'),
logger = require('./lib/logger')('scheduler'),
CronJob = require('cron').CronJob;
exports.init = function(app, callback) {
var distributor = app.distributor,
var logger = app.lib.logger('scheduler'),
projectJobs = {};
app.on('projectLoaded', function(project) {
app.projects.on('projectLoaded', function(project) {
var time = project.buildEvery && project.buildEvery.time;
if (time) {
logger.log('Start job for loaded project "%s"', project.name);
logger.log(
'Start job for loaded project "%s" by schedule "%s"',
project.name,
time
);
projectJobs[project.name] = {};
projectJobs[project.name].job = new CronJob({
cronTime: time,
onTick: function() {
logger.log('Run project "%s"', project.name);
distributor.run({
app.builds.create({
projectName: project.name,
withScmChangesOnly: project.buildEvery.withScmChangesOnly,
initiator: {type: 'scheduler'}
@ -29,7 +32,7 @@ exports.init = function(app, callback) {
}
});
app.on('projectUnloaded', function(project) {
app.projects.on('projectUnloaded', function(project) {
if (project.name in projectJobs) {
logger.log('Stop job for unloaded project "%s"', project.name);
projectJobs[project.name].job.stop();

2
static/js/app/components/projects/view/index.jade
View File

@ -32,7 +32,7 @@ div
p Current successfully streak:
if lastDoneBuild
span= this.state.project.doneBuildsStreak
span= this.state.project.doneBuildsStreak.buildsCount
else
| -

4
static/js/app/components/terminal/terminal.js
View File

@ -46,7 +46,9 @@ define([
var preloader = document.getElementsByClassName(
'terminal_preloader'
)[0];
preloader.parentNode.removeChild(preloader);
if (preloader) {
preloader.parentNode.removeChild(preloader);
}
},
componentWillUnmount: function() {
window.onscroll = null;

17
test/distributor/blocking.js
View File

@ -4,6 +4,7 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock,
Steppy = require('twostep').Steppy;
@ -119,11 +120,11 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run 2 non-blocking projects in parallel', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2'
}];
}]);
});
itRunParallelProjects();
@ -131,12 +132,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 2 blocked by 1', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2',
blockedBy: ['project1']
}];
}]);
});
itRunSequentialProjects();
@ -144,12 +145,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 1 blocks 2', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project2']
}, {
name: 'project2'
}];
}]);
});
itRunSequentialProjects();
@ -159,7 +160,7 @@ describe('Distributor blocking with max 2 executors count', function() {
'should run 1, 2 in parallel, when 1 block 3, 2 blocked by 3',
function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project3']
}, {
@ -167,7 +168,7 @@ describe('Distributor blocking with max 2 executors count', function() {
blockedBy: ['project3']
}, {
name: 'project3'
}];
}]);
});
itRunParallelProjects();

8
test/distributor/helpers.js
View File

@ -1,7 +1,8 @@
'use strict';
var Node = require('../../lib/node').Node,
EventEmitter = require('events').EventEmitter;
EventEmitter = require('events').EventEmitter,
ProjectsCollection = require('../../lib/project').ProjectsCollection;
exports.createNodeMock = function(executorRun) {
@ -17,3 +18,8 @@ exports.createNodeMock = function(executorRun) {
};
};
exports.createProjectsMock = function(configs) {
var projects = new ProjectsCollection({});
projects.configs = configs;
return projects;
};

9
test/distributor/main.js
View File

@ -3,12 +3,13 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor main', function() {
var distributor,
projects = [{name: 'project1'}];
projects = createProjectsMock([{name: 'project1'}]);
var expectUpdateBuild = function(distributor, build, number, conditionsHash) {
var conditions = conditionsHash[number];
@ -154,7 +155,7 @@ describe('Distributor main', function() {
var originalRunNext = distributor._runNext;
distributor._runNext = function() {
distributor.cancel({buildId: 1}, function(err) {
distributor.cancel(1, function(err) {
cancelError = err;
});
originalRunNext.apply(distributor, arguments);
@ -197,7 +198,7 @@ describe('Distributor main', function() {
var originalRunNext = distributor._runNext;
distributor._runNext = function() {
distributor.cancel({buildId: 2}, function(err) {
distributor.cancel(2, function(err) {
cancelError = err;
});
originalRunNext.apply(distributor, arguments);

6
test/distributor/runSelfAfterCatchRev.js
View File

@ -2,9 +2,9 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
helpers = require('../helpers'),
createProjectsMock = require('./helpers').createProjectsMock,
path = require('path');
describe('Distributor run self after catch', function() {
var distributor, executorRunSpy, scmDataSpy;
@ -17,7 +17,7 @@ describe('Distributor run self after catch', function() {
helpers.removeDirIfExists(workspacePath, done);
distributor = new Distributor({
projects: [{
projects: createProjectsMock([{
name: 'project1',
dir: __dirname,
scm: helpers.repository.scm,
@ -25,7 +25,7 @@ describe('Distributor run self after catch', function() {
{type: 'shell', cmd: 'echo 1'}
],
catchRev: {comment: /.*/}
}],
}]),
nodes: nodes
});

39
test/distributor/triggerAfter.js
View File

@ -3,7 +3,8 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor trigger after', function() {
@ -13,14 +14,14 @@ describe('Distributor trigger after', function() {
describe('done when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{status: 'done', project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -39,11 +40,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -77,7 +82,9 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run totally 1 time', function() {
@ -91,14 +98,14 @@ describe('Distributor trigger after', function() {
describe('status is not set when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -117,11 +124,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -155,11 +166,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {