introduce projects collection

This commit is contained in:
oleg 2016-01-05 14:18:20 +03:00
parent e138922db6
commit 7dd920ce28
13 changed files with 241 additions and 171 deletions

27
app.js
View File

@ -10,7 +10,7 @@ var env = process.env.NODE_ENV || 'development',
_ = require('underscore'),
reader = require('./lib/reader'),
notifier = require('./lib/notifier'),
project = require('./lib/project'),
ProjectsCollection = require('./lib/project').ProjectsCollection,
libLogger = require('./lib/logger'),
EventEmitter = require('events').EventEmitter,
validateConfig = require('./lib/validateConfig');
@ -205,16 +205,15 @@ Steppy(
db.init(app.config.paths.db, {db: dbBackend}, this.slot());
},
function() {
// load all projects for the first time
project.loadAll(app.config.paths.projects, this.slot());
app.projects = new ProjectsCollection({
db: db,
reader: reader,
baseDir: app.config.paths.projects
});
completeUncompletedBuilds(this.slot());
},
function(err, projects) {
// note that `app.projects` is live variable
app.projects = projects;
logger.log('Loaded projects: ', _(app.projects).pluck('name'));
function(err) {
require('./distributor').init(app, this.slot());
},
function(err, distributor) {
@ -235,15 +234,17 @@ Steppy(
require('./scheduler').init(app, this.slot());
// notify about first project loading
_(app.projects).each(function(project) {
app.emit('projectLoaded', project);
});
// init resources
require('./resources')(app);
},
function() {
// load projects after all plugins to provide ability for plugins to
// handle `projectLoaded` event
app.projects.loadAll(this.slot());
},
function(err) {
logger.log('Loaded projects: ', app.projects.pluck('name'));
var host = app.config.http.host,
port = app.config.http.port;
logger.log('Start http server on %s:%s', host, port);

View File

@ -3,9 +3,6 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
Distributor = require('./lib/distributor').Distributor,
getAvgProjectBuildDuration = (
require('./lib/project').getAvgProjectBuildDuration
),
db = require('./db'),
logger = require('./lib/logger')('distributor');
@ -20,7 +17,7 @@ exports.init = function(app, callback) {
if (_(build.project).has('avgBuildDuration')) {
this.pass(build.project.avgBuildDuration);
} else {
getAvgProjectBuildDuration(build.project.name, this.slot());
app.projects.getAvgBuildDuration(build.project.name, this.slot());
}
},
function(err, avgBuildDuration) {

View File

@ -2,8 +2,7 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
querystring = require('querystring'),
libProject = require('./lib/project');
querystring = require('querystring');
/*
* Pure rest api on pure nodejs follows below
*/
@ -59,7 +58,7 @@ module.exports = function(app) {
Steppy(
function() {
var projectName = req.body.project,
project = _(app.projects).findWhere({name: projectName});
project = app.projects.get(projectName);
if (project) {
res.statusCode = 204;
@ -92,10 +91,7 @@ module.exports = function(app) {
throw new Error('Access token doesn`t match');
}
libProject.remove({
baseDir: app.config.paths.projects,
name: projectName
}, this.slot());
app.projects.remove(projectName, this.slot());
},
function() {
logger.log('Project "%s" cleaned up', projectName);
@ -123,28 +119,22 @@ module.exports = function(app) {
if (!newProjectName) throw new Error('new project name is not set');
var curProject = _(app.projects).findWhere({name: projectName});
var curProject = app.projects.get(projectName);
if (!curProject) {
throw new Error('Project "' + projectName + '" not found');
}
this.pass(curProject);
var newProject = _(app.projects).findWhere({name: newProjectName});
var newProject = app.projects.get(newProjectName);
if (newProject) {
throw new Error(
'Project name "' + newProjectName + '" already used'
);
}
libProject.rename({
baseDir: app.config.paths.projects,
name: projectName,
newName: newProjectName
}, this.slot());
app.projects.rename(projectName, newProjectName, this.slot());
},
function(err, curProject) {
curProject.name = newProjectName;
function(err) {
res.statusCode = 204;
res.end();
},

View File

@ -272,10 +272,7 @@ Distributor.prototype.run = function(params, callback) {
};
Steppy(
function() {
project = _(self.projects).chain()
.findWhere({name: params.projectName})
.clone()
.value();
project = _(self.projects.get(params.projectName)).clone();
if (params.withScmChangesOnly) {
self.nodes[0].hasScmChanges(project, this.slot());

View File

@ -4,17 +4,33 @@ var Steppy = require('twostep').Steppy,
fs = require('fs'),
path = require('path'),
_ = require('underscore'),
reader = require('./reader'),
db = require('../db'),
utils = require('./utils'),
SpawnCommand = require('./command/spawn').Command,
validateParams = require('./validateParams');
validateParams = require('./validateParams'),
EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
/*
* Projects collection it's something similar to backbone collection.
* But contrasting to backbone there is no model of a single project, when you
* receive project from collection you just get a json.
* General id for the particular project is a `name` of that project.
*/
function ProjectsCollection(params) {
this.db = params.db;
this.reader = params.reader;
this.baseDir = params.baseDir;
this.configs = [];
}
exports.ProjectsCollection = ProjectsCollection;
inherits(ProjectsCollection, EventEmitter);
/**
* Validates and returns given `config` to the `callback`(err, config)
*/
exports.validateConfig = function(config, callback) {
ProjectsCollection.prototype.validateConfig = function(config, callback) {
Steppy(
function() {
validateParams(config, {
@ -60,50 +76,16 @@ exports.validateConfig = function(config, callback) {
);
};
/**
* Loads and returns project
*/
exports.load = function(baseDir, name, callback) {
var dir = path.join(baseDir, name);
ProjectsCollection.prototype._getProjectPath = function(name) {
return path.join(this.baseDir, name);
}
ProjectsCollection.prototype._loadConfig = function(dir, callback) {
var self = this;
Steppy(
function() {
fs.readdir(dir, this.slot());
},
function(err, dirContent) {
exports.loadConfig(dir, this.slot());
},
function(err, config) {
config.name = name;
config.dir = dir;
exports.validateConfig(config, this.slot());
},
callback
);
};
/**
* Loads all projects from `baseDir` and returns array of projects
*/
exports.loadAll = function(baseDir, callback) {
Steppy(
function() {
fs.readdir(baseDir, this.slot());
},
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
exports.load(baseDir, dir, loadGroup.slot());
});
},
callback
);
};
exports.loadConfig = function(dir, callback) {
Steppy(
function() {
reader.load(dir, 'config', this.slot());
self.reader.load(dir, 'config', this.slot());
},
function(err, config) {
// convert steps object to array
@ -132,38 +114,111 @@ exports.loadConfig = function(dir, callback) {
);
};
exports.saveConfig = function(config, dir, callback) {
fs.writeFile(
path.join(dir, 'config.json'),
JSON.stringify(config, null, 4),
callback
);
};
/**
* Loads project to collection
*/
ProjectsCollection.prototype.load = function(name, callback) {
var self = this,
dir = self._getProjectPath(name);
exports.create = function(baseDir, config, callback) {
var dir;
Steppy(
function() {
dir = path.join(baseDir, config.name);
fs.mkdir(dir, this.slot());
self._loadConfig(dir, this.slot());
},
function(err) {
exports.saveConfig(config, baseDir, this.slot());
function(err, config) {
config.name = name;
config.dir = dir;
self.validateConfig(config, this.slot());
},
function(err) {
exports.load(dir, this.slot());
function(err, config) {
self.configs.push(config);
self.emit('projectLoaded', config);
this.pass(null);
},
callback
);
};
exports.getAvgProjectBuildDuration = function(projectName, callback) {
ProjectsCollection.prototype.unload = function(name, callback) {
callback = callback || _.noop;
var self = this;
Steppy(
function() {
var index = _(self.configs).findIndex(function(config) {
return config.name === name;
});
if (index === -1) {
throw new Error('Can`t unload not loaded project: "' + name + '"');
}
var unloadedConfig = self.configs.splice(index, 1)[0];
self.emit('projectUnloaded', unloadedConfig);
this.pass(null);
},
callback
);
};
ProjectsCollection.prototype.get = function(name) {
return _(this.configs).findWhere({name: name});
};
ProjectsCollection.prototype.getAll = function(name) {
return this.configs;
};
ProjectsCollection.prototype.findWhere = function(params) {
return _(this.configs).findWhere(params);
};
ProjectsCollection.prototype.where = function(params) {
return _(this.configs).where(params);
};
ProjectsCollection.prototype.filter = function(iterator) {
return _(this.configs).filter(iterator);
};
ProjectsCollection.prototype.pluck = function(attribute) {
return _(this.configs).pluck(attribute);
};
/**
* Loads all projects (from `this.baseDir`)
*/
ProjectsCollection.prototype.loadAll = function(callback) {
var self = this;
Steppy(
function() {
fs.readdir(self.baseDir, this.slot());
},
function(err, dirs) {
var loadGroup = this.makeGroup();
_(dirs).each(function(dir) {
self.load(dir, loadGroup.slot());
});
},
callback
);
};
/*
* Calculates average build duration for the given project
*/
ProjectsCollection.prototype.getAvgBuildDuration = function(name, callback) {
var self = this;
Steppy(
function() {
// get last done builds to calc avg build time
db.builds.find({
self.db.builds.find({
start: {
projectName: projectName,
projectName: name,
status: 'done',
descCreateDate: ''
},
@ -181,24 +236,28 @@ exports.getAvgProjectBuildDuration = function(projectName, callback) {
);
};
exports.remove = function(params, callback) {
ProjectsCollection.prototype.remove = function(name, callback) {
var self = this;
Steppy(
function() {
db.builds.find({
start: {projectName: params.name, descCreateDate: ''}
self.db.builds.find({
start: {projectName: name, descCreateDate: ''}
}, this.slot());
new SpawnCommand().run({cmd: 'rm', args: [
'-Rf', path.join(params.baseDir, params.name)
'-Rf', self._getProjectPath(name)
]}, this.slot());
self.unload(name, this.slot());
},
function(err, builds) {
if (builds.length) {
db.builds.del(builds, this.slot());
self.db.builds.del(builds, this.slot());
var logLinesRemoveGroup = this.makeGroup();
_(builds).each(function(build) {
db.logLines.remove({
self.db.logLines.remove({
start: {buildId: build.id}
}, logLinesRemoveGroup.slot());
});
@ -210,24 +269,32 @@ exports.remove = function(params, callback) {
);
};
exports.rename = function(params, callback) {
ProjectsCollection.prototype.rename = function(name, newName, callback) {
var self = this;
Steppy(
function() {
fs.rename(
path.join(params.baseDir, params.name),
path.join(params.baseDir, params.newName),
self._getProjectPath(name),
self._getProjectPath(newName),
this.slot()
);
db.builds.multiUpdate(
{start: {projectName: params.name, descCreateDate: ''}},
self.db.builds.multiUpdate(
{start: {projectName: name, descCreateDate: ''}},
function(build) {
build.project.name = params.newName;
build.project.name = newName;
return build;
},
this.slot()
);
},
function() {
// just update currently loaded project name by link
self.get(name).name = newName;
this.pass(null);
},
callback
);
};

View File

@ -2,45 +2,37 @@
var _ = require('underscore'),
path = require('path'),
chokidar = require('chokidar'),
project = require('./lib/project'),
logger = require('./lib/logger')('projects watcher');
chokidar = require('chokidar');
exports.init = function(app, callback) {
var logger = app.lib.logger('projects watcher');
// start file watcher for reloading projects on change
var syncProject = function(filename, fileInfo) {
var baseDir = app.config.paths.projects,
projectName = path.relative(
baseDir,
path.dirname(filename)
);
var projectName = path.relative(
app.config.paths.projects,
path.dirname(filename)
);
var projectIndex = _(app.projects).findIndex(function(project) {
return project.name === projectName;
});
if (projectIndex !== -1) {
if (app.projects.get(projectName)) {
logger.log('Unload project: "' + projectName + '"');
var unloadedProject = app.projects.splice(projectIndex, 1)[0];
app.emit('projectUnloaded', unloadedProject);
app.projects.unload(projectName);
}
// on add or change (info is falsy on unlink)
if (fileInfo) {
logger.log('Load project "' + projectName + '" on change');
project.load(baseDir, projectName, function(err, project) {
app.projects.load(projectName, function(err) {
if (err) {
return logger.error(
'Error during load project "' + projectName + '": ',
err.stack || err
);
}
app.projects.push(project);
logger.log(
'Project "' + projectName + '" loaded:',
JSON.stringify(project, null, 4)
JSON.stringify(app.projects.get(projectName), null, 4)
);
app.emit('projectLoaded', project);
});
}
};

View File

@ -2,8 +2,6 @@
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
getAvgProjectBuildDuration =
require('../lib/project').getAvgProjectBuildDuration,
createBuildDataResource = require('../distributor').createBuildDataResource,
logger = require('../lib/logger')('projects resource'),
db = require('../db');
@ -19,15 +17,17 @@ module.exports = function(app) {
});
resource.use('readAll', function(req, res) {
var filteredProjects = app.projects,
var filteredProjects = app.projects.getAll(),
nameQuery = req.data && req.data.nameQuery;
if (nameQuery) {
filteredProjects = _(filteredProjects).filter(function(project) {
filteredProjects = app.projects.filter(function(project) {
return project.name.indexOf(nameQuery) !== -1;
});
}
filteredProjects = _(filteredProjects).sortBy('name');
res.send(filteredProjects);
});
@ -35,9 +35,9 @@ module.exports = function(app) {
var project;
Steppy(
function() {
project = _(app.projects).findWhere(params.condition);
project = app.projects.findWhere(params.condition);
getAvgProjectBuildDuration(project.name, this.slot());
app.projects.getAvgBuildDuration(project.name, this.slot());
// get last done build
db.builds.find({

View File

@ -1,24 +1,27 @@
'use strict';
var _ = require('underscore'),
logger = require('./lib/logger')('scheduler'),
CronJob = require('cron').CronJob;
exports.init = function(app, callback) {
var distributor = app.distributor,
var logger = app.lib.logger('scheduler'),
projectJobs = {};
app.on('projectLoaded', function(project) {
app.projects.on('projectLoaded', function(project) {
var time = project.buildEvery && project.buildEvery.time;
if (time) {
logger.log('Start job for loaded project "%s"', project.name);
logger.log(
'Start job for loaded project "%s" by schedule "%s"',
project.name,
time
);
projectJobs[project.name] = {};
projectJobs[project.name].job = new CronJob({
cronTime: time,
onTick: function() {
logger.log('Run project "%s"', project.name);
distributor.run({
app.distributor.run({
projectName: project.name,
withScmChangesOnly: project.buildEvery.withScmChangesOnly,
initiator: {type: 'scheduler'}
@ -29,7 +32,7 @@ exports.init = function(app, callback) {
}
});
app.on('projectUnloaded', function(project) {
app.projects.on('projectUnloaded', function(project) {
if (project.name in projectJobs) {
logger.log('Stop job for unloaded project "%s"', project.name);
projectJobs[project.name].job.stop();

View File

@ -4,6 +4,7 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock,
Steppy = require('twostep').Steppy;
@ -119,11 +120,11 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run 2 non-blocking projects in parallel', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2'
}];
}]);
});
itRunParallelProjects();
@ -131,12 +132,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 2 blocked by 1', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
}, {
name: 'project2',
blockedBy: ['project1']
}];
}]);
});
itRunSequentialProjects();
@ -144,12 +145,12 @@ describe('Distributor blocking with max 2 executors count', function() {
describe('should run project1, then 2, when 1 blocks 2', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project2']
}, {
name: 'project2'
}];
}]);
});
itRunSequentialProjects();
@ -159,7 +160,7 @@ describe('Distributor blocking with max 2 executors count', function() {
'should run 1, 2 in parallel, when 1 block 3, 2 blocked by 3',
function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
blocks: ['project3']
}, {
@ -167,7 +168,7 @@ describe('Distributor blocking with max 2 executors count', function() {
blockedBy: ['project3']
}, {
name: 'project3'
}];
}]);
});
itRunParallelProjects();

View File

@ -1,7 +1,8 @@
'use strict';
var Node = require('../../lib/node').Node,
EventEmitter = require('events').EventEmitter;
EventEmitter = require('events').EventEmitter,
ProjectsCollection = require('../../lib/project').ProjectsCollection;
exports.createNodeMock = function(executorRun) {
@ -17,3 +18,8 @@ exports.createNodeMock = function(executorRun) {
};
};
exports.createProjectsMock = function(configs) {
var projects = new ProjectsCollection({});
projects.configs = configs;
return projects;
};

View File

@ -3,12 +3,13 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor main', function() {
var distributor,
projects = [{name: 'project1'}];
projects = createProjectsMock([{name: 'project1'}]);
var expectUpdateBuild = function(distributor, build, number, conditionsHash) {
var conditions = conditionsHash[number];

View File

@ -2,9 +2,9 @@ var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
helpers = require('../helpers'),
createProjectsMock = require('./helpers').createProjectsMock,
path = require('path');
describe('Distributor run self after catch', function() {
var distributor, executorRunSpy, scmDataSpy;
@ -17,7 +17,7 @@ describe('Distributor run self after catch', function() {
helpers.removeDirIfExists(workspacePath, done);
distributor = new Distributor({
projects: [{
projects: createProjectsMock([{
name: 'project1',
dir: __dirname,
scm: helpers.repository.scm,
@ -25,7 +25,7 @@ describe('Distributor run self after catch', function() {
{type: 'shell', cmd: 'echo 1'}
],
catchRev: {comment: /.*/}
}],
}]),
nodes: nodes
});

View File

@ -3,7 +3,8 @@
var Distributor = require('../../lib/distributor').Distributor,
expect = require('expect.js'),
sinon = require('sinon'),
createNodeMock = require('./helpers').createNodeMock;
createNodeMock = require('./helpers').createNodeMock,
createProjectsMock = require('./helpers').createProjectsMock;
describe('Distributor trigger after', function() {
@ -13,14 +14,14 @@ describe('Distributor trigger after', function() {
describe('done when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{status: 'done', project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -39,11 +40,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -77,7 +82,9 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run totally 1 time', function() {
@ -91,14 +98,14 @@ describe('Distributor trigger after', function() {
describe('status is not set when project is done', function() {
before(function() {
projects = [{
projects = createProjectsMock([{
name: 'project1',
trigger: {
after: [{project: 'project2'}]
}
}, {
name: 'project2'
}];
}]);
executorRunSpy = sinon.stub().callsArgAsync(1);
sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
executorRunSpy
@ -117,11 +124,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {
@ -155,11 +166,15 @@ describe('Distributor trigger after', function() {
});
it('should run project1 at first call', function() {
expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
expect(executorRunSpy.getCall(0).thisValue.project).eql(
projects.get('project1')
);
});
it('should run project2 at second call', function() {
expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
expect(executorRunSpy.getCall(1).thisValue.project).eql(
projects.get('project2')
);
});
it('should run totally 2 times', function() {