diff --git a/app.js b/app.js index 9615aa2..c42dbcb 100644 --- a/app.js +++ b/app.js @@ -10,7 +10,7 @@ var env = process.env.NODE_ENV || 'development', _ = require('underscore'), reader = require('./lib/reader'), notifier = require('./lib/notifier'), - project = require('./lib/project'), + ProjectsCollection = require('./lib/project').ProjectsCollection, libLogger = require('./lib/logger'), EventEmitter = require('events').EventEmitter, validateConfig = require('./lib/validateConfig'); @@ -205,16 +205,15 @@ Steppy( db.init(app.config.paths.db, {db: dbBackend}, this.slot()); }, function() { - // load all projects for the first time - project.loadAll(app.config.paths.projects, this.slot()); + app.projects = new ProjectsCollection({ + db: db, + reader: reader, + baseDir: app.config.paths.projects + }); completeUncompletedBuilds(this.slot()); }, - function(err, projects) { - // note that `app.projects` is live variable - app.projects = projects; - logger.log('Loaded projects: ', _(app.projects).pluck('name')); - + function(err) { require('./distributor').init(app, this.slot()); }, function(err, distributor) { @@ -235,15 +234,17 @@ Steppy( require('./scheduler').init(app, this.slot()); - // notify about first project loading - _(app.projects).each(function(project) { - app.emit('projectLoaded', project); - }); - // init resources require('./resources')(app); }, + function() { + // load projects after all plugins to provide ability for plugins to + // handle `projectLoaded` event + app.projects.loadAll(this.slot()); + }, function(err) { + logger.log('Loaded projects: ', app.projects.pluck('name')); + var host = app.config.http.host, port = app.config.http.port; logger.log('Start http server on %s:%s', host, port); diff --git a/distributor.js b/distributor.js index c3a053b..7fda602 100644 --- a/distributor.js +++ b/distributor.js @@ -3,9 +3,6 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), Distributor = require('./lib/distributor').Distributor, - getAvgProjectBuildDuration = ( - require('./lib/project').getAvgProjectBuildDuration - ), db = require('./db'), logger = require('./lib/logger')('distributor'); @@ -20,7 +17,7 @@ exports.init = function(app, callback) { if (_(build.project).has('avgBuildDuration')) { this.pass(build.project.avgBuildDuration); } else { - getAvgProjectBuildDuration(build.project.name, this.slot()); + app.projects.getAvgBuildDuration(build.project.name, this.slot()); } }, function(err, avgBuildDuration) { diff --git a/httpApi.js b/httpApi.js index 3661d87..ee3c21e 100644 --- a/httpApi.js +++ b/httpApi.js @@ -2,8 +2,7 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), - querystring = require('querystring'), - libProject = require('./lib/project'); + querystring = require('querystring'); /* * Pure rest api on pure nodejs follows below */ @@ -59,7 +58,7 @@ module.exports = function(app) { Steppy( function() { var projectName = req.body.project, - project = _(app.projects).findWhere({name: projectName}); + project = app.projects.get(projectName); if (project) { res.statusCode = 204; @@ -92,10 +91,7 @@ module.exports = function(app) { throw new Error('Access token doesn`t match'); } - libProject.remove({ - baseDir: app.config.paths.projects, - name: projectName - }, this.slot()); + app.projects.remove(projectName, this.slot()); }, function() { logger.log('Project "%s" cleaned up', projectName); @@ -123,28 +119,22 @@ module.exports = function(app) { if (!newProjectName) throw new Error('new project name is not set'); - var curProject = _(app.projects).findWhere({name: projectName}); + var curProject = app.projects.get(projectName); if (!curProject) { throw new Error('Project "' + projectName + '" not found'); } this.pass(curProject); - var newProject = _(app.projects).findWhere({name: newProjectName}); + var newProject = app.projects.get(newProjectName); if (newProject) { throw new Error( 'Project name "' + newProjectName + '" already used' ); } - libProject.rename({ - baseDir: app.config.paths.projects, - name: projectName, - newName: newProjectName - }, this.slot()); + app.projects.rename(projectName, newProjectName, this.slot()); }, - function(err, curProject) { - curProject.name = newProjectName; - + function(err) { res.statusCode = 204; res.end(); }, diff --git a/lib/distributor.js b/lib/distributor.js index 673d78b..2888a21 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -272,10 +272,7 @@ Distributor.prototype.run = function(params, callback) { }; Steppy( function() { - project = _(self.projects).chain() - .findWhere({name: params.projectName}) - .clone() - .value(); + project = _(self.projects.get(params.projectName)).clone(); if (params.withScmChangesOnly) { self.nodes[0].hasScmChanges(project, this.slot()); diff --git a/lib/project.js b/lib/project.js index c29afea..d93eb17 100644 --- a/lib/project.js +++ b/lib/project.js @@ -4,17 +4,33 @@ var Steppy = require('twostep').Steppy, fs = require('fs'), path = require('path'), _ = require('underscore'), - reader = require('./reader'), - db = require('../db'), utils = require('./utils'), SpawnCommand = require('./command/spawn').Command, - validateParams = require('./validateParams'); + validateParams = require('./validateParams'), + EventEmitter = require('events').EventEmitter, + inherits = require('util').inherits; +/* + * Projects collection it's something similar to backbone collection. + * But contrasting to backbone there is no model of a single project, when you + * receive project from collection you just get a json. + * General id for the particular project is a `name` of that project. + */ +function ProjectsCollection(params) { + this.db = params.db; + this.reader = params.reader; + this.baseDir = params.baseDir; + this.configs = []; +} + +exports.ProjectsCollection = ProjectsCollection; + +inherits(ProjectsCollection, EventEmitter); /** * Validates and returns given `config` to the `callback`(err, config) */ -exports.validateConfig = function(config, callback) { +ProjectsCollection.prototype.validateConfig = function(config, callback) { Steppy( function() { validateParams(config, { @@ -60,50 +76,16 @@ exports.validateConfig = function(config, callback) { ); }; -/** - * Loads and returns project - */ -exports.load = function(baseDir, name, callback) { - var dir = path.join(baseDir, name); +ProjectsCollection.prototype._getProjectPath = function(name) { + return path.join(this.baseDir, name); +} + +ProjectsCollection.prototype._loadConfig = function(dir, callback) { + var self = this; + Steppy( function() { - fs.readdir(dir, this.slot()); - }, - function(err, dirContent) { - exports.loadConfig(dir, this.slot()); - }, - function(err, config) { - config.name = name; - config.dir = dir; - - exports.validateConfig(config, this.slot()); - }, - callback - ); -}; - -/** - * Loads all projects from `baseDir` and returns array of projects - */ -exports.loadAll = function(baseDir, callback) { - Steppy( - function() { - fs.readdir(baseDir, this.slot()); - }, - function(err, dirs) { - var loadGroup = this.makeGroup(); - _(dirs).each(function(dir) { - exports.load(baseDir, dir, loadGroup.slot()); - }); - }, - callback - ); -}; - -exports.loadConfig = function(dir, callback) { - Steppy( - function() { - reader.load(dir, 'config', this.slot()); + self.reader.load(dir, 'config', this.slot()); }, function(err, config) { // convert steps object to array @@ -132,38 +114,111 @@ exports.loadConfig = function(dir, callback) { ); }; -exports.saveConfig = function(config, dir, callback) { - fs.writeFile( - path.join(dir, 'config.json'), - JSON.stringify(config, null, 4), - callback - ); -}; +/** + * Loads project to collection + */ +ProjectsCollection.prototype.load = function(name, callback) { + var self = this, + dir = self._getProjectPath(name); -exports.create = function(baseDir, config, callback) { - var dir; Steppy( function() { - dir = path.join(baseDir, config.name); - fs.mkdir(dir, this.slot()); + self._loadConfig(dir, this.slot()); }, - function(err) { - exports.saveConfig(config, baseDir, this.slot()); + function(err, config) { + config.name = name; + config.dir = dir; + + self.validateConfig(config, this.slot()); }, - function(err) { - exports.load(dir, this.slot()); + function(err, config) { + self.configs.push(config); + self.emit('projectLoaded', config); + this.pass(null); }, callback ); }; -exports.getAvgProjectBuildDuration = function(projectName, callback) { +ProjectsCollection.prototype.unload = function(name, callback) { + callback = callback || _.noop; + var self = this; + + Steppy( + function() { + var index = _(self.configs).findIndex(function(config) { + return config.name === name; + }); + + if (index === -1) { + throw new Error('Can`t unload not loaded project: "' + name + '"'); + } + + var unloadedConfig = self.configs.splice(index, 1)[0]; + self.emit('projectUnloaded', unloadedConfig); + + this.pass(null); + }, + callback + ); +}; + +ProjectsCollection.prototype.get = function(name) { + return _(this.configs).findWhere({name: name}); +}; + +ProjectsCollection.prototype.getAll = function(name) { + return this.configs; +}; + +ProjectsCollection.prototype.findWhere = function(params) { + return _(this.configs).findWhere(params); +}; + +ProjectsCollection.prototype.where = function(params) { + return _(this.configs).where(params); +}; + +ProjectsCollection.prototype.filter = function(iterator) { + return _(this.configs).filter(iterator); +}; + +ProjectsCollection.prototype.pluck = function(attribute) { + return _(this.configs).pluck(attribute); +}; + +/** + * Loads all projects (from `this.baseDir`) + */ +ProjectsCollection.prototype.loadAll = function(callback) { + var self = this; + + Steppy( + function() { + fs.readdir(self.baseDir, this.slot()); + }, + function(err, dirs) { + var loadGroup = this.makeGroup(); + _(dirs).each(function(dir) { + self.load(dir, loadGroup.slot()); + }); + }, + callback + ); +}; + +/* + * Calculates average build duration for the given project + */ +ProjectsCollection.prototype.getAvgBuildDuration = function(name, callback) { + var self = this; + Steppy( function() { // get last done builds to calc avg build time - db.builds.find({ + self.db.builds.find({ start: { - projectName: projectName, + projectName: name, status: 'done', descCreateDate: '' }, @@ -181,24 +236,28 @@ exports.getAvgProjectBuildDuration = function(projectName, callback) { ); }; -exports.remove = function(params, callback) { +ProjectsCollection.prototype.remove = function(name, callback) { + var self = this; + Steppy( function() { - db.builds.find({ - start: {projectName: params.name, descCreateDate: ''} + self.db.builds.find({ + start: {projectName: name, descCreateDate: ''} }, this.slot()); new SpawnCommand().run({cmd: 'rm', args: [ - '-Rf', path.join(params.baseDir, params.name) + '-Rf', self._getProjectPath(name) ]}, this.slot()); + + self.unload(name, this.slot()); }, function(err, builds) { if (builds.length) { - db.builds.del(builds, this.slot()); + self.db.builds.del(builds, this.slot()); var logLinesRemoveGroup = this.makeGroup(); _(builds).each(function(build) { - db.logLines.remove({ + self.db.logLines.remove({ start: {buildId: build.id} }, logLinesRemoveGroup.slot()); }); @@ -210,24 +269,32 @@ exports.remove = function(params, callback) { ); }; -exports.rename = function(params, callback) { +ProjectsCollection.prototype.rename = function(name, newName, callback) { + var self = this; + Steppy( function() { fs.rename( - path.join(params.baseDir, params.name), - path.join(params.baseDir, params.newName), + self._getProjectPath(name), + self._getProjectPath(newName), this.slot() ); - db.builds.multiUpdate( - {start: {projectName: params.name, descCreateDate: ''}}, + self.db.builds.multiUpdate( + {start: {projectName: name, descCreateDate: ''}}, function(build) { - build.project.name = params.newName; + build.project.name = newName; return build; }, this.slot() ); }, + function() { + // just update currently loaded project name by link + self.get(name).name = newName; + + this.pass(null); + }, callback ); }; diff --git a/projectsWatcher.js b/projectsWatcher.js index db5684d..77aa1a7 100644 --- a/projectsWatcher.js +++ b/projectsWatcher.js @@ -2,45 +2,37 @@ var _ = require('underscore'), path = require('path'), - chokidar = require('chokidar'), - project = require('./lib/project'), - logger = require('./lib/logger')('projects watcher'); + chokidar = require('chokidar'); exports.init = function(app, callback) { + var logger = app.lib.logger('projects watcher'); + // start file watcher for reloading projects on change var syncProject = function(filename, fileInfo) { - var baseDir = app.config.paths.projects, - projectName = path.relative( - baseDir, - path.dirname(filename) - ); + var projectName = path.relative( + app.config.paths.projects, + path.dirname(filename) + ); - var projectIndex = _(app.projects).findIndex(function(project) { - return project.name === projectName; - }); - - if (projectIndex !== -1) { + if (app.projects.get(projectName)) { logger.log('Unload project: "' + projectName + '"'); - var unloadedProject = app.projects.splice(projectIndex, 1)[0]; - app.emit('projectUnloaded', unloadedProject); + app.projects.unload(projectName); } // on add or change (info is falsy on unlink) if (fileInfo) { logger.log('Load project "' + projectName + '" on change'); - project.load(baseDir, projectName, function(err, project) { + app.projects.load(projectName, function(err) { if (err) { return logger.error( 'Error during load project "' + projectName + '": ', err.stack || err ); } - app.projects.push(project); logger.log( 'Project "' + projectName + '" loaded:', - JSON.stringify(project, null, 4) + JSON.stringify(app.projects.get(projectName), null, 4) ); - app.emit('projectLoaded', project); }); } }; diff --git a/resources/projects.js b/resources/projects.js index 3eb1c8c..3dd4a79 100644 --- a/resources/projects.js +++ b/resources/projects.js @@ -2,8 +2,6 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), - getAvgProjectBuildDuration = - require('../lib/project').getAvgProjectBuildDuration, createBuildDataResource = require('../distributor').createBuildDataResource, logger = require('../lib/logger')('projects resource'), db = require('../db'); @@ -19,15 +17,17 @@ module.exports = function(app) { }); resource.use('readAll', function(req, res) { - var filteredProjects = app.projects, + var filteredProjects = app.projects.getAll(), nameQuery = req.data && req.data.nameQuery; if (nameQuery) { - filteredProjects = _(filteredProjects).filter(function(project) { + filteredProjects = app.projects.filter(function(project) { return project.name.indexOf(nameQuery) !== -1; }); } + filteredProjects = _(filteredProjects).sortBy('name'); + res.send(filteredProjects); }); @@ -35,9 +35,9 @@ module.exports = function(app) { var project; Steppy( function() { - project = _(app.projects).findWhere(params.condition); + project = app.projects.findWhere(params.condition); - getAvgProjectBuildDuration(project.name, this.slot()); + app.projects.getAvgBuildDuration(project.name, this.slot()); // get last done build db.builds.find({ diff --git a/scheduler.js b/scheduler.js index 869cedb..e60983b 100644 --- a/scheduler.js +++ b/scheduler.js @@ -1,24 +1,27 @@ 'use strict'; var _ = require('underscore'), - logger = require('./lib/logger')('scheduler'), CronJob = require('cron').CronJob; exports.init = function(app, callback) { - var distributor = app.distributor, + var logger = app.lib.logger('scheduler'), projectJobs = {}; - app.on('projectLoaded', function(project) { + app.projects.on('projectLoaded', function(project) { var time = project.buildEvery && project.buildEvery.time; if (time) { - logger.log('Start job for loaded project "%s"', project.name); + logger.log( + 'Start job for loaded project "%s" by schedule "%s"', + project.name, + time + ); projectJobs[project.name] = {}; projectJobs[project.name].job = new CronJob({ cronTime: time, onTick: function() { logger.log('Run project "%s"', project.name); - distributor.run({ + app.distributor.run({ projectName: project.name, withScmChangesOnly: project.buildEvery.withScmChangesOnly, initiator: {type: 'scheduler'} @@ -29,7 +32,7 @@ exports.init = function(app, callback) { } }); - app.on('projectUnloaded', function(project) { + app.projects.on('projectUnloaded', function(project) { if (project.name in projectJobs) { logger.log('Stop job for unloaded project "%s"', project.name); projectJobs[project.name].job.stop(); diff --git a/test/distributor/blocking.js b/test/distributor/blocking.js index 651c147..756ca69 100644 --- a/test/distributor/blocking.js +++ b/test/distributor/blocking.js @@ -4,6 +4,7 @@ var Distributor = require('../../lib/distributor').Distributor, expect = require('expect.js'), sinon = require('sinon'), createNodeMock = require('./helpers').createNodeMock, + createProjectsMock = require('./helpers').createProjectsMock, Steppy = require('twostep').Steppy; @@ -119,11 +120,11 @@ describe('Distributor blocking with max 2 executors count', function() { describe('should run 2 non-blocking projects in parallel', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', }, { name: 'project2' - }]; + }]); }); itRunParallelProjects(); @@ -131,12 +132,12 @@ describe('Distributor blocking with max 2 executors count', function() { describe('should run project1, then 2, when 2 blocked by 1', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', }, { name: 'project2', blockedBy: ['project1'] - }]; + }]); }); itRunSequentialProjects(); @@ -144,12 +145,12 @@ describe('Distributor blocking with max 2 executors count', function() { describe('should run project1, then 2, when 1 blocks 2', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', blocks: ['project2'] }, { name: 'project2' - }]; + }]); }); itRunSequentialProjects(); @@ -159,7 +160,7 @@ describe('Distributor blocking with max 2 executors count', function() { 'should run 1, 2 in parallel, when 1 block 3, 2 blocked by 3', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', blocks: ['project3'] }, { @@ -167,7 +168,7 @@ describe('Distributor blocking with max 2 executors count', function() { blockedBy: ['project3'] }, { name: 'project3' - }]; + }]); }); itRunParallelProjects(); diff --git a/test/distributor/helpers.js b/test/distributor/helpers.js index 0740d4b..39f0734 100644 --- a/test/distributor/helpers.js +++ b/test/distributor/helpers.js @@ -1,7 +1,8 @@ 'use strict'; var Node = require('../../lib/node').Node, - EventEmitter = require('events').EventEmitter; + EventEmitter = require('events').EventEmitter, + ProjectsCollection = require('../../lib/project').ProjectsCollection; exports.createNodeMock = function(executorRun) { @@ -17,3 +18,8 @@ exports.createNodeMock = function(executorRun) { }; }; +exports.createProjectsMock = function(configs) { + var projects = new ProjectsCollection({}); + projects.configs = configs; + return projects; +}; diff --git a/test/distributor/main.js b/test/distributor/main.js index e7eab9f..c4b3dab 100644 --- a/test/distributor/main.js +++ b/test/distributor/main.js @@ -3,12 +3,13 @@ var Distributor = require('../../lib/distributor').Distributor, expect = require('expect.js'), sinon = require('sinon'), - createNodeMock = require('./helpers').createNodeMock; + createNodeMock = require('./helpers').createNodeMock, + createProjectsMock = require('./helpers').createProjectsMock; describe('Distributor main', function() { var distributor, - projects = [{name: 'project1'}]; + projects = createProjectsMock([{name: 'project1'}]); var expectUpdateBuild = function(distributor, build, number, conditionsHash) { var conditions = conditionsHash[number]; diff --git a/test/distributor/runSelfAfterCatchRev.js b/test/distributor/runSelfAfterCatchRev.js index 06761e4..fdf8484 100644 --- a/test/distributor/runSelfAfterCatchRev.js +++ b/test/distributor/runSelfAfterCatchRev.js @@ -2,9 +2,9 @@ var Distributor = require('../../lib/distributor').Distributor, expect = require('expect.js'), sinon = require('sinon'), helpers = require('../helpers'), + createProjectsMock = require('./helpers').createProjectsMock, path = require('path'); - describe('Distributor run self after catch', function() { var distributor, executorRunSpy, scmDataSpy; @@ -17,7 +17,7 @@ describe('Distributor run self after catch', function() { helpers.removeDirIfExists(workspacePath, done); distributor = new Distributor({ - projects: [{ + projects: createProjectsMock([{ name: 'project1', dir: __dirname, scm: helpers.repository.scm, @@ -25,7 +25,7 @@ describe('Distributor run self after catch', function() { {type: 'shell', cmd: 'echo 1'} ], catchRev: {comment: /.*/} - }], + }]), nodes: nodes }); diff --git a/test/distributor/triggerAfter.js b/test/distributor/triggerAfter.js index 6ca78a8..2461e5f 100644 --- a/test/distributor/triggerAfter.js +++ b/test/distributor/triggerAfter.js @@ -3,7 +3,8 @@ var Distributor = require('../../lib/distributor').Distributor, expect = require('expect.js'), sinon = require('sinon'), - createNodeMock = require('./helpers').createNodeMock; + createNodeMock = require('./helpers').createNodeMock, + createProjectsMock = require('./helpers').createProjectsMock; describe('Distributor trigger after', function() { @@ -13,14 +14,14 @@ describe('Distributor trigger after', function() { describe('done when project is done', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', trigger: { after: [{status: 'done', project: 'project2'}] } }, { name: 'project2' - }]; + }]); executorRunSpy = sinon.stub().callsArgAsync(1); sinon.stub(Distributor.prototype, '_createNode', createNodeMock( executorRunSpy @@ -39,11 +40,15 @@ describe('Distributor trigger after', function() { }); it('should run project1 at first call', function() { - expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]); + expect(executorRunSpy.getCall(0).thisValue.project).eql( + projects.get('project1') + ); }); it('should run project2 at second call', function() { - expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]); + expect(executorRunSpy.getCall(1).thisValue.project).eql( + projects.get('project2') + ); }); it('should run totally 2 times', function() { @@ -77,7 +82,9 @@ describe('Distributor trigger after', function() { }); it('should run project1 at first call', function() { - expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]); + expect(executorRunSpy.getCall(0).thisValue.project).eql( + projects.get('project1') + ); }); it('should run totally 1 time', function() { @@ -91,14 +98,14 @@ describe('Distributor trigger after', function() { describe('status is not set when project is done', function() { before(function() { - projects = [{ + projects = createProjectsMock([{ name: 'project1', trigger: { after: [{project: 'project2'}] } }, { name: 'project2' - }]; + }]); executorRunSpy = sinon.stub().callsArgAsync(1); sinon.stub(Distributor.prototype, '_createNode', createNodeMock( executorRunSpy @@ -117,11 +124,15 @@ describe('Distributor trigger after', function() { }); it('should run project1 at first call', function() { - expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]); + expect(executorRunSpy.getCall(0).thisValue.project).eql( + projects.get('project1') + ); }); it('should run project2 at second call', function() { - expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]); + expect(executorRunSpy.getCall(1).thisValue.project).eql( + projects.get('project2') + ); }); it('should run totally 2 times', function() { @@ -155,11 +166,15 @@ describe('Distributor trigger after', function() { }); it('should run project1 at first call', function() { - expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]); + expect(executorRunSpy.getCall(0).thisValue.project).eql( + projects.get('project1') + ); }); it('should run project2 at second call', function() { - expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]); + expect(executorRunSpy.getCall(1).thisValue.project).eql( + projects.get('project2') + ); }); it('should run totally 2 times', function() {