merge with master (package.json and terminal index merged manually)

README.md

@@ -6,7 +6,7 @@ work in progress...
 
 [![Build Status](https://travis-ci.org/node-ci/nci.svg?branch=master)](https://travis-ci.org/node-ci/nci)
 
-## TODO for release 0.9
+## TODO for release 0.5
 
 * ~~Dashboard (builds list, projects autocomlete)~~
 * ~~Build page (build info(dates, changes, etc), console)~~
@@ -33,15 +33,25 @@ work in progress...
 
 * ~~git checkout before reset~~
 * slow move out from build page (with lot of output) to main page - several sec
+at ff (ff very slow on remove/replace terminal element)
 * ~~when long line appear console output row numbers not on the same line with
 content~~
-* some "undefined" comments in scm changes
+* ~~some "undefined" comments in scm changes~~
 * projects list scroll
 * ~~Error during send: TypeError: Cannot read property 'changes' of undefined~~
 * ~~Builds loss~~
 * ~~error on git after change branch: fatal: ambiguous argument '18a8ea4..branch':
 unknown revision or path not in the working tree.~~
 * "Uncaught TypeError: Cannot read property 'name' of undefined" at item.js (jade)
+* strange git with merge commits changes detection, e.g. whem update from
+"0.3.7" commit to master "new build timeline style, sime layout fixes" and
+"add some responsive styles to build timeline, revert in-progress pulsate
+animation" appear but should not.
+* include fonts and other external static (if any)
+* build console doesn't stick to the bottom at ff
+* ~~more strict server and project configs valifation~~
+* ui browser tests needed
+* use one from: jquery or native browser methods
 
 
 ## Feature requests
@@ -54,6 +64,8 @@ unknown revision or path not in the working tree.~~
 down to the output which could be very long)~~
 * speed up build points animation at ff (maybe borrow something from animate.css?)
 * current successfully streak icons at project page
+* cancell in progress build + buld/step timeout
+* rev hash link to repo web ui
 
 
 ## Roadmap

app.js

@@ -10,9 +10,11 @@ var env = process.env.NODE_ENV || 'development',
 	_ = require('underscore'),
 	reader = require('./lib/reader'),
 	notifier = require('./lib/notifier'),
-	project = require('./lib/project'),
+	ProjectsCollection = require('./lib/project').ProjectsCollection,
+	BuildsCollection = require('./lib/build').BuildsCollection,
 	libLogger = require('./lib/logger'),
-	EventEmitter = require('events').EventEmitter;
+	EventEmitter = require('events').EventEmitter,
+	validateConfig = require('./lib/validateConfig');
 
 var app = new EventEmitter(),
 	logger = libLogger('app'),
@@ -61,7 +63,6 @@ app.lib = {};
 app.lib.reader = reader;
 app.lib.notifier = notifier;
 app.lib.logger = libLogger;
-app.lib.project = project;
 
 var configDefaults = {
 	notify: {},
@@ -182,6 +183,11 @@ Steppy(
 
 		reader.load(app.config.paths.data, 'config', this.slot());
 	},
+	function(err, mkdirResult, config) {
+		this.pass(mkdirResult);
+
+		validateConfig(config, this.slot());
+	},
 	function(err, mkdirResult, config) {
 		_(app.config).defaults(config);
 		_(app.config).defaults(configDefaults);
@@ -200,20 +206,22 @@ Steppy(
 		db.init(app.config.paths.db, {db: dbBackend}, this.slot());
 	},
 	function() {
-		// load all projects for the first time
+		app.projects = new ProjectsCollection({
-		project.loadAll(app.config.paths.projects, this.slot());
+			db: db,
+			reader: reader,
+			baseDir: app.config.paths.projects
+		});
 
 		completeUncompletedBuilds(this.slot());
 	},
-	function(err, projects) {
+	function(err) {
-		// note that `app.projects` is live variable
-		app.projects = projects;
-		logger.log('Loaded projects: ', _(app.projects).pluck('name'));
-
 		require('./distributor').init(app, this.slot());
 	},
 	function(err, distributor) {
-		app.distributor = distributor;
+		app.builds = new BuildsCollection({
+			db: db,
+			distributor: distributor
+		});
 
 		// register other plugins
 		require('./lib/notifier/console').register(app);
@@ -230,15 +238,17 @@ Steppy(
 
 		require('./scheduler').init(app, this.slot());
 
-		// notify about first project loading
-		_(app.projects).each(function(project) {
-			app.emit('projectLoaded', project);
-		});
-
 		// init resources
 		require('./resources')(app);
 	},
+	function() {
+		// load projects after all plugins to provide ability for plugins to
+		// handle `projectLoaded` event
+		app.projects.loadAll(this.slot());
+	},
 	function(err) {
+		logger.log('Loaded projects: ', _(app.projects.getAll()).pluck('name'));
+
 		var host = app.config.http.host,
 			port = app.config.http.port;
 		logger.log('Start http server on %s:%s', host, port);

app/components/builds/view/index.jade

@@ -14,7 +14,7 @@ mixin statusBadge(build)
 .row
 	if this.state.build
 		.col-sm-3.hidden-xs
-			BuildSidebar(projectName=this.state.build.project.name)
+			BuildSidebar(projectName=this.state.build.project.name, currentBuild=this.state.build)
 
 		.col-sm-9
 			h1.page-header

app/components/builds/view/sidebar/index.jade

@@ -1,6 +1,8 @@
 .builds.builds__timeline.builds__timeline-small
 	each item in this.state.items
-		.builds_item(key=item.id, class="builds_item__#{item.status}")
+		- var buildItemClasses = ['builds_item__' + item.status];
+		- if (item.id === this.props.currentBuild.id) buildItemClasses.push('builds_item__current');
+		.builds_item(key=item.id, class=buildItemClasses)
 			.builds_inner
 				.row
 					.builds_header

app/components/projects/view/index.jade

@@ -32,7 +32,7 @@ div
 
 		p Current successfully streak: 
 			if lastDoneBuild
-				span= this.state.project.doneBuildsStreak
+				span= this.state.project.doneBuildsStreak.buildsCount
 			else
 				| -
 

app/components/terminal/index.js

@@ -4,6 +4,7 @@ var _ = require('underscore'),
 	React = require('react'),
 	Reflux = require('reflux'),
 	terminalStore = require('../../stores/terminal'),
+	buildStore = require('../../stores/build'),
 	ansiUp = require('ansi_up'),
 	template = require('./index.jade');
 
@@ -18,9 +19,28 @@ var Component = React.createClass({
 		this.listenTo(terminalStore, this.updateItems);
 		var node = document.getElementsByClassName('terminal')[0];
 		this.initialScrollPosition = node.getBoundingClientRect().top;
+		if (this.props.showPreloader) {
+			this.getTerminal().insertAdjacentHTML('afterend',
+				'<img src="/images/preloader.gif" class="terminal_preloader"/>'
+			);
+
+			this.listenTo(buildStore, function(build) {
+				if (build.completed) {
+					this.removePreloader();
+				}
+			});
+		}
 
 		window.onscroll = this.onScroll;
 	},
+	removePreloader: function() {
+		var preloader = document.getElementsByClassName(
+			'terminal_preloader'
+		)[0];
+		if (preloader) {
+			preloader.parentNode.removeChild(preloader);
+		}
+	},
 	componentWillUnmount: function() {
 		window.onscroll = null;
 	},
@@ -90,6 +110,9 @@ var Component = React.createClass({
 			this.data = build.data;
 			this.renderBuffer();
 		}
+		if (this.props.showPreloader && build.buildCompleted) {
+			this.removePreloader();
+		}
 	},
 	shouldComponentUpdate: function() {
 		return false;

app/styles/components/builds.less

@@ -177,6 +177,21 @@
 						border-right-color: darken(@well-bg, 10%);
 						top: 13px;
 					}
+
+					&__current {
+						&:before {
+							left: 10px;
+							border-right-color: @component-active-bg;
+							top: 13px;
+						}
+
+						.builds {
+							&_inner {
+								border-left: 6px solid @component-active-bg;
+							}
+						}
+
+					}
 				}
 
 				&_header {

data/config.yaml

@@ -1,7 +1,7 @@
 
-plugins:
+# plugins:
-    # - nci-mail-notification
+#     - nci-mail-notification
-    # - nci-jabber-notification
+#     - nci-jabber-notification
 
 nodes:
     - type: local
@@ -14,7 +14,6 @@ http:
 
 storage:
     backend: memdown
-    # backend: leveldown
 
 notify:
     mail:

distributor.js

@@ -3,9 +3,6 @@
 var Steppy = require('twostep').Steppy,
 	_ = require('underscore'),
 	Distributor = require('./lib/distributor').Distributor,
-	getAvgProjectBuildDuration = (
-		require('./lib/project').getAvgProjectBuildDuration
-	),
 	db = require('./db'),
 	logger = require('./lib/logger')('distributor');
 
@@ -18,13 +15,21 @@ exports.init = function(app, callback) {
 			Steppy(
 				function() {
 					if (_(build.project).has('avgBuildDuration')) {
-						this.pass(build.project.avgBuildDuration);
+						this.pass(null);
 					} else {
-						getAvgProjectBuildDuration(build.project.name, this.slot());
+						app.builds.getRecent({
+							projectName: build.project.name,
+							status: 'done',
+							limit: 10
+						}, this.slot());
 					}
 				},
-				function(err, avgBuildDuration) {
+				function(err, doneBuilds) {
-					build.project.avgBuildDuration = avgBuildDuration;
+					if (doneBuilds) {
+						build.project.avgBuildDuration = (
+							app.builds.getAvgBuildDuration(doneBuilds)
+						);
+					}
 
 					db.builds.put(build, this.slot());
 				},
@@ -44,116 +49,7 @@ exports.init = function(app, callback) {
 		}
 	});
 
-	var buildDataResourcesHash = {};
+	distributor.on('buildLogLines', function(build, lines) {
-
-	// create resource for build data
-	var createBuildDataResource = function(buildId) {
-		if (buildId in buildDataResourcesHash) {
-			return;
-		}
-		var buildDataResource = app.dataio.resource('build' + buildId);
-		buildDataResource.on('connection', function(client) {
-			var callback = this.async();
-			Steppy(
-				function() {
-					db.logLines.find({
-						start: {buildId: buildId},
-					}, this.slot());
-				},
-				function(err, lines) {
-					client.emit('sync', 'data', {lines: lines});
-					this.pass(true);
-				},
-				function(err) {
-					if (err) {
-						logger.error(
-							'error during read log for "' + buildId + '":',
-							err.stack || err
-						);
-					}
-					callback();
-				}
-			);
-		});
-		buildDataResourcesHash[buildId] = buildDataResource;
-	};
-
-	exports.createBuildDataResource = createBuildDataResource;
-
-	var buildsResource = app.dataio.resource('builds');
-
-	distributor.on('buildUpdate', function(build, changes) {
-		if (build.status === 'queued') {
-			createBuildDataResource(build.id);
-		}
-
-		// notify about build's project change, coz building affects project
-		// related stat (last build date, avg build time, etc) 
-		if (changes.completed) {
-			var projectsResource = app.dataio.resource('projects');
-			projectsResource.clientEmitSyncChange({name: build.project.name});
-		}
-
-		buildsResource.clientEmitSync('change', {
-			buildId: build.id, changes: changes
-		});
-	});
-
-	distributor.on('buildCancel', function(build) {
-		buildsResource.clientEmitSync('cancel', {buildId: build.id});
-	});
-
-	var buildLogLineNumbersHash = {},
-		lastLinesHash = {};
-
-	distributor.on('buildData', function(build, data) {
-		var cleanupText = function(text) {
-			return text.replace('\r', '');
-		};
-
-		var splittedData  = data.split('\n'),
-			logLineNumber = buildLogLineNumbersHash[build.id] || 0;
-
-		lastLinesHash[build.id] = lastLinesHash[build.id] || '';
-
-		// if we don't have last line, so we start new line
-		if (!lastLinesHash[build.id]) {
-			logLineNumber++;
-		}
-		lastLinesHash[build.id] += _(splittedData).first();
-
-		var lines = [{
-			text: cleanupText(lastLinesHash[build.id]),
-			buildId: build.id,
-			number: logLineNumber
-		}];
-
-		if (splittedData.length > 1) {
-			// if we have last '' we have to take all except last
-			// this shown that string ends with eol
-			if (_(splittedData).last() === '') {
-				lastLinesHash[build.id] = '';
-				splittedData = _(splittedData.slice(1)).initial();
-			} else {
-				lastLinesHash[build.id] = _(splittedData).last();
-				splittedData = _(splittedData).tail();
-			}
-
-			lines = lines.concat(_(splittedData).map(function(line) {
-				return {
-					text: cleanupText(line),
-					buildId: build.id,
-					number: ++logLineNumber
-				};
-			}));
-		}
-
-		buildLogLineNumbersHash[build.id] = logLineNumber;
-		app.dataio.resource('build' + build.id).clientEmitSync(
-			'data',
-			{lines: lines}
-		);
-
 		// write build logs to db
 		db.logLines.put(lines, function(err) {
 			if (err) {

docs/developing-plugins/builds-collection.md

@@ -0,0 +1,50 @@
+
+## BuildsCollection()
+
+  Facade entity which accumulates operations with currently running and
+  db saved builds.
+
+## BuildsCollection.create(params:Object, [callback(err)]:Function)
+
+  Create build by running given project.
+  - `params.projectName` - project to build
+  - `params.withScmChangesOnly` - if true then build will be started only if
+  there is scm changes for project
+  - `params.queueQueued` - if true then currently queued project can be queued
+  again
+  - `params.initiator` - contains information about initiator of the build,
+  must contain `type` property e.g. when one build triggers another:
+  initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
+
+## BuildsCollection.cancel(id:Number, [callback(err)]:Function)
+
+  Cancel build by id.
+  Note that only queued build can be canceled currently.
+
+## BuildsCollection.get(id:Number, callback(err,build):Function)
+
+  Get build by id.
+
+## BuildsCollection.getLogLines(params:Object, callback(err,logLinesData):Function)
+
+  Get log lines for the given build.
+  - `params.buildId` - target build
+  - `params.from` - if set then lines from that number will be returned
+  - `params.to` - if set then lines to that number will be returned
+
+## BuildsCollection.getAvgBuildDuration(builds:Array.<Object>)
+
+  Calculate average build duration for the given builds.
+
+## BuildsCollection.getRecent(params:Object, callback(err,builds):Function)
+
+  Get builds sorted by date in descending order.
+  - `params.projectName` - optional project filter
+  - `params.status` - optional status filter, can be used only when
+  `params.projectName` is set. When used builds in the result will contain
+  only following fields: id, number, startDate, endDate
+
+## BuildsCollection.getDoneStreak(params:Object, callback(err,doneStreak):Function)
+
+  Get info about current done builds streak.
+  - `params.projectName` - optional project filter

docs/developing-plugins/projects-collection.md

@@ -0,0 +1,55 @@
+
+## ProjectsCollection()
+
+  Projects collection contains all currently loaded projects and provides
+  operations for manipulating with them.
+  All projects stored on disk in `baseDir` and loaded to memory so
+  they can be received (by `get`, `getAll` and other methods) in a sync way.
+  Note that id for the particular project is a `name` of that project.
+
+## ProjectsCollection.validateConfig(config:Object, callback(err,config):Function)
+
+  Validate and return given config.
+
+## ProjectsCollection.load(name:String, [callback(err)]:Function)
+
+  Load project to collection.
+  `projectLoaded` event with loaded config as argument will be emitted after
+  load.
+
+## ProjectsCollection.loadAll([callback(err)]:Function)
+
+  Load all projects (from `this.baseDir`).
+  Calls `load` for every project in a base dir.
+
+## ProjectsCollection.unload(name:String, [callback(err)]:Function)
+
+  Unload project from collection
+  `projectUnloaded` event with unloaded config as argument will be emitted
+  after unload.
+
+## ProjectsCollection.get(name:String)
+
+  Get project config by name.
+  Returns config object or undefined if project is not found.
+
+## ProjectsCollection.getAll()
+
+  Get configs for all currently loaded projects.
+  Returns array of config objects.
+
+## ProjectsCollection.filter(predicate:Function)
+
+  Get project configs which match to predicate.
+  Returns array of config objects or empty array if there is no matched
+  project.
+
+## ProjectsCollection.remove(name:String, [callback(err)]:Function)
+
+  Remove project by name.
+  Calls `unload`, removes project from disk and db.
+
+## ProjectsCollection.rename(name:String, [callback(err)]:Function)
+
+  Rename project.
+  Renames project on disk and db, also changes name for loaded project.

httpApi.js

@@ -48,22 +48,25 @@ router.getRoute = function(req) {
 };
 
 module.exports = function(app) {
-	var logger = app.lib.logger('http api');
+	var logger = app.lib.logger('http api'),
+		accessToken = (Math.random() * Math.random()).toString(36).substring(2);
+
+	logger.log('access token is: %s', accessToken);
 
 	// run building of a project
 	router.post('/api/0.1/builds', function(req, res, next) {
 		Steppy(
 			function() {
 				var projectName = req.body.project,
-					project = _(app.projects).findWhere({name: projectName});
+					project = app.projects.get(projectName);
 
 				if (project) {
 					res.statusCode = 204;
 					logger.log('Run project "%s"', projectName);
-					app.distributor.run({
+					app.builds.create({
 						projectName: projectName,
 						withScmChangesOnly: req.body.withScmChangesOnly,
-						skipQueued: req.body.skipQueued,
+						queueQueued: req.body.queueQueued,
 						initiator: {type: 'httpApi'}
 					});
 				} else {
@@ -76,16 +79,19 @@ module.exports = function(app) {
 		);
 	});
 
-	// TODO: restrict access with some sort of token
 	router.del('/api/0.1/projects/:name', function(req, res, next) {
-		var projectName = req.params.name;
+		var token = req.body.token,
+			projectName = req.params.name;
+
 		Steppy(
 			function() {
 				logger.log('Cleaning up project "%s"', projectName);
-				app.lib.project.remove({
+
-					baseDir: app.config.paths.projects,
+				if (token !== accessToken) {
-					name: projectName
+					throw new Error('Access token doesn`t match');
-				}, this.slot());
+				}
+
+				app.projects.remove(projectName, this.slot());
 			},
 			function() {
 				logger.log('Project "%s" cleaned up', projectName);
@@ -97,7 +103,8 @@ module.exports = function(app) {
 	});
 
 	router.patch('/api/0.1/projects/:name', function(req, res, next) {
-		var projectName = req.params.name,
+		var token = req.body.token,
+			projectName = req.params.name,
 			newProjectName = req.body.name;
 
 		Steppy(
@@ -106,30 +113,28 @@ module.exports = function(app) {
 					'Rename project "%s" to "%s"', projectName, newProjectName
 				);
 
+				if (token !== accessToken) {
+					throw new Error('Access token doesn`t match');
+				}
+
 				if (!newProjectName) throw new Error('new project name is not set');
 
-				var curProject = _(app.projects).findWhere({name: projectName});
+				var curProject = app.projects.get(projectName);
 				if (!curProject) {
 					throw new Error('Project "' + projectName + '" not found');
 				}
 				this.pass(curProject);
 
-				var newProject = _(app.projects).findWhere({name: newProjectName});
+				var newProject = app.projects.get(newProjectName);
 				if (newProject) {
 					throw new Error(
 						'Project name "' + newProjectName + '" already used'
 					);
 				}
 
-				app.lib.project.rename({
+				app.projects.rename(projectName, newProjectName, this.slot());
-					baseDir: app.config.paths.projects,
-					name: projectName,
-					newName: newProjectName
-				}, this.slot());
 			},
-			function(err, curProject) {
+			function(err) {
-				curProject.name = newProjectName;
-
 				res.statusCode = 204;
 				res.end();
 			},

lib/build.js

@@ -0,0 +1,222 @@
+'use strict';
+
+var Steppy = require('twostep').Steppy,
+	_ = require('underscore'),
+	EventEmitter = require('events').EventEmitter,
+	inherits = require('util').inherits;
+
+/**
+ * Facade entity which accumulates operations with currently running and
+ * db saved builds.
+ */
+function BuildsCollection(params) {
+	this.db = params.db;
+	this.distributor = params.distributor;
+
+	this._proxyDistributorEvent('buildUpdate', 'buildUpdated');
+	this._proxyDistributorEvent('buildCancel', 'buildCanceled');
+	this._proxyDistributorEvent('buildLogLines', 'buildLogLines');
+}
+
+exports.BuildsCollection = BuildsCollection;
+
+inherits(BuildsCollection, EventEmitter);
+
+BuildsCollection.prototype._proxyDistributorEvent = function(source, dest) {
+	var self = this;
+
+	self.distributor.on(source, function() {
+		self.emit.apply(self, [dest].concat(_(arguments).toArray()));
+	});
+};
+
+/**
+ * Create build by running given project.
+ * - `params.projectName` - project to build
+ * - `params.withScmChangesOnly` - if true then build will be started only if
+ * there is scm changes for project
+ * - `params.queueQueued` - if true then currently queued project can be queued
+ * again
+ * - `params.initiator` - contains information about initiator of the build,
+ * must contain `type` property e.g. when one build triggers another:
+ * initiator: {type: 'build', id: 123, number: 10, project: {name: 'project1'}
+ *
+ * @param {Object} params
+ * @param {Function} [callback(err)]
+ */
+BuildsCollection.prototype.create = function(params, callback) {
+	this.distributor.run(params, callback);
+};
+
+/**
+ * Cancel build by id.
+ * Note that only queued build can be canceled currently.
+ *
+ * @param {Number} id
+ * @param {Function} [callback(err)]
+ */
+BuildsCollection.prototype.cancel = function(id, callback) {
+	this.distributor.cancel(id, callback);
+};
+
+/**
+ * Get build by id.
+ *
+ * @param {Number} id
+ * @param {Function} callback(err,build)
+ */
+BuildsCollection.prototype.get = function(id, callback) {
+	this.db.builds.find({start: {id: id}}, function(err, builds) {
+		callback(err, builds && builds[0]);
+	});
+};
+
+/**
+ * Get log lines for the given build.
+ * - `params.buildId` - target build
+ * - `params.from` - if set then lines from that number will be returned
+ * - `params.to` - if set then lines to that number will be returned
+ *
+ * @param {Object} params
+ * @param {Function} callback(err,logLinesData)
+ */
+BuildsCollection.prototype.getLogLines = function(params, callback) {
+	var self = this;
+
+	Steppy(
+		function() {
+			var findParams = {
+				start: {buildId: params.buildId},
+				end: {buildId: params.buildId}
+			};
+			if (params.from) findParams.start.number = params.from;
+			if (params.to) findParams.end.number = params.to;
+
+			var count = params.from && params.to ? params.to - params.from + 1: 0;
+
+			self.db.logLines.find(findParams, this.slot());
+
+			this.pass(count);
+		},
+		function(err, logLines, count) {
+			this.pass({
+				lines: logLines,
+				isLast: count ? logLines.length < count : true
+			});
+		},
+		callback
+	);	
+};
+
+BuildsCollection.prototype.getLogLinesTail = function(params, callback) {
+	var self = this;
+
+	Steppy(
+		function() {
+			var findParams = {
+				reverse: true,
+				start: {buildId: params.buildId},
+				limit: params.limit
+			};
+
+			self.db.logLines.find(findParams, this.slot());
+		},
+		function(err, logLines) {
+			var lines = logLines.reverse(),
+				total = logLines.length ? logLines[logLines.length - 1].number : 0;
+
+			this.pass({lines: lines, total: total});
+		},
+		callback
+	);	
+};
+
+/**
+ * Calculate average build duration for the given builds.
+ *
+ * @param {Object[]} builds
+ */
+BuildsCollection.prototype.getAvgBuildDuration = function(builds) {
+	var durationsSum = _(builds).reduce(function(sum, build) {
+		return sum + (build.endDate - build.startDate);
+	}, 0);
+	return Math.round(durationsSum / builds.length);
+};
+
+/**
+ * Get builds sorted by date in descending order.
+ * - `params.projectName` - optional project filter
+ * - `params.status` - optional status filter, can be used only when
+ * `params.projectName` is set. When used builds in the result will contain
+ * only following fields: id, number, startDate, endDate
+ *
+ * @param {Object} params
+ * @param {Function} callback(err,builds)
+ */
+BuildsCollection.prototype.getRecent = function(params, callback) {
+	params.limit = params.limit || 20;
+	var self = this;
+
+	Steppy(
+		function() {
+			var findParams = {start: {}, limit: params.limit};
+
+			// such condition for match one of projections:
+			// projectName, descCreateDate
+			// projectName, status, descCreateDate
+			// or just descCreateDate projection if project name is not set
+			if (params.projectName) {
+				findParams.start.projectName = params.projectName;
+				if (params.status) findParams.start.status = params.status;
+			}
+
+			findParams.start.descCreateDate = '';
+
+			self.db.builds.find(findParams, this.slot());
+		},
+		callback
+	);
+};
+
+/**
+ * Get info about current done builds streak.
+ * - `params.projectName` - optional project filter
+ *
+ * @param {Object} params
+ * @param {Function} callback(err,doneStreak)
+ */
+BuildsCollection.prototype.getDoneStreak = function(params, callback) {
+	var self = this;
+
+	Steppy(
+		function() {
+			var start = {};
+
+			if (params.projectName) start.projectName = params.projectName;
+
+			start.descCreateDate = '';
+
+			// tricky but effective streak counting inside filter goes below
+			var doneBuildsStreakCallback = _(this.slot()).once(),
+				doneBuildsStreak = {buildsCount: 0};
+
+			self.db.builds.find({
+				start: start,
+				filter: function(build) {
+					// error exits streak
+					if (build.status === 'error') {
+						doneBuildsStreakCallback(null, doneBuildsStreak);
+						return true;
+					}
+					if (build.status === 'done') {
+						doneBuildsStreak.buildsCount++;
+					}
+				},
+				limit: 1
+			}, function(err) {
+				doneBuildsStreakCallback(err, doneBuildsStreak);
+			});
+		},
+		callback
+	);
+};

lib/distributor.js

@@ -27,6 +27,9 @@ function Distributor(params) {
 	};
 
 	self.projects = params.projects;
+
+	self.buildLogLineNumbersHash = {},
+	self.lastLinesHash = {};
 }
 
 inherits(Distributor, EventEmitter);
@@ -101,7 +104,7 @@ Distributor.prototype._runNext = function(callback) {
 			});
 
 			executor.on('data', function(data) {
-				self.emit('buildData', build, data);
+				self._onBuildData(build, data);
 			});
 
 			executor.once('scmData', function(scmData) {
@@ -115,7 +118,7 @@ Distributor.prototype._runNext = function(callback) {
 							id: build.id,
 							number: build.number,
 							project: {name: build.project.name}
-						},
+						}
 					});
 				}
 			});
@@ -127,6 +130,57 @@ Distributor.prototype._runNext = function(callback) {
 	);
 };
 
+Distributor.prototype._onBuildData = function(build, data) {
+	var self = this;
+
+	self.emit('buildData', build, data);
+
+	var cleanupText = function(text) {
+		return text.replace('\r', '');
+	};
+
+	var splittedData  = data.split('\n'),
+		logLineNumber = self.buildLogLineNumbersHash[build.id] || 0;
+
+	self.lastLinesHash[build.id] = self.lastLinesHash[build.id] || '';
+
+	// if we don't have last line, so we start new line
+	if (!self.lastLinesHash[build.id]) {
+		logLineNumber++;
+	}
+	self.lastLinesHash[build.id] += _(splittedData).first();
+
+	var lines = [{
+		text: cleanupText(self.lastLinesHash[build.id]),
+		buildId: build.id,
+		number: logLineNumber
+	}];
+
+	if (splittedData.length > 1) {
+		// if we have last '' we have to take all except last
+		// this shown that string ends with eol
+		if (_(splittedData).last() === '') {
+			self.lastLinesHash[build.id] = '';
+			splittedData = _(splittedData.slice(1)).initial();
+		} else {
+			self.lastLinesHash[build.id] = _(splittedData).last();
+			splittedData = _(splittedData).tail();
+		}
+
+		lines = lines.concat(_(splittedData).map(function(line) {
+			return {
+				text: cleanupText(line),
+				buildId: build.id,
+				number: ++logLineNumber
+			};
+		}));
+	}
+
+	self.buildLogLineNumbersHash[build.id] = logLineNumber;
+
+	self.emit('buildLogLines', build, lines);
+};
+
 Distributor.prototype._updateWaitReasons = function() {
 	var self = this;
 	_(self.queue).each(function(item) {
@@ -164,7 +218,7 @@ Distributor.prototype._onBuildComplete = function(build, callback) {
 								id: build.id,
 								number: build.number,
 								project: {name: build.project.name}
-							},
+							}
 						}, triggerAfterGroup.slot());
 					}
 				});
@@ -232,17 +286,21 @@ Distributor.prototype._updateBuild = function(build, changes, callback) {
 	);
 };
 
-Distributor.prototype.cancel = function(params, callback) {
+Distributor.prototype.cancel = function(id, callback) {
+	callback = callback || function(err) {
+		if (err) logger.error('Error during cancel: ', err.stack || err);
+	};
 	var self = this;
+
 	Steppy(
 		function() {
 			var queueItemIndex = _(self.queue).findIndex(function(item) {
-				return item.build.id === params.buildId;
+				return item.build.id === id;
 			});
 
 			if (queueItemIndex === -1) {
 				throw new Error(
-					'Build with id "' + params.buildId + '" not found for cancel'
+					'Build with id "' + id + '" not found for cancel'
 				);
 			}
 
@@ -263,19 +321,15 @@ Distributor.prototype.cancel = function(params, callback) {
 };
 
 Distributor.prototype.run = function(params, callback) {
+	callback = callback || function(err) {
+		if (err) logger.error('Error during run: ', err.stack || err);
+	};
 	var self = this,
 		project;
-	callback = callback || function(err) {
+
-		if (err) {
-			logger.error('Error during run: ', err.stack || err);
-		}
-	};
 	Steppy(
 		function() {
-			project = _(self.projects).chain()
+			project = _(self.projects.get(params.projectName)).clone();
-				.findWhere({name: params.projectName})
-				.clone()
-				.value();
 
 			if (params.withScmChangesOnly) {
 				self.nodes[0].hasScmChanges(project, this.slot());
@@ -292,7 +346,7 @@ Distributor.prototype.run = function(params, callback) {
 				return callback();
 			}
 
-			if (params.skipQueued) {
+			if (!params.queueQueued) {
 				var queuedItem = _(self.queue).find(function(item) {
 					return item.project.name === project.name;
 				});

lib/executor/base.js

@@ -17,7 +17,7 @@ inherits(Executor, EventEmitter);
 
 Executor.prototype.throttledEmit = _(function() {
 	this.emit.apply(this, arguments);
-}).throttle(1500);
+}).throttle(500);
 
 Executor.prototype._getSources = function(params, callback) {
 };

lib/project.js

@@ -4,65 +4,92 @@ var Steppy = require('twostep').Steppy,
 	fs = require('fs'),
 	path = require('path'),
 	_ = require('underscore'),
-	reader = require('./reader'),
-	db = require('../db'),
 	utils = require('./utils'),
-	SpawnCommand = require('./command/spawn').Command;
+	SpawnCommand = require('./command/spawn').Command,
-
+	validateParams = require('./validateParams'),
+	EventEmitter = require('events').EventEmitter,
+	inherits = require('util').inherits;
 
 /**
- * Validates and returns given `config` to the `callback`(err, config)
+ * Projects collection contains all currently loaded projects and provides
+ * operations for manipulating with them.
+ * All projects stored on disk in `baseDir` and loaded to memory so
+ * they can be received (by `get`, `getAll` and other methods) in a sync way.
+ * Note that id for the particular project is a `name` of that project.
  */
-exports.validateConfig = function(config, callback) {
+function ProjectsCollection(params) {
-	callback(null, config);
+	this.db = params.db;
-};
+	this.reader = params.reader;
+	this.baseDir = params.baseDir;
+	this.configs = [];
+}
+
+exports.ProjectsCollection = ProjectsCollection;
+
+inherits(ProjectsCollection, EventEmitter);
 
 /**
- * Loads and returns project
+ * Validate and return given config.
+ *
+ * @param {Object} config
+ * @param {Function} callback(err,config)
  */
-exports.load = function(baseDir, name, callback) {
+ProjectsCollection.prototype.validateConfig = function(config, callback) {
-	var dir = path.join(baseDir, name);
 	Steppy(
 		function() {
-			fs.readdir(dir, this.slot());
+			validateParams(config, {
+				type: 'object',
+				properties: {
+					scm: {
+						type: 'object',
+						required: true,
+						properties: {
+							type: {enum: ['git', 'mercurial'], required: true},
+							repository: {type: 'string', required: true},
+							rev: {type: 'string', required: true}
+						}
 					},
-		function(err, dirContent) {
+					steps: {
-			exports.loadConfig(dir, this.slot());
+						type: 'array',
+						required: true,
+						items: {
+							type: 'object',
+							properties: {
+								cmd: {type: 'string', required: true},
+								name: {type: 'string'},
+								type: {enum: ['shell']},
+								shell: {type: 'string'}
+							}
+						}
+					}
 				},
-		function(err, config) {
+				additionalProperties: true
-			exports.validateConfig(config, this.slot());
-		},
-		function(err, config) {
-			config.name = name;
-			config.dir = dir;
-			this.pass(config);
-		},
-		callback
-	);
-};
-
-/**
- * Loads all projects from `baseDir` and returns array of projects
- */
-exports.loadAll = function(baseDir, callback) {
-	Steppy(
-		function() {
-			fs.readdir(baseDir, this.slot());
-		},
-		function(err, dirs) {
-			var loadGroup = this.makeGroup();
-			_(dirs).each(function(dir) {
-				exports.load(baseDir, dir, loadGroup.slot());
 			});
+
+			this.pass(null);
 		},
-		callback
+		function(err) {
+			if (err) {
+				err.message = (
+					'Error during validation of project "' + config.name +
+					'": ' + err.message
+				);
+			}
+			callback(err, config);
+		}
 	);
 };
 
-exports.loadConfig = function(dir, callback) {
+ProjectsCollection.prototype._getProjectPath = function(name) {
+	return path.join(this.baseDir, name);
+};
+
+ProjectsCollection.prototype._loadConfig = function(dir, callback) {
+	var self = this;
+
 	Steppy(
 		function() {
-			reader.load(dir, 'config', this.slot());
+			self.reader.load(dir, 'config', this.slot());
 		},
 		function(err, config) {
 			// convert steps object to array
@@ -82,7 +109,7 @@ exports.loadConfig = function(dir, callback) {
 			// apply defaults
 			_(config.steps).each(function(step) {
 				if (!step.type) step.type = 'shell';
-				if (!step.name) step.name = utils.prune(step.cmd, 40);
+				if (!step.name && step.cmd) step.name = utils.prune(step.cmd, 40);
 			});
 
 			this.pass(config);
@@ -91,73 +118,158 @@ exports.loadConfig = function(dir, callback) {
 	);
 };
 
-exports.saveConfig = function(config, dir, callback) {
+/**
-	fs.writeFile(
+ * Load project to collection.
-		path.join(dir, 'config.json'),
+ * `projectLoaded` event with loaded config as argument will be emitted after
-		JSON.stringify(config, null, 4),
+ * load.
-		callback
+ *
-	);
+ * @param {String} name
-};
+ * @param {Function} [callback(err)]
+ */
+ProjectsCollection.prototype.load = function(name, callback) {
+	callback = callback || _.noop;
+	var self = this,
+		dir = self._getProjectPath(name);
 
-exports.create = function(baseDir, config, callback) {
-	var dir;
 	Steppy(
 		function() {
-			dir = path.join(baseDir, config.name);
+			if (self.get(name)) {
-			fs.mkdir(dir, this.slot());
+				throw new Error(
+					'Can`t load already loaded project "' + name + '"'
+				);
+			}
+
+			self._loadConfig(dir, this.slot());
 		},
-		function(err) {
+		function(err, config) {
-			exports.saveConfig(config, baseDir, this.slot());
+			config.name = name;
+			config.dir = dir;
+
+			self.validateConfig(config, this.slot());
 		},
-		function(err) {
+		function(err, config) {
-			exports.load(dir, this.slot());
+			self.configs.push(config);
+			self.emit('projectLoaded', config);
+			this.pass(null);
 		},
 		callback
 	);
 };
 
-exports.getAvgProjectBuildDuration = function(projectName, callback) {
+/**
+ * Load all projects (from `this.baseDir`).
+ * Calls `load` for every project in a base dir.
+ *
+ * @param {Function} [callback(err)]
+ */
+ProjectsCollection.prototype.loadAll = function(callback) {
+	callback = callback || _.noop;
+	var self = this;
+
 	Steppy(
 		function() {
-			// get last done builds to calc avg build time
+			fs.readdir(self.baseDir, this.slot());
-			db.builds.find({
-				start: {
-					projectName: projectName,
-					status: 'done',
-					descCreateDate: ''
 		},
-				limit: 10
+		function(err, dirs) {
-			}, this.slot());
+			var loadGroup = this.makeGroup();
-		},
+			_(dirs).each(function(dir) {
-		function(err, doneBuilds) {
+				self.load(dir, loadGroup.slot());
-			var durationsSum = _(doneBuilds).reduce(function(memo, build) {
+			});
-				return memo + (build.endDate - build.startDate);
-			}, 0);
-
-			this.pass(Math.round(durationsSum / doneBuilds.length));
 		},
 		callback
 	);
 };
 
-exports.remove = function(params, callback) {
+/**
+ * Unload project from collection
+ * `projectUnloaded` event with unloaded config as argument will be emitted
+ * after unload.
+ *
+ * @param {String} name
+ * @param {Function} [callback(err)]
+ */
+ProjectsCollection.prototype.unload = function(name, callback) {
+	callback = callback || _.noop;
+	var self = this;
+
 	Steppy(
 		function() {
-			db.builds.find({
+			var index = _(self.configs).findIndex(function(config) {
-				start: {projectName: params.name, descCreateDate: ''}
+				return config.name === name;
+			});
+
+			if (index === -1) {
+				throw new Error('Can`t unload not loaded project: "' + name + '"');
+			}
+
+			var unloadedConfig = self.configs.splice(index, 1)[0];
+			self.emit('projectUnloaded', unloadedConfig);
+
+			this.pass(null);
+		},
+		callback
+	);
+};
+
+/**
+ * Get project config by name.
+ * Returns config object or undefined if project is not found.
+ *
+ * @param {String} name
+ */
+ProjectsCollection.prototype.get = function(name) {
+	return _(this.configs).findWhere({name: name});
+};
+
+/**
+ * Get configs for all currently loaded projects.
+ * Returns array of config objects.
+ */
+ProjectsCollection.prototype.getAll = function() {
+	return this.configs;
+};
+
+/**
+ * Get project configs which match to predicate.
+ * Returns array of config objects or empty array if there is no matched
+ * project.
+ *
+ * @param {Function} predicate
+ */
+ProjectsCollection.prototype.filter = function(predicate) {
+	return _(this.configs).filter(predicate);
+};
+
+/**
+ * Remove project by name.
+ * Calls `unload`, removes project from disk and db.
+ *
+ * @param {String} name
+ * @param {Function} [callback(err)]
+ */
+ProjectsCollection.prototype.remove = function(name, callback) {
+	callback = callback || _.noop;
+	var self = this;
+
+	Steppy(
+		function() {
+			self.db.builds.find({
+				start: {projectName: name, descCreateDate: ''}
 			}, this.slot());
 
 			new SpawnCommand().run({cmd: 'rm', args: [
-				'-Rf', path.join(params.baseDir, params.name)
+				'-Rf', self._getProjectPath(name)
 			]}, this.slot());
+
+			self.unload(name, this.slot());
 		},
 		function(err, builds) {
 			if (builds.length) {
-				db.builds.del(builds, this.slot());
+				self.db.builds.del(builds, this.slot());
 
 				var logLinesRemoveGroup = this.makeGroup();
 				_(builds).each(function(build) {
-					db.logLines.remove({
+					self.db.logLines.remove({
 						start: {buildId: build.id}
 					}, logLinesRemoveGroup.slot());
 				});
@@ -169,24 +281,40 @@ exports.remove = function(params, callback) {
 	);
 };
 
-exports.rename = function(params, callback) {
+/**
+ * Rename project.
+ * Renames project on disk and db, also changes name for loaded project.
+ *
+ * @param {String} name
+ * @param {Function} [callback(err)]
+ */
+ProjectsCollection.prototype.rename = function(name, newName, callback) {
+	callback = callback || _.noop;
+	var self = this;
+
 	Steppy(
 		function() {
 			fs.rename(
-				path.join(params.baseDir, params.name),
+				self._getProjectPath(name),
-				path.join(params.baseDir, params.newName),
+				self._getProjectPath(newName),
 				this.slot()
 			);
 
-			db.builds.multiUpdate(
+			self.db.builds.multiUpdate(
-				{start: {projectName: params.name, descCreateDate: ''}},
+				{start: {projectName: name, descCreateDate: ''}},
 				function(build) {
-					build.project.name = params.newName;
+					build.project.name = newName;
 					return build;
 				},
 				this.slot()
 			);
 		},
+		function() {
+			// just update currently loaded project name by link
+			self.get(name).name = newName;
+
+			this.pass(null);
+		},
 		callback
 	);
 };

lib/scm/git.js

@@ -15,9 +15,8 @@ inherits(Scm, ParentScm);
 
 Scm.prototype.defaultRev = 'master';
 
-// use 2 invisible separators as fields separator
+Scm.prototype._fieldsSeparator = '\u2063' + '\u2063';
-Scm.prototype._fieldsSeparator = String.fromCharCode(2063);
+Scm.prototype._linesSeparator = '\u2028' + '\u2028';
-Scm.prototype._fieldsSeparator += Scm.prototype._fieldsSeparator;
 
 Scm.prototype._revTemplate = [
 	'%h', '%cn', '%cd', '%s', '%d'
@@ -149,15 +148,16 @@ Scm.prototype.getChanges = function(rev1, rev2, callback) {
 
 			self.run({cmd: 'git', args: [
 				'log', rev1 ? rev1 + '..' + rev2 : rev2,
-				'--pretty=' + self._revTemplate
+				'--pretty=' + self._revTemplate + self._linesSeparator
 			]}, this.slot());
 		},
 		function(err, currentRev, stdout) {
 			// always skip last line - it's empty
-			var rows = stdout.split('\n').slice(0, -1);
+			var rows = stdout.split(self._linesSeparator).slice(0, -1);
 
 			var changes = _(rows).map(function(str) {
-				return self._parseRev(str);
+				// remove line break which git log add between commits
+				return self._parseRev(str.replace(/^\n/, ''));
 			});
 
 			this.pass(currentRev, changes);

lib/validateConfig.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var Steppy = require('twostep').Steppy,
+	validateParams = require('./validateParams');
+
+module.exports = function(config, callback) {
+	Steppy(
+		function() {
+			validateParams(config, {
+				type: 'object',
+				properties: {
+					plugins: {
+						type: 'array',
+						items: {type: 'string'}
+					},
+					nodes: {
+						type: 'array',
+						required: true,
+						items: {
+							type: 'object',
+							properties: {
+								type: {type: 'string', enum: ['local']},
+								maxExecutorsCount: {type: 'integer'}
+							}
+						},
+						minItems: 1
+					},
+					storage: {
+						type: 'object',						
+						required: true,
+						properties: {
+							backend: {type: 'string', required: true}
+						}
+					},
+					notify: {
+						type: 'object'
+					}
+				},
+				additionalProperties: true
+			});
+
+			this.pass(null);
+		},
+		function(err) {
+			if (err) {
+				err.message = (
+					'Error during validation server config: "' + err.message
+				);
+			}
+			callback(err, config);
+		}
+	);
+};

lib/validateParams.js

@@ -0,0 +1,19 @@
+'use strict';
+
+var conform = require('conform'),
+	_ = require('underscore');
+
+module.exports = function(params, schema, validateOptions) {
+	var defaultValidateOptions = {
+		additionalProperties: false,
+		failOnFirstError: true,
+		cast: true,
+		castSource: true,
+		applyDefaultValue: true
+	};
+
+	conform.validate(params, schema, _({}).extend(
+		defaultValidateOptions, validateOptions
+	));
+	return params;
+};

package.json

@@ -1,13 +1,13 @@
 {
   "name": "nci",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Continuous integration server written in node.js",
   "bin": {
     "nci": "bin/nci"
   },
   "scripts": {
     "makeTestRepos": "rm -rf test/repos/{mercurial,git}; cd test/repos/ && tar -xf mercurial.tar.gz && tar -xf git.tar.gz",
-    "test": "npm run makeTestRepos && mocha --bail --reporter=spec --timeout 4000",
+    "test": "npm run makeTestRepos && mocha --bail --reporter=spec --timeout 10000",
     "build-less": "lessc app/styles/index.less > static/css/index.css",
     "build-js": "browserify  app/app.js -t ./transforms/jade.js | uglifyjs -mc > static/js/app.build.js",
     "watch-less": "chokidar --initial --silent app/styles/**/*.less app/styles/index.less -c 'npm run build-less'",
@@ -15,6 +15,9 @@
     "watch-server": "nodemon --ignore app --ignore static --ignore node_modules --ignore data app.js",
     "dev": "parallelshell 'npm run build-fonts' 'npm run watch-less' 'npm run watch-js' 'npm run watch-server'",
     "sync": "npm install && npm prune",
+    "docProjectsCollection": "dox --api --skipSingleStar < lib/project.js | sed '/^  - \\[ProjectsCollection/ d' > docs/developing-plugins/projects-collection.md",
+    "docBuildsCollection": "dox --api --skipSingleStar < lib/build.js | sed '/^  - \\[BuildsCollection/ d' > docs/developing-plugins/builds-collection.md",
+    "doc": "nrun docProjectsCollection && nrun docBuildsCollection",
     "build-fonts": "cp ./node_modules/bootstrap/fonts/* ./static/fonts/ & cp ./node_modules/font-awesome/fonts/* ./static/fonts/",
     "build-clean": "rm static/index.html",
     "build-html": "jade views/index.jade --obj '{\"env\": \"production\"}' -o static/",
@@ -52,6 +55,7 @@
     "browserify": "12.0.1",
     "chokidar": "1.0.3",
     "colors": "1.1.2",
+    "conform": "0.2.12",
     "cron": "1.0.9",
     "data.io": "0.3.0",
     "font-awesome": "4.5.0",
@@ -71,6 +75,7 @@
   },
   "devDependencies": {
     "chokidar-cli": "1.2.0",
+    "dox": "0.8.0",
     "expect.js": "0.3.1",
     "jade": "1.11.0",
     "jshint": "2.9.1-rc1",

projectsWatcher.js

@@ -2,45 +2,37 @@
 
 var _ = require('underscore'),
 	path = require('path'),
-	chokidar = require('chokidar'),
+	chokidar = require('chokidar');
-	project = require('./lib/project'),
-	logger = require('./lib/logger')('projects watcher');
 
 exports.init = function(app, callback) {
+	var logger = app.lib.logger('projects watcher');
+
 	// start file watcher for reloading projects on change
 	var syncProject = function(filename, fileInfo) {
-		var baseDir = app.config.paths.projects,
+		var projectName = path.relative(
-			projectName = path.relative(
+			app.config.paths.projects,
-				baseDir,
 			path.dirname(filename)
 		);
 
-		var projectIndex = _(app.projects).findIndex(function(project) {
+		if (app.projects.get(projectName)) {
-			return project.name === projectName;
-		});
-
-		if (projectIndex !== -1) {
 			logger.log('Unload project: "' + projectName + '"');
-			var unloadedProject = app.projects.splice(projectIndex, 1)[0];
+			app.projects.unload(projectName);
-			app.emit('projectUnloaded', unloadedProject);
 		}
 
 		// on add or change (info is falsy on unlink)
 		if (fileInfo) {
 			logger.log('Load project "' + projectName + '" on change');
-			project.load(baseDir, projectName, function(err, project) {
+			app.projects.load(projectName, function(err) {
 				if (err) {
 					return logger.error(
 						'Error during load project "' + projectName + '": ',
 						err.stack || err
 					);
 				}
-				app.projects.push(project);
 				logger.log(
 					'Project "' + projectName + '" loaded:',
-					JSON.stringify(project, null, 4)
+					JSON.stringify(app.projects.get(projectName), null, 4)
 				);
-				app.emit('projectLoaded', project);
 			});
 		}
 	};

resources/builds.js

@@ -2,31 +2,22 @@
 
 var Steppy = require('twostep').Steppy,
 	_ = require('underscore'),
-	db = require('../db'),
-	utils = require('../lib/utils'),
 	logger = require('../lib/logger')('builds resource');
 
 module.exports = function(app) {
-	var resource = app.dataio.resource('builds'),
+	var resource = app.dataio.resource('builds');
-		distributor = app.distributor;
 
 	resource.use('readAll', function(req, res, next) {
 		Steppy(
 			function() {
-				var data = req.data || {};
+				var data = req.data || {},
+					getParams = {limit: data.limit || 20};
 
-				var start = {};
 				if (data.projectName) {
-					start.projectName = data.projectName;
+					getParams.projectName = data.projectName;
 				}
 
-				start.descCreateDate = data.descCreateDate || '';
+				app.builds.getRecent(getParams, this.slot());
-
-				var findParams = _(data).pick('offset', 'limit');
-				findParams.start = start;
-				findParams.limit = findParams.limit || 20;
-
-				db.builds.find(findParams, this.slot());
 			},
 			function(err, builds) {
 				// omit big fields not needed for list
@@ -49,12 +40,10 @@ module.exports = function(app) {
 	resource.use('read', function(req, res, next) {
 		Steppy(
 			function() {
-				var findParams = {};
+				app.builds.get(req.data.id, this.slot());
-				findParams.start = _(req.data).pick('id');
-				db.builds.find(findParams, this.slot());
 			},
 			function(err, build) {
-				res.send(build[0]);
+				res.send(build);
 			},
 			next
 		);
@@ -63,19 +52,13 @@ module.exports = function(app) {
 	resource.use('getBuildLogTail', function(req, res, next) {
 		Steppy(
 			function() {
-				var findParams = {
+				app.builds.getLogLinesTail({
-					reverse: true,
+					buildId: req.data.buildId,
-					start: {buildId: req.data.buildId},
 					limit: req.data.length
-				};
+				}, this.slot());
-
-				db.logLines.find(findParams, this.slot());
 			},
-			function(err, logLines) {
+			function(err, tail) {
-				var lines = logLines.reverse(),
+				res.send(tail);
-					total = logLines.length ? logLines[0].number : 0;
-
-				res.send({lines: lines, total: total});
 			},
 			next
 		);
@@ -84,23 +67,13 @@ module.exports = function(app) {
 	resource.use('getBuildLogLines', function(req, res, next) {
 		Steppy(
 			function() {
-				var buildId = req.data.buildId,
+				app.builds.getLogLines(
-					from = req.data.from,
+					_(req.data).pick('buildId', 'from', 'to'),
-					to = req.data.to,
+					this.slot()
-					count = to - from;
+				);
-
-				db.logLines.find({
-					start: {buildId: buildId, number: from},
-					end: {buildId: buildId, number: to}
-				}, this.slot());
-
-				this.pass(count);
 			},
-			function(err, logLines, count) {
+			function(err, logLinesData) {
-				res.send({
+				res.send(logLinesData);
-					lines: logLines,
-					isLast: logLines.length < count
-				});
 			},
 			next
 		);
@@ -111,7 +84,7 @@ module.exports = function(app) {
 			function() {
 				var buildId = req.data.buildId;
 				logger.log('Cancel build: "%s"', buildId);
-				distributor.cancel({buildId: buildId}, this.slot());
+				app.builds.cancel(buildId, this.slot());
 			},
 			function() {
 				res.send();

resources/helpers.js

@@ -0,0 +1,37 @@
+'use strict';
+
+var Steppy = require('twostep').Steppy,
+	logger = require('../lib/logger')('create build resource');
+
+var buildDataResourcesHash = {};
+
+// create resource for build data
+exports.createBuildDataResource = function(app, buildId) {
+	if (buildId in buildDataResourcesHash) {
+		return;
+	}
+	var buildDataResource = app.dataio.resource('build' + buildId);
+	buildDataResource.on('connection', function(client) {
+		var callback = this.async();
+		Steppy(
+			function() {
+				app.builds.getLogLines({buildId: buildId}, this.slot());
+			},
+			function(err, logLinesData) {
+				client.emit('sync', 'data', {lines: logLinesData.lines});
+
+				this.pass(null);
+			},
+			function(err) {
+				if (err) {
+					logger.error(
+						'error during read log for "' + buildId + '":',
+						err.stack || err
+					);
+				}
+				callback();
+			}
+		);
+	});
+	buildDataResourcesHash[buildId] = buildDataResource;
+};

resources/index.js

@@ -1,11 +1,42 @@
 'use strict';
 
 var _ = require('underscore'),
-	errorHandler = require('./errorHandler');
+	errorHandler = require('./errorHandler'),
+	helpers = require('./helpers');
 
 module.exports = function(app) {
 	_(['builds', 'projects']).each(function(resource) {
 		var resource = require('./' + resource)(app);
 		resource.use(errorHandler);
 	});
+
+	var buildsResource = app.dataio.resource('builds');
+
+	app.builds.on('buildUpdated', function(build, changes) {
+		if (build.status === 'queued') {
+			helpers.createBuildDataResource(app, build.id);
+		}
+
+		// notify about build's project change, coz building affects project
+		// related stat (last build date, avg build time, etc)
+		if (changes.completed) {
+			var projectsResource = app.dataio.resource('projects');
+			projectsResource.clientEmitSyncChange(build.project.name);
+		}
+
+		buildsResource.clientEmitSync('change', {
+			buildId: build.id, changes: changes
+		});
+	});
+
+	app.builds.on('buildCanceled', function(build) {
+		buildsResource.clientEmitSync('cancel', {buildId: build.id});
+	});
+
+	app.builds.on('buildLogLines', function(build, lines) {
+		app.dataio.resource('build' + build.id).clientEmitSync(
+			'data',
+			{lines: lines}
+		);
+	});
 };

resources/projects.js

@@ -2,80 +2,50 @@
 
 var Steppy = require('twostep').Steppy,
 	_ = require('underscore'),
-	getAvgProjectBuildDuration =
+	helpers = require('./helpers'),
-		require('../lib/project').getAvgProjectBuildDuration,
+	logger = require('../lib/logger')('projects resource');
-	createBuildDataResource = require('../distributor').createBuildDataResource,
-	logger = require('../lib/logger')('projects resource'),
-	db = require('../db');
 
 module.exports = function(app) {
-
+	var resource = app.dataio.resource('projects');
-	var resource = app.dataio.resource('projects'),
-		distributor = app.distributor;
 
 	resource.use('createBuildDataResource', function(req, res) {
-		createBuildDataResource(req.data.buildId);
+		helpers.createBuildDataResource(app, req.data.buildId);
 		res.send();
 	});
 
 	resource.use('readAll', function(req, res) {
-		var filteredProjects = app.projects,
+		var filteredProjects = app.projects.getAll(),
 			nameQuery = req.data && req.data.nameQuery;
 
 		if (nameQuery) {
-			filteredProjects = _(filteredProjects).filter(function(project) {
+			filteredProjects = app.projects.filter(function(project) {
 				return project.name.indexOf(nameQuery) !== -1;
 			});
 		}
 
+		filteredProjects = _(filteredProjects).sortBy('name');
+
 		res.send(filteredProjects);
 	});
 
-	var getProject = function(params, callback) {
+	// get project with additional fields
+	var getProject = function(name, callback) {
 		var project;
 		Steppy(
 			function() {
-				project = _(app.projects).findWhere(params.condition);
+				project = _(app.projects.get(name)).clone();
 
-				getAvgProjectBuildDuration(project.name, this.slot());
+				app.builds.getRecent({
-
-				// get last done build
-				db.builds.find({
-					start: {
 					projectName: project.name,
 					status: 'done',
-						descCreateDate: ''
+					limit: 10
-					},
-					limit: 1
 				}, this.slot());
 
-				// tricky but effective streak counting inside filter goes below
+				app.builds.getDoneStreak({projectName: project.name}, this.slot());
-				var doneBuildsStreakCallback = _(this.slot()).once(),
-					doneBuildsStreak = 0;
-
-				db.builds.find({
-					start: {
-						projectName: project.name,
-						descCreateDate: ''
 			},
-					filter: function(build) {
+			function(err, doneBuilds, doneBuildsStreak) {
-						// error exits streak
+				project.avgBuildDuration = app.builds.getAvgBuildDuration(doneBuilds);
-						if (build.status === 'error') {
+				project.lastDoneBuild = doneBuilds[0];
-							doneBuildsStreakCallback(null, doneBuildsStreak);
-							return true;
-						}
-						if (build.status === 'done') {
-							doneBuildsStreak++;
-						}
-					},
-					limit: 1
-				}, function(err) {
-					doneBuildsStreakCallback(err, doneBuildsStreak);
-				});
-			},
-			function(err, avgProjectBuildDuration, lastDoneBuilds, doneBuildsStreak) {
-				project.lastDoneBuild = lastDoneBuilds[0];
-				project.avgBuildDuration = avgProjectBuildDuration;
 				project.doneBuildsStreak = doneBuildsStreak;
 
 				this.pass(project);
@@ -84,12 +54,12 @@ module.exports = function(app) {
 		);
 	};
 
-	// resource custom method which finds project by condition
+	// resource custom method which finds project by name
 	// and emits event about it change to clients
-	resource.clientEmitSyncChange = function(condition) {
+	resource.clientEmitSyncChange = function(name) {
 		Steppy(
 			function() {
-				getProject({condition: condition}, this.slot());
+				getProject(name, this.slot());
 			},
 			function(err, project) {
 				resource.clientEmitSync('change', {project: project});
@@ -106,7 +76,7 @@ module.exports = function(app) {
 	resource.use('read', function(req, res) {
 		Steppy(
 			function() {
-				getProject({condition: req.data}, this.slot());
+				getProject(req.data.name, this.slot());
 			},
 			function(err, project) {
 				res.send(project);
@@ -117,9 +87,10 @@ module.exports = function(app) {
 	resource.use('run', function(req, res) {
 		var projectName = req.data.projectName;
 		logger.log('Run the project: "%s"', projectName);
-		distributor.run({
+		app.builds.create({
 			projectName: projectName,
-			initiator: {type: 'user'}
+			initiator: {type: 'user'},
+			queueQueued: true
 		});
 		res.send();
 	});

scheduler.js

@@ -1,24 +1,27 @@
 'use strict';
 
 var _ = require('underscore'),
-	logger = require('./lib/logger')('scheduler'),
 	CronJob = require('cron').CronJob;
 
 exports.init = function(app, callback) {
 
-	var distributor = app.distributor,
+	var logger = app.lib.logger('scheduler'),
 		projectJobs = {};
 
-	app.on('projectLoaded', function(project) {
+	app.projects.on('projectLoaded', function(project) {
 		var time = project.buildEvery && project.buildEvery.time;
 		if (time) {
-			logger.log('Start job for loaded project "%s"', project.name);
+			logger.log(
+				'Start job for loaded project "%s" by schedule "%s"',
+				project.name,
+				time
+			);
 			projectJobs[project.name] = {};
 			projectJobs[project.name].job = new CronJob({
 				cronTime: time,
 				onTick: function() {
 					logger.log('Run project "%s"', project.name);
-					distributor.run({
+					app.builds.create({
 						projectName: project.name,
 						withScmChangesOnly: project.buildEvery.withScmChangesOnly,
 						initiator: {type: 'scheduler'}
@@ -29,7 +32,7 @@ exports.init = function(app, callback) {
 		}
 	});
 
-	app.on('projectUnloaded', function(project) {
+	app.projects.on('projectUnloaded', function(project) {
 		if (project.name in projectJobs) {
 			logger.log('Stop job for unloaded project "%s"', project.name);
 			projectJobs[project.name].job.stop();

test/distributor/blocking.js

@@ -4,6 +4,7 @@ var Distributor = require('../../lib/distributor').Distributor,
 	expect = require('expect.js'),
 	sinon = require('sinon'),
 	createNodeMock = require('./helpers').createNodeMock,
+	createProjectsMock = require('./helpers').createProjectsMock,
 	Steppy = require('twostep').Steppy;
 
 
@@ -119,11 +120,11 @@ describe('Distributor blocking with max 2 executors count', function() {
 
 	describe('should run 2 non-blocking projects in parallel', function() {
 		before(function() {
-			projects = [{
+			projects = createProjectsMock([{
 				name: 'project1',
 			}, {
 				name: 'project2'
-			}];
+			}]);
 		});
 
 		itRunParallelProjects();
@@ -131,12 +132,12 @@ describe('Distributor blocking with max 2 executors count', function() {
 
 	describe('should run project1, then 2, when 2 blocked by 1', function() {
 		before(function() {
-			projects = [{
+			projects = createProjectsMock([{
 				name: 'project1',
 			}, {
 				name: 'project2',
 				blockedBy: ['project1']
-			}];
+			}]);
 		});
 
 		itRunSequentialProjects();
@@ -144,12 +145,12 @@ describe('Distributor blocking with max 2 executors count', function() {
 
 	describe('should run project1, then 2, when 1 blocks 2', function() {
 		before(function() {
-			projects = [{
+			projects = createProjectsMock([{
 				name: 'project1',
 				blocks: ['project2']
 			}, {
 				name: 'project2'
-			}];
+			}]);
 		});
 
 		itRunSequentialProjects();
@@ -159,7 +160,7 @@ describe('Distributor blocking with max 2 executors count', function() {
 		'should run 1, 2 in parallel, when 1 block 3, 2 blocked by 3',
 		function() {
 			before(function() {
-				projects = [{
+				projects = createProjectsMock([{
 					name: 'project1',
 					blocks: ['project3']
 				}, {
@@ -167,7 +168,7 @@ describe('Distributor blocking with max 2 executors count', function() {
 					blockedBy: ['project3']
 				}, {
 					name: 'project3'
-				}];
+				}]);
 			});
 
 			itRunParallelProjects();

test/distributor/helpers.js

@@ -1,7 +1,8 @@
 'use strict';
 
 var Node = require('../../lib/node').Node,
-	EventEmitter = require('events').EventEmitter;
+	EventEmitter = require('events').EventEmitter,
+	ProjectsCollection = require('../../lib/project').ProjectsCollection;
 
 
 exports.createNodeMock = function(executorRun) {
@@ -17,3 +18,8 @@ exports.createNodeMock = function(executorRun) {
 	};
 };
 
+exports.createProjectsMock = function(configs) {
+	var projects = new ProjectsCollection({});
+	projects.configs = configs;
+	return projects;
+};

test/distributor/main.js

@@ -3,12 +3,13 @@
 var Distributor = require('../../lib/distributor').Distributor,
 	expect = require('expect.js'),
 	sinon = require('sinon'),
-	createNodeMock = require('./helpers').createNodeMock;
+	createNodeMock = require('./helpers').createNodeMock,
+	createProjectsMock = require('./helpers').createProjectsMock;
 
 
 describe('Distributor main', function() {
 	var distributor,
-		projects = [{name: 'project1'}];
+		projects = createProjectsMock([{name: 'project1'}]);
 
 	var expectUpdateBuild = function(distributor, build, number, conditionsHash) {
 		var conditions = conditionsHash[number];
@@ -154,7 +155,7 @@ describe('Distributor main', function() {
 
 				var originalRunNext = distributor._runNext;
 				distributor._runNext = function() {
-					distributor.cancel({buildId: 1}, function(err) {
+					distributor.cancel(1, function(err) {
 						cancelError = err;
 					});
 					originalRunNext.apply(distributor, arguments);
@@ -197,7 +198,7 @@ describe('Distributor main', function() {
 
 				var originalRunNext = distributor._runNext;
 				distributor._runNext = function() {
-					distributor.cancel({buildId: 2}, function(err) {
+					distributor.cancel(2, function(err) {
 						cancelError = err;
 					});
 					originalRunNext.apply(distributor, arguments);

test/distributor/runSelfAfterCatchRev.js

@@ -2,9 +2,9 @@ var Distributor = require('../../lib/distributor').Distributor,
 	expect = require('expect.js'),
 	sinon = require('sinon'),
 	helpers = require('../helpers'),
+	createProjectsMock = require('./helpers').createProjectsMock,
 	path = require('path');
 
-
 describe('Distributor run self after catch', function() {
 	var distributor, executorRunSpy, scmDataSpy;
 
@@ -17,7 +17,7 @@ describe('Distributor run self after catch', function() {
 			helpers.removeDirIfExists(workspacePath, done);
 
 			distributor = new Distributor({
-				projects: [{
+				projects: createProjectsMock([{
 					name: 'project1',
 					dir: __dirname,
 					scm: helpers.repository.scm,
@@ -25,7 +25,7 @@ describe('Distributor run self after catch', function() {
 						{type: 'shell', cmd: 'echo 1'}
 					],
 					catchRev: {comment: /.*/}
-				}],
+				}]),
 				nodes: nodes
 			});
 

test/distributor/triggerAfter.js

@@ -3,7 +3,8 @@
 var Distributor = require('../../lib/distributor').Distributor,
 	expect = require('expect.js'),
 	sinon = require('sinon'),
-	createNodeMock = require('./helpers').createNodeMock;
+	createNodeMock = require('./helpers').createNodeMock,
+	createProjectsMock = require('./helpers').createProjectsMock;
 
 
 describe('Distributor trigger after', function() {
@@ -13,14 +14,14 @@ describe('Distributor trigger after', function() {
 
 	describe('done when project is done', function() {
 		before(function() {
-			projects = [{
+			projects = createProjectsMock([{
 				name: 'project1',
 				trigger: {
 					after: [{status: 'done', project: 'project2'}]
 				}
 			}, {
 				name: 'project2'
-			}];
+			}]);
 			executorRunSpy = sinon.stub().callsArgAsync(1);
 			sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
 				executorRunSpy
@@ -39,11 +40,15 @@ describe('Distributor trigger after', function() {
 		});
 
 		it('should run project1 at first call', function() {
-			expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
+			expect(executorRunSpy.getCall(0).thisValue.project).eql(
+				projects.get('project1')
+			);
 		});
 
 		it('should run project2 at second call', function() {
-			expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
+			expect(executorRunSpy.getCall(1).thisValue.project).eql(
+				projects.get('project2')
+			);
 		});
 
 		it('should run totally 2 times', function() {
@@ -77,7 +82,9 @@ describe('Distributor trigger after', function() {
 		});
 
 		it('should run project1 at first call', function() {
-			expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
+			expect(executorRunSpy.getCall(0).thisValue.project).eql(
+				projects.get('project1')
+			);
 		});
 
 		it('should run totally 1 time', function() {
@@ -91,14 +98,14 @@ describe('Distributor trigger after', function() {
 
 	describe('status is not set when project is done', function() {
 		before(function() {
-			projects = [{
+			projects = createProjectsMock([{
 				name: 'project1',
 				trigger: {
 					after: [{project: 'project2'}]
 				}
 			}, {
 				name: 'project2'
-			}];
+			}]);
 			executorRunSpy = sinon.stub().callsArgAsync(1);
 			sinon.stub(Distributor.prototype, '_createNode', createNodeMock(
 				executorRunSpy
@@ -117,11 +124,15 @@ describe('Distributor trigger after', function() {
 		});
 
 		it('should run project1 at first call', function() {
-			expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
+			expect(executorRunSpy.getCall(0).thisValue.project).eql(
+				projects.get('project1')
+			);
 		});
 
 		it('should run project2 at second call', function() {
-			expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
+			expect(executorRunSpy.getCall(1).thisValue.project).eql(
+				projects.get('project2')
+			);
 		});
 
 		it('should run totally 2 times', function() {
@@ -155,11 +166,15 @@ describe('Distributor trigger after', function() {
 		});
 
 		it('should run project1 at first call', function() {
-			expect(executorRunSpy.getCall(0).thisValue.project).eql(projects[0]);
+			expect(executorRunSpy.getCall(0).thisValue.project).eql(
+				projects.get('project1')
+			);
 		});
 
 		it('should run project2 at second call', function() {
-			expect(executorRunSpy.getCall(1).thisValue.project).eql(projects[1]);
+			expect(executorRunSpy.getCall(1).thisValue.project).eql(
+				projects.get('project2')
+			);
 		});
 
 		it('should run totally 2 times', function() {