persistent in-memory builds using nlevel and memdown

db.js

@@ -0,0 +1,64 @@
+'use strict';
+
+var nlevel = require('nlevel'),
+	ldb = nlevel.db('path/to/db/ignored/for/memdown', {
+		db: require('memdown'),
+		valueEncoding: 'json'
+	});
+
+exports.builds = new nlevel.DocsSection(ldb, 'builds', {
+	projections: [
+		{key: {createDate: 1}, value: pickId},
+		{key: {descCreateDate: descCreateDate, id: 1}},
+		{key: {project: 1, descCreateDate: descCreateDate, id: 1}}
+	]
+});
+
+exports.builds.idGenerator = getNextId;
+
+// TODO: move to nlevel
+var superPut = nlevel.DocsSection.prototype.put;
+nlevel.DocsSection.prototype.put = function(docs, callback) {
+	var self = this;
+	if (!Array.isArray(docs)) docs = [docs];
+	if (this.idGenerator && docs[0] && 'id' in docs[0] === false) {
+		if (docs.every(function(doc) { return 'id' in doc === false; })) {
+			this.idGenerator(function(err, id) {
+				if (err) return callback(err);
+				docs.forEach(function(doc) {
+					doc.id = id;
+					id++;
+				});
+				superPut.call(self, docs, callback);
+			});
+		} else {
+			return callback(new Error(
+				'Documents with id and without should not be ' +
+				'mixed on put when id generator is set'
+			));
+		}
+	} else {
+		return superPut.call(this, docs, callback);
+	}
+};
+
+function getNextId(callback) {
+	this.find({
+		start: {createDate: ''}, limit: 1, reverse: true
+	}, function(err, docs) {
+		callback(err, !err && docs[0] && ++docs[0].id || 1);
+	});
+}
+
+function pickId(doc) {
+	return {id: doc.id};
+}
+
+// reversed date - for sorting forward (it's fatster for leveldb then
+// reverse: true, see levelup reverse notes for details) but have documents
+// sorted by some date in descending order
+var maxTime = new Date('03:14:07 UTC 2138-01-19').getTime();
+
+function descCreateDate(doc) {
+	return maxTime - doc.createDate;
+}

lib/distributor.js

@@ -99,6 +99,7 @@ Distributor.prototype.run = function(project, params, callback) {
 			self._updateBuild({
 				project: project,
 				params: params,
+				createDate: Date.now(),
 				status: 'queued'
 			}, this.slot());
 		},

package.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "data.io": "0.3.0",
     "jade": "1.9.2",
+    "nlevel": "1.0.0",
     "node-static": "0.7.6",
     "socket.io": "1.3.5",
     "twostep": "0.4.1",
@@ -41,6 +42,7 @@
     "gulp-less": "3.0.3",
     "gulp-nodemon": "2.0.3",
     "gulp-react-jade-amd": "git://github.com/vladimir-polyakov/gulp-react-jade-amd",
+    "memdown": "1.0.0",
     "mocha": "1.18.2",
     "nodemon": "1.3.7"
   }

resources/builds.js

@@ -1,7 +1,29 @@
 'use strict';
 
+var Steppy = require('twostep').Steppy,
+	_ = require('underscore'),
+	db = require('../db');
+
 module.exports = function(app) {
 	var resource = app.dataio.resource('builds');
 
+	resource.use('read', function(req, res) {
+		Steppy(
+			function() {
+				var findParams = _(req.data).pick('offset', 'limit');
+				findParams.limit = findParams.limit || 20;
+				findParams.start = {descCreateDate: ''};
+
+				db.builds.find(findParams, this.slot());
+			},
+			function(err, builds) {
+				res.send(builds);
+			},
+			function(err) {
+				console.log(err.stack || err)
+			}
+		);
+	});
+
 	return resource;
 };

resources/projects.js

@@ -1,8 +1,10 @@
 'use strict';
 
-var _ = require('underscore'),
+var Steppy = require('twostep').Steppy,
+	_ = require('underscore'),
 	project = require('../lib/project'),
-	Distributor = require('../lib/distributor').Distributor;
+	Distributor = require('../lib/distributor').Distributor,
+	db = require('../db');
 
 var projects, projectsHash;
 
@@ -19,25 +21,34 @@ project.loadAll('projects', function(err, loadedProjects) {
 });
 
 module.exports = function(app) {
-	var buildsSequnce = 0;
 
 	var distributor = new Distributor({
 		nodes: [{type: 'local', maxExecutorsCount: 1}],
 		onBuildUpdate: function(build, callback) {
-			var buildsResource = app.dataio.resource('builds');
+			Steppy(
-			if (build.status === 'queued') {
+				function() {
-				build.id = ++buildsSequnce;
+					db.builds.put(build, this.slot());
-				// create resource for build data
+				},
-				var buildDataResource = app.dataio.resource('build' + build.id);
+				function() {
-				buildDataResource.on('connection', function(client) {
+					var buildsResource = app.dataio.resource('builds');
-					client.emit('sync', 'data', '< collected data >');
+
-				});
+					if (build.status === 'queued') {
-			}
+						// create resource for build data
-			buildsResource.clientEmitSync(
+						var buildDataResource = app.dataio.resource('build' + build.id);
-				build.status === 'queued' ? 'create' : 'update',
+						buildDataResource.on('connection', function(client) {
-				build
+							client.emit('sync', 'data', '< collected data >');
+						});
+					}
+
+					buildsResource.clientEmitSync(
+						build.status === 'queued' ? 'create' : 'update',
+						build
+					);
+
+					this.pass(build);
+				},
+				callback
 			);
-			callback(null, build);
 		},
 		onBuildData: function(build, data) {
 			app.dataio.resource('build' + build.id).clientEmitSync('data', data);

static/js/app/app.js

@@ -2,13 +2,15 @@
 
 define([
 	'react', 'templates/app/index', 'app/components/index',
-	'app/actions/project'
+	'app/actions/project', 'app/actions/build'
 ], function(
-	React, template, Components, ProjectActions
+	React, template, Components,
+	ProjectActions, BuildActions
 ) {
 	React.render(template({
 		App: Components.App
 	}), document.getElementById('content'));
 
 	ProjectActions.readAll();
+	BuildActions.readAll();
 });

static/js/app/stores/builds.js

@@ -22,13 +22,14 @@ define([
 		},
 
 		init: function() {
-			resource.subscribe(this._onAction);
+			resource.subscribe('create', 'update', this._onAction);
 		},
 
 		onReadAll: function() {
 			var self = this;
 			resource.sync('read', function(err, builds) {
-				self.trigger(builds);
+				self.builds = builds;
+				self.trigger(self.builds);
 			});
 		}
 	});