From a0662ba631cbe3b71a030a0dfc16e01c3890a0bc Mon Sep 17 00:00:00 2001 From: oleg Date: Sat, 9 May 2015 22:53:19 +0300 Subject: [PATCH] persistent in-memory builds using nlevel and memdown --- db.js | 64 ++++++++++++++++++++++++++++++++++ lib/distributor.js | 1 + package.json | 2 ++ resources/builds.js | 22 ++++++++++++ resources/projects.js | 43 ++++++++++++++--------- static/js/app/app.js | 6 ++-- static/js/app/stores/builds.js | 5 +-- 7 files changed, 123 insertions(+), 20 deletions(-) create mode 100644 db.js diff --git a/db.js b/db.js new file mode 100644 index 0000000..e60f71a --- /dev/null +++ b/db.js @@ -0,0 +1,64 @@ +'use strict'; + +var nlevel = require('nlevel'), + ldb = nlevel.db('path/to/db/ignored/for/memdown', { + db: require('memdown'), + valueEncoding: 'json' + }); + +exports.builds = new nlevel.DocsSection(ldb, 'builds', { + projections: [ + {key: {createDate: 1}, value: pickId}, + {key: {descCreateDate: descCreateDate, id: 1}}, + {key: {project: 1, descCreateDate: descCreateDate, id: 1}} + ] +}); + +exports.builds.idGenerator = getNextId; + +// TODO: move to nlevel +var superPut = nlevel.DocsSection.prototype.put; +nlevel.DocsSection.prototype.put = function(docs, callback) { + var self = this; + if (!Array.isArray(docs)) docs = [docs]; + if (this.idGenerator && docs[0] && 'id' in docs[0] === false) { + if (docs.every(function(doc) { return 'id' in doc === false; })) { + this.idGenerator(function(err, id) { + if (err) return callback(err); + docs.forEach(function(doc) { + doc.id = id; + id++; + }); + superPut.call(self, docs, callback); + }); + } else { + return callback(new Error( + 'Documents with id and without should not be ' + + 'mixed on put when id generator is set' + )); + } + } else { + return superPut.call(this, docs, callback); + } +}; + +function getNextId(callback) { + this.find({ + start: {createDate: ''}, limit: 1, reverse: true + }, function(err, docs) { + callback(err, !err && docs[0] && ++docs[0].id || 1); + }); +} + +function pickId(doc) { + return {id: doc.id}; +} + +// reversed date - for sorting forward (it's fatster for leveldb then +// reverse: true, see levelup reverse notes for details) but have documents +// sorted by some date in descending order +var maxTime = new Date('03:14:07 UTC 2138-01-19').getTime(); + +function descCreateDate(doc) { + return maxTime - doc.createDate; +} diff --git a/lib/distributor.js b/lib/distributor.js index 579db10..a729c96 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -99,6 +99,7 @@ Distributor.prototype.run = function(project, params, callback) { self._updateBuild({ project: project, params: params, + createDate: Date.now(), status: 'queued' }, this.slot()); }, diff --git a/package.json b/package.json index aa0e846..f7536c9 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "dependencies": { "data.io": "0.3.0", "jade": "1.9.2", + "nlevel": "1.0.0", "node-static": "0.7.6", "socket.io": "1.3.5", "twostep": "0.4.1", @@ -41,6 +42,7 @@ "gulp-less": "3.0.3", "gulp-nodemon": "2.0.3", "gulp-react-jade-amd": "git://github.com/vladimir-polyakov/gulp-react-jade-amd", + "memdown": "1.0.0", "mocha": "1.18.2", "nodemon": "1.3.7" } diff --git a/resources/builds.js b/resources/builds.js index d1c626d..a91f8a9 100644 --- a/resources/builds.js +++ b/resources/builds.js @@ -1,7 +1,29 @@ 'use strict'; +var Steppy = require('twostep').Steppy, + _ = require('underscore'), + db = require('../db'); + module.exports = function(app) { var resource = app.dataio.resource('builds'); + resource.use('read', function(req, res) { + Steppy( + function() { + var findParams = _(req.data).pick('offset', 'limit'); + findParams.limit = findParams.limit || 20; + findParams.start = {descCreateDate: ''}; + + db.builds.find(findParams, this.slot()); + }, + function(err, builds) { + res.send(builds); + }, + function(err) { + console.log(err.stack || err) + } + ); + }); + return resource; }; diff --git a/resources/projects.js b/resources/projects.js index 43653c4..cccbfe8 100644 --- a/resources/projects.js +++ b/resources/projects.js @@ -1,8 +1,10 @@ 'use strict'; -var _ = require('underscore'), +var Steppy = require('twostep').Steppy, + _ = require('underscore'), project = require('../lib/project'), - Distributor = require('../lib/distributor').Distributor; + Distributor = require('../lib/distributor').Distributor, + db = require('../db'); var projects, projectsHash; @@ -19,25 +21,34 @@ project.loadAll('projects', function(err, loadedProjects) { }); module.exports = function(app) { - var buildsSequnce = 0; var distributor = new Distributor({ nodes: [{type: 'local', maxExecutorsCount: 1}], onBuildUpdate: function(build, callback) { - var buildsResource = app.dataio.resource('builds'); - if (build.status === 'queued') { - build.id = ++buildsSequnce; - // create resource for build data - var buildDataResource = app.dataio.resource('build' + build.id); - buildDataResource.on('connection', function(client) { - client.emit('sync', 'data', '< collected data >'); - }); - } - buildsResource.clientEmitSync( - build.status === 'queued' ? 'create' : 'update', - build + Steppy( + function() { + db.builds.put(build, this.slot()); + }, + function() { + var buildsResource = app.dataio.resource('builds'); + + if (build.status === 'queued') { + // create resource for build data + var buildDataResource = app.dataio.resource('build' + build.id); + buildDataResource.on('connection', function(client) { + client.emit('sync', 'data', '< collected data >'); + }); + } + + buildsResource.clientEmitSync( + build.status === 'queued' ? 'create' : 'update', + build + ); + + this.pass(build); + }, + callback ); - callback(null, build); }, onBuildData: function(build, data) { app.dataio.resource('build' + build.id).clientEmitSync('data', data); diff --git a/static/js/app/app.js b/static/js/app/app.js index 3bfc0c4..cd05d14 100644 --- a/static/js/app/app.js +++ b/static/js/app/app.js @@ -2,13 +2,15 @@ define([ 'react', 'templates/app/index', 'app/components/index', - 'app/actions/project' + 'app/actions/project', 'app/actions/build' ], function( - React, template, Components, ProjectActions + React, template, Components, + ProjectActions, BuildActions ) { React.render(template({ App: Components.App }), document.getElementById('content')); ProjectActions.readAll(); + BuildActions.readAll(); }); diff --git a/static/js/app/stores/builds.js b/static/js/app/stores/builds.js index 1c9d0fe..960e74d 100644 --- a/static/js/app/stores/builds.js +++ b/static/js/app/stores/builds.js @@ -22,13 +22,14 @@ define([ }, init: function() { - resource.subscribe(this._onAction); + resource.subscribe('create', 'update', this._onAction); }, onReadAll: function() { var self = this; resource.sync('read', function(err, builds) { - self.trigger(builds); + self.builds = builds; + self.trigger(self.builds); }); } });