From b2fa9bea111f2552210a2ea70981bc942ca29ef8 Mon Sep 17 00:00:00 2001 From: oleg Date: Mon, 30 Nov 2015 23:08:54 +0300 Subject: [PATCH] manage log lines using db directly --- app.js | 3 +- db.js | 115 +++++++++++++++++---- distributor.js | 2 +- package.json | 3 +- resources/builds.js | 6 +- static/js/app/components/buildLog/index.js | 3 +- 6 files changed, 106 insertions(+), 26 deletions(-) diff --git a/app.js b/app.js index f65a864..803df97 100644 --- a/app.js +++ b/app.js @@ -181,8 +181,7 @@ Steppy( logger.log('Server config:', JSON.stringify(app.config, null, 4)); db.init(app.config.paths.db, { - db: require(app.config.storage.backend), - valueEncoding: 'json' + db: require(app.config.storage.backend) }, this.slot()); }, function() { diff --git a/db.js b/db.js index 10ef5be..ed5dd17 100644 --- a/db.js +++ b/db.js @@ -4,14 +4,17 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), nlevel = require('nlevel'), path = require('path'), - utils = require('./lib/utils'); + utils = require('./lib/utils'), + through = require('through'); exports.init = function(dbPath, params, callback) { callback = _.after(2, callback); var maindbPath = path.join(dbPath, 'main'), - mainDb = nlevel.db(maindbPath, params, callback); + mainDb = nlevel.db(maindbPath, _({ + valueEncoding: 'json' + }).defaults(params), callback); exports.builds = new nlevel.DocsSection(mainDb, 'builds', { projections: [ @@ -82,22 +85,98 @@ exports.init = function(dbPath, params, callback) { var buildLogsDbPath = path.join(dbPath, 'buildLogs'), buildLogsDb = nlevel.db(buildLogsDbPath, params, callback); - exports.logLines = new nlevel.DocsSection(buildLogsDb, 'logLines', { - projections: [ - { - key: { - buildId: 1, - numberStr: function(logLine) { - return utils.toNumberStr(logLine.number); - } - }, - value: function(logLine) { - return _(logLine).pick('number', 'text'); - } - } - ], - withUniqueId: false - }); + // custom optimized emplementation for storing log lines + exports.logLines = {}; + + exports.logLines.separator = '~'; + exports.logLines.end = '\xff'; + + exports.logLines._getStrKey = function(line) { + return line.buildId + this.separator + ( + _(line).has('number') ? utils.toNumberStr(line.number) : '' + ); + }; + + exports.logLines._parseData = function(data) { + var keyParts = data.key.split(this.separator), + buildId = Number(keyParts[0]), + number = Number(keyParts[1]); + + return {buildId: buildId, number: number, text: data.value}; + }; + + exports.logLines.put = function(lines, callback) { + var self = this; + lines = _(lines).isArray() ? lines : [lines]; + + var operations = _(lines).map(function(line) { + return { + type: 'put', + key: self._getStrKey(line), + value: line.text + }; + }); + + buildLogsDb.batch(operations, callback); + }; + + exports.logLines.createReadStream = function(params) { + var self = this; + if (!params.start && params.end) { + new Error('`end` selected without `start`'); + } + + params.start = self._getStrKey(params.start); + params.end = params.end ? self._getStrKey(params.end) : params.start; + // add end character + params.end += self.end; + // swap `start` `end` conditions when reverse is set + if (params.reverse) { + var prevStart = params.start; + params.start = params.end; + params.end = prevStart; + } + + var resultStream = through(function(data) { + this.emit('data', _(data).isObject() ? self._parseData(data) : data); + }); + + return buildLogsDb.createReadStream(params) + .on('error', function(err) { + resultStream.emit('error', err); + }) + .pipe(resultStream) + }; + + exports.logLines.find = function(params, callback) { + var self = this; + callback = _(callback).once(); + + var lines = []; + self.createReadStream(params) + .on('error', callback) + .on('data', function(line) { + lines.push(line); + }) + .on('end', function() { + callback(null, lines); + }) + }; + + exports.logLines.remove = function(params, callback) { + var self = this; + callback = _(callback).once(); + + var operations = []; + self.createReadStream(_({values: false}).extend(params)) + .on('error', callback) + .on('data', function(key) { + operations.push({type: 'del', key: key}); + }) + .on('end', function() { + buildLogsDb.batch(operations, callback); + }); + }; }; /* diff --git a/distributor.js b/distributor.js index d8151b1..365a818 100644 --- a/distributor.js +++ b/distributor.js @@ -57,7 +57,7 @@ exports.init = function(app, callback) { Steppy( function() { db.logLines.find({ - start: {buildId: buildId, numberStr: ''}, + start: {buildId: buildId}, }, this.slot()); }, function(err, lines) { diff --git a/package.json b/package.json index 087382f..333dcf4 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "node-static": "0.7.6", "socket.io": "1.3.5", "twostep": "0.4.1", - "underscore": "1.8.3" + "underscore": "1.8.3", + "through": "2.3.6" }, "devDependencies": { "bower": "1.4.1", diff --git a/resources/builds.js b/resources/builds.js index 6dc7ddb..946a749 100644 --- a/resources/builds.js +++ b/resources/builds.js @@ -65,7 +65,7 @@ module.exports = function(app) { function() { var findParams = { reverse: true, - start: {buildId: req.data.buildId, numberStr: ''}, + start: {buildId: req.data.buildId}, limit: req.data.length }; @@ -90,8 +90,8 @@ module.exports = function(app) { count = to - from; db.logLines.find({ - start: {buildId: buildId, numberStr: utils.toNumberStr(from)}, - end: {buildId: buildId, numberStr: utils.toNumberStr(to)} + start: {buildId: buildId, number: from}, + end: {buildId: buildId, number: to} }, this.slot()); this.pass(count); diff --git a/static/js/app/components/buildLog/index.js b/static/js/app/components/buildLog/index.js index 3822419..5364483 100644 --- a/static/js/app/components/buildLog/index.js +++ b/static/js/app/components/buildLog/index.js @@ -14,11 +14,12 @@ define([ return React.createClass({ mixins: [ Reflux.connectFilter(buildLogStore, 'data', function(data) { - data.output = _(data.lines).pluck('text').join(''); + data.output = _(data.lines).pluck('text').join('
'); data.output = data.output.replace( /(.*)\n/gi, '$1' ); + data.output = ansiUp.ansi_to_html(data.output); return data; })