diff --git a/db.js b/db.js index a649aa0..dd72a6b 100644 --- a/db.js +++ b/db.js @@ -2,13 +2,17 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), - nlevel = require('nlevel'); + nlevel = require('nlevel'), + path = require('path'); exports.init = function(dbPath, params, callback) { - var ldb = nlevel.db(dbPath, params, callback); + callback = _.after(2, callback); - exports.builds = new nlevel.DocsSection(ldb, 'builds', { + var maindbPath = path.join(dbPath, 'main'), + mainDb = nlevel.db(maindbPath, params, callback); + + exports.builds = new nlevel.DocsSection(mainDb, 'builds', { projections: [ {key: {createDate: 1}, value: pickId}, {key: {descCreateDate: descCreateDate, id: 1}}, @@ -73,6 +77,17 @@ exports.init = function(dbPath, params, callback) { callback ); }; + + var buildLogsDbPath = path.join(dbPath, 'main'), + buildLogsDb = nlevel.db(buildLogsDbPath, params, callback); + + exports.logLines = new nlevel.DocsSection(buildLogsDb, 'logLines', { + projections: [ + {key: {buildId: 1, numberStr: 1}, value: function(logLine) { + return _(logLine).pick('number', 'text'); + }} + ] + }); }; /* @@ -81,6 +96,12 @@ exports.init = function(dbPath, params, callback) { */ nlevel.DocsSection.prototype._beforePut = function(docs, callback) { var self = this; + + // Quit early if beforePut is not set + if (!self.beforePut) { + return callback(); + } + Steppy( function() { if (self._beforePutInProgress) { diff --git a/distributor.js b/distributor.js index 0e04670..eaf7bc2 100644 --- a/distributor.js +++ b/distributor.js @@ -9,7 +9,8 @@ var Steppy = require('twostep').Steppy, db = require('./db'), path = require('path'), fs = require('fs'), - logger = require('./lib/logger')('distributor'); + logger = require('./lib/logger')('distributor'), + utils = require('./lib/utils'); exports.init = function(app, callback) { @@ -102,6 +103,8 @@ exports.init = function(app, callback) { var writeStreamsHash = {}; + var buildLogLineNumbersHash = {}; + distributor.on('buildData', function(build, data) { if (!/\n$/.test(data)) { data += '\n'; @@ -126,6 +129,31 @@ exports.init = function(app, callback) { app.dataio.resource('build' + build.id).clientEmitSync( 'data', data ); + + // write build logs to db + if (buildLogLineNumbersHash[build.id]) { + buildLogLineNumbersHash[build.id]++; + } else { + buildLogLineNumbersHash[build.id] = 1; + } + + var logLineNumber = buildLogLineNumbersHash[build.id], + logLineId = build.id + '-' + logLineNumber; + + db.logLines.put({ + id: logLineId, + buildId: build.id, + numberStr: utils.toNumberStr(logLineNumber), + number: logLineNumber, + text: data + }, function(err) { + if (err) { + logger.error( + 'Error during write log line "' + logLineId + '":', + err.stack || err + ); + } + }); }); callback(null, distributor); diff --git a/lib/utils.js b/lib/utils.js index 3981aef..755c83d 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -10,3 +10,13 @@ exports.prune = function(str, length) { return result.replace(/ $/, words.length ? '...' : ''); }; + +exports.lpad = function(str, length, chr) { + chr = chr || '0'; + while (str.length < length) str = chr + str; + return str; +}; + +exports.toNumberStr = function(number) { + return exports.lpad(String(number), 20); +}; diff --git a/resources/builds.js b/resources/builds.js index 9ff15f4..c865569 100644 --- a/resources/builds.js +++ b/resources/builds.js @@ -2,7 +2,8 @@ var Steppy = require('twostep').Steppy, _ = require('underscore'), - db = require('../db'); + db = require('../db'), + utils = require('../lib/utils'); module.exports = function(app) { var resource = app.dataio.resource('builds'); @@ -55,5 +56,47 @@ module.exports = function(app) { ); }); + resource.use('getBuildLogTail', function(req, res, next) { + Steppy( + function() { + var findParams = { + reverse: true, + start: {buildId: req.data.buildId, numberStr: ''}, + limit: req.data.length + }; + + db.logLines.find(findParams, this.slot()); + }, + function(err, logLines) { + var lines = _(logLines).pluck('text').reverse(), + total = logLines.length ? logLines[0].number : 0; + + res.send({lines: lines, total: total}); + }, + next + ); + }); + + resource.use('getBuildLogLines', function(req, res, next) { + Steppy( + function() { + var buildId = req.data.buildId, + from = req.data.from, + to = req.data.to; + + db.logLines.find({ + start: {buildId: buildId, numberStr: utils.toNumberStr(from)}, + end: {buildId: buildId, numberStr: utils.toNumberStr(to)} + }, this.slot()); + }, + function(err, logLines) { + res.send({ + lines: _(logLines).pluck('text') + }); + }, + next + ); + }); + return resource; }; diff --git a/static/js/app/actions/buildLog.js b/static/js/app/actions/buildLog.js new file mode 100644 index 0000000..26d4142 --- /dev/null +++ b/static/js/app/actions/buildLog.js @@ -0,0 +1,10 @@ +'use strict'; + +define(['reflux'], function(Reflux) { + var Actions = Reflux.createActions([ + 'getTail', + 'getLines' + ]); + + return Actions; +}); diff --git a/static/js/app/app.js b/static/js/app/app.js index e3cb4be..779a2d6 100644 --- a/static/js/app/app.js +++ b/static/js/app/app.js @@ -24,7 +24,12 @@ define([ path: 'projects/:name', handler: Components.Project.View }), - Route({name: 'build', path: 'builds/:id', handler: Components.Build.View}) + Route({name: 'build', path: 'builds/:id', handler: Components.Build.View}), + Route({ + name: 'buildLog', + path: 'builds/:buildId/log', + handler: Components.BuildLog + }) ) ); diff --git a/static/js/app/components/buildLog/index.jade b/static/js/app/components/buildLog/index.jade new file mode 100644 index 0000000..7477e2b --- /dev/null +++ b/static/js/app/components/buildLog/index.jade @@ -0,0 +1,19 @@ + +- var buildId = this.props.params.buildId; +- var total = this.state.data.total; +- var output = this.state.data.output; + +div + | build: + span= buildId + +div + | lines in total: + span= total + +div + | from: + input(type="text", value=this.state.from, onChange=this.onFromChange) +br +.terminal + .terminal_code(ref="code")!= output diff --git a/static/js/app/components/buildLog/index.js b/static/js/app/components/buildLog/index.js new file mode 100644 index 0000000..f973856 --- /dev/null +++ b/static/js/app/components/buildLog/index.js @@ -0,0 +1,41 @@ +'use strict'; + +define([ + 'react', 'reflux', 'app/actions/buildLog', 'app/stores/buildLog', + 'ansi_up', 'underscore', 'templates/app/components/buildLog/index' +], function( + React, Reflux, BuildLogActions, buildLogStore, + ansiUp, _, template +) { + var chunkSize = 20; + + return React.createClass({ + mixins: [ + Reflux.connectFilter(buildLogStore, 'data', function(data) { + data.output = data.lines.join(''); + data.output = data.output.replace( + /(.*)\n/gi, + '$1' + ); + data.output = ansiUp.ansi_to_html(data.output); + return data; + }) + ], + statics: { + willTransitionTo: function(transition, params, query) { + BuildLogActions.getTail({buildId: params.buildId, length: chunkSize}); + } + }, + onFromChange: function(event) { + var from = Number(event.target.value); + this.setState({from: from}); + + BuildLogActions.getLines({ + buildId: this.props.params.buildId, + from: from, + to: from + chunkSize - 1 + }); + }, + render: template + }); +}); diff --git a/static/js/app/components/index.js b/static/js/app/components/index.js index a2d7480..26e9d52 100644 --- a/static/js/app/components/index.js +++ b/static/js/app/components/index.js @@ -1,17 +1,20 @@ 'use strict'; define([ - 'app/components/projects/index', - 'app/components/builds/index', - 'app/components/app/index', - 'app/components/header/index', - 'app/components/dashboard/index' -], function(ProjectsComponents, BuildsComponents, App, Header, Dashboard) { + 'app/components/projects/index', 'app/components/builds/index', + 'app/components/app/index', 'app/components/header/index', + 'app/components/dashboard/index', 'app/components/buildLog/index' +], function( + ProjectsComponents, BuildsComponents, + App, Header, + Dashboard, BuildLogComponent +) { return { App: App, Header: Header, Project: ProjectsComponents, Build: BuildsComponents, - Dashboard: Dashboard + Dashboard: Dashboard, + BuildLog: BuildLogComponent }; }); diff --git a/static/js/app/stores/buildLog.js b/static/js/app/stores/buildLog.js new file mode 100644 index 0000000..6e4eb81 --- /dev/null +++ b/static/js/app/stores/buildLog.js @@ -0,0 +1,41 @@ +'use strict'; + +define([ + 'reflux', 'app/actions/buildLog', 'app/resources' +], function( + Reflux, BuildLogActions, resources +) { + var resource = resources.builds; + + var Store = Reflux.createStore({ + listenables: BuildLogActions, + data: { + lines: [], + total: 0 + }, + + getInitialState: function() { + return this.data; + }, + + onGetTail: function(params) { + var self = this; + resource.sync('getBuildLogTail', params, function(err, data) { + if (err) throw err; + self.data = data; + self.trigger(self.data); + }); + }, + + onGetLines: function(params) { + var self = this; + resource.sync('getBuildLogLines', params, function(err, data) { + if (err) throw err; + self.data.lines = data.lines; + self.trigger(self.data); + }); + } + }); + + return Store; +});