store build logs in db + sample output

db.js

@@ -2,13 +2,17 @@
 
 var Steppy = require('twostep').Steppy,
 	_ = require('underscore'),
-	nlevel = require('nlevel');
+	nlevel = require('nlevel'),
+	path = require('path');
 
 
 exports.init = function(dbPath, params, callback) {
-	var ldb = nlevel.db(dbPath, params, callback);
+	callback = _.after(2, callback);
 
-	exports.builds = new nlevel.DocsSection(ldb, 'builds', {
+	var maindbPath = path.join(dbPath, 'main'),
+		mainDb = nlevel.db(maindbPath, params, callback);
+
+	exports.builds = new nlevel.DocsSection(mainDb, 'builds', {
 		projections: [
 			{key: {createDate: 1}, value: pickId},
 			{key: {descCreateDate: descCreateDate, id: 1}},
@@ -73,6 +77,17 @@ exports.init = function(dbPath, params, callback) {
 			callback
 		);
 	};
+
+	var buildLogsDbPath = path.join(dbPath, 'main'),
+		buildLogsDb = nlevel.db(buildLogsDbPath, params, callback);
+
+	exports.logLines = new nlevel.DocsSection(buildLogsDb, 'logLines', {
+		projections: [
+			{key: {buildId: 1, numberStr: 1}, value: function(logLine) {
+				return _(logLine).pick('number', 'text');
+			}}
+		]
+	});
 };
 
 /*
@@ -81,6 +96,12 @@ exports.init = function(dbPath, params, callback) {
  */
 nlevel.DocsSection.prototype._beforePut = function(docs, callback) {
 	var self = this;
+
+	// Quit early if beforePut is not set
+	if (!self.beforePut) {
+		return callback();
+	}
+
 	Steppy(
 		function() {
 			if (self._beforePutInProgress) {

distributor.js

@@ -9,7 +9,8 @@ var Steppy = require('twostep').Steppy,
 	db = require('./db'),
 	path = require('path'),
 	fs = require('fs'),
-	logger = require('./lib/logger')('distributor');
+	logger = require('./lib/logger')('distributor'),
+	utils = require('./lib/utils');
 
 
 exports.init = function(app, callback) {
@@ -102,6 +103,8 @@ exports.init = function(app, callback) {
 
 	var writeStreamsHash = {};
 
+	var buildLogLineNumbersHash = {};
+
 	distributor.on('buildData', function(build, data) {
 		if (!/\n$/.test(data)) {
 			data += '\n';
@@ -126,6 +129,31 @@ exports.init = function(app, callback) {
 		app.dataio.resource('build' + build.id).clientEmitSync(
 			'data', data
 		);
+
+		// write build logs to db
+		if (buildLogLineNumbersHash[build.id]) {
+			buildLogLineNumbersHash[build.id]++;
+		} else {
+			buildLogLineNumbersHash[build.id] = 1;
+		}
+
+		var logLineNumber = buildLogLineNumbersHash[build.id],
+			logLineId = build.id + '-' + logLineNumber;
+
+		db.logLines.put({
+			id: logLineId,
+			buildId: build.id,
+			numberStr: utils.toNumberStr(logLineNumber),
+			number: logLineNumber,
+			text: data
+		}, function(err) {
+			if (err) {
+				logger.error(
+					'Error during write log line "' + logLineId + '":',
+					err.stack || err
+				);
+			}
+		});
 	});
 
 	callback(null, distributor);

lib/utils.js

@@ -10,3 +10,13 @@ exports.prune = function(str, length) {
 
 	return result.replace(/ $/, words.length ? '...' : '');
 };
+
+exports.lpad = function(str, length, chr) {
+	chr = chr || '0';
+	while (str.length < length) str = chr + str;
+	return str;
+};
+
+exports.toNumberStr = function(number) {
+	return exports.lpad(String(number), 20);
+};

resources/builds.js

@@ -2,7 +2,8 @@
 
 var Steppy = require('twostep').Steppy,
 	_ = require('underscore'),
-	db = require('../db');
+	db = require('../db'),
+	utils = require('../lib/utils');
 
 module.exports = function(app) {
 	var resource = app.dataio.resource('builds');
@@ -55,5 +56,47 @@ module.exports = function(app) {
 		);
 	});
 
+	resource.use('getBuildLogTail', function(req, res, next) {
+		Steppy(
+			function() {
+				var findParams = {
+					reverse: true,
+					start: {buildId: req.data.buildId, numberStr: ''},
+					limit: req.data.length
+				};
+
+				db.logLines.find(findParams, this.slot());
+			},
+			function(err, logLines) {
+				var lines = _(logLines).pluck('text').reverse(),
+					total = logLines.length ? logLines[0].number : 0;
+
+				res.send({lines: lines, total: total});
+			},
+			next
+		);
+	});
+
+	resource.use('getBuildLogLines', function(req, res, next) {
+		Steppy(
+			function() {
+				var buildId = req.data.buildId,
+					from = req.data.from,
+					to = req.data.to;
+
+				db.logLines.find({
+					start: {buildId: buildId, numberStr: utils.toNumberStr(from)},
+					end: {buildId: buildId, numberStr: utils.toNumberStr(to)}
+				}, this.slot());
+			},
+			function(err, logLines) {
+				res.send({
+					lines: _(logLines).pluck('text')
+				});
+			},
+			next
+		);
+	});
+
 	return resource;
 };

static/js/app/actions/buildLog.js

@@ -0,0 +1,10 @@
+'use strict';
+
+define(['reflux'], function(Reflux) {
+	var Actions = Reflux.createActions([
+		'getTail',
+		'getLines'
+	]);
+
+	return Actions;
+});

static/js/app/app.js

@@ -24,7 +24,12 @@ define([
 				path: 'projects/:name',
 				handler: Components.Project.View
 			}),
-			Route({name: 'build', path: 'builds/:id', handler: Components.Build.View})
+			Route({name: 'build', path: 'builds/:id', handler: Components.Build.View}),
+			Route({
+				name: 'buildLog',
+				path: 'builds/:buildId/log',
+				handler: Components.BuildLog
+			})
 		)
 	);
 

static/js/app/components/buildLog/index.jade

@@ -0,0 +1,19 @@
+
+- var buildId = this.props.params.buildId;
+- var total = this.state.data.total;
+- var output = this.state.data.output;
+
+div
+	| build: 
+	span= buildId
+
+div
+	| lines in total: 
+	span= total
+
+div
+	| from: 
+	input(type="text", value=this.state.from, onChange=this.onFromChange)
+br
+.terminal
+	.terminal_code(ref="code")!= output

static/js/app/components/buildLog/index.js

@@ -0,0 +1,41 @@
+'use strict';
+
+define([
+	'react', 'reflux', 'app/actions/buildLog', 'app/stores/buildLog',
+	'ansi_up', 'underscore', 'templates/app/components/buildLog/index'
+], function(
+	React, Reflux, BuildLogActions, buildLogStore,
+	ansiUp, _, template
+) {
+	var chunkSize = 20;
+
+	return React.createClass({
+		mixins: [
+			Reflux.connectFilter(buildLogStore, 'data', function(data) {
+				data.output = data.lines.join('');
+				data.output = data.output.replace(
+					/(.*)\n/gi,
+					'<span class="terminal_code_newline">$1</span>'
+				);
+				data.output = ansiUp.ansi_to_html(data.output);
+				return data;
+			})
+		],
+		statics: {
+			willTransitionTo: function(transition, params, query) {
+				BuildLogActions.getTail({buildId: params.buildId, length: chunkSize});
+			}
+		},
+		onFromChange: function(event) {
+			var from = Number(event.target.value);
+			this.setState({from: from});
+
+			BuildLogActions.getLines({
+				buildId: this.props.params.buildId,
+				from: from,
+				to: from + chunkSize - 1
+			});
+		},
+		render: template
+	});
+});

static/js/app/components/index.js

@@ -1,17 +1,20 @@
 'use strict';
 
 define([
-	'app/components/projects/index',
+	'app/components/projects/index', 'app/components/builds/index',
-	'app/components/builds/index',
+	'app/components/app/index', 'app/components/header/index',
-	'app/components/app/index',
+	'app/components/dashboard/index', 'app/components/buildLog/index'
-	'app/components/header/index',
+], function(
-	'app/components/dashboard/index'
+	ProjectsComponents, BuildsComponents,
-], function(ProjectsComponents, BuildsComponents, App, Header, Dashboard) {
+	App, Header,
+	Dashboard, BuildLogComponent
+) {
 	return {
 		App: App,
 		Header: Header,
 		Project: ProjectsComponents,
 		Build: BuildsComponents,
-		Dashboard: Dashboard
+		Dashboard: Dashboard,
+		BuildLog: BuildLogComponent
 	};
 });

static/js/app/stores/buildLog.js

@@ -0,0 +1,41 @@
+'use strict';
+
+define([
+	'reflux', 'app/actions/buildLog', 'app/resources'
+], function(
+	Reflux, BuildLogActions, resources
+) {
+	var resource = resources.builds;
+
+	var Store = Reflux.createStore({
+		listenables: BuildLogActions,
+		data: {
+			lines: [],
+			total: 0
+		},
+
+		getInitialState: function() {
+			return this.data;
+		},
+
+		onGetTail: function(params) {
+			var self = this;
+			resource.sync('getBuildLogTail', params, function(err, data) {
+				if (err) throw err;
+				self.data = data;
+				self.trigger(self.data);
+			});
+		},
+
+		onGetLines: function(params) {
+			var self = this;
+			resource.sync('getBuildLogLines', params, function(err, data) {
+				if (err) throw err;
+				self.data.lines = data.lines;
+				self.trigger(self.data);
+			});
+		}
+	});
+
+	return Store;
+});