diff --git a/lib/command/base.js b/lib/command/base.js index 3ae7ce8..a9431b3 100644 --- a/lib/command/base.js +++ b/lib/command/base.js @@ -5,6 +5,7 @@ var EventEmitter = require('events').EventEmitter, function Command(params) { params = params || {}; + this.emitIn = params.emitIn; this.emitOut = params.emitOut; } diff --git a/lib/command/spawn.js b/lib/command/spawn.js index 2bf1ac7..24174f1 100644 --- a/lib/command/spawn.js +++ b/lib/command/spawn.js @@ -3,7 +3,7 @@ var spawn = require('child_process').spawn, ParentCommand = require('./base').Command, inherits = require('util').inherits, - utils = require('../utils'); + _ = require('underscore'); function Command(params) { params = params || {}; @@ -21,20 +21,29 @@ inherits(Command, ParentCommand); Command.prototype.run = function(params, callback) { var self = this, stdout = self.collectOut ? '' : null; + if (!params.cmd) return callback(new Error('`cmd` is not set')); if (!params.args) return callback(new Error('`args` is not set')); - callback = utils.once(callback); + callback = _(callback).once(); params.options = params.options || {}; params.options.cwd = params.options.cwd || this.cwd; + var cmd = spawn(params.cmd, params.args, params.options); + + if (self.emitIn) { + self.emit('stdin', params.cmd + ' ' + params.args.join(' ')); + } + cmd.stdout.on('data', function(data) { if (self.emitOut) self.emit('stdout', data); if (self.collectOut) stdout += data; }); + cmd.stderr.on('data', function(data) { callback(new Error('Spawned command outputs to stderr: ' + data)); cmd.kill(); }); + cmd.on('close', function(code) { var err = null; if (code !== 0) err = new Error( @@ -42,5 +51,6 @@ Command.prototype.run = function(params, callback) { ); callback(err, stdout); }); + return cmd; }; diff --git a/lib/distributor.js b/lib/distributor.js index 93814bd..be30ddf 100644 --- a/lib/distributor.js +++ b/lib/distributor.js @@ -17,6 +17,9 @@ function Distributor(params) { self.onBuildUpdate = params.onBuildUpdate || function(build, callback) { callback(null, build); }; + + self.onBuildData = params.onBuildData || function(build, data) { + }; } exports.Distributor = Distributor; @@ -55,10 +58,22 @@ Distributor.prototype._runNext = function(callback) { self.queue.splice(queueItemIndex, 1); var stepCallback = this.slot(); - node.run(queueItem.project, build.params, function(err) { + var executor = node.run(queueItem.project, build.params, function(err) { build.status = err ? 'error' : 'done'; build.error = err; - self._updateBuild(build, stepCallback); + self._updateBuild(build, function(err, build) { + // try to run next project from the queue + self._runNext(stepCallback); + }); + }); + + executor.on('currentStep', function(stepLabel) { + build.currentStep = stepLabel; + self._updateBuild(build); + }); + + executor.on('data', function(data) { + self.onBuildData(build, data); }); }, callback @@ -66,6 +81,7 @@ Distributor.prototype._runNext = function(callback) { }; Distributor.prototype._updateBuild = function(build, callback) { + callback = callback || _.noop; this.onBuildUpdate(build, callback); }; @@ -76,7 +92,7 @@ Distributor.prototype.run = function(project, params, callback) { self._updateBuild({ project: project, params: params, - status: 'waiting' + status: 'queued' }, this.slot()); }, function(err, build) { diff --git a/lib/executor/base.js b/lib/executor/base.js index 3de0f6a..36f13cb 100644 --- a/lib/executor/base.js +++ b/lib/executor/base.js @@ -2,7 +2,10 @@ var Steppy = require('twostep').Steppy, path = require('path'), - _ = require('underscore'); + _ = require('underscore'), + EventEmitter = require('events').EventEmitter, + inherits = require('util').inherits, + utils = require('../utils'); function Executor(params) { this.project = params.project; @@ -11,6 +14,8 @@ function Executor(params) { exports.Executor = Executor; +inherits(Executor, EventEmitter); + Executor.prototype._getSources = function(params, callback) { }; @@ -23,11 +28,14 @@ Executor.prototype.run = function(params, callback) { project = _({}).extend(self.project, params); Steppy( function() { + self.emit('currentStep', 'get sources'); self._getSources(project.scm, this.slot()); }, function() { - var funcs = project.steps.map(function(step) { + var funcs = project.steps.map(function(step, index) { return function() { + var stepLabel = step.name || utils.prune(step.cmd, 15); + self.emit('currentStep', stepLabel); self._runStep(step, this.slot()); }; }); diff --git a/lib/executor/local.js b/lib/executor/local.js index fa8f625..b7d721c 100644 --- a/lib/executor/local.js +++ b/lib/executor/local.js @@ -5,7 +5,8 @@ var Steppy = require('twostep').Steppy, ParentExecutor = require('./base').Executor, createScm = require('../scm').createScm, createCommand = require('../command').createCommand, - fs = require('fs'); + fs = require('fs'), + _ = require('underscore'); function Executor(params) { ParentExecutor.call(this, params); @@ -61,7 +62,21 @@ Executor.prototype._runStep = function(params, callback) { } // set command cwd to executor cwd params.cwd = self.cwd; - var command = createCommand(params); + var command = createCommand( + _({ + emitIn: true, + emitOut: true + }).extend(params) + ); + + command.on('stdin', function(data) { + self.emit('data', '> ' + String(data)); + }); + + command.on('stdout', function(data) { + self.emit('data', String(data)); + }); + command.run(params, this.slot()) }, callback diff --git a/lib/node.js b/lib/node.js index 9801aee..1057e8c 100644 --- a/lib/node.js +++ b/lib/node.js @@ -39,4 +39,6 @@ Node.prototype.run = function(project, params, callback) { delete self.executors[project.name]; callback(err); }); + + return this.executors[project.name]; }; diff --git a/lib/utils.js b/lib/utils.js index 53c72e0..90cf13c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,24 +1,12 @@ 'use strict'; -['Function', 'String', 'Number', 'Date', 'RegExp'].forEach(function(name) { - exports['is' + name] = function(obj) { - return toString.call(obj) == '[object ' + name + ']'; - }; -}); +exports.prune = function(str, length) { + var result = '', + words = str.split(' '); -exports.isObject = function(obj) { - return obj === Object(obj); -}; - -exports.noop = function() {}; - -exports.slice = Array.prototype.slice; - -exports.once = function(func) { - var isCalled = false; - return function() { - if (isCalled) return; - func.apply(this, arguments); - isCalled = true; - }; + do { + result += words.shift() + ' '; + } while (words.length && result.length < length); + + return result.replace(/ $/, result.length <= length ? '' : '...'); }; diff --git a/projects/project1/config.json b/projects/project1/config.json index d58fa60..aadae04 100644 --- a/projects/project1/config.json +++ b/projects/project1/config.json @@ -6,7 +6,11 @@ "rev": "default" }, "steps": [ + {"type": "shell", "cmd": "sleep 2 && echo \"hello, cur dir is `pwd`\""}, + {"type": "shell", "name": "sleep", "cmd": "sleep 4"}, {"type": "shell", "cmd": "echo 1 > 1.txt"}, - {"type": "shell", "cmd": "echo 2 > 2.txt"} + {"type": "shell", "cmd": "sleep 4"}, + {"type": "shell", "cmd": "echo 2 > 2.txt"}, + {"type": "shell", "cmd": "cat 1.txt 2.txt"} ] } \ No newline at end of file diff --git a/projects/project2/config.json b/projects/project2/config.json index 91b2c61..9661323 100644 --- a/projects/project2/config.json +++ b/projects/project2/config.json @@ -7,6 +7,8 @@ }, "steps": [ {"type": "shell", "cmd": "echo 11 > 11.txt"}, - {"type": "shell", "cmd": "echo 22 > 22.txt"} + {"type": "shell", "cmd": "sleep 4"}, + {"type": "shell", "cmd": "echo 22 > 22.txt"}, + {"type": "shell", "cmd": "cat 11.txt 22.txt"} ] } \ No newline at end of file diff --git a/resources/projects.js b/resources/projects.js index 5bd40ef..43653c4 100644 --- a/resources/projects.js +++ b/resources/projects.js @@ -19,16 +19,28 @@ project.loadAll('projects', function(err, loadedProjects) { }); module.exports = function(app) { + var buildsSequnce = 0; var distributor = new Distributor({ nodes: [{type: 'local', maxExecutorsCount: 1}], onBuildUpdate: function(build, callback) { var buildsResource = app.dataio.resource('builds'); + if (build.status === 'queued') { + build.id = ++buildsSequnce; + // create resource for build data + var buildDataResource = app.dataio.resource('build' + build.id); + buildDataResource.on('connection', function(client) { + client.emit('sync', 'data', '< collected data >'); + }); + } buildsResource.clientEmitSync( - build.status === 'waiting' ? 'create' : 'update', + build.status === 'queued' ? 'create' : 'update', build ); callback(null, build); + }, + onBuildData: function(build, data) { + app.dataio.resource('build' + build.id).clientEmitSync('data', data); } }); diff --git a/static/js/app/actions/build.js b/static/js/app/actions/build.js new file mode 100644 index 0000000..4c99c9e --- /dev/null +++ b/static/js/app/actions/build.js @@ -0,0 +1,10 @@ +'use strict'; + +define(['reflux'], function(Reflux) { + var Actions = Reflux.createActions([ + 'readConsoleOutput', + 'readAll' + ]); + + return Actions; +}); diff --git a/static/js/app/app.js b/static/js/app/app.js index c565143..3bfc0c4 100644 --- a/static/js/app/app.js +++ b/static/js/app/app.js @@ -6,31 +6,9 @@ define([ ], function( React, template, Components, ProjectActions ) { - //var projectsTemplate = _($('#projects-template').html()).template(); - //$('#content').on('click', '.js-projects .js-run', function() { - //var projectName = $(this).parent('.js-project').data('name'); - //projects.sync('run', {projectName: projectName}, function(err, result) { - //$('#content').append( - //(err && err.message) - //); - //}); - //}); - - //projects.sync('read', function(err, projects) { - //console.log('read complete'); - ////$('#content').html( - ////(err && err.message) || - ////projectsTemplate({projects: projects}) - ////); - //}); - - //builds.subscribe(function(data, action) { - //$('#content').append(action.action + ': ' + JSON.stringify(data)); - //}); - React.render(template({ App: Components.App - }), document.getElementById('react-content')); + }), document.getElementById('content')); ProjectActions.readAll(); }); diff --git a/static/js/app/components/app.jade b/static/js/app/components/app.jade index 50d6c32..2f5f553 100644 --- a/static/js/app/components/app.jade +++ b/static/js/app/components/app.jade @@ -1,6 +1,11 @@ -div - h1 application +h1 nci - ProjectsList() +.row + .col-md-4 + ProjectsList() + .col-md-4 + BuildsList() + .col-md-4 + Console() diff --git a/static/js/app/components/app.js b/static/js/app/components/app.js index a26cc50..eec75ae 100644 --- a/static/js/app/components/app.js +++ b/static/js/app/components/app.js @@ -3,12 +3,16 @@ define([ 'react', 'app/components/projects/index', + 'app/components/builds/index', + 'app/components/console/index', 'templates/app/components/app' -], function(React, Projects, template) { +], function(React, Projects, Builds, Console, template) { var Component = React.createClass({ render: function() { return template({ - ProjectsList: Projects.List + ProjectsList: Projects.List, + BuildsList: Builds.List, + Console: Console.Console }); } }); diff --git a/static/js/app/components/builds/index.js b/static/js/app/components/builds/index.js new file mode 100644 index 0000000..c4246ac --- /dev/null +++ b/static/js/app/components/builds/index.js @@ -0,0 +1,11 @@ +'use strict'; + +define([ + 'app/components/builds/item', + 'app/components/builds/list' +], function(Item, List) { + return { + Item: Item, + List: List + }; +}); diff --git a/static/js/app/components/builds/item.jade b/static/js/app/components/builds/item.jade new file mode 100644 index 0000000..60a4d58 --- /dev/null +++ b/static/js/app/components/builds/item.jade @@ -0,0 +1,5 @@ +li.list-group-item + span.badge= item.status + a.pull-right(href="javascript:void(0);", onClick=onBuildSelect(item.id), style={marginRight: '5px'}) show console output + span Build # + span= item.id diff --git a/static/js/app/components/builds/item.js b/static/js/app/components/builds/item.js new file mode 100644 index 0000000..30f2b3e --- /dev/null +++ b/static/js/app/components/builds/item.js @@ -0,0 +1,20 @@ +'use strict'; + +define([ + 'react', 'app/actions/build', 'templates/app/components/builds/item' +], function(React, BuildActions, template) { + var Component = React.createClass({ + onBuildSelect: function(buildId) { + console.log('on build select'); + BuildActions.readConsoleOutput(buildId); + }, + render: function() { + return template({ + item: this.props.item, + onBuildSelect: this.onBuildSelect + }); + } + }); + + return Component; +}); diff --git a/static/js/app/components/builds/list.jade b/static/js/app/components/builds/list.jade new file mode 100644 index 0000000..eb5ff4b --- /dev/null +++ b/static/js/app/components/builds/list.jade @@ -0,0 +1,6 @@ +div + h2 builds list + - console.log(items) + ul.list-group + each build, index in items + Item(item=build, key=build.id) diff --git a/static/js/app/components/builds/list.js b/static/js/app/components/builds/list.js new file mode 100644 index 0000000..eff7974 --- /dev/null +++ b/static/js/app/components/builds/list.js @@ -0,0 +1,32 @@ +'use strict'; + +define([ + 'react', + 'reflux', + './item', + 'app/stores/builds', + 'templates/app/components/builds/list' +], function(React, Reflux, Item, buildsStore, template) { + var Component = React.createClass({ + mixins: [Reflux.ListenerMixin], + componentDidMount: function() { + this.listenTo(buildsStore, this.updateItems); + }, + updateItems: function(items) { + this.setState({items: items}); + }, + render: function() { + return template({ + Item: Item, + items: this.state.items + }); + }, + getInitialState: function() { + return { + items: [] + }; + } + }); + + return Component; +}); diff --git a/static/js/app/components/console/console.jade b/static/js/app/components/console/console.jade new file mode 100644 index 0000000..04a9757 --- /dev/null +++ b/static/js/app/components/console/console.jade @@ -0,0 +1,3 @@ +if name + h2= name + pre= data diff --git a/static/js/app/components/console/console.js b/static/js/app/components/console/console.js new file mode 100644 index 0000000..e58750a --- /dev/null +++ b/static/js/app/components/console/console.js @@ -0,0 +1,29 @@ +'use strict'; + +define([ + 'react', + 'reflux', + 'app/stores/console', + 'templates/app/components/console/console' +], function(React, Reflux, consoleStore, template) { + var Component = React.createClass({ + mixins: [Reflux.ListenerMixin], + componentDidMount: function() { + this.listenTo(consoleStore, this.updateItems); + }, + updateItems: function(data) { + this.setState({data: data}); + }, + render: function() { + return template(this.state.data); + }, + getInitialState: function() { + return { + name: '', + data: '' + }; + } + }); + + return Component; +}); diff --git a/static/js/app/components/console/index.js b/static/js/app/components/console/index.js new file mode 100644 index 0000000..a8f6baf --- /dev/null +++ b/static/js/app/components/console/index.js @@ -0,0 +1,9 @@ +'use strict'; + +define([ + 'app/components/console/console', +], function(Console) { + return { + Console: Console + }; +}); diff --git a/static/js/app/components/index.js b/static/js/app/components/index.js index d96b3be..7515a50 100644 --- a/static/js/app/components/index.js +++ b/static/js/app/components/index.js @@ -2,10 +2,12 @@ define([ 'app/components/projects/index', + 'app/components/builds/index', 'app/components/app', -], function(ProjectsComponents, App) { +], function(ProjectsComponents, BuildsComponents, App) { return { App: App, - ProjectsComponents: ProjectsComponents + ProjectsComponents: ProjectsComponents, + BuildsComponents: BuildsComponents }; }); diff --git a/static/js/app/components/projects/item.jade b/static/js/app/components/projects/item.jade index c570ef9..05e60a6 100644 --- a/static/js/app/components/projects/item.jade +++ b/static/js/app/components/projects/item.jade @@ -1 +1,3 @@ -h1(onClick=onProjectSelect(item.name))= item.name +li.list-group-item + a.pull-right(href="javascript:void(0);", onClick=onProjectSelect(item.name)) start build + span= item.name diff --git a/static/js/app/components/projects/list.jade b/static/js/app/components/projects/list.jade index 183bc8d..da10117 100644 --- a/static/js/app/components/projects/list.jade +++ b/static/js/app/components/projects/list.jade @@ -1,4 +1,4 @@ -div - h2 projects list +h2 projects list +ul.list-group each project, index in items Item(item=project, key=project.name) diff --git a/static/js/app/connect.js b/static/js/app/connect.js new file mode 100644 index 0000000..264901c --- /dev/null +++ b/static/js/app/connect.js @@ -0,0 +1,8 @@ +'use strict'; + +define([ + 'socketio', 'dataio' +], function(socketio, dataio) { + // Do it because we use connect in console store + return dataio(socketio.connect()); +}); diff --git a/static/js/app/index.jade b/static/js/app/index.jade index e45e67d..a021249 100644 --- a/static/js/app/index.jade +++ b/static/js/app/index.jade @@ -1,2 +1,2 @@ -div +.container-fluid App() diff --git a/static/js/app/resources.js b/static/js/app/resources.js index 1f45c29..4b4db09 100644 --- a/static/js/app/resources.js +++ b/static/js/app/resources.js @@ -1,10 +1,6 @@ 'use strict'; -define([ - 'socketio', 'dataio' -], function(socketio, dataio) { - var connect = dataio(socketio.connect()); - +define(['app/connect'], function(connect) { var projects = connect.resource('projects'); var builds = connect.resource('builds'); diff --git a/static/js/app/stores/builds.js b/static/js/app/stores/builds.js new file mode 100644 index 0000000..1c9d0fe --- /dev/null +++ b/static/js/app/stores/builds.js @@ -0,0 +1,37 @@ +'use strict'; + +define([ + 'underscore', + 'reflux', 'app/actions/build', 'app/resources' +], function(_, Reflux, BuildActions, resources) { + var resource = resources.builds; + + var Store = Reflux.createStore({ + listenables: BuildActions, + builds: [], + + _onAction: function(build, action) { + var oldBuild = _(this.builds).findWhere({id: build.id}); + if (oldBuild) { + _(oldBuild).extend(build); + } else { + this.builds.unshift(build); + } + + this.trigger(this.builds); + }, + + init: function() { + resource.subscribe(this._onAction); + }, + + onReadAll: function() { + var self = this; + resource.sync('read', function(err, builds) { + self.trigger(builds); + }); + } + }); + + return Store; +}); diff --git a/static/js/app/stores/console.js b/static/js/app/stores/console.js new file mode 100644 index 0000000..11948ff --- /dev/null +++ b/static/js/app/stores/console.js @@ -0,0 +1,34 @@ +'use strict'; + +define([ + 'underscore', + 'reflux', 'app/actions/build', 'app/connect' +], function(_, Reflux, BuildActions, connect) { + var Store = Reflux.createStore({ + listenables: BuildActions, + + output: '', + + init: function() { + console.log('init builds console output'); + }, + + onReadConsoleOutput: function(buildId) { + this.output = '' + + var resourceName = 'build' + buildId, + self = this; + + connect.resource(resourceName).unsubscribeAll(); + connect.resource(resourceName).subscribe(function(data) { + self.output += data; + self.trigger({ + name: 'Console for build #' + buildId, + data: data + }); + }); + } + }); + + return Store; +}); diff --git a/static/js/dataio.js b/static/js/dataio.js new file mode 100644 index 0000000..4a774dd --- /dev/null +++ b/static/js/dataio.js @@ -0,0 +1,19 @@ +'use strict'; + +define(['_dataio'], function(dataio) { + return function(socket) { + var connect = dataio(socket); + + /* + * Extend Resource + */ + var resource = connect.resource('__someResource__'), + resourcePrototype = Object.getPrototypeOf(resource); + + resourcePrototype.unsubscribeAll = function() { + this.socket.removeAllListeners(); + }; + + return connect; + }; +}); \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js index dcc88d0..c63d257 100644 --- a/static/js/main.js +++ b/static/js/main.js @@ -6,7 +6,7 @@ require.config({ underscore: 'libs/underscore/underscore', react: 'libs/react/react-with-addons', reflux: 'libs/reflux/dist/reflux', - dataio: '/data.io', + _dataio: '/data.io', socketio: '/socket.io/socket.io.js', jquery: 'libs/jquery/jquery' } diff --git a/test/distributor.js b/test/distributor.js index 406101d..3978b54 100644 --- a/test/distributor.js +++ b/test/distributor.js @@ -2,7 +2,8 @@ var Distributor = require('../lib/distributor').Distributor, Node = require('../lib/node').Node, - expect = require('expect.js'); + expect = require('expect.js'), + EventEmitter = require('events').EventEmitter; describe('Distributor', function() { @@ -13,7 +14,9 @@ describe('Distributor', function() { return function(params) { var node = new Node(params); node._createExecutor = function() { - return {run: executorRun}; + var executor = new EventEmitter(); + executor.run = executorRun; + return executor; }; return node; }; @@ -43,7 +46,7 @@ describe('Distributor', function() { it('instance should be created without errors', function() { var number = 1; var conditionsHash = { - 1: {queue: {length: 0}, build: {status: 'waiting'}}, + 1: {queue: {length: 0}, build: {status: 'queued'}}, 2: {queue: {length: 1}, build: {status: 'in-progress'}}, 3: {queue: {length: 0}, build: {status: 'done'}}, 4: 'Should never happend' @@ -92,7 +95,7 @@ describe('Distributor', function() { it('instance should be created without errors', function() { var number = 1; var conditionsHash = { - 1: {queue: {length: 0}, build: {status: 'waiting'}}, + 1: {queue: {length: 0}, build: {status: 'queued'}}, 2: {queue: {length: 1}, build: {status: 'in-progress'}}, 3: { queue: {length: 0}, diff --git a/views/index.jade b/views/index.jade index b75945a..208540e 100644 --- a/views/index.jade +++ b/views/index.jade @@ -1,26 +1,14 @@ doctype html html head - title test + title nci + + // do it temporary + link(href="/js/libs/bootstrap/dist/css/bootstrap.css", rel="stylesheet", type="text/css") + script(data-main="/js/main" src="/js/libs/requirejs/require.js") script(type="text/javascript"). require(['app/app']); body - h1 hello world - - - script#projects-template(type="text/template") - |