martin/nci
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/nci.git synced 2025-01-10 21:45:08 +00:00
Code Issues Projects Releases Wiki Activity

merge with master (manually merge package.json)

Browse Source
This commit is contained in:
oleg 2016-01-09 23:08:26 +03:00
commit 9d25492eb1
15 changed files with 135 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -52,6 +52,7 @@ animation" appear but should not.
* ~~more strict server and project configs valifation~~
* ui browser tests needed
* use one from: jquery or native browser methods
* cleanup project steps (remove cwd, options) inside build by migration
## Feature requests

99
app.js
View File

@ -2,8 +2,7 @@
var env = process.env.NODE_ENV || 'development',
db = require('./db'),
http = require('http'),
nodeStatic = require('node-static'),
httpServer = require('./lib/httpServer'),
path = require('path'),
fs = require('fs'),
Steppy = require('twostep').Steppy,
@ -14,49 +13,51 @@ var env = process.env.NODE_ENV || 'development',
BuildsCollection = require('./lib/build').BuildsCollection,
libLogger = require('./lib/logger'),
EventEmitter = require('events').EventEmitter,
validateConfig = require('./lib/validateConfig');
validateConfig = require('./lib/validateConfig'),
utils = require('./lib/utils');
var app = new EventEmitter(),
logger = libLogger('app'),
httpApi;
logger = libLogger('app');
var staticPath = path.join(__dirname, 'static'),
staticServer = new nodeStatic.Server(staticPath),
staticDataServer;
var staticPath = path.join(__dirname, 'static');
var server = http.createServer(function(req, res) {
if (req.url.indexOf('/api/') === 0) {
return httpApi(req, res);
}
var httpServerLogger = libLogger('http server');
if (new RegExp('^/projects/(\\w|-)+/workspace').test(req.url)) {
return staticDataServer.serve(req, res);
}
app.httpServer = httpServer.create();
if (req.url.indexOf('/data.io.js') === -1) {
if (/(js|css|fonts|images)/.test(req.url)) {
staticServer.serve(req, res);
} else {
// serve index for all app pages
if (env === 'development') {
var jade = require('jade');
// Compile a function
var index = jade.compileFile(__dirname + '/views/index.jade');
res.write(index({env: env}));
res.end();
} else {
// serve index for all other pages (/builds/:id, etc)
fs.createReadStream(path.join(staticPath, 'index.html'))
.pipe(res);
}
}
app.httpServer.on('error', function(err, req, res) {
httpServerLogger.error(
'Error processing request ' + req.method + ' ' + req.url + ':',
err.stack || err
);
if (!res.headersSent) {
res.statusCode = 500;
res.end();
}
});
var socketio = require('socket.io')(server);
app.httpServer.addRequestListener(function(req, res, next) {
var start = Date.now();
res.on('finish', function() {
var end = Date.now();
httpServerLogger.log(
'[%s] %s %s %s - %s ms',
new Date(end).toUTCString(),
req.method,
req.url,
res.statusCode,
end - start
);
});
next();
});
var socketio = require('socket.io')(app.httpServer);
var dataio = require('./dataio')(socketio);
app.server = server;
app.dataio = dataio;
app.lib = {};
@ -149,7 +150,6 @@ Steppy(
// path to root dir (with projects, builds etc)
app.config.paths.data = path.join(process.cwd(), 'data');
staticDataServer = new nodeStatic.Server(app.config.paths.data);
app.config.paths.projects = path.join(app.config.paths.data, 'projects');
app.config.paths.db = path.join(app.config.paths.data, 'db');
@ -192,7 +192,7 @@ Steppy(
_(app.config).defaults(config);
_(app.config).defaults(configDefaults);
logger.log('Server config:', JSON.stringify(app.config, null, 4));
logger.log('Server config:', utils.toPrettyJson(app.config));
var dbBackend = require(app.config.storage.backend);
@ -230,14 +230,8 @@ Steppy(
require(plugin).register(app);
});
httpApi = require('./httpApi')(app);
notifier.init(app.config.notify, this.slot());
require('./projectsWatcher').init(app, this.slot());
require('./scheduler').init(app, this.slot());
// init resources
require('./resources')(app);
},
@ -245,6 +239,25 @@ Steppy(
// load projects after all plugins to provide ability for plugins to
// handle `projectLoaded` event
app.projects.loadAll(this.slot());
// serve index for all app pages, add this listener after all other
// listeners
app.httpServer.addRequestListener(function(req, res, next) {
if (req.url.indexOf('/data.io.js') === -1) {
if (env === 'development') {
var jade = require('jade');
// Compile a function
var index = jade.compileFile(__dirname + '/views/index.jade');
res.write(index({env: env}));
res.end();
} else {
fs.createReadStream(path.join(staticPath, 'index.html'))
.pipe(res);
}
} else {
next();
}
});
},
function(err) {
logger.log('Loaded projects: ', _(app.projects.getAll()).pluck('name'));
@ -252,7 +265,7 @@ Steppy(
var host = app.config.http.host,
port = app.config.http.port;
logger.log('Start http server on %s:%s', host, port);
app.server.listen(port, host);
app.httpServer.listen(port, host);
},
function(err) {
if (err) throw err;

13
data/config.yaml
View File

@ -1,7 +1,11 @@
# plugins:
plugins:
- nci-projects-reloader
- nci-static-server
- nci-rest-api-server
# - nci-mail-notification
# - nci-jabber-notification
# - nci-scheduler
nodes:
- type: local
@ -11,9 +15,16 @@ http:
host: 127.0.0.1
port: 3000
url: http://127.0.0.1:3000
static:
locations:
- url: !!js/regexp ^/(js|css|fonts|images)/
root: static/
- url: !!js/regexp ^/projects/(\w|-)+/workspace/
root: data/
storage:
backend: memdown
# backend: leveldown
notify:
mail:

1
docs/developing-plugins/builds-collection.md
View File

@ -43,6 +43,7 @@
- `params.status` - optional status filter, can be used only when
`params.projectName` is set. When used builds in the result will contain
only following fields: id, number, startDate, endDate
- `params.limit` - maximum builds count to get
## BuildsCollection.getDoneStreak(params:Object, callback(err,doneStreak):Function)

2
docs/developing-plugins/projects-collection.md
View File

@ -49,7 +49,7 @@
Remove project by name.
Calls `unload`, removes project from disk and db.
## ProjectsCollection.rename(name:String, [callback(err)]:Function)
## ProjectsCollection.rename(name:String, newName:String, [callback(err)]:Function)
Rename project.
Renames project on disk and db, also changes name for loaded project.

186
httpApi.js
View File

@ -1,186 +0,0 @@
'use strict';
var Steppy = require('twostep').Steppy,
_ = require('underscore'),
querystring = require('querystring');
/*
* Pure rest api on pure nodejs follows below
*/
var router = {};
router.routes = {};
_(['get', 'post', 'patch', 'delete']).each(function(method) {
router[method] = function(path, handler) {
this.routes[method] = this.routes[method] || [];
var keys = [],
regExpStr = path.replace(/:(\w+)/g, function(match, name) {
keys.push(name);
return '(.+)';
});
this.routes[method].push({
regExp: new RegExp('^' + regExpStr + '$'),
handler: handler,
keys: keys
});
};
});
router.del = router['delete'];
router.getRoute = function(req) {
var parts,
route = _(this.routes[req.method.toLowerCase()]).find(function(route) {
parts = route.regExp.exec(req.path);
return parts;
});
if (route && route.keys.length) {
route.params = {};
_(route.keys).each(function(key, index) {
route.params[key] = parts[index + 1];
});
}
return route;
};
module.exports = function(app) {
var logger = app.lib.logger('http api'),
accessToken = (Math.random() * Math.random()).toString(36).substring(2);
logger.log('access token is: %s', accessToken);
// run building of a project
router.post('/api/0.1/builds', function(req, res, next) {
Steppy(
function() {
var projectName = req.body.project,
project = app.projects.get(projectName);
if (project) {
res.statusCode = 204;
logger.log('Run project "%s"', projectName);
app.builds.create({
projectName: projectName,
withScmChangesOnly: req.body.withScmChangesOnly,
queueQueued: req.body.queueQueued,
initiator: {type: 'httpApi'}
});
} else {
res.statusCode = 404;
}
res.end();
},
next
);
});
router.del('/api/0.1/projects/:name', function(req, res, next) {
var token = req.body.token,
projectName = req.params.name;
Steppy(
function() {
logger.log('Cleaning up project "%s"', projectName);
if (token !== accessToken) {
throw new Error('Access token doesn`t match');
}
app.projects.remove(projectName, this.slot());
},
function() {
logger.log('Project "%s" cleaned up', projectName);
res.statusCode = 204;
res.end();
},
next
);
});
router.patch('/api/0.1/projects/:name', function(req, res, next) {
var token = req.body.token,
projectName = req.params.name,
newProjectName = req.body.name;
Steppy(
function() {
logger.log(
'Rename project "%s" to "%s"', projectName, newProjectName
);
if (token !== accessToken) {
throw new Error('Access token doesn`t match');
}
if (!newProjectName) throw new Error('new project name is not set');
var curProject = app.projects.get(projectName);
if (!curProject) {
throw new Error('Project "' + projectName + '" not found');
}
this.pass(curProject);
var newProject = app.projects.get(newProjectName);
if (newProject) {
throw new Error(
'Project name "' + newProjectName + '" already used'
);
}
app.projects.rename(projectName, newProjectName, this.slot());
},
function(err) {
res.statusCode = 204;
res.end();
},
next
);
});
return function(req, res) {
Steppy(
function() {
var stepCallback = this.slot();
var urlParts = req.url.split('?');
req.path = urlParts[0];
req.query = querystring.parse(urlParts[1]);
req.setEncoding('utf-8');
var bodyString = '';
req.on('data', function(data) {
bodyString += data;
});
req.on('end', function() {
var body = bodyString ? JSON.parse(bodyString) : {};
stepCallback(null, body);
});
req.on('error', stepCallback);
},
function(err, body) {
req.body = body;
var route = router.getRoute(req);
if (route) {
req.params = route.params;
route.handler(req, res, this.slot());
} else {
res.statusCode = 404;
res.end();
}
},
function(err) {
logger.error('Error occurred during request: ', err.stack || err);
res.statusCode = 500;
res.end();
}
);
};
};

2
lib/build.js
View File

@ -149,12 +149,12 @@ BuildsCollection.prototype.getAvgBuildDuration = function(builds) {
* - `params.status` - optional status filter, can be used only when
* `params.projectName` is set. When used builds in the result will contain
* only following fields: id, number, startDate, endDate
* - `params.limit` - maximum builds count to get
*
* @param {Object} params
* @param {Function} callback(err,builds)
*/
BuildsCollection.prototype.getRecent = function(params, callback) {
params.limit = params.limit || 20;
var self = this;
Steppy(

6
lib/executor/local.js
View File

@ -110,8 +110,10 @@ Executor.prototype._getSources = function(params, callback) {
);
};
Executor.prototype._runStep = function(params, callback) {
var self = this;
Executor.prototype._runStep = function(step, callback) {
var self = this,
params = _(step).clone();
Steppy(
function() {
if (params.type !== 'shell') {

40
lib/httpServer.js Normal file
View File

@ -0,0 +1,40 @@
'use strict';
var http = require('http'),
inherits = require('util').inherits;
function Server() {
var self = this;
self.requestListeners = [];
return http.Server.call(self, function(req, res) {
self._processRequestListeners(req, res, 0);
});
}
inherits(Server, http.Server);
Server.prototype.addRequestListener = function(requestListener) {
this.requestListeners.push(requestListener);
};
Server.prototype._processRequestListeners = function(req, res, index) {
var self = this;
self.requestListeners[index](req, res, function(err) {
if (err) {
self.emit('error', err, req, res);
} else {
index++;
if (self.requestListeners[index]) {
self._processRequestListeners(req, res, index);
}
}
});
};
exports.create = function() {
return new Server;
};

5
lib/project.js
View File

@ -40,6 +40,10 @@ ProjectsCollection.prototype.validateConfig = function(config, callback) {
validateParams(config, {
type: 'object',
properties: {
name: {
type: 'string',
pattern: /^(\w|-)+$/
},
scm: {
type: 'object',
required: true,
@ -286,6 +290,7 @@ ProjectsCollection.prototype.remove = function(name, callback) {
* Renames project on disk and db, also changes name for loaded project.
*
* @param {String} name
* @param {String} newName
* @param {Function} [callback(err)]
*/
ProjectsCollection.prototype.rename = function(name, newName, callback) {

10
lib/utils.js
View File

@ -22,3 +22,13 @@ exports.prune = function(str, length) {
exports.toNumberStr = function(number) {
return exports.lpad(String(number), 20);
};
exports.toPrettyJson = function(data) {
return JSON.stringify(data, function(key, value) {
if (_(value).isRegExp()) {
return 'RegExp ' + String(value);
} else {
return value;
}
}, 4);
};

6
package.json
View File

@ -53,16 +53,13 @@
"ansi_up": "1.3.0",
"bootstrap": "3.3.6",
"browserify": "12.0.1",
"chokidar": "1.0.3",
"colors": "1.1.2",
"conform": "0.2.12",
"cron": "1.0.9",
"data.io": "0.3.0",
"font-awesome": "4.5.0",
"history": "1.13.1",
"moment": "2.10.6",
"nlevel": "1.0.3",
"node-static": "0.7.6",
"react-dom": "0.14.3",
"react-jade": "2.5.0",
"react-router": "0.13.5",
@ -82,6 +79,9 @@
"less": "2.5.3",
"memdown": "1.1.0",
"mocha": "1.18.2",
"nci-projects-reloader": "0.1.1",
"nci-rest-api-server": "0.1.1",
"nci-static-server": "0.1.0",
"nci-yaml-reader": "0.1.0",
"nodemon": "1.3.7",
"nrun": "0.1.4",

55
projectsWatcher.js
View File

@ -1,55 +0,0 @@
'use strict';
var _ = require('underscore'),
path = require('path'),
chokidar = require('chokidar');
exports.init = function(app, callback) {
var logger = app.lib.logger('projects watcher');
// start file watcher for reloading projects on change
var syncProject = function(filename, fileInfo) {
var projectName = path.relative(
app.config.paths.projects,
path.dirname(filename)
);
if (app.projects.get(projectName)) {
logger.log('Unload project: "' + projectName + '"');
app.projects.unload(projectName);
}
// on add or change (info is falsy on unlink)
if (fileInfo) {
logger.log('Load project "' + projectName + '" on change');
app.projects.load(projectName, function(err) {
if (err) {
return logger.error(
'Error during load project "' + projectName + '": ',
err.stack || err
);
}
logger.log(
'Project "' + projectName + '" loaded:',
JSON.stringify(app.projects.get(projectName), null, 4)
);
});
}
};
// NOTE: currently after add remove and then add same file events will
// not be emitted
var watcher = chokidar.watch(
path.join(app.config.paths.projects, '*', 'config.*'),
{ignoreInitial: true, depth: 1}
);
watcher.on('add', syncProject);
watcher.on('change', syncProject);
watcher.on('unlink', syncProject);
watcher.on('error', function(err) {
logger.error('File watcher error occurred: ', err.stack || err);
});
callback();
};

2
resources/builds.js
View File

@ -11,7 +11,7 @@ module.exports = function(app) {
Steppy(
function() {
var data = req.data || {},
getParams = {limit: data.limit || 20};
getParams = {limit: Number(data.limit) || 20};
if (data.projectName) {
getParams.projectName = data.projectName;

44
scheduler.js
View File

@ -1,44 +0,0 @@
'use strict';
var _ = require('underscore'),
CronJob = require('cron').CronJob;
exports.init = function(app, callback) {
var logger = app.lib.logger('scheduler'),
projectJobs = {};
app.projects.on('projectLoaded', function(project) {
var time = project.buildEvery && project.buildEvery.time;
if (time) {
logger.log(
'Start job for loaded project "%s" by schedule "%s"',
project.name,
time
);
projectJobs[project.name] = {};
projectJobs[project.name].job = new CronJob({
cronTime: time,
onTick: function() {
logger.log('Run project "%s"', project.name);
app.builds.create({
projectName: project.name,
withScmChangesOnly: project.buildEvery.withScmChangesOnly,
initiator: {type: 'scheduler'}
});
},
start: true
});
}
});
app.projects.on('projectUnloaded', function(project) {
if (project.name in projectJobs) {
logger.log('Stop job for unloaded project "%s"', project.name);
projectJobs[project.name].job.stop();
delete projectJobs[project.name];
}
});
callback();
};