martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

does xml now too

Browse Source
This commit is contained in:
Martin Donnelly 2017-05-31 16:20:56 +01:00
parent 95cbb9e0a9
commit 96611ae34c
1 changed files with 51 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
jobs-corpus.js
View File

@ -2,7 +2,7 @@ var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = requi
'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require( 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require(
'path').join, str = read(join(__dirname, '/templates/rss.ejs'), 'path').join, str = read(join(__dirname, '/templates/rss.ejs'),
'utf8'), feeds = {}; 'utf8'), feeds = {};
var RSS = require('rss');
var log4js = require('log4js'); var log4js = require('log4js');
var logger = log4js.getLogger(); var logger = log4js.getLogger();
const stopwords = require('stopwords-en'); const stopwords = require('stopwords-en');
@ -204,49 +204,68 @@ var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off) // Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('error'); rss_braider.logger.level('error');
rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { function doFeeds() {
if (err) { rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
return console.log(err); if (err) {
} return console.log(err);
}
var j = JSON.parse(data); var j = JSON.parse(data);
j = processJson(j); j = processJson(j);
/*for (let mm in j) { /*for (let mm in j) {
console.log(mm); console.log(mm);
}*/ }*/
var ejsOutput = ejs.compile(str)(j); var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/dist/" + outputFile + ".html", fs.writeFile(__dirname + "/dist/" + outputFile + ".html",
ejsOutput, ejsOutput,
function(err) { function (err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
console.log("The file was saved!"); console.log("The file was saved!");
}); });
fs.writeFile(__dirname + "/dist/" + outputFile + ".json", fs.writeFile(__dirname + "/dist/" + outputFile + ".json",
data, data,
function(err) { function (err) {
if (err) { if (err) {
logger.error(err); logger.error(err);
return console.log(err); return console.log(err);
} }
console.log("The file was saved!");
});
saveCorpus();
var newfeed = new RSS({}, j.items);
// console.log(newfeed);
fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", newfeed.xml(' '), function(err) {
if (err) {
return console.log(err);
}
console.log("The XML file was saved!");
});
//
});
}
console.log("The file was saved!");
});
saveCorpus();
});
function scoreEntry(s) { function scoreEntry(s) {
let score = 0; let score = 0;
let used = []; let used = [];
for(let t of s) { for(let t of s) {
console.log(`processing: ${t}`); //console.log(`processing: ${t}`);
if (!corpus.hasOwnProperty(t)) { if (!corpus.hasOwnProperty(t)) {
corpus[t] = {score:0, common:1}; corpus[t] = {score:0, common:1};
@ -304,7 +323,7 @@ function getNoneStopWords(sentence) {
function doFeeds() { function doFeedsXML() {
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) { rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);