martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

does xml now too

Browse Source
This commit is contained in:
Martin Donnelly 2017-05-31 16:20:56 +01:00
parent 95cbb9e0a9
commit 96611ae34c
1 changed files with 51 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
jobs-corpus.js
View File

@ -2,7 +2,7 @@ var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = requi
'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require(
'path').join, str = read(join(__dirname, '/templates/rss.ejs'),
'utf8'), feeds = {};
var RSS = require('rss');
var log4js = require('log4js');
var logger = log4js.getLogger();
const stopwords = require('stopwords-en');
@ -204,49 +204,68 @@ var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('error');
rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
if (err) {
return console.log(err);
}
function doFeeds() {
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
if (err) {
return console.log(err);
}
var j = JSON.parse(data);
var j = JSON.parse(data);
j = processJson(j);
/*for (let mm in j) {
console.log(mm);
}*/
var ejsOutput = ejs.compile(str)(j);
j = processJson(j);
/*for (let mm in j) {
console.log(mm);
}*/
var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/dist/" + outputFile + ".html",
ejsOutput,
function(err) {
fs.writeFile(__dirname + "/dist/" + outputFile + ".html",
ejsOutput,
function (err) {
if (err) {
return console.log(err);
}
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
console.log("The file was saved!");
});
fs.writeFile(__dirname + "/dist/" + outputFile + ".json",
data,
function(err) {
if (err) {
logger.error(err);
return console.log(err);
}
fs.writeFile(__dirname + "/dist/" + outputFile + ".json",
data,
function (err) {
if (err) {
logger.error(err);
return console.log(err);
}
console.log("The file was saved!");
});
saveCorpus();
var newfeed = new RSS({}, j.items);
// console.log(newfeed);
fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", newfeed.xml(' '), function(err) {
if (err) {
return console.log(err);
}
console.log("The XML file was saved!");
});
//
});
}
console.log("The file was saved!");
});
saveCorpus();
});
function scoreEntry(s) {
let score = 0;
let used = [];
for(let t of s) {
console.log(`processing: ${t}`);
//console.log(`processing: ${t}`);
if (!corpus.hasOwnProperty(t)) {
corpus[t] = {score:0, common:1};
@ -304,7 +323,7 @@ function getNoneStopWords(sentence) {
function doFeeds() {
function doFeedsXML() {
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) {
if (err) {
return console.log(err);