martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added some new feeds

Browse Source
This commit is contained in:
Martin Donnelly 2018-06-20 11:22:12 +01:00
parent 7f893a0b15
commit 56543b6cd7
4 changed files with 165 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
.eslintrc
View File

@ -1,22 +1,55 @@
{
"parserOptions": {
"ecmaVersion": 2017,
"sourceType": "module",
"ecmaFeatures": {
"jsx": false
}
},
"env": {
"browser": true,
"commonjs": true,
"es6": true,
"node": true
},
"parserOptions": {
"sourceType": "module"
"node": true,
"es6": true
},
"rules": {
"no-const-assign": "warn",
"no-this-before-super": "warn",
"no-undef": "warn",
"no-unreachable": "warn",
"no-unused-vars": "warn",
"constructor-super": "warn",
"valid-typeof": "warn",
"indent": [2, 2, {SwitchCase: 1}]
"arrow-spacing": "error",
"block-scoped-var": "error",
"block-spacing": "error",
"brace-style": ["error", "stroustrup", {}],
"camelcase": "error",
"comma-dangle": ["error", "never"],
"comma-spacing": ["error", { "before": false, "after": true }],
"comma-style": [1, "last"],
"consistent-this": [1, "_this"],
"curly": [1, "multi"],
"eol-last": 1,
"eqeqeq": 1,
"func-names": 1,
"indent": ["error", 2, { "SwitchCase": 1 }],
"lines-around-comment": ["error", { "beforeBlockComment": true, "allowArrayStart": true }],
"max-len": [1, 180, 2], // 2 spaces per tab, max 80 chars per line
"new-cap": 1,
"newline-before-return": "error",
"no-array-constructor": 1,
"no-inner-declarations": [1, "both"],
"no-mixed-spaces-and-tabs": 1,
"no-multi-spaces": 2,
"no-new-object": 1,
"no-shadow-restricted-names": 1,
"object-curly-spacing": ["error", "always"],
"padded-blocks": ["error", { "blocks": "never", "switches": "always" }],
"prefer-const": "error",
"prefer-template": "error",
"one-var": 0,
"quote-props": ["error", "always"],
"quotes": [1, "single"],
"radix": 1,
"semi": [1, "always"],
"space-before-blocks": [1, "always"],
"space-infix-ops": 1,
"vars-on-top": 1,
"no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 1 }],
"spaced-comment": ["error", "always", { "markers": ["/"] }]
}
}

136
.idea/workspace.xml
View File

@ -2,8 +2,10 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="">
<change beforePath="$PROJECT_DIR$/.eslintrc" beforeDir="false" afterPath="$PROJECT_DIR$/.eslintrc" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/feeds/jobs-special.json" beforeDir="false" afterPath="$PROJECT_DIR$/feeds/jobs-special.json" afterDir="false" />
<change beforePath="$PROJECT_DIR$/gather.js" beforeDir="false" afterPath="$PROJECT_DIR$/gather.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
</list>
<ignored path="rinser.iws" />
<ignored path=".idea/workspace.xml" />
@ -25,11 +27,11 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="true">
<file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="834">
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" />
<state relative-caret-position="677">
<caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
</state>
</provider>
</entry>
@ -70,11 +72,20 @@
</provider>
</entry>
</file>
<file leaf-file-name="gather.js" pinned="false" current-in-tab="false">
<file leaf-file-name="gather.js" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="835">
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" />
<state relative-caret-position="346">
<caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name=".eslintrc" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="583">
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
</state>
</provider>
</entry>
@ -105,15 +116,6 @@
</provider>
</entry>
</file>
<file leaf-file-name="RssBraider.js" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="11" column="4" selection-start-line="11" selection-start-column="4" selection-end-line="11" selection-end-column="4" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
@ -126,9 +128,6 @@
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>nodejs</find>
<find>node</find>
<find>getstop</find>
<find>stripped</find>
<find>scot</find>
<find>html</find>
@ -156,6 +155,9 @@
<find>jobsite</find>
<find>4E2AC50E02AD128B</find>
<find>aspx</find>
<find>cw</find>
<find>braider_options</find>
<find>fs_writeFile</find>
</findStrings>
<replaceStrings>
<replace />
@ -207,7 +209,6 @@
<option value="$PROJECT_DIR$/dist/jobs-special.xml" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch.txt" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" />
<option value="$PROJECT_DIR$/.eslintrc" />
<option value="$PROJECT_DIR$/jobs-new.js" />
<option value="$PROJECT_DIR$/jobs-corpus.js" />
<option value="$PROJECT_DIR$/package.json" />
@ -225,9 +226,10 @@
<option value="$PROJECT_DIR$/feeds/fit.json" />
<option value="$PROJECT_DIR$/feeds/tech.json" />
<option value="$PROJECT_DIR$/feeds/bored.json" />
<option value="$PROJECT_DIR$/gather.js" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt" />
<option value="$PROJECT_DIR$/feeds/jobs-special.json" />
<option value="$PROJECT_DIR$/.eslintrc" />
<option value="$PROJECT_DIR$/gather.js" />
</list>
</option>
</component>
@ -297,11 +299,6 @@
<item name="rinser" type="b2602c69:ProjectViewProjectNode" />
<item name="Rinser" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="rinser" type="b2602c69:ProjectViewProjectNode" />
<item name="Rinser" type="462c0819:PsiDirectoryNode" />
<item name="feeds" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
@ -341,10 +338,10 @@
<property name="node.js.detected.package.jscs" value="true" />
<property name="node.js.detected.package.prettier" value="true" />
<property name="node.js.detected.package.standard" value="true" />
<property name="node.js.path.for.package.eslint" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" />
<property name="node.js.path.for.package.eslint" value="node" />
<property name="node.js.path.for.package.jscs" value="project" />
<property name="node.js.path.for.package.prettier" value="project" />
<property name="node.js.path.for.package.standard" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" />
<property name="node.js.path.for.package.standard" value="node" />
<property name="node.js.selected.package.eslint" value="$PROJECT_DIR$/node_modules/eslint" />
<property name="node.js.selected.package.jscs" value="" />
<property name="node.js.selected.package.prettier" value="" />
@ -352,7 +349,7 @@
<property name="nodejs_interpreter_path" value="$USER_HOME$/.nvm/versions/node/v8.11.3/bin/node" />
<property name="nodejs_package_manager_path" value="npm" />
<property name="recentsLimit" value="5" />
<property name="settings.editor.selected.configurable" value="settings.nodejs" />
<property name="settings.editor.selected.configurable" value="settings.javascript.linters.eslint" />
</component>
<component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS">
@ -560,14 +557,7 @@
<workItem from="1520180489311" duration="993000" />
<workItem from="1528921135827" duration="17718000" />
<workItem from="1529358967248" duration="4995000" />
<workItem from="1529481827961" duration="2667000" />
</task>
<task id="LOCAL-00068" summary="fixed plugins">
<created>1471949182374</created>
<option name="number" value="00068" />
<option name="presentableId" value="LOCAL-00068" />
<option name="project" value="LOCAL" />
<updated>1471949182375</updated>
<workItem from="1529481827961" duration="3711000" />
</task>
<task id="LOCAL-00069" summary="fixed plugins">
<created>1472027998982</created>
@ -905,22 +895,29 @@
<option name="project" value="LOCAL" />
<updated>1529363450787</updated>
</task>
<option name="localTasksCounter" value="117" />
<task id="LOCAL-00117" summary="added some new feeds">
<created>1529489048452</created>
<option name="number" value="00117" />
<option name="presentableId" value="LOCAL-00117" />
<option name="project" value="LOCAL" />
<updated>1529489048452</updated>
</task>
<option name="localTasksCounter" value="118" />
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="135583000" />
<option name="totallyTimeSpent" value="136627000" />
</component>
<component name="ToolWindowManager">
<frame x="17" y="43" width="1569" height="1106" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Gulp" order="2" sideWeight="0.40669373" side_tool="true" visible="true" weight="0.20219639" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5933063" visible="true" weight="0.20219639" />
<window_info id="Gulp" order="2" sideWeight="0.40695915" side_tool="true" visible="true" weight="0.20219639" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5930408" visible="true" weight="0.20219639" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.36999437" side_tool="true" weight="0.24975222" />
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.0" />
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.3296146" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.63000566" weight="0.24975222" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.4838976" weight="0.32838774" />
@ -1020,7 +1017,6 @@
<component name="VcsManagerConfiguration">
<option name="CHECK_CODE_SMELLS_BEFORE_PROJECT_COMMIT" value="false" />
<option name="CHECK_NEW_TODO" value="false" />
<MESSAGE value="Updating to Version 3" />
<MESSAGE value="limit image size to 900px" />
<MESSAGE value="force images through our image server" />
<MESSAGE value="Max images out at 1140px wide" />
@ -1045,7 +1041,8 @@
<MESSAGE value="all feeds using stone" />
<MESSAGE value="added gather" />
<MESSAGE value="added bored" />
<option name="LAST_COMMIT_MESSAGE" value="added bored" />
<MESSAGE value="added some new feeds" />
<option name="LAST_COMMIT_MESSAGE" value="added some new feeds" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
@ -1060,11 +1057,6 @@
<line>17</line>
<option name="timeStamp" value="8" />
</line-breakpoint>
<line-breakpoint enabled="true" type="javascript">
<url>file://$PROJECT_DIR$/gather.js</url>
<line>70</line>
<option name="timeStamp" value="10" />
</line-breakpoint>
</breakpoints>
<option name="time" value="11" />
</breakpoint-manager>
@ -1153,13 +1145,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" column="8" selection-start-line="19" selection-start-column="8" selection-end-line="19" selection-end-column="8" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/feedparser/History.md">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="284">
@ -1335,13 +1320,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="835">
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/jobs.js">
<provider selected="true" editor-type-id="text-editor" />
</entry>
@ -1376,13 +1354,6 @@
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/jobs-new.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="945">
@ -1392,8 +1363,29 @@
</entry>
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="834">
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" />
<state relative-caret-position="677">
<caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="583">
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="346">
<caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
</state>
</provider>
</entry>

118
gather.js
View File

@ -1,23 +1,16 @@
const RssBraider = require('rss-braider');
const fs = require('fs');
const ejs = require(
'ejs');
const read = require('fs').readFileSync;
const join = require('path').join;
const cron = require('node-cron');
const util = require('util')
const fs_writeFile = util.promisify(fs.writeFile)
const util = require('util');
const fsWriteFile = util.promisify(fs.writeFile);
const str = read(
join(__dirname, '/templates/rss.ejs'),
'utf8');
const feeds = {};
const log4js = require('log4js');
const logger = log4js.getLogger();
logger.level = 'trace';
let timerID = 0;
// Pull feeds from config files: ,
// feeds.simple_test_feed = require("./config/feed").feed;
@ -32,99 +25,58 @@ feeds.tech = require('./feeds/tech.json');
feeds.fit = require('./feeds/fit.json');
feeds.bored = require('./feeds/bored.json');
const braider_options = {
feeds: feeds,
indent: ' ',
date_sort_order: 'desc', // Newest first
log_level: 'warn',
dedupe_fields: ['link', 'guid'],
plugins_directories: [__dirname + '/plugins/']
const braiderOptions = {
feeds,
'indent': ' ',
'date_sort_order': 'desc', // Newest first
'log_level': 'warn',
'dedupe_fields': ['link', 'guid'],
'plugins_directories': [`${__dirname }/plugins/`]
};
const rss_braider = RssBraider.createClient(braider_options);
const rssBraider = RssBraider.createClient(braiderOptions);
// Override logging level (debug, info, warn, err, off)
//rss_braider.logger.level('off');
// rssBraider.logger.level('off');
function gatherV2(feedName, fileName, mode) {
return new Promise((resolve, reject) => {
const modeSuffix = {'rss': '.xml', 'json': '.json'};
const modeSuffix = { 'rss': '.xml', 'json': '.json' };
logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`);
rss_braider.processFeed(feedName, mode, (err, data) => {
rssBraider.processFeed(feedName, mode, (err, data) => {
if (err) {
logger.error(err);
return reject(err);
}
fs_writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function(){
fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () {
return resolve(`${feedName} saved`);
});
});
});
}
function toHour(extra = 0) {
const now = new Date();
async function gather(feedName, fileName) {
logger.info(`Gathering ${feedName}...`);
rss_braider.processFeed(feedName, 'json', function (err, data) {
console.log('A');
if (err) {
logger.error(err);
return err;
}
console.log('B');
const j = JSON.parse(data);
console.log('C');
const ejsOutput = ejs.compile(str)(j);
console.log('D');
fs.writeFile(__dirname + '/dist/' + fileName + '.html',
ejsOutput,
function (err) {
if (err) {
logger.error(err);
return console.error(err);
}
console.log('The file was saved!');
});
fs.writeFile(__dirname + '/dist/' + fileName + '.json',
data,
function (err) {
if (err) {
logger.error(err);
return console.error(err);
}
console.log('The file was saved!');
});
});
return (3600000 - (now.getTime() % 3600000)) + extra;
}
async function gatherXML(feedName, fileName) {
rss_braider.processFeed(feedName, 'rss', function (err, data) {
if (err) {
rssBraider.processFeed(feedName, 'rss', function (err, data) {
if (err)
return console.log(err);
}
console.log('Saving', __dirname + '/dist/' + fileName + '.xml');
fs.writeFile(__dirname + '/dist/' + fileName + '.xml', data, function (err) {
if (err) {
console.log('Saving', `${__dirname }/dist/${ fileName }.xml`);
fs.writeFile(`${__dirname }/dist/${ fileName }.xml`, data, function (err) {
if (err)
return console.log(err);
}
console.log('The file was saved!');
});
});
}
async function main() {
await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
logger.debug(d);
@ -164,10 +116,20 @@ async function main() {
// await gatherXML('jobsLocal','jobs-local');
}
cron.schedule('3 * * * *', function () {
function tick() {
const extra = Math.floor((Math.random() * 60) + 1) * 1000;
main();
return -1;
});
// main();
timerID = setTimeout(
() => tick(),
toHour(extra)
);
}
(() => {
logger.info('Gather started...');
timerID = setTimeout(
() => tick(),
toHour()
);
})();

6
package-lock.json generated
View File

@ -159,9 +159,9 @@
}
},
"acorn": {
"version": "5.6.2",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-5.6.2.tgz",
"integrity": "sha512-zUzo1E5dI2Ey8+82egfnttyMlMZ2y0D8xOCO3PNPPlYXpl8NZvF6Qk9L9BEtJs+43FqEmfBViDqc5d1ckRDguw==",
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz",
"integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==",
"dev": true
},
"acorn-jsx": {