martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added some new feeds

Browse Source
This commit is contained in:
Martin Donnelly 2018-06-20 11:22:12 +01:00
parent 7f893a0b15
commit 56543b6cd7
4 changed files with 165 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
.eslintrc
View File

@ -1,22 +1,55 @@
{ {
"env": { "parserOptions": {
"browser": true, "ecmaVersion": 2017,
"commonjs": true, "sourceType": "module",
"es6": true, "ecmaFeatures": {
"node": true "jsx": false
},
"parserOptions": {
"sourceType": "module"
},
"rules": {
"no-const-assign": "warn",
"no-this-before-super": "warn",
"no-undef": "warn",
"no-unreachable": "warn",
"no-unused-vars": "warn",
"constructor-super": "warn",
"valid-typeof": "warn",
"indent": [2, 2, {SwitchCase: 1}]
} }
},
"env": {
"browser": true,
"node": true,
"es6": true
},
"rules": {
"arrow-spacing": "error",
"block-scoped-var": "error",
"block-spacing": "error",
"brace-style": ["error", "stroustrup", {}],
"camelcase": "error",
"comma-dangle": ["error", "never"],
"comma-spacing": ["error", { "before": false, "after": true }],
"comma-style": [1, "last"],
"consistent-this": [1, "_this"],
"curly": [1, "multi"],
"eol-last": 1,
"eqeqeq": 1,
"func-names": 1,
"indent": ["error", 2, { "SwitchCase": 1 }],
"lines-around-comment": ["error", { "beforeBlockComment": true, "allowArrayStart": true }],
"max-len": [1, 180, 2], // 2 spaces per tab, max 80 chars per line
"new-cap": 1,
"newline-before-return": "error",
"no-array-constructor": 1,
"no-inner-declarations": [1, "both"],
"no-mixed-spaces-and-tabs": 1,
"no-multi-spaces": 2,
"no-new-object": 1,
"no-shadow-restricted-names": 1,
"object-curly-spacing": ["error", "always"],
"padded-blocks": ["error", { "blocks": "never", "switches": "always" }],
"prefer-const": "error",
"prefer-template": "error",
"one-var": 0,
"quote-props": ["error", "always"],
"quotes": [1, "single"],
"radix": 1,
"semi": [1, "always"],
"space-before-blocks": [1, "always"],
"space-infix-ops": 1,
"vars-on-top": 1,
"no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 1 }],
"spaced-comment": ["error", "always", { "markers": ["/"] }]
}
} }

136
.idea/workspace.xml
View File

@ -2,8 +2,10 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment=""> <list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="">
<change beforePath="$PROJECT_DIR$/.eslintrc" beforeDir="false" afterPath="$PROJECT_DIR$/.eslintrc" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/feeds/jobs-special.json" beforeDir="false" afterPath="$PROJECT_DIR$/feeds/jobs-special.json" afterDir="false" /> <change beforePath="$PROJECT_DIR$/gather.js" beforeDir="false" afterPath="$PROJECT_DIR$/gather.js" afterDir="false" />
<change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
</list> </list>
<ignored path="rinser.iws" /> <ignored path="rinser.iws" />
<ignored path=".idea/workspace.xml" /> <ignored path=".idea/workspace.xml" />
@ -25,11 +27,11 @@
</component> </component>
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300"> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="true"> <file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json"> <entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="834"> <state relative-caret-position="677">
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" /> <caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -70,11 +72,20 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="gather.js" pinned="false" current-in-tab="false"> <file leaf-file-name="gather.js" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/gather.js"> <entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="835"> <state relative-caret-position="346">
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" /> <caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
</state>
</provider>
</entry>
</file>
<file leaf-file-name=".eslintrc" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="583">
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -105,15 +116,6 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="RssBraider.js" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="165">
<caret line="11" column="4" selection-start-line="11" selection-start-column="4" selection-end-line="11" selection-end-column="4" />
</state>
</provider>
</entry>
</file>
</leaf> </leaf>
</component> </component>
<component name="FileTemplateManagerImpl"> <component name="FileTemplateManagerImpl">
@ -126,9 +128,6 @@
</component> </component>
<component name="FindInProjectRecents"> <component name="FindInProjectRecents">
<findStrings> <findStrings>
<find>nodejs</find>
<find>node</find>
<find>getstop</find>
<find>stripped</find> <find>stripped</find>
<find>scot</find> <find>scot</find>
<find>html</find> <find>html</find>
@ -156,6 +155,9 @@
<find>jobsite</find> <find>jobsite</find>
<find>4E2AC50E02AD128B</find> <find>4E2AC50E02AD128B</find>
<find>aspx</find> <find>aspx</find>
<find>cw</find>
<find>braider_options</find>
<find>fs_writeFile</find>
</findStrings> </findStrings>
<replaceStrings> <replaceStrings>
<replace /> <replace />
@ -207,7 +209,6 @@
<option value="$PROJECT_DIR$/dist/jobs-special.xml" /> <option value="$PROJECT_DIR$/dist/jobs-special.xml" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch.txt" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/scratch.txt" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" />
<option value="$PROJECT_DIR$/.eslintrc" />
<option value="$PROJECT_DIR$/jobs-new.js" /> <option value="$PROJECT_DIR$/jobs-new.js" />
<option value="$PROJECT_DIR$/jobs-corpus.js" /> <option value="$PROJECT_DIR$/jobs-corpus.js" />
<option value="$PROJECT_DIR$/package.json" /> <option value="$PROJECT_DIR$/package.json" />
@ -225,9 +226,10 @@
<option value="$PROJECT_DIR$/feeds/fit.json" /> <option value="$PROJECT_DIR$/feeds/fit.json" />
<option value="$PROJECT_DIR$/feeds/tech.json" /> <option value="$PROJECT_DIR$/feeds/tech.json" />
<option value="$PROJECT_DIR$/feeds/bored.json" /> <option value="$PROJECT_DIR$/feeds/bored.json" />
<option value="$PROJECT_DIR$/gather.js" />
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt" />
<option value="$PROJECT_DIR$/feeds/jobs-special.json" /> <option value="$PROJECT_DIR$/feeds/jobs-special.json" />
<option value="$PROJECT_DIR$/.eslintrc" />
<option value="$PROJECT_DIR$/gather.js" />
</list> </list>
</option> </option>
</component> </component>
@ -297,11 +299,6 @@
<item name="rinser" type="b2602c69:ProjectViewProjectNode" /> <item name="rinser" type="b2602c69:ProjectViewProjectNode" />
<item name="Rinser" type="462c0819:PsiDirectoryNode" /> <item name="Rinser" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="rinser" type="b2602c69:ProjectViewProjectNode" />
<item name="Rinser" type="462c0819:PsiDirectoryNode" />
<item name="feeds" type="462c0819:PsiDirectoryNode" />
</path>
</expand> </expand>
<select /> <select />
</subPane> </subPane>
@ -341,10 +338,10 @@
<property name="node.js.detected.package.jscs" value="true" /> <property name="node.js.detected.package.jscs" value="true" />
<property name="node.js.detected.package.prettier" value="true" /> <property name="node.js.detected.package.prettier" value="true" />
<property name="node.js.detected.package.standard" value="true" /> <property name="node.js.detected.package.standard" value="true" />
<property name="node.js.path.for.package.eslint" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" /> <property name="node.js.path.for.package.eslint" value="node" />
<property name="node.js.path.for.package.jscs" value="project" /> <property name="node.js.path.for.package.jscs" value="project" />
<property name="node.js.path.for.package.prettier" value="project" /> <property name="node.js.path.for.package.prettier" value="project" />
<property name="node.js.path.for.package.standard" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" /> <property name="node.js.path.for.package.standard" value="node" />
<property name="node.js.selected.package.eslint" value="$PROJECT_DIR$/node_modules/eslint" /> <property name="node.js.selected.package.eslint" value="$PROJECT_DIR$/node_modules/eslint" />
<property name="node.js.selected.package.jscs" value="" /> <property name="node.js.selected.package.jscs" value="" />
<property name="node.js.selected.package.prettier" value="" /> <property name="node.js.selected.package.prettier" value="" />
@ -352,7 +349,7 @@
<property name="nodejs_interpreter_path" value="$USER_HOME$/.nvm/versions/node/v8.11.3/bin/node" /> <property name="nodejs_interpreter_path" value="$USER_HOME$/.nvm/versions/node/v8.11.3/bin/node" />
<property name="nodejs_package_manager_path" value="npm" /> <property name="nodejs_package_manager_path" value="npm" />
<property name="recentsLimit" value="5" /> <property name="recentsLimit" value="5" />
<property name="settings.editor.selected.configurable" value="settings.nodejs" /> <property name="settings.editor.selected.configurable" value="settings.javascript.linters.eslint" />
</component> </component>
<component name="RecentsManager"> <component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS"> <key name="MoveFile.RECENT_KEYS">
@ -560,14 +557,7 @@
<workItem from="1520180489311" duration="993000" /> <workItem from="1520180489311" duration="993000" />
<workItem from="1528921135827" duration="17718000" /> <workItem from="1528921135827" duration="17718000" />
<workItem from="1529358967248" duration="4995000" /> <workItem from="1529358967248" duration="4995000" />
<workItem from="1529481827961" duration="2667000" /> <workItem from="1529481827961" duration="3711000" />
</task>
<task id="LOCAL-00068" summary="fixed plugins">
<created>1471949182374</created>
<option name="number" value="00068" />
<option name="presentableId" value="LOCAL-00068" />
<option name="project" value="LOCAL" />
<updated>1471949182375</updated>
</task> </task>
<task id="LOCAL-00069" summary="fixed plugins"> <task id="LOCAL-00069" summary="fixed plugins">
<created>1472027998982</created> <created>1472027998982</created>
@ -905,22 +895,29 @@
<option name="project" value="LOCAL" /> <option name="project" value="LOCAL" />
<updated>1529363450787</updated> <updated>1529363450787</updated>
</task> </task>
<option name="localTasksCounter" value="117" /> <task id="LOCAL-00117" summary="added some new feeds">
<created>1529489048452</created>
<option name="number" value="00117" />
<option name="presentableId" value="LOCAL-00117" />
<option name="project" value="LOCAL" />
<updated>1529489048452</updated>
</task>
<option name="localTasksCounter" value="118" />
<servers /> <servers />
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="135583000" /> <option name="totallyTimeSpent" value="136627000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="17" y="43" width="1569" height="1106" extended-state="0" /> <frame x="17" y="43" width="1569" height="1106" extended-state="0" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info id="Gulp" order="2" sideWeight="0.40669373" side_tool="true" visible="true" weight="0.20219639" /> <window_info id="Gulp" order="2" sideWeight="0.40695915" side_tool="true" visible="true" weight="0.20219639" />
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5933063" visible="true" weight="0.20219639" /> <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5930408" visible="true" weight="0.20219639" />
<window_info anchor="bottom" id="TODO" order="6" /> <window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" /> <window_info anchor="bottom" id="Docker" order="7" />
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.36999437" side_tool="true" weight="0.24975222" /> <window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.36999437" side_tool="true" weight="0.24975222" />
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.0" /> <window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.3296146" />
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.63000566" weight="0.24975222" /> <window_info anchor="bottom" id="Run" order="2" sideWeight="0.63000566" weight="0.24975222" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" /> <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.4838976" weight="0.32838774" /> <window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.4838976" weight="0.32838774" />
@ -1020,7 +1017,6 @@
<component name="VcsManagerConfiguration"> <component name="VcsManagerConfiguration">
<option name="CHECK_CODE_SMELLS_BEFORE_PROJECT_COMMIT" value="false" /> <option name="CHECK_CODE_SMELLS_BEFORE_PROJECT_COMMIT" value="false" />
<option name="CHECK_NEW_TODO" value="false" /> <option name="CHECK_NEW_TODO" value="false" />
<MESSAGE value="Updating to Version 3" />
<MESSAGE value="limit image size to 900px" /> <MESSAGE value="limit image size to 900px" />
<MESSAGE value="force images through our image server" /> <MESSAGE value="force images through our image server" />
<MESSAGE value="Max images out at 1140px wide" /> <MESSAGE value="Max images out at 1140px wide" />
@ -1045,7 +1041,8 @@
<MESSAGE value="all feeds using stone" /> <MESSAGE value="all feeds using stone" />
<MESSAGE value="added gather" /> <MESSAGE value="added gather" />
<MESSAGE value="added bored" /> <MESSAGE value="added bored" />
<option name="LAST_COMMIT_MESSAGE" value="added bored" /> <MESSAGE value="added some new feeds" />
<option name="LAST_COMMIT_MESSAGE" value="added some new feeds" />
</component> </component>
<component name="XDebuggerManager"> <component name="XDebuggerManager">
<breakpoint-manager> <breakpoint-manager>
@ -1060,11 +1057,6 @@
<line>17</line> <line>17</line>
<option name="timeStamp" value="8" /> <option name="timeStamp" value="8" />
</line-breakpoint> </line-breakpoint>
<line-breakpoint enabled="true" type="javascript">
<url>file://$PROJECT_DIR$/gather.js</url>
<line>70</line>
<option name="timeStamp" value="10" />
</line-breakpoint>
</breakpoints> </breakpoints>
<option name="time" value="11" /> <option name="time" value="11" />
</breakpoint-manager> </breakpoint-manager>
@ -1153,13 +1145,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="285">
<caret line="19" column="8" selection-start-line="19" selection-start-column="8" selection-end-line="19" selection-end-column="8" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/feedparser/History.md"> <entry file="file://$PROJECT_DIR$/node_modules/feedparser/History.md">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="284"> <state relative-caret-position="284">
@ -1335,13 +1320,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="835">
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/jobs.js"> <entry file="file://$PROJECT_DIR$/jobs.js">
<provider selected="true" editor-type-id="text-editor" /> <provider selected="true" editor-type-id="text-editor" />
</entry> </entry>
@ -1376,13 +1354,6 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/jobs-new.js"> <entry file="file://$PROJECT_DIR$/jobs-new.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="945"> <state relative-caret-position="945">
@ -1392,8 +1363,29 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json"> <entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="834"> <state relative-caret-position="677">
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" /> <caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="210">
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/.eslintrc">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="583">
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/gather.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="346">
<caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
</state> </state>
</provider> </provider>
</entry> </entry>

130
gather.js
View File

@ -1,23 +1,16 @@
const RssBraider = require('rss-braider'); const RssBraider = require('rss-braider');
const fs = require('fs'); const fs = require('fs');
const ejs = require(
'ejs');
const read = require('fs').readFileSync;
const join = require('path').join;
const cron = require('node-cron');
const util = require('util') const util = require('util');
const fs_writeFile = util.promisify(fs.writeFile) const fsWriteFile = util.promisify(fs.writeFile);
const str = read(
join(__dirname, '/templates/rss.ejs'),
'utf8');
const feeds = {}; const feeds = {};
const log4js = require('log4js'); const log4js = require('log4js');
const logger = log4js.getLogger(); const logger = log4js.getLogger();
logger.level = 'trace'; logger.level = 'trace';
let timerID = 0;
// Pull feeds from config files: , // Pull feeds from config files: ,
// feeds.simple_test_feed = require("./config/feed").feed; // feeds.simple_test_feed = require("./config/feed").feed;
@ -32,99 +25,58 @@ feeds.tech = require('./feeds/tech.json');
feeds.fit = require('./feeds/fit.json'); feeds.fit = require('./feeds/fit.json');
feeds.bored = require('./feeds/bored.json'); feeds.bored = require('./feeds/bored.json');
const braider_options = { const braiderOptions = {
feeds: feeds, feeds,
indent: ' ', 'indent': ' ',
date_sort_order: 'desc', // Newest first 'date_sort_order': 'desc', // Newest first
log_level: 'warn', 'log_level': 'warn',
dedupe_fields: ['link', 'guid'], 'dedupe_fields': ['link', 'guid'],
plugins_directories: [__dirname + '/plugins/'] 'plugins_directories': [`${__dirname }/plugins/`]
}; };
const rss_braider = RssBraider.createClient(braider_options); const rssBraider = RssBraider.createClient(braiderOptions);
// Override logging level (debug, info, warn, err, off) // Override logging level (debug, info, warn, err, off)
//rss_braider.logger.level('off'); // rssBraider.logger.level('off');
function gatherV2(feedName, fileName, mode) { function gatherV2(feedName, fileName, mode) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const modeSuffix = {'rss': '.xml', 'json': '.json'}; const modeSuffix = { 'rss': '.xml', 'json': '.json' };
logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`); logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`);
rss_braider.processFeed(feedName, mode, (err, data) => { rssBraider.processFeed(feedName, mode, (err, data) => {
if (err) { if (err) {
logger.error(err); logger.error(err);
return reject(err);
}
fs_writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function(){ return reject(err);
return resolve(`${feedName} saved`); }
});
fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () {
return resolve(`${feedName} saved`);
}); });
});
}); });
} }
function toHour(extra = 0) {
const now = new Date();
async function gather(feedName, fileName) { return (3600000 - (now.getTime() % 3600000)) + extra;
logger.info(`Gathering ${feedName}...`);
rss_braider.processFeed(feedName, 'json', function (err, data) {
console.log('A');
if (err) {
logger.error(err);
return err;
}
console.log('B');
const j = JSON.parse(data);
console.log('C');
const ejsOutput = ejs.compile(str)(j);
console.log('D');
fs.writeFile(__dirname + '/dist/' + fileName + '.html',
ejsOutput,
function (err) {
if (err) {
logger.error(err);
return console.error(err);
}
console.log('The file was saved!');
});
fs.writeFile(__dirname + '/dist/' + fileName + '.json',
data,
function (err) {
if (err) {
logger.error(err);
return console.error(err);
}
console.log('The file was saved!');
});
});
} }
async function gatherXML(feedName, fileName) { async function gatherXML(feedName, fileName) {
rss_braider.processFeed(feedName, 'rss', function (err, data) { rssBraider.processFeed(feedName, 'rss', function (err, data) {
if (err) { if (err)
return console.log(err); return console.log(err);
}
console.log('Saving', __dirname + '/dist/' + fileName + '.xml'); console.log('Saving', `${__dirname }/dist/${ fileName }.xml`);
fs.writeFile(__dirname + '/dist/' + fileName + '.xml', data, function (err) { fs.writeFile(`${__dirname }/dist/${ fileName }.xml`, data, function (err) {
if (err) { if (err)
return console.log(err); return console.log(err);
}
console.log('The file was saved!'); console.log('The file was saved!');
}); });
}); });
} }
async function main() { async function main() {
await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => { await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
logger.debug(d); logger.debug(d);
@ -164,10 +116,20 @@ async function main() {
// await gatherXML('jobsLocal','jobs-local'); // await gatherXML('jobsLocal','jobs-local');
} }
function tick() {
cron.schedule('3 * * * *', function () { const extra = Math.floor((Math.random() * 60) + 1) * 1000;
main(); main();
return -1;
});
// main(); timerID = setTimeout(
() => tick(),
toHour(extra)
);
}
(() => {
logger.info('Gather started...');
timerID = setTimeout(
() => tick(),
toHour()
);
})();

6
package-lock.json generated
View File

@ -159,9 +159,9 @@
} }
}, },
"acorn": { "acorn": {
"version": "5.6.2", "version": "5.7.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-5.6.2.tgz", "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz",
"integrity": "sha512-zUzo1E5dI2Ey8+82egfnttyMlMZ2y0D8xOCO3PNPPlYXpl8NZvF6Qk9L9BEtJs+43FqEmfBViDqc5d1ckRDguw==", "integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==",
"dev": true "dev": true
}, },
"acorn-jsx": { "acorn-jsx": {