added some new feeds
This commit is contained in:
parent
7f893a0b15
commit
56543b6cd7
71
.eslintrc
71
.eslintrc
@ -1,22 +1,55 @@
|
||||
{
|
||||
"env": {
|
||||
"browser": true,
|
||||
"commonjs": true,
|
||||
"es6": true,
|
||||
"node": true
|
||||
},
|
||||
"parserOptions": {
|
||||
|
||||
"sourceType": "module"
|
||||
},
|
||||
"rules": {
|
||||
"no-const-assign": "warn",
|
||||
"no-this-before-super": "warn",
|
||||
"no-undef": "warn",
|
||||
"no-unreachable": "warn",
|
||||
"no-unused-vars": "warn",
|
||||
"constructor-super": "warn",
|
||||
"valid-typeof": "warn",
|
||||
"indent": [2, 2, {SwitchCase: 1}]
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2017,
|
||||
"sourceType": "module",
|
||||
"ecmaFeatures": {
|
||||
"jsx": false
|
||||
}
|
||||
},
|
||||
"env": {
|
||||
"browser": true,
|
||||
"node": true,
|
||||
"es6": true
|
||||
},
|
||||
"rules": {
|
||||
"arrow-spacing": "error",
|
||||
"block-scoped-var": "error",
|
||||
"block-spacing": "error",
|
||||
"brace-style": ["error", "stroustrup", {}],
|
||||
"camelcase": "error",
|
||||
"comma-dangle": ["error", "never"],
|
||||
"comma-spacing": ["error", { "before": false, "after": true }],
|
||||
"comma-style": [1, "last"],
|
||||
"consistent-this": [1, "_this"],
|
||||
"curly": [1, "multi"],
|
||||
"eol-last": 1,
|
||||
"eqeqeq": 1,
|
||||
"func-names": 1,
|
||||
"indent": ["error", 2, { "SwitchCase": 1 }],
|
||||
"lines-around-comment": ["error", { "beforeBlockComment": true, "allowArrayStart": true }],
|
||||
"max-len": [1, 180, 2], // 2 spaces per tab, max 80 chars per line
|
||||
"new-cap": 1,
|
||||
"newline-before-return": "error",
|
||||
"no-array-constructor": 1,
|
||||
"no-inner-declarations": [1, "both"],
|
||||
"no-mixed-spaces-and-tabs": 1,
|
||||
"no-multi-spaces": 2,
|
||||
"no-new-object": 1,
|
||||
"no-shadow-restricted-names": 1,
|
||||
"object-curly-spacing": ["error", "always"],
|
||||
"padded-blocks": ["error", { "blocks": "never", "switches": "always" }],
|
||||
"prefer-const": "error",
|
||||
"prefer-template": "error",
|
||||
"one-var": 0,
|
||||
"quote-props": ["error", "always"],
|
||||
"quotes": [1, "single"],
|
||||
"radix": 1,
|
||||
"semi": [1, "always"],
|
||||
"space-before-blocks": [1, "always"],
|
||||
"space-infix-ops": 1,
|
||||
"vars-on-top": 1,
|
||||
"no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 1 }],
|
||||
"spaced-comment": ["error", "always", { "markers": ["/"] }]
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -2,8 +2,10 @@
|
||||
<project version="4">
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="">
|
||||
<change beforePath="$PROJECT_DIR$/.eslintrc" beforeDir="false" afterPath="$PROJECT_DIR$/.eslintrc" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/feeds/jobs-special.json" beforeDir="false" afterPath="$PROJECT_DIR$/feeds/jobs-special.json" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/gather.js" beforeDir="false" afterPath="$PROJECT_DIR$/gather.js" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/package-lock.json" beforeDir="false" afterPath="$PROJECT_DIR$/package-lock.json" afterDir="false" />
|
||||
</list>
|
||||
<ignored path="rinser.iws" />
|
||||
<ignored path=".idea/workspace.xml" />
|
||||
@ -25,11 +27,11 @@
|
||||
</component>
|
||||
<component name="FileEditorManager">
|
||||
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
|
||||
<file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="true">
|
||||
<file leaf-file-name="jobs-special.json" pinned="false" current-in-tab="false">
|
||||
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="834">
|
||||
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" />
|
||||
<state relative-caret-position="677">
|
||||
<caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
@ -70,11 +72,20 @@
|
||||
</provider>
|
||||
</entry>
|
||||
</file>
|
||||
<file leaf-file-name="gather.js" pinned="false" current-in-tab="false">
|
||||
<file leaf-file-name="gather.js" pinned="false" current-in-tab="true">
|
||||
<entry file="file://$PROJECT_DIR$/gather.js">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="835">
|
||||
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" />
|
||||
<state relative-caret-position="346">
|
||||
<caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
</file>
|
||||
<file leaf-file-name=".eslintrc" pinned="false" current-in-tab="false">
|
||||
<entry file="file://$PROJECT_DIR$/.eslintrc">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="583">
|
||||
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
@ -105,15 +116,6 @@
|
||||
</provider>
|
||||
</entry>
|
||||
</file>
|
||||
<file leaf-file-name="RssBraider.js" pinned="false" current-in-tab="false">
|
||||
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="165">
|
||||
<caret line="11" column="4" selection-start-line="11" selection-start-column="4" selection-end-line="11" selection-end-column="4" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
</file>
|
||||
</leaf>
|
||||
</component>
|
||||
<component name="FileTemplateManagerImpl">
|
||||
@ -126,9 +128,6 @@
|
||||
</component>
|
||||
<component name="FindInProjectRecents">
|
||||
<findStrings>
|
||||
<find>nodejs</find>
|
||||
<find>node</find>
|
||||
<find>getstop</find>
|
||||
<find>stripped</find>
|
||||
<find>scot</find>
|
||||
<find>html</find>
|
||||
@ -156,6 +155,9 @@
|
||||
<find>jobsite</find>
|
||||
<find>4E2AC50E02AD128B</find>
|
||||
<find>aspx</find>
|
||||
<find>cw</find>
|
||||
<find>braider_options</find>
|
||||
<find>fs_writeFile</find>
|
||||
</findStrings>
|
||||
<replaceStrings>
|
||||
<replace />
|
||||
@ -207,7 +209,6 @@
|
||||
<option value="$PROJECT_DIR$/dist/jobs-special.xml" />
|
||||
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch.txt" />
|
||||
<option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" />
|
||||
<option value="$PROJECT_DIR$/.eslintrc" />
|
||||
<option value="$PROJECT_DIR$/jobs-new.js" />
|
||||
<option value="$PROJECT_DIR$/jobs-corpus.js" />
|
||||
<option value="$PROJECT_DIR$/package.json" />
|
||||
@ -225,9 +226,10 @@
|
||||
<option value="$PROJECT_DIR$/feeds/fit.json" />
|
||||
<option value="$PROJECT_DIR$/feeds/tech.json" />
|
||||
<option value="$PROJECT_DIR$/feeds/bored.json" />
|
||||
<option value="$PROJECT_DIR$/gather.js" />
|
||||
<option value="$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt" />
|
||||
<option value="$PROJECT_DIR$/feeds/jobs-special.json" />
|
||||
<option value="$PROJECT_DIR$/.eslintrc" />
|
||||
<option value="$PROJECT_DIR$/gather.js" />
|
||||
</list>
|
||||
</option>
|
||||
</component>
|
||||
@ -297,11 +299,6 @@
|
||||
<item name="rinser" type="b2602c69:ProjectViewProjectNode" />
|
||||
<item name="Rinser" type="462c0819:PsiDirectoryNode" />
|
||||
</path>
|
||||
<path>
|
||||
<item name="rinser" type="b2602c69:ProjectViewProjectNode" />
|
||||
<item name="Rinser" type="462c0819:PsiDirectoryNode" />
|
||||
<item name="feeds" type="462c0819:PsiDirectoryNode" />
|
||||
</path>
|
||||
</expand>
|
||||
<select />
|
||||
</subPane>
|
||||
@ -341,10 +338,10 @@
|
||||
<property name="node.js.detected.package.jscs" value="true" />
|
||||
<property name="node.js.detected.package.prettier" value="true" />
|
||||
<property name="node.js.detected.package.standard" value="true" />
|
||||
<property name="node.js.path.for.package.eslint" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" />
|
||||
<property name="node.js.path.for.package.eslint" value="node" />
|
||||
<property name="node.js.path.for.package.jscs" value="project" />
|
||||
<property name="node.js.path.for.package.prettier" value="project" />
|
||||
<property name="node.js.path.for.package.standard" value="$USER_HOME$/.nvm/versions/node/v8.1.3/bin/node" />
|
||||
<property name="node.js.path.for.package.standard" value="node" />
|
||||
<property name="node.js.selected.package.eslint" value="$PROJECT_DIR$/node_modules/eslint" />
|
||||
<property name="node.js.selected.package.jscs" value="" />
|
||||
<property name="node.js.selected.package.prettier" value="" />
|
||||
@ -352,7 +349,7 @@
|
||||
<property name="nodejs_interpreter_path" value="$USER_HOME$/.nvm/versions/node/v8.11.3/bin/node" />
|
||||
<property name="nodejs_package_manager_path" value="npm" />
|
||||
<property name="recentsLimit" value="5" />
|
||||
<property name="settings.editor.selected.configurable" value="settings.nodejs" />
|
||||
<property name="settings.editor.selected.configurable" value="settings.javascript.linters.eslint" />
|
||||
</component>
|
||||
<component name="RecentsManager">
|
||||
<key name="MoveFile.RECENT_KEYS">
|
||||
@ -560,14 +557,7 @@
|
||||
<workItem from="1520180489311" duration="993000" />
|
||||
<workItem from="1528921135827" duration="17718000" />
|
||||
<workItem from="1529358967248" duration="4995000" />
|
||||
<workItem from="1529481827961" duration="2667000" />
|
||||
</task>
|
||||
<task id="LOCAL-00068" summary="fixed plugins">
|
||||
<created>1471949182374</created>
|
||||
<option name="number" value="00068" />
|
||||
<option name="presentableId" value="LOCAL-00068" />
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1471949182375</updated>
|
||||
<workItem from="1529481827961" duration="3711000" />
|
||||
</task>
|
||||
<task id="LOCAL-00069" summary="fixed plugins">
|
||||
<created>1472027998982</created>
|
||||
@ -905,22 +895,29 @@
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1529363450787</updated>
|
||||
</task>
|
||||
<option name="localTasksCounter" value="117" />
|
||||
<task id="LOCAL-00117" summary="added some new feeds">
|
||||
<created>1529489048452</created>
|
||||
<option name="number" value="00117" />
|
||||
<option name="presentableId" value="LOCAL-00117" />
|
||||
<option name="project" value="LOCAL" />
|
||||
<updated>1529489048452</updated>
|
||||
</task>
|
||||
<option name="localTasksCounter" value="118" />
|
||||
<servers />
|
||||
</component>
|
||||
<component name="TimeTrackingManager">
|
||||
<option name="totallyTimeSpent" value="135583000" />
|
||||
<option name="totallyTimeSpent" value="136627000" />
|
||||
</component>
|
||||
<component name="ToolWindowManager">
|
||||
<frame x="17" y="43" width="1569" height="1106" extended-state="0" />
|
||||
<editor active="true" />
|
||||
<layout>
|
||||
<window_info id="Gulp" order="2" sideWeight="0.40669373" side_tool="true" visible="true" weight="0.20219639" />
|
||||
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5933063" visible="true" weight="0.20219639" />
|
||||
<window_info id="Gulp" order="2" sideWeight="0.40695915" side_tool="true" visible="true" weight="0.20219639" />
|
||||
<window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.5930408" visible="true" weight="0.20219639" />
|
||||
<window_info anchor="bottom" id="TODO" order="6" />
|
||||
<window_info anchor="bottom" id="Docker" order="7" />
|
||||
<window_info anchor="bottom" id="Event Log" order="7" sideWeight="0.36999437" side_tool="true" weight="0.24975222" />
|
||||
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.0" />
|
||||
<window_info anchor="bottom" id="Version Control" order="7" sideWeight="0.59560066" visible="true" weight="0.3296146" />
|
||||
<window_info anchor="bottom" id="Run" order="2" sideWeight="0.63000566" weight="0.24975222" />
|
||||
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
|
||||
<window_info anchor="bottom" id="Terminal" order="7" sideWeight="0.4838976" weight="0.32838774" />
|
||||
@ -1020,7 +1017,6 @@
|
||||
<component name="VcsManagerConfiguration">
|
||||
<option name="CHECK_CODE_SMELLS_BEFORE_PROJECT_COMMIT" value="false" />
|
||||
<option name="CHECK_NEW_TODO" value="false" />
|
||||
<MESSAGE value="Updating to Version 3" />
|
||||
<MESSAGE value="limit image size to 900px" />
|
||||
<MESSAGE value="force images through our image server" />
|
||||
<MESSAGE value="Max images out at 1140px wide" />
|
||||
@ -1045,7 +1041,8 @@
|
||||
<MESSAGE value="all feeds using stone" />
|
||||
<MESSAGE value="added gather" />
|
||||
<MESSAGE value="added bored" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value="added bored" />
|
||||
<MESSAGE value="added some new feeds" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value="added some new feeds" />
|
||||
</component>
|
||||
<component name="XDebuggerManager">
|
||||
<breakpoint-manager>
|
||||
@ -1060,11 +1057,6 @@
|
||||
<line>17</line>
|
||||
<option name="timeStamp" value="8" />
|
||||
</line-breakpoint>
|
||||
<line-breakpoint enabled="true" type="javascript">
|
||||
<url>file://$PROJECT_DIR$/gather.js</url>
|
||||
<line>70</line>
|
||||
<option name="timeStamp" value="10" />
|
||||
</line-breakpoint>
|
||||
</breakpoints>
|
||||
<option name="time" value="11" />
|
||||
</breakpoint-manager>
|
||||
@ -1153,13 +1145,6 @@
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/.eslintrc">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="285">
|
||||
<caret line="19" column="8" selection-start-line="19" selection-start-column="8" selection-end-line="19" selection-end-column="8" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/node_modules/feedparser/History.md">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="284">
|
||||
@ -1335,13 +1320,6 @@
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/gather.js">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="835">
|
||||
<caret line="172" column="3" selection-start-line="172" selection-start-column="3" selection-end-line="172" selection-end-column="3" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/jobs.js">
|
||||
<provider selected="true" editor-type-id="text-editor" />
|
||||
</entry>
|
||||
@ -1376,13 +1354,6 @@
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="210">
|
||||
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/jobs-new.js">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="945">
|
||||
@ -1392,8 +1363,29 @@
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/feeds/jobs-special.json">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="834">
|
||||
<caret line="127" column="117" selection-start-line="127" selection-start-column="117" selection-end-line="127" selection-end-column="117" />
|
||||
<state relative-caret-position="677">
|
||||
<caret line="106" column="105" lean-forward="true" selection-start-line="106" selection-start-column="105" selection-end-line="106" selection-end-column="105" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$APPLICATION_CONFIG_DIR$/scratches/scratch_10.txt">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="210">
|
||||
<caret line="14" column="325" selection-start-line="14" selection-start-column="314" selection-end-line="14" selection-end-column="325" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/.eslintrc">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="583">
|
||||
<caret line="55" lean-forward="true" selection-start-line="55" selection-end-line="55" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
<entry file="file://$PROJECT_DIR$/gather.js">
|
||||
<provider selected="true" editor-type-id="text-editor">
|
||||
<state relative-caret-position="346">
|
||||
<caret line="122" column="23" lean-forward="true" selection-start-line="122" selection-start-column="23" selection-end-line="122" selection-end-column="23" />
|
||||
</state>
|
||||
</provider>
|
||||
</entry>
|
||||
|
130
gather.js
130
gather.js
@ -1,23 +1,16 @@
|
||||
const RssBraider = require('rss-braider');
|
||||
const fs = require('fs');
|
||||
const ejs = require(
|
||||
'ejs');
|
||||
const read = require('fs').readFileSync;
|
||||
const join = require('path').join;
|
||||
const cron = require('node-cron');
|
||||
|
||||
const util = require('util')
|
||||
const fs_writeFile = util.promisify(fs.writeFile)
|
||||
const util = require('util');
|
||||
const fsWriteFile = util.promisify(fs.writeFile);
|
||||
|
||||
const str = read(
|
||||
join(__dirname, '/templates/rss.ejs'),
|
||||
'utf8');
|
||||
const feeds = {};
|
||||
|
||||
const log4js = require('log4js');
|
||||
const logger = log4js.getLogger();
|
||||
|
||||
logger.level = 'trace';
|
||||
let timerID = 0;
|
||||
|
||||
// Pull feeds from config files: ,
|
||||
// feeds.simple_test_feed = require("./config/feed").feed;
|
||||
@ -32,99 +25,58 @@ feeds.tech = require('./feeds/tech.json');
|
||||
feeds.fit = require('./feeds/fit.json');
|
||||
feeds.bored = require('./feeds/bored.json');
|
||||
|
||||
const braider_options = {
|
||||
feeds: feeds,
|
||||
indent: ' ',
|
||||
date_sort_order: 'desc', // Newest first
|
||||
log_level: 'warn',
|
||||
dedupe_fields: ['link', 'guid'],
|
||||
plugins_directories: [__dirname + '/plugins/']
|
||||
const braiderOptions = {
|
||||
feeds,
|
||||
'indent': ' ',
|
||||
'date_sort_order': 'desc', // Newest first
|
||||
'log_level': 'warn',
|
||||
'dedupe_fields': ['link', 'guid'],
|
||||
'plugins_directories': [`${__dirname }/plugins/`]
|
||||
};
|
||||
const rss_braider = RssBraider.createClient(braider_options);
|
||||
const rssBraider = RssBraider.createClient(braiderOptions);
|
||||
|
||||
// Override logging level (debug, info, warn, err, off)
|
||||
//rss_braider.logger.level('off');
|
||||
// rssBraider.logger.level('off');
|
||||
|
||||
function gatherV2(feedName, fileName, mode) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const modeSuffix = {'rss': '.xml', 'json': '.json'};
|
||||
logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`);
|
||||
rss_braider.processFeed(feedName, mode, (err, data) => {
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
return reject(err);
|
||||
}
|
||||
const modeSuffix = { 'rss': '.xml', 'json': '.json' };
|
||||
logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`);
|
||||
rssBraider.processFeed(feedName, mode, (err, data) => {
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
|
||||
fs_writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function(){
|
||||
return resolve(`${feedName} saved`);
|
||||
});
|
||||
return reject(err);
|
||||
}
|
||||
|
||||
fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () {
|
||||
return resolve(`${feedName} saved`);
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function toHour(extra = 0) {
|
||||
const now = new Date();
|
||||
|
||||
async function gather(feedName, fileName) {
|
||||
logger.info(`Gathering ${feedName}...`);
|
||||
rss_braider.processFeed(feedName, 'json', function (err, data) {
|
||||
console.log('A');
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
return err;
|
||||
}
|
||||
|
||||
console.log('B');
|
||||
|
||||
const j = JSON.parse(data);
|
||||
|
||||
console.log('C');
|
||||
const ejsOutput = ejs.compile(str)(j);
|
||||
|
||||
console.log('D');
|
||||
fs.writeFile(__dirname + '/dist/' + fileName + '.html',
|
||||
ejsOutput,
|
||||
function (err) {
|
||||
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
return console.error(err);
|
||||
}
|
||||
|
||||
console.log('The file was saved!');
|
||||
});
|
||||
|
||||
fs.writeFile(__dirname + '/dist/' + fileName + '.json',
|
||||
data,
|
||||
function (err) {
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
return console.error(err);
|
||||
}
|
||||
|
||||
console.log('The file was saved!');
|
||||
});
|
||||
});
|
||||
|
||||
return (3600000 - (now.getTime() % 3600000)) + extra;
|
||||
}
|
||||
|
||||
async function gatherXML(feedName, fileName) {
|
||||
rss_braider.processFeed(feedName, 'rss', function (err, data) {
|
||||
if (err) {
|
||||
rssBraider.processFeed(feedName, 'rss', function (err, data) {
|
||||
if (err)
|
||||
return console.log(err);
|
||||
}
|
||||
console.log('Saving', __dirname + '/dist/' + fileName + '.xml');
|
||||
fs.writeFile(__dirname + '/dist/' + fileName + '.xml', data, function (err) {
|
||||
if (err) {
|
||||
|
||||
console.log('Saving', `${__dirname }/dist/${ fileName }.xml`);
|
||||
fs.writeFile(`${__dirname }/dist/${ fileName }.xml`, data, function (err) {
|
||||
if (err)
|
||||
return console.log(err);
|
||||
}
|
||||
|
||||
console.log('The file was saved!');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async function main() {
|
||||
await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
|
||||
logger.debug(d);
|
||||
@ -164,10 +116,20 @@ async function main() {
|
||||
// await gatherXML('jobsLocal','jobs-local');
|
||||
}
|
||||
|
||||
|
||||
cron.schedule('3 * * * *', function () {
|
||||
function tick() {
|
||||
const extra = Math.floor((Math.random() * 60) + 1) * 1000;
|
||||
main();
|
||||
return -1;
|
||||
});
|
||||
|
||||
// main();
|
||||
timerID = setTimeout(
|
||||
() => tick(),
|
||||
toHour(extra)
|
||||
);
|
||||
}
|
||||
|
||||
(() => {
|
||||
logger.info('Gather started...');
|
||||
timerID = setTimeout(
|
||||
() => tick(),
|
||||
toHour()
|
||||
);
|
||||
})();
|
||||
|
6
package-lock.json
generated
6
package-lock.json
generated
@ -159,9 +159,9 @@
|
||||
}
|
||||
},
|
||||
"acorn": {
|
||||
"version": "5.6.2",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-5.6.2.tgz",
|
||||
"integrity": "sha512-zUzo1E5dI2Ey8+82egfnttyMlMZ2y0D8xOCO3PNPPlYXpl8NZvF6Qk9L9BEtJs+43FqEmfBViDqc5d1ckRDguw==",
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz",
|
||||
"integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==",
|
||||
"dev": true
|
||||
},
|
||||
"acorn-jsx": {
|
||||
|
Loading…
Reference in New Issue
Block a user