f2880b661e
* Moved to mongo * UI updated to use mongo * UI is a bit fancier now * Import sql to mongo
67 lines
3.9 KiB
JavaScript
67 lines
3.9 KiB
JavaScript
/**
|
|
* Created by WebStorm.
|
|
* User: martin
|
|
* Date: 16/04/2020
|
|
* Time: 23:35
|
|
|
|
*/
|
|
const CronJob = require('cron').CronJob;
|
|
const IndeedScraper = require('./scrapers/indeed');
|
|
const TotaljobsScraper = require('./scrapers/totaljobs');
|
|
const CwjobsScraper = require('./scrapers/cwjobs');
|
|
const JobserveScraper = require('./scrapers/rss.jobserve');
|
|
const RssS1Jobs = require('./scrapers/rss.s1jobs');
|
|
const RssTechnojobs = require('./scrapers/rss.technojobs');
|
|
|
|
(async function () {
|
|
console.log('Started..');
|
|
const indeedScraper = new IndeedScraper();
|
|
const totaljobsScraper = new TotaljobsScraper();
|
|
const cwjobsScraper = new CwjobsScraper();
|
|
const jobserveScraper = new JobserveScraper();
|
|
const s1jobsScraper = new RssS1Jobs();
|
|
const technojobsScraper = new RssTechnojobs();
|
|
|
|
await indeedScraper.go('london');
|
|
await totaljobsScraper.go('london');
|
|
await cwjobsScraper.go('london');
|
|
|
|
await indeedScraper.go('glasgow');
|
|
await totaljobsScraper.go('glasgow');
|
|
await cwjobsScraper.go('glasgow');
|
|
await indeedScraper.go('edinburgh');
|
|
await totaljobsScraper.go('edinburgh');
|
|
await cwjobsScraper.go('edinburgh');
|
|
await indeedScraper.go('milton keynes');
|
|
await totaljobsScraper.go('milton keynes');
|
|
await cwjobsScraper.go('milton keynes');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/4E2AC50E02AD128B.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/6DA9769BA89834AA.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/EDF47BEA6B31EF.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/3CAD044BEF2BFA.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/C7B25D86D0844A.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/64A3EEF615FA4C.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/CA49421A86CA3F74.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/846CDA8658FF93A3.rss');
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/ED1708BF42EF3513.rss'); // javascript node 2 Jul 2020
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/4C67595E323E3453.rss'); // vuejs 2 Jul 2020
|
|
await jobserveScraper.go('https://www.jobserve.com/MySearch/DCD6B8CE431FE402.rss'); // svelte 2 Jul 2020
|
|
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/m7dp711z2r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/pfvf7o7z2r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/lluqnt8z2r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/tu33qt8z2r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/u3btnz8z2r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml');
|
|
await s1jobsScraper.go('http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml');
|
|
|
|
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
|
|
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
|
|
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
|
|
|
|
})();
|