martin/jobscraper
martin/jobscraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Temporarily turned off stone for rss feeds

Browse Source
This commit is contained in:
Martin Donnelly 2020-06-01 08:59:29 +01:00
parent c18d946a35
commit 819f6704eb
5 changed files with 37 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
grabber.js
View File

@ -59,8 +59,8 @@ const RssTechnojobs = require('./scrapers/rss.technojobs');
await s1jobsScraper.go('http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml');
await s1jobsScraper.go('http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml');
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
}, null, true);
})();

2
scrapers/rss.jobserve.js
View File

@ -14,7 +14,7 @@ class RssJobserve extends MasterRSS {
this.siteurl = 'www.jobserve.com';
this.siteid = 'jobserve';
this.useStone = true;
this.useStone = false;
this.requestOptions = {
'url' : ''
};

2
scrapers/rss.technojobs.js
View File

@ -14,7 +14,7 @@ class RssTechnojobs extends MasterRSS {
this.siteurl = 'www.technojobs.co.uk';
this.siteid = 'technojobs';
this.useStone = true;
this.useStone = false;
this.requestOptions = {
'url' : ''
};

6
test/indeed.js
View File

@ -21,7 +21,7 @@ const indeedScraper = new IndeedScraper();
const page = fs.readFileSync('data/indeed/page2.html');
test.test('Test Indeed scraper', async t => {
const $ = cheerio.load(page);
/*const $ = cheerio.load(page);
indeedScraper.loadPage($);
@ -35,7 +35,9 @@ test.test('Test Indeed scraper', async t => {
await indeedScraper.filterAdverts();
// await indeedScraper.addToDB();
// await indeedScraper.addToDB();*/
await indeedScraper.go('london');
t.end();
});

28
test/testfeeds.js Normal file
View File

@ -0,0 +1,28 @@
/**
* Created by WebStorm.
* User: martin
* Date: 22/05/2020
* Time: 12:18
*/
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const fs = require('fs');
const cheerio = require('cheerio');
const JobserveScraper = require('../scrapers/rss.jobserve');
const RssS1Jobs = require('../scrapers/rss.s1jobs');
const RssTechnojobs = require('../scrapers/rss.technojobs');
const jobserveScraper = new JobserveScraper();
const s1jobsScraper = new RssS1Jobs();
const technojobsScraper = new RssTechnojobs();
test.test('Test Jobserve scraper', async t => {
// await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss');
// await s1jobsScraper.go('http://www.s1jobs.com/xml/m7dp711z2r.xml');
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1');
t.end();
});