martin/jobscraper
martin/jobscraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added scraper test

Browse Source
This commit is contained in:
Martin Donnelly 2020-06-01 09:13:04 +01:00
parent 819f6704eb
commit b43a54c0e4
7 changed files with 1285 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
db/jobs.db
View File

Binary file not shown.

BIN
db/jobstest.db Normal file
View File

Binary file not shown.

13
db/jobstest.db.sql
View File

@ -36,4 +36,17 @@ left join read on read.rid = jobs._id
left join applied on applied.aid = jobs._id
order by jobs._id desc;
DROP TABLE IF EXISTS "accounts";
CREATE TABLE IF NOT EXISTS "accounts" (
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
"username" TEXT NOT NULL,
"password" TEXT NOT NULL,
"email" TEXT NOT NULL
);
INSERT INTO accounts VALUES(1,'bob','plop','bob@plop.com');
INSERT INTO accounts VALUES(2,'martin','MPReoa43','martind2000@gmail.com');
INSERT INTO sqlite_sequence VALUES('accounts',2);
COMMIT;

1222
db/jobstest2.db.sql Normal file
View File

File diff suppressed because one or more lines are too long

BIN
db/jobtest.db Normal file
View File

Binary file not shown.

11
test/indeed.js
View File

@ -38,6 +38,17 @@ test.test('Test Indeed scraper', async t => {
// await indeedScraper.addToDB();*/
await indeedScraper.go('london');
await totaljobsScraper.go('london');
await cwjobsScraper.go('london');
await indeedScraper.go('glasgow');
await totaljobsScraper.go('glasgow');
await cwjobsScraper.go('glasgow');
await indeedScraper.go('edinburgh');
await totaljobsScraper.go('edinburgh');
await cwjobsScraper.go('edinburgh');
await indeedScraper.go('milton keynes');
await totaljobsScraper.go('milton keynes');
await cwjobsScraper.go('milton keynes');
t.end();
});

39
test/testscrapers.js Normal file
View File

@ -0,0 +1,39 @@
/**
* Created by WebStorm.
* User: martin
* Date: 15/04/2020
* Time: 11:56
*/
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const fs = require('fs');
const cheerio = require('cheerio');
const IndeedScraper = require('../scrapers/indeed');
const TotaljobsScraper = require('../scrapers/totaljobs');
const CwjobsScraper = require('../scrapers/cwjobs');
const indeedScraper = new IndeedScraper();
const totaljobsScraper = new TotaljobsScraper();
const cwjobsScraper = new CwjobsScraper();
test.test('Test Indeed scraper', async t => {
await indeedScraper.go('london');
await totaljobsScraper.go('london');
await cwjobsScraper.go('london');
await indeedScraper.go('glasgow');
await totaljobsScraper.go('glasgow');
await cwjobsScraper.go('glasgow');
await indeedScraper.go('edinburgh');
await totaljobsScraper.go('edinburgh');
await cwjobsScraper.go('edinburgh');
await indeedScraper.go('milton keynes');
await totaljobsScraper.go('milton keynes');
await cwjobsScraper.go('milton keynes');
t.end();
});