Jobserve and s1Jobs rss scraper added

.gitignore

@@ -146,3 +146,4 @@ fabric.properties
 /src/bundle.js.map
 /live/
 !/output/
+/db/jobs.db

grabber.js

@@ -8,11 +8,15 @@
 const CronJob = require('cron').CronJob;
 const IndeedScraper = require('./scrapers/indeed');
 const TotaljobsScraper = require('./scrapers/totaljobs');
+const JobserveScraper = require('./scrapers/rss.jobserve');
+const RssS1Jobs = require('./scrapers/rss.s1jobs');
 
 (async function () {
   console.log('Started..');
   const indeedScraper = new IndeedScraper();
   const totaljobsScraper = new TotaljobsScraper();
+  const jobserveScraper = new JobserveScraper();
+  const s1jobsScraper = new RssS1Jobs();
 
   new CronJob('5 6-23/3 * * *', async function() {
     await indeedScraper.go('london');
@@ -24,4 +28,27 @@ const TotaljobsScraper = require('./scrapers/totaljobs');
     await indeedScraper.go('milton keynes');
     await totaljobsScraper.go('milton keynes');
   }, null, true);
+
+  new CronJob('0 6-23/1 * * *', async function() {
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/4E2AC50E02AD128B.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/6DA9769BA89834AA.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/EDF47BEA6B31EF.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/3CAD044BEF2BFA.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/C7B25D86D0844A.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/64A3EEF615FA4C.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/CA49421A86CA3F74.rss');
+    await jobserveScraper.go('https://www.jobserve.com/MySearch/846CDA8658FF93A3.rss');
+
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/m7dp711z2r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/pfvf7o7z2r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/lluqnt8z2r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/tu33qt8z2r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/u3btnz8z2r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml');
+    await s1jobsScraper.go('http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml');
+  }, null, true);
 })();

lib/base.js

@@ -0,0 +1,127 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 12:01
+
+ */
+const filterReject = require('../lib/filter_reject');
+const filterAccept = require('../lib/filter_md_jobs');
+const dbmanager = require('../lib/dbmanager');
+
+class MasterBase {
+
+  /**
+   *
+   */
+  constructor() {
+    this.url = '';
+    this.items = [];
+    this.currentPage = null;
+    this.hosturl = '';
+    this.siteid = '';
+    this.useStone = false;
+    this.requestOptions = {
+      'url'    : '',
+      'proxy'  : 'http://uk.proxymesh.com:31280',
+      'tunnel' : true
+    };
+  }
+
+  /**
+   *
+   * @returns {{summary: string, site: string, postDate: string, location: string, company: string, id: string, title: string, isEasyApply: boolean, salary: string, url: string, timestamp: number}}
+   */
+  newRecord() {
+    const now = ~~(new Date().getTime() / 1000.0);
+
+    return { 'title': '', 'site': this.siteid || '', 'url':'', 'id':'', 'summary':'', 'postDate':'', 'isEasyApply':false, 'location': '', 'company': '', 'salary': '', 'timestamp':now };
+  }
+
+  /**
+   * 
+   * @returns {Promise<void>}
+   */
+  async addToDB() {
+    for(const item of this.items) 
+    // console.log(item);
+
+      dbmanager.insertOne(item)
+        .then((data) => {
+          console.log(data);
+        })
+        .catch((err) => {
+          console.error(err.message || 'Some error occurred while querying the database.');
+        });
+  }
+
+  /**
+   * 
+   * @returns {Promise<void>}
+   */
+  async filterAdverts() {
+    console.log('>> FilterAdverts');
+    console.log(`Currently ${this.items.length} items...`);
+
+    this.items = this.items.filter(filterReject);
+
+    console.log(`After reject ${this.items.length} items...`);
+
+    this.items = this.items.filter(filterAccept);
+
+    console.log(`After accept ${this.items.length} items...`);
+
+    //  console.log(this.items);
+  }
+  
+  /**
+   *
+   * @param newUrl
+   */
+  setStartUrl(newUrl) {
+    this.url = newUrl;
+  }
+
+  /**
+   *
+   * @param page
+   */
+  loadPage(page) {
+    this.currentPage = page;
+  }
+
+  /**
+   *
+   * @param appended
+   * @returns {string}
+   */
+  makeUrl(appended) {
+    return `https://${ this.siteurl }${appended}`;
+  }
+
+  /**
+   *
+   * @param appended
+   * @returns {string}
+   */
+  makeProxyUrl(appended) {
+    return `https://${ this.siteurl }${appended}`;
+  }
+
+  /**
+   *
+   * @param url
+   * @param q
+   * @returns {string}
+   */
+  makeImg(url, q = 75) {
+    return `https://image.silvrtree.co.uk/q${q}/${url}`;
+  }
+
+  async go() {
+    this.items = [];
+    this.rawItems = [];
+  }
+}
+
+module.exports = MasterBase;

lib/filter_md_jobs.js

@@ -5,11 +5,11 @@ module.exports = function (item) {
 
   console.log('My Filter:', (result || resultB === true) ? 'Pass' : 'Reject');
 
-  if (!(result || resultB === true)) {
+ /* if (!(result || resultB === true)) {
     console.log('Result', result);
     console.log('ResultB', resultB);
     console.log(item);
-  }
+  }*/
 
   return (result || resultB === true) ;
 

lib/filter_reject.js

@@ -1,13 +1,12 @@
 module.exports = function (item) {
-  const patt = /(Teachers?|Technical Writer|Data Analyst|WebLogic|WebSphere|Data Scientist|Change Manager|T24|Test Analyst|Insight Analyst|application tester|senior tester|Salesforce|QlikView|Navision|Murex|seo|django|drupal|SHAREPOINT|per annum|ServiceNow|Test Lead|User Researcher|Service Management|\(PERM\)|£\d.K|Remedy|ITSM|Symfony|Zend|Full Time|Technical Business Analyst|BUSINESS ANALYST|AUTOMATION TESTER|FIELD TECHNICIAN|websphere administrator|Research Data Scientist)/ig;
+  const patt = /(Simply Education|Splunk|Coordinators?|Teachers?|Technical Writers?|Data Analyst|WebLogic|WebSphere|Data Scientist|Change Managers?|T24|Test Analyst|Insight Analyst|application tester|senior tester|Salesforce|QlikView|Navision|Murex|seo|django|drupal|SHAREPOINT|per annum|ServiceNow|Test Lead|User Researcher|Service Management|\(PERM\)|£\d.K|Remedy|ITSM|Symfony|Zend|Full Time|Technical Business Analyst|BUSINESS ANALYST|AUTOMATION TESTER|FIELD TECHNICIAN|websphere administrator|Research Data Scientist)/ig;
 
   const engineers = /(Support|Devops|Planning|security|Postgresql|network|sccm|test|data|imac|firewall|vmware)+(?:\s)(?=Engineer)/ig;
-  const developers = /(Java|PHP|Graduate|Access|Oracle ADF|SHAREPOINT|Ruby on Rails|Java Software|IOS|Qlikview|c#|c\+\+|\.net|bi|go lang)+(?:\s)(?=Developer)/ig;
+  const developers = /(Big Data|Java Server Side|Java|PHP|Graduate|Access|Oracle ADF|SHAREPOINT|Ruby on Rails|Java Software|IOS|Qlikview|c#|c\+\+|\.net|bi|go lang|Python)+(?:\s)(?=Developer)/ig;
   const architects = /(Java|PHP|Microsoft)+(?:\s)(?=Architect)/ig;
 
   const antiAd = /sja\d+/gi;
 
-
   const result = patt.test(item.summary) || engineers.test(item.summary) || developers.test(item.summary) || architects.test(item.summary);
   const resultB = patt.test(item.title) || engineers.test(item.title) || developers.test(item.title) || architects.test(item.title);
   const resultC = antiAd.test(item.id);

lib/rss.js

@@ -0,0 +1,109 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 12:01
+
+ */
+const Parser = require('rss-parser');
+let parser;
+
+const MasterBase = require('./base');
+
+const isEmpty = obj => [Object, Array].includes((obj || {}).constructor) && !Object.entries((obj || {})).length;
+
+class MasterRSS extends MasterBase {
+
+  constructor(options = {}) {
+    super();
+    
+    parser = new Parser(options);
+    
+    this.rawItems = [];
+  }
+
+  /**
+   * 
+   * @param file
+   * @returns {Promise<void>}
+   */
+  async loadFeed(file) {
+    console.log('>> retrieveFeed');
+    const feed = await parser.parseString(file).catch((err) => {
+      console.error(err);
+    });
+
+    console.log(feed);
+
+    this.rawItems = [...feed.items];
+  }
+
+  /**
+   * 
+   * @returns {Promise<void>}
+   */
+  async retrieveFeed() {
+    console.log('>> retrieveFeed');
+    let _url;
+
+    if (this.useStone) 
+      _url = `http://45.33.114.116:8080/${encodeURIComponent(this.url)}`;
+    else
+      _url = this.url; 
+
+    console.log(_url);
+    await parser.parseURL(_url)
+      .then((feed) => {
+        // console.log(feed);
+
+        this.rawItems = [...feed.items];
+        console.log(`${this.url} // ${this.rawItems.length}`);
+      })
+      .catch((err) => {
+        console.error('retrieveFeed', err);
+      });
+  }
+
+  /**
+   * 
+   */
+  reduceItems() {
+    if(this.rawItems.length > 0)
+      this.rawItems.forEach(item => {
+        this.items.push(this.reduceItem(item));
+      });
+
+    // console.log(this.items);
+  }
+
+  /**
+   * 
+   * @returns {Promise<void>}
+   */
+  async processFeed() {
+    await this.retrieveFeed();
+
+    if(this.rawItems.length > 0) {
+      this.reduceItems();
+
+      await this.filterAdverts();
+
+      if (this.items.length > 0) await this.addToDB();
+    }
+    else
+      console.log('No items to process');
+  }
+  
+  // -------------
+
+  /**
+   *
+   * @param item
+   */
+  reduceItem(item) {
+
+  }
+
+}
+
+module.exports = MasterRSS;

package-lock.json

@@ -2167,9 +2167,9 @@
       }
     },
     "rss-parser": {
-      "version": "3.7.6",
-      "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.7.6.tgz",
-      "integrity": "sha512-wWWh3/pPLAPgWyfkCC9jB83jSBenU6VPMymfXiysi8wJxaN7KNkW4vU3Jm8jQxExAribFvXREy+RtaL3XQubeA==",
+      "version": "3.8.0",
+      "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.8.0.tgz",
+      "integrity": "sha512-EvjM0r3IymXnk+SCUawLhdH36D2WQCwpEfIyvu4hpPswYuMSk/jlCLOcg24OvXwVdcJcEocAlTZVjGNjOOSJXg==",
       "requires": {
         "entities": "^1.1.1",
         "xml2js": "^0.4.19"

package.json

@@ -18,7 +18,7 @@
     "express": "^4.17.1",
     "fecha": "^4.2.0",
     "present": "^1.0.0",
-    "rss-parser": "^3.7.6",
+    "rss-parser": "^3.8.0",
     "sqlite3": "^4.1.1",
     "tape": "^4.13.2",
     "tape-promise": "^4.0.0"

scrapers/indeed.js

@@ -80,7 +80,7 @@ class IndeedScraper extends MasterScraper {
     await this.getContent(item.url)
         .then((html) => {
           const $ = cheerio.load(html);
-          newItem.summary = $('#jobDescriptionText').text().trim();
+          newItem.summary = $('#jobDescriptionText').html();
 
         })
         .catch((err) => console.error(err));

scrapers/rss.jobserve.js

@@ -0,0 +1,80 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 12:01
+
+ */
+
+const MasterRSS = require('../lib/rss');
+
+class RssJobserve extends MasterRSS {
+  constructor() {
+    super();
+    
+    this.siteurl = 'www.jobserve.com';
+    this.siteid = 'jobserve';
+    this.useStone = true;
+    this.requestOptions = {
+      'url'    : ''
+    };
+
+    this.guidRegex = /\/en\/(.*?)\./;
+    this.imgRegex = /src="(.+?)"/g;
+    this.locationRegEx = /Location:(.*?) /;
+    this.companyRegEx = /Advertiser:(?: )?(.*?)(?:Reference|Start|Email)/;
+    this.rateRegEx = /Rate:(?: )?(.*?)(?: |Reference|Start|Email|Type)/;
+  }
+
+  reduceItem(item) {
+    const newObj = this.newRecord();
+
+    /*
+    console.log('++ reduce');
+    console.log(item);
+*/
+
+    const location = this.locationRegEx.exec(item.contentSnippet);
+    const company = this.companyRegEx.exec(item.contentSnippet);
+    const rate = this.rateRegEx.exec(item.contentSnippet);
+    const id = this.guidRegex.exec(item.link);
+
+    const _content = item.content.replace(this.imgRegex, (full, part) => {
+      return `src="${this.makeImg(part)}"`;
+    });
+
+    if (location !== null) {
+      const _location = location[1].trim();
+      if (_location.length <= 30) 
+        newObj.location = _location;
+    } 
+    
+    if (company !== null) 
+      newObj.company = company[1].trim();
+
+    if (rate !== null)
+      newObj.salary = rate[1].trim().slice(0, 55);
+
+    if (id !== null)
+      newObj.id = id[1].trim();
+
+    newObj.title = item.title;
+    newObj.postDate = item.isoDate;
+    newObj.url = item.link;
+    newObj.summary = _content;
+    
+    /* console.log(newObj);
+    console.log('-- reduce');*/
+
+    return newObj;
+  }
+  
+  async go(url) {
+    super.go();
+    this.setStartUrl(url);
+    
+    await this.processFeed();
+  }
+}
+
+module.exports = RssJobserve;

scrapers/rss.s1jobs.js

@@ -0,0 +1,63 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 17:20
+
+ */
+const MasterRSS = require('../lib/rss');
+
+class RssS1Jobs extends MasterRSS {
+  constructor() {
+    super({
+      'customFields':{
+        'item': [
+          ['summary', 'content'],
+          ['issued', 'pubDate']
+        ]
+      }
+    });
+
+    this.siteurl = 'www.s1jobs.com';
+    this.siteid = 's1jobs';
+    this.useStone = true;
+    this.requestOptions = {
+      'url'    : ''
+    };
+
+    this.guidRegex = /\/en\/(.*?)\./;
+    this.imgRegex = /src="(.+?)"/g;
+    this.locationRegEx = /Location:(.*?) /;
+    this.companyRegEx = /Advertiser:(?: )?(.*?)(?:Reference|Start|Email)/;
+    this.rateRegEx = /Rate:(?: )?(.*?)(?: |Reference|Start|Email|Type)/;
+  }
+
+  reduceItem(item) {
+    const newObj = this.newRecord();
+
+    /*console.log('++ reduce');
+    console.log(item);
+*/
+    newObj.title = item.title;
+    newObj.postDate = item.isoDate;
+    newObj.url = item.link;
+    newObj.summary = item.content;
+    newObj.id = item.id;
+
+/*
+    console.log(newObj);
+    console.log('-- reduce');
+*/
+
+    return newObj;
+  }
+
+  async go(url) {
+    super.go();
+    this.setStartUrl(url);
+
+    await this.processFeed();
+  }
+}
+
+module.exports = RssS1Jobs;

test/data/s1jobs/m7dp711z2r.xml

@@ -0,0 +1,190 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="/xml/feed-style.xsl" media="screen"?>
+<feed version="0.3" xmlns="http://purl.org/atom/ns#">
+  <title>s1jobs Search Results : IT jobs in Glasgow</title>
+  <link rel="alternate" type="text/html"  href="http://www.s1jobs.com/search/results.cgi?page=1&#x26;s1jobs2003=1&#x26;r__=20030528&#x26;industrySector=IT&#x26;publicsector=publicsector&#x26;order_by=date_posted&#x26;location=Glasgow%2C%20Renfrewshire%20East%2C%20Dunbartonshire%20East%20--%20%20-%20-%20Glasgow"/>
+
+  <modified>2020-05-22T17:08:44+01:00</modified>
+  <author><name>s1jobs technical team</name></author>
+
+  <entry>
+    <title>SQL Server DBA, Edinburgh, &#xA3;45000.00 - &#xA3;55000.00 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760132964.html"/>
+    <id>tag:s1jobs.com,2005:760132964</id>
+    <issued>2020-05-22T17:02:10+00:00</issued>
+    <modified>2020-05-22T17:02:10+00:00</modified>
+    <summary>SQL Server DBA - 12 Month FTC - Competitive Salary  Your new company Your new company is a leading renewable energy company based in Edinburgh who have went from strength to strength in recent years. They are now looking to expand their team and bring in a resource for 12 months.  Your new role Your...</summary>
+  </entry>
+
+  <entry>
+    <title>Data Designer, Glasgow, &#xA3;40000 - &#xA3;47000 per annum + excellent benefits</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760125718.html"/>
+    <id>tag:s1jobs.com,2005:760125718</id>
+    <issued>2020-05-22T15:04:35+00:00</issued>
+    <modified>2020-05-22T15:04:35+00:00</modified>
+    <summary> Harvey Nash are delighted to be working with our Glasgow based client to appoint a critical Technical resource to their expanding team. This will be a hands on senior technical role within the data development space, working to make sure that technical data solutions truly meet business/ system...</summary>
+  </entry>
+
+  <entry>
+    <title>Data Engineer, Edinburgh, &#xA3;45,000 - &#xA3;65,000</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760124045.html"/>
+    <id>tag:s1jobs.com,2005:760124045</id>
+    <issued>2020-05-22T14:13:11+00:00</issued>
+    <modified>2020-05-22T14:13:11+00:00</modified>
+    <summary>As a data engineer would you love to test your skills while working within a market leading and award winning organization with  7,000 employees in 50 locations? You might not know it but an exciting business currently managing $1.5BN of global media data analytics work are hiring and committed to...</summary>
+  </entry>
+
+  <entry>
+    <title>Technical Operations Manager - Middleware, Edinburgh, car allowance, pension, bonus, private health cover</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760122696.html"/>
+    <id>tag:s1jobs.com,2005:760122696</id>
+    <issued>2020-05-22T13:50:38+00:00</issued>
+    <modified>2020-05-22T13:50:38+00:00</modified>
+    <summary>Technical Operations Manager - Middleware - Permanent - EdinburghHead Resourcing is currently recruiting for an experienced Technical Operations Manager with a demonstrable background in the middleware technologies in an enterprise level organisation.  Our client is going through a significant...</summary>
+  </entry>
+
+  <entry>
+    <title>BI Manager, Edinburgh, &#xA3;40000.00 - &#xA3;48000.00 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760121474.html"/>
+    <id>tag:s1jobs.com,2005:760121474</id>
+    <issued>2020-05-22T13:10:43+00:00</issued>
+    <modified>2020-05-22T13:10:43+00:00</modified>
+    <summary>BI Manager - 12 Month FTC - &#xA3;48,000 + Competitive Benefits  Your new company Our client are a market leading care and housing provider who are forward thinking and well respected. Our client now have an opportunity to join them as a BI Manager.  Your new role This is an exciting role where you will...</summary>
+  </entry>
+
+  <entry>
+    <title>Oracle/SQL Developer - Migration, Glasgow, &#xA3;50000 - &#xA3;65000 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760120486.html"/>
+    <id>tag:s1jobs.com,2005:760120486</id>
+    <issued>2020-05-22T13:07:57+00:00</issued>
+    <modified>2020-05-22T13:07:57+00:00</modified>
+    <summary>The Developer will work primarily on designing and building data migrations between the new student system and existing business critical systems such as Oracle Finance to support the University business and processes.Skills/Experience Required for the role;- Strong understanding and significant...</summary>
+  </entry>
+
+  <entry>
+    <title>MI Reporting Analyst, Motherwell, up to &#xA3;40k</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760118047.html"/>
+    <id>tag:s1jobs.com,2005:760118047</id>
+    <issued>2020-05-22T12:39:40+00:00</issued>
+    <modified>2020-05-22T12:39:40+00:00</modified>
+    <summary>My client has become the UK&#x2019;s No.1 rated home insurer, as rated by over 12,000 customers. Their best in class service levels, aligned with our proprietary technology and passionate team of people, have also resulted in us being one of the UK&#x2019;s fastest growing home insurers.Building on...</summary>
+  </entry>
+
+  <entry>
+    <title>PEGA Technical Analyst, Edinburgh</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760111068.html"/>
+    <id>tag:s1jobs.com,2005:760111068</id>
+    <issued>2020-05-21T22:56:24+00:00</issued>
+    <modified>2020-05-21T22:56:24+00:00</modified>
+    <summary>Job description:One of Core-Asset&#x27;s clients has an excellent opportunity for a PEGA Technical Analyst role to join their Digital Solution IT department in Edinburgh or Essex.The purpose of the Digital Solution IT department is to provide technical support and progressive e-business solutions to meet...</summary>
+  </entry>
+
+  <entry>
+    <title>Senior Software Engineer, Dundee, &#xA3;Competitive</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760109841.html"/>
+    <id>tag:s1jobs.com,2005:760109841</id>
+    <issued>2020-05-21T18:27:29+00:00</issued>
+    <modified>2020-05-21T18:27:29+00:00</modified>
+    <summary>Findmypast is looking for Software Engineers who want to make a difference to join our growing Dundee team. We want passionate people who use the right technology for solving problems and like to deliver in a lean-agile environment. Together we try to find new ways to create a great experience for...</summary>
+  </entry>
+
+  <entry>
+    <title>Senior Information Security Architect, Edinburgh, &#xA3;59,539 - &#xA3;64,095</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760099787.html"/>
+    <id>tag:s1jobs.com,2005:760099787</id>
+    <issued>2020-05-21T13:47:47+00:00</issued>
+    <modified>2020-05-21T13:47:47+00:00</modified>
+    <summary>Digital and SecuritySenior Information Security ArchitectPermanentBand &#x2013; 8bSalary Scale - &#xA3;59,539 - &#xA3;64,095Hours &#x2013; 37.5Location &#x2013; FlexiblePost Reference &#x2013; 22435This is a new role in the Digital and Security (DaS) Cyber Security team and an opportunity for you to...</summary>
+  </entry>
+
+  <entry>
+    <title>Senior Software Developer, Glasgow, &#xA3;54000.00 - &#xA3;64000.00 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760099543.html"/>
+    <id>tag:s1jobs.com,2005:760099543</id>
+    <issued>2020-05-21T12:46:40+00:00</issued>
+    <modified>2020-05-21T12:46:40+00:00</modified>
+    <summary>A large central government authority is seeking to appoint a Senior Software Developer based in Glasgow.  Your new company   Our client is a large public sector organisation based in Glasgow, who are renowned for their development environment and use of the up-to-date technologies. The company is...</summary>
+  </entry>
+
+  <entry>
+    <title>Product Manager, Edinburgh, &#xA3;40000.00 - &#xA3;50000.00 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760097346.html"/>
+    <id>tag:s1jobs.com,2005:760097346</id>
+    <issued>2020-05-21T11:12:03+00:00</issued>
+    <modified>2020-05-21T11:12:03+00:00</modified>
+    <summary>Product Manager, Software Company, Central Edinburgh Your new company  This exciting new opportunity is with a well-established software company, located in central Edinburgh. This company provide tailored solutions to their clients and are looking to hire an experienced product manager for their...</summary>
+  </entry>
+
+  <entry>
+    <title>Software Engineer - Python / Django, Edinburgh</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760095984.html"/>
+    <id>tag:s1jobs.com,2005:760095984</id>
+    <issued>2020-05-21T10:51:40+00:00</issued>
+    <modified>2020-05-21T10:51:40+00:00</modified>
+    <summary>Software Engineer - Python / Django  Harvey Nash is excited to be working with a fantastic small business in Edinburgh. Our client created the leading platform that allows graduates and undergraduates to connect with the best employers in the UK and Europe. Originally founded in London 6 years ago...</summary>
+  </entry>
+
+  <entry>
+    <title>Frontend Software Engineer, Edinburgh</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760093842.html"/>
+    <id>tag:s1jobs.com,2005:760093842</id>
+    <issued>2020-05-21T09:45:14+00:00</issued>
+    <modified>2020-05-21T09:45:14+00:00</modified>
+    <summary>Frontend Software Engineer - Edinburgh Harvey Nash is excited to be working with an exciting small business in Edinburgh. Our client created the leading platform that allows graduates and undergraduates to connect with the best employers in the UK and Europe. Originally founded in London 6 years ago...</summary>
+  </entry>
+
+  <entry>
+    <title>Migration Developer, Glasgow, &#xA3;45000 - &#xA3;55000 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760090311.html"/>
+    <id>tag:s1jobs.com,2005:760090311</id>
+    <issued>2020-05-20T16:37:35+00:00</issued>
+    <modified>2020-05-20T16:37:35+00:00</modified>
+    <summary>Would you like to test your ability while playing an integral part in a large-scale transformation project integrating systems to a new platform? You have the opportunity to join a leading organisation who are committed to onboarding new members remotely before moving back to their Glasgow base long...</summary>
+  </entry>
+
+  <entry>
+    <title>.NET Developer, Glasgow, Up to &#xA3;39800.00 per annum</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760087023.html"/>
+    <id>tag:s1jobs.com,2005:760087023</id>
+    <issued>2020-05-20T16:00:57+00:00</issued>
+    <modified>2020-05-20T16:00:57+00:00</modified>
+    <summary>.NET DeveloperPermanent&#xA3;39,800 plus excellent benefitsCan be located in Glasgow, Edinburgh or DundeeBadenoch + Clark is partnering with a high-profile, national service to deliver a niche technical appointment during a period of transformational change. We are looking for a .NET Developer who has an...</summary>
+  </entry>
+
+  <entry>
+    <title>Migration Analyst/ Developer, Glasgow, &#xA3;40000 - &#xA3;65000 per annum + including public sector pension</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760081101.html"/>
+    <id>tag:s1jobs.com,2005:760081101</id>
+    <issued>2020-05-20T12:38:56+00:00</issued>
+    <modified>2020-05-20T12:38:56+00:00</modified>
+    <summary>Eden Scott is working with our respected University client who is looking to recruit Migration Analyst / Developer(s) to work with their Student Information Systems which is fundamental to their success and form a key part in their decision making. This is a key hire as they deliver a flagship...</summary>
+  </entry>
+
+  <entry>
+    <title>Integration Developer, Glasgow, &#xA3;40000 - &#xA3;65000 per annum + including public sector pension</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760079137.html"/>
+    <id>tag:s1jobs.com,2005:760079137</id>
+    <issued>2020-05-20T11:40:33+00:00</issued>
+    <modified>2020-05-20T11:40:33+00:00</modified>
+    <summary>Eden Scott is working with our respected University client who is looking to recruit an Integration Developer to work with their Student Information Systems which is fundamental to their success and form a key part in their decision making. This is a key hire as they deliver a flagship strategic...</summary>
+  </entry>
+
+  <entry>
+    <title>Front End Developer, Edinburgh, Salary Dependant on Experience</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760078929.html"/>
+    <id>tag:s1jobs.com,2005:760078929</id>
+    <issued>2020-05-20T11:29:29+00:00</issued>
+    <modified>2020-05-20T11:29:29+00:00</modified>
+    <summary>Software Developer  Scotland - Remote WorkingAre you an Angular Developer looking for new challenge? Would you like to join software house in role working from home? Would you enjoy joining growing team working on award winning products? Then you will want to discuss this role with us. You will be...</summary>
+  </entry>
+
+  <entry>
+    <title>SITS Developer, Glasgow, &#xA3;40000 - &#xA3;65000 per annum + public sector benefits package</title>
+    <link rel="alternate" type="text/html" href="http://www.s1jobs.com/job/760078473.html"/>
+    <id>tag:s1jobs.com,2005:760078473</id>
+    <issued>2020-05-20T11:16:17+00:00</issued>
+    <modified>2020-05-20T11:16:17+00:00</modified>
+    <summary>Eden Scott are working with our respected University client who are looking to recruit a SITS Developer to work with their Student Information Systems which is fundamental to their success and form a key part in their decision making. This is a key hire as they deliver a flagship strategic project...</summary>
+  </entry>
+
+</feed>

test/jobserve.js

@@ -0,0 +1,48 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 12:18
+
+ */
+const tape = require('tape');
+const _test = require('tape-promise').default; // <---- notice 'default'
+const test = _test(tape); // decorate tape
+
+const fs = require('fs');
+const cheerio = require('cheerio');
+
+const JobserveScraper = require('../scrapers/rss.jobserve');
+
+const jobserveScraper = new JobserveScraper();
+
+const feed = fs.readFileSync('test/data/jobserve/74C7C106F0D8ADE9.rss');
+
+test.test('Test Jobserve scraper', async t => {
+  // await jobserveScraper.go('https://www.jobserve.com/MySearch/49403B7B37C2EA61.rss');
+
+ /* await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/4E2AC50E02AD128B.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/6DA9769BA89834AA.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/EDF47BEA6B31EF.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/3CAD044BEF2BFA.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/C7B25D86D0844A.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/64A3EEF615FA4C.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/CA49421A86CA3F74.rss');
+  await jobserveScraper.go('https://www.jobserve.com/MySearch/846CDA8658FF93A3.rss');*/
+
+
+  await jobserveScraper.loadFeed(feed);
+
+  /*
+  jobserveScraper.reduceItems();
+
+  await jobserveScraper.filterAdverts();
+*/
+  // await jobserveScraper.addToDB();
+
+  t.end();
+});

test/s1jobs.js

@@ -0,0 +1,37 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 21:01
+
+ */
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 22/05/2020
+ * Time: 12:18
+
+ */
+const tape = require('tape');
+const _test = require('tape-promise').default; // <---- notice 'default'
+const test = _test(tape); // decorate tape
+
+const fs = require('fs');
+const cheerio = require('cheerio');
+
+const RssS1Jobs = require('../scrapers/rss.s1jobs');
+
+const s1jobsScraper = new RssS1Jobs();
+
+const feed = fs.readFileSync('test/data/s1jobs/m7dp711z2r.xml');
+
+test.test('Test Jobserve scraper', async t => {
+  await s1jobsScraper.loadFeed(feed);
+
+  s1jobsScraper.reduceItems();
+
+  await s1jobsScraper.filterAdverts();
+  await s1jobsScraper.addToDB();
+
+  t.end();
+});