fixe for new euronews ampstructure

gulp/build.js

@@ -1,16 +1,14 @@
-const gulp = require('gulp'),
-
-  autoprefixer = require('gulp-autoprefixer'),
-  cssnano = require('gulp-cssnano'),
-  uglify = require('gulp-uglify'),
-
-  rename = require('gulp-rename'),
-  concat = require('gulp-concat'),
-  cache = require('gulp-cache'),
-  htmlmin = require('gulp-htmlmin'),
-  inject = require('gulp-inject'),
-  del = require('del'),
-  htmlreplace = require('gulp-html-replace');
+const gulp = require('gulp');
+const autoprefixer = require('gulp-autoprefixer');
+const cssnano = require('gulp-cssnano');
+const uglify = require('gulp-uglify');
+const rename = require('gulp-rename');
+const concat = require('gulp-concat');
+const cache = require('gulp-cache');
+const htmlmin = require('gulp-htmlmin');
+const inject = require('gulp-inject');
+const del = require('del');
+const htmlreplace = require('gulp-html-replace');
 
 const scss = require('gulp-scss');
 const sass = require('gulp-sass');
@@ -56,13 +54,13 @@ gulp.task('vendor', function() {
     .pipe(concat('vendor.js'))
 
     /* .pipe(uglify({ 'mangle': false }))*/
-    .pipe(gulp.dest(`live/js`));
+    .pipe(gulp.dest('live/js'));
 });
 
 gulp.task('fonts', function() {
   return gulp.src('src/fonts.list')
     .pipe(googleWebFonts(fontOptions))
-    .pipe(gulp.dest(`live/fonts`))
+    .pipe(gulp.dest('live/fonts'))
   ;
 });
 
@@ -71,7 +69,6 @@ gulp.task('gotham', function() {
   gulp.src(['fonts/GothamSSm-Black.otf', 'fonts/GothamSSm-Bold.otf', 'fonts/GothamSSm-Book.otf', 'fonts/GothamSSm-Light.otf', 'fonts/GothamSSm-Medium.otf']).pipe(gulp.dest('live/fonts'));
 });
 
-
 gulp.task('fujicons', function() {
   gulp.src(['fonts/fujicons.css']).pipe(gulp.dest('live/fonts'));
   gulp.src(['fonts/fujicons.ttf']).pipe(gulp.dest('live/fonts'));

gulpfile.js

@@ -6,5 +6,4 @@ requireDir('./gulp');
 
 gulp.task('default', ['bundleBackbone', 'styles', 'copy', 'customMUI', 'vendor', 'fonts', 'gotham', 'fujicons']);
 
-
 gulp.task('live', ['liveBackbone', 'styles', 'copy', 'customMUI', 'vendor', 'fonts', 'gotham', 'fujicons']);

mongo.yml

@@ -0,0 +1,20 @@
+# Use root/example as user/password credentials
+version: '3.1'
+
+services:
+
+  mongo:
+    image: mongo
+    restart: always
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: example

notes/scrath_5.json

@@ -0,0 +1,72 @@
+{
+  "response": {
+    "result": [
+      {
+        "cards": [
+          {
+            "card_id": "9953c0e9-5b77-3162-bcf4-e19fcd1f541f",
+            "rendering_engine": "custom",
+            "type": "VENUE_INFO",
+            "type_display_name": "Venues",
+            "ttl": 1518712127,
+            "layout": {
+              "template": "venue_info"
+            },
+            "data": {
+              "name": "Meadow Centre",
+              "category": "Gym Pool",
+              "iconUrl": "https://ss3.4sqi.net/img/categories_v2/parks_outdoors/pool_64.png",
+              "id": "4bd49bb9cfa7b713237424da",
+              "provider": "foursquare",
+              "tips": [
+                "Great swimming pool with a wave machine every 30 minutes - but could be warmer"
+              ],
+              "images": [
+                "https://igx.4sqi.net/img/general/6016x4016/212169387_Sk6ObrC2WhtKFKZgTQG5UDeFJr9BH1WmjhUI_qkRqlw.jpg"
+              ],
+              "telephone": "",
+              "address": "",
+              "city": "",
+              "state": "",
+              "zip": "",
+              "latitude": 55.947152440274564,
+              "longitude": -4.564007641621031,
+              "twitter": {
+                "handle": null,
+                "viewIntent": null,
+                "tweetIntent": null
+              },
+              "yelp": {
+                "url": "https://m.yelp.com/biz/meadow-leisure-centre-dumbarton?adjust_creative=ogmBMO91tbdmscbTIaQEdA&utm_campaign=yelp_api&utm_medium=api_v2_search&utm_source=ogmBMO91tbdmscbTIaQEdA",
+                "rating": 3.5,
+                "reviewCount": 2,
+                "viewIntent": "https://m.yelp.com/biz/meadow-leisure-centre-dumbarton?adjust_creative=ogmBMO91tbdmscbTIaQEdA&utm_campaign=yelp_api&utm_medium=api_v2_search&utm_source=ogmBMO91tbdmscbTIaQEdA#Intent;action=android.intent.action.VIEW;end"
+              }
+            },
+            "modules": {},
+            "reason": "",
+            "notify": false,
+            "ranking_arguments": {
+              "poi_latitude": 55.947152440274564,
+              "poi_longitude": -4.564007641621031,
+              "USER_LOCATION:OTHER": 1.0,
+              "req_longitude": -4.5666,
+              "req_latitude": 55.95,
+              "USER_LOCATION_POI_CONFIDENCE:HIGH": 1.0,
+              "STREAM_TYPE": 1.0,
+              "rule_score": 0.0,
+              "TIER": 0.0,
+              "USER_DIST_FROM_REQ_LOC:WITHIN_100M": 1.0,
+              "score": 0.0
+            },
+            "instrumentation": {
+              "rid": "3un9llld8b9pf",
+              "bucket": "ga"
+            }
+          }
+        ]
+      }
+    ],
+    "error": null
+  }
+}

package.json

@@ -6,12 +6,15 @@
   "scripts": {
     "test": "mocha"
   },
+  "engines": {
+    "node": ">=8.9.1"
+  },
   "author": "",
   "license": "ISC",
   "dependencies": {
     "apicache": "^1.2.1",
     "backbone": "^1.3.3",
-    "browserify": "^16.1.0",
+    "browserify": "^16.2.3",
     "cheerio": "^1.0.0-rc.2",
     "dark-sky-api": "^0.6.3",
     "dateformat": "^3.0.3",
@@ -22,14 +25,14 @@
     "fecha": "^2.3.3",
     "feedme": "^1.2.0",
     "geolocation": "^0.2.0",
-    "gulp-autoprefixer": "^5.0.0",
+    "gulp-autoprefixer": "^4.1.0",
     "gulp-bump": "^3.1.0",
     "gulp-cache": "^1.0.2",
     "gulp-concat": "^2.6.1",
     "gulp-cssnano": "^2.1.2",
     "gulp-html-replace": "^1.6.2",
     "gulp-htmlmin": "^4.0.0",
-    "gulp-inject": "^4.3.1",
+    "gulp-inject": "^4.3.0",
     "gulp-uglify": "^3.0.0",
     "hh-mm-ss": "^1.2.0",
     "humanize-duration": "^3.13.0",
@@ -60,12 +63,12 @@
   "devDependencies": {
     "expect.js": "^0.3.1",
     "gulp": "^3.9.1",
-    "gulp-google-webfonts": "^1.0.0",
+    "gulp-google-webfonts": "0.0.14",
     "gulp-rename": "^1.2.2",
     "gulp-sass": "^3.1.0",
     "gulp-scss": "^1.4.0",
     "gulp-sourcemaps": "^2.6.4",
-    "gulp-strip-debug": "^3.0.0",
+    "gulp-strip-debug": "^2.0.0",
     "gulp-uglify-es": "^1.0.1",
     "lodash.assign": "^4.2.0",
     "mocha": "^5.0.4",
@@ -74,6 +77,8 @@
     "require-dir": "^1.0.0",
     "requirejs": "^2.3.5",
     "sinon": "^4.4.6",
+    "tape": "^4.9.2",
+    "tape-promise": "^4.0.0",
     "vinyl-buffer": "^1.0.1",
     "vinyl-source-stream": "^2.0.0",
     "watchify": "^3.11.0",

server/RightByMe.js

@@ -96,7 +96,8 @@ function doFSVenueExplore(ll) {
           payLoad = reduceExplore(fsP1);
         }
 
-        jsonfile.writeFileSync(`output/${payLoad.id}-FSVenueExplore.json`, fsData);
+        logger.debug(payLoad);
+        // jsonfile.writeFileSync(`output/${payLoad.id}-FSVenueExplore.json`, fsData);
 
         return resolve(payLoad);
       }

server/reducers/euronews.js

@@ -20,18 +20,31 @@ function reduceArticle(body = '') {
 
   const html = [];
 
-  // stuff.children().each(function () {
-  stuff.each(function () {
-    let line = $(this).html().replace('amp-img', 'img');
+  const content = $('div.article__content');
+
+  for (let top = 0, topLen = content.length; top < topLen; top++) {
+    const children = $(content[top]).children();
+
+    for (let index = 0, len = children.length; index < len; index++) {
+      let line = $.html($(children[index])).replace('amp-img', 'img');
+      const tag = children[index].name;
 
       const symbol = /src=(['"])(http[s]?:\/\/)/.exec(line) || [];
+
+      if (tag === 'amp-twitter') {
+        const tweetid = $(children[index]).data('tweetid');
+        line = `<amp-twitter width="375"
+  height="472"
+  layout="responsive" data-tweetid="${tweetid}" > </amp-twitter>`;
+      }
       // logger.debug(symbol);
 
       if (symbol.length !== 0)
         line = line.replace(/src=['"]http[s]?:\/\//, `src=${symbol[1]}https://image.silvrtree.co.uk/640,fit,q80/${symbol[2]}`);
 
       html.push(line);
-  });
+    }
+  }
 
   html.push('<div class="endbumper"></div>');
   // const outputHTML = html.join('').replace(htmlTidy, '');
@@ -41,6 +54,8 @@ function reduceArticle(body = '') {
   obj.image = image;
   obj.html = outputHTML;
 
+  logger.debug(JSON.stringify(obj));
+
   return obj;
 }
 

src/v1/index.html

@@ -90,6 +90,9 @@
 <script src="js/vendor.js" async></script>
 <script type='module' src="js/bundle.js" async></script>
 
+<script async="" src="https://cdn.ampproject.org/v0.js"></script>
+<script async custom-element="amp-twitter" src="https://cdn.ampproject.org/v0/amp-twitter-0.1.js"></script>
+
 <noscript>
   <!-- anchor linking to external file -->
   <h1>Javascript disabled</h1>

test/data/euronews/en002.json

@@ -0,0 +1,5 @@
+{
+  "title": "'Fat Cat' Friday: UK bosses earn average worker's salary in just three days",
+  "image": "https://image.silvrtree.co.uk/640,fit,q80/https://static.euronews.com/articles/stories/03/55/91/28/1000x563_cmsv2_d217a269-5c35-54c4-83e4-21f133b6b4eb-3559128.jpg",
+  "html": "<p>The year has barely started but the UK&apos;s top executives have already made more money than the typical British worker will earn in the next twelve months, <a href=\"http://highpaycentre.org/blog/its-fatcat-friday-ceo-pay-for-2019-surpasses-the-amount-the-average-uk-work\">figures released on Friday</a> have revealed.</p><p>According to the High Pay Centre and the Chartered Institute of Personnel and Development (CIPD), the CEOs of Britain&apos;s leading 100 companies make an average of &#xA3;1,020 per hour (&#x20AC;1,133). Their mean yearly pay package hovers just around &#xA3;3.9 million (&#x20AC;4.3 million).</p><div class=\"clear\"></div><p>By their calculations, the average FTSE 100 CEO will have matched the typical worker&apos;s annual salary of just over &#xA3;29,500 (&#x20AC;32,770) by early afternoon on January 4. The two bodies have dubbed the date &quot;Fat Cat&quot; Friday.</p><div class=\"clear\"></div><h2>&apos;Corporate governance failure&apos;</h2><p>The report highlights that pay levels have skyrocketed from around 60 times the average UK full-time salary at the turn of the millennium to about 133 times today.</p><p>It also argues that companies&apos; performances do not justify the pay rise.</p><p>&quot;Excessive executive pay represents a massive corporate governance failure and is a barrier to a fairer economy,&quot; Luke Hildyard, director of the High Pay Centre, said.</p><p>&quot;Corporate boards are too willing to spend millions on top executives without any real justification, while the wider workforce is treated as a cost to be minimised,&quot; he added.</p><p>Charles Cotton, Senior Advisor at the CIPD, said the disparity can hurt companies&apos; reputations.</p><p>&quot;Overly generous executive pay awards are one of the major factors responsible for low levels of trust in business &#x2014; particularly when pay has been stagnating across the wider workforce &#x2014; and therefore represent a grave threat to businesses&apos; ability to thrive in the long term,&quot; he explained.</p><h2>&apos;Cod statistics&apos;?</h2><p>The Trade Union Congress, a federation of British trade unions, expressed outrage at the findings and called for &quot;radical change.&quot;</p><amp-twitter width=\"390\" height=\"330\" layout=\"responsive\" data-tweetid=\"1081094982470172672\" data-conversation=\"none\" data-cards=\"hidden\"> </amp-twitter><p>But Matthew Lesh, Head of Research at the Adam Smith Institute, a free-market think tank, described the figures as &quot;cod statistics.&quot;</p><p>&quot;In a global market for CEOs, British firms must be able to compete for top CEOs who provide immense value to companies.</p><p>&quot;Limits on executive pay would drive top British talent and companies offshore, ultimately leading to fewer jobs and lower pay for workers,&quot; he added.</p><amp-twitter width=\"390\" height=\"330\" layout=\"responsive\" data-tweetid=\"1081173633924509696\" data-conversation=\"none\" data-cards=\"hidden\"> </amp-twitter><div class=\"clear\"></div><div class=\"endbumper\"></div>"
+}

test/data/euronews/test.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+    <script async custom-element="amp-twitter" src="https://cdn.ampproject.org/v0/amp-twitter-0.1.js"></script>
+</head>
+<body>
+<p>The year has barely started but the UK&apos;s top executives have already made more money than the typical British worker will earn in
+    the next twelve months, <a
+            href="http://highpaycentre.org/blog/its-fatcat-friday-ceo-pay-for-2019-surpasses-the-amount-the-average-uk-work">figures
+        released on Friday</a> have revealed.</p>
+<p>According to the High Pay Centre and the Chartered Institute of Personnel and Development (CIPD), the CEOs of Britain&apos;s leading 100
+    companies make an average of &#xA3;1,020 per hour (&#x20AC;1,133). Their mean yearly pay package hovers just around &#xA3;3.9 million (&#x20AC;4.3
+    million).</p>
+<div class="clear"></div>
+<p>By their calculations, the average FTSE 100 CEO will have matched the typical worker&apos;s annual salary of just over &#xA3;29,500 (&#x20AC;32,770)
+    by early afternoon on January 4. The two bodies have dubbed the date &quot;Fat Cat&quot; Friday.</p>
+<div class="clear"></div>
+<h2>&apos;Corporate governance failure&apos;</h2>
+<p>The report highlights that pay levels have skyrocketed from around 60 times the average UK full-time salary at the turn of the millennium
+    to about 133 times today.</p>
+<p>It also argues that companies&apos; performances do not justify the pay rise.</p>
+<p>&quot;Excessive executive pay represents a massive corporate governance failure and is a barrier to a fairer economy,&quot; Luke
+    Hildyard, director of the High Pay Centre, said.</p>
+<p>&quot;Corporate boards are too willing to spend millions on top executives without any real justification, while the wider workforce is
+    treated as a cost to be minimised,&quot; he added.</p>
+<p>Charles Cotton, Senior Advisor at the CIPD, said the disparity can hurt companies&apos; reputations.</p>
+<p>&quot;Overly generous executive pay awards are one of the major factors responsible for low levels of trust in business &#x2014;
+    particularly when pay has been stagnating across the wider workforce &#x2014; and therefore represent a grave threat to businesses&apos;
+    ability to thrive in the long term,&quot; he explained.</p>
+<h2>&apos;Cod statistics&apos;?</h2>
+<p>The Trade Union Congress, a federation of British trade unions, expressed outrage at the findings and called for &quot;radical change.&quot;</p>
+
+<amp-twitter width="390"
+             height="330"
+             layout="responsive"
+             data-tweetid="855178606556856320">
+    <blockquote placeholder>
+        <p>What are 5 common misconceptions people often have about AMP? Find out on today&#39;s installment of Amplify:
+            <a href="https://t.co/kaSvV8SQtI">https://t.co/kaSvV8SQtI</a>
+            <a href="https://t.co/Cu9VYOmiKV">pic.twitter.com/Cu9VYOmiKV</a>
+        </p>&mdash; AMP Project (@AMPhtml)
+        <a href="https://twitter.com/AMPhtml/status/855178606556856320">April 20, 2017</a>
+    </blockquote>
+    <div fallback>
+        An error occurred while retrieving the Tweet. It might have been deleted.
+    </div>
+</amp-twitter>
+
+
+<amp-twitter width="390"
+             height="330"
+             layout="responsive"
+             data-tweetid="855178606556856320"
+             data-conversation="none"
+             data-cards="hidden">
+</amp-twitter>
+
+
+<amp-twitter width="375" height="472" layout="responsive" data-tweetid="1081094982470172672">
+
+</amp-twitter>
+<p>But Matthew Lesh, Head of Research at the Adam Smith Institute, a free-market think tank, described the figures as &quot;cod statistics.&quot;</p>
+<p>&quot;In a global market for CEOs, British firms must be able to compete for top CEOs who provide immense value to companies.</p>
+<p>&quot;Limits on executive pay would drive top British talent and companies offshore, ultimately leading to fewer jobs and lower pay for
+    workers,&quot; he added.</p>
+<amp-twitter width="375" height="472" layout="responsive" data-tweetid="1081173633924509696">
+
+</amp-twitter>
+<div class="clear"></div>
+<div class="endbumper"></div>
+</body>
+</html>

test/euronews.scrape.js

@@ -0,0 +1,26 @@
+const tape = require('tape');
+const _test = require('tape-promise').default; // <---- notice 'default'
+const test = _test(tape); // decorate tape
+
+const fs = require('fs');
+const jsonfile = require('jsonfile');
+
+const cheerio = require('cheerio');
+
+const { reduceArticle } = require('../server/reducers/euronews');
+
+test('Euronews', async t => {
+  t.test('Reduce a page', async t => {
+    const psDetail = fs.readFileSync('test/data/euronews/en002.html');
+    const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json');
+
+
+    const output = await reduceArticle(psDetail);
+
+    t.deepEquals(output, expectedJSON, 'Extracted Details from Page');
+
+    t.end();
+  });
+
+  t.end();
+});