Merge branch 'docker-part-2' into 'dev'

Further docker work, moved from Pm2 to plain cron

See merge request martind2000/lot!3

Docker/Dockerfile

@@ -0,0 +1,20 @@
+# FROM node:current-slim
+FROM node:current-alpine
+ARG VERSION
+ENV VERSION ${VERSION:-development}
+
+WORKDIR /app
+
+COPY ./package*.json /app/
+
+COPY ./Docker/cronjobs /etc/crontabs/root
+
+COPY ./app /app/app
+
+COPY ./libs /app/libs
+
+COPY ./data /app/data
+
+RUN npm install
+
+CMD ["crond", "-f", "-d", "8"]

Docker/cronjobs

@@ -0,0 +1,3 @@
+0 * * * * echo "$(date '+%Y-%m-%d_%H%M%S')" 2>&1
+10 15 * * 2,5 npm run predict 2>&1
+45 9 * * 3,6 npm run retrieve 2>&1

Docker/ecosystem.config.json

@@ -0,0 +1,22 @@
+[
+  {
+    "name": "Predictor",
+    "script": "app/predict.js",
+    "env": {
+      "NODE_ENV": "production"
+    },
+    "autorestart": false,
+    "instances": 1,
+    "cron_restart": "10 15 * * 2,5"
+  },
+  {
+    "name": "Retriever",
+    "script": "app/retriever.js",
+    "env": {
+      "NODE_ENV": "production"
+    },
+    "autorestart": false,
+    "instances": 1,
+    "cron_restart": "45 9 * * 3,6"
+  }
+]

Docker/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -ex
+
+pm2-runtime start ecosystem.config.json --raw --env production

Makefile

@@ -0,0 +1,33 @@
+PROJECT = lot
+VERSION = $(shell git rev-parse --short HEAD)
+
+ECR_REPO = mail.caliban.io:5000
+#APP_IMAGE = 482681734622.dkr.ecr.eu-west-1.amazonaws.com/$(PROJECT):$(VERSION)
+APP_IMAGE = $(ECR_REPO)/$(PROJECT):$(VERSION)
+NO_CACHE = true
+
+
+#build docker image
+build:
+	docker build ./Docker/. -t $(APP_IMAGE) --build-arg VERSION=$(VERSION)  --no-cache=$(NO_CACHE) --compress
+.PHONY: build
+
+#push docker image to registry
+push: build
+	docker push $(APP_IMAGE)
+.PHONY: push
+
+#push docker image to registry
+run: build
+	docker run $(APP_IMAGE)
+.PHONY: run
+ver:
+	@echo '$(VERSION)'
+	#echo $ERSION
+.PHONY: ver
+
+tar:
+	# docker build . -t $(APP_IMAGE) --build-arg VERSION=$(VERSION)  --no-cache=$(NO_CACHE)
+	tar -C ./ -czvf ./archive.tar.gz   'package.json' 'ncas/' 'helpers/' -X *.js
+
+.PHONY: build

app/predict.js

@@ -0,0 +1,31 @@
+const loader = require('../libs/loader');
+const lot = require('../libs/lot');
+const lotv2 = require('../libs/lotV2');
+const pusher = require('../libs/pusher');
+const log4js = require('log4js');
+const logger = log4js.getLogger();
+
+logger.level = 'debug';
+
+function run(data) {
+  const v1 = lot.calculate(data);
+  const v2 = lotv2.calculate(data);
+
+  const msg = `V2: ${v2} stars: ${v1.mainstars}
+V1: ${v1.mainline} stars: ${v1.mainstars}
+Other: ${v1.otherLine} stars: ${v1.otherStars}
+`;
+
+  pusher.push(msg);
+  logger.debug(msg);
+  // loader.save(data);
+}
+
+function go() {
+  logger.info('GO!');
+  loader.load(run);
+}
+
+(() => {
+  go();
+})();

app/retriever.js

@@ -0,0 +1,5 @@
+const { retrieveNew } = require('../libs/retriever');
+
+(() => {
+  retrieveNew();
+})();

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.5'
+
+services:
+  lot:
+    container_name: Lot
+    build:
+      context: .
+      dockerfile: ./Docker/Dockerfile
+    image: silvrtree-lot
+    volumes:
+      - ./data_store:/app/data

libs/pusher.js

@@ -5,22 +5,21 @@
  * Time: 16:35
  *
  */
-var Pushover = require('node-pushover'), dateFormat = require('dateformat');
+const Pushover = require('node-pushover')/* , dateFormat = require('dateformat')*/;
-var push = new Pushover({
+const push = new Pushover({
-  'token': 'aqnos2j4v4pjpry3hvnxq9646eup23',
+  'token': 'aqnos2j4v4pjpry3hvnxq9646eup23', 'user': 'BE2vgFxdHJw91lVGMRYvZDDmVa5cCM'
-  'user': 'BE2vgFxdHJw91lVGMRYvZDDmVa5cCM'
 });
 
-
+const logger = require('log4js').getLogger();
-var logger = require('log4js').getLogger();
 
 const prefix = process.env.NODE_ENV === 'production' ? 'Production' : 'Dev';
 const title = `mdLot ${ prefix } Alert`;
 module.exports = {
 
   'push' : function(contents) {
-    var now = new Date();
+    const now = new Date();
-    var msg = `Updated at ${ dateFormat(now, 'dddd, mmmm dS, yyyy, HH:MM:ss') }\n\n${ contents }`;
+    // var msg = `Updated at ${ dateFormat(now, 'dddd, mmmm dS, yyyy, HH:MM:ss') }\n\n${ contents }`;
+    const msg = `Updated at ${new Date(now).toLocaleString()}\n\n${contents}`;
     push.send(title, msg);
   }
 };

libs/retriever.js

@@ -1,30 +1,36 @@
-const jsonfile = require("jsonfile");
+const jsonfile = require('jsonfile');
 
-const fetch = require("node-fetch");
+const fetch = require('node-fetch');
-const fecha = require("fecha");
+const fecha = require('fecha');
 
-const { scrapeResults } = require("./scraper");
+const { scrapeResults } = require('./scraper');
 
-const file = "data/data.json";
+const file = 'data/data.json';
 
-// https://www.euro-millions.com/results
+function lastEvent() {
+  const dateThen = new Date();
+
+  // Tuesday and Friday
+  // Sun, Mon, *Tue*, Wed, Thu, *Fri*, Sat
+  // [2,  3    ,4    ,1    ,2   ,3   ,1]
+
+  const daySubtractor = [2, 3, 4, 1, 2, 3, 1];
+
+  const dayNow = dateThen.getDay();
+
+  dateThen.setDate(dateThen.getDate() - daySubtractor[dayNow]);
+
+  return fecha.format(dateThen, '/DD-MM-YYYY');
+}
 
 function retrieveNew() {
-  console.log("retrieveNew...");
+  const ystring = lastEvent();
-  const now = new Date().getTime();
-
-  // https://www.euro-millions.com/results/05-02-2019
-
-  const yesterday = new Date(now - 8.64e7);
-
-  const ystring = fecha.format(yesterday, "/DD-MM-YYYY");
 
   const lotData = jsonfile.readFileSync(file);
+
   const u = `https://www.euro-millions.com/results${ystring}`;
 
-  console.log("retrieving:", u);
+  console.log('retrieving:', u);
-
-  // https://www.euro-millions.com/results/15-02-2019
 
   fetch(u)
     .then((res) => res.text())

oldapp/app.js

@@ -43,3 +43,6 @@ cron.schedule("45 9 * * 3,6", () => {
 });
 
 logger.info("Lot V3 started...");
+
+
+retrieveNew();

oldapp/pusher.js

@@ -5,14 +5,13 @@
  * Time: 16:35
  *
  */
-var Pushover = require('node-pushover'), dateFormat = require('dateformat');
+var Pushover = require('node-pushover')/*, dateFormat = require('dateformat')*/;
 var push = new Pushover({
   'token': 'aqnos2j4v4pjpry3hvnxq9646eup23',
   'user': 'BE2vgFxdHJw91lVGMRYvZDDmVa5cCM'
 });
 
-
+const logger = require('log4js').getLogger();
-var logger = require('log4js').getLogger();
 
 const prefix = process.env.NODE_ENV === 'production' ? 'Production' : 'Dev';
 const title = `mdLot ${ prefix } Alert`;
@@ -20,7 +19,8 @@ module.exports = {
 
   'push' : function(contents) {
     var now = new Date();
-    var msg = `Updated at ${ dateFormat(now, 'dddd, mmmm dS, yyyy, HH:MM:ss') }\n\n${ contents }`;
+    // var msg = `Updated at ${ dateFormat(now, 'dddd, mmmm dS, yyyy, HH:MM:ss') }\n\n${ contents }`;
+    var msg = `Updated at ${ now.format(('dddd, mmmm dS, yyyy, HH:MM')) }\n\n${ contents }`;
     push.send(title, msg);
   }
 };

package.json

@@ -6,14 +6,16 @@
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
     "start": "node lot.js",
-    "retrieve": "node app.js"
+    "retrieve": "node ./app/retriever.js",
+    "predict" : "node ./app/predict.js"
   },
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "jsonfile": "^2.2.3",
+    "eslint": "^8.15.0",
-    "node-cron": "^1.0.0",
+    "jsonfile": "^6.1.0",
-    "tape": "^4.9.2",
+    "node-cron": "^3.0.0",
+    "tape": "^5.5.3",
     "tape-promise": "^4.0.0",
     "ultrases": "^0.1.3"
   },
@@ -22,12 +24,10 @@
   },
   "dependencies": {
     "cheerio": "^1.0.0-rc.2",
-    "dateformat": "^2.0.0",
+    "fecha": "^4.2.3",
-    "fecha": "^3.0.2",
+    "log4js": "^6.4.6",
-    "jsonfile": "^2.4.0",
-    "log4js": "^2.3.3",
     "node-fetch": "^2.3.0",
-    "node-pushover": "^0.2.2",
+    "node-pushover": "^1.0.0",
     "sugar": "^2.0.1",
     "sugar-date": "^2.0.0"
   }