From 78a95206a38c7c6620017371020cc912f473a55e Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Sun, 24 Jun 2018 21:13:31 +0100 Subject: [PATCH] init --- .DS_Store | Bin 0 -> 8196 bytes .arcconfig | 3 ++ .gitattributes | 10 +++++ .gitignore | 6 +++ README.md | 88 ++++++++++++++++++++++++++++++++++++++++ minio/README.md | 70 ++++++++++++++++++++++++++++++++ minio/Vagrantfile | 41 +++++++++++++++++++ minio/bootstrap.sh | 17 ++++++++ mongodb | 1 + nginx/README.md | 35 ++++++++++++++++ nginx/conf/nginx.conf | 62 ++++++++++++++++++++++++++++ nginx/logs/nginx.pid | 1 + nginx/make-ssl-keys.bat | 9 ++++ nginx/make-ssl-keys.sh | 10 +++++ nginx/reload-nginx.bat | 1 + nginx/reload-nginx.sh | 12 ++++++ nginx/start-nginx.bat | 6 +++ nginx/start-nginx.sh | 9 ++++ nginx/stop-nginx.bat | 1 + nginx/stop-nginx.sh | 6 +++ 20 files changed, 388 insertions(+) create mode 100644 .DS_Store create mode 100644 .arcconfig create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 README.md create mode 100644 minio/README.md create mode 100644 minio/Vagrantfile create mode 100644 minio/bootstrap.sh create mode 160000 mongodb create mode 100644 nginx/README.md create mode 100644 nginx/conf/nginx.conf create mode 100644 nginx/logs/nginx.pid create mode 100644 nginx/make-ssl-keys.bat create mode 100755 nginx/make-ssl-keys.sh create mode 100644 nginx/reload-nginx.bat create mode 100755 nginx/reload-nginx.sh create mode 100644 nginx/start-nginx.bat create mode 100755 nginx/start-nginx.sh create mode 100644 nginx/stop-nginx.bat create mode 100755 nginx/stop-nginx.sh diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..a0a54a09636d6ce2aa3234e1fadbeff7e7fab878 GIT binary patch literal 8196 zcmeI1U2GLa6oAhurQIKCr-1$Ar`Hs*wN1HAxwJxxw}0AN2*kGZ54r5^F74{x-Ewzt zOQ|)D81=yj`X({a#P~qK#20z-1$_~tCSaQ2&w~&8;-k@M;*)1)_BL=!6XSzQ*hyy2 z+&OdR=ghY|IeP&ByK`nMKs5j;l(}R&6?aKo&-0p86t)zTM6w6)z=U*q-k(i7YkEfw zkq06VL>`Dd5P2Z-!2iJmbZ7IVtaI;6ZB#}ch&*s#dVs$lVwAZ|hH_a-@#>&Lga9OY z2p|&GDGm@eu}~&Mxh$nHr7>mnfT1adAqGlw(#LprlF3jmODW9(r8!_YGlmfgirL97 zCb|PArHsnR1Ca;TdVoJZrI4Y>-G%Af<99Djn$Od-j_n8W_$@5jP`t6ER8cfV+oU}+ zG~>+=1xAqbyN&E5|8>*|X3}AO*z=s#x{*F+Tk|Qlwa0SAzFz{$+Q*~RCV zUU*CH48ykrlo#yu;@WLG{#;Mm;oB^FGr!Gm^4nB3{ek`gZ7{_)&(Z3S%m%jS`p2xS zPcp^IecJKBl;T;YR`>f@UPI*o)n@D#}VlIV;l|^(K`W%CJw%mMtw>TxGsG z{2U#tXwloWgu+a9#4*yABbz$3{R$gYN9UwFW#bm@prVdVd6^7t(ihVvyL9~opZ1X9 z2S;WtmzH#$<}RzQsp_L8jdNX>Y4QctCmZCE3m2&M$#_VB0yWS=m&h<22McE59J~gv z!+E#>Z^QfWAzX!Pa2>va8*mf8hVS7=_!)kM-{5!n0|8a6#A>X;TCB%C7{^xJj|Z?5 z4`L4V4$ za&<*{UJr$<4F!Hjm4e5Tc%{#3g4gl8^4i;{J^3H;`t2qO>DC7;>vlD^v~_fLtsZ5; z3Es|;07sG`h{+Qn&hQQtInK4)wrlzbfn}X3ggFm6E9LMjE44(3{*Yd;5HBqJxm#~g zh-g+M#(K0#*VqPvS3D`Td*YhTHWH_b(C$rW8Y?D{rS`G?q%9$2LTPOQTQY{VvP#ujYHju5^lupgho zK>~0DM{yh{a1u>CMflnHEc%4sJf6V?d=by#O9bM1ydXe)2j7)^tzy?7@)iobM3n;U z=7zGa0Qkq7>U2e7zzsJELsuJ?j3 zB3+E~Ny/mongodb.pem` | WARNING: This path is **relative to `node_server`**! Ignore the bad naming, this should point to a PEM file, not a Base64 value! The SSL is encryption only and the CA of the cert is not verified, so *any* pem file will do. The one from the MongoDB setup is a reasonable one to use for dev. Recommended location is `../../mongodb.pem` so it sits outside the git repo and doesn't have to be ignored every commit.| +| `minioEndpoint` | "172.16.0.21" | Minio local IP | +| `minioPort` | "9000" | Minio local Port | +| `minioAccessKey` | "XXXXXXXXXXXXXXXXXXXX" | See /minio/README.md | +| `minioSecretKey` | "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" | See /minio/README.md | +| `minioStorageBucket` | "dev" | Name of Minio bucket | + +Other required fields will be notified at startup. + +## OPTIONAL Debug Server environment + +The following optional environment variables can be useful depending on what +parts of the server you are wanting to debu + +| Setting | Value | Reason | +|--------------------|----------------------|--------| +| `debug` | `core:*,ComServe:*` | Defines which `debug()` statements in the code to enable. See the [debug module on NPM](https://www.npmjs.com/package/debug) for details | + +## Optional Code Modifications for local dev + +The following code modifications can make it easier to do local development: + +### Removing HMAC checks + +When using Postman or similar to simulate a device and call the commands you want +to test, it is easier to disable the HMAC checking (as HMACs are hard to calculate +in Postman). + +At the top of `ComServe/auth.js` -> `checkHMAC()`, add: +``` +next(null); +return; +``` +to remove the checks. + +### Debug the webconsole + +If you are debugging the webconsole project, and it's running on http instead of +https, you need to change the CORS config or the browser will block your requests. + +At the top of `swagger_api\api_cors_middleware.js` change: + +`const ORIGIN_PROTOCOL = 'https';` +to: +`const ORIGIN_PROTOCOL = 'http';` diff --git a/minio/README.md b/minio/README.md new file mode 100644 index 0000000..43cdaee --- /dev/null +++ b/minio/README.md @@ -0,0 +1,70 @@ +# Summary +This directory contains scripts for starting a VM running Minio - an open source object storage +server with an Amazon S3 compatibl API. + +The sections are: + +* **Connection Information**: The connection details for the object store +* **Setup**: Steps required for setup +* **Maintenance**: How to bring minio up again if the VM wasn't suspended properly + +Note, the information on connecting a local server to Minio is contained in the top-level README.md. + +# Connection information: + +* URL for web frontend: http://172.16.0.21:9000/ +* Access Key & Secret Key: Displayed on first run, or available from `/root/.minio/config.json` + +* Environment variables for dev server: +``` + "minioEndpoint": "172.16.0.21", + "minioPort": "9000", + "minioAccessKey": "XXXXXXXXXXXXXXXXXXXX", + "minioSecretKey": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "minioStorageBucket": "dev" +``` + +# Setup + +## Pre-requisites + +1. Install [Virtual Box](https://www.virtualbox.org/). +2. Install [Vagrant](https://www.vagrantup.com/). + +## To Start + +1. `vagrant up` from the shell/console +2. Go to http://172.16.0.21:9000/ +3. Log in with Access Key and Secret Key (displayed on first run or in `/root/.minio/config.json`) +4. Click the *`+`* sign in the bottom right and choose *"Create Bucket"* +5. Name the bucket *"dev"* + +NOTE: Steps 2-5 are only needed on first start + +The first time, This will create and start an entirely new VM, download and install all packages, etc. +After that, this will only start up the suspended VM again. + +NOTE: if the PC is rebooted without suspending the VM, the VM will be "powered down" and will not +have minio running next time it is resumed with `mongo up`. See *Maintenance* section below for +how to bring minio up again. + +## To Stop + +1. `vagrant suspend` + +This will suspend the VM, retaining all data. + +## To Reset + +1. `vagrant destroy` +2. `vagrant up` + +This will entirely reset the VM, losing all previous data. You have been warned! + +# Maintenance +Login to the VM from the host and restart minio: + +1. `vagrant ssh` +2. `sudo killall minio` to kill any running minio +3. `cd minio` +3. `sudo ./minio server ./data &` to start it again diff --git a/minio/Vagrantfile b/minio/Vagrantfile new file mode 100644 index 0000000..63f5783 --- /dev/null +++ b/minio/Vagrantfile @@ -0,0 +1,41 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # + # Mongo box definition + # + config.vm.define "minio" do |minio| + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + minio.vm.box = "ubuntu/xenial64" + + config.vm.provider "virtualbox" do |v| + v.memory = 2048 + v.cpus = 2 + end + + # Create a private network, which allows host-only access to the machine + minio.vm.network "private_network", ip: "172.16.0.21" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # minio.vm.synced_folder "./data", "/vagrant_data" + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # sudo apt-get update + # sudo apt-get install -y apache2 + # SHELL + minio.vm.provision :shell, path: "bootstrap.sh" + end + +end diff --git a/minio/bootstrap.sh b/minio/bootstrap.sh new file mode 100644 index 0000000..82799a4 --- /dev/null +++ b/minio/bootstrap.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# +# Make a directory for the minio binary +# +mkdir minio +cd minio + +# +# Download and run the minio binary as specified at: +# https://www.minio.io/downloads.html#download-server-linux-x64 +# +wget https://dl.minio.io/server/minio/release/linux-amd64/minio + +chmod +x minio + +./minio server ./data & diff --git a/mongodb b/mongodb new file mode 160000 index 0000000..8f89fe1 --- /dev/null +++ b/mongodb @@ -0,0 +1 @@ +Subproject commit 8f89fe12d7bb4f36494b80515bf76ef2ce4538b6 diff --git a/nginx/README.md b/nginx/README.md new file mode 100644 index 0000000..9bef199 --- /dev/null +++ b/nginx/README.md @@ -0,0 +1,35 @@ +This uses Nginx as the SSL offload / proxy for your local debug server. + +The nginx server will server `https://localhost` and will proxy that straight +through to `http://localhost` (where your development server should be running). + + +## Pre-requisites + +1. Download nginx from http://nginx.org/en/download.html + +2. Unzip to /nginx directory below this one. + NOTE: the downloaded zip files like have a nested folder with the version + number, e.g. nginx-1.11.9. You MUST rename and move folders such that + you end up with nginx/nginx.exe being a valid exe. + +3. Run "make-ssl-keys.bat" to create self-signed keys. + WARNING: You will be asked questions. + The important one is "Common Name" + Set this to "localhost" or the IP address that you expect to connect + to your local dev server on. + +## To Start nginx + +1. Run "start-nginx.exe" +2. Look in the process list and see if nginx.exe is running. + +If not, try just running nginx\nginx.exe from the command line to view any errors. + +WARNING: you cannot stop nginx using Ctrl+C. See below for how to stop it. + +## To Stop nginx + +1. Run "stop-nginx.exe" + +Alternatively, kill the nginx processes using task manager diff --git a/nginx/conf/nginx.conf b/nginx/conf/nginx.conf new file mode 100644 index 0000000..a348966 --- /dev/null +++ b/nginx/conf/nginx.conf @@ -0,0 +1,62 @@ + +#user nobody; +worker_processes 1; + +#error_log logs/error.log; +#error_log logs/error.log notice; +error_log logs/error.log info; + +#pid logs\nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + #include mime.types; + #default_type application/octet-stream; + + #log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + # '$status $body_bytes_sent "$http_referer" ' + # '"$http_user_agent" "$http_x_forwarded_for"'; + + #access_log logs/access.log main; + + sendfile on; + #tcp_nopush on; + + #keepalive_timeout 0; + keepalive_timeout 65; + + #gzip on; + + # HTTPS server + # + server { + listen 443 ssl; + server_name localhost; + + ssl_certificate keys/nginx-selfsigned.crt; + ssl_certificate_key keys/nginx-selfsigned.key; + + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 5m; + + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + + location / { + proxy_pass http://localhost:8080; + + # Headers as recommneded at https://www.nginx.com/resources/wiki/start/topics/examples/SSL-Offloader/ + proxy_set_header Accept-Encoding ""; # no backend compression + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-By $server_addr:$server_port; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + } + } + +} diff --git a/nginx/logs/nginx.pid b/nginx/logs/nginx.pid new file mode 100644 index 0000000..a5a49e9 --- /dev/null +++ b/nginx/logs/nginx.pid @@ -0,0 +1 @@ +18984 diff --git a/nginx/make-ssl-keys.bat b/nginx/make-ssl-keys.bat new file mode 100644 index 0000000..e4317de --- /dev/null +++ b/nginx/make-ssl-keys.bat @@ -0,0 +1,9 @@ +@echo off +echo -------------------------------------------------------------------- +echo Making self-signed keys for the HTTPS proxy. +echo You will be asked to answer some questions. The important one is: +echo "Common Name" +echo Set this to localhost, 127.0.0.1., or your machine's IP address +echo -------------------------------------------------------------------- +mkdir conf/keys +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout conf/keys/nginx-selfsigned.key -out conf/keys/nginx-selfsigned.crt \ No newline at end of file diff --git a/nginx/make-ssl-keys.sh b/nginx/make-ssl-keys.sh new file mode 100755 index 0000000..c6c4c25 --- /dev/null +++ b/nginx/make-ssl-keys.sh @@ -0,0 +1,10 @@ +#!/bin/bash +echo "-------------------------------------------------------------------"- +echo "Making self-signed keys for the HTTPS proxy." +echo "You will be asked to answer some questions. The important one is:" +echo "'Common Name'" +echo "Set this to localhost, 127.0.0.1., or your machine's IP address" +echo "--------------------------------------------------------------------" +mkdir conf/keys +openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout conf/keys/nginx-selfsigned.key -out conf/keys/nginx-selfsigned.crt + diff --git a/nginx/reload-nginx.bat b/nginx/reload-nginx.bat new file mode 100644 index 0000000..11dcc2d --- /dev/null +++ b/nginx/reload-nginx.bat @@ -0,0 +1 @@ +nginx\nginx.exe -s reload \ No newline at end of file diff --git a/nginx/reload-nginx.sh b/nginx/reload-nginx.sh new file mode 100755 index 0000000..f13ea3f --- /dev/null +++ b/nginx/reload-nginx.sh @@ -0,0 +1,12 @@ +#!/bin/bash +echo "Stopping nginx..." +if [[ `uname` == 'Darwin' ]]; then + echo "Mac" + nginx -s reload + # /usr/local/Cellar/nginx/1.12.0/bin/nginx -s reload +else + echo Linux + nginx -s reload +fi + +nginx\nginx.exe -s quit diff --git a/nginx/start-nginx.bat b/nginx/start-nginx.bat new file mode 100644 index 0000000..00f70e5 --- /dev/null +++ b/nginx/start-nginx.bat @@ -0,0 +1,6 @@ +@echo off +echo Making sure directories already exist... +mkdir logs +mkdir temp +echo Starting nginx... +START /B nginx\nginx.exe diff --git a/nginx/start-nginx.sh b/nginx/start-nginx.sh new file mode 100755 index 0000000..18b240b --- /dev/null +++ b/nginx/start-nginx.sh @@ -0,0 +1,9 @@ +#!/bin/bash +echo "Making sure directories already exist..." +echo -n "Starting nginx " + +mkdir -p /usr/local/var/log/nginx/ +mkdir -p /usr/local/var/run/nginx/ + +nginx -c ${PWD}/conf/nginx.conf +echo "nginx configured." diff --git a/nginx/stop-nginx.bat b/nginx/stop-nginx.bat new file mode 100644 index 0000000..1e9a32e --- /dev/null +++ b/nginx/stop-nginx.bat @@ -0,0 +1 @@ +nginx\nginx.exe -s quit \ No newline at end of file diff --git a/nginx/stop-nginx.sh b/nginx/stop-nginx.sh new file mode 100755 index 0000000..5efd17e --- /dev/null +++ b/nginx/stop-nginx.sh @@ -0,0 +1,6 @@ +#!/bin/bash +echo "Stopping nginx..." + +nginx -s quit + +# nginx\nginx.exe -s quit