https support (#207)

* Add databox labels to services

* Create certs with sha-256

* Add unsecure port to container manager

* Tidying

* Update Tests to Https

* Fix multiple IPs in travis tests

Author: ktg

.gitignore

@@ -57,6 +57,8 @@ typings/
 # dotenv environment variables file
 .env
 
+.idea/
+
 certs/
 
 databox-arbiter/

databox-components

@@ -3,5 +3,5 @@ core-export-service https://github.com/me-box/core-export-service.git master
 store-json https://github.com/me-box/store-json.git master
 core-store https://github.com/me-box/core-store.git master
 platform-app-server https://github.com/me-box/platform-app-server.git master
-core-container-manager https://github.com/me-box/core-container-manager.git master
+core-container-manager https://github.com/ktg/core-container-manager master
 core-network https://github.com/me-box/core-network.git master

databox-install-component

@@ -29,14 +29,14 @@ usage() {
 seedManifest() {
   log "Uploading Manifest for ${1} ..."
   STATUS=$(curl --cookie-jar ./databox-jar -sL -w "%{http_code}\\n" http://127.0.0.1:8181/ -o /dev/null)
-  test_assert $STATUS 200 "Seeding manifest"
+  test_assert ${STATUS} 200 "Seeding manifest"
 
   PAYLOAD=$(<${1}/databox-manifest.json)
   PAYLOAD="manifest=${PAYLOAD}"
 
   EXPECTED='{"success":true}'
   RES=$(curl --cookie ./databox-jar -s -X POST -d "${PAYLOAD}" -L 'http://127.0.0.1:8181/app/post')
-  test_assert $RES $EXPECTED "Uploading manifest"
+  test_assert ${RES} ${EXPECTED} "Uploading manifest"
 
   rm ./databox-jar
 }
@@ -51,15 +51,15 @@ installStores() {
 
 buildImage() {
   cd ${1}
-  log "[$(datef) $ME]: Starting build ${1} ..."
+  log "Starting build ${1} ..."
   OUTPUT=$(docker build -t $1 -t $1:${DATABOX_VERSION} -f Dockerfile${DATABOX_ARCH} .)
   test_assert $? 0 "Build ${1}" "$OUTPUT"
   cd ..
 }
 
 pullChanges() {
   cd ${1}
-  log "[$(datef) $ME]: Pulling Changes ${1} ..."
+  log "Pulling Changes ${1} ..."
   OUTPUT=$(git pull)
   test_assert $? 0 "Git pull ${1}" "$OUTPUT"
   cd ..
@@ -73,12 +73,12 @@ fi
 
 COMPONENT=$1
 
-if [[ $COMPONENT != *"/"* ]]; then
+if [[ ${COMPONENT} != *"/"* ]]; then
   COMPONENT="me-box/${COMPONENT}"
 fi
 
 oIFS="$IFS"
-IFS=/ arr=( $COMPONENT )
+IFS=/ arr=( ${COMPONENT} )
 IFS="$oIFS"
 USER=${arr[0]}
 COMPONENT_NAME=${arr[1]}
@@ -87,13 +87,13 @@ if [ ! -d "./$COMPONENT_NAME" ]; then
   log "Getting the code..... "
   OUTPUT=$(git clone "https://github.com/${COMPONENT}.git")
   test_assert $? 0 "Pulling the code from https://github.com/${COMPONENT}.git \n\n ${OUTPUT}"
-  buildImage $COMPONENT_NAME
-  seedManifest $COMPONENT_NAME
+  buildImage ${COMPONENT_NAME}
+  seedManifest ${COMPONENT_NAME}
   installStores
 else
   log "${COMPONENT_NAME} already installed, rebuilding and reseeding"
-  pullChanges $COMPONENT_NAME
-  buildImage $COMPONENT_NAME
-  seedManifest $COMPONENT_NAME
+  pullChanges ${COMPONENT_NAME}
+  buildImage ${COMPONENT_NAME}
+  seedManifest ${COMPONENT_NAME}
   installStores
 fi

databox-start

@@ -55,23 +55,29 @@ assert_or_die $? 1 "Databox is already running!"
 
 ## extract a host interface IP address
 err "extracting host interface IP address ..."
-ips=($(ifconfig | sed -En 's/127.0.0.1//;s/172.//;s/169.//;s/.inet (addr:)?(([0-9]+.){3}[0-9]+).*/\2/p'))
-EXT_IP=$ips
+ips=($(ifconfig | grep "inet " | grep -v 127.0.0.1|awk 'match($0, /([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)/) {print substr($0,RSTART,RLENGTH)}'))
+EXT_IP=${ips}
 if [[ "${#ips[@]}" -gt "1" ]]; then
   err "More than one IP found! Please select one:"
-  select ip in ${ips[*]}; do
-    case $ip in
-      exit )
-        die 0 "exiting ..."
-        break
-        ;;
-
-      * )
-        EXT_IP=$ip
-        break
-        ;;
-    esac
-  done
+  if [[ "${DATABOX_TESTING}" == 1 ]]
+  then
+    err "running in testing mode select second IP ${ips[1]}"
+    EXT_IP=${ips[1]}
+  else
+    select ip in ${ips[*]}; do
+      case ${ip} in
+        exit )
+          die 0 "exiting ..."
+          break
+          ;;
+
+        * )
+          EXT_IP=${ip}
+          break
+          ;;
+      esac
+    done
+  fi
 fi
 err "host interface IP address = ${EXT_IP}"
 
@@ -111,24 +117,26 @@ then
 fi
 
 err "Starting version ${DATABOX_VERSION}"
-export DOCKER_REPO=$DOCKER_REPO
-export DATABOX_VERSION=$DATABOX_VERSION
-export DATABOX_CORE_IMAGE_VERSION=$DATABOX_CORE_IMAGE_VERSION
+export DOCKER_REPO=${DOCKER_REPO}
+export DATABOX_VERSION=${DATABOX_VERSION}
+export DATABOX_CORE_IMAGE_VERSION=${DATABOX_CORE_IMAGE_VERSION}
 
 function _exec {
   docker run \
          --net=host -ti --rm -v "$(pwd -P)":/cwd -w /cwd \
-         $DARGS $NODE_IMAGE "$@"
+         ${DARGS} ${NODE_IMAGE} "$@"
 }
 
 if [ ! -d "node_modules" ]; then
   _exec npm install -loglevel silent
 fi
 
-err "starting the Databox swarm"
-docker swarm init --advertise-addr $EXT_IP > /dev/null
+err "Starting the Databox swarm"
+docker swarm init --advertise-addr ${EXT_IP} > /dev/null
 mkdir -p ./certs
-_exec node ./src/createCerts.js
+
+err "Creating Certificates"
+_exec node ./src/createCerts.js ${ips[@]}
 
 if [ "$DEV" == "1" ]
 then
@@ -139,7 +147,7 @@ then
   assert_or_die $? 0 "Problem building core images"
 fi
 
-err "starting Databox"
+err "Starting Databox"
 
 docker network create -d overlay --attachable databox-system-net
 
@@ -153,13 +161,13 @@ assert_or_die $? 0 "Problem with docker stack deploy"
 if [ "$DEV" == "1" ]
 then
   #source ./databox-install-component driver-sensingkit
-  echo "sleeping..."
+  err "Sleeping..."
   sleep 10
   source ./databox-install-component driver-os-monitor
   source ./databox-install-component app-os-monitor
 fi
 
-err "Databox started! Visit http://127.0.0.1:8989"
+err "Databox started! Visit http://localhost:8448"
 if [ -z "$DATABOX_TESTING" ]
 then
   docker service logs databox_container-manager -f

databox-stop

@@ -29,24 +29,25 @@ then
 fi
 
 ## stop Databox
-err "stopping Databox ..."
+err "Stopping Databox ..."
+export DOCKER_REPO=""
 docker stack rm databox
 docker-compose -f ./docker-databox-appstore.yaml down
 
 rm -f .resolv.conf
-err "stopping and remvoing databox-bridge ..."
+err "Stopping and Removing databox-bridge ..."
 docker-compose -f ./docker-core-network.yaml down
-err "removing network databox-system-net ..."
+err "Removing network databox-system-net ..."
 docker network rm databox-system-net >/dev/null 2>&1
 
-err "waiting ..."
+err "Waiting ..."
 sleep 10 # give docker some time to remove the networks etc
-err "forcing ..."
+err "Forcing ..."
 docker swarm leave --force
 
 if [[ $(docker ps -aq -f status=created -f status=exited -f "label=databox.type") ]]
 then
-  err "Removing  and exited containers ..."
+  err "Removing exited containers ..."
   docker rm $(docker ps -aq -f status=created -f status=exited -f "label=databox.type")
 fi
-err "done!"
+err "Done!"

databox-test

@@ -34,33 +34,33 @@ sleep 60
 docker ps
 
 #can we see the CM UI
-STATUS=$(curl -sL -w "%{http_code}\\n" "http://127.0.0.1:8989/" -o /dev/null)
+STATUS=$(curl -sL --insecure -w "%{http_code}\\n" "https://127.0.0.1:8989/" -o /dev/null)
 test_assert $STATUS 200 "Is core-container-manager available?"
 
 #is the arbiter up
-RES=$(curl -sL  "http://127.0.0.1:8989/arbiter/status")
+RES=$(curl -sL  --insecure "https://127.0.0.1:8989/arbiter/status")
 test_assert "$RES" "active" "Is the arbiter available?"
 
 #is the arbiter /cat endpoint ok
 EXPECTED='{"catalogue-metadata":[{"rel":"urn:X-hypercat:rels:isContentType","val":"application/vnd.hypercat.catalogue+json"},{"rel":"urn:X-hypercat:rels:hasDescription:en","val":"Databox Root Catalogue"},{"rel":"urn:X-hypercat:rels:hasHomepage","val":"http://www.databoxproject.uk/"},{"rel":"urn:X-hypercat:rels:containsContentType","val":"application/vnd.hypercat.catalogue+json"},{"rel":"urn:Xhypercat:rels:supportsSearch","val":"urn:X-hypercat:search:simple"}],"items":[]}'
-RES=$(curl -sL  "http://127.0.0.1:8989/arbiter/cat")
+RES=$(curl -sL  --insecure "https://127.0.0.1:8989/arbiter/cat")
 test_assert "$RES" "$EXPECTED" "Is the arbiter cat endpoint is ok?"
 
 #can we install the OS monitor driver
 PAYLOAD='{"manifest-version":1,"name":"driver-os-monitor","version":"0.1.0","description":"A Databox driver to monitor the local os","author":"Anthony Brown <[email protected]> ()","license":"MIT","databox-type":"driver","tags":["monitor","os"],"homepage":"https://github.com/me-box/driver-os-monitor","repository":{"type":"git","url":"git+https://github.com/me-box/driver-os-monitor.git"},"resource-requirements":{"store":"store-json"}}'
-RES=$(curl -s -H 'Content-Type: application/json' -X POST -d "${PAYLOAD}" -L 'http://127.0.0.1:8989/api/install')
+RES=$(curl -s  --insecure -H 'Content-Type: application/json' -X POST -d "${PAYLOAD}" -L 'https://127.0.0.1:8989/api/install')
 EXPECTED='{"status":200,"msg":"Success"}'
 test_assert "$RES" "$EXPECTED" "Can we install the driver-os-monitor?"
 
 #can we install the OS monitor app
 PAYLOAD='{"manifest-version":1,"name":"app-os-monitor","databox-type":"app","version":"0.1.0","description":"An app in golang to plot the output of the os monitor driver.","author":"Tosh Brown <[email protected]>","license":"MIT","tags":["template","app","nodejs"],"homepage":"https://github.com/me-box/app-os-monitor","repository":{"type":"git","url":"git+https://github.com/me-box/app-os-monitor"},"packages":[{"name":"OS monitor Plotter","purpose":"To visualize your databox load and free memory","install":"required","risks":"None.","benefits":"You can see the data!","datastores":["loadavg1","loadavg5","loadavg15","freemem"],"enabled":true}],"allowed-combinations":[],"datasources":[{"type":"loadavg1","required":true,"name":"loadavg1","clientid":"loadavg1","granularities":[],"hypercat":{"item-metadata":[{"rel":"urn:X-hypercat:rels:hasDescription:en","val":"Databox load average 1 minuet"},{"rel":"urn:X-hypercat:rels:isContentType","val":"text/json"},{"rel":"urn:X-databox:rels:hasVendor","val":"Databox Inc."},{"rel":"urn:X-databox:rels:hasType","val":"loadavg1"},{"rel":"urn:X-databox:rels:hasDatasourceid","val":"loadavg1"},{"rel":"urn:X-databox:rels:hasStoreType","val":"store-json"},{"rel":"urn:X-databox:rels:hasUnit","val":"%"}],"href":"https://driver-os-monitor-store-json:8080/loadavg1"}},{"type":"loadavg5","required":true,"name":"loadavg5","clientid":"loadavg5","granularities":[],"hypercat":{"item-metadata":[{"rel":"urn:X-hypercat:rels:hasDescription:en","val":"Databox load average 5 minuets"},{"rel":"urn:X-hypercat:rels:isContentType","val":"text/json"},{"rel":"urn:X-databox:rels:hasVendor","val":"Databox Inc."},{"rel":"urn:X-databox:rels:hasType","val":"loadavg5"},{"rel":"urn:X-databox:rels:hasDatasourceid","val":"loadavg5"},{"rel":"urn:X-databox:rels:hasStoreType","val":"store-json"},{"rel":"urn:X-databox:rels:hasUnit","val":"%"}],"href":"https://driver-os-monitor-store-json:8080/loadavg5"}},{"type":"loadavg15","required":true,"name":"loadavg15","clientid":"loadavg15","granularities":[],"hypercat":{"item-metadata":[{"rel":"urn:X-hypercat:rels:hasDescription:en","val":"Databox load average 15 minuets"},{"rel":"urn:X-hypercat:rels:isContentType","val":"text/json"},{"rel":"urn:X-databox:rels:hasVendor","val":"Databox Inc."},{"rel":"urn:X-databox:rels:hasType","val":"loadavg15"},{"rel":"urn:X-databox:rels:hasDatasourceid","val":"loadavg15"},{"rel":"urn:X-databox:rels:hasStoreType","val":"store-json"},{"rel":"urn:X-databox:rels:hasUnit","val":"%"}],"href":"https://driver-os-monitor-store-json:8080/loadavg15"}},{"type":"freemem","required":true,"name":"freemem","clientid":"freemem","granularities":[],"hypercat":{"item-metadata":[{"rel":"urn:X-hypercat:rels:hasDescription:en","val":"Free memory in bytes"},{"rel":"urn:X-hypercat:rels:isContentType","val":"text/json"},{"rel":"urn:X-databox:rels:hasVendor","val":"Databox Inc."},{"rel":"urn:X-databox:rels:hasType","val":"freemem"},{"rel":"urn:X-databox:rels:hasDatasourceid","val":"freemem"},{"rel":"urn:X-databox:rels:hasStoreType","val":"store-json"},{"rel":"urn:X-databox:rels:hasUnit","val":"bytes"}],"href":"https://driver-os-monitor-store-json:8080/freemem"}}],"export-whitelist":[{"url":"https://export.amar.io/","description":"Exports the calculated sentiment to amar.io"}],"resource-requirements":{}}'
-RES=$(curl -s -H 'Content-Type: application/json' -X POST -d "${PAYLOAD}" -L 'http://127.0.0.1:8989/api/install')
+RES=$(curl -s  --insecure -H 'Content-Type: application/json' -X POST -d "${PAYLOAD}" -L 'https://127.0.0.1:8989/api/install')
 EXPECTED='{"status":200,"msg":"Success"}'
 test_assert "$RES" "$EXPECTED" "Can we install the app-os-monitor?"
 
 #is the OS monitor app up
 sleep 30
-STATUS=$(curl -sL  -w "%{http_code}\\n" "http://127.0.0.1:8989/app-os-monitor/ui" -o /dev/null)
+STATUS=$(curl -sL --insecure -w "%{http_code}\\n" "https://127.0.0.1:8989/app-os-monitor/ui" -o /dev/null)
 test_assert "$STATUS" 200 "Is the app-os-monitor/ui available?"
 
 exit 0

docker-compose.yaml

@@ -6,6 +6,7 @@ services:
         image: ${DOCKER_REPO}container-manager:${DATABOX_CORE_IMAGE_VERSION}
         ports:
             - '8989:8989'
+            - '8448:8448'
         volumes:
             - '/var/run/docker.sock:/var/run/docker.sock'
             - './docker-compose.yaml:/cfg/docker-compose.yaml'
@@ -29,6 +30,8 @@ services:
             - arbiter
         deploy:
             replicas: 1
+            labels:
+                databox.type: "system"
             placement:
                 constraints:
                     - 'node.role == manager'
@@ -44,6 +47,8 @@ services:
             - DATABOX_EXPORT_SERVICE_KEY
         deploy:
             replicas: 1
+            labels:
+                databox.type: "system"
             endpoint_mode: dnsrr
             placement:
                 constraints:
@@ -64,6 +69,8 @@ services:
             - DATABOX_EXPORT_SERVICE_KEY
         deploy:
             replicas: 1
+            labels:
+                databox.type: "system"
             endpoint_mode: dnsrr
             placement:
                 constraints:

scripts/utils.sh

@@ -3,15 +3,14 @@
 function red {
   echo "\033[0;31m${1}\033[0m"
 }
+
 function green {
   echo "\033[0;32m${1}\033[0m"
 }
 
-ME='databox-build-core'
-
 function datef
 {
-  date +'%Y-%m-%dT%H:%M:%S%z'
+  date +'%Y-%m-%d %H:%M:%S'
 }
 
 log() {

src/createCerts.js

@@ -3,31 +3,30 @@ process.setMaxListeners(200);
 const conman = require('./container-manager.js');
 const Config = require('./config.json');
 const httpsHelper = require('./https-helper');
-const DATABOX_DEV = process.env.DATABOX_DEV
-
-var containerMangerUIServer = null;
 
 httpsHelper.init()
 	.then(() => {
+		let proms = Config.requiredArbiterSecrets.map((name) => {
+			return conman.generateArbiterToken(name);
+		});
 
-			let proms = Config.requiredArbiterSecrets.map((name)=>{
-				return conman.generateArbiterToken(name);
-			});
+		proms = proms.concat(Config.requiredHTTPSecrets.map((name) => {
+			return httpsHelper.createClientCert(name);
+		}));
 
-			let proms1 = Config.requiredHTTPSecrets.map((name)=>{
-				return httpsHelper.createClientCert(name);
-			});
-			
-			return Promise.all(proms1.concat(proms));
+		const ips = process.argv.slice(2);
+		if (ips.indexOf('127.0.0.1') === -1) {
+			ips.push('127.0.0.1')
+		}
+		proms.push(httpsHelper.createClientCert('container-manager', ips));
 
+		return Promise.all(proms);
 	})
-	.then(()=>{
+	.then(() => {
 		console.log("Done!");
 	})
-		
 	.catch(err => {
 		console.log(err);
 		const stack = new Error().stack;
 		console.log(stack);
 	});
-