[20.02.20] Kubernetes deployments, pipeline and env changes

configuration/coinbase.env

@@ -1,2 +1,2 @@
-API_KEY=""
+API_KEY=B1efIaTHpUk5Bh1W
-API_SECRET=""
+API_SECRET=nqBiADFQ2I8UeyLhHaSH0IdeqkFcCq1K

configuration/dbgateway.env

@@ -1,2 +0,0 @@
-URI="http://localhost"
-PORT="8080"

configuration/kubernetes/deployment.yaml

@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    name: LABEL
+  name: RESOURCE_NAME
+  namespace: default
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: RESOURCE_NAME
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: RESOURCE_NAME
+    spec:
+      containers:
+      - image: REPOSITORY/IMAGE
+        name: RESOURCE_NAME
+        env:
+          - name: KUBERNETES_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: CONTAINER_CORE_LIMIT
+            valueFrom:
+              resourceFieldRef:
+                resource: limits.cpu
+          - name: CONTAINER_MAX_MEMORY
+            valueFrom:
+              resourceFieldRef:
+                resource: limits.memory
+
+          - name: DATABASE_URL
+            valueFrom:
+              secretKeyRef:
+                name: db-gateway
+                key: db.url
+          - name: DATABASE_PORT
+            valueFrom:
+              secretKeyRef:
+                name: db-gateway
+                key: db.port
+          - name: COINBASE_KEY
+            valueFrom:
+              secretKeyRef:
+                name: coinbase-secret
+                key: coinbase.api.key
+          - name: COINBASE_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: coinbase-secret
+                key: coinbase.api.secret
+        ports:
+        - containerPort: 9090
+          name: RESOURCE_NAME
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 9090
+          timeoutSeconds: 2
+          successThreshold: 2
+          failureThreshold: 2
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 0.25
+            memory: 128Mi
+          limits:
+            cpu: 0.25
+            memory: 128Mi
+      restartPolicy: Always
+      imagePullSecrets:
+        - name: registry-secret

configuration/kubernetes/service.yaml

@@ -0,0 +1,14 @@
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    name: LABEL
+  name: RESOURCE_NAME
+spec:
+  selector:
+    app: RESOURCE_NAME
+  ports:
+    - port: 9090
+      targetPort: 9090
+  sessionAffinity: None
+  type: ClusterIP

configuration/pipelines/build.groovy

@@ -0,0 +1,100 @@
+#!/usr/bin/env groovy
+
+env.APPLICATION_NAME    = 'price-collector'
+env.APPLICATION_LABEL   = 'pricing'
+env.GIT_BRANCH          = 'master'
+env.GIT_REPOSITORY_PATH = "github.com/andyjk15/${env.APPLICATION_NAME}.git"
+env.GIT_REPOSITORY_URL  = "https://${env.GIT_REPOSITORY_PATH}"
+env.GITHUB_CREDENTIALS_ID = 'Github'
+env.DOCKER_REPOSITORY   = 'registry.cryptosky.me'
+env.DOCKER_REPOSITORY_URL = "https://${env.DOCKER_REPOSITORY}"
+env.DOCKER_REPOSITORY_TCP = "tcp://${env.DOCKER_REPOSITORY}:4243"
+
+env.NAMESPACE           = 'production'
+env.SLAVE_LABEL         = "cryptosky-aio-build"
+
+def mvn( String gloals ) {
+    sh "mvn -s configuration/settings.xml --show-version --batch-mode ${gloals}"
+}
+
+String get_application_version() {
+    "1.0.0-b${env.BUILD_NUMBER}"
+}
+
+String executeShellScript( String shellPath, String arg1 = '', String arg2 = '', String arg3 = '', String arg4 = '' ) {
+    sh "./${shellPath} ${arg1} ${arg2} ${arg3} ${arg4}"
+}
+
+try {
+    timestamps {
+    node ("${env.SLAVE_LABEL}") {
+        stage('Initialise') {
+            checkout([$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'Github', url: 'https://github.com/andyjk15/db-gateway.git']]])
+
+            env.APPLICATION_VERSION = get_application_version()
+
+            withCredentials(
+                    [usernamePassword(
+                            credentialsId: 'doctl',
+                            passwordVariable: 'DOCTL_TOKEN',
+                            usernameVariable: 'DOCTL_USERNAME'
+                    )]
+            ) {
+                sh "doctl auth init --access-token ${DOCTL_TOKEN}"
+                sh "doctl kubernetes cluster kubeconfig save cryptosky-kubernetes-cluster-production"
+            }
+        }
+
+        stage('Test Artifact') {
+            try {
+//                mvn 'verify -DskipUTs -DskipTests'
+            } finally {
+//                mvn 'test'
+            }
+        }
+
+        stage('Build Image') {
+//            mvn 'clean package -DskipTests'
+
+            executeShellScript("configuration/scripts/mapVarsToConfigs.sh",
+                    env.DOCKER_REPOSITORY,
+                    env.APPLICATION_NAME,
+                    env.APPLICATION_VERSION,
+                    env.APPLICATION_LABEL)
+
+        }
+
+        stage('Tag Repository') {
+
+            withDockerServer([uri: "${env.DOCKER_REPOSITORY_TCP}"]) {
+                withDockerRegistry([credentialsId: 'Registry', url: "${env.DOCKER_REPOSITORY_URL}"]) {
+                    def image = docker.build("${env.DOCKER_REPOSITORY}/${env.APPLICATION_NAME}:${env.APPLICATION_VERSION}")
+                    image.push()
+                    def latest = docker.build("${env.DOCKER_REPOSITORY}/${env.APPLICATION_NAME}:latest")
+                    latest.push()
+                }
+            }
+            withCredentials(
+                    [usernamePassword(
+                            credentialsId: env.GITHUB_CREDENTIALS_ID,
+                            passwordVariable: 'GIT_PASSWORD',
+                            usernameVariable: 'GIT_USERNAME'
+                    )]
+            ) {
+                sh "git tag ${env.APPLICATION_VERSION}"
+                sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@${env.GIT_REPOSITORY_PATH} ${env.APPLICATION_VERSION}"
+            }
+        }
+
+        stage('Deploy') {
+            executeShellScript("configuration/scripts/deployToKubernetes.sh",
+                env.APPLICATION_NAME)
+        }
+    }
+    }
+} catch ( exception ) {
+    currentBuild.result = 'FAILURE'
+    throw exception
+} finally {
+    currentBuild.result = 'SUCCESS'
+}

configuration/scripts/deployToKubernetes.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+APPLICATION_NAME=$1
+
+kubectl apply -f configuration/kubernetes/deployment.yaml
+kubectl apply -f configuration/kubernetes/service.yaml
+
+kubectl get pods
+
+kubectl rollout status deployment/${APPLICATION_NAME}

configuration/scripts/mapVarsToConfigs.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+DOCKER_REPOSITORY=$1
+APPLICATION_NAME=$2
+APPLICATION_VERSION=$3
+APPLICATION_LABEL=$4
+
+sed -i "s/REPOSITORY/${DOCKER_REPOSITORY}/g" configuration/kubernetes/deployment.yaml
+sed -i "s/IMAGE/${APPLICATION_NAME}:${APPLICATION_VERSION}/g" configuration/kubernetes/deployment.yaml
+sed -i "s/RESOURCE_NAME/${APPLICATION_NAME}/g" configuration/kubernetes/deployment.yaml
+sed -i "s/LABEL/${APPLICATION_LABEL}/g" configuration/kubernetes/deployment.yaml
+
+sed -i "s/RESOURCE_NAME/${APPLICATION_NAME}/g" configuration/kubernetes/service.yaml
+sed -i "s/LABEL/${APPLICATION_LABEL}/g" configuration/kubernetes/service.yaml

src/main.py

@@ -8,7 +8,7 @@ def callCollector(args):
 
 if __name__=='__main__':
     # Dynamically create new child for each currency
-    currencies = [ "btc_gbp" ]
+    currencies = [ "btc_usd" ]
 
     for i in range(len(currencies)):
         Thread(target=callCollector, args=[currencies[i]]).start()

src/pricing/gemini.py

@@ -5,17 +5,13 @@ from currency_converter import CurrencyConverter
 
 def geminiPublicTicker(type):
 
-    converter = CurrencyConverter()
-
     try:
-        uri = "https://api.gemini.com/v1/pubticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
+        uri = "https://api.gemini.com/v1/pubticker/" + type.lower().replace('_', '')
 
         response = requests.request("GET", uri)
         response = json.loads(response.text)
 
-        price = (converter.convert(float(response['last']), 'USD', 'GBP') +
+        price = float(response['last']) + float(response['ask']) + float(response['bid'])/3
-                 converter.convert(float(response['ask']), 'USD', 'GBP') +
-                 converter.convert(float(response['bid']), 'USD', 'GBP'))/3
         price = round(price, 3)
         return price
     except KeyError as e:
@@ -26,20 +22,18 @@ def geminiPublicTicker(type):
 
 def geminiHighLowVol(type):
 
-    converter = CurrencyConverter()
-
     try:
-        uri = "https://api.gemini.com/v2/ticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
+        uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
         response = requests.request("GET", uri)
         response = json.loads(response.text)
 
-        high = converter.convert(float(response['high']), 'USD', 'GBP')
+        high = float(response['high'])
-        low = converter.convert(float(response['low']), 'USD', 'GBP')
+        low = float(response['low'])
 
-        uri = "https://api.gemini.com/v1/pubticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
+        uri = "https://api.gemini.com/v1/pubticker/" + type.lower().replace('_', '')
         response = requests.request("GET", uri)
         response = json.loads(response.text)
-        vol = converter.convert(float(response['volume']['BTC']), 'USD', 'GBP')
+        vol = float(response['volume']['USD'])
 
         return high, low, vol
     except KeyError as e:
@@ -49,15 +43,13 @@ def geminiHighLowVol(type):
 
 def geminiOpenClose(type):
 
-    converter = CurrencyConverter()
-
     try:
-        uri = "https://api.gemini.com/v2/ticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
+        uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
         response = requests.request("GET", uri)
         response = json.loads(response.text)
 
-        open = converter.convert(float(response['open']), 'USD', 'GBP')
+        open = float(response['open'])
-        close = converter.convert(float(response['close']), 'USD', 'GBP')
+        close = float(response['close'])
 
         return open, close
     except KeyError as e:

src/utils/databaseConnect.py

@@ -2,16 +2,11 @@
 
 import requests, os, json
 
-from dotenv import load_dotenv
-from pathlib import Path  # python3 only
-env_path = Path('.') / 'configuration/dbgateway.env'
-load_dotenv(dotenv_path=env_path)
-
 class keys():
 
     def __init__(self):
-        self.uri = os.getenv("URI")
+        self.uri = os.getenv("DATABASE_URL")
-        self.port = os.getenv("PORT")
+        self.port = os.getenv("DATABASE_PORT")
 
 def send(query):