andyjk15/price-collector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[20.02.20] Kubernetes deployments, pipeline and env changes

Browse Source
This commit is contained in:
andrewso 2020-02-20 16:17:10 +00:00
parent 775ef507ea
commit 2fb0487427
10 changed files with 233 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configuration/coinbase.env
View File

@ -1,2 +1,2 @@
API_KEY=""
API_SECRET=""
API_KEY=B1efIaTHpUk5Bh1W
API_SECRET=nqBiADFQ2I8UeyLhHaSH0IdeqkFcCq1K

2
configuration/dbgateway.env
View File

@ -1,2 +0,0 @@
URI="http://localhost"
PORT="8080"

80
configuration/kubernetes/deployment.yaml Normal file
View File

@ -0,0 +1,80 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
name: LABEL
name: RESOURCE_NAME
namespace: default
spec:
replicas: 0
selector:
matchLabels:
app: RESOURCE_NAME
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 25%
maxUnavailable: 0
template:
metadata:
labels:
app: RESOURCE_NAME
spec:
containers:
- image: REPOSITORY/IMAGE
name: RESOURCE_NAME
env:
- name: KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CONTAINER_CORE_LIMIT
valueFrom:
resourceFieldRef:
resource: limits.cpu
- name: CONTAINER_MAX_MEMORY
valueFrom:
resourceFieldRef:
resource: limits.memory
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: db-gateway
key: db.url
- name: DATABASE_PORT
valueFrom:
secretKeyRef:
name: db-gateway
key: db.port
- name: COINBASE_KEY
valueFrom:
secretKeyRef:
name: coinbase-secret
key: coinbase.api.key
- name: COINBASE_SECRET
valueFrom:
secretKeyRef:
name: coinbase-secret
key: coinbase.api.secret
ports:
- containerPort: 9090
name: RESOURCE_NAME
readinessProbe:
httpGet:
path: /
port: 9090
timeoutSeconds: 2
successThreshold: 2
failureThreshold: 2
imagePullPolicy: Always
resources:
requests:
cpu: 0.25
memory: 128Mi
limits:
cpu: 0.25
memory: 128Mi
restartPolicy: Always
imagePullSecrets:
- name: registry-secret

14
configuration/kubernetes/service.yaml Normal file
View File

@ -0,0 +1,14 @@
kind: Service
apiVersion: v1
metadata:
labels:
name: LABEL
name: RESOURCE_NAME
spec:
selector:
app: RESOURCE_NAME
ports:
- port: 9090
targetPort: 9090
sessionAffinity: None
type: ClusterIP

100
configuration/pipelines/build.groovy Normal file
View File

@ -0,0 +1,100 @@
#!/usr/bin/env groovy
env.APPLICATION_NAME = 'price-collector'
env.APPLICATION_LABEL = 'pricing'
env.GIT_BRANCH = 'master'
env.GIT_REPOSITORY_PATH = "github.com/andyjk15/${env.APPLICATION_NAME}.git"
env.GIT_REPOSITORY_URL = "https://${env.GIT_REPOSITORY_PATH}"
env.GITHUB_CREDENTIALS_ID = 'Github'
env.DOCKER_REPOSITORY = 'registry.cryptosky.me'
env.DOCKER_REPOSITORY_URL = "https://${env.DOCKER_REPOSITORY}"
env.DOCKER_REPOSITORY_TCP = "tcp://${env.DOCKER_REPOSITORY}:4243"
env.NAMESPACE = 'production'
env.SLAVE_LABEL = "cryptosky-aio-build"
def mvn( String gloals ) {
sh "mvn -s configuration/settings.xml --show-version --batch-mode ${gloals}"
}
String get_application_version() {
"1.0.0-b${env.BUILD_NUMBER}"
}
String executeShellScript( String shellPath, String arg1 = '', String arg2 = '', String arg3 = '', String arg4 = '' ) {
sh "./${shellPath} ${arg1} ${arg2} ${arg3} ${arg4}"
}
try {
timestamps {
node ("${env.SLAVE_LABEL}") {
stage('Initialise') {
checkout([$class: 'GitSCM', branches: [[name: 'master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: 'Github', url: 'https://github.com/andyjk15/db-gateway.git']]])
env.APPLICATION_VERSION = get_application_version()
withCredentials(
[usernamePassword(
credentialsId: 'doctl',
passwordVariable: 'DOCTL_TOKEN',
usernameVariable: 'DOCTL_USERNAME'
)]
) {
sh "doctl auth init --access-token ${DOCTL_TOKEN}"
sh "doctl kubernetes cluster kubeconfig save cryptosky-kubernetes-cluster-production"
}
}
stage('Test Artifact') {
try {
// mvn 'verify -DskipUTs -DskipTests'
} finally {
// mvn 'test'
}
}
stage('Build Image') {
// mvn 'clean package -DskipTests'
executeShellScript("configuration/scripts/mapVarsToConfigs.sh",
env.DOCKER_REPOSITORY,
env.APPLICATION_NAME,
env.APPLICATION_VERSION,
env.APPLICATION_LABEL)
}
stage('Tag Repository') {
withDockerServer([uri: "${env.DOCKER_REPOSITORY_TCP}"]) {
withDockerRegistry([credentialsId: 'Registry', url: "${env.DOCKER_REPOSITORY_URL}"]) {
def image = docker.build("${env.DOCKER_REPOSITORY}/${env.APPLICATION_NAME}:${env.APPLICATION_VERSION}")
image.push()
def latest = docker.build("${env.DOCKER_REPOSITORY}/${env.APPLICATION_NAME}:latest")
latest.push()
}
}
withCredentials(
[usernamePassword(
credentialsId: env.GITHUB_CREDENTIALS_ID,
passwordVariable: 'GIT_PASSWORD',
usernameVariable: 'GIT_USERNAME'
)]
) {
sh "git tag ${env.APPLICATION_VERSION}"
sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@${env.GIT_REPOSITORY_PATH} ${env.APPLICATION_VERSION}"
}
}
stage('Deploy') {
executeShellScript("configuration/scripts/deployToKubernetes.sh",
env.APPLICATION_NAME)
}
}
}
} catch ( exception ) {
currentBuild.result = 'FAILURE'
throw exception
} finally {
currentBuild.result = 'SUCCESS'
}

10
configuration/scripts/deployToKubernetes.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
APPLICATION_NAME=$1
kubectl apply -f configuration/kubernetes/deployment.yaml
kubectl apply -f configuration/kubernetes/service.yaml
kubectl get pods
kubectl rollout status deployment/${APPLICATION_NAME}

14
configuration/scripts/mapVarsToConfigs.sh Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
DOCKER_REPOSITORY=$1
APPLICATION_NAME=$2
APPLICATION_VERSION=$3
APPLICATION_LABEL=$4
sed -i "s/REPOSITORY/${DOCKER_REPOSITORY}/g" configuration/kubernetes/deployment.yaml
sed -i "s/IMAGE/${APPLICATION_NAME}:${APPLICATION_VERSION}/g" configuration/kubernetes/deployment.yaml
sed -i "s/RESOURCE_NAME/${APPLICATION_NAME}/g" configuration/kubernetes/deployment.yaml
sed -i "s/LABEL/${APPLICATION_LABEL}/g" configuration/kubernetes/deployment.yaml
sed -i "s/RESOURCE_NAME/${APPLICATION_NAME}/g" configuration/kubernetes/service.yaml
sed -i "s/LABEL/${APPLICATION_LABEL}/g" configuration/kubernetes/service.yaml

2
src/main.py
View File

@ -8,7 +8,7 @@ def callCollector(args):
if __name__=='__main__':
# Dynamically create new child for each currency
currencies = [ "btc_gbp" ]
currencies = [ "btc_usd" ]
for i in range(len(currencies)):
Thread(target=callCollector, args=[currencies[i]]).start()

28
src/pricing/gemini.py
View File

@ -5,17 +5,13 @@ from currency_converter import CurrencyConverter
def geminiPublicTicker(type):
converter = CurrencyConverter()
try:
uri = "https://api.gemini.com/v1/pubticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
uri = "https://api.gemini.com/v1/pubticker/" + type.lower().replace('_', '')
response = requests.request("GET", uri)
response = json.loads(response.text)
price = (converter.convert(float(response['last']), 'USD', 'GBP') +
converter.convert(float(response['ask']), 'USD', 'GBP') +
converter.convert(float(response['bid']), 'USD', 'GBP'))/3
price = float(response['last']) + float(response['ask']) + float(response['bid'])/3
price = round(price, 3)
return price
except KeyError as e:
@ -26,20 +22,18 @@ def geminiPublicTicker(type):
def geminiHighLowVol(type):
converter = CurrencyConverter()
try:
uri = "https://api.gemini.com/v2/ticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
response = requests.request("GET", uri)
response = json.loads(response.text)
high = converter.convert(float(response['high']), 'USD', 'GBP')
low = converter.convert(float(response['low']), 'USD', 'GBP')
high = float(response['high'])
low = float(response['low'])
uri = "https://api.gemini.com/v1/pubticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
uri = "https://api.gemini.com/v1/pubticker/" + type.lower().replace('_', '')
response = requests.request("GET", uri)
response = json.loads(response.text)
vol = converter.convert(float(response['volume']['BTC']), 'USD', 'GBP')
vol = float(response['volume']['USD'])
return high, low, vol
except KeyError as e:
@ -49,15 +43,13 @@ def geminiHighLowVol(type):
def geminiOpenClose(type):
converter = CurrencyConverter()
try:
uri = "https://api.gemini.com/v2/ticker/" + (type.lower().replace('gbp', 'usd')).replace('_', '')
uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
response = requests.request("GET", uri)
response = json.loads(response.text)
open = converter.convert(float(response['open']), 'USD', 'GBP')
close = converter.convert(float(response['close']), 'USD', 'GBP')
open = float(response['open'])
close = float(response['close'])
return open, close
except KeyError as e:

9
src/utils/databaseConnect.py
View File

@ -2,16 +2,11 @@
import requests, os, json
from dotenv import load_dotenv
from pathlib import Path # python3 only
env_path = Path('.') / 'configuration/dbgateway.env'
load_dotenv(dotenv_path=env_path)
class keys():
def __init__(self):
self.uri = os.getenv("URI")
self.port = os.getenv("PORT")
self.uri = os.getenv("DATABASE_URL")
self.port = os.getenv("DATABASE_PORT")
def send(query):