[3.03.20] Proper logging with timestamps and pod restart

configuration/kubernetes/deployment.yaml

@@ -6,7 +6,7 @@ metadata:
   name: RESOURCE_NAME
   namespace: production
 spec:
-  replicas: 1
+  replicas: 0
   selector:
     matchLabels:
       app: RESOURCE_NAME

src/pricing/bitfinex.py

@@ -2,8 +2,7 @@
 
 import requests, json, sys
 
-def bitfinexPublicTicker(type):
+def bitfinexPublicTicker(type, logger):
-
     try:
         uri = "https://api.bitfinex.com/v2/tickers?symbols=" + "t"+type.upper().replace('_', '')
 
@@ -14,12 +13,11 @@ def bitfinexPublicTicker(type):
         price = round(price, 3)
         return price
     except KeyError as e:
-        print("Bitfinex Spot Price Error: %s" % str(e))
+        logger.error("Bitfinex Spot Price Error: {}".format(e))
-        sys.stdout.flush()
         price = 0
         return price
 
-def bitfinexHighLowVol(type):
+def bitfinexHighLowVol(type, logger):
 
     try:
         uri = "https://api.bitfinex.com/v2/tickers?symbols=" + "t"+type.upper().replace('_', '')
@@ -33,6 +31,5 @@ def bitfinexHighLowVol(type):
 
         return high, low, vol
     except KeyError as e:
-        print("Bitfinex High Low Volume Error: %s" % str(e))
+        logger.error("Bitfinex High Low Volume Error: {}".format(e))
-        sys.stdout.flush()
         return 0, 0, 0

src/pricing/coinbase.py

@@ -4,16 +4,13 @@ import sys, os
 
 from coinbase.wallet.client import Client
 
-# from dotenv import load_dotenv
-# load_dotenv()
-
 class keys():
 
     def __init__(self):
         self.api_key = os.getenv('COINBASE_KEY')
         self.api_secret = os.getenv("COINBASE_SECRET")
 
-def coinbasePublicTicker(type):
+def coinbasePublicTicker(type, logger):
 
     api_key = keys().api_key
     api_secret = keys().api_secret
@@ -27,7 +24,6 @@ def coinbasePublicTicker(type):
         price = round(price, 3)
         return price
     except KeyError as e:
-        print("Coinbase Spot Price Error: %s" % str(e))
+        logger.error("Coinbase Spot Price Error: {}".format(e))
-        sys.stdout.flush()
         price = 0
         return price

src/pricing/collector.py

@@ -10,27 +10,25 @@ from pricing.bitfinex import bitfinexPublicTicker, bitfinexHighLowVol
 from pricing.coinbase import coinbasePublicTicker
 from pricing.gemini import geminiPublicTicker, geminiHighLowVol, geminiOpenClose
 
-from src.utils.databaseConnect import send
+from utils.databaseConnect import send
 
-btc_usd="/home/price-collector/src/resources/queries/V1_INSERT_NEW_PRICE_RECORD_BTC.graphql"
+import logging as logger
 
-def getInsertForType(type):
+logger.basicConfig(
-    if type == "btc_usd":
+    level=logger.INFO,
-        with open(btc_usd, 'r') as s:
+    format="%(asctime)s: %(levelname)s -- %(message)s",
-            sql = s.read()
+    datefmt='%Y-%m-%d %H:%M:%S'
-            s.close()
+)
-            return sql
+
-    elif type == "":
+btc_usd="src/resources/queries/V1_INSERT_NEW_PRICE_RECORD_BTC.graphql"
-        return ""
 
 def averager(type):
 
-    # FORMAT FOR RFC-3339
     timestamp = datetime.now()# + timedelta(hours=1)
 
-    coinbase_P = coinbasePublicTicker(type)
+    coinbase_P = coinbasePublicTicker(type, logger)
-    bitfinex_P = bitfinexPublicTicker(type)
+    bitfinex_P = bitfinexPublicTicker(type, logger)
-    gemini_P = geminiPublicTicker(type)
+    gemini_P = geminiPublicTicker(type, logger)
 
     if coinbase_P == 0 or bitfinex_P == 0 or gemini_P == 0:
         if coinbase_P and bitfinex_P == 0:
@@ -48,14 +46,13 @@ def averager(type):
 
     averagePrice = round(averagePrice, 3)
 
-    print("Price: ", averagePrice)
+    logger.info("Hourly Price for ({}) is {}".format(timestamp ,averagePrice))
-    sys.stdout.flush()
 
     return averagePrice, timestamp
 
 def getHighLowVol(type):
-    bH, bL, bV = bitfinexHighLowVol(type)
+    bH, bL, bV = bitfinexHighLowVol(type, logger)
-    gH, gL, gV = geminiHighLowVol(type)
+    gH, gL, gV = geminiHighLowVol(type, logger)
 
     if ( bH == 0 or bL == 0 or bV == 0 ) or ( gH == 0 or gL == 0 or gL == 0):
         if bH == 0:
@@ -87,24 +84,24 @@ def sendToGateway(c_type, timestamp, av_price, high, low, vol, o_price, c_price)
 
     query = data % (timestamp, type, av_price, high, low, o_price, c_price, vol)
 
-    print(query)
+    logger.info("Query sending down to db-gateway -- ({})".format(query))
 
-    status, response = send(query)
+    status, response = send(query, logger)
 
-    print("Status: ", status)
+    if status != '200':
-    sys.stdout.flush()
+        logger.critical("Query wasn't executed properly, view logs. Status = {}".format(status))
-
+        logger.error("With Response of : {}".format(response))
-    print("Response: ", response)
+    else:
-    sys.stdout.flush()
+        logger.info("Query executed successfully with Status = {}".format(status))
+        logger.info("With Response of : {}".format(response))
 
 def getOpenClose(type):
-    open, close = geminiOpenClose(type)
+    open, close = geminiOpenClose(type, logger)
     return open, close
 
 # Dynamically Spin up Child process for each type wanting to track
 def collector(c_type):
-    print("Console: ", "== Historical Price Collector ==")
+    logger.info("== Historical Price Collector ==")
-    sys.stdout.flush()
 
     while True:
         av_price, timestamp = averager(c_type)

src/pricing/gemini.py

@@ -2,7 +2,7 @@
 
 import requests, json, sys
 
-def geminiPublicTicker(type):
+def geminiPublicTicker(type, logger):
 
     try:
         uri = "https://api.gemini.com/v1/pubticker/" + type.lower().replace('_', '')
@@ -15,13 +15,11 @@ def geminiPublicTicker(type):
 
         return price
     except KeyError as e:
-        print("Gemini Spot Price Error: %s" % str(e))
+        logger.error("Gemini Spot Price Error: {}".format(e))
-        sys.stdout.flush()
         price = 0
         return price
 
-def geminiHighLowVol(type):
+def geminiHighLowVol(type, logger):
-
     try:
         uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
         response = requests.request("GET", uri)
@@ -37,12 +35,10 @@ def geminiHighLowVol(type):
 
         return high, low, vol
     except KeyError as e:
-        print("Gemini High Low Volume Error: %s" % str(e))
+        logger.error("Gemini High Low Volume Error: {}".format(e))
-        sys.stdout.flush()
         return 0, 0, 0
 
-def geminiOpenClose(type):
+def geminiOpenClose(type, logger):
-
     try:
         uri = "https://api.gemini.com/v2/ticker/" + type.lower().replace('_', '')
         response = requests.request("GET", uri)
@@ -53,6 +49,6 @@ def geminiOpenClose(type):
 
         return open, close
     except KeyError as e:
-        print("Gemini Open Close Error: %s" % str(e))
+        logger.error("Gemini Open Close Error: {}".format(e))
         sys.stdout.flush()
         return 0, 0

src/utils/databaseConnect.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import requests, os, json
+import requests, os, json, sys
 
 class keys():
 
@@ -8,16 +8,20 @@ class keys():
         self.uri = os.getenv("DATABASE_URL")
         self.port = os.getenv("DATABASE_PORT")
 
-def send(query):
+def send(query, logger):
+    try:
+        uri = keys().uri + "/graphql"
+        headers = {'Content-type': 'application/json'}
 
-    uri = keys().uri + "/graphql"
+        response = requests.post(uri, json={'query': query}, headers=headers)
-    headers = {'Content-type': 'application/json'}
 
-    print(uri)
+        statusCode = response.status_code
+        response = json.loads(response.text)
 
-    response = requests.post(uri, json={'query': query}, headers=headers)
+        return statusCode, response
-
+    except requests.exceptions.HTTPError as e:
-    statusCode = response.status_code
+        logger.critical("Unable to send data down to db-gateway: {}".format(e))
-    response = json.loads(response.text)
+        sys.exit(1)
-
+    except requests.exceptions.RequestException as e:
-    return statusCode, response
+        logger.critical("Unable to send data down to db-gateway: {}".format(e))
+        sys.exit(1)