andyjk15/tweet-collector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[09.10.2020] Handling of incomplete read when API gets overloaded

Browse Source
This commit is contained in:
Andy Sotheran 2020-10-09 22:57:05 +01:00
parent 0903b69b8e
commit 06c95da108
1 changed files with 32 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/tweets/collector.py
View File

@ -18,7 +18,10 @@ from src.utils.sentimentAnalyser import callSentimentAnalyser
from src.utils.activemqConnect import activeMQSender
from src.utils.jsonLogger import log
from http.client import IncompleteRead
hourStack = []
dumpStack = []
class keys():
@ -60,8 +63,12 @@ class Streamer():
wait_on_rate_limit_notify=True)
log("Streaming Tweets", 'INFO')
while True:
try:
stream = Stream(auth=api.auth, listener=listener, tweet_mode='extended')
stream.filter(languages=["en"], track=hashtag)
except IncompleteRead:
continue
class Listener(StreamListener):
@ -79,10 +86,10 @@ class Listener(StreamListener):
if 'extended_tweet' in data['retweeted_status']:
#if tweet is over the 140 word limit
text = data['retweeted_status']['extended_tweet']['full_text']
self.processTweet(text)
dumpStack.append(text)
else:
text = data['retweeted_status']['text']
self.processTweet(text)
dumpStack.append(text)
else:
# Else if a normal Tweeet
if 'extended_tweet' in data:
@ -90,8 +97,11 @@ class Listener(StreamListener):
text = data['extended_tweet']['full_text']
self.processTweet(text)
def processTweet(self, text):
removedLines = fixLines(text)
def processTweet():
if len(dumpStack) != 0:
for tweet in dumpStack:
removedLines = fixLines(tweet)
removedSpecialChars = cleanTweet(removedLines)
removedSpacing = removeSpacing(removedSpecialChars[0])
tweetLength = checkLength(removedSpacing)
@ -143,6 +153,8 @@ def createHourJob():
global timeF
timeF = timeFunction()
processTweet()
if len(hourStack) != 0:
for item in hourStack:
ovPos = ovPos + item['pos']