andyjk15/tweet-collector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[13.10.20] Testing

Browse Source
This commit is contained in:
andrewso 2020-10-13 09:31:39 +01:00
parent 50e2ef40fd
commit 9baf56f267
1 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/tweets/collector.py
View File

@ -54,6 +54,7 @@ class Streamer():
def stream_tweets(self, hashtag): def stream_tweets(self, hashtag):
listener = Listener(hashtag) listener = Listener(hashtag)
auth = OAuthHandler(keys().api_key, keys().api_secret) auth = OAuthHandler(keys().api_key, keys().api_secret)
log("Authorising with twitter API...", 'INFO') log("Authorising with twitter API...", 'INFO')
@ -114,43 +115,39 @@ class Listener(StreamListener):
def processTweet(text, type): def processTweet(text, type):
log("hourStack size :: [{}]".format(len(hourStack)), "INFO") log("hourStack size :: [{}]".format(len(hourStack)), "INFO")
# processStack = dumpStack.copy()
# dumpStack.clear()
# log("Processing [{}] Tweet...".format(text), 'INFO') # log("Processing [{}] Tweet...".format(text), 'INFO')
# if len(processStack) != 0:
# for tweet in processStack:
removedLines = fixLines(text) removedLines = fixLines(text)
log("fixLines", 'DEBUG') # log("fixLines", 'DEBUG')
removedSpecialChars = cleanTweet(removedLines) removedSpecialChars = cleanTweet(removedLines)
log("cleanTweet", 'DEBUG') # log("cleanTweet", 'DEBUG')
removedSpacing = removeSpacing(removedSpecialChars[0]) removedSpacing = removeSpacing(removedSpecialChars[0])
log("removeSpacing", 'DEBUG') # log("removeSpacing", 'DEBUG')
tweetLength = checkLength(removedSpacing) tweetLength = checkLength(removedSpacing)
log("checkLength", 'DEBUG') # log("checkLength", 'DEBUG')
if tweetLength == True: if tweetLength == True:
checkIfEnglish = detectLaguage(removedSpecialChars[0]) checkIfEnglish = detectLaguage(removedSpecialChars[0])
log("Boop2", 'DEBUG') # log("Boop2", 'DEBUG')
if checkIfEnglish == True: if checkIfEnglish == True:
tweetText = remove_non_ascii(removedSpacing) tweetText = remove_non_ascii(removedSpacing)
log("Boop3", 'DEBUG') # log("Boop3", 'DEBUG')
# log("Cleaned Tweet: {}".format(tweetText), 'INFO') # log("Cleaned Tweet: {}".format(tweetText), 'INFO')
cleanedTweet = tweetText + ' ' + removedSpecialChars[1] cleanedTweet = tweetText + ' ' + removedSpecialChars[1]
if callSpamFilter(cleanedTweet) != 'spam': if callSpamFilter(cleanedTweet) != 'spam':
log("After spam call", 'DEBUG') # log("After spam call", 'DEBUG')
pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet) pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet)
log("After Sentiment Call", 'DEBUG') # log("After Sentiment Call", 'DEBUG')
if compound != 0.0 | neu <= 0.8: if compound != 0.0 | neu <= 0.8:
log("if not 0.0 or above 0.8", 'DEBUG') log("if not 0.0 or above 0.8", 'DEBUG')