andyjk15/tweet-collector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[12.10.20] Testing

Browse Source
This commit is contained in:
andrewso 2020-10-12 11:51:37 +01:00
parent 167932e390
commit 477cab8f09
1 changed files with 31 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/tweets/collector.py
View File

@ -90,52 +90,55 @@ class Listener(StreamListener):
if 'extended_tweet' in data['retweeted_status']: if 'extended_tweet' in data['retweeted_status']:
#if tweet is over the 140 word limit #if tweet is over the 140 word limit
text = data['retweeted_status']['extended_tweet']['full_text'] text = data['retweeted_status']['extended_tweet']['full_text']
dumpStack.append({'type': self.hashtag, 'tweet': text}) processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
else: else:
text = data['retweeted_status']['text'] text = data['retweeted_status']['text']
dumpStack.append({'type': self.hashtag, 'tweet': text}) processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
else: else:
# Else if a normal Tweeet # Else if a normal Tweeet
if 'extended_tweet' in data: if 'extended_tweet' in data:
# If tweet is over 140 word limit # If tweet is over 140 word limit
text = data['extended_tweet']['full_text'] text = data['extended_tweet']['full_text']
dumpStack.append({'type': self.hashtag, 'tweet': text}) processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
def processTweet(): def processTweet(text, type):
processStack = dumpStack.copy() # processStack = dumpStack.copy()
dumpStack.clear() # dumpStack.clear()
log("Processing [{}] Tweets...".format(len(processStack)), 'INFO') # log("Processing [{}] Tweets...".format(len(processStack)), 'INFO')
if len(processStack) != 0: # if len(processStack) != 0:
for tweet in processStack: # for tweet in processStack:
removedLines = fixLines(tweet["tweet"]) removedLines = fixLines(text)
removedSpecialChars = cleanTweet(removedLines) removedSpecialChars = cleanTweet(removedLines)
removedSpacing = removeSpacing(removedSpecialChars[0]) removedSpacing = removeSpacing(removedSpecialChars[0])
tweetLength = checkLength(removedSpacing) tweetLength = checkLength(removedSpacing)
if tweetLength == True: if tweetLength == True:
checkIfEnglish = detectLaguage(removedSpecialChars[0]) checkIfEnglish = detectLaguage(removedSpecialChars[0])
if checkIfEnglish == True: if checkIfEnglish == True:
tweetText = remove_non_ascii(removedSpacing) tweetText = remove_non_ascii(removedSpacing)
# log("Cleaned Tweet: {}".format(tweetText), 'INFO') # log("Cleaned Tweet: {}".format(tweetText), 'INFO')
cleanedTweet = tweetText + ' ' + removedSpecialChars[1] cleanedTweet = tweetText + ' ' + removedSpecialChars[1]
if callSpamFilter(cleanedTweet) != 'spam': if callSpamFilter(cleanedTweet) != 'spam':
pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet) pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet)
if compound != 0.0: if compound != 0.0:
hourTweet = {'pos': pos, 'neu': neu, 'neg': neg, 'compound': compound, 'type': tweet["type"]} hourTweet = {'pos': pos, 'neu': neu, 'neg': neg, 'compound': compound, 'type': type}
hourStack.append(hourTweet) hourStack.append(hourTweet)
processStack.clear() # processStack.clear()
else: # else:
log("Dump Stack was Empty", 'WARN') # log("Dump Stack was Empty", 'WARN')
def collector(hashtag): def collector(hashtag):
@ -165,7 +168,7 @@ def createHourJob():
global timeF global timeF
timeF = timeFunction() timeF = timeFunction()
processTweet() # processTweet()
log("Extracting sentiment scores...", 'INFO') log("Extracting sentiment scores...", 'INFO')