andyjk15/tweet-collector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[12.10.20] Testing

Browse Source
This commit is contained in:
andrewso 2020-10-12 11:51:37 +01:00
parent 167932e390
commit 477cab8f09
1 changed files with 31 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/tweets/collector.py
View File

@ -90,52 +90,55 @@ class Listener(StreamListener):
if 'extended_tweet' in data['retweeted_status']:
#if tweet is over the 140 word limit
text = data['retweeted_status']['extended_tweet']['full_text']
dumpStack.append({'type': self.hashtag, 'tweet': text})
processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
else:
text = data['retweeted_status']['text']
dumpStack.append({'type': self.hashtag, 'tweet': text})
processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
else:
# Else if a normal Tweeet
if 'extended_tweet' in data:
# If tweet is over 140 word limit
text = data['extended_tweet']['full_text']
dumpStack.append({'type': self.hashtag, 'tweet': text})
processTweet(text, self.hashtag)
# dumpStack.append({'type': self.hashtag, 'tweet': text})
def processTweet():
def processTweet(text, type):
processStack = dumpStack.copy()
dumpStack.clear()
# processStack = dumpStack.copy()
# dumpStack.clear()
log("Processing [{}] Tweets...".format(len(processStack)), 'INFO')
# log("Processing [{}] Tweets...".format(len(processStack)), 'INFO')
if len(processStack) != 0:
for tweet in processStack:
removedLines = fixLines(tweet["tweet"])
removedSpecialChars = cleanTweet(removedLines)
removedSpacing = removeSpacing(removedSpecialChars[0])
tweetLength = checkLength(removedSpacing)
if tweetLength == True:
# if len(processStack) != 0:
# for tweet in processStack:
removedLines = fixLines(text)
removedSpecialChars = cleanTweet(removedLines)
removedSpacing = removeSpacing(removedSpecialChars[0])
tweetLength = checkLength(removedSpacing)
if tweetLength == True:
checkIfEnglish = detectLaguage(removedSpecialChars[0])
checkIfEnglish = detectLaguage(removedSpecialChars[0])
if checkIfEnglish == True:
if checkIfEnglish == True:
tweetText = remove_non_ascii(removedSpacing)
tweetText = remove_non_ascii(removedSpacing)
# log("Cleaned Tweet: {}".format(tweetText), 'INFO')
# log("Cleaned Tweet: {}".format(tweetText), 'INFO')
cleanedTweet = tweetText + ' ' + removedSpecialChars[1]
cleanedTweet = tweetText + ' ' + removedSpecialChars[1]
if callSpamFilter(cleanedTweet) != 'spam':
pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet)
if callSpamFilter(cleanedTweet) != 'spam':
pos, neu, neg, compound = callSentimentAnalyser(cleanedTweet)
if compound != 0.0:
hourTweet = {'pos': pos, 'neu': neu, 'neg': neg, 'compound': compound, 'type': tweet["type"]}
if compound != 0.0:
hourTweet = {'pos': pos, 'neu': neu, 'neg': neg, 'compound': compound, 'type': type}
hourStack.append(hourTweet)
processStack.clear()
else:
log("Dump Stack was Empty", 'WARN')
hourStack.append(hourTweet)
# processStack.clear()
# else:
# log("Dump Stack was Empty", 'WARN')
def collector(hashtag):
@ -165,7 +168,7 @@ def createHourJob():
global timeF
timeF = timeFunction()
processTweet()
# processTweet()
log("Extracting sentiment scores...", 'INFO')