diff --git a/document.aux b/document.aux index bed4910..16c48a2 100644 --- a/document.aux +++ b/document.aux @@ -31,15 +31,15 @@ \newlabel{glossary}{{3}{4}{Glossary}{section.3}{}} \abx@aux@cite{SaTdpsmm} \abx@aux@segm{0}{0}{SaTdpsmm} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {4}Introduction}{8}{section.4}} -\newlabel{introduction}{{4}{8}{Introduction}{section.4}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {5}Problem Articulation}{10}{section.5}} -\newlabel{problem}{{5}{10}{Problem Articulation}{section.5}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Problem Statement}{10}{subsection.5.1}} -\newlabel{statement}{{5.1}{10}{Problem Statement}{subsection.5.1}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Stakeholders}{10}{subsection.5.2}} -\newlabel{stakeholders}{{5.2}{10}{Stakeholders}{subsection.5.2}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Project Motivation}{11}{subsection.5.3}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {4}Introduction}{9}{section.4}} +\newlabel{introduction}{{4}{9}{Introduction}{section.4}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {5}Problem Articulation}{11}{section.5}} +\newlabel{problem}{{5}{11}{Problem Articulation}{section.5}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Problem Statement}{11}{subsection.5.1}} +\newlabel{statement}{{5.1}{11}{Problem Statement}{subsection.5.1}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Stakeholders}{11}{subsection.5.2}} +\newlabel{stakeholders}{{5.2}{11}{Stakeholders}{subsection.5.2}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Project Motivation}{12}{subsection.5.3}} \abx@aux@cite{nlAeiBTCPSO} \abx@aux@segm{0}{0}{nlAeiBTCPSO} \abx@aux@cite{BTCFTsent} @@ -47,9 +47,7 @@ \abx@aux@cite{BTCRNN} \abx@aux@segm{0}{0}{BTCRNN} \abx@aux@segm{0}{0}{SaTdpsmm} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.4}Technical Specification}{13}{subsection.5.4}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.5}Project Constraints}{15}{subsection.5.5}} -\newlabel{constraints}{{5.5}{15}{Project Constraints}{subsection.5.5}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {5.4}Technical Specification}{14}{subsection.5.4}} \abx@aux@cite{ISO9000} \abx@aux@segm{0}{0}{ISO9000} \@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {6}Quality Goals}{16}{section.6}} @@ -168,111 +166,122 @@ \abx@aux@cite{SpamCScratch} \abx@aux@segm{0}{0}{SpamCScratch} \abx@aux@segm{0}{0}{SpamCScratch} +\abx@aux@cite{RMSEMAE} +\abx@aux@segm{0}{0}{RMSEMAE} +\abx@aux@cite{MSE} +\abx@aux@segm{0}{0}{MSE} +\abx@aux@segm{0}{0}{RMSEMAE} +\abx@aux@cite{MAPE} +\abx@aux@segm{0}{0}{MAPE} \@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {7.9}Addictive Smoothing}{31}{subsection.7.9}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {7.10}Regression Performance Metrics}{31}{subsection.7.10}} \abx@aux@cite{TweepyDoc} \abx@aux@segm{0}{0}{TweepyDoc} \abx@aux@segm{0}{0}{SearchTweets} \abx@aux@segm{0}{0}{ConStream} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {8}Solution Approach}{32}{section.8}} -\newlabel{solution}{{8}{32}{Solution Approach}{section.8}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.1}Data gathering}{32}{subsection.8.1}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.2}Data pre-processing}{33}{subsection.8.2}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.3}Spam Filtering}{33}{subsection.8.3}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {8}Solution Approach}{33}{section.8}} +\newlabel{solution}{{8}{33}{Solution Approach}{section.8}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.1}Data gathering}{33}{subsection.8.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.2}Data pre-processing}{34}{subsection.8.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.3}Spam Filtering}{34}{subsection.8.3}} \abx@aux@segm{0}{0}{VADERPaper} \abx@aux@segm{0}{0}{VADERPaper} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.4}Language Detection}{34}{subsection.8.4}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.5}Sentiment Analysis}{34}{subsection.8.5}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.4}Language Detection}{35}{subsection.8.4}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.5}Sentiment Analysis}{35}{subsection.8.5}} \abx@aux@segm{0}{0}{VADERPaper} \abx@aux@segm{0}{0}{LexiconSocSent} \abx@aux@segm{0}{0}{LSTM} \abx@aux@segm{0}{0}{StPNSentA} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.6}Neural Network}{35}{subsection.8.6}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.6}Neural Network}{36}{subsection.8.6}} \abx@aux@segm{0}{0}{LSTMforetime} \abx@aux@segm{0}{0}{LSTMforetime} \abx@aux@cite{TFvsThe} \abx@aux@segm{0}{0}{TFvsThe} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.7}Price Forecasting}{37}{subsection.8.7}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.8}Frontend Application}{37}{subsection.8.8}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.9}With reference to Initial PID}{37}{subsection.8.9}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.10}Solution Summary}{38}{subsection.8.10}} -\newlabel{summary}{{8.10}{38}{Solution Summary}{subsection.8.10}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.11}Data flow Overview}{39}{subsection.8.11}} -\newlabel{data-flow}{{8.11}{39}{Data flow Overview}{subsection.8.11}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {9}System Design}{40}{section.9}} -\newlabel{Design}{{9}{40}{System Design}{section.9}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {9.1}Dataflow Designs}{40}{subsection.9.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.7}Price Forecasting}{38}{subsection.8.7}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.8}Frontend Application}{38}{subsection.8.8}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.9}With reference to Initial PID}{38}{subsection.8.9}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.10}Solution Summary}{39}{subsection.8.10}} +\newlabel{summary}{{8.10}{39}{Solution Summary}{subsection.8.10}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {8.11}Data flow Overview}{40}{subsection.8.11}} +\newlabel{data-flow}{{8.11}{40}{Data flow Overview}{subsection.8.11}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {9}System Design}{41}{section.9}} +\newlabel{Design}{{9}{41}{System Design}{section.9}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {9.1}Dataflow Designs}{41}{subsection.9.1}} \abx@aux@segm{0}{0}{VADERPaper} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {9.2}Interface Design}{47}{subsection.9.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {9.2}Interface Design}{48}{subsection.9.2}} \abx@aux@cite{btcCharts} \abx@aux@segm{0}{0}{btcCharts} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {10}Implementation}{49}{section.10}} -\newlabel{implementation}{{10}{49}{Implementation}{section.10}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.1}Data collection}{49}{subsection.10.1}} -\newlabel{collection}{{10.1}{49}{Data collection}{subsection.10.1}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.1}Price Time-Series Historical Data}{49}{subsubsection.10.1.1}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {1}Historical price collection and averaging per exchange}{49}{lstlisting.1}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.2}Price Time-Series Live Data}{50}{subsubsection.10.1.2}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {2}Extraction of Price from exchanges}{50}{lstlisting.2}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.3}Historical Tweet Collection}{51}{subsubsection.10.1.3}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {3}Sample Curl request - data saved to json and python scripted called to process data}{51}{lstlisting.3}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {4}Sift-text python script - used alongside Curl command in Listing 4}{51}{lstlisting.4}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.4}Live Tweet Collection}{53}{subsubsection.10.1.4}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {5}Tweepy Streamer setup}{53}{lstlisting.5}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.2}Data pre-processing}{55}{subsection.10.2}} -\newlabel{processing}{{10.2}{55}{Data pre-processing}{subsection.10.2}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.1}Tweet Filtering}{55}{subsubsection.10.2.1}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {6}Basic data filtering and processing function - defined in 'tweet\_collector.py'}{55}{lstlisting.6}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.2}Language detection filtering}{56}{subsubsection.10.2.2}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {7}Language detection and filter function \cite {langdectNLTK}}{56}{lstlisting.7}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {10}Implementation}{50}{section.10}} +\newlabel{implementation}{{10}{50}{Implementation}{section.10}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.1}Data collection}{50}{subsection.10.1}} +\newlabel{collection}{{10.1}{50}{Data collection}{subsection.10.1}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.1}Price Time-Series Historical Data}{50}{subsubsection.10.1.1}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {1}Historical price collection and averaging per exchange}{50}{lstlisting.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.2}Price Time-Series Live Data}{51}{subsubsection.10.1.2}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {2}Extraction of Price from exchanges}{51}{lstlisting.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.3}Historical Tweet Collection}{52}{subsubsection.10.1.3}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {3}Sample Curl request - data saved to json and python scripted called to process data}{52}{lstlisting.3}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {4}Sift-text python script - used alongside Curl command in Listing 4}{52}{lstlisting.4}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.1.4}Live Tweet Collection}{54}{subsubsection.10.1.4}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {5}Tweepy Streamer setup}{54}{lstlisting.5}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.2}Data pre-processing}{56}{subsection.10.2}} +\newlabel{processing}{{10.2}{56}{Data pre-processing}{subsection.10.2}{}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.1}Tweet Filtering}{56}{subsubsection.10.2.1}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {6}Basic data filtering and processing function - defined in 'tweet\_collector.py'}{56}{lstlisting.6}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.2}Language detection filtering}{57}{subsubsection.10.2.2}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {7}Language detection and filter function \cite {langdectNLTK}}{57}{lstlisting.7}} \abx@aux@cite{langdectNLTK} \abx@aux@segm{0}{0}{langdectNLTK} \abx@aux@segm{0}{0}{langdectNLTK} \abx@aux@cite{LanNgram} \abx@aux@segm{0}{0}{LanNgram} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.3}Spam filter - Tokenisation, Ngrams, Stopword removal and Stemming}{58}{subsubsection.10.2.3}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {8}pre-processing of data prior to being used by the spam filter}{58}{lstlisting.8}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.2.3}Spam filter - Tokenisation, Ngrams, Stopword removal and Stemming}{59}{subsubsection.10.2.3}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {8}pre-processing of data prior to being used by the spam filter}{59}{lstlisting.8}} \abx@aux@cite{StemvsLem} \abx@aux@segm{0}{0}{StemvsLem} \abx@aux@segm{0}{0}{StemvsLem} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.3}Spam Filtering}{59}{subsection.10.3}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {9}Spam filter training Class - \textit {tweet\_collector.py}}{59}{lstlisting.9}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.3}Spam Filtering}{60}{subsection.10.3}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {9}Spam filter training Class - \textit {tweet\_collector.py}}{60}{lstlisting.9}} \abx@aux@cite{NValgor} \abx@aux@segm{0}{0}{NValgor} \abx@aux@segm{0}{0}{SpamCScratch} \abx@aux@cite{SpamOrHamGit} \abx@aux@segm{0}{0}{SpamOrHamGit} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.1}Naive Bayes model}{62}{subsubsection.10.3.1}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {10}classifer class of spam\_filter.py}{62}{lstlisting.10}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.2}Classification}{63}{subsubsection.10.3.2}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {11}Classify Function of Parent classifier class of spam\_filter.py}{63}{lstlisting.11}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.3}Predict}{64}{subsubsection.10.3.3}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {12}Predict function of parent classifier class of spam\_filter.py}{64}{lstlisting.12}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.4}Metrics}{64}{subsubsection.10.3.4}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {13}Metrics function for calculating the performance and accuracy of the model}{64}{lstlisting.13}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.4}Sentiment Analysis}{65}{subsection.10.4}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {14}VADER polarity classification}{65}{lstlisting.14}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.5}Recurrent Neural Network - LSTM}{66}{subsection.10.5}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.5.1}Dataset Creation}{66}{subsubsection.10.5.1}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {15}Dataset creation and preprocessing}{67}{lstlisting.15}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.5.2}Training and Testing Model}{68}{subsubsection.10.5.2}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {16}LSTM model creation\tmspace +\thinmuskip {.1667em} layering\tmspace +\thinmuskip {.1667em} compiling and fitting}{68}{lstlisting.16}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.6}Future Prediction Forecasting}{70}{subsection.10.6}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {17}Forecasting future price of next hour for Bitcoin}{70}{lstlisting.17}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.7}User Interface}{72}{subsection.10.7}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.7.1}Key Functions}{72}{subsubsection.10.7.1}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {18}AJAX request and plotting performance data to HTML table}{72}{lstlisting.18}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {19}Chart creation with AJAX request}{73}{lstlisting.19}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.7.2}Final Interface}{75}{subsubsection.10.7.2}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {11}Testing Metrics and Accuracy}{76}{section.11}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {12}Discussion of Results}{76}{section.12}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {13}Project Evaluation}{77}{section.13}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {14}Discussion: Contribution and Reflection}{77}{section.14}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {14.1}Limitations}{77}{subsection.14.1}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {15}Conclusion and Future Improvements}{78}{section.15}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {15.1}Conclusion}{78}{subsection.15.1}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {15.2}Future Improvements}{78}{subsection.15.2}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {16}Appendices}{83}{section.16}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {16.1}Appendix A - Project Initiation Document}{83}{subsection.16.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.1}Naive Bayes model}{63}{subsubsection.10.3.1}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {10}classifer class of spam\_filter.py}{63}{lstlisting.10}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.2}Classification}{64}{subsubsection.10.3.2}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {11}Classify Function of Parent classifier class of spam\_filter.py}{64}{lstlisting.11}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.3}Predict}{65}{subsubsection.10.3.3}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {12}Predict function of parent classifier class of spam\_filter.py}{65}{lstlisting.12}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.3.4}Metrics}{65}{subsubsection.10.3.4}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {13}Metrics function for calculating the performance and accuracy of the model}{65}{lstlisting.13}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.4}Sentiment Analysis}{66}{subsection.10.4}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {14}VADER polarity classification}{66}{lstlisting.14}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.5}Recurrent Neural Network - LSTM}{67}{subsection.10.5}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.5.1}Dataset Creation}{67}{subsubsection.10.5.1}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {15}Dataset creation and preprocessing}{68}{lstlisting.15}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.5.2}Training and Testing Model}{69}{subsubsection.10.5.2}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {16}LSTM model creation\tmspace +\thinmuskip {.1667em} layering\tmspace +\thinmuskip {.1667em} compiling and fitting}{69}{lstlisting.16}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.6}Future Prediction Forecasting}{71}{subsection.10.6}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {17}Forecasting future price of next hour for Bitcoin}{71}{lstlisting.17}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {10.7}User Interface}{73}{subsection.10.7}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.7.1}Key Functions}{73}{subsubsection.10.7.1}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {18}AJAX request and plotting performance data to HTML table}{73}{lstlisting.18}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {19}Chart creation with AJAX request}{74}{lstlisting.19}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {10.7.2}Final Interface}{76}{subsubsection.10.7.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {11}Testing Metrics and Accuracy}{77}{section.11}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {11.1}Integration Testing}{77}{subsection.11.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {11.2}Accuracy of Model \& Results}{78}{subsection.11.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {11.2.1}Results Discussion}{78}{subsubsection.11.2.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{\numberline {11.2.2}Execution Speeds}{80}{subsubsection.11.2.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {12}Project Evaluation}{81}{section.12}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {13}Discussion: Contribution and Reflection}{81}{section.13}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {13.1}Limitations}{81}{subsection.13.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {14}Conclusion and Future Improvements}{82}{section.14}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {14.1}Conclusion}{82}{subsection.14.1}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {14.2}Future Improvements}{82}{subsection.14.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{\numberline {15}Appendices}{87}{section.15}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {15.1}Appendix A - Project Initiation Document}{87}{subsection.15.1}} \abx@aux@refcontextdefaultsdone \abx@aux@defaultrefcontext{0}{SaTdpsmm}{none/global//global/global} \abx@aux@defaultrefcontext{0}{nlAeiBTCPSO}{none/global//global/global} @@ -309,6 +318,9 @@ \abx@aux@defaultrefcontext{0}{StudyNBC}{none/global//global/global} \abx@aux@defaultrefcontext{0}{TFIDFBOW}{none/global//global/global} \abx@aux@defaultrefcontext{0}{SpamCScratch}{none/global//global/global} +\abx@aux@defaultrefcontext{0}{RMSEMAE}{none/global//global/global} +\abx@aux@defaultrefcontext{0}{MSE}{none/global//global/global} +\abx@aux@defaultrefcontext{0}{MAPE}{none/global//global/global} \abx@aux@defaultrefcontext{0}{TweepyDoc}{none/global//global/global} \abx@aux@defaultrefcontext{0}{TFvsThe}{none/global//global/global} \abx@aux@defaultrefcontext{0}{btcCharts}{none/global//global/global} @@ -317,4 +329,4 @@ \abx@aux@defaultrefcontext{0}{StemvsLem}{none/global//global/global} \abx@aux@defaultrefcontext{0}{NValgor}{none/global//global/global} \abx@aux@defaultrefcontext{0}{SpamOrHamGit}{none/global//global/global} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {16.2}Appendix B - Log book}{96}{subsection.16.2}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{\numberline {15.2}Appendix B - Log book}{100}{subsection.15.2}} diff --git a/document.bbl b/document.bbl index c6d6bc5..eeca792 100644 --- a/document.bbl +++ b/document.bbl @@ -1186,6 +1186,94 @@ \verb https://towardsdatascience.com/spam-classifier-in-python-from-scratch-27a98ddd8e73 \endverb \endentry + \entry{RMSEMAE}{inproceedings}{} + \name{author}{1}{}{% + {{hash=4557151fc83b7534d523a502a84a9c9c}{% + family={Swalin}, + familyi={S\bibinitperiod}, + given={Alvira}, + giveni={A\bibinitperiod}}}% + } + \list{organization}{1}{% + {Medium}% + } + \strng{namehash}{4557151fc83b7534d523a502a84a9c9c} + \strng{fullhash}{4557151fc83b7534d523a502a84a9c9c} + \strng{bibnamehash}{4557151fc83b7534d523a502a84a9c9c} + \strng{authorbibnamehash}{4557151fc83b7534d523a502a84a9c9c} + \strng{authornamehash}{4557151fc83b7534d523a502a84a9c9c} + \strng{authorfullhash}{4557151fc83b7534d523a502a84a9c9c} + \field{sortinit}{6} + \field{sortinithash}{57e57fb8451e7fcfa45d1e069f6d3136} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{title}{Choosing the Right Metric for Evaluating Machine Learning Models} + \field{year}{Apr 7, 2018} + \verb{urlraw} + \verb https://medium.com/usf-msds/choosing-the-right-metric-for-machine-learning-models-part-1-a99d7d7414e4 + \endverb + \verb{url} + \verb https://medium.com/usf-msds/choosing-the-right-metric-for-machine-learning-models-part-1-a99d7d7414e4 + \endverb + \endentry + \entry{MSE}{inproceedings}{} + \name{author}{1}{}{% + {{hash=ef4b13671a2486f2f568d40a7a46a066}{% + family={Binieli}, + familyi={B\bibinitperiod}, + given={Moshe}, + giveni={M\bibinitperiod}}}% + } + \list{organization}{1}{% + {Medium}% + } + \strng{namehash}{ef4b13671a2486f2f568d40a7a46a066} + \strng{fullhash}{ef4b13671a2486f2f568d40a7a46a066} + \strng{bibnamehash}{ef4b13671a2486f2f568d40a7a46a066} + \strng{authorbibnamehash}{ef4b13671a2486f2f568d40a7a46a066} + \strng{authornamehash}{ef4b13671a2486f2f568d40a7a46a066} + \strng{authorfullhash}{ef4b13671a2486f2f568d40a7a46a066} + \field{sortinit}{6} + \field{sortinithash}{57e57fb8451e7fcfa45d1e069f6d3136} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{title}{Machine learning: an introduction to mean squared error and regression lines} + \field{year}{Oct 16, 2018} + \verb{urlraw} + \verb https://medium.freecodecamp.org/machine-learning-mean-squared-error-regression-line-c7dde9a26b93 + \endverb + \verb{url} + \verb https://medium.freecodecamp.org/machine-learning-mean-squared-error-regression-line-c7dde9a26b93 + \endverb + \endentry + \entry{MAPE}{inproceedings}{} + \name{author}{1}{}{% + {{hash=d764afef638302d6579ce78222a92818}{% + family={Stephanie}, + familyi={S\bibinitperiod}}}% + } + \list{organization}{1}{% + {Statistics HowTo}% + } + \strng{namehash}{d764afef638302d6579ce78222a92818} + \strng{fullhash}{d764afef638302d6579ce78222a92818} + \strng{bibnamehash}{d764afef638302d6579ce78222a92818} + \strng{authorbibnamehash}{d764afef638302d6579ce78222a92818} + \strng{authornamehash}{d764afef638302d6579ce78222a92818} + \strng{authorfullhash}{d764afef638302d6579ce78222a92818} + \field{sortinit}{6} + \field{sortinithash}{57e57fb8451e7fcfa45d1e069f6d3136} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{title}{Mean absolute percentage error (MAPE)} + \field{year}{Sep 8, 2017} + \verb{urlraw} + \verb https://www.statisticshowto.datasciencecentral.com/mean-absolute-percentage-error-mape/ + \endverb + \verb{url} + \verb https://www.statisticshowto.datasciencecentral.com/mean-absolute-percentage-error-mape/ + \endverb + \endentry \entry{TweepyDoc}{inproceedings}{} \name{author}{1}{}{% {{hash=1e48a059b3f2c3703bea8d54a9e002c9}{% @@ -1285,8 +1373,8 @@ \strng{authorbibnamehash}{41b3b5daa9b33f52b08087b54f96f115} \strng{authornamehash}{41b3b5daa9b33f52b08087b54f96f115} \strng{authorfullhash}{41b3b5daa9b33f52b08087b54f96f115} - \field{sortinit}{7} - \field{sortinithash}{c818dd9105a2852444fc9f5e145c294e} + \field{sortinit}{8} + \field{sortinithash}{07edf88d4ea82509b9c4b4d13f41c452} \field{labelnamesource}{author} \field{labeltitlesource}{title} \field{title}{Detecting Text Language With Python and NLTK} @@ -1314,8 +1402,8 @@ \strng{authorbibnamehash}{90ecaa1e27a33fbd3ab56223485ab402} \strng{authornamehash}{90ecaa1e27a33fbd3ab56223485ab402} \strng{authorfullhash}{90ecaa1e27a33fbd3ab56223485ab402} - \field{sortinit}{7} - \field{sortinithash}{c818dd9105a2852444fc9f5e145c294e} + \field{sortinit}{8} + \field{sortinithash}{07edf88d4ea82509b9c4b4d13f41c452} \field{labelnamesource}{author} \field{labeltitlesource}{title} \field{title}{A tutorial on Automatic Language Identification - ngram based} @@ -1343,8 +1431,8 @@ \strng{authorbibnamehash}{b26f0ccdcc8e766bb4785cce3a550346} \strng{authornamehash}{b26f0ccdcc8e766bb4785cce3a550346} \strng{authorfullhash}{b26f0ccdcc8e766bb4785cce3a550346} - \field{sortinit}{7} - \field{sortinithash}{c818dd9105a2852444fc9f5e145c294e} + \field{sortinit}{8} + \field{sortinithash}{07edf88d4ea82509b9c4b4d13f41c452} \field{labelnamesource}{author} \field{labeltitlesource}{title} \field{title}{What is the difference between stemming and lemmatization} diff --git a/document.bcf b/document.bcf index c94621e..07dc988 100644 --- a/document.bcf +++ b/document.bcf @@ -1985,28 +1985,32 @@ TFIDFBOW SpamCScratch SpamCScratch - TweepyDoc - SearchTweets - ConStream - VADERPaper - VADERPaper - VADERPaper - LexiconSocSent - LSTM - StPNSentA - LSTMforetime - LSTMforetime - TFvsThe - VADERPaper - btcCharts - langdectNLTK - langdectNLTK - LanNgram - StemvsLem - StemvsLem - NValgor - SpamCScratch - SpamOrHamGit + RMSEMAE + MSE + RMSEMAE + MAPE + TweepyDoc + SearchTweets + ConStream + VADERPaper + VADERPaper + VADERPaper + LexiconSocSent + LSTM + StPNSentA + LSTMforetime + LSTMforetime + TFvsThe + VADERPaper + btcCharts + langdectNLTK + langdectNLTK + LanNgram + StemvsLem + StemvsLem + NValgor + SpamCScratch + SpamOrHamGit * diff --git a/document.blg b/document.blg index e71e0d0..32c4f34 100644 --- a/document.blg +++ b/document.blg @@ -1,23 +1,26 @@ [0] Config.pm:302> INFO - This is Biber 2.9 [0] Config.pm:305> INFO - Logfile is 'document.blg' -[18] biber:313> INFO - === Sat Apr 27, 2019, 18:31:54 -[33] Biber.pm:371> INFO - Reading 'document.bcf' -[79] Biber.pm:854> INFO - Using all citekeys in bib section 0 -[89] Biber.pm:3981> INFO - Processing section 0 -[96] Biber.pm:4154> INFO - Looking for bibtex format file 'report.bib' for section 0 -[97] bibtex.pm:1468> INFO - LaTeX decoding ... -[133] bibtex.pm:1294> INFO - Found BibTeX data source 'report.bib' -[145] Utils.pm:169> WARN - year field 'Aug 2, 2017' in entry 'SpamOrHamGit' is not an integer - this will probably not sort properly. -[193] Utils.pm:169> WARN - Name "Mairal, J., Ponce, J., Sapiro, G., Zisserman, A." has too many commas: skipping name -[196] Utils.pm:169> WARN - year field 'Aug 2, 2017' in entry 'SpamCScratch' is not an integer - this will probably not sort properly. -[217] Utils.pm:169> WARN - year field 'Mar 13, 2016' in entry 'LSTMdia' is not an integer - this will probably not sort properly. -[257] Utils.pm:169> WARN - year field 'Feb 26, 2018' in entry 'StemvsLem' is not an integer - this will probably not sort properly. -[260] Utils.pm:169> WARN - BibTeX subsystem: warning: comma(s) at end of name (removing) -[260] Utils.pm:169> WARN - BibTeX subsystem: author, warning: comma(s) at end of name (removing) -[281] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'variable = shifted' with 'variable = non-ignorable' -[281] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'normalization = NFD' with 'normalization = prenormalized' -[281] Biber.pm:3809> INFO - Sorting list 'none/global//global/global' of type 'entry' with template 'none' and locale 'en-US' -[281] Biber.pm:3815> INFO - No sort tailoring available for locale 'en-US' -[303] bbl.pm:617> INFO - Writing 'document.bbl' with encoding 'ascii' -[316] bbl.pm:720> INFO - Output to document.bbl -[317] Biber.pm:110> INFO - WARNINGS: 7 +[20] biber:313> INFO - === Sat Apr 27, 2019, 20:51:27 +[36] Biber.pm:371> INFO - Reading 'document.bcf' +[89] Biber.pm:854> INFO - Using all citekeys in bib section 0 +[99] Biber.pm:3981> INFO - Processing section 0 +[108] Biber.pm:4154> INFO - Looking for bibtex format file 'report.bib' for section 0 +[109] bibtex.pm:1468> INFO - LaTeX decoding ... +[150] bibtex.pm:1294> INFO - Found BibTeX data source 'report.bib' +[156] Utils.pm:169> WARN - year field 'Aug 2, 2017' in entry 'SpamOrHamGit' is not an integer - this will probably not sort properly. +[161] Utils.pm:169> WARN - year field 'Aug 2, 2017' in entry 'SpamCScratch' is not an integer - this will probably not sort properly. +[164] Utils.pm:169> WARN - year field 'Oct 16, 2018' in entry 'MSE' is not an integer - this will probably not sort properly. +[196] Utils.pm:169> WARN - year field 'Feb 26, 2018' in entry 'StemvsLem' is not an integer - this will probably not sort properly. +[208] Utils.pm:169> WARN - year field 'Sep 8, 2017' in entry 'MAPE' is not an integer - this will probably not sort properly. +[233] Utils.pm:169> WARN - year field 'Apr 7, 2018' in entry 'RMSEMAE' is not an integer - this will probably not sort properly. +[265] Utils.pm:169> WARN - year field 'Mar 13, 2016' in entry 'LSTMdia' is not an integer - this will probably not sort properly. +[291] Utils.pm:169> WARN - Name "Mairal, J., Ponce, J., Sapiro, G., Zisserman, A." has too many commas: skipping name +[299] Utils.pm:169> WARN - BibTeX subsystem: warning: comma(s) at end of name (removing) +[300] Utils.pm:169> WARN - BibTeX subsystem: author, warning: comma(s) at end of name (removing) +[323] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'variable = shifted' with 'variable = non-ignorable' +[323] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'normalization = NFD' with 'normalization = prenormalized' +[323] Biber.pm:3809> INFO - Sorting list 'none/global//global/global' of type 'entry' with template 'none' and locale 'en-US' +[323] Biber.pm:3815> INFO - No sort tailoring available for locale 'en-US' +[348] bbl.pm:617> INFO - Writing 'document.bbl' with encoding 'ascii' +[364] bbl.pm:720> INFO - Output to document.bbl +[364] Biber.pm:110> INFO - WARNINGS: 10 diff --git a/document.log b/document.log index fc3a208..46f88c1 100644 --- a/document.log +++ b/document.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.16) 27 APR 2019 19:19 +This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.16) 28 APR 2019 01:42 entering extended mode restricted \write18 enabled. %&-line parsing enabled. @@ -979,7 +979,7 @@ LaTeX Info: Redefining \nameref on input line 108. \@outlinefile=\write5 \openout5 = `document.out'. - + File: images/reading_logo.png Graphic file (type png) Package pdftex.def Info: images/reading_logo.png used on input line 111. @@ -997,262 +997,267 @@ File: umsb.fd 2013/01/14 v3.01 AMS symbols B {/var/lib/texmf/fonts/map/pdftex/updmap/pdftex.map} <./images/reading_logo.png> ] [2] [3] [4] -(./document.toc [5] [6]) +(./document.toc [5] [6] [7]) \tf@toc=\write6 \openout6 = `document.toc'. - [7] [8] [9] -LaTeX Font Info: Try loading font information for OMS+cmr on input line 207. + [8] [9] [10] +LaTeX Font Info: Try loading font information for OMS+cmr on input line 215. (/usr/share/texlive/texmf-dist/tex/latex/base/omscmr.fd File: omscmr.fd 2014/09/29 v2.5h Standard LaTeX font definitions ) LaTeX Font Info: Font shape `OMS/cmr/m/n' in size <12> not available -(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 207. - [10] [11] [12] -Underfull \hbox (badness 10000) in paragraph at lines 223--225 +(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 215. + [11] [12] [13] +Underfull \hbox (badness 10000) in paragraph at lines 231--233 [] -Underfull \hbox (badness 10000) in paragraph at lines 232--234 +Underfull \hbox (badness 10000) in paragraph at lines 240--242 [] -Underfull \hbox (badness 10000) in paragraph at lines 242--244 +Underfull \hbox (badness 10000) in paragraph at lines 250--252 [] -[13] -Underfull \hbox (badness 10000) in paragraph at lines 252--254 +[14] +Underfull \hbox (badness 10000) in paragraph at lines 260--262 [] -Underfull \hbox (badness 10000) in paragraph at lines 261--263 +Underfull \hbox (badness 10000) in paragraph at lines 269--271 [] -[14] [15] [16] [17] [18] [19] [20] [21] [22] - +[15] [16] [17] [18] [19] [20] [21] [22] + File: images/perceptron.png Graphic file (type png) -Package pdftex.def Info: images/perceptron.png used on input line 378. +Package pdftex.def Info: images/perceptron.png used on input line 377. (pdftex.def) Requested size: 284.52713pt x 170.72142pt. [23 <./images/perceptron.png>] - + File: images/rnn_ffn.png Graphic file (type png) -Package pdftex.def Info: images/rnn_ffn.png used on input line 398. +Package pdftex.def Info: images/rnn_ffn.png used on input line 397. (pdftex.def) Requested size: 426.80307pt x 170.72112pt. - -[24 <./images/rnn_ffn.png>] - + [24 <./images/rnn_ffn.png>] + File: images/lstm.png Graphic file (type png) -Package pdftex.def Info: images/lstm.png used on input line 416. +Package pdftex.def Info: images/lstm.png used on input line 415. (pdftex.def) Requested size: 256.07123pt x 199.1616pt. [25 <./images/lstm.png>] [26] [27] [28] [29] -Underfull \hbox (badness 10000) in paragraph at lines 527--530 +Underfull \hbox (badness 10000) in paragraph at lines 526--529 [] [30] -Underfull \hbox (badness 10000) in paragraph at lines 544--546 +Underfull \hbox (badness 10000) in paragraph at lines 543--545 [] -Overfull \hbox (9.8438pt too wide) detected at line 549 +Overfull \hbox (9.8438pt too wide) detected at line 548 \OML/cmm/m/it/12 P\OT1/cmr/m/n/12 (\OML/cmm/m/it/12 w\OMS/cmsy/m/n/12 j\OML/cmm /m/it/12 spam\OT1/cmr/m/n/12 ) = [] [] -[31] -Underfull \hbox (badness 10000) in paragraph at lines 556--558 +[31] [32] +Underfull \hbox (badness 10000) in paragraph at lines 565--567 [] -Underfull \hbox (badness 10000) in paragraph at lines 560--562 +Underfull \hbox (badness 10000) in paragraph at lines 569--571 [] -Underfull \hbox (badness 10000) in paragraph at lines 563--567 +Underfull \hbox (badness 10000) in paragraph at lines 572--576 [] -Underfull \hbox (badness 10000) in paragraph at lines 568--570 +Underfull \hbox (badness 10000) in paragraph at lines 577--579 [] -Underfull \hbox (badness 10000) in paragraph at lines 571--575 +Underfull \hbox (badness 10000) in paragraph at lines 580--584 [] -[32] [33] [34] [35] [36] [37] [38] - +[33] [34] [35] [36] [37] [38] [39] + File: images/Generic_Flow.png Graphic file (type png) -Package pdftex.def Info: images/Generic_Flow.png used on input line 679. +Package pdftex.def Info: images/Generic_Flow.png used on input line 677. (pdftex.def) Requested size: 512.13474pt x 227.62009pt. -Overfull \hbox (71.28728pt too wide) in paragraph at lines 679--680 +Overfull \hbox (71.28728pt too wide) in paragraph at lines 677--678 [][] [] -[39 <./images/Generic_Flow.png (PNG copy)>] -Underfull \hbox (badness 10000) in paragraph at lines 690--692 +[40 <./images/Generic_Flow.png (PNG copy)>] +Underfull \hbox (badness 10000) in paragraph at lines 688--690 [] - + File: images/Dataflow.png Graphic file (type png) -Package pdftex.def Info: images/Dataflow.png used on input line 695. +Package pdftex.def Info: images/Dataflow.png used on input line 693. (pdftex.def) Requested size: 512.09683pt x 227.62125pt. -Overfull \hbox (71.24937pt too wide) in paragraph at lines 695--697 +Overfull \hbox (71.24937pt too wide) in paragraph at lines 693--695 [] [] -[40 <./images/Dataflow.png (PNG copy)>] - +[41 <./images/Dataflow.png (PNG copy)>] + File: images/Data_Collector.png Graphic file (type png) -Package pdftex.def Info: images/Data_Collector.png used on input line 703. +Package pdftex.def Info: images/Data_Collector.png used on input line 701. (pdftex.def) Requested size: 426.78574pt x 227.61746pt. - [41 <./images/Data_Collector.png (PNG copy)>] - + [42 <./images/Data_Collector.png (PNG copy)>] + File: images/Analysis_Engine.png Graphic file (type png) -Package pdftex.def Info: images/Analysis_Engine.png used on input line 718. +Package pdftex.def Info: images/Analysis_Engine.png used on input line 716. (pdftex.def) Requested size: 483.67276pt x 227.62561pt. -Overfull \hbox (42.8253pt too wide) in paragraph at lines 718--720 +Overfull \hbox (42.8253pt too wide) in paragraph at lines 716--718 [] [] -[42 <./images/Analysis_Engine.png (PNG copy)>] [43] - +[43 <./images/Analysis_Engine.png (PNG copy)>] [44] + File: images/Neural_Network.png Graphic file (type png) -Package pdftex.def Info: images/Neural_Network.png used on input line 736. +Package pdftex.def Info: images/Neural_Network.png used on input line 734. (pdftex.def) Requested size: 483.6893pt x 341.42757pt. -Overfull \hbox (42.84184pt too wide) in paragraph at lines 736--738 +Overfull \hbox (42.84184pt too wide) in paragraph at lines 734--736 [] [] -[44 <./images/Neural_Network.png (PNG copy)>] - +[45 <./images/Neural_Network.png (PNG copy)>] + File: images/Future_Predictions.png Graphic file (type png) -Package pdftex.def Info: images/Future_Predictions.png used on input line 750. +Package pdftex.def Info: images/Future_Predictions.png used on input line 748. (pdftex.def) Requested size: 512.1362pt x 227.62119pt. -Overfull \hbox (71.28874pt too wide) in paragraph at lines 750--752 +Overfull \hbox (71.28874pt too wide) in paragraph at lines 748--750 [] [] -[45 <./images/Future_Predictions.png (PNG copy)>] - +[46 <./images/Future_Predictions.png (PNG copy)>] + File: images/Frontend_Application.png Graphic file (type png) Package pdftex.def Info: images/Frontend_Application.png used on input line 76 -3. +1. (pdftex.def) Requested size: 284.52162pt x 256.07664pt. - [46 <./images/Frontend_Application.png (PNG copy)>] - + [47 <./images/Frontend_Application.png (PNG copy)>] + File: images/interface_design.png Graphic file (type png) -Package pdftex.def Info: images/interface_design.png used on input line 782. +Package pdftex.def Info: images/interface_design.png used on input line 777. (pdftex.def) Requested size: 227.61479pt x 369.88063pt. - [47 <./images/interface_design.png>] [48] -Underfull \hbox (badness 10000) in paragraph at lines 793--795 + [48 <./images/interface_design.png>] [49] +Underfull \hbox (badness 10000) in paragraph at lines 788--790 [] (/usr/share/texlive/texmf-dist/tex/latex/listings/lstlang1.sty File: lstlang1.sty 2015/06/04 1.6 listings language file -) [49] [50] -[51] [52] -Underfull \hbox (badness 10000) in paragraph at lines 943--945 +) [50] [51] +[52] [53] +Underfull \hbox (badness 10000) in paragraph at lines 938--940 [] -[53] [54] [55] [56] [57] [58] [59] [60] [61] +[54] [55] [56] [57] [58] [59] [60] [61] [62] LaTeX Font Info: Font shape `OMS/cmr/m/n' in size <10> not available -(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 1221. +(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 1216. - [62] [63] [64] [65] [66] [67] -[68] [69] [70] [71] [72] -Underfull \hbox (badness 10000) in paragraph at lines 1552--1554 + [63] [64] [65] [66] [67] [68] +[69] [70] [71] [72] [73] +Underfull \hbox (badness 10000) in paragraph at lines 1548--1550 [] -[73] [74] - +[74] [75] + File: images/final_interface.png Graphic file (type png) -Package pdftex.def Info: images/final_interface.png used on input line 1619. +Package pdftex.def Info: images/final_interface.png used on input line 1615. (pdftex.def) Requested size: 398.32816pt x 540.60803pt. - [75 <./images/final_interface.png>] [76] [77] [78] -LaTeX Font Info: Try loading font information for T1+cmtt on input line 1678 + [76 <./images/final_interface.png>] [77] [78] [79] [80] [81] [82] +LaTeX Font Info: Try loading font information for T1+cmtt on input line 1790 . (/usr/share/texlive/texmf-dist/tex/latex/base/t1cmtt.fd File: t1cmtt.fd 2014/09/29 v2.5h Standard LaTeX font definitions ) -Overfull \hbox (40.38213pt too wide) in paragraph at lines 1679--1679 +Overfull \hbox (40.38213pt too wide) in paragraph at lines 1791--1791 \T1/cmr/m/n/12 works,'' To-wards Data Sci-ence, 2018. [On-line]. Avail-able: [] $\T1/cmtt/m/n/12 https : / / towardsdatascience . [] -[79] -Overfull \hbox (83.66737pt too wide) in paragraph at lines 1679--1679 +[83] +Overfull \hbox (83.66737pt too wide) in paragraph at lines 1791--1791 \T1/cmr/m/n/12 works,'' Ma-chine Larn-ing Mas-tery, 2017. [On-line]. Avail-able : []$\T1/cmtt/m/n/12 https : / / machinelearningmastery . [] -Overfull \hbox (28.45175pt too wide) in paragraph at lines 1679--1679 +Overfull \hbox (28.45175pt too wide) in paragraph at lines 1791--1791 \T1/cmr/m/n/12 lem,'' Su-per Data Sci-ence, 2018. [On-line]. Avail-able: []$\T1 /cmtt/m/n/12 https : / / www . superdatascience . [] -[80] -Overfull \hbox (7.75049pt too wide) in paragraph at lines 1679--1679 +[84] +Overfull \hbox (7.75049pt too wide) in paragraph at lines 1791--1791 \T1/cmr/m/n/12 2019. [On-line]. Avail-able: []$\T1/cmtt/m/n/12 https : / / medi um . com / datadriveninvestor / overview -[] [] -[81] -Overfull \hbox (9.24751pt too wide) in paragraph at lines 1679--1679 +[85] +Overfull \hbox (7.25049pt too wide) in paragraph at lines 1791--1791 +\T1/cmr/m/n/12 2017. [On-line]. Avail-able: []$\T1/cmtt/m/n/12 https : / / www +. statisticshowto . datasciencecentral . + [] + + +Overfull \hbox (9.24751pt too wide) in paragraph at lines 1791--1791 \T1/cmr/m/n/12 [On-line]. Avail-able: []$\T1/cmtt/m/n/12 http : / / blog . alej andronolla . com / 2013 / 05 / 15 / detecting -[] [] -Overfull \hbox (0.88026pt too wide) in paragraph at lines 1679--1679 +Overfull \hbox (0.88026pt too wide) in paragraph at lines 1791--1791 []\T1/cmr/m/n/12 P. Cryp-tog-ra-phy, ``A tu-to-rial on au-to-matic lan-guage id en-ti-fi-ca-tion - ngram based,'' [] -[82] +[86] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf used on input line 1684. +Package pdftex.def Info: PID.pdf used on input line 1796. (pdftex.def) Requested size: 597.551pt x 845.07512pt. @@ -1260,7 +1265,7 @@ pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf used on input line 1684. +Package pdftex.def Info: PID.pdf used on input line 1796. (pdftex.def) Requested size: 597.551pt x 845.07512pt. @@ -1270,234 +1275,234 @@ rsion <1.7>, but at most version <1.5> allowed pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1684. +Package pdftex.def Info: PID.pdf , page1 used on input line 1796. (pdftex.def) Requested size: 597.551pt x 845.07512pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1684. +Package pdftex.def Info: PID.pdf , page1 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[83] +[87] File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1684. +Package pdftex.def Info: PID.pdf , page1 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1684. +Package pdftex.def Info: PID.pdf , page1 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1684. +Package pdftex.def Info: PID.pdf , page1 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. - [84 <./PID.pdf>] + [88 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1684. +Package pdftex.def Info: PID.pdf , page2 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1684. +Package pdftex.def Info: PID.pdf , page2 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[85 <./PID.pdf>] - -pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve -rsion <1.7>, but at most version <1.5> allowed - -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[86 <./PID.pdf>] - -pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve -rsion <1.7>, but at most version <1.5> allowed - -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[87 <./PID.pdf>] - -pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve -rsion <1.7>, but at most version <1.5> allowed - -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[88 <./PID.pdf>] - -pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve -rsion <1.7>, but at most version <1.5> allowed - -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1684. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1684. +Package pdftex.def Info: PID.pdf , page2 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [89 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1796. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [90 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. [91 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. [92 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. [93 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1684. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1796. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. [94 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1796. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1796. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1684. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1796. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. [95 <./PID.pdf>] -Package atveryend Info: Empty hook `BeforeClearDocument' on input line 1689. - [96] -Package atveryend Info: Empty hook `AfterLastShipout' on input line 1689. + +pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve +rsion <1.7>, but at most version <1.5> allowed + +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page9 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page9 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page9 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +[96 <./PID.pdf>] + +pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve +rsion <1.7>, but at most version <1.5> allowed + +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page10 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page10 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page10 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +[97 <./PID.pdf>] + +pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve +rsion <1.7>, but at most version <1.5> allowed + +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page11 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page11 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page11 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +[98 <./PID.pdf>] + +pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve +rsion <1.7>, but at most version <1.5> allowed + +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1796. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +[99 <./PID.pdf>] +Package atveryend Info: Empty hook `BeforeClearDocument' on input line 1801. + [100] +Package atveryend Info: Empty hook `AfterLastShipout' on input line 1801. (./document.aux) -Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 1689. -Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 1689. +Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 1801. +Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 1801. Package rerunfilecheck Info: File `document.out' has not changed. -(rerunfilecheck) Checksum: A2611AC8121E52D2F7538E5EDEC264FB;13147. +(rerunfilecheck) Checksum: 44E9A77D9054042DAB657D617BF8D6A7;13734. Package logreq Info: Writing requests to 'document.run.xml'. \openout1 = `document.run.xml'. ) Here is how much of TeX's memory you used: - 25087 strings out of 492982 - 394874 string characters out of 6134895 + 25106 strings out of 492982 + 395294 string characters out of 6134895 1018656 words of memory out of 5000000 - 27432 multiletter control sequences out of 15000+600000 + 27440 multiletter control sequences out of 15000+600000 20231 words of font info for 58 fonts, out of 8000000 for 9000 1141 hyphenation exceptions out of 8191 45i,18n,78p,2008b,1838s stack positions out of 5000i,500n,10000p,200000b,80000s @@ -1518,10 +1523,10 @@ ic/cm-super/sfrm1200.pfb> -Output written on document.pdf (96 pages, 2156782 bytes). +Output written on document.pdf (100 pages, 2170811 bytes). PDF statistics: - 2027 PDF objects out of 2073 (max. 8388607) - 1851 compressed objects within 19 object streams - 857 named destinations out of 1000 (max. 500000) - 797 words of extra memory for PDF output out of 10000 (max. 10000000) + 2079 PDF objects out of 2487 (max. 8388607) + 1899 compressed objects within 19 object streams + 868 named destinations out of 1000 (max. 500000) + 821 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/document.out b/document.out index 876f30a..9e1c19b 100644 --- a/document.out +++ b/document.out @@ -7,30 +7,30 @@ \BOOKMARK [2][-]{subsection.5.2}{\376\377\000S\000t\000a\000k\000e\000h\000o\000l\000d\000e\000r\000s}{section.5}% 7 \BOOKMARK [2][-]{subsection.5.3}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000M\000o\000t\000i\000v\000a\000t\000i\000o\000n}{section.5}% 8 \BOOKMARK [2][-]{subsection.5.4}{\376\377\000T\000e\000c\000h\000n\000i\000c\000a\000l\000\040\000S\000p\000e\000c\000i\000f\000i\000c\000a\000t\000i\000o\000n}{section.5}% 9 -\BOOKMARK [2][-]{subsection.5.5}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000C\000o\000n\000s\000t\000r\000a\000i\000n\000t\000s}{section.5}% 10 -\BOOKMARK [1][-]{section.6}{\376\377\000Q\000u\000a\000l\000i\000t\000y\000\040\000G\000o\000a\000l\000s}{}% 11 -\BOOKMARK [2][-]{subsection.6.1}{\376\377\000P\000r\000o\000c\000e\000s\000s\000\040\000D\000e\000s\000c\000r\000i\000p\000t\000i\000o\000n}{section.6}% 12 -\BOOKMARK [2][-]{subsection.6.2}{\376\377\000Q\000u\000a\000l\000i\000t\000y\000\040\000O\000b\000j\000e\000c\000t\000i\000v\000e\000s}{section.6}% 13 -\BOOKMARK [2][-]{subsection.6.3}{\376\377\000T\000o\000o\000l\000s\000\040\000t\000o\000\040\000E\000n\000s\000u\000r\000e\000\040\000Q\000u\000a\000l\000i\000t\000y}{section.6}% 14 -\BOOKMARK [1][-]{section.7}{\376\377\000L\000i\000t\000e\000r\000a\000t\000u\000r\000e\000\040\000R\000e\000v\000i\000e\000w}{}% 15 -\BOOKMARK [2][-]{subsection.7.1}{\376\377\000E\000x\000i\000s\000t\000i\000n\000g\000\040\000T\000o\000o\000l\000s}{section.7}% 16 -\BOOKMARK [2][-]{subsection.7.2}{\376\377\000R\000e\000l\000a\000t\000e\000d\000\040\000r\000e\000s\000e\000a\000r\000c\000h}{section.7}% 17 -\BOOKMARK [2][-]{subsection.7.3}{\376\377\000D\000a\000t\000a\000\040\000C\000o\000l\000l\000e\000c\000t\000i\000o\000n}{section.7}% 18 -\BOOKMARK [3][-]{subsubsection.7.3.1}{\376\377\000T\000w\000i\000t\000t\000e\000r\000\040\000a\000n\000d\000\040\000T\000w\000i\000t\000t\000e\000r\000\040\000A\000P\000I}{subsection.7.3}% 19 -\BOOKMARK [3][-]{subsubsection.7.3.2}{\376\377\000T\000w\000e\000e\000p\000y\000\040\000P\000y\000t\000h\000o\000n\000\040\000P\000a\000c\000k\000a\000g\000e}{subsection.7.3}% 20 -\BOOKMARK [2][-]{subsection.7.4}{\376\377\000S\000e\000n\000t\000i\000m\000e\000n\000t\000\040\000A\000n\000a\000l\000y\000s\000i\000s}{section.7}% 21 -\BOOKMARK [3][-]{subsubsection.7.4.1}{\376\377\000N\000a\000t\000u\000r\000a\000l\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000P\000r\000o\000c\000e\000s\000s\000i\000n\000g}{subsection.7.4}% 22 -\BOOKMARK [3][-]{subsubsection.7.4.2}{\376\377\000V\000a\000l\000e\000n\000c\000e\000\040\000A\000w\000a\000r\000e\000\040\000D\000i\000c\000t\000i\000o\000n\000a\000r\000y\000\040\000a\000n\000d\000\040\000s\000E\000n\000t\000i\000m\000e\000n\000t\000\040\000R\000e\000a\000s\000o\000n\000i\000n\000g}{subsection.7.4}% 23 -\BOOKMARK [2][-]{subsection.7.5}{\376\377\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000s}{section.7}% 24 -\BOOKMARK [3][-]{subsubsection.7.5.1}{\376\377\000R\000e\000c\000u\000r\000r\000e\000n\000t\000\040\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000\040\000\050\000R\000N\000N\000\051}{subsection.7.5}% 25 -\BOOKMARK [3][-]{subsubsection.7.5.2}{\376\377\000L\000o\000n\000g\000-\000S\000h\000o\000r\000t\000\040\000T\000e\000r\000m\000\040\000M\000e\000m\000o\000r\000y\000\040\000\050\000L\000S\000T\000M\000\051}{subsection.7.5}% 26 -\BOOKMARK [3][-]{subsubsection.7.5.3}{\376\377\000K\000e\000r\000a\000s\000\040\000a\000n\000d\000\040\000T\000e\000n\000s\000o\000r\000F\000l\000o\000w}{subsection.7.5}% 27 -\BOOKMARK [3][-]{subsubsection.7.5.4}{\376\377\000O\000p\000t\000i\000m\000i\000s\000e\000r\000s}{subsection.7.5}% 28 -\BOOKMARK [2][-]{subsection.7.6}{\376\377\000M\000a\000c\000h\000i\000n\000e\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.7}% 29 -\BOOKMARK [3][-]{subsubsection.7.6.1}{\376\377\000N\000a\000i\000v\000e\000\040\000B\000a\000y\000e\000s}{subsection.7.6}% 30 -\BOOKMARK [2][-]{subsection.7.7}{\376\377\000B\000a\000g\000\040\000O\000f\000\040\000W\000o\000r\000d\000s}{section.7}% 31 -\BOOKMARK [2][-]{subsection.7.8}{\376\377\000T\000F\000-\000I\000D\000F}{section.7}% 32 -\BOOKMARK [2][-]{subsection.7.9}{\376\377\000A\000d\000d\000i\000c\000t\000i\000v\000e\000\040\000S\000m\000o\000o\000t\000h\000i\000n\000g}{section.7}% 33 +\BOOKMARK [1][-]{section.6}{\376\377\000Q\000u\000a\000l\000i\000t\000y\000\040\000G\000o\000a\000l\000s}{}% 10 +\BOOKMARK [2][-]{subsection.6.1}{\376\377\000P\000r\000o\000c\000e\000s\000s\000\040\000D\000e\000s\000c\000r\000i\000p\000t\000i\000o\000n}{section.6}% 11 +\BOOKMARK [2][-]{subsection.6.2}{\376\377\000Q\000u\000a\000l\000i\000t\000y\000\040\000O\000b\000j\000e\000c\000t\000i\000v\000e\000s}{section.6}% 12 +\BOOKMARK [2][-]{subsection.6.3}{\376\377\000T\000o\000o\000l\000s\000\040\000t\000o\000\040\000E\000n\000s\000u\000r\000e\000\040\000Q\000u\000a\000l\000i\000t\000y}{section.6}% 13 +\BOOKMARK [1][-]{section.7}{\376\377\000L\000i\000t\000e\000r\000a\000t\000u\000r\000e\000\040\000R\000e\000v\000i\000e\000w}{}% 14 +\BOOKMARK [2][-]{subsection.7.1}{\376\377\000E\000x\000i\000s\000t\000i\000n\000g\000\040\000T\000o\000o\000l\000s}{section.7}% 15 +\BOOKMARK [2][-]{subsection.7.2}{\376\377\000R\000e\000l\000a\000t\000e\000d\000\040\000r\000e\000s\000e\000a\000r\000c\000h}{section.7}% 16 +\BOOKMARK [2][-]{subsection.7.3}{\376\377\000D\000a\000t\000a\000\040\000C\000o\000l\000l\000e\000c\000t\000i\000o\000n}{section.7}% 17 +\BOOKMARK [3][-]{subsubsection.7.3.1}{\376\377\000T\000w\000i\000t\000t\000e\000r\000\040\000a\000n\000d\000\040\000T\000w\000i\000t\000t\000e\000r\000\040\000A\000P\000I}{subsection.7.3}% 18 +\BOOKMARK [3][-]{subsubsection.7.3.2}{\376\377\000T\000w\000e\000e\000p\000y\000\040\000P\000y\000t\000h\000o\000n\000\040\000P\000a\000c\000k\000a\000g\000e}{subsection.7.3}% 19 +\BOOKMARK [2][-]{subsection.7.4}{\376\377\000S\000e\000n\000t\000i\000m\000e\000n\000t\000\040\000A\000n\000a\000l\000y\000s\000i\000s}{section.7}% 20 +\BOOKMARK [3][-]{subsubsection.7.4.1}{\376\377\000N\000a\000t\000u\000r\000a\000l\000\040\000L\000a\000n\000g\000u\000a\000g\000e\000\040\000P\000r\000o\000c\000e\000s\000s\000i\000n\000g}{subsection.7.4}% 21 +\BOOKMARK [3][-]{subsubsection.7.4.2}{\376\377\000V\000a\000l\000e\000n\000c\000e\000\040\000A\000w\000a\000r\000e\000\040\000D\000i\000c\000t\000i\000o\000n\000a\000r\000y\000\040\000a\000n\000d\000\040\000s\000E\000n\000t\000i\000m\000e\000n\000t\000\040\000R\000e\000a\000s\000o\000n\000i\000n\000g}{subsection.7.4}% 22 +\BOOKMARK [2][-]{subsection.7.5}{\376\377\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000s}{section.7}% 23 +\BOOKMARK [3][-]{subsubsection.7.5.1}{\376\377\000R\000e\000c\000u\000r\000r\000e\000n\000t\000\040\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000\040\000\050\000R\000N\000N\000\051}{subsection.7.5}% 24 +\BOOKMARK [3][-]{subsubsection.7.5.2}{\376\377\000L\000o\000n\000g\000-\000S\000h\000o\000r\000t\000\040\000T\000e\000r\000m\000\040\000M\000e\000m\000o\000r\000y\000\040\000\050\000L\000S\000T\000M\000\051}{subsection.7.5}% 25 +\BOOKMARK [3][-]{subsubsection.7.5.3}{\376\377\000K\000e\000r\000a\000s\000\040\000a\000n\000d\000\040\000T\000e\000n\000s\000o\000r\000F\000l\000o\000w}{subsection.7.5}% 26 +\BOOKMARK [3][-]{subsubsection.7.5.4}{\376\377\000O\000p\000t\000i\000m\000i\000s\000e\000r\000s}{subsection.7.5}% 27 +\BOOKMARK [2][-]{subsection.7.6}{\376\377\000M\000a\000c\000h\000i\000n\000e\000\040\000L\000e\000a\000r\000n\000i\000n\000g}{section.7}% 28 +\BOOKMARK [3][-]{subsubsection.7.6.1}{\376\377\000N\000a\000i\000v\000e\000\040\000B\000a\000y\000e\000s}{subsection.7.6}% 29 +\BOOKMARK [2][-]{subsection.7.7}{\376\377\000B\000a\000g\000\040\000O\000f\000\040\000W\000o\000r\000d\000s}{section.7}% 30 +\BOOKMARK [2][-]{subsection.7.8}{\376\377\000T\000F\000-\000I\000D\000F}{section.7}% 31 +\BOOKMARK [2][-]{subsection.7.9}{\376\377\000A\000d\000d\000i\000c\000t\000i\000v\000e\000\040\000S\000m\000o\000o\000t\000h\000i\000n\000g}{section.7}% 32 +\BOOKMARK [2][-]{subsection.7.10}{\376\377\000R\000e\000g\000r\000e\000s\000s\000i\000o\000n\000\040\000P\000e\000r\000f\000o\000r\000m\000a\000n\000c\000e\000\040\000M\000e\000t\000r\000i\000c\000s}{section.7}% 33 \BOOKMARK [1][-]{section.8}{\376\377\000S\000o\000l\000u\000t\000i\000o\000n\000\040\000A\000p\000p\000r\000o\000a\000c\000h}{}% 34 \BOOKMARK [2][-]{subsection.8.1}{\376\377\000D\000a\000t\000a\000\040\000g\000a\000t\000h\000e\000r\000i\000n\000g}{section.8}% 35 \BOOKMARK [2][-]{subsection.8.2}{\376\377\000D\000a\000t\000a\000\040\000p\000r\000e\000-\000p\000r\000o\000c\000e\000s\000s\000i\000n\000g}{section.8}% 36 @@ -70,13 +70,16 @@ \BOOKMARK [3][-]{subsubsection.10.7.1}{\376\377\000K\000e\000y\000\040\000F\000u\000n\000c\000t\000i\000o\000n\000s}{subsection.10.7}% 70 \BOOKMARK [3][-]{subsubsection.10.7.2}{\376\377\000F\000i\000n\000a\000l\000\040\000I\000n\000t\000e\000r\000f\000a\000c\000e}{subsection.10.7}% 71 \BOOKMARK [1][-]{section.11}{\376\377\000T\000e\000s\000t\000i\000n\000g\000\040\000M\000e\000t\000r\000i\000c\000s\000\040\000a\000n\000d\000\040\000A\000c\000c\000u\000r\000a\000c\000y}{}% 72 -\BOOKMARK [1][-]{section.12}{\376\377\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n\000\040\000o\000f\000\040\000R\000e\000s\000u\000l\000t\000s}{}% 73 -\BOOKMARK [1][-]{section.13}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000E\000v\000a\000l\000u\000a\000t\000i\000o\000n}{}% 74 -\BOOKMARK [1][-]{section.14}{\376\377\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n\000:\000\040\000C\000o\000n\000t\000r\000i\000b\000u\000t\000i\000o\000n\000\040\000a\000n\000d\000\040\000R\000e\000f\000l\000e\000c\000t\000i\000o\000n}{}% 75 -\BOOKMARK [2][-]{subsection.14.1}{\376\377\000L\000i\000m\000i\000t\000a\000t\000i\000o\000n\000s}{section.14}% 76 -\BOOKMARK [1][-]{section.15}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n\000\040\000a\000n\000d\000\040\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{}% 77 -\BOOKMARK [2][-]{subsection.15.1}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n}{section.15}% 78 -\BOOKMARK [2][-]{subsection.15.2}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{section.15}% 79 -\BOOKMARK [1][-]{section.16}{\376\377\000A\000p\000p\000e\000n\000d\000i\000c\000e\000s}{}% 80 -\BOOKMARK [2][-]{subsection.16.1}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000A\000\040\000-\000\040\000P\000r\000o\000j\000e\000c\000t\000\040\000I\000n\000i\000t\000i\000a\000t\000i\000o\000n\000\040\000D\000o\000c\000u\000m\000e\000n\000t}{section.16}% 81 -\BOOKMARK [2][-]{subsection.16.2}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000B\000\040\000-\000\040\000L\000o\000g\000\040\000b\000o\000o\000k}{section.16}% 82 +\BOOKMARK [2][-]{subsection.11.1}{\376\377\000I\000n\000t\000e\000g\000r\000a\000t\000i\000o\000n\000\040\000T\000e\000s\000t\000i\000n\000g}{section.11}% 73 +\BOOKMARK [2][-]{subsection.11.2}{\376\377\000A\000c\000c\000u\000r\000a\000c\000y\000\040\000o\000f\000\040\000M\000o\000d\000e\000l\000\040\000\046\000\040\000R\000e\000s\000u\000l\000t\000s}{section.11}% 74 +\BOOKMARK [3][-]{subsubsection.11.2.1}{\376\377\000R\000e\000s\000u\000l\000t\000s\000\040\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n}{subsection.11.2}% 75 +\BOOKMARK [3][-]{subsubsection.11.2.2}{\376\377\000E\000x\000e\000c\000u\000t\000i\000o\000n\000\040\000S\000p\000e\000e\000d\000s}{subsection.11.2}% 76 +\BOOKMARK [1][-]{section.12}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000E\000v\000a\000l\000u\000a\000t\000i\000o\000n}{}% 77 +\BOOKMARK [1][-]{section.13}{\376\377\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n\000:\000\040\000C\000o\000n\000t\000r\000i\000b\000u\000t\000i\000o\000n\000\040\000a\000n\000d\000\040\000R\000e\000f\000l\000e\000c\000t\000i\000o\000n}{}% 78 +\BOOKMARK [2][-]{subsection.13.1}{\376\377\000L\000i\000m\000i\000t\000a\000t\000i\000o\000n\000s}{section.13}% 79 +\BOOKMARK [1][-]{section.14}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n\000\040\000a\000n\000d\000\040\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{}% 80 +\BOOKMARK [2][-]{subsection.14.1}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n}{section.14}% 81 +\BOOKMARK [2][-]{subsection.14.2}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{section.14}% 82 +\BOOKMARK [1][-]{section.15}{\376\377\000A\000p\000p\000e\000n\000d\000i\000c\000e\000s}{}% 83 +\BOOKMARK [2][-]{subsection.15.1}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000A\000\040\000-\000\040\000P\000r\000o\000j\000e\000c\000t\000\040\000I\000n\000i\000t\000i\000a\000t\000i\000o\000n\000\040\000D\000o\000c\000u\000m\000e\000n\000t}{section.15}% 84 +\BOOKMARK [2][-]{subsection.15.2}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000B\000\040\000-\000\040\000L\000o\000g\000\040\000b\000o\000o\000k}{section.15}% 85 diff --git a/document.pdf b/document.pdf index d5ddfc2..38cc344 100644 Binary files a/document.pdf and b/document.pdf differ diff --git a/document.synctex.gz b/document.synctex.gz index c1dac8f..995051d 100644 Binary files a/document.synctex.gz and b/document.synctex.gz differ diff --git a/document.tex b/document.tex index 8ea6e47..12a0240 100644 --- a/document.tex +++ b/document.tex @@ -157,6 +157,14 @@ LSTM - Long-Short Term Memory Neural Network + RMSE - Root Mean Squared Error + + MSE - Mean Squared Error + + MAE - Mean Absolute Error + + MAPE - Mean Absolute Percentage Error + \newpage \begin{center} @@ -272,20 +280,11 @@ This project will only be coded to predict an hour ahead as the model will be trained on an hourly basis as the data is gathered per hour. Predictions for further in the future can be modelled but will be seen as a future improvement to the system. The detail of a interface may be subject of change through this project due to time contraints and the focus being the investigation of the impact social media has on market predictions. - - \subsection{Project Constraints}\label{constraints} - The following constraints are recognisted in this project - - \begin{itemize} - \item ... - \end{itemize} \newpage \section{Quality Goals} Although this project will be an investigation it is important to ensure that a specific level of quality is met and maintained throughout development to produce a final solution that fully meets the requirements set out in the technical specifiction. This section will outline the quality objectives, processes used to aid in ensuring quality, quality goals and the deveopment pipeline. - \textit{Goals} - \subsection{Process Description} To maintain a level of quality through the project's development lifecycle various processes are to be defined. As stated in ISO 9001:2015 \cite{ISO9000} "Consistent and predictable results are achieved more effectively and efficiently when activities are understood and managed as interrelated processes". To achieve this, clearly defined processes of how this system should be developed along with tools to achieve this will need to be defined before the development of this project, to ensure an effective, transparent and well-tested system is created, of which should be outlined in the solution approach. @@ -294,7 +293,7 @@ The most important aspect of testing of the system is to ensure that predictions for Bitcoin price are calculated for the next hour interval. Testing should be conducted around this to both determine the accuracy of the predictions made by the system, and to validate the optimisers and other methods used for model creation. Suitable performance and accuracy metrics should be chosen and implemented to aid in justifying the accuracy of the developed solution for a regression model. - The development of this project should follow an agile methodology as closely as and when possible. Prior to functions and components being created after another has finished development, said completed function should be tested and fully operational with the existing functions and components it uses or provides for. Component testing should be conducted to determine what functions are operating as intended along with the development of the system. + The development of this project should follow an agile methodology as closely as and when possible. Prior to functions and components being created after another has finished development, said completed function should be tested and fully operational with the existing functions and components it uses or provides for. Integration testing should be conducted to determine what functions are operating as intended along with the development of the system. Lastly, the testing of the user interface shouldn't be considered as part of the testing of the system due to the focus of this project not formulating around the development of the interface, but rather around the back-end prediction system. As long as what is intended to be displayed to the user and are displayed correctly as per the user interface design such testing will be sufficient. @@ -547,6 +546,16 @@ For TF-IDF: \[P(w|spam) = \frac{TF(w)*IDF(w) + \alpha}{\sum _{a\ words\ x\ \epsilon\ train\ dataset} TF(x|spam)*IDF(x) + \alpha\sum_{a\ words\ x\ \epsilon\ train\ dataset}}\] + + \subsection{Regression Performance Metrics} + Due to the problem statement and concept behind this project is that of a regression method, forecasting and predicting a future value, performance and accuracy of such a model will ultimatly need to be identified. + + \begin{itemize} + \item RMSE - Root Mean Squared Error: Represents the sample standard deviation of difference between predicted values and observed values, known as residuals. It identifies how concentrated the data is around the line of best fit, lower the better. \cite{RMSEMAE} + \item MSE - Mean Squared Error: Is the average squared difference between the estimated values. \textit{"MSE is a risk function, corresponding to the expected value of the squared error loss."} \cite{MSE}. + \item MAE - Mean Absolute Error: Is the average of the absolute difference between the predicted values and observed values, and is a linear score. This means that the idividual differences are weighted equally in the average. \cite{RMSEMAE} + \item MAPE - Mean Absolute Percentage Error: Is a measure of how accuracte a forecast system is, it measures the accuracy as a percentage error for each time period minus the actual valuse divided by the actual values. \cite{MAPE} + \end{itemize} \newpage @@ -660,17 +669,6 @@ \subsection{Solution Summary}\label{summary} The overall solution, concerning the problem statement, is to create a system mainly consisting of; a frontend application that will display plotting, predicted and true, performance metric data to the user as a clear and concise form. The backend system behind the price forecasting will consist of various subsystem responsible for data collection, filtering, data pre-processing, sentiment analysis, network training, validation and training and future price predictions. Each stage will consist of relevant tools and techniques for performing their required task. - %The tools and techniques that will be used for this project are as follows, with relation to the relevent part of the system. - - %\begin{itemize} - % \item Data Collector - % \subitem Prices: Historical - None, will be extrapolated from a historical price CSV, Live - Request and Coinbase Client api package - % \subitem Tweets: Historical - Request or Curl from the Twitter API, Live - Tweepy - % \item Natural Language pre-processing - Regex, (Tokenisation, Stemming, Stopword removal), ngrams, language detection - % \item Spam filter - Natural language pre-processing techniques (above), Naive Bayes Classifier - % \item Neural Network - Tensorflow, Keras, Sc - - %\end{itemize} \newpage \subsection{Data flow Overview}\label{data-flow} @@ -678,7 +676,7 @@ \includegraphics[width=18cm,height=8cm]{images/Generic_Flow.png} \begin{center} - \textit{Figure 3: Basic Dataflow diagram of systems in the project and how data could possibly flow} + \textit{Figure 4: Basic Dataflow diagram of systems in the project and how data could possibly flow} \end{center} \newpage @@ -693,7 +691,7 @@ \textbf{Dataflow overview of entire system:} \begin{center} \includegraphics[width=18cm,height=8cm]{images/Dataflow.png} - \textit{Figure 4: Overall Dataflow diagram of the entire system} + \textit{Figure 5: Overall Dataflow diagram of the entire system} \end{center} This dataflow diagram shows the overall concept of how the data is intended to flow through the system, from being processed and manipulated through each component and what the outputs are of each. Due to the size, this will be broken up and individually explained. \newpage @@ -701,7 +699,7 @@ \textbf{Data collector} \begin{center} \includegraphics[width=15cm,height=8cm]{images/Data_Collector.png} - \textit{Figure 5: Data collector Dataflow diagram} + \textit{Figure 6: Data collector Dataflow diagram} \end{center} This dataflow diagram shows the part of the system responsible for the collection and processing of both historical data. This is split into three parts: Price collector, Tweet collector and tweet normalisation and natural language pre-processing. \begin{itemize} @@ -716,7 +714,7 @@ \textbf{Analysis Engine} \begin{center} \includegraphics[width=17cm,height=8cm]{images/Analysis_Engine.png} - \textit{Figure 6: Analysis Engine Dataflow diagram} + \textit{Figure 7: Analysis Engine Dataflow diagram} \end{center} This dataflow diagram shows the part of the system that is responsible for training a spam filter, creating the model that'll be used to identify if the tweets from the data collector are unwanted - spam. This system is also responsible for assigning the polarity classification to the tweet through sentiment analysis conducted by the VADER package \cite{VADERPaper}. \begin{itemize} @@ -734,9 +732,9 @@ \textbf{Neural Network} \begin{center} \includegraphics[width=17cm,height=12cm]{images/Neural_Network.png} - \textit{Figure 7: Neural Network layout Dataflow diagram} + \textit{Figure 8: Neural Network layout Dataflow diagram} \end{center} - The dataflow diagram in \textit{figure 7} shows the part of the system that is responsible for training and creating the neural network model. The dataflow diagram shows how the network will be trained, and the layers of a possible solution to the network. The model shows four layers which may not be the solution that will be implemented but is there to show a representation of a number of layers that could be applied. + The dataflow diagram in \textit{figure 8} shows the part of the system that is responsible for training and creating the neural network model. The dataflow diagram shows how the network will be trained, and the layers of a possible solution to the network. The model shows four layers which may not be the solution that will be implemented but is there to show a representation of a number of layers that could be applied. \begin{itemize} \item Merging of Datasets - Data from both historical datasets are merged to create one dataset with mapped price and sentiment for each hour. *This is a specific process that is different from the system that does not include sentiment for predictions, the merge process doesn't occur in that system/model. \item Training and Testing - Data is split into two samples of training and testing, 75:25 respectively. **This also doesn't occur in the system that doesn't model with the sentiment. @@ -748,9 +746,9 @@ \textbf{Future Price Forecasting} \begin{center} \includegraphics[width=18cm,height=8cm]{images/Future_Predictions.png} - \textit{Figure 8: Price Forecasting Dataflow diagram} + \textit{Figure 9: Price Forecasting Dataflow diagram} \end{center} - The dataflow diagram in \textit{figure 8} shows how the forecasting system would be implemented. This dataflow shows how it will read live data of both sentiment and price data, merge, split and conduct regression using the trained neural network model to predict the next hour price. + The dataflow diagram in \textit{figure 9} shows how the forecasting system would be implemented. This dataflow shows how it will read live data of both sentiment and price data, merge, split and conduct regression using the trained neural network model to predict the next hour price. \begin{itemize} \item Data merging - (Doesn't occur with the system that doesn't include sentiment in price predictions). Data is consolidated from both historical and live data up to 5 iterations. This is due to after the initial hour there will only be a singular record of price and sentiment data, in which no prediction could be made from this as there isn't a sufficient amount of data. \item Prediction - This data is then fitted to the neural network model and predictions for the next time-step hour are made. @@ -762,7 +760,7 @@ \begin{center} \includegraphics[width=10cm,height=9cm]{images/Frontend_Application.png} \newline - \textit{Figure 9: Front-end Application Dataflow diagram} + \textit{Figure 10: Front-end Application Dataflow diagram} \end{center} The above dataflow diagram shows the data flow for the front-end application and how the data is read into the system from the data files generated by the backend application (Neural network). \begin{itemize} @@ -771,26 +769,23 @@ \item Charting and Tables - Accesses the loaded data from the AJAX requests and plots the data. Prediction data, only with sentiment and prices are plotted into a table. There will be separate charts and tables displaying the data from the backend that hasn't used sentiment in predictions to aid in establishing a correlation between sentiment and price and whether it affects the hourly price (Aiming to solve the problem statement) \item Stakeholders - There will be the four stakeholders, outlined in the problem articulation section, that would be the primary users of this application. \end{itemize} - - %\subsection{UML Component Design} \subsection{Interface Design} - - + \begin{figure}[hbt!] \centering \includegraphics[width=8cm,height=13cm]{images/interface_design.png} \end{figure} \begin{center} - \textit{Figure 10: Interface design} + \textit{Figure 11: Interface design} \end{center} - \textit{Figure 10} above shows the basic idea of the interface design that will be presented to the stakeholders and aims to be the interface that these stakeholders will use to aid in their market decisions of Bitcoin. The interface, although simplistic, provides all the necessary information that any of these stakeholders would need, it also provides information to allow visual comparison on how sentiment affects the hourly price of Bitcoin, represented as the two charts. The comparison will aid in solving the problem statement later in the conclusion of the project. + \textit{Figure 11} above shows the basic idea of the interface design that will be presented to the stakeholders and aims to be the interface that these stakeholders will use to aid in their market decisions of Bitcoin. The interface, although simplistic, provides all the necessary information that any of these stakeholders would need, it also provides information to allow visual comparison on how sentiment affects the hourly price of Bitcoin, represented as the two charts. The comparison will aid in solving the problem statement later in the conclusion of the project. \newpage \begin{center} \section{Implementation}\label{implementation} \end{center} - This section will outline the method and process of development of the system to satisfy the chosen solution, technical specification and the problem statement. Each section of the system will be outlined and discussed with relevant codes snippets of essential methods from the system to highlight the processing of data throughout. Some sections will reference code snippets in \textit{Appendix B}, due to the size of some crucial methods. + This section will outline the method and process of development of the system to satisfy the chosen solution, technical specification and the problem statement. Each section of the system will be outlined and discussed with relevant codes snippets of essential methods from the system to highlight the processing of data throughout. Additionally, the order in which the following sections are show was not the order in which they were developed, the order in which they are shown is to represent the order of how the data flows through the system, see \textit{section 9 - Ssytem Design} for an understanding of the flow of data through the system. \newline \subsection{Data collection}\label{collection} @@ -969,7 +964,7 @@ def processTweet(tweet, tweetFilter): The tweet text can be nested in multiple attributes in the response; this depends on a few factors of what the tweet is and how it was posted on Twitter. If a user retweeted the tweet, the text of the tweet would be nested under \textit{'retweeted\_status'} in the JSON response, also there is a check to see if the tweets are above the original twitter tweet character limit (140 characters). This is a possible legacy parameter in the Twitter API but is checked upon data response. If an attribute \textit{'extended\_tweet'} exists the character limit for the tweet exceeds 140 but is under the 280 characters hard limit of Twitter, this exact filtering is the same if it in a non-retweeted tweet. - As for the key facts about this function; the length of the tweet is checked to be above 5 (tokenised) due to any tweets with fewer words will not contain enough information to be given a proper polarity classification and almost always returns as 100\% neutral, which is of no use and will have no effect on the hours average sentiment. The entire code in the function is encapsulated in a try-catch to check if data was received and handles non-responses and missing data. If there was no data the issue is ignored unless a connection between the streamer and API is broken it otherwise exits the script. + As for the key facts about this function; the length of the tweet is checked to be above 5 (tokenised) due to any tweets with fewer words will not contain enough information to be given a proper polarity classification and almost always returns as 100\% neutral, which is of no use and will have no effect on the hours average sentiment. The entire code in the function is encapsulated in a try-catch to check if data was received and handles non-responses and missing data. If there was no data the issue is ignored unless a connection between the streamer and API is broken it otherwise exits the script. As for the processing of the tweet the code in \textit{Listing 4} is used. \newpage \subsection{Data pre-processing}\label{processing} @@ -1431,6 +1426,7 @@ def create_sets(self, data, lookback, sentiment): # Model summary of params and dropout at each layer history = self.model.fit(train_X, train_Y, epochs=200, batch_size=1000, validation_data=(test_X, test_Y), verbose=0, shuffle=False, callbacks=[TQDMCallback()]) + ## fit model using train data and validate using the test data yhat = self.model.predict(test_X) @@ -1443,7 +1439,7 @@ def create_sets(self, data, lookback, sentiment): As per the discussion in the literature review and outlined in the solution approach, the Adam optimiser was used for the compilation of the model. The loss was calculated using the mean squared error and the metrics calculated and returned to present the accuracy of the model in prediction were: mean squared error, root mean squared error, mean absolute error and mean absolute percentage error. Both the metrics and predictions made are saved to a CSV that is then presented to users in the server-hosted UI. - The model was fitted on the training sets (X, Y) over 200 epochs with a batch size of 1000 on about ~11000 records - was about the total amount of data used to train for a year. Predictions are then made using the test set resulting in the predictions of \textit{'yhat'} which is inverted and rescaled to get original price values to save to a CSV and which are displayed on the user interface. + The model was fitted on the training sets (X, Y) over 200 epochs with a batch size of 1000 on about ~11000 records - was about the total amount of data used to train for a year. Predictions are then made using the test set resulting in the predictions of \textit{'yhat'} which is inverted and rescaled to get original price values to save to a CSV and which are displayed on the user interface. The model is also validated using the test data as specified in the \textit{model.fit} method. \newpage @@ -1513,7 +1509,7 @@ def create_sets(self, data, lookback, sentiment): This section describes and discusses how the user interface is implemented, and what functions were used to both load data from the server and to display this data as graphical plots and tables on the interface. The aim of the interface, although simple in design, is to display relevant and useful information to the stakeholders in aiding in market decisions. - The interface is a simple HTML page that uses JQuery and AJAX requests in conjunction to display the required data and consists of three charts and two tables - each displaying predictions and performance metrics. A snippet of the final interface can be seen in \textit{Figure 11} later in the section. This section will describe the reasoning behind the data displayed and the layout. + The interface is a simple HTML page that uses JQuery and AJAX requests in conjunction to display the required data and consists of three charts and two tables - each displaying predictions and performance metrics. A snippet of the final interface can be seen in \textit{Figure 12} later in the section. This section will describe the reasoning behind the data displayed and the layout. \subsubsection{Key Functions} @@ -1549,7 +1545,7 @@ def create_sets(self, data, lookback, sentiment): \end{lstlisting} - The data displayed in these tables consist of the perforance metrics of the trained model of the network: root mean squared error, mean squard error, mean absolute error, mean absolute percentage error and loss of the network. Another table on the interface shows the the hourly predictions made by the LSTM network, showing: date and time of prediction, next hourly prediction, current hourly price, hourly sentiment and the suggested market action (\textit{Buy} or \textit{Sell}) based on the hard-coded 0.25 (25\%) threshold in the network code. \textit{See figure 11 below for visual representation} + The data displayed in these tables consist of the perforance metrics of the trained model of the network: root mean squared error, mean squard error, mean absolute error, mean absolute percentage error and loss of the network. Another table on the interface shows the the hourly predictions made by the LSTM network, showing: date and time of prediction, next hourly prediction, current hourly price, hourly sentiment and the suggested market action (\textit{Buy} or \textit{Sell}) based on the hard-coded 0.25 (25\%) threshold in the network code. \textit{See figure 12 below for visual representation} \newline \textbf{Graph Generation} @@ -1618,7 +1614,7 @@ def create_sets(self, data, lookback, sentiment): \centering \includegraphics[width=14cm,height=19cm]{images/final_interface.png} \begin{center} - \textit{Figure 11: Price Forecasting Dataflow diagram} + \textit{Figure 12: Price Forecasting Dataflow diagram} \end{center} \end{figure} @@ -1627,18 +1623,123 @@ def create_sets(self, data, lookback, sentiment): \section{Testing Metrics and Accuracy} This section discusses the performance metrics used to validate the network, any additional validation step during training, speed of execution of the two networks, and outlines what these mean regarding the performance of the network. - Due to the project being primarily an investigation of the use of discussed tools on how these would be used for forecasting the price of the next hour of Bitcoin, based on both historical and live price, and sentiment data. + Due to that the project is primarily an investigation of the use of discussed tools on how these would be used for forecasting the price of the next hour of Bitcoin, based on both historic and live price, and sentiment data, testing is focused around the accuracy of the predicted hourly values and the model generated by the LSTM network. A discussion of the accuracy of the predicted values is discussed in the 'Discussion of Results' section along with comparing both models - with and without sentiment embedded to fulfil the problem statement. - mean bias Error + \subsection{Integration Testing} + Integration testing occurred throughout the development of this system, the goal of this type of testing is the ensure that each of the functions within the system was working with previous functions that it either uses or provides functionality too and to conform to an agile methodology when possible. + If and when a function was completed, prior to moving on to the development of another, the function was tested with real and fake data - depending on the functionality the function performed. This was to ensure that the function was either being access or used other functions correctly. - k-fold cross validation was attempted, but issues with continuous data - How would this work what will it show or validate? + An example of when such testing occurred, though not documented, between components is that of the \textit{FilterSpam} class in the \textit{tweet\_collector} script. This was a significant class to test the operation of and was due to both the multiple calling of functions within the class from throughout the \textit{tweet\_collector} script and from the functions of the class to individual functions in the \textit{spam\_filter} script. For a precise example, using \textit{Listing 9} and function \textit{testTweet}, this function is only ever called upon receiving data from the Tweepy streamer (\textit{Listing 4}), however, this function calls two other functions of \textit{processTweets} (\textit{Listing 8}) and \textit{classify} (\textit{Listing 11}) from the \textit{spam\_filter} script (detailed in the implementation section). Not only do each of these functions need to be tested that the correct data is being processed and returned, but the entire function is reliant on the classifier being already trained for classification which is handled by the function in the class, whom of which also call other functions from the \textit{spam\_filter} script (\textit{Listing 10}). - Validation params for network compiling, what do that do? + This example was one of the most complex classes and set of functions to test as there was a lot of jumping back and forth and sequencing needed to ensure that the classifier was trained correctly and that streamed tweets are classified correctly. Similar testing occurred with all other functions and classes of the system at each stage of development. - speed of execution - both networks are ~ 3:15 and no sent ~ 2:50 over 200 epochs on a batch size of 1000 + \subsection{Accuracy of Model \& Results} + Due to the type of neural network used and its use case, suitable regression metrics were chosen to identify the accuracy of the model used for forecasting the next hour price of Bitcoin. As mentiod in the implementation section, subsection \textit{Training and Testing Model} the metrics used are as follows: - \section{Discussion of Results} + \begin{center} + \begin{tabular}{c|c|c|c|c} + \textbf{RMSE} & \textbf{MSE} & \textbf{MAE} & \textbf{MAPE} & \textbf{Loss}\\ + \hline + \multirow{1}{*}{} 100 $\pm$ 20 & 0.3 $\pm$ 0.1 & 3.0 $\pm$ 0.5 & 10\% $\pm$ 5\% & 0.3 $\pm$ 0.1 \\ + \end{tabular} + \newline + + Per execution of the neural network - \textbf{with} sentiment embedded + \end{center} + +\begin{center} + \begin{tabular}{c|c|c|c|c} + \textbf{RMSE} & \textbf{MSE} & \textbf{MAE} & \textbf{MAPE} & \textbf{Loss}\\ + \hline + \multirow{1}{*}{} 100 $\pm$ 25 & 0.3 $\pm$ 0.15 & 3.0 $\pm$ 0.5 & 10\% $\pm$ 6\% & 0.3 $\pm$ 0.15 \\ + \end{tabular} + \newline + + Per execution of the neural network - \textbf{without} sentiment embedded +\end{center} + + \subsubsection{Results Discussion} + + The results, although exceptionally close for each network (with and without sentiment embedded), averaged over 50 runs of each network show that the model with sentiment embedded generally performs better than the model without sentiment used in predictions. The model with sentiment embedded has an average RMSE of 100 give or take 20 per each run which shows that on average the predictions are closer to the regression line than the model without sentiment. Additionally, the mean absolute percentage error discrepancy is less for the model with sentiment embedded than the model without. This, however, does not provide a clear distinction between the two models that is needed to justify what is outlined in the problem statement. + + A clearer distinction between the performance of the two models is when the metics are showing the two model performing almost as accurate, as in the RMSE of both has been 105 and 106, the closest it has been on the 50 runs during testing. This also saw the mean absolute percentage error have only 0.2\% difference between the two models. Unfortunatly, the difference between the two models can only be seen when a portion of predictions are made. The models were left to run for 48 hours each which returned 48 predictions each of which could be compared to the next hour price. + + \begin{center} + \begin{tabular}{c|c|c|c} + \textbf{Created at} & \textbf{Prediction} & \textbf{Current Price} & \textbf{Current Sentiment}\\ + \hline + \multirow{1}{*}{} 2019-04-22 6pm & 5308.333 & 5318.4119 & 0.24312407 \\ + \multirow{1}{*}{} 2019-04-22 7pm & 5309.2754 & 5373.438 & 0.21355466 \\ + \multirow{1}{*}{} 2019-04-22 8pm & 5310.557 & 5413.161 & 0.28671014 \\ + \multirow{1}{*}{} 2019-04-22 9pm & 5317.3716 & 5375.6269 & 0.22499429\\ + \multirow{1}{*}{} 2019-04-22 10pm & 5337.3213 & 5373.607 & 0.25170501 \\ + \multirow{1}{*}{} 2019-04-22 11pm & 5370.356 & 5386.581 & 0.26898607\\ + \multirow{1}{*}{} 2019-04-23 12am & 5386.6113 & 5392.774 & 0.22517575\\ + \multirow{1}{*}{} 2019-04-23 1am & 5386.9487 & 5387.8319 & 0.27451984\\ + \multirow{1}{*}{} 2019-04-23 2am & 5379.05 & 5380.0669 & 0.23613823\\ + \multirow{1}{*}{} 2019-04-23 3am & 5384.681 & 5386.57 & 0.24832858\\ + \multirow{1}{*}{} 2019-04-23 4am & 5388.9434 & 5399.268 & 0.25803705\\ + \multirow{1}{*}{} 2019-04-23 5am & 5386.557 & 5429.906 & 0.25804942\\ + \multirow{1}{*}{} 2019-04-23 6am & 5385.1934 & 5510.472 & 0.25270584\\ + \multirow{1}{*}{} 2019-04-23 7am & 5389.97 & 5533.843 & 0.34432973\\ + \multirow{1}{*}{} 2019-04-23 8am & 5406.9917 & 5531.68 & 0.34782233\\ + \multirow{1}{*}{} 2019-04-23 9am & 5449.7676 & 5534.522 & 0.27746379\\ + \end{tabular} + \newline + + 15 records of predictions - \textbf{with} sentiment embedded + \end{center} + +\begin{center} + \begin{tabular}{c|c|c} + \textbf{Created at} & \textbf{Prediction} & \textbf{Current Price}\\ + \hline + \multirow{1}{*}{} 2019-04-22 6pm & 5373.431 & 5318.4119 \\ + \multirow{1}{*}{} 2019-04-22 7pm & 5381.814 & 5373.438 \\ + \multirow{1}{*}{} 2019-04-22 8pm & 5381.952 & 5413.161 \\ + \multirow{1}{*}{} 2019-04-22 9pm & 5388.013 & 5375.6269 \\ + \multirow{1}{*}{} 2019-04-22 10pm & 5410 & 5373.607 \\ + \multirow{1}{*}{} 2019-04-22 11pm & 5442.346 & 5386.581 \\ + \multirow{1}{*}{} 2019-04-23 12am & 5457.733 & 5392.774 \\ + \multirow{1}{*}{} 2019-04-23 1am & 5460.422 & 5387.8319 \\ + \multirow{1}{*}{} 2019-04-23 2am & 5451.898 & 5380.0669 \\ + \multirow{1}{*}{} 2019-04-23 3am & 5457.16 & 5386.57 \\ + \multirow{1}{*}{} 2019-04-23 4am & 5461.436 & 5399.268 \\ + \multirow{1}{*}{} 2019-04-23 5am & 5459.212 & 5429.906 \\ + \multirow{1}{*}{} 2019-04-23 6am & 5457.895 & 5510.472 \\ + \multirow{1}{*}{} 2019-04-23 7am & 5461.916 & 5533.843 \\ + \multirow{1}{*}{} 2019-04-23 8am & 5479.55 & 5531.68 \\ + \multirow{1}{*}{} 2019-04-23 9am & 5523.663 & 5534.522 \\ + \end{tabular} + \newline + + 15 records of predictions - \textbf{without} sentiment embedded +\end{center} + + On visual inspection, after both models had made 48 predictions, \textit{See above tables} showing 10 records as an example, it can be seen that the model with the sentiment embedded both follows the current price more closely and is not as conservative in its predictions as the model without sentiment. + How conservative the model without sentiment embedded is can seen in the five values between, 1 am to 5 am, where it attempts to correct itself to the actual value but slow to do so, then predicts a higher price for the next hour. At a point, the predicted value somewhat resembles that of the actual price but only due to the exact price rising substantially. + This model through the data shown, shows that it takes much longer to change the prediction to the actual real value of the next hour compared to the model with the embedded sentiment. + Also, neither model handles spikes in prices very well which is more noticeable with the model with sentiment as it follows the exact price more closely. + + Another factor that can be identified by the results shown above is that the model with the sentiment embedded in with the price data doesn't react adequately for when the sentiment spikes - regardless of a price spike. This could suggest that the data has not been predicted using enough data of both price and sentiment, due to only training on the last five live prices and sentiment and not 1000 samples to match the batch size the model was trained on during model creation. An improvement to the model could be made during predictions of the next hour price where it continuously predicts on the data available until it has 1000 records, then only predicts using the last 1000 records of live data. At this point, 1000 hours into predictions, said predictions might become more accurate than what is presented for evaluation at the time of writing due to matching the training batch sample size of the trained model of the network. + + Another result that can be drawn from the data presented, separate from the models' performance, is that of how sentiment affects the following hours' price. It can be seen in a number of the records presented that the sentiment occasionally spikes, at 1 am spikes to \textit{0.27451984} from the previous hour of \textit{0.22517575}, and the spike is not represented in the following hours' price. This also occurs on a previous hour at 8 pm which saw a spike of sentiment, a \textit{0.07405548} difference from the previous hour but saw the price drop almost \$40. This could indicate that the sentiment of the hours does not directly affect the next hours' price but affects it over possibly several hours, and also shows that there is not a direct correlation in sentiment spikes to price spikes. + + \subsubsection{Execution Speeds} + Each of the models are trained and validated using both generated training and testing dataset over 200 epochs and a batch size of 1000 on 11000 records total of data, within minutes. The network with sentiment embedded take marginly longer to create the prediction model. + + \begin{center} + \begin{tabular}{c|c} + \textbf{Network} & \textbf{Speed} \\ + \hline + \multirow{1}{*}{} With Sentiment & 3:15 $\pm$ 0:20 secs\\ + \multirow{1}{*}{} Without Sentiment & 2:50 $\pm$ 0:20 secs\\ + + \end{tabular} + \newline + + Speed of executions of each network, averaged over the 50 testing runs + \end{center} \newpage @@ -1652,11 +1753,13 @@ def create_sets(self, data, lookback, sentiment): \subsection{Limitations} + How would changing epoch and batch size affect performance? + \newpage \section{Conclusion and Future Improvements} \subsection{Conclusion} - + Interesting what would a days prediction would show due to sentiment not directly affecting the next hour price \subsection{Future Improvements} Shifting the predicted data by and hour and sequencing over previous data - will also allow proper use of look-back windows @@ -1672,6 +1775,15 @@ def create_sets(self, data, lookback, sentiment): Compare Scikit-learns in-built naive bayes algorithms and other variations performance for spam filtering against hand-coded version Look into adding to the VADER lexicon for increasing performance and accuracy for topic domain of stock market language and what sentiment would be assigned to words + + IMPLEMENT AND IDENTIFY R2 stat and Mean Bias ERROR + + (R2 and mean bias error would be suitable metrics to show how conservative the model is and the difference between the predicted and true price it is) - never got to implement + + mean bias Error + + k-fold cross validation was attempted, but issues with continuous data + How would this work what will it show or validate? \newpage \nocite{*} diff --git a/document.toc b/document.toc index ef7b905..7ff54fa 100644 --- a/document.toc +++ b/document.toc @@ -6,19 +6,17 @@ \defcounter {refsection}{0}\relax \contentsline {section}{\numberline {3}Glossary}{4}{section.3} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {4}Introduction}{8}{section.4} +\contentsline {section}{\numberline {4}Introduction}{9}{section.4} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {5}Problem Articulation}{10}{section.5} +\contentsline {section}{\numberline {5}Problem Articulation}{11}{section.5} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {5.1}Problem Statement}{10}{subsection.5.1} +\contentsline {subsection}{\numberline {5.1}Problem Statement}{11}{subsection.5.1} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {5.2}Stakeholders}{10}{subsection.5.2} +\contentsline {subsection}{\numberline {5.2}Stakeholders}{11}{subsection.5.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {5.3}Project Motivation}{11}{subsection.5.3} +\contentsline {subsection}{\numberline {5.3}Project Motivation}{12}{subsection.5.3} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {5.4}Technical Specification}{13}{subsection.5.4} -\defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {5.5}Project Constraints}{15}{subsection.5.5} +\contentsline {subsection}{\numberline {5.4}Technical Specification}{14}{subsection.5.4} \defcounter {refsection}{0}\relax \contentsline {section}{\numberline {6}Quality Goals}{16}{section.6} \defcounter {refsection}{0}\relax @@ -66,100 +64,108 @@ \defcounter {refsection}{0}\relax \contentsline {subsection}{\numberline {7.9}Addictive Smoothing}{31}{subsection.7.9} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {8}Solution Approach}{32}{section.8} +\contentsline {subsection}{\numberline {7.10}Regression Performance Metrics}{31}{subsection.7.10} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.1}Data gathering}{32}{subsection.8.1} +\contentsline {section}{\numberline {8}Solution Approach}{33}{section.8} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.2}Data pre-processing}{33}{subsection.8.2} +\contentsline {subsection}{\numberline {8.1}Data gathering}{33}{subsection.8.1} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.3}Spam Filtering}{33}{subsection.8.3} +\contentsline {subsection}{\numberline {8.2}Data pre-processing}{34}{subsection.8.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.4}Language Detection}{34}{subsection.8.4} +\contentsline {subsection}{\numberline {8.3}Spam Filtering}{34}{subsection.8.3} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.5}Sentiment Analysis}{34}{subsection.8.5} +\contentsline {subsection}{\numberline {8.4}Language Detection}{35}{subsection.8.4} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.6}Neural Network}{35}{subsection.8.6} +\contentsline {subsection}{\numberline {8.5}Sentiment Analysis}{35}{subsection.8.5} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.7}Price Forecasting}{37}{subsection.8.7} +\contentsline {subsection}{\numberline {8.6}Neural Network}{36}{subsection.8.6} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.8}Frontend Application}{37}{subsection.8.8} +\contentsline {subsection}{\numberline {8.7}Price Forecasting}{38}{subsection.8.7} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.9}With reference to Initial PID}{37}{subsection.8.9} +\contentsline {subsection}{\numberline {8.8}Frontend Application}{38}{subsection.8.8} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.10}Solution Summary}{38}{subsection.8.10} +\contentsline {subsection}{\numberline {8.9}With reference to Initial PID}{38}{subsection.8.9} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {8.11}Data flow Overview}{39}{subsection.8.11} +\contentsline {subsection}{\numberline {8.10}Solution Summary}{39}{subsection.8.10} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {9}System Design}{40}{section.9} +\contentsline {subsection}{\numberline {8.11}Data flow Overview}{40}{subsection.8.11} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {9.1}Dataflow Designs}{40}{subsection.9.1} +\contentsline {section}{\numberline {9}System Design}{41}{section.9} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {9.2}Interface Design}{47}{subsection.9.2} +\contentsline {subsection}{\numberline {9.1}Dataflow Designs}{41}{subsection.9.1} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {10}Implementation}{49}{section.10} +\contentsline {subsection}{\numberline {9.2}Interface Design}{48}{subsection.9.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.1}Data collection}{49}{subsection.10.1} +\contentsline {section}{\numberline {10}Implementation}{50}{section.10} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.1.1}Price Time-Series Historical Data}{49}{subsubsection.10.1.1} +\contentsline {subsection}{\numberline {10.1}Data collection}{50}{subsection.10.1} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.1.2}Price Time-Series Live Data}{50}{subsubsection.10.1.2} +\contentsline {subsubsection}{\numberline {10.1.1}Price Time-Series Historical Data}{50}{subsubsection.10.1.1} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.1.3}Historical Tweet Collection}{51}{subsubsection.10.1.3} +\contentsline {subsubsection}{\numberline {10.1.2}Price Time-Series Live Data}{51}{subsubsection.10.1.2} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.1.4}Live Tweet Collection}{53}{subsubsection.10.1.4} +\contentsline {subsubsection}{\numberline {10.1.3}Historical Tweet Collection}{52}{subsubsection.10.1.3} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.2}Data pre-processing}{55}{subsection.10.2} +\contentsline {subsubsection}{\numberline {10.1.4}Live Tweet Collection}{54}{subsubsection.10.1.4} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.2.1}Tweet Filtering}{55}{subsubsection.10.2.1} +\contentsline {subsection}{\numberline {10.2}Data pre-processing}{56}{subsection.10.2} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.2.2}Language detection filtering}{56}{subsubsection.10.2.2} +\contentsline {subsubsection}{\numberline {10.2.1}Tweet Filtering}{56}{subsubsection.10.2.1} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.2.3}Spam filter - Tokenisation, Ngrams, Stopword removal and Stemming}{58}{subsubsection.10.2.3} +\contentsline {subsubsection}{\numberline {10.2.2}Language detection filtering}{57}{subsubsection.10.2.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.3}Spam Filtering}{59}{subsection.10.3} +\contentsline {subsubsection}{\numberline {10.2.3}Spam filter - Tokenisation, Ngrams, Stopword removal and Stemming}{59}{subsubsection.10.2.3} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.3.1}Naive Bayes model}{62}{subsubsection.10.3.1} +\contentsline {subsection}{\numberline {10.3}Spam Filtering}{60}{subsection.10.3} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.3.2}Classification}{63}{subsubsection.10.3.2} +\contentsline {subsubsection}{\numberline {10.3.1}Naive Bayes model}{63}{subsubsection.10.3.1} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.3.3}Predict}{64}{subsubsection.10.3.3} +\contentsline {subsubsection}{\numberline {10.3.2}Classification}{64}{subsubsection.10.3.2} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.3.4}Metrics}{64}{subsubsection.10.3.4} +\contentsline {subsubsection}{\numberline {10.3.3}Predict}{65}{subsubsection.10.3.3} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.4}Sentiment Analysis}{65}{subsection.10.4} +\contentsline {subsubsection}{\numberline {10.3.4}Metrics}{65}{subsubsection.10.3.4} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.5}Recurrent Neural Network - LSTM}{66}{subsection.10.5} +\contentsline {subsection}{\numberline {10.4}Sentiment Analysis}{66}{subsection.10.4} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.5.1}Dataset Creation}{66}{subsubsection.10.5.1} +\contentsline {subsection}{\numberline {10.5}Recurrent Neural Network - LSTM}{67}{subsection.10.5} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.5.2}Training and Testing Model}{68}{subsubsection.10.5.2} +\contentsline {subsubsection}{\numberline {10.5.1}Dataset Creation}{67}{subsubsection.10.5.1} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.6}Future Prediction Forecasting}{70}{subsection.10.6} +\contentsline {subsubsection}{\numberline {10.5.2}Training and Testing Model}{69}{subsubsection.10.5.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {10.7}User Interface}{72}{subsection.10.7} +\contentsline {subsection}{\numberline {10.6}Future Prediction Forecasting}{71}{subsection.10.6} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.7.1}Key Functions}{72}{subsubsection.10.7.1} +\contentsline {subsection}{\numberline {10.7}User Interface}{73}{subsection.10.7} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{\numberline {10.7.2}Final Interface}{75}{subsubsection.10.7.2} +\contentsline {subsubsection}{\numberline {10.7.1}Key Functions}{73}{subsubsection.10.7.1} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {11}Testing Metrics and Accuracy}{76}{section.11} +\contentsline {subsubsection}{\numberline {10.7.2}Final Interface}{76}{subsubsection.10.7.2} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {12}Discussion of Results}{76}{section.12} +\contentsline {section}{\numberline {11}Testing Metrics and Accuracy}{77}{section.11} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {13}Project Evaluation}{77}{section.13} +\contentsline {subsection}{\numberline {11.1}Integration Testing}{77}{subsection.11.1} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {14}Discussion: Contribution and Reflection}{77}{section.14} +\contentsline {subsection}{\numberline {11.2}Accuracy of Model \& Results}{78}{subsection.11.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {14.1}Limitations}{77}{subsection.14.1} +\contentsline {subsubsection}{\numberline {11.2.1}Results Discussion}{78}{subsubsection.11.2.1} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {15}Conclusion and Future Improvements}{78}{section.15} +\contentsline {subsubsection}{\numberline {11.2.2}Execution Speeds}{80}{subsubsection.11.2.2} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {15.1}Conclusion}{78}{subsection.15.1} +\contentsline {section}{\numberline {12}Project Evaluation}{81}{section.12} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {15.2}Future Improvements}{78}{subsection.15.2} +\contentsline {section}{\numberline {13}Discussion: Contribution and Reflection}{81}{section.13} \defcounter {refsection}{0}\relax -\contentsline {section}{\numberline {16}Appendices}{83}{section.16} +\contentsline {subsection}{\numberline {13.1}Limitations}{81}{subsection.13.1} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {16.1}Appendix A - Project Initiation Document}{83}{subsection.16.1} +\contentsline {section}{\numberline {14}Conclusion and Future Improvements}{82}{section.14} \defcounter {refsection}{0}\relax -\contentsline {subsection}{\numberline {16.2}Appendix B - Log book}{96}{subsection.16.2} +\contentsline {subsection}{\numberline {14.1}Conclusion}{82}{subsection.14.1} +\defcounter {refsection}{0}\relax +\contentsline {subsection}{\numberline {14.2}Future Improvements}{82}{subsection.14.2} +\defcounter {refsection}{0}\relax +\contentsline {section}{\numberline {15}Appendices}{87}{section.15} +\defcounter {refsection}{0}\relax +\contentsline {subsection}{\numberline {15.1}Appendix A - Project Initiation Document}{87}{subsection.15.1} +\defcounter {refsection}{0}\relax +\contentsline {subsection}{\numberline {15.2}Appendix B - Log book}{100}{subsection.15.2} diff --git a/report.bib b/report.bib index fd33f8e..cf49721 100644 --- a/report.bib +++ b/report.bib @@ -428,4 +428,34 @@ year={Aug 2, 2017}, organization={Github}, url={https://github.com/tejank10/Spam-or-Ham} +} + +@inproceedings{MAPE, + title={Mean absolute percentage error (MAPE)}, + author={Stephanie}, + booktitle={}, + pages={}, + year={Sep 8, 2017}, + organization={Statistics HowTo}, + url={https://www.statisticshowto.datasciencecentral.com/mean-absolute-percentage-error-mape/} +} + +@inproceedings{RMSEMAE, + title={Choosing the Right Metric for Evaluating Machine Learning Models}, + author={Alvira Swalin}, + booktitle={}, + pages={}, + year={Apr 7, 2018}, + organization={Medium}, + url={https://medium.com/usf-msds/choosing-the-right-metric-for-machine-learning-models-part-1-a99d7d7414e4} +} + +@inproceedings{MSE, + title={Machine learning: an introduction to mean squared error and regression lines}, + author={Moshe Binieli}, + booktitle={}, + pages={}, + year={Oct 16, 2018}, + organization={Medium}, + url={https://medium.freecodecamp.org/machine-learning-mean-squared-error-regression-line-c7dde9a26b93} } \ No newline at end of file