diff --git a/document.aux b/document.aux index e5d8216..676658c 100644 --- a/document.aux +++ b/document.aux @@ -206,33 +206,33 @@ \@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {6}Spam filter initialisation and training functions}{50}{lstlisting.6}} \@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {7}Tweepy Streamer setup}{51}{lstlisting.7}} \@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {8}Tweepy Stream: 'on\_data' method}{52}{lstlisting.8}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Data processing}{55}{section*.53}} -\newlabel{processing}{{}{55}{Data processing}{section*.53}{}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Preprocessing}{55}{section*.54}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Data pre-processing}{55}{section*.53}} +\newlabel{processing}{{}{55}{Data pre-processing}{section*.53}{}} \@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {9}Basic data filtering and processing function - defined in 'tweet\_collector.py'}{55}{lstlisting.9}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {paragraph}{Text Cleaning}{55}{section*.56}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {paragraph}{Ngram based Language detection filtering}{55}{section*.57}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Spam Filtering}{55}{section*.58}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {10}Spam filter training Class}{55}{lstlisting.10}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {paragraph}{Naive Bayes model}{57}{section*.59}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Sentiment Analysis}{57}{section*.60}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{VADER}{57}{section*.61}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Recurrent Neural Network - LSTM}{57}{section*.62}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Training and Testing Model}{57}{section*.63}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Scoring and Validation}{57}{section*.64}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Future Prediction Forecasting}{58}{section*.65}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Main File 'Main.py'}{58}{section*.66}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Miscellaneous}{58}{section*.67}} -\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {11}keys class - loads API keys for access}{58}{lstlisting.11}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Testing Metrics and Accuracy}{59}{section*.68}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Project Evaluation}{60}{section*.69}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Discussion: Contribution and Reflection}{60}{section*.70}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Limitations}{60}{section*.71}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Conclusion and Future Improvements}{61}{section*.72}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Conclusion}{61}{section*.73}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Future Improvements}{61}{section*.74}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Appendices}{66}{section*.76}} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Appendix A - Project Initiation Document}{66}{section*.77}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {10}Language detection and filter function \cite {36}}{56}{lstlisting.10}} +\abx@aux@cite{36} +\abx@aux@segm{0}{0}{36} +\abx@aux@segm{0}{0}{36} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Spam Filtering}{57}{section*.54}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {11}Spam filter training Class}{57}{lstlisting.11}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Sentiment Analysis}{59}{section*.55}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{VADER}{59}{section*.56}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Recurrent Neural Network - LSTM}{59}{section*.57}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Training and Testing Model}{59}{section*.58}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsubsection}{Scoring and Validation}{59}{section*.59}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Future Prediction Forecasting}{59}{section*.60}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Main File 'Main.py'}{59}{section*.61}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Miscellaneous}{59}{section*.62}} +\@writefile{lol}{\defcounter {refsection}{0}\relax }\@writefile{lol}{\contentsline {lstlisting}{\numberline {12}keys class - loads API keys for access}{59}{lstlisting.12}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Testing Metrics and Accuracy}{60}{section*.63}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Project Evaluation}{61}{section*.64}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Discussion: Contribution and Reflection}{61}{section*.65}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Limitations}{61}{section*.66}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Conclusion and Future Improvements}{62}{section*.67}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Conclusion}{62}{section*.68}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Future Improvements}{62}{section*.69}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {section}{Appendices}{67}{section*.71}} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Appendix A - Project Initiation Document}{67}{section*.72}} \abx@aux@refcontextdefaultsdone \abx@aux@defaultrefcontext{0}{1}{none/global//global/global} \abx@aux@defaultrefcontext{0}{2}{none/global//global/global} @@ -269,4 +269,5 @@ \abx@aux@defaultrefcontext{0}{33}{none/global//global/global} \abx@aux@defaultrefcontext{0}{34}{none/global//global/global} \abx@aux@defaultrefcontext{0}{35}{none/global//global/global} -\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Appendix B - Log book}{79}{section*.78}} +\abx@aux@defaultrefcontext{0}{36}{none/global//global/global} +\@writefile{toc}{\defcounter {refsection}{0}\relax }\@writefile{toc}{\contentsline {subsection}{Appendix B - Log book}{80}{section*.73}} diff --git a/document.bbl b/document.bbl index e02b972..96c6fc2 100644 --- a/document.bbl +++ b/document.bbl @@ -1181,6 +1181,35 @@ \verb http://api.bitcoincharts.com/v1/csv/ \endverb \endentry + \entry{36}{inproceedings}{} + \name{author}{1}{}{% + {{hash=41b3b5daa9b33f52b08087b54f96f115}{% + family={Nolla}, + familyi={N\bibinitperiod}, + given={Alejandro}, + giveni={A\bibinitperiod}}}% + } + \list{organization}{1}{% + {Alejandro Nolla Blog}% + } + \strng{namehash}{41b3b5daa9b33f52b08087b54f96f115} + \strng{fullhash}{41b3b5daa9b33f52b08087b54f96f115} + \strng{bibnamehash}{41b3b5daa9b33f52b08087b54f96f115} + \strng{authorbibnamehash}{41b3b5daa9b33f52b08087b54f96f115} + \strng{authornamehash}{41b3b5daa9b33f52b08087b54f96f115} + \strng{authorfullhash}{41b3b5daa9b33f52b08087b54f96f115} + \field{sortinit}{6} + \field{sortinithash}{57e57fb8451e7fcfa45d1e069f6d3136} + \field{labelnamesource}{author} + \field{labeltitlesource}{title} + \field{title}{Detecting Text Language With Python and NLTK} + \verb{urlraw} + \verb http://blog.alejandronolla.com/2013/05/15/detecting-text-language-with-python-and-nltk/ + \endverb + \verb{url} + \verb http://blog.alejandronolla.com/2013/05/15/detecting-text-language-with-python-and-nltk/ + \endverb + \endentry \enddatalist \endrefsection \endinput diff --git a/document.bcf b/document.bcf index cc2a9da..4570ef1 100644 --- a/document.bcf +++ b/document.bcf @@ -1992,6 +1992,8 @@ 34 12 35 + 36 + 36 * diff --git a/document.blg b/document.blg index 42671e0..ab4eb46 100644 --- a/document.blg +++ b/document.blg @@ -1,20 +1,20 @@ [0] Config.pm:302> INFO - This is Biber 2.9 [0] Config.pm:305> INFO - Logfile is 'document.blg' -[22] biber:313> INFO - === Thu Apr 25, 2019, 20:17:38 -[39] Biber.pm:371> INFO - Reading 'document.bcf' -[92] Biber.pm:854> INFO - Using all citekeys in bib section 0 -[103] Biber.pm:3981> INFO - Processing section 0 -[112] Biber.pm:4154> INFO - Looking for bibtex format file 'report.bib' for section 0 -[114] bibtex.pm:1468> INFO - LaTeX decoding ... -[147] bibtex.pm:1294> INFO - Found BibTeX data source 'report.bib' +[19] biber:313> INFO - === Thu Apr 25, 2019, 21:12:18 +[33] Biber.pm:371> INFO - Reading 'document.bcf' +[78] Biber.pm:854> INFO - Using all citekeys in bib section 0 +[88] Biber.pm:3981> INFO - Processing section 0 +[95] Biber.pm:4154> INFO - Looking for bibtex format file 'report.bib' for section 0 +[96] bibtex.pm:1468> INFO - LaTeX decoding ... +[127] bibtex.pm:1294> INFO - Found BibTeX data source 'report.bib' +[184] Utils.pm:169> WARN - Name "Mairal, J., Ponce, J., Sapiro, G., Zisserman, A." has too many commas: skipping name [196] Utils.pm:169> WARN - year field 'Mar 13, 2016' in entry '23' is not an integer - this will probably not sort properly. -[203] Utils.pm:169> WARN - Name "Mairal, J., Ponce, J., Sapiro, G., Zisserman, A." has too many commas: skipping name -[266] Utils.pm:169> WARN - BibTeX subsystem: warning: comma(s) at end of name (removing) -[266] Utils.pm:169> WARN - BibTeX subsystem: author, warning: comma(s) at end of name (removing) -[286] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'normalization = NFD' with 'normalization = prenormalized' -[286] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'variable = shifted' with 'variable = non-ignorable' -[286] Biber.pm:3809> INFO - Sorting list 'none/global//global/global' of type 'entry' with template 'none' and locale 'en-US' -[286] Biber.pm:3815> INFO - No sort tailoring available for locale 'en-US' -[306] bbl.pm:617> INFO - Writing 'document.bbl' with encoding 'ascii' -[318] bbl.pm:720> INFO - Output to document.bbl -[319] Biber.pm:110> INFO - WARNINGS: 4 +[238] Utils.pm:169> WARN - BibTeX subsystem: warning: comma(s) at end of name (removing) +[238] Utils.pm:169> WARN - BibTeX subsystem: author, warning: comma(s) at end of name (removing) +[257] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'variable = shifted' with 'variable = non-ignorable' +[257] UCollate.pm:68> INFO - Overriding locale 'en-US' defaults 'normalization = NFD' with 'normalization = prenormalized' +[257] Biber.pm:3809> INFO - Sorting list 'none/global//global/global' of type 'entry' with template 'none' and locale 'en-US' +[257] Biber.pm:3815> INFO - No sort tailoring available for locale 'en-US' +[275] bbl.pm:617> INFO - Writing 'document.bbl' with encoding 'ascii' +[287] bbl.pm:720> INFO - Output to document.bbl +[287] Biber.pm:110> INFO - WARNINGS: 4 diff --git a/document.log b/document.log index 5a878cc..cf0d8ca 100644 --- a/document.log +++ b/document.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.16) 25 APR 2019 20:17 +This is pdfTeX, Version 3.14159265-2.6-1.40.18 (TeX Live 2017/Debian) (preloaded format=pdflatex 2018.10.16) 25 APR 2019 21:27 entering extended mode restricted \write18 enabled. %&-line parsing enabled. @@ -972,7 +972,7 @@ LaTeX Info: Redefining \nameref on input line 91. \@outlinefile=\write5 \openout5 = `document.out'. - + File: images/reading_logo.png Graphic file (type png) Package pdftex.def Info: images/reading_logo.png used on input line 94. @@ -1063,19 +1063,19 @@ Missing character: There is no Missing character: There is no € in font cmr12! Missing character: There is no ™ in font cmr12! [20] - + File: images/perceptron.png Graphic file (type png) Package pdftex.def Info: images/perceptron.png used on input line 338. (pdftex.def) Requested size: 284.52713pt x 170.72142pt. [21 <./images/perceptron.png>] - + File: images/rnn_ffn.png Graphic file (type png) Package pdftex.def Info: images/rnn_ffn.png used on input line 358. (pdftex.def) Requested size: 426.80307pt x 170.72112pt. [22 <./images/rnn_ffn.png>] - + File: images/lstm.png Graphic file (type png) Package pdftex.def Info: images/lstm.png used on input line 376. @@ -1121,7 +1121,7 @@ Missing character: There is no Missing character: There is no € in font cmr12! Missing character: There is no ™ in font cmr12! [33] [34] [35] - + File: images/Generic_Flow.png Graphic file (type png) Package pdftex.def Info: images/Generic_Flow.png used on input line 609. @@ -1136,7 +1136,7 @@ Underfull \hbox (badness 10000) in paragraph at lines 620--622 [] - + File: images/Dataflow.png Graphic file (type png) Package pdftex.def Info: images/Dataflow.png used on input line 625. @@ -1147,13 +1147,13 @@ Overfull \hbox (71.24937pt too wide) in paragraph at lines 625--627 [] [37 <./images/Dataflow.png (PNG copy)>] - + File: images/Data_Collector.png Graphic file (type png) Package pdftex.def Info: images/Data_Collector.png used on input line 633. (pdftex.def) Requested size: 426.78574pt x 227.61746pt. [38 <./images/Data_Collector.png (PNG copy)>] - + File: images/Analysis_Engine.png Graphic file (type png) Package pdftex.def Info: images/Analysis_Engine.png used on input line 648. @@ -1164,7 +1164,7 @@ Overfull \hbox (42.8253pt too wide) in paragraph at lines 648--650 [] [39 <./images/Analysis_Engine.png (PNG copy)>] - + File: images/Neural_Network.png Graphic file (type png) Package pdftex.def Info: images/Neural_Network.png used on input line 664. @@ -1175,7 +1175,7 @@ Overfull \hbox (42.84184pt too wide) in paragraph at lines 664--666 [] [40] [41 <./images/Neural_Network.png (PNG copy)>] - + File: images/Future_Predictions.png Graphic file (type png) Package pdftex.def Info: images/Future_Predictions.png used on input line 678. @@ -1187,14 +1187,14 @@ Overfull \hbox (71.28874pt too wide) in paragraph at lines 678--680 [] [42 <./images/Future_Predictions.png (PNG copy)>] - + File: images/Frontend_Application.png Graphic file (type png) Package pdftex.def Info: images/Frontend_Application.png used on input line 69 1. (pdftex.def) Requested size: 284.52162pt x 256.07664pt. [43 <./images/Frontend_Application.png (PNG copy)>] - + File: images/interface_design.png Graphic file (type png) Package pdftex.def Info: images/interface_design.png used on input line 710. @@ -1221,55 +1221,64 @@ Underfull \hbox (badness 10000) in paragraph at lines 971--973 [] [51] -LaTeX Font Info: Try loading font information for OML+cmr on input line 985. +LaTeX Font Info: Try loading font information for OML+cmr on input line 984. (/usr/share/texlive/texmf-dist/tex/latex/base/omlcmr.fd File: omlcmr.fd 2014/09/29 v2.5h Standard LaTeX font definitions ) LaTeX Font Info: Font shape `OML/cmr/m/n' in size <10> not available -(Font) Font shape `OML/cmm/m/it' tried instead on input line 985. +(Font) Font shape `OML/cmm/m/it' tried instead on input line 984. [52] [53] [54] +Underfull \hbox (badness 10000) in paragraph at lines 1075--1077 + + [] + Missing character: There is no  in font cmr10! Missing character: There is no £ in font cmr10! - [55] [56] [57] [58] [59] [60] [61] -Overfull \hbox (5.27716pt too wide) in paragraph at lines 1234--1234 + +Underfull \hbox (badness 10000) in paragraph at lines 1114--1117 + + [] + +[55] [56] [57] [58] [59] [60] [61] [62] +Overfull \hbox (5.27716pt too wide) in paragraph at lines 1297--1297 \OT1/cmr/m/it/12 ence on sig-nal pro-cess-ing, com-mu-ni-ca-tion, power and em- bed-ded sys-tem (SCOPES)\OT1/cmr/m/n/12 , [] -Overfull \hbox (42.7786pt too wide) in paragraph at lines 1234--1234 +Overfull \hbox (42.7786pt too wide) in paragraph at lines 1297--1297 \OT1/cmr/m/n/12 works,'' To-wards Data Sci-ence, 2018. [On-line]. Avail-able: [ ]$\OT1/cmtt/m/n/12 https : / / towardsdatascience . [] -[62] -Overfull \hbox (86.07425pt too wide) in paragraph at lines 1234--1234 +[63] +Overfull \hbox (86.07425pt too wide) in paragraph at lines 1297--1297 \OT1/cmr/m/n/12 works,'' Ma-chine Larn-ing Mas-tery, 2017. [On-line]. Avail-abl e: []$\OT1/cmtt/m/n/12 https : / / machinelearningmastery . [] -Overfull \hbox (30.84552pt too wide) in paragraph at lines 1234--1234 +Overfull \hbox (30.84552pt too wide) in paragraph at lines 1297--1297 \OT1/cmr/m/n/12 lem,'' Su-per Data Sci-ence, 2018. [On-line]. Avail-able: []$\O T1/cmtt/m/n/12 https : / / www . superdatascience . [] -[63] -Overfull \hbox (9.16136pt too wide) in paragraph at lines 1234--1234 +[64] +Overfull \hbox (9.16136pt too wide) in paragraph at lines 1297--1297 \OT1/cmr/m/n/12 2019. [On-line]. Avail-able: []$\OT1/cmtt/m/n/12 https : / / me dium . com / datadriveninvestor / overview -[] [] -[64] [65] +[65] [66] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf used on input line 1239. +Package pdftex.def Info: PID.pdf used on input line 1302. (pdftex.def) Requested size: 597.551pt x 845.07512pt. @@ -1277,7 +1286,7 @@ pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf used on input line 1239. +Package pdftex.def Info: PID.pdf used on input line 1302. (pdftex.def) Requested size: 597.551pt x 845.07512pt. @@ -1287,234 +1296,234 @@ rsion <1.7>, but at most version <1.5> allowed pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1239. +Package pdftex.def Info: PID.pdf , page1 used on input line 1302. (pdftex.def) Requested size: 597.551pt x 845.07512pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1239. +Package pdftex.def Info: PID.pdf , page1 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[66] +[67] File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1239. +Package pdftex.def Info: PID.pdf , page1 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1239. +Package pdftex.def Info: PID.pdf , page1 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page1 used on input line 1239. +Package pdftex.def Info: PID.pdf , page1 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. - [67 <./PID.pdf>] + [68 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1239. +Package pdftex.def Info: PID.pdf , page2 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1239. +Package pdftex.def Info: PID.pdf , page2 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) -Package pdftex.def Info: PID.pdf , page2 used on input line 1239. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -[68 <./PID.pdf>] - -pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve -rsion <1.7>, but at most version <1.5> allowed - -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1239. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1239. -(pdftex.def) Requested size: 562.1644pt x 795.0303pt. -File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page3 used on input line 1239. +Package pdftex.def Info: PID.pdf , page2 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [69 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page4 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page3 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [70 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page5 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page4 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [71 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page6 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page5 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [72 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page7 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page6 used on input line 1302. (pdftex.def) Requested size: 562.1644pt x 795.0303pt. [73 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1239. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1302. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1239. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1302. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page8 used on input line 1239. -(pdftex.def) Requested size: 795.0303pt x 562.1644pt. + +Package pdftex.def Info: PID.pdf , page7 used on input line 1302. +(pdftex.def) Requested size: 562.1644pt x 795.0303pt. [74 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page9 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page8 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. [75 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page9 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page9 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page10 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page9 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. [76 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page10 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page10 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page11 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page10 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. [77 <./PID.pdf>] pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve rsion <1.7>, but at most version <1.5> allowed - + File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page11 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page11 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. File: PID.pdf Graphic file (type pdf) - -Package pdftex.def Info: PID.pdf , page12 used on input line 1239. + +Package pdftex.def Info: PID.pdf , page11 used on input line 1302. (pdftex.def) Requested size: 795.0303pt x 562.1644pt. [78 <./PID.pdf>] -Package atveryend Info: Empty hook `BeforeClearDocument' on input line 1243. - [79] -Package atveryend Info: Empty hook `AfterLastShipout' on input line 1243. + +pdfTeX warning: /usr/bin/pdflatex (file ./PID.pdf): PDF inclusion: found PDF ve +rsion <1.7>, but at most version <1.5> allowed + +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1302. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1302. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +File: PID.pdf Graphic file (type pdf) + +Package pdftex.def Info: PID.pdf , page12 used on input line 1302. +(pdftex.def) Requested size: 795.0303pt x 562.1644pt. +[79 <./PID.pdf>] +Package atveryend Info: Empty hook `BeforeClearDocument' on input line 1306. + [80] +Package atveryend Info: Empty hook `AfterLastShipout' on input line 1306. (./document.aux) -Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 1243. -Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 1243. +Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 1306. +Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 1306. Package rerunfilecheck Info: File `document.out' has not changed. -(rerunfilecheck) Checksum: 285E84D8F0115130141DAF2B9AB7C433;10818. +(rerunfilecheck) Checksum: 283D22108CCCB057050FE149F93F8AC8;10716. Package logreq Info: Writing requests to 'document.run.xml'. \openout1 = `document.run.xml'. ) Here is how much of TeX's memory you used: - 24568 strings out of 492982 - 385699 string characters out of 6134895 - 1048130 words of memory out of 5000000 - 27195 multiletter control sequences out of 15000+600000 + 24631 strings out of 492982 + 386648 string characters out of 6134895 + 1047130 words of memory out of 5000000 + 27213 multiletter control sequences out of 15000+600000 13923 words of font info for 54 fonts, out of 8000000 for 9000 1141 hyphenation exceptions out of 8191 45i,18n,78p,2008b,1819s stack positions out of 5000i,500n,10000p,200000b,80000s @@ -1533,10 +1542,10 @@ fb> -Output written on document.pdf (79 pages, 1486919 bytes). +Output written on document.pdf (80 pages, 1495523 bytes). PDF statistics: - 1554 PDF objects out of 1728 (max. 8388607) - 1405 compressed objects within 15 object streams - 588 named destinations out of 1000 (max. 500000) - 696 words of extra memory for PDF output out of 10000 (max. 10000000) + 1611 PDF objects out of 1728 (max. 8388607) + 1460 compressed objects within 15 object streams + 633 named destinations out of 1000 (max. 500000) + 688 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/document.out b/document.out index 3f86170..bba1423 100644 --- a/document.out +++ b/document.out @@ -47,24 +47,23 @@ \BOOKMARK [3][-]{section*.50}{\376\377\000P\000r\000i\000c\000e\000\040\000T\000i\000m\000e\000-\000S\000e\000r\000i\000e\000s\000\040\000L\000i\000v\000e\000\040\000D\000a\000t\000a}{section*.48}% 47 \BOOKMARK [3][-]{section*.51}{\376\377\000H\000i\000s\000t\000o\000r\000i\000c\000a\000l\000\040\000T\000w\000e\000e\000t\000\040\000C\000o\000l\000l\000e\000c\000t\000i\000o\000n}{section*.48}% 48 \BOOKMARK [3][-]{section*.52}{\376\377\000L\000i\000v\000e\000\040\000T\000w\000e\000e\000t\000\040\000C\000o\000l\000l\000e\000c\000t\000i\000o\000n}{section*.48}% 49 -\BOOKMARK [2][-]{section*.53}{\376\377\000D\000a\000t\000a\000\040\000p\000r\000o\000c\000e\000s\000s\000i\000n\000g}{section*.47}% 50 -\BOOKMARK [3][-]{section*.54}{\376\377\000P\000r\000e\000p\000r\000o\000c\000e\000s\000s\000i\000n\000g}{section*.53}% 51 -\BOOKMARK [3][-]{section*.58}{\376\377\000S\000p\000a\000m\000\040\000F\000i\000l\000t\000e\000r\000i\000n\000g}{section*.53}% 52 -\BOOKMARK [2][-]{section*.60}{\376\377\000S\000e\000n\000t\000i\000m\000e\000n\000t\000\040\000A\000n\000a\000l\000y\000s\000i\000s}{section*.47}% 53 -\BOOKMARK [3][-]{section*.61}{\376\377\000V\000A\000D\000E\000R}{section*.60}% 54 -\BOOKMARK [2][-]{section*.62}{\376\377\000R\000e\000c\000u\000r\000r\000e\000n\000t\000\040\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000\040\000-\000\040\000L\000S\000T\000M}{section*.47}% 55 -\BOOKMARK [3][-]{section*.63}{\376\377\000T\000r\000a\000i\000n\000i\000n\000g\000\040\000a\000n\000d\000\040\000T\000e\000s\000t\000i\000n\000g\000\040\000M\000o\000d\000e\000l}{section*.62}% 56 -\BOOKMARK [3][-]{section*.64}{\376\377\000S\000c\000o\000r\000i\000n\000g\000\040\000a\000n\000d\000\040\000V\000a\000l\000i\000d\000a\000t\000i\000o\000n}{section*.62}% 57 -\BOOKMARK [2][-]{section*.65}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000P\000r\000e\000d\000i\000c\000t\000i\000o\000n\000\040\000F\000o\000r\000e\000c\000a\000s\000t\000i\000n\000g}{section*.47}% 58 -\BOOKMARK [2][-]{section*.66}{\376\377\000M\000a\000i\000n\000\040\000F\000i\000l\000e\000\040\000'\000M\000a\000i\000n\000.\000p\000y\000'}{section*.47}% 59 -\BOOKMARK [2][-]{section*.67}{\376\377\000M\000i\000s\000c\000e\000l\000l\000a\000n\000e\000o\000u\000s}{section*.47}% 60 -\BOOKMARK [1][-]{section*.68}{\376\377\000T\000e\000s\000t\000i\000n\000g\000\040\000M\000e\000t\000r\000i\000c\000s\000\040\000a\000n\000d\000\040\000A\000c\000c\000u\000r\000a\000c\000y}{}% 61 -\BOOKMARK [1][-]{section*.69}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000E\000v\000a\000l\000u\000a\000t\000i\000o\000n}{}% 62 -\BOOKMARK [1][-]{section*.70}{\376\377\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n\000:\000\040\000C\000o\000n\000t\000r\000i\000b\000u\000t\000i\000o\000n\000\040\000a\000n\000d\000\040\000R\000e\000f\000l\000e\000c\000t\000i\000o\000n}{}% 63 -\BOOKMARK [2][-]{section*.71}{\376\377\000L\000i\000m\000i\000t\000a\000t\000i\000o\000n\000s}{section*.70}% 64 -\BOOKMARK [1][-]{section*.72}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n\000\040\000a\000n\000d\000\040\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{}% 65 -\BOOKMARK [2][-]{section*.73}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n}{section*.72}% 66 -\BOOKMARK [2][-]{section*.74}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{section*.72}% 67 -\BOOKMARK [1][-]{section*.76}{\376\377\000A\000p\000p\000e\000n\000d\000i\000c\000e\000s}{}% 68 -\BOOKMARK [2][-]{section*.77}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000A\000\040\000-\000\040\000P\000r\000o\000j\000e\000c\000t\000\040\000I\000n\000i\000t\000i\000a\000t\000i\000o\000n\000\040\000D\000o\000c\000u\000m\000e\000n\000t}{section*.76}% 69 -\BOOKMARK [2][-]{section*.78}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000B\000\040\000-\000\040\000L\000o\000g\000\040\000b\000o\000o\000k}{section*.76}% 70 +\BOOKMARK [2][-]{section*.53}{\376\377\000D\000a\000t\000a\000\040\000p\000r\000e\000-\000p\000r\000o\000c\000e\000s\000s\000i\000n\000g}{section*.47}% 50 +\BOOKMARK [3][-]{section*.54}{\376\377\000S\000p\000a\000m\000\040\000F\000i\000l\000t\000e\000r\000i\000n\000g}{section*.53}% 51 +\BOOKMARK [2][-]{section*.55}{\376\377\000S\000e\000n\000t\000i\000m\000e\000n\000t\000\040\000A\000n\000a\000l\000y\000s\000i\000s}{section*.47}% 52 +\BOOKMARK [3][-]{section*.56}{\376\377\000V\000A\000D\000E\000R}{section*.55}% 53 +\BOOKMARK [2][-]{section*.57}{\376\377\000R\000e\000c\000u\000r\000r\000e\000n\000t\000\040\000N\000e\000u\000r\000a\000l\000\040\000N\000e\000t\000w\000o\000r\000k\000\040\000-\000\040\000L\000S\000T\000M}{section*.47}% 54 +\BOOKMARK [3][-]{section*.58}{\376\377\000T\000r\000a\000i\000n\000i\000n\000g\000\040\000a\000n\000d\000\040\000T\000e\000s\000t\000i\000n\000g\000\040\000M\000o\000d\000e\000l}{section*.57}% 55 +\BOOKMARK [3][-]{section*.59}{\376\377\000S\000c\000o\000r\000i\000n\000g\000\040\000a\000n\000d\000\040\000V\000a\000l\000i\000d\000a\000t\000i\000o\000n}{section*.57}% 56 +\BOOKMARK [2][-]{section*.60}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000P\000r\000e\000d\000i\000c\000t\000i\000o\000n\000\040\000F\000o\000r\000e\000c\000a\000s\000t\000i\000n\000g}{section*.47}% 57 +\BOOKMARK [2][-]{section*.61}{\376\377\000M\000a\000i\000n\000\040\000F\000i\000l\000e\000\040\000'\000M\000a\000i\000n\000.\000p\000y\000'}{section*.47}% 58 +\BOOKMARK [2][-]{section*.62}{\376\377\000M\000i\000s\000c\000e\000l\000l\000a\000n\000e\000o\000u\000s}{section*.47}% 59 +\BOOKMARK [1][-]{section*.63}{\376\377\000T\000e\000s\000t\000i\000n\000g\000\040\000M\000e\000t\000r\000i\000c\000s\000\040\000a\000n\000d\000\040\000A\000c\000c\000u\000r\000a\000c\000y}{}% 60 +\BOOKMARK [1][-]{section*.64}{\376\377\000P\000r\000o\000j\000e\000c\000t\000\040\000E\000v\000a\000l\000u\000a\000t\000i\000o\000n}{}% 61 +\BOOKMARK [1][-]{section*.65}{\376\377\000D\000i\000s\000c\000u\000s\000s\000i\000o\000n\000:\000\040\000C\000o\000n\000t\000r\000i\000b\000u\000t\000i\000o\000n\000\040\000a\000n\000d\000\040\000R\000e\000f\000l\000e\000c\000t\000i\000o\000n}{}% 62 +\BOOKMARK [2][-]{section*.66}{\376\377\000L\000i\000m\000i\000t\000a\000t\000i\000o\000n\000s}{section*.65}% 63 +\BOOKMARK [1][-]{section*.67}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n\000\040\000a\000n\000d\000\040\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{}% 64 +\BOOKMARK [2][-]{section*.68}{\376\377\000C\000o\000n\000c\000l\000u\000s\000i\000o\000n}{section*.67}% 65 +\BOOKMARK [2][-]{section*.69}{\376\377\000F\000u\000t\000u\000r\000e\000\040\000I\000m\000p\000r\000o\000v\000e\000m\000e\000n\000t\000s}{section*.67}% 66 +\BOOKMARK [1][-]{section*.71}{\376\377\000A\000p\000p\000e\000n\000d\000i\000c\000e\000s}{}% 67 +\BOOKMARK [2][-]{section*.72}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000A\000\040\000-\000\040\000P\000r\000o\000j\000e\000c\000t\000\040\000I\000n\000i\000t\000i\000a\000t\000i\000o\000n\000\040\000D\000o\000c\000u\000m\000e\000n\000t}{section*.71}% 68 +\BOOKMARK [2][-]{section*.73}{\376\377\000A\000p\000p\000e\000n\000d\000i\000x\000\040\000B\000\040\000-\000\040\000L\000o\000g\000\040\000b\000o\000o\000k}{section*.71}% 69 diff --git a/document.pdf b/document.pdf index 5f5abf6..18b0a75 100644 Binary files a/document.pdf and b/document.pdf differ diff --git a/document.synctex.gz b/document.synctex.gz index 9360597..c69833a 100644 Binary files a/document.synctex.gz and b/document.synctex.gz differ diff --git a/document.tex b/document.tex index f28ca93..4e581f5 100644 --- a/document.tex +++ b/document.tex @@ -978,8 +978,7 @@ from tweepy import OAuthHandler from tweepy import Stream from tweepy.streaming import StreamListener import csv -... - +... def on_data(self, data): ## Check time limit for under an hour - if limit reached kill script if (time.time() - self.start_time) < self.limit: @@ -1025,7 +1024,7 @@ def on_data(self, data): # Check if tweet is long enough to perform polarity classification on (> 5 words (checked through tokenisation)) if tweetLength == True: checkIfEnglish = utilityFuncs().detectLaguage(removedSpecialChars[0]) - # Check if the text in tweet is predominatly English, if not drop + # Check if the text in tweet is predominantly English, if not drop if checkIfEnglish == True: tweetText = utilityFuncs().remove_non_ascii(removedSpacing) print("Cleaned Tweet: ", tweetText) @@ -1068,115 +1067,179 @@ def on_data(self, data): ... \end{lstlisting} - As for key facts about this function; the tweets length is checked to be above 5 (tokenised) due to any tweets will less words do not contain enough information to be given a proper polarity classification and almost always returns as 100\% neutral, which is of no use and will have no affect on the hours average sentiment. Entire code in the function is encapsulated in a try catch to check if data was recieved and handles non-responses and missing data by simply ignore that there was no data, unless a connection between the streamer and API is broken it otherwise exits the script. + As for the key facts about this function; the length of the tweet is checked to be above 5 (tokenised) due to any tweets with fewer words will not contain enough information to be given a proper polarity classification and almost always returns as 100\% neutral, which is of no use and will have no effect on the hours average sentiment. The entire code in the function is encapsulated in a try-catch to check if data was received and handles non-responses and missing data. If there was no data the issue is ignored unless a connection between the streamer and API is broken it otherwise exits the script. \newpage - \subsection{Data processing}\label{processing} - \subsubsection{Preprocessing} + \subsection{Data pre-processing}\label{processing} + Various techniques and tools have been utilised throughout the development of the system to process the data appropriately so it can be parsed by VADER, spam filter and neural network. This section will cover the crucial functions that provide such functionalities and that are called throughout the system, as seen in some of the above code snippets. - \paragraph{Tweet Filtering} + \newline + + \textbf{Tweet Filtering} + + Various 'Utility Functions' have been used to initially filter out unwanted data from tweet text. These functions called by, both live tweet (\textit{tweet\_collector.py}) and historical tweet (\textit{sift\_text.py}) processing, prior any polarity classification or storing of tweet data to CSV files. \begin{lstlisting}[language=python, caption=Basic data filtering and processing function - defined in 'tweet\_collector.py'] import re import emoji as ji +## Key packages used ... - class utilityFuncs(): def cleanTweet(self, text): # Function to clean tweets, removes links and special characters return re.sub(r'([^0-9A-Za-z \-\%\£\$ \t])|(@[A-Za-z0-9]+)|(http\S+)', '', text), ' '.join(c for c in text if c in ji.UNICODE_EMOJI) + # Also removes emojis from text - later re-added due to VADER supporting emoticons def removeSpacing(self, text): return re.sub(r'( +)', ' ', text) + # Removes extra spacing that may be left between words def fixLines(self, text): return re.sub(r"([\r\n])", " ", text) + # Removes line breaks and new lines from text def remove_non_ascii(self, text): return ''.join(i for i in text if ord(i)<128) + # User after language detections and removes non-English characters from text + + def checkLength(self, text): + tokens = text.split() + if len(tokens) <= 5: # Tokenisation + return False + else: + return True \end{lstlisting} - \paragraph{Text Cleaning} - \paragraph{Ngram based Language detection filtering} + + Due to VADER being a lexicon-based sentiment analyser little data pre-processing needs conducting on the tweet text. The functions above primarily remove unnecessary text from the tweet that will either provide no insight into public opinion or can obstruct a proper classification of the sentiment - such as the existence of URLs in the given text. Additionally, the 'clean\_tweet' function removes the emojis in the given text if any are presently using the emoji package - which in turn is another lexicon that compares the given text to any emoticon contained within the lexicon. These are removed at this stage but are later re-added back to the text as VADER support emoticon classification. + The last function in 'utility functions', 'checkLength' splits the text up into individual words (tokens - a process of tokenisation), this is used to check the total length of a tweet. If the tweet is less than five words it is dropped from classification, this is due to + \newline + + \textbf{Language detection filtering} + This feature of the system is used as an additional filter for filtering out non-English tweets. As discussed in the solution approach, Tweepy/Twitter API provides a means to filter out non-English based tweets, this, however, doesn't work if the user has settings on Twitter set to be English as a prefered language and the region 'en'. Due to this non-English characters can still be within collected tweets; thus these are detected and filtered with the below function. + + \begin{lstlisting}[language=python, caption=Language detection and filter function \cite{36}] +def detectLaguage(self, text): + """ + Calculate the probability of given text is written in several languages + Using nltk stopwords and comparing to all supported languages + + There are other ways to identify this - TextBlob.detect_language and Ngrams + """ + language_ratios = {} + + # Split words up into tokens - tokenisation + tokens = wordpunct_tokenize(text) + + # Shift to lower case + words = [word.lower() for word in tokens] + + # Compute per language in nltk number of stopwords in text + for language in stopwords.fileids(): + stopwords_set = set(stopwords.words(language)) + words_set = set(words) + common_elements = words_set.intersection(stopwords_set) + + language_ratios[language] = len(common_elements) + # Form ratio scores for each language detected from stopword comparison + + ratios = language_ratios + highest_ratio = max(ratios, key=ratios.get) + # Extract highest ratio language used in given text + + print("Console: Text is - ", highest_ratio) + sys.stdout.flush() + + if highest_ratio == 'english': + return True + else: + return False + # If text is not predominately English drop tweet + \end{lstlisting} + + This function uses several natural languages pre-processing techniques to identify the most predominant language for a given text. It accomplishes this by first tokenising the text into tokens and converting them to lower case - this is so that the stopwords can be identified. For each of the languages supported by the Natural Language Toolkit Python package, the stopwords are identified in the text and compared to the stopwords in the language corpus' in NLTK. The ratios for the individual languages are formed, and then the predominant language identified. If the language is not predominantly English, the tweet is dropped. + + Ngrams could also be used + + \textbf{Spam filter - Tokenisation, Ngrams, Stopword removal and Stemming} \subsubsection{Spam Filtering} \begin{lstlisting}[language=python, caption=Spam filter training Class] - class filterSpam(object): +class filterSpam(object): - def __init__(self, training_set): - self.training_set = training_set - ## initialises function and globalises training set for use in every function where needed + def __init__(self, training_set): + self.training_set = training_set + ## initialises function and globalises training set for use in every function where needed - def trainFilter(self): - self.dataset() ## Split dataset 75:25 - self.train() ## Train based on training dataset + def trainFilter(self): + self.dataset() ## Split dataset 75:25 + self.train() ## Train based on training dataset - def dataset(self): - self.data = pd.read_csv(self.training_set) + def dataset(self): + self.data = pd.read_csv(self.training_set) - self.data['class'] = self.data['classes'].map({'ham': 0, 'spam': 1}) - # Remap labels of 'Spam' and 'Ham' to 1:0 respectively + self.data['class'] = self.data['classes'].map({'ham': 0, 'spam': 1}) + # Remap labels of 'Spam' and 'Ham' to 1:0 respectively - self.data.drop(['classes'], axis=1, inplace=True) - # Drop old labels + self.data.drop(['classes'], axis=1, inplace=True) + # Drop old labels - self.trainIndex, self.testIndex = list(), list() - for i in range(self.data.shape[0]): - if np.random.uniform(0, 1) < 0.75: # Random shuffle data of 75% + self.trainIndex, self.testIndex = list(), list() + for i in range(self.data.shape[0]): + if np.random.uniform(0, 1) < 0.75: # Random shuffle data of 75% self.trainIndex += [i] # Create training index - else: + else: self.testIndex += [i] # Create testing index - self.trainData = self.data.loc[self.trainIndex] - self.testData = self.data.loc[self.testIndex] - # Define datasets by getting values from first 75% and then 25% + self.trainData = self.data.loc[self.trainIndex] + self.testData = self.data.loc[self.testIndex] + # Define datasets by getting values from first 75% and then 25% - self.trainData.reset_index(inplace=True) - self.testData.reset_index(inplace=True) - # Reset indexes + self.trainData.reset_index(inplace=True) + self.testData.reset_index(inplace=True) + # Reset indexes - self.trainData.drop(['index'], axis=1, inplace=True) - self.testData.drop(['index'], axis=1, inplace=True) - # Drop old index + self.trainData.drop(['index'], axis=1, inplace=True) + self.testData.drop(['index'], axis=1, inplace=True) + # Drop old index - def train(self): - self.spamFilter = spam_filter.classifier(self.trainData) - # Initialise the spam filter with the 75% dataset +def train(self): + self.spamFilter = spam_filter.classifier(self.trainData) + # Initialise the spam filter with the 75% dataset - self.spamFilter.train() - # Train + self.spamFilter.train() + # Train - def testData_Prediction(self): - prediction = self.spamFilter.predict(self.testData['tweet']) +def testData_Prediction(self): + prediction = self.spamFilter.predict(self.testData['tweet']) + return prediction - return prediction +def testPrediction(self): - def testPrediction(self): + # Test Spam/Ham tweets - should return True and False respectivly + spam = spam_filter.processTweet("Earn more than 0015 btc free No deposit No investment Free Bitcoins - Earn $65 free btc in 5 minutes bitcoin freebtc getbtc") - # Test Spam/Ham tweets - should return True and False respectivly - spam = spam_filter.processTweet("Earn more than 0015 btc free No deposit No investment Free Bitcoins - Earn $65 free btc in 5 minutes bitcoin freebtc getbtc") + ham = spam_filter.processTweet("Bitcoin closed with some gains in month of February") - ham = spam_filter.processTweet("Bitcoin closed with some gains in month of February") + hamTweet = self.spamFilter.classify(ham) + spamTweet = self.spamFilter.classify(spam) - hamTweet = self.spamFilter.classify(ham) - spamTweet = self.spamFilter.classify(spam) + print("Console: ", "Spam Tweet -- ", spamTweet) + sys.stdout.flush() + print("Console: ", "Ham Tweet -- ", hamTweet) + sys.stdout.flush() - print("Console: ", "Spam Tweet -- ", spamTweet) - sys.stdout.flush() - print("Console: ", "Ham Tweet -- ", hamTweet) - sys.stdout.flush() +def filterStatistics(self, prediction): + spam_filter.metrics(self.testData['class'], prediction) - def filterStatistics(self, prediction): - spam_filter.metrics(self.testData['class'], prediction) +def testTweet(self, tweet): - def testTweet(self, tweet): + processed = spam_filter.processTweet(tweet) + classified = self.spamFilter.classify(processed) - processed = spam_filter.processTweet(tweet) - classified = self.spamFilter.classify(processed) - - return classified + return classified \end{lstlisting} - \paragraph{Naive Bayes model} + \textbf{Naive Bayes model} \subsection{Sentiment Analysis} \subsubsection{VADER} diff --git a/document.toc b/document.toc index adea065..36a2a99 100644 --- a/document.toc +++ b/document.toc @@ -98,50 +98,42 @@ \defcounter {refsection}{0}\relax \contentsline {subsubsection}{Live Tweet Collection}{50}{section*.52} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Data processing}{55}{section*.53} +\contentsline {subsection}{Data pre-processing}{55}{section*.53} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{Preprocessing}{55}{section*.54} +\contentsline {subsubsection}{Spam Filtering}{57}{section*.54} \defcounter {refsection}{0}\relax -\contentsline {paragraph}{Text Cleaning}{55}{section*.56} +\contentsline {subsection}{Sentiment Analysis}{59}{section*.55} \defcounter {refsection}{0}\relax -\contentsline {paragraph}{Ngram based Language detection filtering}{55}{section*.57} +\contentsline {subsubsection}{VADER}{59}{section*.56} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{Spam Filtering}{55}{section*.58} +\contentsline {subsection}{Recurrent Neural Network - LSTM}{59}{section*.57} \defcounter {refsection}{0}\relax -\contentsline {paragraph}{Naive Bayes model}{57}{section*.59} +\contentsline {subsubsection}{Training and Testing Model}{59}{section*.58} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Sentiment Analysis}{57}{section*.60} +\contentsline {subsubsection}{Scoring and Validation}{59}{section*.59} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{VADER}{57}{section*.61} +\contentsline {subsection}{Future Prediction Forecasting}{59}{section*.60} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Recurrent Neural Network - LSTM}{57}{section*.62} +\contentsline {subsection}{Main File 'Main.py'}{59}{section*.61} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{Training and Testing Model}{57}{section*.63} +\contentsline {subsection}{Miscellaneous}{59}{section*.62} \defcounter {refsection}{0}\relax -\contentsline {subsubsection}{Scoring and Validation}{57}{section*.64} +\contentsline {section}{Testing Metrics and Accuracy}{60}{section*.63} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Future Prediction Forecasting}{58}{section*.65} +\contentsline {section}{Project Evaluation}{61}{section*.64} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Main File 'Main.py'}{58}{section*.66} +\contentsline {section}{Discussion: Contribution and Reflection}{61}{section*.65} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Miscellaneous}{58}{section*.67} +\contentsline {subsection}{Limitations}{61}{section*.66} \defcounter {refsection}{0}\relax -\contentsline {section}{Testing Metrics and Accuracy}{59}{section*.68} +\contentsline {section}{Conclusion and Future Improvements}{62}{section*.67} \defcounter {refsection}{0}\relax -\contentsline {section}{Project Evaluation}{60}{section*.69} +\contentsline {subsection}{Conclusion}{62}{section*.68} \defcounter {refsection}{0}\relax -\contentsline {section}{Discussion: Contribution and Reflection}{60}{section*.70} +\contentsline {subsection}{Future Improvements}{62}{section*.69} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Limitations}{60}{section*.71} +\contentsline {section}{Appendices}{67}{section*.71} \defcounter {refsection}{0}\relax -\contentsline {section}{Conclusion and Future Improvements}{61}{section*.72} +\contentsline {subsection}{Appendix A - Project Initiation Document}{67}{section*.72} \defcounter {refsection}{0}\relax -\contentsline {subsection}{Conclusion}{61}{section*.73} -\defcounter {refsection}{0}\relax -\contentsline {subsection}{Future Improvements}{61}{section*.74} -\defcounter {refsection}{0}\relax -\contentsline {section}{Appendices}{66}{section*.76} -\defcounter {refsection}{0}\relax -\contentsline {subsection}{Appendix A - Project Initiation Document}{66}{section*.77} -\defcounter {refsection}{0}\relax -\contentsline {subsection}{Appendix B - Log book}{79}{section*.78} +\contentsline {subsection}{Appendix B - Log book}{80}{section*.73} diff --git a/report.bib b/report.bib index 6b7ef7b..0d42c66 100644 --- a/report.bib +++ b/report.bib @@ -348,4 +348,14 @@ year={}, organization={Bitcoin Charts}, url={http://api.bitcoincharts.com/v1/csv/} +} + +@inproceedings{36, + title={Detecting Text Language With Python and NLTK}, + author={Alejandro Nolla}, + booktitle={}, + pages={}, + year={}, + organization={Alejandro Nolla Blog}, + url={http://blog.alejandronolla.com/2013/05/15/detecting-text-language-with-python-and-nltk/} } \ No newline at end of file