Modify UnicodeDecodeError text. You'll use utf-8

Author: neoneo40

ch05/classify.py

@@ -54,6 +54,7 @@ def prepare_sent_features():
         if not text:
             meta[pid]['AvgSentLen'] = meta[pid]['AvgWordLen'] = 0
         else:
+            text = text.decode('utf-8')
             sent_lens = [len(nltk.word_tokenize(
                 sent)) for sent in nltk.sent_tokenize(text)]
             meta[pid]['AvgSentLen'] = np.mean(sent_lens)