Node demo fix, Readme

Author: arnicas

4. Naive Bayes Classification.ipynb

@@ -1,7 +1,7 @@
 {
  "metadata": {
   "name": "",
-  "signature": "sha256:ad3e77c84b6a81dc5eb1f4aab46bfb3ce903969463e671f758759e3e251a1e38"
+  "signature": "sha256:0df1e645a00490baf1a4cd243e17fdb996e55ad258e665d8b406f4982d82660f"
  },
  "nbformat": 3,
  "nbformat_minor": 0,
@@ -21,7 +21,7 @@
      "cell_type": "markdown",
      "metadata": {},
      "source": [
-      "This is an example of going from labeled text to machine classification, first with NLTK and then the Python machine learning library scikit-learn. Examples updated from my OpenVis Conf talk here, which is more entertaining: https://www.youtube.com/watch?v=f41U936WqPM\n",
+      "This is an example of going from labeled text to machine classification, first with NLTK and then the Python machine learning library scikit-learn. Examples updated from my OpenVis Conf talk here, which is more entertaining: https://www.youtube.com/watch?v=f41U936WqPM and slides: http://www.slideshare.net/arnicas/the-bones-of-a-bestseller\n",
       "\n",
       "###Warning: Rated NC-17. Using text samples from \"50 Shades of Gray\"! (Because spam is boring.)###\n",
       "\n"
@@ -719,6 +719,10 @@
      "source": [
       "###Perkins discusses some of the differences in performance in his book. Basically, for machine learning problems, sklearn is highly optimized.\n",
       "\n",
+      "Let's look at a visualization of the accuracy of one of the runs, from my [Openvis Conf talk](http://www.slideshare.net/arnicas/the-bones-of-a-bestseller):\n",
+      "\n",
+      "http://www.ghostweather.com/essays/talks/openvisconf/text_scores/rollover.html\n",
+      "\n",
       "**Now, optionally, you can look at notebook 5 on using sklearn to do the same things!**"
      ]
     },

README.md

@@ -1,2 +1,24 @@
 # NLP-in-Python
-Intro to some NLP concepts in Python for a class
+
+Intro to some NLP concepts and libraries in Python for a class at CMU,
+Feb 2015.
+
+Lots of libraries are required - see [here](Installation.md) for install info.
+
+Notebook viewer links:
+
+0. [Reading in Files](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/0.%20Reading%20Files.ipynb)
+
+1. [Tokenizing, Stemming, POS](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/1.Tokenizing%2C%20Stemming%2C%20POS.ipynb)
+
+2. [Wordclouds](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/2.%20WordClouds.ipynb)
+
+3. [TF-IDF, Clustering, Pattern](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/3.%20TF-IDF%2C%20Clustering%2C%20Pattern.ipynb)
+
+Doing some TF-IDF NLP in node's package "natural" (but caveats apply):[here](utils/booksNodeTfIdf.js)
+
+4. [Naive Bayes Classification](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/4.%20Naive%20Bayes%20Classification.ipynb) - the infamous 50 Shades Sex Scene Detector
+
+5. [Naive Bayes in Scikit-Learn](http://nbviewer.ipython.org/github/arnicas/NLP-in-Python/blob/master/5.%20Naive%20Bayes%20in%20Scikit-Learn.ipynb)
+
+

utils/booksNodeTfIdf.js

@@ -1,25 +1,33 @@
 
-/*
+/************
 
-This uses code from the natural package, particularly:
+Lynn Cherny, 2/2015:
+
+This uses code from the node natural package, particularly:
 https://github.com/NaturalNode/natural/blob/master/lib/natural/tfidf/tfidf.js
 which implements tf-idf in its own way and removes stopwords, apparently.
+
 https://github.com/NaturalNode/natural/blob/master/lib/natural/util/stopwords.js
 
-*/
+To run this, you need to 
+>npm install natural
+> node booksNodeTfIdf.js <files dir without trailing / ! > <word>
+
+*************/
 
 var natural = require('natural'),
     TfIdf = natural.TfIdf,
     tfidf = new TfIdf();
 
 // Borrowed from Shiffman's node examples for his AtoZ Programming class
-// We can get command line arguments in a node program
+// We can get command line arguments in a node program:
 if (process.argv.length < 4) {
   console.log('Oops, you forgot to pass in a directory path and file to evaluate.');
   process.exit(1);
 }
 
-// The 'fs' (file system) module allows us to read and write files
+// The 'fs' (file system) module allows us to read and write files:
+
 var fs = require('fs');
 // A path for all the files of data
 var path = process.argv[2];
@@ -31,30 +39,33 @@ var files = fs.readdirSync(path);
 // Total word count of document
 var totalwords = 0;
 
+function printTopN(docNumber, N) {
+  var sorted = tfidf.listTerms(docNumber).sort(function(a, b) {
+    return b.tfidf - a.tfidf;
+    }).slice(0,N);
+
+  console.log("Top terms for doc " + docNumber + ":")
+  sorted.forEach(function (a) {
+    console.log("  ", a.term, a.tfidf);
+  });
+}
+
+// Add each of the files to the model
+
 files.forEach(function(d,i) {
   console.log('Doc #' + i + ' is ' + path + '/' + d);
   tfidf.addFileSync(path + '/' + d);
 });
 
 setTimeout(function () {
-  // doing it in a timeout because otherwise the files.each above may not have finished
+  // doing it in a timeout because otherwise the files.each above may not
+  // have finished
   files.forEach(function(d,i) {
     printTopN(i, 6);
   });
-    console.log('Looking up tfidf for: ' + word);
+  console.log('Looking up tfidf for: ' + word);
   tfidf.tfidfs(word, function(i, measure) {
-      console.log('document #' + i + ' is ' + measure);
+      console.log('document #' + i + ' with ' + measure);
   });
 }, 500);
 
-function printTopN(docNumber, N) {
-  var sorted = tfidf.listTerms(docNumber).sort(function(a, b) {
-    return b.tfidf - a.tfidf;
-    }).slice(0,N);
-
-  console.log("Top terms for doc " + docNumber + ":")
-  sorted.forEach(function (a) {
-    console.log("  ", a.term, a.tfidf);
-  });
-}
-