HW1 Pull

Makefile

@@ -0,0 +1,11 @@
+PLACE_HOLDER=<\!--markdown goes here-->
+SLIDE_SOURCES=$(wildcard slides/*.markdown)
+SLIDES=$(SLIDE_SOURCES:.markdown=.html)
+
+all: index.html $(SLIDES)
+
+index.html: README.markdown
+	pandoc README.markdown -t html -c slides/production/common.css > index.html
+
+%.html: %.markdown slides/slide_template.html
+	sed -e "/$(PLACE_HOLDER)/r $<" < slides/slide_template.html | sed -e "s/$(PLACE_HOLDER)//" > $@

README.markdown

@@ -0,0 +1,43 @@
+# Data Mining 290
+
+### Description
+Learn how to obtain, clean, visualize, understand, model, and
+predict the world around you using data. Grading will consist of homework
+(30%), a midterm (30%), and a project (40%).
+
+### Instructor
+Jimmy Retzlaff <jretz@ischool>
+
+### GSI
+Shreyas <shreyas@ischool>
+
+### Textbook
+Han, J., Kamber, M., & Pei, J. (2011). _Data Mining: Concepts and Techniques_, Third Edition *(3rd ed.)*. Morgan Kaufmann.
+
+### Course Discussion
+[Info 290T: Data Mining on Piazza](https://piazza.com/berkeley/spring2014/info290t03)
+
+---
+
+# Syllabus
+DM[0-9]+ indicates chapters from the text, _Data Mining_.
+
+| Date   | Readings                                                                                                                                                                                                                                   | Slides                                                                                                                                                 | Homework / Project                                                                                                      |
+|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| Jan 23 | [Try Github](http://try.github.com) ; [A Taxonomy of Data Science](http://www.dataists.com/2010/09/a-taxonomy-of-data-science/)                                                                                                            | [Class Intro](slides/2014-01-23-Intro.html) ; [Tools Intro](https://speakerdeck.com/seekshreyas/introduction-to-git-and-github) by GUEST: Shreyas      | [Git Intro](slides/2014-01-23-Lab.html)                                                                                 |
+| Jan 30 | DM1 ; [The Yelp Factor: Are Consumer Reviews Good for Business?](http://hbswk.hbs.edu/item/6836.html)                                                                                                                                      | [Case Studies](slides/2014-01-30-CaseStudies.html) ; [Obtaining Data](slides/2014-01-30-Obtaining-Data.html)                                           | [Obtain & Explore Data](slides/2014-01-30-Lab.html)                                                                     |
+| Feb 6  | DM2, DM3                                                                                                                                                                                                                                   | [Probability](slides/2014-02-06-Probability.html) ; [Preprocessing](slides/2014-02-06-Preprocessing.html)                                              | [Data Stats](slides/2014-02-06-Lab.html)                                                                                |
+| Feb 13 | DM4, [Apache Hadoop: Petabytes and Terawatts](http://www.youtube.com/watch?v=SS27F-hYWfU) ([slides](http://prezi.com/u0ukvqzpyh5p/apache-hadoop-petabytes-and-terawatts/)); [mrjob docs](http://packages.python.org/mrjob/) (for homework) | [Data Warehouse](slides/2014-02-13-Data-Warehouse.html) ; [MapReduce](slides/2014-02-13-MapReduce.html)                                                | [Project Details](slides/2014-02-13-Project.html) ; [mrjob](slides/2014-02-13-mrjob.html)                               |
+| Feb 20 | DM8                                                                                                                                                                                                                                        | [Decision Trees](slides/2014-02-20-Decision-Trees.html); [Naive Bayes](slides/2014-02-20-Bayes.html)                                                   | [Gini Index](slides/2014-02-20-Gini.html)                                                                               |
+| Feb 27 | DM[9.1-9.3], 9.5 ; [Understanding the Bias-Variance Tradeoff](http://scott.fortmann-roe.com/docs/BiasVariance.html)                                                                                                                        | [SVM](slides/2014-02-27-SVM.html) ; [Neural Networks](slides/2014-02-27-Neural-Network.html)                                                           | [Neural Network Back Propagation](slides/2014-02-27-Lab-NN.html)                                                        |
+| Mar 6  | DM10                                                                                                                                                                                                                                       | [Clustering - Partitioning](slides/2014-03-06-Clustering.html) ; [Clustering - Hierarchical & Density](slides/2014-03-06-Hierarchical.html)            | [K-Means](slides/2014-03-06-k-means.html)                                                                               |
+| Mar 13 | DM11.1                                                                                                                                                                                                                                     | [Review](slides/2014-03-13-Review.html)                                                                                                                | prepare 1 cheat sheet                                                                                                   |
+| Mar 20 | 1 cheat sheet                                                                                                                                                                                                                              | *Midterm*                                                                                                                                              |                                                                                                                         |
+| Mar 27 | HOLIDAY                                                                                                                                                                                                                                    |                                                                                                                                                        |                                                                                                                         |
+| Apr 3  | DM6                                                                                                                                                                                                                                        | [Advanced Clustering](slides/2014-03-13-Advanced-Cluster.html) ; [Frequent Patterns](slides/2014-04-03-Frequent-Pattern.html)                          | [AWS](slides/2014-04-03-AWS.html) ; Project Proposal due April 9                                                        |
+| Apr 10 | DM11.3; [PageRank](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf); [Uncovering Social Network Sybils in the Wild](http://arxiv.org/pdf/1106.5321)                                                                                      | [Graphs](slides/2014-04-10-Graphs.html); [PageRank](slides/2014-04-10-PageRank.html)                                                                   | [Adjacency Representations](slides/2014-04-10-AdjacencyRepresentations.html)                                            |
+| Apr 17 | DM12; [Shazam Audio Search](http://www.ee.columbia.edu/~dpwe/papers/Wang03-shazam.pdf)                                                                                                                                                     | [Outliers](slides/2014-04-17-Outliers.html); [Images & Audio](slides/2014-04-17-Multimedia.html)                                                       | [Midterm Review](slides/2014-04-17-Midterm-HW.html)                                                                     |
+| Apr 24 | [Embedded Plots](https://groups.google.com/group/gsofgs/attach/2f1cdd7a999c3ad8/embedded-plots.pdf?part=2&authuser=0) ; [Data-Driven Documents](http://vis.stanford.edu/files/2011-D3-InfoVis.pdf)                                         | [Visualization](slides/2014-04-24-Visualization.html) ; [Yelp's Visualizations](slides/2014-04-24-Yelp-Visualization.html)                             | [D3 Intro](http://vogievetsky.github.io/IntroD3/); [D3 Lab](slides/2014-04-24-D3.html)                                  |
+| May 1  | [A Few Useful Things to Know about Machine Learning](http://homes.cs.washington.edu/~pedrod/papers/cacm12.pdf) ; [Top 10 Algorithms in Data Mining](http://www.cs.uvm.edu/~icdm/algorithms/10Algorithms-08.pdf)                            | [In Real Life](slides/2014-05-01-Real-World.html)                                                                                                      | Project Data and Presentation due May 8th                                                                               |
+| May 8  |                                                                                                                                                                                                                                            | Final Presentation                                                                                                                                     | Project Code & Papers due May 14th                                                                                      |
+| May 15 |                                                                                                                                                                                                                                            |                                                                                                                                                        | Bye!                                                                                                                    |

code/gini.py

@@ -6,22 +6,32 @@
 import fileinput
 import csv
 
-(cmte_id, cand_id, cand_nm, contbr_nm, contbr_city, contbr_st, contbr_zip,
-contbr_employer, contbr_occupation, contb_receipt_amt, contb_receipt_dt,
-receipt_desc, memo_cd, memo_text, form_tp, file_num, tran_id, election_tp) = range(18)
-
+(
+    CMTE_ID, AMNDT_IND, RPT_TP, TRANSACTION_PGI, IMAGE_NUM, TRANSACTION_TP,
+    ENTITY_TP, NAME, CITY, STATE, ZIP_CODE, EMPLOYER, OCCUPATION,
+    TRANSACTION_DT, TRANSACTION_AMT, OTHER_ID, CAND_ID, TRAN_ID, FILE_NUM,
+    MEMO_CD, MEMO_TEXT, SUB_ID
+) = range(22)
+
+CANDIDATES = {
+    'P80003338': 'Obama',
+    'P80003353': 'Romney',
+}
 
 ############### Set up variables
 # TODO: declare datastructures
 
 ############### Read through files
-for row in csv.reader(fileinput.input()):
-    if not fileinput.isfirstline():
-        ###
-        # TODO: replace line below with steps to save information to calculate
-        # Gini Index
-        row[cand_nm], row[contbr_zip]
-        ##/
+for row in csv.reader(fileinput.input(), delimiter='|'):
+    candidate_id = row[CAND_ID]
+    if candidate_id not in CANDIDATES:
+        continue
+
+    candidate_name = CANDIDATES[candidate_id]
+    zip_code = row[ZIP_CODE]
+    ###
+    # TODO: save information to calculate Gini Index
+    ##/
 
 ###
 # TODO: calculate the values below: