diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cb841c2 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +PLACE_HOLDER=<\!--markdown goes here--> +SLIDE_SOURCES=$(wildcard slides/*.markdown) +SLIDES=$(SLIDE_SOURCES:.markdown=.html) + +all: index.html $(SLIDES) + +index.html: README.markdown + pandoc README.markdown -t html -c slides/production/common.css > index.html + +%.html: %.markdown slides/slide_template.html + sed -e "/$(PLACE_HOLDER)/r $<" < slides/slide_template.html | sed -e "s/$(PLACE_HOLDER)//" > $@ diff --git a/README.markdown b/README.markdown new file mode 100644 index 0000000..9040087 --- /dev/null +++ b/README.markdown @@ -0,0 +1,43 @@ +# Data Mining 290 + +### Description +Learn how to obtain, clean, visualize, understand, model, and +predict the world around you using data. Grading will consist of homework +(30%), a midterm (30%), and a project (40%). + +### Instructor +Jimmy Retzlaff <jretz@ischool> + +### GSI +Shreyas <shreyas@ischool> + +### Textbook +Han, J., Kamber, M., & Pei, J. (2011). _Data Mining: Concepts and Techniques_, Third Edition *(3rd ed.)*. Morgan Kaufmann. + +### Course Discussion +[Info 290T: Data Mining on Piazza](https://piazza.com/berkeley/spring2014/info290t03) + +--- + +# Syllabus +DM[0-9]+ indicates chapters from the text, _Data Mining_. + +| Date | Readings | Slides | Homework / Project | +|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| +| Jan 23 | [Try Github](http://try.github.com) ; [A Taxonomy of Data Science](http://www.dataists.com/2010/09/a-taxonomy-of-data-science/) | [Class Intro](slides/2014-01-23-Intro.html) ; [Tools Intro](https://speakerdeck.com/seekshreyas/introduction-to-git-and-github) by GUEST: Shreyas | [Git Intro](slides/2014-01-23-Lab.html) | +| Jan 30 | DM1 ; [The Yelp Factor: Are Consumer Reviews Good for Business?](http://hbswk.hbs.edu/item/6836.html) | [Case Studies](slides/2014-01-30-CaseStudies.html) ; [Obtaining Data](slides/2014-01-30-Obtaining-Data.html) | [Obtain & Explore Data](slides/2014-01-30-Lab.html) | +| Feb 6 | DM2, DM3 | [Probability](slides/2014-02-06-Probability.html) ; [Preprocessing](slides/2014-02-06-Preprocessing.html) | [Data Stats](slides/2014-02-06-Lab.html) | +| Feb 13 | DM4, [Apache Hadoop: Petabytes and Terawatts](http://www.youtube.com/watch?v=SS27F-hYWfU) ([slides](http://prezi.com/u0ukvqzpyh5p/apache-hadoop-petabytes-and-terawatts/)); [mrjob docs](http://packages.python.org/mrjob/) (for homework) | [Data Warehouse](slides/2014-02-13-Data-Warehouse.html) ; [MapReduce](slides/2014-02-13-MapReduce.html) | [Project Details](slides/2014-02-13-Project.html) ; [mrjob](slides/2014-02-13-mrjob.html) | +| Feb 20 | DM8 | [Decision Trees](slides/2014-02-20-Decision-Trees.html); [Naive Bayes](slides/2014-02-20-Bayes.html) | [Gini Index](slides/2014-02-20-Gini.html) | +| Feb 27 | DM[9.1-9.3], 9.5 ; [Understanding the Bias-Variance Tradeoff](http://scott.fortmann-roe.com/docs/BiasVariance.html) | [SVM](slides/2014-02-27-SVM.html) ; [Neural Networks](slides/2014-02-27-Neural-Network.html) | [Neural Network Back Propagation](slides/2014-02-27-Lab-NN.html) | +| Mar 6 | DM10 | [Clustering - Partitioning](slides/2014-03-06-Clustering.html) ; [Clustering - Hierarchical & Density](slides/2014-03-06-Hierarchical.html) | [K-Means](slides/2014-03-06-k-means.html) | +| Mar 13 | DM11.1 | [Review](slides/2014-03-13-Review.html) | prepare 1 cheat sheet | +| Mar 20 | 1 cheat sheet | *Midterm* | | +| Mar 27 | HOLIDAY | | | +| Apr 3 | DM6 | [Advanced Clustering](slides/2014-03-13-Advanced-Cluster.html) ; [Frequent Patterns](slides/2014-04-03-Frequent-Pattern.html) | [AWS](slides/2014-04-03-AWS.html) ; Project Proposal due April 9 | +| Apr 10 | DM11.3; [PageRank](http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf); [Uncovering Social Network Sybils in the Wild](http://arxiv.org/pdf/1106.5321) | [Graphs](slides/2014-04-10-Graphs.html); [PageRank](slides/2014-04-10-PageRank.html) | [Adjacency Representations](slides/2014-04-10-AdjacencyRepresentations.html) | +| Apr 17 | DM12; [Shazam Audio Search](http://www.ee.columbia.edu/~dpwe/papers/Wang03-shazam.pdf) | [Outliers](slides/2014-04-17-Outliers.html); [Images & Audio](slides/2014-04-17-Multimedia.html) | [Midterm Review](slides/2014-04-17-Midterm-HW.html) | +| Apr 24 | [Embedded Plots](https://groups.google.com/group/gsofgs/attach/2f1cdd7a999c3ad8/embedded-plots.pdf?part=2&authuser=0) ; [Data-Driven Documents](http://vis.stanford.edu/files/2011-D3-InfoVis.pdf) | [Visualization](slides/2014-04-24-Visualization.html) ; [Yelp's Visualizations](slides/2014-04-24-Yelp-Visualization.html) | [D3 Intro](http://vogievetsky.github.io/IntroD3/); [D3 Lab](slides/2014-04-24-D3.html) | +| May 1 | [A Few Useful Things to Know about Machine Learning](http://homes.cs.washington.edu/~pedrod/papers/cacm12.pdf) ; [Top 10 Algorithms in Data Mining](http://www.cs.uvm.edu/~icdm/algorithms/10Algorithms-08.pdf) | [In Real Life](slides/2014-05-01-Real-World.html) | Project Data and Presentation due May 8th | +| May 8 | | Final Presentation | Project Code & Papers due May 14th | +| May 15 | | | Bye! | diff --git a/README.org b/README.org deleted file mode 100644 index b501fb3..0000000 --- a/README.org +++ /dev/null @@ -1,48 +0,0 @@ -* Data Mining 290 :slide: - + Description :: Learn how to obtain, clean, visualize, understand, model, and - predict the world around you using data. Grading will consist of homework - (30%), midterm (30%), project (40%). - + Instructor :: Jim Blomo - + GSI :: Shreyas - + Textbook :: Han, J., Kamber, M., & Pei, J. (2011). _Data Mining: Concepts and Techniques_, Third Edition *(3rd ed.)*. Morgan Kaufmann. - - -* Syllabus :slide: -DM[0-9]+ indicates chapters from the text, _Data Mining_. - -| Date | Readings | Slides | Homework / Project | -|------+----------+--------+--------------------| -| Jan 25 | [[http://try.github.com][Try Github]] ; [[http://www.dataists.com/2010/09/a-taxonomy-of-data-science/][A Taxonomy of Data Science]] | [[file:slides/2013-01-25-Intro.html][Class Intro]] ; Tools Intro by /GUEST: Shreyas/ | [[ https://github.com/seekshreyas/Introduction-to-Git-Github][Git Intro]] | -| Feb 1 | DM1 ; [[http://hbswk.hbs.edu/item/6836.html][The Yelp Factor: Are Consumer Reviews Good for Business?]] | [[file:slides/2013-02-01-CaseStudies.html][Case Studies]] ; [[file:slides/2013-02-01-Obtaining-Data.html][Obtaining Data]] | [[file:slides/2013-02-01-Lab.html][Obtain & Explore Data]] | -| Feb 8 | DM2, DM3 | [[file:slides/2013-02-08-Probability.html][Probability]] ; [[file:slides/2013-02-08-Preprocessing.html][Preprocessing]] | [[file:slides/2013-02-08-Lab.html][Data Stats]] | -| Feb 15 | DM4, [[http://www.youtube.com/watch?v=SS27F-hYWfU][Apache Hadoop: Petabytes and Terawatts]] ([[http://prezi.com/u0ukvqzpyh5p/apache-hadoop-petabytes-and-terawatts/][slides]]); [[http://packages.python.org/mrjob/][mrjob docs]] (for homework) | [[file:slides/2013-02-15-Data-Warehouse.html][Data Warehouse]] ; [[file:slides/2013-02-15-MapReduce.html][MapReduce]] | [[file:slides/2013-02-15-Project.html][Project Details]] ; [[file:slides/2013-02-15-mrjob.html][mrjob]] | -| Feb 22 | DM8 | [[file:slides/2013-02-22-Decision-Trees.html][Decision Trees]]; [[file:slides/2013-02-22-Bayes.html][Naive Bayes]] | [[file:slides/2013-02-22-Gini.html][Gini Index]] | -| Mar 1 | DM[9.1-9.3], 9.5 ; [[http://scott.fortmann-roe.com/docs/BiasVariance.html][Understanding the Bias-Variance Tradeoff]] | [[file:slides/2013-03-01-SVM.html][SVM]] ; [[file:slides/2013-03-01-Neural-Network.html][Neural Networks]] | [[file:slides/2013-03-01-Lab-NN.html][Neural Network Back Propagation]] | -| Mar 8 | DM10 | [[file:slides/2013-03-07-Clustering.html][Agglomerative - Clustering]] ; [[file:slides/2013-03-07-Hierarchical.html][Hierarchical, Density - Clustering]] | [[file:slides/2013-03-07-k-means.html][K-Means]] | -| Mar 15 | DM11.1 | [[file:slides/2013-03-15-Review.html][Review]] | prepare 1 cheat sheet | -| Mar 22 | 1 cheat sheet | *Midterm* | - | -| Mar 29 | HOLIDAY -| Apr 5 | DM6 | [[file:slides/2013-03-15-Advanced-Cluster.html][Advanced Clustering]] ; [[file:slides/2013-04-05-Frequent-Pattern.html][Frequent Pattern]] | [[file:slides/2013-04-05-AWS.html][AWS]] ; Project Proposal Due | -| Apr 12 | DM11.3; [[http://ilpubs.stanford.edu:8090/422/1/1999-66.pdf][PageRank]]; [[http://arxiv.org/pdf/1106.5321][Uncovering Social Network Sybils in the Wild]] | [[file:slides/2013-04-12-Graphs.html][Graphs]]; [[file:slides/2013-04-12-PageRank.html][PageRank]] | [[file:slides/2013-04-12-AdjacencyRepresentations.html][Adjacency Representations]] | -| Apr 19 | [[file:slides/2013-04-19-Nonlinear.pdf][Non-linear regression]] | GUEST: Gene Lee Ceaser's [[file:slides/RM Pricing Strategy.ppt][Pricing Strategy]]; [[file:slides/Campus Recruiting Deck_2012_UC Berkeley.ppt][Ceaser's Recruiting]]| [[file:slides/2013-04-19-Elasticity.html][Price Elasticity]] | -| Apr 26 | DM12; [[http://www.ee.columbia.edu/~dpwe/papers/Wang03-shazam.pdf][Shazam Audio Search]] | [[file:slides/2013-04-26-Outliers.html][Outliers]]; [[file:slides/2013-04-26-Multimedia.html][Images & Audio]] | [[file:slides/2013-04-26-Midterm-HW.html][Midterm Review]] | -| May 3 | [[https://groups.google.com/group/gsofgs/attach/2f1cdd7a999c3ad8/embedded-plots.pdf?part=2&authuser=0][Embedded Plots]] ; [[http://vis.stanford.edu/files/2011-D3-InfoVis.pdf][Data-Driven Documents]]| [[file:slides/2013-05-03-Visualization.html][Visualization]] ; [[file:slides/2013-05-03-Yelp-Visualization.html][Yelp's Visualizations]] | [[http://vogievetsky.github.io/IntroD3/][D3 Intro]] [[file:slides/2013-05-03-D3.html][D3 Lab]] | -| May 10 | [[http://homes.cs.washington.edu/~pedrod/papers/cacm12.pdf][A Few Useful Things to Know about Machine Learning]] ; [[http://www.cs.uvm.edu/~icdm/algorithms/10Algorithms-08.pdf][Top 10 Algorithms in Data Mining]] | [[file:slides/2013-05-10-Real-World.html][In Real Life]] ; Presentations | May 16th: Project Papers Due | -| May 17 | - | Final Presentation | Bye! | - - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -Fork me on GitHub -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/code/gini.py b/code/gini.py index d164946..8cb8734 100755 --- a/code/gini.py +++ b/code/gini.py @@ -6,22 +6,32 @@ import fileinput import csv -(cmte_id, cand_id, cand_nm, contbr_nm, contbr_city, contbr_st, contbr_zip, -contbr_employer, contbr_occupation, contb_receipt_amt, contb_receipt_dt, -receipt_desc, memo_cd, memo_text, form_tp, file_num, tran_id, election_tp) = range(18) - +( + CMTE_ID, AMNDT_IND, RPT_TP, TRANSACTION_PGI, IMAGE_NUM, TRANSACTION_TP, + ENTITY_TP, NAME, CITY, STATE, ZIP_CODE, EMPLOYER, OCCUPATION, + TRANSACTION_DT, TRANSACTION_AMT, OTHER_ID, CAND_ID, TRAN_ID, FILE_NUM, + MEMO_CD, MEMO_TEXT, SUB_ID +) = range(22) + +CANDIDATES = { + 'P80003338': 'Obama', + 'P80003353': 'Romney', +} ############### Set up variables # TODO: declare datastructures ############### Read through files -for row in csv.reader(fileinput.input()): - if not fileinput.isfirstline(): - ### - # TODO: replace line below with steps to save information to calculate - # Gini Index - row[cand_nm], row[contbr_zip] - ##/ +for row in csv.reader(fileinput.input(), delimiter='|'): + candidate_id = row[CAND_ID] + if candidate_id not in CANDIDATES: + continue + + candidate_name = CANDIDATES[candidate_id] + zip_code = row[ZIP_CODE] + ### + # TODO: save information to calculate Gini Index + ##/ ### # TODO: calculate the values below: diff --git a/code/k_means.py b/code/k_means.py index 36ce300..d0c1344 100644 --- a/code/k_means.py +++ b/code/k_means.py @@ -3,8 +3,18 @@ # ##/ -dataset = [-13.65089255716321, -0.5409562932238607, -88.4726466247223, 39.30158828358612, 4.066458182574449, 64.64143300482378, 38.68269424751338, 33.42013676314311, 31.18603331719732, -0.2027616409406292, 45.13590038987272, 30.791899783552395, 61.1727490302448, 18.167220741624856, 88.88077709786394, -1.3808002119514704, 50.14991362212521, 55.92029956281276, -6.759813255299466, 34.28290084421072] -k = 2 # number of clusters +dataset = [ + -13.65089255716321, -0.5409562932238607, -88.4726466247223, + 39.30158828358612, 4.066458182574449, 64.64143300482378, + 38.68269424751338, 33.42013676314311, 31.18603331719732, + -0.2027616409406292, 45.13590038987272, 30.791899783552395, + 61.1727490302448, 18.167220741624856, 88.88077709786394, + -1.3808002119514704, 50.14991362212521, 55.92029956281276, + -6.759813255299466, 34.28290084421072 +] + +k = 2 # number of clusters + ### # Helper functions @@ -15,9 +25,10 @@ def pick_centroids(xs, num): """Return list of num centroids given a list of numbers in xs""" ### # TODO select and return centroids - return [1,2] + return [1, 2] ##/ + def distance(a, b): """Return the distance of numbers a and b""" ### @@ -25,6 +36,7 @@ def distance(a, b): return 0 ##/ + def centroid(xs): """Return the centroid number given a list of numbers, xs""" ### @@ -32,6 +44,7 @@ def centroid(xs): return 0 ##/ + def cluster(xs, centroids): """Return a list of clusters centered around the given centroids. Clusters are lists of numbers.""" @@ -40,13 +53,15 @@ def cluster(xs, centroids): for x in xs: # find the closest cluster to x - dist, cluster_id = min((distance(x, c), cluster_id) - for cluster_id, c in enumerate(centroids)) + dist, cluster_id = min( + (distance(x, c), cluster_id) for cluster_id, c in enumerate(centroids) + ) # place x in cluster clusters[cluster_id].append(x) return clusters + def iterate_centroids(xs, centroids): """Return stable centroids given a dataset and initial centroids""" @@ -77,5 +92,3 @@ def iterate_centroids(xs, centroids): for centroid, cluster in zip(final_centroids, final_clusters): print "Centroid: %s" % centroid print "Cluster contents: %r" % cluster - - diff --git a/code/review_word_count.py b/code/review_word_count.py index b49aef1..1c92f1a 100644 --- a/code/review_word_count.py +++ b/code/review_word_count.py @@ -5,6 +5,7 @@ WORD_RE = re.compile(r"[\w']+") + class ReviewWordCount(MRJob): INPUT_PROTOCOL = JSONValueProtocol @@ -24,7 +25,10 @@ def steps(self): extract_words: => count_words: => """ - return [self.mr(self.extract_words, self.count_words)] + return [ + self.mr(self.extract_words, self.count_words), + ] + if __name__ == '__main__': ReviewWordCount.run() diff --git a/code/stats.py b/code/stats.py index a67a59f..b85fb0c 100755 --- a/code/stats.py +++ b/code/stats.py @@ -1,15 +1,17 @@ #!/usr/bin/python """This script can be used to analyze data in the 2012 Presidential Campaign, -available from http://www.fec.gov/disclosurep/PDownload.do""" +available from ftp://ftp.fec.gov/FEC/2012/pas212.zip - data dictionary is at +http://www.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml +""" import fileinput import csv -total = 0 +total = 0.0 -for row in csv.reader(fileinput.input()): +for row in csv.reader(fileinput.input(), delimiter='|'): if not fileinput.isfirstline(): - total += float(row[9]) + total += float(row[14]) ### # TODO: calculate other statistics here # You may need to store numbers in an array to access them together @@ -29,7 +31,7 @@ # square root can be calculated with N**0.5 print "Standard Deviation: " -##### Comma separated list of unique candidate names +##### Comma separated list of unique candidate ID numbers print "Candidates: " def minmax_normalize(value): @@ -39,9 +41,8 @@ def minmax_normalize(value): # TODO: replace line below with the actual calculations norm = value ###/ - + return norm ##### Normalize some sample values print "Min-max normalized values: %r" % map(minmax_normalize, [2500, 50, 250, 35, 8, 100, 19]) - diff --git a/code/unique_review.py b/code/unique_review.py index b041679..c20c820 100644 --- a/code/unique_review.py +++ b/code/unique_review.py @@ -5,11 +5,12 @@ WORD_RE = re.compile(r"[\w']+") + class UniqueReview(MRJob): INPUT_PROTOCOL = JSONValueProtocol def extract_words(self, _, record): - """Take in a record, filter by type=review, yield """ + """Take in a record, yield """ if record['type'] == 'review': ### # TODO: for each word in the review, yield the correct key,value @@ -34,7 +35,7 @@ def count_unique_words(self, review_id, unique_word_counts): """Output the number of unique words for a given review_id""" ### # TODO: summarize unique_word_counts and output the result - # + # ##/ def aggregate_max(self, review_id, unique_word_count): @@ -61,9 +62,12 @@ def steps(self): reducer1: mapper2: ... """ - return [self.mr(self.extract_words, self.count_reviews), - self.mr(reducer=self.count_unique_words), - self.mr(self.aggregate_max, self.select_max)] + return [ + self.mr(self.extract_words, self.count_reviews), + self.mr(reducer=self.count_unique_words), + self.mr(self.aggregate_max, self.select_max), + ] + if __name__ == '__main__': UniqueReview.run() diff --git a/code/user_similarity.py b/code/user_similarity.py index 18329aa..596186b 100644 --- a/code/user_similarity.py +++ b/code/user_similarity.py @@ -1,12 +1,13 @@ from mrjob.job import MRJob from mrjob.protocol import JSONValueProtocol + class UserSimilarity(MRJob): INPUT_PROTOCOL = JSONValueProtocol ### # TODO: write the functions needed to - # 1) find potential matches, + # 1) find potential matches, # 2) calculate the Jaccard between users, with a user defined as a set of # reviewed businesses ##/ @@ -17,8 +18,10 @@ def steps(self): reducer1: mapper2: ... """ - return [self.mr(mapper=self.mapper1, reducer=self.reducer1), - self.mr(mapper=...)] + return [ + self.mr(mapper=self.mapper1, reducer=self.reducer1), + self.mr(mapper=...), + ] if __name__ == '__main__': diff --git a/index.html b/index.html index e6694e2..be71fab 100644 --- a/index.html +++ b/index.html @@ -1,119 +1,141 @@ - - - + + -README - - - - - - - - - - - - - - + + + + + + - -
- -
- -
-

README

- - -
-

Table of Contents

-
- -
-
- -
-

1 Data Mining 290    slide

-
- -
-
Description
Learn how to obtain, clean, visualize, understand, model, and - predict the world around you using data. Grading will consist of homework - (30%), midterm (30%), project (40%). -
-
Instructor
Jim Blomo <jblomo@ischool> -
-
GSI
Shreyas <shreyas@ischool> -
-
Textbook
Han, J., Kamber, M., & Pei, J. (2011). Data Mining: Concepts and Techniques, Third Edition (3rd ed.). Morgan Kaufmann. -
-
- - - -
- -
- -
-

2 Syllabus    slide

-
- -

DM[0-9]+ indicates chapters from the text, Data Mining. -

- - -- +

Data Mining 290

+

Description

+

Learn how to obtain, clean, visualize, understand, model, and predict the world around you using data. Grading will consist of homework (30%), a midterm (30%), and a project (40%).

+

Instructor

+

Jimmy Retzlaff <jretz@ischool>

+

GSI

+

Shreyas <shreyas@ischool>

+

Textbook

+

Han, J., Kamber, M., & Pei, J. (2011). Data Mining: Concepts and Techniques, Third Edition (3rd ed.). Morgan Kaufmann.

+

Course Discussion

+

Info 290T: Data Mining on Piazza

+
+

Syllabus

+

DM[0-9]+ indicates chapters from the text, Data Mining.

+
- + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DateReadingsSlidesHomework / Project
DateReadingsSlidesHomework / Project
Jan 25Try Github ; A Taxonomy of Data ScienceClass Intro ; Tools Intro by GUEST: ShreyasGit Intro
Feb 1DM1 ; The Yelp Factor: Are Consumer Reviews Good for Business?Case Studies ; Obtaining DataObtain & Explore Data
Feb 8DM2, DM3Probability ; PreprocessingData Stats
Feb 15DM4, Apache Hadoop: Petabytes and Terawatts (slides); mrjob docs (for homework)Data Warehouse ; MapReduceProject Details ; mrjob
Feb 22DM8Decision Trees; Naive BayesGini Index
Mar 1DM[9.1-9.3], 9.5 ; Understanding the Bias-Variance TradeoffSVM ; Neural NetworksNeural Network Back Propagation
Mar 8DM10Agglomerative - Clustering ; Hierarchical, Density - ClusteringK-Means
Mar 15DM11.1Reviewprepare 1 cheat sheet
Mar 221 cheat sheetMidterm-
Mar 29HOLIDAY
Apr 5DM6Advanced Clustering ; Frequent PatternAWS ; Project Proposal Due
Apr 12DM11.3; PageRank; Uncovering Social Network Sybils in the WildGraphs; PageRankAdjacency Representations
Apr 19Non-linear regressionGUEST: Gene Lee Ceaser's Pricing Strategy; Ceaser's RecruitingPrice Elasticity
Apr 26DM12; Shazam Audio SearchOutliers; Images & AudioMidterm Review
May 3Embedded Plots ; Data-Driven DocumentsVisualization ; Yelp's VisualizationsD3 Intro D3 Lab
May 10A Few Useful Things to Know about Machine Learning ; Top 10 Algorithms in Data MiningIn Real Life ; PresentationsMay 16th: Project Papers Due
May 17-Final PresentationBye!
Jan 23Try Github ; A Taxonomy of Data ScienceClass Intro ; Tools Intro by GUEST: ShreyasGit Intro
Jan 30DM1 ; The Yelp Factor: Are Consumer Reviews Good for Business?Case Studies ; Obtaining DataObtain & Explore Data
Feb 6DM2, DM3Probability ; PreprocessingData Stats
Feb 13DM4, Apache Hadoop: Petabytes and Terawatts (slides); mrjob docs (for homework)Data Warehouse ; MapReduceProject Details ; mrjob
Feb 20DM8Decision Trees; Naive BayesGini Index
Feb 27DM[9.1-9.3], 9.5 ; Understanding the Bias-Variance TradeoffSVM ; Neural NetworksNeural Network Back Propagation
Mar 6DM10Clustering - Partitioning ; Clustering - Hierarchical & DensityK-Means
Mar 13DM11.1Reviewprepare 1 cheat sheet
Mar 201 cheat sheetMidterm
Mar 27HOLIDAY
Apr 3DM6Advanced Clustering ; Frequent PatternsAWS ; Project Proposal due April 9
Apr 10DM11.3; PageRank; Uncovering Social Network Sybils in the WildGraphs; PageRankAdjacency Representations
Apr 17DM12; Shazam Audio SearchOutliers; Images & AudioMidterm Review
Apr 24Embedded Plots ; Data-Driven DocumentsVisualization ; Yelp's VisualizationsD3 Intro; D3 Lab
May 1A Few Useful Things to Know about Machine Learning ; Top 10 Algorithms in Data MiningIn Real LifeProject Data and Presentation due May 8th
May 8Final PresentationProject Code & Papers due May 14th
May 15Bye!
- - - - - - - -Fork me on GitHub - -
-
-
- -
-

Date: 2013-05-10 00:54:43 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
diff --git a/nn-train.txt b/nn-train.txt index 43977f1..9d0b82f 100644 --- a/nn-train.txt +++ b/nn-train.txt @@ -1,15 +1,17 @@ -err_1 = -err_2 = -err_3 = +err_6 = -0.11346127339699999 +err_5 = -0.0011326458827956695 err_4 = -err_5 = -err_6 = -w_13 = -w_14 = -w_15 = -w_23 = -w_24 = -w_25 = -w_36 = +err_3 = + +w_56 = 0.37298917134759924 w_46 = -w_56 = +w_36 = + +err_2 = +err_1 = +w_25 = +w_24 = +w_23 = +w_15 = +w_14 = +w_13 = diff --git a/slides/2013-01-25-Intro.html b/slides/2013-01-25-Intro.html deleted file mode 100644 index 11919c8..0000000 --- a/slides/2013-01-25-Intro.html +++ /dev/null @@ -1,887 +0,0 @@ - - - - -2013-01-25-Intro - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-01-25-Intro

- - - - -
-

1 Data Mining i290    slide

-
- -
    -
  • Jim Blomo & Shreyas -
    • -
- - -
- -
- -
-

2 Course Goals    slide

-
- -
    -
  • Extract information from data -
    • -
  • Understand techniques to find patterns -
    • -
  • Apply algorithms to real data sets -
    • -
- - -
- -
- -
-

3 We'll Do Stuff    slide

-
- -
    -
  • 30%: 10 Homework Assignments -
    • -
  • 30%: 1 Midterm -
    • -
  • 40%: 1 Project: Find, Mine, Report on Data -
    • -
- - -
- -
-

3.1 Homework Details    notes

-
- -
    -
  • HW due at midnight Thursday before class -
    • -
  • Each 24 hours late is 10% off -
    • -
  • HW will be turned in by GitHub pull request -
    • -
  • Project will be submitted by email & presentation -
    • -
- - -
-
- -
- -
-

4 But Don't Worry    slide

-
- -
    -
  • This isn't a programming class -
    • -
  • Grades are based on understanding of the concepts, not the craziest project -
    • -
  • Shreyas & I are here to help -
    • -
- - -
- -
-

4.1 Help    notes

-
- -
    -
  • We realize there's a wide range of technical skill -
    • -
  • We will help get anyone up to speed in these technical areas -
    • -
- - -
-
- -
- -
-

5 This is a Graduate class    slide

-
- -
    -
  • Perform well without supervision -
    • -
  • Readings from both book and online documentation -
    • -
  • TMTOWTDI -
    • -
  • Getting frameworks working on your computer -
    • -
- - -
- -
-

5.1 Style    notes

-
- -
    -
  • More firehouse than spoon feed, you'll need to follow up for - understanding -
    • -
  • Honor system: No copying code or answers. Helping each other with - concepts is encouraged, but document it. -
    • -
  • Everybody has a different workflow. We'll be covering the most basic. - Great if you want to do something different, but realize we may not be able - to help you as much. -
    • -
  • Non ISchool students should email student ID from EDU account to shreyas and - jblomo and we will get them ischool accounts. -
    • -
  • You may want to use other frameworks for your projects. Great! But again, - we may not be familiar with them -
    • -
- - -
-
- -
- -
-

6 Prerequisites    slide

-
- -
    -
  • Basic probability: P(A), P(A or B), P(A and B), P(A | B) -
    • -
  • Basic programming: Python -
    • -
  • Basic command line: SSH, downloading, copying large files, running programs - against data -
    • -
  • Textbook: Han, J., Kamber, M., & Pei, J. (2011). Data Mining: Concepts and Techniques, Third Edition (3rd ed.). Morgan Kaufmann. -
    • -
  • Technology will be available on ischool.berkeley.edu -
    • -
- - -
- -
-

6.1 Basics    notes

-
- -
    -
  • "Probability of A", "Probability of A or B" "A and B" "A given B" -
    • -
  • Most assignments filling in algorithm code -
    • -
  • Project you may use any language, though we suggest Python. -
    • -
  • We'll introduce any specific frameworks -
    • -
  • Command line: cp, mv, less… Imagine you have a 10G file, how are you - going to inspect the contents? -
    • -
- - -
-
- -
- -
-

7 Material    slide

-
- -
    -
  • Process: from find data to mining it to visualizing results -
    • -
  • Algorithms: all intuitively motivated, some rigorously studied -
    • -
  • Programming: using algorithms against data sets -
    • -
  • Discovery: finding information in self-defined project -
    • -
- - -
- -
-

7.1 What will we learn?    notes

-
- -
    -
  • Data mining not just about algorithms. We'll learn how to obtain, clean, - and store data. -
    • -
  • In real life, this is 70% of the job! -
    • -
  • We'll cover many different algorithms, and dive in depth on several of - them. But we're not going to get into any hairy math proofs -
    • -
  • Programming is the best way to precisely describe an algorithm. It is also - the way data mining is used in the real world. -
    • -
  • Your own project should emphasize your passion. Again, real world requires - you to grab data and squeeze information out of it without external help -
    • -
- - -
-
- -
- -
-

8 Lectures & Labs    slide

-
- -
    -
  • Start with Q&A for at least 10 minutes -
    • -
  • Expect to be asked a question -
    • -
  • Breaks -
    • -
  • Lab: Stick around and get the first question of HW done -
    • -
  • Slides on http://jblomo.github.com/datamining290/ -
    • -
- - -
- -
-

8.1 Helpful tips    notes

-
- -
    -
  • Helpful to me if you say your name -
    • -
  • Sorry, I tend to forget names -
    • -
  • If I am not calling on you, check to make sure you are on the class list! -
    • -
  • I'm not taking attendance, but let me know if you can't make it so I - won't call on you -
    • -
- - -
-
- -
- -
-

9 Office Hours    slide two_col

-
- -
    -
  • We'll stay after class -
    • -
  • or schedule a Skype call -
    • -
  • Piazza for questions and - announcements -
    • -
  • Wait list will be processed normally until 3rd week… then I'll accept - everyone who's participated in class if we have physical room -
    • -
- -

img/Office_Hours.png -

-
- -
-

9.1 Details    notes

-
- - - - -
-
- -
- -
-

10 Questions?    slide

-
- - -
- -
- -
-

11 Schedule    slide

-
- -

Available at GitHub Syllabus page -

    -
  • Jan 25 Class Intro ; Tools Intro by GUEST: Shreyas -
      -
    • lab: Git Intro -
      • -
    - -
    • -
  • Feb 1 Case Studies ; Obtaining Data -
    • -
  • Feb 8 Probability ; Preprocessing -
    • -
  • Feb 15 MapReduce, Data Warehouse -
    • -
  • Feb 22 Decision Trees; Naive Bayes -
    • -
  • Mar 1 SVM ; Neural Networks -
    • -
  • Mar 8 Clustering ; Review -
      -
    • lab: Project Proposal Due -
      • -
    - -
    • -
  • Mar 15 Midterm -
      -
    • lab: - -
      • -
    - -
    • -
  • Mar 21 Dimensionality Curse ; Graph Mining -
    • -
  • Mar 29 HOLIDAY -
    • -
  • Apr 5 Pattern ; Evaluations -
    • -
  • Apr 12 Collaborative Filtering; PageRank -
    • -
  • Apr 19 Feature Extraction ; Evaluation -
    • -
  • Apr 26 Images ; Audio -
    • -
  • May 3 Visualization ; HTML -
    • -
  • May 10 In Real Life ; Review -
      -
    • lab: - -
      • -
    - -
    • -
  • May 17 Final Presentation -
      -
    • lab: Bye! -
      • -
    - -
    • -
- - -
- -
- -
-

12 Hi, I'm Jim Blomo    slide two_col

-
- -

*Hello Class!* -

-
    -
  • Cal EECS -
    • -
  • A9 - Amazon Search -
    • -
  • PBworks -
    • -
  • Yelp -
    • -
  • Lecturer -
    • -
- - -
- -
- -
-

13 Hi, I'm Shreyas    slide

-
- -
    -
  • First year Grad Student (MIMS '14) -
    • -
  • Also TA'd Analyzing Big Data class -
    • -
  • I can be reached at seekshreyas@gmail.com -
    • -
- - -
- -
- -
-

14 Data is Important    slide

-
- -
    -
  • Making decisions is a core part of humanity -
    • -
  • Data can help you make better decisions -
    • -
  • Challenge: extract information from data to improve decisions -
    • -
- - -
- -
-

14.1 Decisions    notes

-
- -
    -
  • From big to small; from planning to execution -
    • -
  • Business questions: what is the ROI of this feature? Where to concentrate - development? -
    • -
  • Personal questions: Where to eat dinner tonight? What movie to see? -
    • -
  • Improving decisions means improving quality of life -
    • -
- - -
-
- -
- -
-

15 Data is Important    slide center

-
- - - - -
- -
-

15.1 Nice example of data mining    notes

-
- -
    -
  • Stop at 3:51 -
    • -
  • Had to work with external parties to get data (Yelp, city of Seattle) -
    • -
  • Had to clean data (literally, sometimes he was just handed paper receipts) -
    • -
  • Used regression analysis to discover patterns -
    • -
  • created follow up questions -
    • -
  • Used result to understand the meaning behind the data -
    • -
- - -
-
- -
- -
-

16 Data Mining ecosystem    slide

-
- -
    -
  • Data mining is part of a process to make decisions from data -
    • -
  • Intersection between statistics, computer science, data management, machine - learning -
    • -
  • Analysis & visualization often required -
    • -
- - -
- -
-

16.1 Ecosystem    notes

-
- -
    -
  • We'll talk about several ways to think about the process from data to - knowledge -
    • -
  • No universally agreed process, or black-and-white boundaries -
    • -
  • Analysis: used at the beginning of investigations to understand data - characteristics -
    • -
  • Visualization: better understanding of the results of analysis or data - mining -
    • -
- - -
- -
- -
-

16.2 Analysis vs. Data Mining    slide two_col

-
- -
    -
  • Analysis: manually investigating data. No algorithms. -
    • -
  • Statistical qualities: mean, median, standard deviation -
    • -
  • Histograms (manually set buckets) -
    • -
  • Counts / Percentages -
    • -
- - - -
    -
  • Data Mining: discovering patterns though automated algorithms -
    • -
  • Regressions: fitting data to a model -
    • -
  • Clustering: grouping data without manually set descriptions -
    • -
  • Classification: identifying divisive features -
    • -
- - -
- -
-

16.2.1 Pedantic    notes

-
- -
    -
  • Difference is subtle, but important for both the project and your resume -
    • -
- - -
-
- -
- -
-

16.3 Machine Learning    slide two_col

-
- -
    -
  • Programs that can learn from data -
    • -
  • Focus on prediction, based on verified training data -
    • -
  • Used in two ways: during DM, after DM -
    • -
- -

img/Terminator.jpg -

-
- -
-

16.3.1 Uses    notes

-
- -
-
During
assume we have training data, train on it, see how useful trained - program is or find outliers -
-
After
Discover clusters, verify and label clusters. Use labelled clusters - to train a program to recognize new data points -
-
- - -
-
- -
- -
-

16.4 Probability & Statistics    slide two_col

-
- -

img/Poisson_cdf.svg.png -

    -
  • Data describes real world events -
    • -
  • Probability can describe real world expected events -
    • -
  • Distributions can be used to summarize data, understand the factors behind - its creation -
    • -
- - -
- -
-

16.4.1 Uses    notes

-
- -
    -
  • Can "fit" data to a distribution, find outliers that are unexpected -
    • -
  • An example: Poisson distribution describes the expectation of a particular - number of events occurring. -
      -
    • Eg. pieces of mail. average is 4, but it can vary. Is getting 7 or more - pieces of mail really an outlier? -
      • -
    - -
    • -
- - -
-
-
- -
- -
-

17 Process    slide two_col

-
- -
    -
  • Knowledge Discovery in Databases (KDD) -
    • -
  • Selection -
    • -
  • Pre-processing -
    • -
  • Transformation -
    • -
  • Data Mining -
    • -
  • Interpretation/Evaluation -
    • -
- - - -
    -
  • Cross Industry Standard Process for Data Mining -
    • -
  • Business Understanding -
    • -
  • Data Understanding -
    • -
  • Data Preparation -
    • -
  • Modeling -
    • -
  • Evaluation -
    • -
  • Deployment -
    • -
- - -
- -
-

17.1 Common Themes    notes

-
- -
    -
  • Figure out what you want to do -
    • -
  • Get the data -
    • -
  • Make sure it's OK -
    • -
  • Understanding -
    • -
  • Make a decision, test its effectiveness -
    • -
  • Reading will cover another process, aimed at "Data Science", but basically - applies to Data Mining -
    • -
- - -
-
- -
- -
-

18 Break    slide

-
- - - - - - - - -
-
-
- -
-

Date: 2013-02-01 11:00:01 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-01-25-Intro.org b/slides/2013-01-25-Intro.org deleted file mode 100644 index 0003609..0000000 --- a/slides/2013-01-25-Intro.org +++ /dev/null @@ -1,260 +0,0 @@ -* Data Mining i290 :slide: - + Jim Blomo & Shreyas - -* Course Goals :slide: - + Extract *information* from *data* - + Understand techniques to find patterns - + Apply algorithms to real data sets - -* We'll Do Stuff :slide: - + 30%: 10 Homework Assignments - + 30%: 1 Midterm - + 40%: 1 Project: Find, Mine, Report on Data -** Homework Details :notes: - + HW due at midnight Thursday before class - + Each 24 hours late is 10% off - + HW will be turned in by GitHub pull request - + Project will be submitted by email & presentation - -* But Don't Worry :slide: - + This isn't a programming class - + Grades are based on understanding of the concepts, not the craziest project - + Shreyas & I are here to help -** Help :notes: - + We realize there's a wide range of technical skill - + We will help get anyone up to speed in these technical areas - -* This is a Graduate class :slide: - + Perform well without supervision - + Readings from both book and online documentation - + TMTOWTDI - + Getting frameworks working on your computer -** Style :notes: - + More firehouse than spoon feed, you'll need to follow up for - understanding - + Honor system: No copying code or answers. Helping each other with - concepts is encouraged, but document it. - + Everybody has a different workflow. We'll be covering the most basic. - Great if you want to do something different, but realize we may not be able - to help you as much. - + Non ISchool students should email student ID from EDU account to shreyas and - jblomo and we will get them ischool accounts. - + You may want to use other frameworks for your projects. Great! But again, - we may not be familiar with them - -* Prerequisites :slide: - + Basic probability: P(A), P(A or B), P(A and B), P(A | B) - + Basic programming: Python - + Basic command line: SSH, downloading, copying large files, running programs - against data - + Textbook: Han, J., Kamber, M., & Pei, J. (2011). _Data Mining: Concepts and Techniques_, Third Edition *(3rd ed.)*. Morgan Kaufmann. - + Technology will be available on =ischool.berkeley.edu= -** Basics :notes: - + "Probability of A", "Probability of A or B" "A and B" "A given B" - + Most assignments filling in algorithm code - + Project you may use any language, though we suggest Python. - + We'll introduce any specific frameworks - + Command line: cp, mv, less... Imagine you have a 10G file, how are you - going to inspect the contents? - -* Material :slide: - + Process: from find data to mining it to visualizing results - + Algorithms: all intuitively motivated, some rigorously studied - + Programming: using algorithms against data sets - + Discovery: finding information in self-defined project -** What will we learn? :notes: - + Data mining not just about algorithms. We'll learn how to obtain, clean, - and store data. - + In real life, this is 70% of the job! - + We'll cover many different algorithms, and dive in depth on several of - them. But we're not going to get into any hairy math proofs - + Programming is the best way to precisely describe an algorithm. It is also - the way data mining is used in the real world. - + Your own project should emphasize your passion. Again, real world requires - you to grab data and squeeze information out of it without external help - -* Lectures & Labs :slide: - + Start with Q&A for at least 10 minutes - + Expect to be asked a question - + Breaks - + Lab: Stick around and get the first question of HW done - + Slides on http://jblomo.github.com/datamining290/ -** Helpful tips :notes: - + Helpful to me if you say your name - + Sorry, I tend to forget names - + If I am not calling on you, check to make sure you are on the class list! - + I'm not taking attendance, but let me know if you can't make it so I - won't call on you - -* Office Hours :slide:two_col: - + We'll stay after class - + or schedule a Skype call - + [[https://piazza.com/class#spring2013/i290][Piazza]] for questions and - announcements - + Wait list will be processed normally until 3rd week... then I'll accept - everyone who's participated in class if we have physical room - [[file:img/Office_Hours.png]] -** Details :notes: - + I expect that everyone will be able to get into the class - + img src: http://statweb.calpoly.edu/srein/ - -* *Questions?* :slide: - -* Schedule :slide: -Available at [[http://jblomo.github.com/datamining290/][GitHub Syllabus page]] - + Jan 25 Class Intro ; Tools Intro by /GUEST: Shreyas/ - + lab: Git Intro - + Feb 1 Case Studies ; Obtaining Data - + Feb 8 Probability ; Preprocessing - + Feb 15 MapReduce, Data Warehouse - + Feb 22 Decision Trees; Naive Bayes - + Mar 1 SVM ; Neural Networks - + Mar 8 Clustering ; Review - + lab: Project Proposal Due - + Mar 15 *Midterm* - + lab: - - + Mar 21 Dimensionality Curse ; Graph Mining - + Mar 29 HOLIDAY - + Apr 5 Pattern ; Evaluations - + Apr 12 Collaborative Filtering; PageRank - + Apr 19 Feature Extraction ; Evaluation - + Apr 26 Images ; Audio - + May 3 Visualization ; HTML - + May 10 In Real Life ; Review - + lab: - - + May 17 Final Presentation - + lab: Bye! - -* Hi, I'm Jim Blomo :slide:two_col: -*[[https://www.dropbox.com/s/obnsldacg355wqn/2013-01-08%2021.50.03.mp4][Hello Class!]]* - - + Cal EECS - + A9 - Amazon Search - + PBworks - + Yelp - + Lecturer - -* Hi, I'm Shreyas :slide: - + First year Grad Student (MIMS '14) - + Also TA'd Analyzing Big Data class - + I can be reached at =seekshreyas@gmail.com= - -* Data is Important :slide: - + Making decisions is a core part of humanity - + Data can help you make better decisions - + Challenge: extract information from data to improve decisions -** Decisions :notes: - + From big to small; from planning to execution - + Business questions: what is the ROI of this feature? Where to concentrate - development? - + Personal questions: Where to eat dinner tonight? What movie to see? - + Improving decisions means improving quality of life - -* Data is Important :slide:center: -#+BEGIN_HTML - -#+END_HTML -** Nice example of data mining :notes: - + Stop at 3:51 - + Had to work with external parties to get data (Yelp, city of Seattle) - + Had to clean data (literally, sometimes he was just handed paper receipts) - + Used regression analysis to discover patterns - + created follow up questions - + Used result to understand the meaning behind the data - -* Data Mining ecosystem :slide: - + Data mining is part of a process to make decisions from data - + Intersection between statistics, computer science, data management, machine - learning - + Analysis & visualization often required -** Ecosystem :notes: - + We'll talk about several ways to think about the process from data to - knowledge - + No universally agreed process, or black-and-white boundaries - + Analysis: used at the beginning of investigations to understand data - characteristics - + Visualization: better understanding of the results of analysis or data - mining - -** Analysis vs. Data Mining :slide:two_col: - + *Analysis*: manually investigating data. No algorithms. - + Statistical qualities: mean, median, standard deviation - + Histograms (manually set buckets) - + Counts / Percentages - - - + *Data Mining*: discovering patterns though automated algorithms - + Regressions: fitting data to a model - + Clustering: grouping data without manually set descriptions - + Classification: identifying divisive features -*** Pedantic :notes: - + Difference is subtle, but important for both the project and your resume - -** Machine Learning :slide:two_col: - + Programs that can learn from data - + Focus on prediction, based on verified training data - + Used in two ways: during DM, after DM - [[file:img/Terminator.jpg]] -*** Uses :notes: - + During :: assume we have training data, train on it, see how useful trained - program is or find outliers - + After :: Discover clusters, verify and label clusters. Use labelled clusters - to train a program to recognize new data points - -** Probability & Statistics :slide:two_col: - [[file:img/Poisson_cdf.svg.png]] - + Data describes real world events - + Probability can describe real world *expected* events - + Distributions can be used to summarize data, understand the factors behind - its creation -*** Uses :notes: - + Can "fit" data to a distribution, find outliers that are unexpected - + An example: Poisson distribution describes the expectation of a particular - number of events occurring. - + Eg. pieces of mail. average is 4, but it can vary. Is getting 7 or more - pieces of mail really an outlier? - -* Process :slide:two_col: - + *Knowledge Discovery in Databases (KDD)* - + Selection - + Pre-processing - + Transformation - + Data Mining - + Interpretation/Evaluation - - - + *Cross Industry Standard Process for Data Mining* - + Business Understanding - + Data Understanding - + Data Preparation - + Modeling - + Evaluation - + Deployment -** Common Themes :notes: - + Figure out what you want to do - + Get the data - + Make sure it's OK - + Understanding - + Make a decision, test its effectiveness - + Reading will cover another process, aimed at "Data Science", but basically - applies to Data Mining - -* *Break* :slide: - - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-01-CaseStudies.html b/slides/2013-02-01-CaseStudies.html deleted file mode 100644 index 64f2a2e..0000000 --- a/slides/2013-02-01-CaseStudies.html +++ /dev/null @@ -1,740 +0,0 @@ - - - - -2013-02-01-CaseStudies - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-01-CaseStudies

- - - - -
-

1 Questions    slide

-
- - - - -
- -
-

1.1 Memory    notes

-
- -
    -
  • Spaced intervals makes memorizing easiest -
    • -
- - -
-
- -
- -
-

2 Case Studies    slide

-
- - -
- -
- -
-

3 Process    slide two_col

-
- -
    -
  • Knowledge Discovery in Databases (KDD) -
    • -
  • Selection -
    • -
  • Pre-processing -
    • -
  • Transformation -
    • -
  • Data Mining -
    • -
  • Interpretation/Evaluation -
    • -
- - - -
    -
  • Cross Industry Standard Process for Data Mining -
    • -
  • Business Understanding -
    • -
  • Data Understanding -
    • -
  • Data Preparation -
    • -
  • Modeling -
    • -
  • Evaluation -
    • -
  • Deployment -
    • -
- - -
- -
-

3.1 Data to Knowledge    notes

-
- -
    -
  • We learned last week that the goal of data mining is to turn raw data into - knowledge -
    • -
- - -
-
- -
- -
-

4 Search Engine Logs    slide

-
- - - - -
193.139.1 jim [10/Oct/2013:13:55:36 -0700] "GET /search?query=headache HTTP/1.1" 200 9288
-282.482.3 shreyas [10/Oct/2013:13:56:36 -0700] "GET /search?query=bananas HTTP/1.1" 200 2929
-345.114.1 steven [10/Oct/2013:13:56:37 -0700] "GET /search?query=cold HTTP/1.1" 200 8232
-10.328.52 anne [10/Oct/2013:13:56:39 -0700] "GET /search?query=flu+shot HTTP/1.1" 200 2342
-10.328.52 lily [10/Oct/2013:13:57:40 -0700] "GET /search?query=i290 HTTP/1.1" 200 2342
-
- - -
    -
  • what is a common theme in these queries? -
    • -
- - -
- -
-

4.1 Raw Data    notes

-
- -
    -
  • raw data comes in many forms -
    • -
  • often well use tech examples: eg search engine logs -
    • -
  • these have information like user, IP, date-time, HTTP version, query -
    • -
  • can we extract actionable information from it? -
    • -
- - -
-
- -
- -
-

5 Flu Trends    slide two_col

-
- -
    -
  • Use dates to plot trends over time -
    • -
  • Use IPs to show activity per state or city -
    • -
  • Other ideas? -
    • -
- - -

- img/flu-trends.png -

-
- -
-

5.1 Other ideas    notes

-
- -
    -
  • What other information could you extract from log data? -
    • -
  • Spread of flu over countries, cities? -
    • -
  • Time of day? Do people notice in the morning? -
    • -
  • correlated with any other activity? (eg. travel) -
    • -
  • best day of the week to call in sick (and get away with it)? -
    • -
- - -
-
- -
- -
-

6 Asking Questions    slide

-
- -
    -
  • Many potential discoveries within search logs -
    • -
  • Asking meaningful questions is a difficult but essential part of data - mining -
    • -
  • Algorithms can answer questions for you, but it can't ask them -
    • -
- - -
- -
-

6.1 No magic    notes

-
- -
    -
  • Data mining is not a magical machine into which one throws data and gets - out interesting facts -
    • -
  • Data + question + algorithm suited for question => potential insights -
    • -
- - -
-
- -
- -
-

7 Data Mining Process    slide animate

-
- -
    -
  • Data cleaning -
    • -
  • Data integration -
    • -
  • Data selection -
    • -
  • Data transformation -
    • -
  • Data mining* -
    • -
  • Pattern evaluation -
    • -
  • Knowledge presentation -
    • -
- - -
- -
-

7.1 We cover the full process    notes

-
- -
-
Cleaning
remove abuse requests, "Estimates for Connecticut for weeks - 2012-12-16 to 2013-01-06 were affected by a software glitch" -
-
Integration
Collecting logs from different data centers, maybe from - different formats (over the years) -
-
Selection
IPs, dates, queries -
-
Transformation
IP to location. Dates to local time. -
-
Mining
what words are associated with the flu? cold? fever? other - languages? -
-
Evaluation
This year worse than last, peaking later. -
-
Presentation
plotting, cartograms -
-
- - -
-
- -
- -
-

8 Data Preparation    slide

-
- -
    -
  • Collecting, cleaning, integrating takes > 50% of the time in real world - situations -
    • -
  • Explains difficulty in finding good candidates for Data Scientist roles -
    • -
- - -
- -
-

8.1 Data Scientist    notes

-
- -
    -
  • In industry, most companies are hiring engineers to interact with the full - stack, so that they can collect data -
    • -
  • If preperation is > 50% and they hire you just for algorithms, they need ot - hire > 1 other person just to support you -
    • -
  • How many of you like just preparing data? -
    • -
- - -
-
- -
- -
-

9 Transactional Data    slide

-
- -
    -
  • Discrete history of events, containing some minimum amount of data: -
    • -
  • Subject: Who initiated action? -
    • -
  • Verb: What was done? -
    • -
  • Object: What was it done to? -
    • -
  • Timestamp: When? -
    • -
- - -
- -
-

9.1 Storage    notes

-
- -
    -
  • Most common example is purchase history -
    • -
  • Subject: user ID, or name -
    • -
  • Verb: In logs, can vary. In databases, you'll have a purchases table, so - verb is assumed to be "purchased" -
    • -
  • Object: product IDs (or in web logs, web pages) -
    • -
  • Timestamp: Make sure you account for timezones -
    • -
  • Other Data: previous page, extra info about action (purchase with CC? - Cash?) -
    • -
- - -
-
- -
- -
-

10 Other Data    slide

-
- -
    -
  • Often does not contain timestamps -
    • -
  • Spatial Data -
    • -
  • Multimedia -
    • -
- -

img/moonlight_sonata.jpg -

-
- -
-

10.1 Data    notes

-
- -
    -
  • Maps in general can be used to find interesting information: where are - cities typically located? What are properties of well planned cities? -
    • -
  • Videos have a time component, but are not transactional. -
    • -
  • Music can be seen non-linearly and analyzed -
    • -
  • img: http://flyingpudding.com/projects/viz_music/ -
    • -
- - -
-
- -
- -
-

11 Purpose of Data Mining    slide

-
- -
-
Purpose
Obtaining actionable knowledge -
-
Descriptive
explains data already seen -
-
Predictive
Immediately understand new data -
-
- - -
- -
-

11.1 Tasks    notes

-
- -
    -
  • At Amazon, dashboards for different countries -
    • -
  • Americans shopped at work; Germans shopped early morning, early evening; Japanese shopped late at night -
    • -
  • Can help with capacity planning, ideas for discounts, warehouse staffing -
    • -
  • Predictive: at Yelp, what business are you most likely to want to review - next? As you have activity, instantly understand what is the best - recommendation -
    • -
- - -
-
- -
- -
-

12 Types of Models    slide animate

-
- -
    -
  • Classifiers -
    • -
  • Regressions -
    • -
  • Clustering -
    • -
  • Outlier -
    • -
- - -
- -
-

12.1 Details    notes

-
- -
-
Classifiers
describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description -
-
Regressions
Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms -
-
Clustering
find "natural" groups of data without labels -
-
Outlier
find anomalous transactions, eg. finding fraud for credit cards -
-
- - -
- -
- -
-

12.2 Tip of the Iceburg    slide two_col

-
- -

img/iceberg11.jpg -

    -
  • Thousands of ways to calculate a model -
    • -
  • Combinatorially more ways to combine them -
    • -
  • In technique, large amount of overlap between purpose -
    • -
- -
- -
- -
-

12.3 Survey    notes

-
- -
    -
  • ML and DM fields churn these models out -
    • -
  • Newest methods combine multiple models (boosting & bagging) -
    • -
  • We're going to cover these in much greater detail in the course -
    • -
- - -
-
- -
- -
-

13 Your own examples    slide animate

-
- -
    -
  • Classifiers -
    • -
  • Regressions -
    • -
  • Clustering -
    • -
  • Outlier -
    • -
- - -
- -
-

13.1 Examples    notes

-
- -
-
Classifiers
Newly opened business -
-
Regressions
Revenue estimates for a franchise store -
-
Clustering
Movie genres -
-
Outlier
Bot vs human web traffic -
-
- - -
-
- -
- -
-

14 Machine Learning    slide

-
- -
-
Supervised
Given data with a label, predict data without a - label -
-
Unsupervised
Given data without labels, group "similar" items - together -
-
Semi-supervised
Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases -
-
Active
Starting with unlabeled data, select the most helpful cases for a - human to label -
-
- - -
- -
- -
-

15 Matching    slide

-
- -
    -
  • Categories for businesses, where some business have correct labels, but not sure how precise categories should be -
    • -
  • Comparing search results algorithms: some queries return the same results, some return very different businesses -
    • -
  • Spam filter with existing corpus -
    • -
  • Demographic information about customers -
    • -
- - -
- -
-

15.1 Details    notes

-
- -
    -
  • Matching with the type of learning -
    • -
- - -
-
- -
- -
-

16 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-02-01 13:38:13 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-01-CaseStudies.org b/slides/2013-02-01-CaseStudies.org deleted file mode 100644 index f0f73dc..0000000 --- a/slides/2013-02-01-CaseStudies.org +++ /dev/null @@ -1,209 +0,0 @@ -* Questions :slide: - + Differences from [[http://bid.berkeley.edu/cs294-1-spring13/index.php/Main_Page][CS 294 Behavioral Data Mining]] - + Course Load - + Readings: after lecture -** Memory :notes: - + Spaced intervals makes memorizing easiest - -* Case Studies :slide: - -* Process :slide:two_col: - + *Knowledge Discovery in Databases (KDD)* - + Selection - + Pre-processing - + Transformation - + Data Mining - + Interpretation/Evaluation - - - + *Cross Industry Standard Process for Data Mining* - + Business Understanding - + Data Understanding - + Data Preparation - + Modeling - + Evaluation - + Deployment -** Data to Knowledge :notes: - + We learned last week that the goal of data mining is to turn raw data into - knowledge - -* Search Engine Logs :slide: -#+begin_src log -193.139.1 jim [10/Oct/2013:13:55:36 -0700] "GET /search?query=headache HTTP/1.1" 200 9288 -282.482.3 shreyas [10/Oct/2013:13:56:36 -0700] "GET /search?query=bananas HTTP/1.1" 200 2929 -345.114.1 steven [10/Oct/2013:13:56:37 -0700] "GET /search?query=cold HTTP/1.1" 200 8232 -10.328.52 anne [10/Oct/2013:13:56:39 -0700] "GET /search?query=flu+shot HTTP/1.1" 200 2342 -10.328.52 lily [10/Oct/2013:13:57:40 -0700] "GET /search?query=i290 HTTP/1.1" 200 2342 -#+end_src - - + what is a common theme in these queries? -** Raw Data :notes: - + raw data comes in many forms - + often well use tech examples: eg search engine logs - + these have information like user, IP, date-time, HTTP version, query - + can we extract actionable information from it? - -* Flu Trends :slide:two_col: - + Use dates to plot trends over time - + Use IPs to show activity per state or city - + Other ideas? - - [[file:img/flu-trends.png]] -** Other ideas :notes: - + What other information could you extract from log data? - + Spread of flu over countries, cities? - + Time of day? Do people notice in the morning? - + correlated with any other activity? (eg. travel) - + best day of the week to call in sick (and get away with it)? - -* Asking Questions :slide: - + Many potential discoveries within search logs - + Asking meaningful questions is a difficult but essential part of data - mining - + Algorithms can answer questions for you, but it can't ask them -** No magic :notes: - + Data mining is not a magical machine into which one throws data and gets - out interesting facts - + Data + question + algorithm suited for question => potential insights - -* Data Mining Process :slide:animate: - + Data cleaning - + Data integration - + Data selection - + Data transformation - + Data mining* - + Pattern evaluation - + Knowledge presentation -** We cover the full process :notes: - + Cleaning :: remove abuse requests, "Estimates for Connecticut for weeks - 2012-12-16 to 2013-01-06 were affected by a software glitch" - + Integration :: Collecting logs from different data centers, maybe from - different formats (over the years) - + Selection :: IPs, dates, queries - + Transformation :: IP to location. Dates to local time. - + Mining :: what words are associated with the flu? cold? fever? other - languages? - + Evaluation :: This year worse than last, peaking later. - + Presentation :: plotting, cartograms - -* Data Preparation :slide: - + Collecting, cleaning, integrating takes > 50% of the time in real world - situations - + Explains difficulty in finding good candidates for Data Scientist roles -** Data Scientist :notes: - + In industry, most companies are hiring engineers to interact with the full - stack, so that they can collect data - + If preperation is > 50% and they hire you just for algorithms, they need ot - hire > 1 other person just to support you - + How many of you like just preparing data? - -* Transactional Data :slide: - + Discrete history of events, containing some minimum amount of data: - + Subject: Who initiated action? - + Verb: What was done? - + Object: What was it done to? - + Timestamp: When? -** Storage :notes: - + Most common example is purchase history - + Subject: user ID, or name - + Verb: In logs, can vary. In databases, you'll have a purchases table, so - verb is assumed to be "purchased" - + Object: product IDs (or in web logs, web pages) - + Timestamp: Make sure you account for timezones - + Other Data: previous page, extra info about action (purchase with CC? - Cash?) - -* Other Data :slide: - + Often does not contain timestamps - + Spatial Data - + Multimedia -[[file:img/moonlight_sonata.jpg]] -** Data :notes: - + Maps in general can be used to find interesting information: where are - cities typically located? What are properties of well planned cities? - + Videos have a time component, but are not transactional. - + Music can be seen non-linearly and analyzed - + img: http://flyingpudding.com/projects/viz_music/ - -* Purpose of Data Mining :slide: - + Purpose :: Obtaining *actionable knowledge* - + Descriptive :: explains data already seen - + Predictive :: Immediately understand new data -** Tasks :notes: - + At Amazon, dashboards for different countries - + Americans shopped at work; Germans shopped early morning, early evening; Japanese shopped late at night - + Can help with capacity planning, ideas for discounts, warehouse staffing - + Predictive: at Yelp, what business are you most likely to want to review - next? As you have activity, instantly understand what is the best - recommendation - -* Types of Models :slide:animate: - + Classifiers - + Regressions - + Clustering - + Outlier -** Details :notes: - + Classifiers :: describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description - + Regressions :: Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms - + Clustering :: find "natural" groups of data *without labels* - + Outlier :: find anomalous transactions, eg. finding fraud for credit cards - -** Tip of the Iceburg :slide:two_col: -[[file:img/iceberg11.jpg]] - + Thousands of ways to calculate a model - + Combinatorially more ways to combine them - + In technique, large amount of overlap between purpose -** Survey :notes: - + ML and DM fields churn these models out - + Newest methods combine multiple models (boosting & bagging) - + We're going to cover these in much greater detail in the course - -* Your own examples :slide:animate: - + Classifiers - + Regressions - + Clustering - + Outlier -** Examples :notes: - + Classifiers :: Newly opened business - + Regressions :: Revenue estimates for a franchise store - + Clustering :: Movie genres - + Outlier :: Bot vs human web traffic - -* Machine Learning :slide: - + Supervised :: Given data with a label, predict data without a - label - + Unsupervised :: Given data without labels, group "similar" items - together - + Semi-supervised :: Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases - + Active :: Starting with unlabeled data, select the most helpful cases for a - human to label - -* Matching :slide: - + Categories for businesses, where some business have correct labels, but not sure how precise categories should be - + Comparing search results algorithms: some queries return the same results, some return very different businesses - + Spam filter with existing corpus - + Demographic information about customers -** Details :notes: - + Matching with the type of learning - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-01-Lab.html b/slides/2013-02-01-Lab.html deleted file mode 100644 index d7e8537..0000000 --- a/slides/2013-02-01-Lab.html +++ /dev/null @@ -1,312 +0,0 @@ - - - - -2013-02-01-Lab - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-01-Lab

- - - - -
-

1 Lab: Obtain and Explore Data    slide

-
- -
    -
  • Setup GitHub account -
    • -
  • Find a data set or external API -
    • -
  • Superficially examine it -
    • -
  • Summarize findings -
    • -
  • Submit assignment via GitHub -
    • -
- - -
- -
- -
-

2 Why GitHub?    slide

-
- -
    -
  • git tool is standard in industry -
    • -
  • GitHub provides best tools for sharing, commenting code -
    • -
  • This assignment will not have code, just practice submitting -
    • -
- - -
- -
- -
-

3 Setup GitHub account    slide

-
- - - - -
- -
- -
-

4 Setup git repository on ischool server    slide

-
- -
    -
  • On the server ischool.berkeley.edu -
    • -
- - - - -
$ git clone git://github.com/jblomo/datamining290.git
-
- -
    -
  • On the server, in the datamining290 directory run -
    • -
- - - - -
$ git remote rename origin jblomo
-
- - -
- -
- -
-

5 Connect it to GitHub    slide

-
- -
    -
  • After you recieve your free micro account on GitHub, create a private repository called datamining290 -
    • -
  • It will provide you with an SSH git path, let's call it PATH -
    • -
  • You must use the SSH PATH starting with git:// -
    • -
  • On the server, in the datamining290 directory, run -
    • -
- - - - -
$ git remote add origin PATH
-$ git push origin master
-
- - -
- -
- -
-

6 Share with us    slide

-
- -
    -
  • Hopefully you now have a private copy of my repository -
    • -
  • Add Shreyas and me (users: seekshreyas, jblomo) as a contributor to your private repository -
    • -
- - -
- -
- -
-

7 Obtain Data    slide

-
- -
    -
  • Look through the links in slides for interesting data sets, or find your own -
    • -
  • Or find a service API, like NYTimes -
    • -
  • Explore the data available to answer the following questions -
    • -
- - -
- -
- -
-

8 Questions    slide

-
- -
    -
  • What are the types of data available to you? -
    • -
  • For data sets: how many records are in the data set? -
    • -
  • For API: what are the limits on fetching data? -
    • -
  • Provide an "interesting" record, explain its properties and why it is - interesting -
    • -
  • What are 3 questions you could answer using your data? -
    • -
- - -
- -
- -
-

9 Submit Homework    slide

-
- -
    -
  • On the ischool server, create a branch called hw-obtain-data -
    • -
  • Create a text file to write the solution, a simple editor to use is pico -
    • -
  • git add the file -
    • -
  • git commit the change -
    • -
  • git push origin hw-obtain-data to put it on GitHub -
    • -
  • on github, submit a "pull request" from the hw-obtain-data branch to your master branch -
    • -
- - -
- -
-

9.1 Pull Requests    notes

-
- -
    -
  • Pull requests are a way of showing updates in a way that lets me provide - comments, get notifications -
    • -
  • This is the first time I've tried it for class, so you're on the cutting - edge. Hopefully it will work, give me feedback if it is not -
    • -
- - -
-
- -
- -
-

10 Going Forward    slide

-
- -
    -
  • Other homework assignments will be completing code -
    • -
  • General work-flow: -
      -
    • Start a new branch -
      • -
    • Add required files -
      • -
    • push to GitHub -
      • -
    • Submit Pull Request -
      • -
    - -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-02-01 17:25:49 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-01-Lab.org b/slides/2013-02-01-Lab.org deleted file mode 100644 index ffc2413..0000000 --- a/slides/2013-02-01-Lab.org +++ /dev/null @@ -1,93 +0,0 @@ -* Lab: Obtain and Explore Data :slide: - + Setup GitHub account - + Find a data set or external API - + Superficially examine it - + Summarize findings - + Submit assignment via GitHub - -* Why GitHub? :slide: - + =git= tool is standard in industry - + GitHub provides best tools for sharing, commenting code - + This assignment will not have code, just practice submitting - -* Setup GitHub account :slide: - + Create a [[https://github.com/signup/free][GitHub Account]], making sure to - use your .edu address - + Use [[https://github.com/edu][GitHub/Edu]] to request a free micro plan: - these let us use private accounts - + Setup a [[https://help.github.com/articles/generating-ssh-keys][GitHub SSH Key]] - -* Setup git repository on ischool server :slide: - + On the server ischool.berkeley.edu -#+begin_src bash -$ git clone git://github.com/jblomo/datamining290.git -#+end_src - + On the server, in the datamining290 directory run -#+begin_src bash -$ git remote rename origin jblomo -#+end_src - -* Connect it to GitHub :slide: - + After you recieve your free micro account on GitHub, create a private repository called datamining290 - + It will provide you with an SSH git path, let's call it PATH - + You must use the *SSH* PATH starting with =git://= - + On the server, in the datamining290 directory, run -#+begin_src html -$ git remote add origin PATH -$ git push origin master -#+end_src - -* Share with us :slide: - + Hopefully you now have a private copy of my repository - + Add Shreyas and me (users: seekshreyas, jblomo) as a contributor to your private repository - -* Obtain Data :slide: - + Look through the links in slides for interesting data sets, or find your own - + Or find a service API, like NYTimes - + Explore the data available to answer the following questions - -* Questions :slide: - + What are the types of data available to you? - + For data sets: how many records are in the data set? - + For API: what are the limits on fetching data? - + Provide an "interesting" record, explain its properties and why it is - interesting - + What are 3 questions you could answer using your data? - -* Submit Homework :slide: - + On the ischool server, create a branch called =hw-obtain-data= - + Create a text file to write the solution, a simple editor to use is =pico= - + =git add= the file - + =git commit= the change - + =git push origin hw-obtain-data= to put it on GitHub - + on github, submit a "pull request" from the =hw-obtain-data= branch to your master branch -** Pull Requests :notes: - + Pull requests are a way of showing updates in a way that lets me provide - comments, get notifications - + This is the first time I've tried it for class, so you're on the cutting - edge. Hopefully it will work, give me feedback if it is not - -* Going Forward :slide: - + Other homework assignments will be completing code - + General work-flow: - + Start a new branch - + Add required files - + push to GitHub - + Submit Pull Request - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-01-Obtaining-Data.html b/slides/2013-02-01-Obtaining-Data.html deleted file mode 100644 index 1376a1b..0000000 --- a/slides/2013-02-01-Obtaining-Data.html +++ /dev/null @@ -1,944 +0,0 @@ - - - - -2013-02-01-Obtaining-Data - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-01-Obtaining-Data

- - - - -
-

1 Obtaining Data    slide

-
- - -
- -
- -
-

2 Ways to Collect    slide two_col

-
- -
    -
  • Operational Data -
    • -
  • Data Warehouse -
    • -
  • Unstructured Data -
    • -
  • External API -
    • -
  • Data Sets -
    • -
- - -

-img/bottlecaps.jpg -

- -
- -
-

2.1 Operational Data    slide

-
- -
    -
  • Most frequent in industry -
    • -
  • Usually stored in databases best suited for transactional use -
    • -
  • Challenge is reorganizing data to suit question -
    • -
- - -
- -
-

2.1.1 Data from production    notes

-
- -
    -
  • Most frequently you'll have data that is being used by the application, - and you'll want to find insights in it -
    • -
  • We'll go into more detail in another class, but online use is - optimized for small queries and small updates -
    • -
  • Frequently just accessing the data in bulk is a software engineering - problem: -
      -
    • ensuring long queries don't hold up production usage -
      • -
    • joining across databases via software -
      • -
    • understanding esoteric columns, like "flags" -
      • -
    - -
    • -
  • Often will want to reorganize data to look like transactional -
    • -
  • img: http://woodwarddesign.ca/blog/2009/03/06/bottle-caps/ -
    • -
- - -
-
- -
- -
-

2.2 Example    slide

-
- -
    -
  • Find the user names with most "liked" reviews on Yelp -
    • -
- - -

-users -

- -- - - - - -
userIDnameflags
25234Jim0x200
- - -

-reviews -

- -- - - - - -
reviewIDbusinessIDuserIDstarstextflags
28252432252344great place!0x1
- - - -

-feedback -

- -- - - - - -
reviewIDsrcUserIDufcFlagsflags
28282050x10x0
- - - -
- -
-

2.2.1 Distributed Data    notes

-
- -
    -
  • At Yelp we have a variety of database tables, and those tables can be - spread across different databases -
    • -
  • At a minimum we frequently need to JOIN across tables to answer queries -
      -
    • eg. matching up user names with reviews from separate tables -
      • -
    - -
    • -
  • It is possible the review table is only indexed on business ID, and so - finding all reviews by a user is really disk intensive: make sure you're - not slowing down the whole site! -
    • -
  • An additional challenge is when the "feedback" tables are in a separate - database: can no longer issue normal SQL queries -
    • -
  • What are these "flag" columns for? -
    • -
  • Exactly: no one knows. Often must look into code, or compare data to - production representation to guess meaning. In Yelp, 0x1 often means - "inactive", so we probably don't want to count that feedback -
    • -
- - -
-
- -
- -
-

2.3 Data Warehouse    slide two_col

-
- -
    -
  • Data located on same system -
    • -
  • Organized for analytics queries -
    • -
  • Requires extra maintenance and understanding of construction -
    • -
- - -

- img/Ikea-Warehouse.jpg -

-
- -
-

2.3.1 No free lunch    notes

-
- -
    -
  • A strong data warehouse can be a big improvement over operational data -
    • -
  • Hopefully, someone has already cleaned, joined data in a way that makes - sense! -
    • -
  • Optimized for long running queries: less fear of brining down website! -
    • -
  • But you must learn how that process was accomplished in order to understand - potential problems -
    • -
  • How to handle missing data? -
    • -
  • We'll go into more detail about how data warehouse schemas compare to - online ones later in the course -
    • -
- - -
-
- -
- -
-

2.4 Unstructured    slide

-
- -
    -
  • Haphazard collection of data -
    • -
  • Unclear what structure should be -
    • -
  • Examples: Web logs, text, multimedia -
    • -
  • Must extract structure eventually -
    • -
- - -
- -
-

2.4.1 Yelp JSON logs    notes

-
- -
    -
  • When developing a web application, new context or details become - important: how long did certain requests take? What link did a user follow - to a website? -
    • -
  • Relational Databases aren't well suited for these wide varieties of - potential attributes that don't apply to all items -
    • -
  • So the current work around is just to write all useful information down in - a log, and extract what is needed later -
    • -
  • Text, like business reviews, another example: desired structure changes - radically between questions: How many words? Characters? What is the sentiment? -
    • -
  • Pictures can contain attributes like color depth, length, width -
    • -
  • First step of data mining is often imposing structure on data: the data is - not inherently unstructured, it just is unclear what the structure should be - until query time -
    • -
- - -
-
- -
- -
-

2.5 Search Logs Example    slide

-
- - - - -
193.139.1 jim [10/Oct/2013:13:55:36 -0700] "GET /search?query=headache HTTP/1.1" 200 9288
-282.482.3 shreyas [10/Oct/2013:13:56:36 -0700] "GET /search?query=bananas HTTP/1.1" 200 2929
-345.114.1 steven [10/Oct/2013:13:56:37 -0700] "GET /search?query=cold HTTP/1.1" 200 8232
-10.328.52 anne [10/Oct/2013:13:56:39 -0700] "GET /search?query=flu+shot HTTP/1.1" 200 2342
-10.328.52 lily [10/Oct/2013:13:57:40 -0700] "GET /search?query=i290 HTTP/1.1" 200 2342
-
- - - - -- - - - - - - - - -
userNamedatequery
jim10/Oct/2013:13:55:36 -0700headache
shreyas10/Oct/2013:13:56:36 -0700bananas
steven10/Oct/2013:13:56:37 -0700cold
anne10/Oct/2013:13:56:39 -0700flu shot
lily10/Oct/2013:13:57:40 -0700i290
- - - -
- -
-

2.5.1 Imposing Structure    notes

-
- -
    -
  • Extract only the rows we know follow a format -
    • -
  • Format queries from some encoding (eg. URL) to standardized format -
    • -
- - -
-
- -
- -
-

2.6 External APIs    slide

-
- -
    -
  • Better documented than internal data! -
    • -
  • More limited in amount and detail -
    • -
  • Commonly HTTP/REST based -
    • -
- - -
- -
-

2.6.1 Motivation    notes

-
- -
    -
  • Companies are often searching for other ways to leverage their data -
    • -
  • Both for immediate business purposes, and for brand recognition -
    • -
  • Twitter more (in)famous example -
    • -
  • NYTimes another good option -
    • -
- - -
-
- -
- -
-

2.7 NYTimes API Example    slide

-
- - - - -
- -
-

2.7.1 Accessing these    notes

-
- -
    -
  • More info on how to access these APIs is in the Web Architecture class, - but feel free to ask Shreyas or I about how best to access them -
    • -
- - -
-
- -
- -
-

2.8 Data Sets    slide

-
- -
    -
  • Download large, curated set of data all at once -
    • -
  • Formats vary, but usually documented -
    • -
  • Can be useful to combine with other datasets or APIs -
    • -
- - -

-img/kaggle-digits.png -

-
- -
-

2.8.1 Research    notes

-
- -
    -
  • Data sets commonly used in research: can compare different techniques on - same data to understand advantages -
    • -
  • Sizes can range to a few MB to GB -
    • -
  • JSON, CSV, XML all potential formats. Cleaning, organization for your - question again becomes an important aspect -
    • -
- - -
-
- -
- -
-

2.9 Data Set Example    slide two_col

-
- - - - -

-img/video.jpg -

-
-
- -
- -
-

3 Exploring Data    slide

-
- -
    -
  • Data sets are frequently too large to fit in standard tools like Excel or - Word -
    • -
  • Simplest to explore on the command line -
    • -
  • Homework will be exploring a data set of your choice -
    • -
- - -
- -
-

3.1 Size    notes

-
- -
    -
  • Some formats will not be easily parsed into Excel: eg. JSON, XML -
    • -
  • Word will be slow, or unworkable for GB size data -
    • -
  • CLI provides many composable tools for text manipulation -
    • -
- - -
-
- -
- -
-

4 Yelp Academic Dataset    slide

-
- -
    -
  • Yelp Data Set covers reviews, users, - businesses -
    • -
  • To download, you'll need to sign up: process takes ~24 hours for approval -
    • -
  • Use .edu email -
    • -
- - -
- -
-

4.1 Example    notes

-
- -
    -
  • We'll use this as an example, you can use any data set of your choice -
    • -
  • Just for HW, don't need to use for project -
    • -
- - -
-
- -
- -
-

5 CLI introduction    slide

-
- -
    -
  • Standard commands available in Learn CLI the hard way -
    • -
  • All example will be run on ischool.berkeley.edu -
    • -
  • Sheyas and I available for more help -
    • -
- - -
- -
-

5.1 Help    notes

-
- -
    -
  • If you're new, don't be intimidated. -
    • -
  • Security policies ensure you can't break anything besides your own files -
    • -
  • Keep backups of important stuff anyway -
    • -
- - -
-
- -
- -
-

6 wget    slide

-
- -
    -
  • Used for downloading files -
    • -
  • Downloading with the browser is fine, but sometimes nice to use faster - connection, or download it directly to machine you're working on -
    • -
- - - - -
$ wget 'http://www.grouplens.org/system/files/ml-100k.zip'
-
- - -
- -
-

6.1 Command    notes

-
- -
    -
  • Just wget URL -
    • -
  • I like to use quotes in case there are special characters in the URL, eg - ? -
    • -
  • Will download to current directory, same name as remote file -
    • -
- - -
-
- -
- -
-

7 scp    slide

-
- -
    -
  • Copy a file to or from a remote machine -
    • -
  • Uses same connection as SSH, but copies data instead -
    • -
  • Example: Copy data you've downloaded in your browser -
    • -
- - - - -
$ scp ~/Downloads/ml-100k.zip jblomo@ischool.berkeley.edu:
-# or
-$ scp ~/Downloads/ml-100k.zip jblomo@ischool.berkeley.edu:i290/movielens-100k.zip
-
- - -
- -
-

7.1 Command    notes

-
- -
    -
  • Trailing : is important: signifies remote machine -
    • -
  • If you don't specify path or filename, will copy the file with the same - name into your home directory -
    • -
- - -
-
- -
- -
-

8 gunzip unzip    slide

-
- -
    -
  • Uncompress data sets for simpler, faster manipulation -
    • -
- - - - -
$ unzip ml-100k.zip
-# or
-$ gunzip yelp_academic_dataset.json.gz
-
- - -
- -
-

8.1 Commands    notes

-
- -
-
unzip
expand potentially many file, leave original alone -
-
gunzip
expand original file, leaving only the uncompressed version -
-
- - -
-
- -
- -
-

9 less    slide

-
- -
    -
  • View a file -
    • -
  • History: original command was called more to see a file a page at a time -
    • -
  • "Less is more" -
    • -
- - - - -
less yelp_academic_dataset.json
-
- - -
- -
- -
-

10 Searching in less    slide

-
- -
    -
  • / (forward slash) lets you input search text -
    • -
  • q will quit -
    • -
- - - - -
/type": "user"
-/type": "review"
-
- - -
- -
-

10.1 Command    notes

-
- -
    -
  • Useful for finding specific instances to investigate -
    • -
- - -
-
- -
- -
-

11 grep    slide

-
- -
    -
  • Find and print lines matching a "regular expression" -
    • -
  • Regular expressions are "find" on steroids, but you can use simple strings -
    • -
- - - - -
$ grep 'type": "review"' yelp_academic_dataset.json
-
- - -
- -
- -
-

12 wc    slide

-
- -
    -
  • "wordcount" counts characters, words, lines -
    • -
  • Most useful in data sets for lines: -l -
    • -
- - - - -
$ wc -l yelp_academic_dataset.json
-474434 yelp_academic_dataset.json
-
- - -
- -
- -
-

13 Composable    slide

-
- -
    -
  • Genius of Unix: do one thing well, compose commands to get what you want -
    • -
  • | pipe characters "sends" output from one program to the input of another -
    • -
  • How many reviews in the data set? -
    • -
- - - - -
$ grep 'type": "review"' yelp_academic_dataset.json | wc -l
-330071
-$ egrep -o 'business_id": "\w+"' yelp_academic_dataset.json  | sort -u | wc -l
-9592
-
- - - - - -
-
-
- -
-

Date: 2013-02-01 13:45:14 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-01-Obtaining-Data.org b/slides/2013-02-01-Obtaining-Data.org deleted file mode 100644 index 6de7701..0000000 --- a/slides/2013-02-01-Obtaining-Data.org +++ /dev/null @@ -1,279 +0,0 @@ -* Obtaining Data :slide: - -* Ways to Collect :slide:two_col: - + Operational Data - + Data Warehouse - + Unstructured Data - + External API - + Data Sets - -[[file:img/bottlecaps.jpg]] - -** Operational Data :slide: - + Most frequent in industry - + Usually stored in databases best suited for transactional use - + Challenge is reorganizing data to suit question -*** Data from production :notes: - + Most frequently you'll have data that is being used by the application, - and you'll want to find insights in it - + We'll go into more detail in another class, but online use is - optimized for small queries and small updates - + Frequently just accessing the data in bulk is a software engineering - problem: - + ensuring long queries don't hold up production usage - + joining across databases via software - + understanding esoteric columns, like "flags" - + Often will want to reorganize data to look like transactional - + img: http://woodwarddesign.ca/blog/2009/03/06/bottle-caps/ - -** Example :slide: - + Find the user names with most "liked" reviews on Yelp - -users -| userID | name | flags | -| 25234 | Jim | 0x200 | - -reviews -| reviewID | businessID | userID | stars | text | flags | -| 282 | 52432 | 25234 | 4 | great place! | 0x1 | - - -feedback -| reviewID | srcUserID | ufcFlags | flags | -| 282 | 8205 | 0x1 | 0x0 | - -*** Distributed Data :notes: - + At Yelp we have a variety of database tables, and those tables can be - spread across different databases - + At a minimum we frequently need to =JOIN= across tables to answer queries - + eg. matching up user names with reviews from separate tables - + It is possible the review table is only indexed on business ID, and so - finding all reviews by a user is really disk intensive: make sure you're - not slowing down the whole site! - + An additional challenge is when the "feedback" tables are in a separate - database: can no longer issue normal SQL queries - + What are these "flag" columns for? - + Exactly: no one knows. Often must look into code, or compare data to - production representation to guess meaning. In Yelp, =0x1= often means - "inactive", so we probably don't want to count that feedback - -** Data Warehouse :slide:two_col: - + Data located on same system - + Organized for analytics queries - + Requires extra maintenance and understanding of construction - - [[file:img/Ikea-Warehouse.jpg]] -*** No free lunch :notes: - + A strong data warehouse can be a big improvement over operational data - + Hopefully, someone has already cleaned, joined data in a way that makes - sense! - + Optimized for long running queries: less fear of brining down website! - + But you must learn how that process was accomplished in order to understand - potential problems - + How to handle missing data? - + We'll go into more detail about how data warehouse schemas compare to - online ones later in the course - -** Unstructured :slide: - + Haphazard collection of data - + Unclear what structure should be - + Examples: Web logs, text, multimedia - + Must extract structure eventually -*** Yelp JSON logs :notes: - + When developing a web application, new context or details become - important: how long did certain requests take? What link did a user follow - to a website? - + Relational Databases aren't well suited for these wide varieties of - potential attributes that don't apply to all items - + So the current work around is just to write all useful information down in - a log, and extract what is needed later - + Text, like business reviews, another example: desired structure changes - radically between questions: How many words? Characters? What is the sentiment? - + Pictures can contain attributes like color depth, length, width - + First step of data mining is often imposing structure on data: the data is - not inherently unstructured, it just is unclear what the structure *should be* - until query time - -** Search Logs Example :slide: -#+begin_src log -193.139.1 jim [10/Oct/2013:13:55:36 -0700] "GET /search?query=headache HTTP/1.1" 200 9288 -282.482.3 shreyas [10/Oct/2013:13:56:36 -0700] "GET /search?query=bananas HTTP/1.1" 200 2929 -345.114.1 steven [10/Oct/2013:13:56:37 -0700] "GET /search?query=cold HTTP/1.1" 200 8232 -10.328.52 anne [10/Oct/2013:13:56:39 -0700] "GET /search?query=flu+shot HTTP/1.1" 200 2342 -10.328.52 lily [10/Oct/2013:13:57:40 -0700] "GET /search?query=i290 HTTP/1.1" 200 2342 -#+end_src - -| userName | date | query | -| jim | 10/Oct/2013:13:55:36 -0700 | headache | -| shreyas | 10/Oct/2013:13:56:36 -0700 | bananas | -| steven | 10/Oct/2013:13:56:37 -0700 | cold | -| anne | 10/Oct/2013:13:56:39 -0700 | flu shot | -| lily | 10/Oct/2013:13:57:40 -0700 | i290 | - -*** Imposing Structure :notes: - + Extract only the rows we know follow a format - + Format queries from some encoding (eg. URL) to standardized format - -** External APIs :slide: - + Better documented than internal data! - + More limited in amount and detail - + Commonly HTTP/REST based -*** Motivation :notes: - + Companies are often searching for other ways to leverage their data - + Both for immediate business purposes, and for brand recognition - + Twitter more (in)famous example - + NYTimes another good option - -** NYTimes API Example :slide: - + [[http://developer.nytimes.com/docs/read/article_search_api][Article Search API]] - + http://api.nytimes.com/svc/search/v1/article?format=json&query=ballot&api-key=6578bab7f8c3808ce4c392edc9a793f0:8:5717915 -*** Accessing these :notes: - + More info on how to access these APIs is in the Web Architecture class, - but feel free to ask Shreyas or I about how best to access them - -** Data Sets :slide: - + Download large, curated set of data all at once - + Formats vary, but usually documented - + Can be useful to combine with other datasets or APIs - -[[file:img/kaggle-digits.png]] -*** Research :notes: - + Data sets commonly used in research: can compare different techniques on - same data to understand advantages - + Sizes can range to a few MB to GB - + JSON, CSV, XML all potential formats. Cleaning, organization for your - question again becomes an important aspect - -** Data Set Example :slide:two_col: - + [[http://www.grouplens.org/node/73][MovieLens Data Sets]] - + [[https://www.kaggle.com/c/digit-recognizer][Kaggle Digit Recognizer]] - + [[https://bitly.com/bundles/hmason/1][Hilary Mason's Data Sets]] - -[[file:img/video.jpg]] - -* Exploring Data :slide: - + Data sets are frequently too large to fit in standard tools like Excel or - Word - + Simplest to explore on the command line - + Homework will be exploring a data set of your choice -** Size :notes: - + Some formats will not be easily parsed into Excel: eg. JSON, XML - + Word will be slow, or unworkable for GB size data - + CLI provides many composable tools for text manipulation - -* Yelp Academic Dataset :slide: - + [[http://yelp.com/academic_dataset][Yelp Data Set]] covers reviews, users, - businesses - + To download, you'll need to sign up: process takes ~24 hours for approval - + Use .edu email -** Example :notes: - + We'll use this as an example, you can use any data set of your choice - + Just for HW, don't need to use for project - -* CLI introduction :slide: - + Standard commands available in [[http://cli.learncodethehardway.com][Learn CLI the hard way]] - + All example will be run on =ischool.berkeley.edu= - + Sheyas and I available for more help -** Help :notes: - + If you're new, don't be intimidated. - + Security policies ensure you can't break anything besides your own files - + Keep backups of important stuff anyway - -* =wget= :slide: - + Used for downloading files - + Downloading with the browser is fine, but sometimes nice to use faster - connection, or download it directly to machine you're working on -#+begin_src bash -$ wget 'http://www.grouplens.org/system/files/ml-100k.zip' -#+end_src -** Command :notes: - + Just =wget URL= - + I like to use quotes in case there are special characters in the URL, eg - =?= - + Will download to current directory, same name as remote file - -* =scp= :slide: - + Copy a file to or from a remote machine - + Uses same connection as SSH, but copies data instead - + Example: Copy data you've downloaded in your browser -#+begin_src bash -$ scp ~/Downloads/ml-100k.zip jblomo@ischool.berkeley.edu: -# or -$ scp ~/Downloads/ml-100k.zip jblomo@ischool.berkeley.edu:i290/movielens-100k.zip -#+end_src -** Command :notes: - + Trailing =:= is important: signifies remote machine - + If you don't specify path or filename, will copy the file with the same - name into your home directory - -* =gunzip= =unzip= :slide: - + Uncompress data sets for simpler, faster manipulation -#+begin_src bash -$ unzip ml-100k.zip -# or -$ gunzip yelp_academic_dataset.json.gz -#+end_src -** Commands :notes: - + unzip :: expand potentially many file, leave original alone - + gunzip :: expand original file, leaving only the uncompressed version - -* =less= :slide: - + View a file - + History: original command was called =more= to see a file a page at a time - + "Less is more" -#+begin_src bash -less yelp_academic_dataset.json -#+end_src - -* Searching in =less= :slide: - + =/= (forward slash) lets you input search text - + =q= will quit -#+begin_src less -/type": "user" -/type": "review" -#+end_src -** Command :notes: - + Useful for finding specific instances to investigate - -* =grep= :slide: - + Find and print lines matching a "regular expression" - + [[http://www.regular-expressions.info/quickstart.html][Regular expressions]] are "find" on steroids, but you can use simple strings -#+begin_src bash -$ grep 'type": "review"' yelp_academic_dataset.json -#+end_src - -* =wc= :slide: - + "wordcount" counts characters, words, lines - + Most useful in data sets for lines: =-l= -#+begin_src bash -$ wc -l yelp_academic_dataset.json -474434 yelp_academic_dataset.json -#+end_src - -* Composable :slide: - + Genius of Unix: do one thing well, compose commands to get what you want - + =|= pipe characters "sends" output from one program to the input of another - + How many reviews in the data set? -#+begin_src bash -$ grep 'type": "review"' yelp_academic_dataset.json | wc -l -330071 -$ egrep -o 'business_id": "\w+"' yelp_academic_dataset.json | sort -u | wc -l -9592 -#+end_src - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-08-Lab.html b/slides/2013-02-08-Lab.html deleted file mode 100644 index 12967f0..0000000 --- a/slides/2013-02-08-Lab.html +++ /dev/null @@ -1,288 +0,0 @@ - - - - -2013-02-08-Lab - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-08-Lab

- - - - -
-

1 Lab: Data Stats    slide

-
- -
    -
  • Obtain California (CA) campaign finance data -
    • -
  • Decompress -
    • -
  • Manually check -
    • -
  • Run code/stats.py -
    • -
  • Edit code/stats.py to add functionality -
    • -
  • Pull Request submission -
    • -
- - -
- -
- -
-

2 Data    slide

-
- -
    -
  • campaign finance data -
    • -
  • We'll just be using CA data -
    • -
  • Decompress and investigate using the tools we discussed -
    • -
- - -
- -
- -
-

3 Code    slide

-
- - - - -
$ git checkout master
-$ git pull jblomo master
-$ git checkout -b hw-stats
-
- -
    -
  • Run and edit code/stats.py -
    • -
- - -
- -
- -
-

4 Stats    slide

-
- -
    -
  • Minimum -
    • -
  • Maxiumum -
    • -
  • Mean -
    • -
  • Median -
    • -
  • Standard Deviation -
    • -
  • Candidates -
    • -
  • Normalized sample contributions -
    • -
- - -
- -
- -
-

5 Extra Credit    slide

-
- -
    -
  • Extra credit is used to get you up to 100% -
    • -
  • On the current assignment -
    • -
  • Also helpful for learning topics more in depth -
    • -
  • You may do partial extra credit -
    • -
- - -
- -
-

5.1 Overall Extra Credit    notes

-
- -
    -
  • EC that applies to overall grade will not be assigned -
    • -
- - -
-
- -
- -
-

6 Extra Credit    slide

-
- -
    -
  • Stats per candidate -
    • -
  • z-score -
    • -
- - -
- -
- -
-

7 Git usage    slide

-
- -
    -
  • All edits, commits, pushes should happen on a hw- or project branch -
    • -
  • git status -
    • -
  • All pulls (typically from jblomo) should happen on master branch -
      -
    • If you use an editor connected to ischool server, make sure you are - either editing or using git -
      • -
    - -
    • -
- - -
- -
-

7.1 Exceptions    notes

-
- -
    -
  • There are exceptions but know what you're trying to do -
    • -
  • External editors can write back files after you've changed git branches -
    • -
- - -
-
- -
- -
-

8 Submission    slide

-
- -
    -
  • GitHub pull request -
    • -
  • If something is going wrong, submit by email: jblomo@ischool, - shreyas@ischool -
    • -
  • We'll help you submit the pull request, but HW will be full credit -
    • -
- - -
- -
-

8.1 Submission    notes

-
- -
    -
  • The pull request is a way for Shreyas and I to easily see changes, grade -
    • -
  • It'll give you good experience, but it is not a fundemental skill of the - class, so I'm not too worried about it -
    • -
- - - - - - - -
-
-
-
- -
-

Date: 2013-02-08 13:51:10 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-08-Lab.org b/slides/2013-02-08-Lab.org deleted file mode 100644 index a693ef0..0000000 --- a/slides/2013-02-08-Lab.org +++ /dev/null @@ -1,78 +0,0 @@ -* Lab: Data Stats :slide: - + Obtain California (CA) [[http://www.fec.gov/disclosurep/PDownload.do][campaign finance data]] - + Decompress - + Manually check - + Run =code/stats.py= - + Edit =code/stats.py= to add functionality - + Pull Request submission - -* Data :slide: - + [[http://www.fec.gov/disclosurep/PDownload.do][campaign finance data]] - + We'll just be using CA data - + Decompress and investigate using the tools we discussed - -* Code :slide: -#+begin_src bash -$ git checkout master -$ git pull jblomo master -$ git checkout -b hw-stats -#+end_src - + Run and edit =code/stats.py= - -* Stats :slide: - + Minimum - + Maximum - + Mean - + Median - + Standard Deviation - + Candidates - + Normalized sample contributions - -* Extra Credit :slide: - + Extra credit is used to get you *up to 100%* - + On the *current assignment* - + Also helpful for learning topics more in depth - + You may do partial extra credit -** Overall Extra Credit :notes: - + EC that applies to overall grade will not be assigned - -* Extra Credit :slide: - + Stats per candidate - + z-score - -* Git usage :slide: - + All edits, commits, pushes should happen on a =hw-= or =project= branch - + =git status= - + All pulls (typically from =jblomo=) should happen on =master= branch - + If you use an editor connected to ischool server, make sure you are - *either* editing *or* using git -** Exceptions :notes: - + There are exceptions but know what you're trying to do - + External editors can write back files *after* you've changed git branches - -* Submission :slide: - + GitHub pull request - + If something is going wrong, submit by email: jblomo@ischool, - shreyas@ischool - + We'll help you submit the pull request, but HW will be full credit -** Submission :notes: - + The pull request is a way for Shreyas and I to easily see changes, grade - + It'll give you good experience, but it is not a fundamental skill of the - class, so I'm not too worried about it - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-08-Preprocessing.html b/slides/2013-02-08-Preprocessing.html deleted file mode 100644 index 4342f83..0000000 --- a/slides/2013-02-08-Preprocessing.html +++ /dev/null @@ -1,940 +0,0 @@ - - - - -2013-02-08-Preprocessing - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-08-Preprocessing

- - - - -
-

1 Preprocessing    slide

-
- - -
- -
- -
-

2 Real World is Dirty    slide

-
- -
-
Incomplete
missing timestamps for actions -
-
Noisy
salary = -10 -
-
Inconsistent
age: 42, birthday: 1997-03-07 -
-
- - -
- -
-

2.1 Types of dirty    notes

-
- -
-
Incomplete
lacking some attribute values, containing only aggregate - data. Eg. We often regret not including timestamps on different actions - like UFCing, instead of tracking total votes (aggregation) -
-
Noisy
Containing errors, like impossible salary data, or decimals in the - wrong place -
-
Inconsistent
If there's every two fields that depend on each other, in a - large dataset you'll find them disagreeing. Errors ofen come from failures: - processes failing halfway into updating -
-
- - -
-
- -
- -
-

3 Causes of Problems    slide

-
- -
    -
  • Humans -
    • -
  • Software -
    • -
  • Hardware -
    • -
- - -
- -
-

3.1 Problems    notes

-
- -
    -
  • Berkeley experiment to measure temperature across campus -
    • -
  • Turned out average on campus much warmer than external weather services - predicted -
    • -
  • But sample data looked in line with predictions -
    • -
  • Problem: one monitoring station right next to air conditioning unit! -
    • -
  • Hardware failure rare, but with large numbers of machines, probable. Eg. - RAM can suffer ~1 bit/hour/gigabyte (ECC can help) -
    • -
- - -
- -
- -
-

3.2 Inconsistent Different Sources    slide

-
- -
    -
  • Great value in combining data sources -
    • -
  • Challenge is merging them together, removing duplicates -
    • -
  • Example: Business names -
    • -
- -
- -
- -
-

3.3 Business names    notes

-
- -
    -
  • Starbucks vs. Starbucks Coffee Shop -
    • -
  • Buck's vs Bucks -
    • -
  • Trying to use address? Stackbucks vs. Starbucks across the street -
    • -
  • Best strategy here is to use DM/ML techniques on the combination of - features to determine likelihood of match. We'll discuss specific - algorithms later in the course -
    • -
- - -
-
- -
- -
-

4 Preprocessing    slide

-
- -
-
Cleaning
fill missing values, smooth noisy data, identify or remove - outliers, resolve inconsistencies -
-
Integration
merging data from multiple sources -
-
Reduction
obtain a smaller data set that can sufficiently answer - important questions -
-
Transformation
change data to a form that is easier to mine or analyze -
-
- - -
- -
-

4.1 Flu Trend Problems (Questions)    notes

-
- -
    -
  • We have millisecond search resolution, but will only be plotting on a per day basis -
    • -
  • We have the exact text of each query, but just care if it is about the flu or not -
    • -
  • Flu Trends, we sometimes see out of control search bots doing 100,000s of searches per day -
    • -
  • Mobile phone searches and web searches hit different machines, software, logs -
    • -
  • We have IPs in the logs, but will by plotting against geographical areas -
    • -
- - -
-
- -
- -
-

5 Missing Values    slide two_col

-
- - - -- - - - - - - - - -
PersonHeight
Jim6'0
Ashley-
Sam5'11
Alice5'9
Kate-
- - -

-img/tallest-shortest-man.jpg -

-
- -
-

5.1 What to do?    notes

-
- -
    -
  • (Heights are made up) -
    • -
  • We want to get an average class height -
    • -
  • Q: What to do with missing rows? -
    • -
  • ignore, fill, constant, average, average wrt gender -
    • -
- - -
-
- -
- -
-

6 Fill Missing Values    slide animate

-
- -
    -
  • Ignore the record -
    • -
  • Find value manually -
    • -
  • Global constant -
    • -
  • Average -
    • -
  • Average with respect to class -
    • -
  • "Most probable" -
    • -
- - -
- -
-

6.1 Details    notes

-
- -
-
Trade-offs
core to engineering -
-
Ignore
simply drop from data set. Hope there are not too many to affect - answer. Drawbacks? When missing values are all same class (skew data) -
-
Find value manually
Even for a small class, might be difficult. Get - ruler, measure them. For historical data, impossible. -
-
Global constant
replace with "N/A" or "6 foot". Can skew data, or cause - data to pop in other analysis (all grouped together) -
-
Average
Mean or median. Either one has potential problems. -
-
Average with respect to class
gender. Average female/male height to fill - in values -
-
"Most probable"
Think of as another step from avg -> class avg. Now - throw in other details: age, family history, shoe size. Then weight - depending on how much those factors are correlated. Pretty soon you have a - regression or Bayesian model, which will cover later -
-
- - -
-
- -
- -
-

7 Normalization    slide

-
- -
    -
  • Type of data transformation to make reasoning and comparison easier -
    • -
  • Is 6' tall? -
    • -
  • Coefficients on attributes in regressions understandable -
    • -
- - -
- -
-

7.1 Context, Comparison    notes

-
- -
    -
  • 6' Might be tall for this class, but not on a basketball team -
    • -
  • How to know when a data point "average" or towards the top of a range? -
    • -
  • For our housing model, we wanted to use sq. footage and # of bedrooms. But - the sq. footage number is huge compared to bedrooms. If we didn't - normalize, a formula for determine house price might seem to indicate that - # of bedrooms was way more important -
    • -
- - -
-
- -
- -
-

8 Min-max    slide

-
- -

img/min-max.gif -

-
- -
-

8.1 New Range    notes

-
- -
    -
  • Typically new range is -
      -
    • [0-1] (thought of as %) -
      • -
    • [-1-1] (though of as bad->good -
      • -
    - -
    • -
- - -
-
- -
- -
-

9 Z-score    slide

-
- -

img/z-score.gif -

-
- -
-

9.1 Uses    notes

-
- -
    -
  • When you want a relative measure of deviation -
    • -
  • When you have a distribution estimate, but are unsure of absolute min-max -
    • -
- - -
-
- -
- -
-

10 Comparison    slide

-
- -

img/outliers.png -img/outliers-minmax-zscore.png -

-
- -
-

10.1 Min-max vs Z-score    notes

-
- -
    -
  • Min-max: Known range -
    • -
  • Z-score: more expressive range -
    • -
  • Min-max: requires knowing min-max -
    • -
  • Z-score: can estimate with sampling or informed guess -
    • -
- - -
-
- -
- -
-

11 Removing Noise    slide

-
- -
-
Binning
create B bins << N data samples, use aggregate statistic of bin - for value -
-
Regression
fit data to a function, use function value -
-
Outlier analysis
find outlying points, understand and/or ignore them -
-
- - -
- -
-

11.1 Monitoring Problem    notes

-
- -
    -
  • For the problem encountered in temperature monitoring, which makes the most - sense? -
    • -
- - -
- -
- -
-

11.2 Trade-offs    slide

-
- -
-
Binning
Simple way to remove outliers, but difficult to pick buckets - correctly -
-
Regression
If one metric is a direct function of another, what extra - information does the value provide? -
-
Outlier analysis
Manual process of understanding outliers, ignoring them - can obscure some analysis (eg. income disparity) -
-
- - -
- -
-

11.2.1 Trade-offs again    notes

-
- -
    -
  • Remember: this class is exposing you to potential tools, up to you to be - asking the right questions, selecting the appropriate algorithms, - interpreting results -
    • -
- - -
-
-
- -
- -
-

12 Data integration    slide

-
- -
    -
  • Merging two data sources -
    • -
  • Problem: uniquely identify a concept in both sources -
    • -
  • Find data points that are very "close" to each other, call them the same - with some probability -
    • -
  • Example: Yelp Menu Data -
    • -
- - -
- -
-

12.1 Yelp Menu Data    notes

-
- -
    -
  • Recently launched menu data -
    • -
  • Takes data about the restaurant menu, find reviews & pictures referring to - the menu item -
    • -
  • Joins them together -
    • -
  • Many different metrics for "close": remember them? -
    • -
- - -
-
- -
- -
-

13 Other measures of "close"    slide

-
- -

Are A and B close? -

- -- - - - - - - - - -
AB
260
5150
6180
10300
13390
- - -
- -
-

13.1 Correlation    notes

-
- -
    -
  • Imagine A and B have several different dimensions, maybe things like - length, height, width, radius -
    • -
  • Are they similar? -
    • -
  • On one hand no: clearly different order of magnitude -
    • -
  • Another way to think about similarity is correlation -
    • -
  • All of B dimensions are 30x of A -
    • -
  • Maybe just using different units! -
    • -
  • If I plotted A and B and x,y, what would the result look like? -
    • -
- - -
-
- -
- -
-

14 Χ2 Correlation Test    slide

-
- -

img/correlation.png - img/chiequation.jpg -

-
- -
-

14.1 Motivation    notes

-
- -
    -
  • Answer: a straight line -
    • -
  • So a correlation coefficient gives a sense of how closely linearly - related two data sets are -
    • -
  • Note, besides positive & negative, the slop does not affect the correlation - score, just how well fit the data is -
    • -
  • Also note I said linear: patterns may still be exhibited, but they are not - linearly related, eg 30x -
    • -
  • Details of test are in book, you are expected to understand it -
    • -
  • Motivation: how different are the observed values from the expected? -
    • -
  • Expected is calculated using probability with the assumptions that the sets - are independent -
    • -
- - -
-
- -
- -
-

15 Covariance & Correlation    slide

-
- -
    -
  • Correlation is "normalized" covariance -
    • -
  • Covariance describes the degree to which two data sets track each other in - units of of the two data sets -
    • -
  • Correlations describes the degree of similarity without units -
    • -
- - -
- -
-

15.1 Use in industry    notes

-
- -
    -
  • Χ2 used most commonly, handy to have an expected [0-1] range -
    • -
  • "Correlation does not imply causation" -
    • -
  • A->B, B->A, C->A,B, A->B->A…, coincidence -
    • -
- - -
-
- -
- -
-

16 Data Reduction    slide two_col

-
- -
-
Dimensionality
remove attributes that are the same or similar to other - attributes -
-
Numerosity
represent or aggregate the data, sometimes with precision loss -
-
Compression
generalized techniques to decrease the number of bytes needed - to store data -
-
- -

img/compress-car.jpg -

-
- -
-

16.1 Deep Dive    notes

-
- -
    -
  • We're only going to cover selected topics in these areas. -
    • -
  • When reading, make sure to understand the intuition behind the other - techniques, but if we don't cover it in lecture, you won't need to - calculate it in midterm -
    • -
  • Ask questions about the concepts you don't understand! That's what - separates this class from a book :) -
    • -
  • But still potentially useful for your projects! -
    • -
  • img: http://www.flickr.com/photos/marcovdz/4520986339/sizes/o/in/photostream/ -
    • -
- - -
-
- -
- -
-

17 Subset Selection    slide

-
- -
    -
  • Two many attributes? -
    • -
  • Ignore some -
    • -
  • Tricky part: which to ignore? -
    • -
  • height x width = area -
    • -
- - -
- -
-

17.1 Simple to Sophisticated    notes

-
- -
    -
  • Ignore the ones that are not helpful -
    • -
  • Ignore an attribute highly correlated with another (cm, in) -
    • -
  • Ignore an attribute that can be built from others -
    • -
- - -
-
- -
- -
-

18 Principal Component Analysis    slide

-
- -

img/GaussianScatterPCA.png -

    -
  • Map data to a locatoin along a few vectors -
    • -
- - -
- -
-

18.1 Higher dimensions    notes

-
- -
    -
  • Remember, 2 dimensions might not make much sense, but becomes useful in - higher number of dimensions -
    • -
  • These points described by two attributes, <x,y> -
    • -
  • What if we wanted to describe them in just 1 dimension? -
    • -
  • Pick some good vectors (in our case 1) -
    • -
  • Describe where a point is located using only those vectors -
    • -
- - -
-
- -
- -
-

19 Netflix and PCA    slide

-
- -
    -
  • A user may have many preferences: Mission Impossible, Love Actually, Man - from Nowhere, … -
    • -
  • Instead of keeping track of every preference, we can summarize -
    • -
  • Action, RomCom, Foreign -
    • -
- - -
- -
-

19.1 Summarize in discovered dimensions    notes

-
- -
    -
  • With 3 or more "categories", we can reconstruct the user's likely - preferences -
    • -
  • Dimensions don't necessarily fit into human notions: probably is not an - "foreign" dimension, but a subtle combination of other aspects -
    • -
- - - - - - - - -
-
-
-
- -
-

Date: 2013-02-08 13:49:51 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-08-Preprocessing.org b/slides/2013-02-08-Preprocessing.org deleted file mode 100644 index c229b9f..0000000 --- a/slides/2013-02-08-Preprocessing.org +++ /dev/null @@ -1,271 +0,0 @@ -* Preprocessing :slide: - -* Real World is Dirty :slide: - + Incomplete :: missing timestamps for actions - + Noisy :: salary = -10 - + Inconsistent :: age: 42, birthday: 1997-03-07 -** Types of dirty :notes: - + Incomplete :: lacking some attribute values, containing only aggregate - data. Eg. We often regret not including timestamps on different actions - like UFCing, instead of tracking total votes (aggregation) - + Noisy :: Containing errors, like impossible salary data, or decimals in the - wrong place - + Inconsistent :: If there's every two fields that depend on each other, in a - large dataset you'll find them disagreeing. Errors ofen come from failures: - processes failing halfway into updating - -* Causes of Problems :slide: - + Humans - + Software - + Hardware -** Problems :notes: - + Berkeley experiment to measure temperature across campus - + Turned out average on campus much warmer than external weather services - predicted - + But sample data looked in line with predictions - + Problem: one monitoring station right next to air conditioning unit! - + Hardware failure rare, but with large numbers of machines, probable. Eg. - RAM can suffer ~1 bit/hour/gigabyte (ECC can help) - -** Inconsistent Different Sources :slide: - + Great value in combining data sources - + Challenge is merging them together, removing duplicates - + Example: Business names -** Business names :notes: - + Starbucks vs. Starbucks Coffee Shop - + Buck's vs Bucks - + Trying to use address? Stackbucks vs. Starbucks across the street - + Best strategy here is to use DM/ML techniques on the *combination* of - features to determine likelihood of match. We'll discuss specific - algorithms later in the course - -* Preprocessing :slide: - + Cleaning :: fill missing values, smooth noisy data, identify or remove - outliers, resolve inconsistencies - + Integration :: merging data from multiple sources - + Reduction :: obtain a smaller data set that can sufficiently answer - important questions - + Transformation :: change data to a form that is easier to mine or analyze -** Flu Trend Problems (Questions) :notes: - + We have millisecond search resolution, but will only be plotting on a per day basis - + We have the exact text of each query, but just care if it is about the flu or not - + Flu Trends, we sometimes see out of control search bots doing 100,000s of searches per day - + Mobile phone searches and web searches hit different machines, software, logs - + We have IPs in the logs, but will by plotting against geographical areas - -* Missing Values :slide:two_col: -| Person | Height | -| Jim | 6'0 | -| Ashley | - | -| Sam | 5'11 | -| Alice | 5'9 | -| Kate | - | - -[[file:img/tallest-shortest-man.jpg]] -** What to do? :notes: - + (Heights are made up) - + We want to get an average class height - + Q: What to do with missing rows? - + ignore, fill, constant, average, average wrt gender - -* Fill Missing Values :slide:animate: - + Ignore the record - + Find value manually - + Global constant - + Average - + Average with respect to class - + "Most probable" -** Details :notes: - + Trade-offs :: core to engineering - + Ignore :: simply drop from data set. Hope there are not too many to affect - answer. Drawbacks? When missing values are all same class (skew data) - + Find value manually :: Even for a small class, might be difficult. Get - ruler, measure them. For historical data, impossible. - + Global constant :: replace with "N/A" or "6 foot". Can skew data, or cause - data to pop in other analysis (all grouped together) - + Average :: Mean or median. Either one has potential problems. - + Average with respect to class :: gender. Average female/male height to fill - in values - + "Most probable" :: Think of as another step from avg -> class avg. Now - throw in other details: age, family history, shoe size. Then weight - depending on how much those factors are correlated. Pretty soon you have a - regression or Bayesian model, which will cover later - -* Normalization :slide: - + Type of data transformation to make reasoning and comparison easier - + Is 6' tall? - + Coefficients on attributes in regressions understandable -** Context, Comparison :notes: - + 6' Might be tall for this class, but not on a basketball team - + How to know when a data point "average" or towards the top of a range? - + For our housing model, we wanted to use sq. footage and # of bedrooms. But - the sq. footage number is huge compared to bedrooms. If we didn't - normalize, a formula for determine house price might seem to indicate that - # of bedrooms was way more important - -* Min-max :slide: -[[file:img/min-max.gif]] -** New Range :notes: - + Typically new range is - + [0-1] (thought of as %) - + [-1-1] (though of as bad->good - -* Z-score :slide: -[[file:img/z-score.gif]] -** Uses :notes: - + When you want a relative measure of deviation - + When you have a distribution estimate, but are unsure of absolute min-max - -* Comparison :slide: -[[file:img/outliers.png]] -[[file:img/outliers-minmax-zscore.png]] -** Min-max vs Z-score :notes: - + Min-max: Known range - + Z-score: more expressive range - + Min-max: requires knowing min-max - + Z-score: can estimate with sampling or informed guess - -* Removing Noise :slide: - + Binning :: create B bins << N data samples, use aggregate statistic of bin - for value - + Regression :: fit data to a function, use function value - + Outlier analysis :: find outlying points, understand and/or ignore them -** Monitoring Problem :notes: - + For the problem encountered in temperature monitoring, which makes the most - sense? - -** Trade-offs :slide: - + Binning :: Simple way to remove outliers, but difficult to pick buckets - correctly - + Regression :: If one metric is a direct function of another, what extra - information does the value provide? - + Outlier analysis :: Manual process of understanding outliers, ignoring them - can obscure some analysis (eg. income disparity) -*** Trade-offs again :notes: - + Remember: this class is exposing you to potential tools, up to you to be - asking the right questions, selecting the appropriate algorithms, - interpreting results - -* Data integration :slide: - + Merging two data sources - + Problem: uniquely identify a concept in both sources - + Find data points that are very "close" to each other, call them the same - with some probability - + Example: [[http://www.yelp.com/menu/tartine-bakery-san-francisco][Yelp Menu Data]] -** Yelp Menu Data :notes: - + Recently launched menu data - + Takes data about the restaurant menu, find reviews & pictures referring to - the menu item - + Joins them together - + Many different metrics for "close": remember them? - -* Other measures of "close" :slide: -Are =A= and =B= close? -| A | B | -| 2 | 60 | -| 5 | 150 | -| 6 | 180 | -| 10 | 300 | -| 13 | 390 | -** Correlation :notes: - + Imagine =A= and =B= have several different dimensions, maybe things like - length, height, width, radius - + Are they similar? - + On one hand no: clearly different order of magnitude - + Another way to think about similarity is correlation - + All of =B= dimensions are 30x of =A= - + Maybe just using different units! - + If I plotted =A= and =B= and x,y, what would the result look like? - -* Χ^2 Correlation Test :slide: - [[file:img/correlation.png]] - [[file:img/chiequation.jpg]] -** Motivation :notes: - + Answer: a straight line - + So a correlation coefficient gives a sense of how closely *linearly* - related two data sets are - + Note, besides positive & negative, the slop does not affect the correlation - score, just how well fit the data is - + Also note I said linear: patterns may still be exhibited, but they are not - linearly related, eg 30x - + Details of test are in book, you are expected to understand it - + Motivation: how different are the observed values from the expected? - + Expected is calculated using probability with the assumptions that the sets - are *independent* - -* Covariance & Correlation :slide: - + Correlation is "normalized" covariance - + Covariance describes the degree to which two data sets track each other in - units of of the two data sets - + Correlations describes the degree of similarity without units -** Use in industry :notes: - + Χ^2 used most commonly, handy to have an expected [0-1] range - + "Correlation does not imply causation" - + A->B, B->A, C->A,B, A->B->A..., coincidence - -* Data Reduction :slide:two_col: - + Dimensionality :: remove attributes that are the same or similar to other - attributes - + Numerosity :: represent or aggregate the data, sometimes with precision loss - + Compression :: generalized techniques to decrease the number of bytes needed - to store data -[[file:img/compress-car.jpg]] -** Deep Dive :notes: - + We're only going to cover selected topics in these areas. - + When reading, make sure to understand the intuition behind the other - techniques, but if we don't cover it in lecture, you won't need to - calculate it in midterm - + Ask questions about the concepts you don't understand! That's what - separates this class from a book :) - + But still potentially useful for your projects! - + img: http://www.flickr.com/photos/marcovdz/4520986339/sizes/o/in/photostream/ - -* Subset Selection :slide: - + Two many attributes? - + *Ignore some* - + Tricky part: which to ignore? - + height x width = area -** Simple to Sophisticated :notes: - + Ignore the ones that are not helpful - + Ignore an attribute highly correlated with another (cm, in) - + Ignore an attribute that can be built from others - -* Principal Component Analysis :slide: -[[file:img/GaussianScatterPCA.png]] - + Map data to a locatoin along a few vectors -** Higher dimensions :notes: - + Remember, 2 dimensions might not make much sense, but becomes useful in - higher number of dimensions - + These points described by two attributes, - + What if we wanted to describe them in just 1 dimension? - + Pick some good vectors (in our case 1) - + Describe where a point is located using only those vectors - -* Netflix and PCA :slide: - + A user may have many preferences: Mission Impossible, Love Actually, Man - from Nowhere, ... - + Instead of keeping track of every preference, we can summarize - + Action, RomCom, Foreign -** Summarize in discovered dimensions :notes: - + With 3 or more "categories", we can reconstruct the user's likely - preferences - + Dimensions don't necessarily fit into human notions: probably is not an - "foreign" dimension, but a subtle combination of other aspects - - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-08-Probability.html b/slides/2013-02-08-Probability.html deleted file mode 100644 index 42b74bc..0000000 --- a/slides/2013-02-08-Probability.html +++ /dev/null @@ -1,855 +0,0 @@ - - - - -2013-02-08-Probability - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-08-Probability

- - - - -
-

1 Probability    slide

-
- - -
- -
- -
-

2 Nomenclature    slide

-
- -
-
Record
a single entity or concept. Also: data object, sample, example, - instance, data point -
-
Feature
a characteristic or way of describing a record. Also: attribute, - dimension, variable, signal -
-
- - -
- -
-

2.1 Slightly different from book    notes

-
- -
    -
  • The meanings do carry different connotations, but are generally - transferable -
    • -
  • Eg. dimensions is usually used in the math domain -
    • -
  • Feature is usually used in the ML domain -
    • -
- - -
- -
- -
-

2.2 Feautre Types    slide

-
- -
-
Binary
True/False. Also: 0/1 -
-
Numeric
Involving numbers. Also: integer, float, double -
-
Ordinal
Feature with sortable values. -
-
Discrete
countable, finite set. Also: classes -
-
Continuous
unbounded numeric number. Also: integer, float, double -
-
Enumerated
feature named, discrete values. Also: nominal, classed -
-
- - -
- -
-

2.2.1 Rain data set    notes

-
- -
    -
  • Stored did/dot not rain -
    • -
  • Stored how many inches it rained -
    • -
  • Stored the day as an integer offset from Jan 1 1970 -
    • -
  • Stored weather information: Sunny, Partly Sunny, Cloudy, Rainy -
    • -
  • Stored barometer reading -
    • -
  • Stored day of the week -
    • -
- - -
-
-
- -
- -
-

3 Central Tendency    slide

-
- -

img/skew-normal.png -

-
- -
-

3.1 Define    notes

-
- -
-
Mean
"average" all data points divided by size of set -
-
Median
middle value -
-
Mode
The value most likely to be picked -
-
discrete
most common value -
-
continuous
max probability density function -
-
- -
-
- - -
- -
- -
-

3.2 Skew Positive    slide

-
- -

img/skew-positive.png -

- -
- -
-

3.3 Skew    notes

-
- -
    -
  • Think about mean - mode -
    • -
  • Or think about where the "tail" is -
    • -
- - -
- -
- -
-

3.4 Skew Negative    slide

-
- -

img/skew-negative.png -

-
-
- -
- -
-

4 The Long Tail    slide two_col

-
- -

img/Long_tail.svg.png -

    -
  • Most popular are very popular -
    • -
  • Everything else, not so much -
    • -
  • But there's a lot of everything else -
    • -
- - -
- -
-

4.1 Movies    notes

-
- -
    -
  • Current releases: millions of people watching -
    • -
  • Older movies are rented by < 1 person a week -
    • -
  • What is the skew? -
    • -
  • Power law distribution (please follow up on Wikipedia or a stats class) -
    • -
  • Distributions are important, but will only be covered as necessary -
    • -
- - -
-
- -
- -
-

5 Dispersion    slide

-
- -

img/dispersion.png -

    -
  • Centrality not the whole story -
    • -
- - -
- -
-

5.1 Differences    notes

-
- -
    -
  • Wildly different data sets can still share many of these characteristics -
    • -
- - -
-
- -
- -
-

6 Quartiles    slide

-
- -

img/quartiles.png -

-
- -
-

6.1 Parts    notes

-
- -
    -
  • Go back to our unskewed normal distribution -
    • -
  • Quartiles divide the data into quarters -
    • -
  • InterQuartile Range is the distance of the middle two quartiles -
    • -
  • BoxPlot is one of the most useful tools for data. For public results, I - almost never want to see scatter plot or bar charts. I want to see box - plots. -
    • -
  • Bottom, we spit it up into standard deviations -
    • -
  • Variance measures, on average, how far points are away from the mean -
    • -
  • Standard deviation is the square root of the variance -
    • -
- - -
-
- -
- -
-

7 Standard Deviation    slide

-
- -

img/stddev.png -

    -
  • Within 1: 68% -
    • -
  • Within 2: 95% -
    • -
  • Within 3: 99.7% -
    • -
- - -
- -
-

7.1 Standard Deviation    notes

-
- -
    -
  • Useful for thinking about what % of outliers you'd like to catch -
    • -
  • We use it for alerting: let us know when we're 2 stddev away from the - median, there's a very small likelihood of that happening -
    • -
- - -
-
- -
- -
-

8 Visualization Tools    slide

-
- -
    -
  • Python: Matplotlib -
    • -
  • R: builtin -
    • -
  • Matlab: builtin -
    • -
  • Octave: builtin (gnuplot) -
    • -
  • HTML: D3.js -
    • -
- - -
- -
-

8.1 Covered later    notes

-
- -
    -
  • Chapter 2 is going to cover some visualization stuff -
    • -
  • We're going to cover visualization a bit later in the course, and more of a - "how its done in industry" -
    • -
  • There is another class on visualization in general -
    • -
- - -
-
- -
- -
-

9 Mathmatical Representation    slide

-
- - - -- - - - - - - -
Bad BoysRobin HoodWaterworld
Prabha132
AJ543
Victor441
- - - - -
[ 1 3 2
-  5 4 3
-  4 4 1 ]
-
- - -
- -
-

9.1 Matrix    notes

-
- -
    -
  • Matrix representations very powerful, as we'll see later in class -
    • -
  • Usually rows are records, columns are attributes -
    • -
  • Sometimes you can think of data in different ways, can take the transpose - of the matrix to get attributes about movies -
    • -
- - -
-
- -
- -
-

10 Waterworld    slide

-
- -

img/waterworld.jpg -

-
- -
- -
-

11 Similarity | Distance    slide

-
- -
    -
  • Two sides of the same coin -
    • -
  • similarity = 1 - distance -
    • -
  • We'll use these metrics for many other algorithms -
    • -
- - -
- -
-

11.1 Core Concept    notes

-
- -
    -
  • Many data mining techniques rely on finding a way to quantify similarity -
    • -
  • When you think about questions like "how similar are two users?" "is this - text plagiarism?" "are these products likely to be purchased together?" -
    • -
  • All are ways of thinking about similarity -
    • -
- - -
-
- -
- -
-

12 Nominal Distance    slide

-
- -
    -
  • Ratio of mismatches to potential matches -
    • -
  • Why can't we take the sum of the mismatches? -
    • -
- - -
- -
-

12.1 Nominal    notes

-
- -
    -
  • Nominal means we can't compare two values: there is no ordering -
    • -
  • All we can do is take ratio of the ones that are exactly the same -
    • -
  • The book describes how to think about this in terms of matrices -
    • -
- - -
-
- -
- -
-

13 Binary Distance    slide

-
- -
    -
  • Could use Nominal Distance: count all exact matches or mismatches -
    • -
  • Could use Numeric Distance: just treat values as 0/1 -
    • -
  • asymmetric binary dissimilarity: don't care about negative matches -
      -
    • mismatches / (positive_matches + mismatches) -
      • -
    - -
    • -
  • asymmetric binary similarity: care more about positive matches than mismatches -
      -
    • positive_matches / (positive_matches + mismatches) -
      • -
    - -
    • -
- - -
- -
-

13.1 Binary    notes

-
- -
    -
  • Nominal problem: for rare attributes, like a disease, two people who - don't have the disease, aren't necessarily very similar -
    • -
- - -
-
- -
- -
-

14 Jaccard Coefficient    slide

-
- -
    -
  • Asymmetric binary similarity -
    • -
  • More commonly used for calculating set similarity -
    • -
  • |intersection| / |union| -
    • -
  • "Jim likes pizza" | "Shreyas likes pizza" -
    • -
- - -
- -
-

14.1 Jaccard    notes

-
- -
    -
  1. Break up into a set -
    2. -
  2. calculate # in intersection -
    3. -
  3. calculate # in union -
    4. -
  4. divide -
    5. -
- - -
-
- -
- -
-

15 Euclidean distance    slide

-
- -
    -
  • Straight line between two points -
    • -
  • Again: usually considered with just (x,y), but can calculate for any number - of dimensions -
    • -
- -

img/euclidean.png -

-
- -
-

15.1 Ordinary    notes

-
- -
    -
  • Distance as you probably learned in grade school -
    • -
- - -
-
- -
- -
-

16 Manhattan distance    slide

-
- -
    -
  • How many blocks would you need to walk between two points? -
    • -
- -

img/manhattan.png -

-
- -
-

16.1 Usefulness    notes

-
- -
    -
  • Obviously useful for maps/directions -
    • -
  • But haven't seen it used much beyond that -
    • -
- - -
-
- -
- -
-

17 Lp norm    slide

-
- -
    -
  • Euclidean distance and Manhattan can be generalized -
    • -
  • Euclidean distance referred to as L2 norm -
    • -
  • Chebyshev distance is L∞ -
    • -
- -

img/lp-norm.png -

-
- -
-

17.1 Lp space    notes

-
- -
    -
  • Important for signal processing, math, other applications -
    • -
  • You may want to study these distances for comparing wave forms, like audio -
    • -
- - -
-
- -
- -
-

18 Ordinal Distance    slide

-
- -
    -
  • Normalize the ordinal rankings -
    • -
  • Use a numerical distance metric -
    • -
- - -
- -
- -
-

19 Cosine Similarity    slide

-
- -
    -
  • Jaccard similarity can work well for sets of roughly equal size -
    • -
  • How to compare sets with a large difference in magnitude? -
    • -
  • Model them as vectors, take the cosign of the angle between -
    • -
- -

img/cosine-similarity.png -

-
- -
-

19.1 Cosign    notes

-
- - - - -
-
- -
- -
-

20 Cosine Example    slide

-
- -
    -
  • "Jim likes pizza" | "Shreyas likes pizza" -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-02-08 13:46:10 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-08-Probability.org b/slides/2013-02-08-Probability.org deleted file mode 100644 index 7a77857..0000000 --- a/slides/2013-02-08-Probability.org +++ /dev/null @@ -1,214 +0,0 @@ -* Probability :slide: - -* Nomenclature :slide: - + Record :: a single entity or concept. Also: data object, sample, example, - instance, data point - + Feature :: a characteristic or way of describing a record. Also: attribute, - dimension, variable, signal -** Slightly different from book :notes: - + The meanings do carry different connotations, but are generally - transferable - + Eg. dimensions is usually used in the math domain - + Feature is usually used in the ML domain - -** Feautre Types :slide: - + Binary :: True/False. Also: 0/1 - + Numeric :: Involving numbers. Also: integer, float, double - + Ordinal :: Feature with sortable values. - + Discrete :: countable, finite set. Also: classes - + Continuous :: unbounded numeric number. Also: integer, float, double - + Enumerated :: feature named, discrete values. Also: nominal, classed -*** Rain data set :notes: - + Stored did/dot not rain - + Stored how many inches it rained - + Stored the day as an integer offset from Jan 1 1970 - + Stored weather information: Sunny, Partly Sunny, Cloudy, Rainy - + Stored barometer reading - + Stored day of the week - -* Central Tendency :slide: -[[file:img/skew-normal.png]] -** Define :notes: - + Mean :: "average" all data points divided by size of set - + Median :: middle value - + Mode :: The value most likely to be picked - + discrete :: most common value - + continuous :: max probability density function - -** Skew Positive :slide: -[[file:img/skew-positive.png]] -** Skew :notes: - + Think about =mean - mode= - + Or think about where the "tail" is - -** Skew Negative :slide: -[[file:img/skew-negative.png]] - -* The Long Tail :slide:two_col: -[[file:img/Long_tail.svg.png]] - + Most popular are *very* popular - + Everything else, not so much - + But there's a lot of everything else -** Movies :notes: - + Current releases: millions of people watching - + Older movies are rented by < 1 person a week - + What is the skew? - + Power law distribution (please follow up on Wikipedia or a stats class) - + Distributions are important, but will only be covered as necessary - -* Dispersion :slide: -[[file:img/dispersion.png]] - + Centrality not the whole story -** Differences :notes: - + Wildly different data sets can still share many of these characteristics - -* Quartiles :slide: -[[file:img/quartiles.png]] -** Parts :notes: - + Go back to our unskewed normal distribution - + Quartiles divide the data into quarters - + InterQuartile Range is the distance of the middle two quartiles - + BoxPlot is one of the most useful tools for data. For public results, I - almost never want to see scatter plot or bar charts. I want to see box - plots. - + Bottom, we spit it up into standard deviations - + Variance measures, on average, how far points are away from the mean - + Standard deviation is the square root of the variance - -* Standard Deviation :slide: - [[file:img/stddev.png]] - + Within 1: 68% - + Within 2: 95% - + Within 3: 99.7% -** Standard Deviation :notes: - + Useful for thinking about what % of outliers you'd like to catch - + We use it for alerting: let us know when we're 2 stddev away from the - median, there's a very small likelihood of that happening - -* Visualization Tools :slide: - + Python: Matplotlib - + R: builtin - + Matlab: builtin - + Octave: builtin (gnuplot) - + HTML: D3.js -** Covered later :notes: - + Chapter 2 is going to cover some visualization stuff - + We're going to cover visualization a bit later in the course, and more of a - "how its done in industry" - + There is another class on visualization in general - -* Mathmatical Representation :slide: -| | Bad Boys | Robin Hood | Waterworld | -| Prabha | 1 | 3 | 2 | -| AJ | 5 | 4 | 3 | -| Victor | 4 | 4 | 1 | -#+begin_src octave -[ 1 3 2 - 5 4 3 - 4 4 1 ] -#+end_src -** Matrix :notes: - + Matrix representations very powerful, as we'll see later in class - + Usually rows are records, columns are attributes - + Sometimes you can think of data in different ways, can take the transpose - of the matrix to get attributes about movies - -* Waterworld :slide: - [[file:img/waterworld.jpg]] - -* Similarity | Distance :slide: - + Two sides of the same coin - + =similarity = 1 - distance= - + We'll use these metrics for many other algorithms -** Core Concept :notes: - + Many data mining techniques rely on finding a way to quantify similarity - + When you think about questions like "how similar are two users?" "is this - text plagiarism?" "are these products likely to be purchased together?" - + All are ways of thinking about similarity - -* Nominal Distance :slide: - + Ratio of mismatches to potential matches - + Why can't we take the sum of the mismatches? -** Nominal :notes: - + Nominal means we can't compare two values: there is no ordering - + All we can do is take ratio of the ones that are exactly the same - + The book describes how to think about this in terms of matrices - -* Binary Distance :slide: - + Could use Nominal Distance: count all exact matches or mismatches - + Could use Numeric Distance: just treat values as 0/1 - + asymmetric binary dissimilarity: don't care about *negative matches* - + =mismatches / (positive_matches + mismatches)= - + asymmetric binary similarity: care more about *positive matches* than mismatches - + =positive_matches / (positive_matches + mismatches)= -** Binary :notes: - + Nominal problem: for rare attributes, like a disease, two people who - *don't* have the disease, aren't necessarily very similar - -* Jaccard Coefficient :slide: - + Asymmetric binary similarity - + More commonly used for calculating set similarity - + =|intersection| / |union|= - + "Jim likes pizza" | "Shreyas likes pizza" -** Jaccard :notes: - 1. Break up into a set - 1. calculate # in intersection - 1. calculate # in union - 1. divide - -* Euclidean distance :slide: - + Straight line between two points - + Again: usually considered with just (x,y), but can calculate for any number - of dimensions - [[file:img/euclidean.png]] -** Ordinary :notes: - + Distance as you probably learned in grade school - -* Manhattan distance :slide: - + How many blocks would you need to walk between two points? - [[file:img/manhattan.png]] -** Usefulness :notes: - + Obviously useful for maps/directions - + But haven't seen it used much beyond that - -* L_p norm :slide: - + Euclidean distance and Manhattan can be generalized - + Euclidean distance referred to as L_2 norm - + Chebyshev distance is L_∞ - [[file:img/lp-norm.png]] -** L_p space :notes: - + Important for signal processing, math, other applications - + You may want to study these distances for comparing wave forms, like audio - -* Ordinal Distance :slide: - + Normalize the ordinal rankings - + Use a numerical distance metric - -* Cosine Similarity :slide: - + Jaccard similarity can work well for sets of roughly equal size - + How to compare sets with a large difference in magnitude? - + Model them as vectors, take the cosign of the angle between - [[file:img/cosine-similarity.png]] -** Cosign :notes: - + Why cosine? Hint: nomalization - + img: http://cs.carleton.edu/cs_comps/0910/netflixprize/final_results/knn/index.html - -* Cosine Example :slide: - + "Jim likes pizza" | "Shreyas likes pizza" - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-15-Data-Warehouse.html b/slides/2013-02-15-Data-Warehouse.html deleted file mode 100644 index 28e56fc..0000000 --- a/slides/2013-02-15-Data-Warehouse.html +++ /dev/null @@ -1,818 +0,0 @@ - - - - -2013-02-15-Data-Warehouse - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-15-Data-Warehouse

- - - - -
-

1 Data Warehouse    slide

-
- - -
- -
- -
-

2 Database Types    slide

-
- -
-
Data Warehouse
Database designed for using data to make decisions -
-
OLAP
OnLine Analytical Processing -
-
OLTP
OnLine Transactional Processing -
-
- - -
- -
-

2.1 Data Mining    notes

-
- -
    -
  • These databases are often the starting point for data mining in companies -
    • -
  • Most of the data sets from companies typical come from exporting some - portion of their data warehouse -
    • -
- - -
-
- -
- -
-

3 Properties    slide

-
- -
-
Subject Oriented
Focus on core business objects -
-
Integrated
Access to as much data as possible -
-
Time Variant
Contains historical data with time parameter -
-
Non-volatile
Updated (relatively) infrequently, in bulk -
-
- - -
- -
-

3.1 Examples    notes

-
- -
    -
  • Yelp users can be directed to a datacenter depending on conditions. This - data probably doesn't need to be in the DW -
    • -
  • Yelp has severl databases: log summaries, user info, salesforce. Most - useful if they are all in the same place -
    • -
  • Operationally, when someone changes their address, we just overwrite it in - the OLTP DB. But DW potentially cares about the old value -
    • -
  • OLTP writes to rows every time someone updates profile, review, etc. Lots of - simultaneous updates. DW: typically once a day, in bulk -
    • -
- - -
-
- -
- -
-

4 OLAP or OLTP?    slide animate

-
- -
    -
  • Transactional Focus vs. Analytic Focus -
    • -
  • Used by Managers, Executives vs. DBAs, programmers -
    • -
  • Contains current information vs. Historical -
    • -
  • Variety of differently summarized data vs normalized -
    • -
  • Short transactions vs. Long queries -
    • -
  • Full table scans vs. Indexes on for fast lookups -
    • -
  • Simultaneous queries: 100s-1000s vs 1-100 -
    • -
  • Simple updates vs Complex queries -
    • -
  • Guarenteed high performance vs Flexibility & Customization -
    • -
- - -
- -
- -
-

5 Overview    slide

-
- -

img/olap-overview.png -

-
- -
-

5.1 From the front    notes

-
- -
    -
  • Analytics team uses charts, reports, et -
    • -
  • Generated from an OLAP server -
    • -
  • Which uses data from a data warehouse (often DW and OLAP server are - integrated) -
    • -
  • Uses a process (ETL) to move the data from other source into DW -
    • -
- - -
-
- -
- -
-

6 Types of Data Warehouses    slide

-
- -
-
Enterprise
turnkey solution, often expensive, sophisticated but complex - ingestion, integration, security features -
-
Data Mart
Smaller, limited in scope. Designed for specific team or - department -
-
Virtual
OLAP built on top of an OLTP database -
-
Cloud
Google BigQuery, Amazon RedShift -
-
- - -
- -
-

6.1 Vendors    notes

-
- -
-
Enterprise
Oracle, Greenplum, AsterData -
-
Data Mart
MySQL, PostgreSQL -
-
Virtual
MySQL, PostgreSQL views or admin interface -
-
- - -
-
- -
- -
-

7 Metadata    slide

-
- -
    -
  • Data about the data being stored -
    • -
  • Overview: schema, languages -
    • -
  • Operational: last update, query latency -
    • -
  • Algorithms: normalization, transformation -
    • -
  • Performance: job dependencies -
    • -
  • Business: ownership, permissions -
    • -
- - -
- -
-

7.1 Considerations    notes

-
- -
    -
  • As soon as several people start using the DW, they'll need to know about - how it is put together -
    • -
  • Metadata often comes as an after thought but is an important part of - scaling -
    • -
- - -
-
- -
- -
-

8 Overview    slide

-
- -

img/olap-overview.png -

-
- -
-

8.1 Data Cubes    notes

-
- -
    -
  • Why are there cubes in the OLAP area? -
    • -
- - -
-
- -
- -
-

9 Datacube    slide two_col

-
- -
    -
  • Way of thinking about multi dimensional data -
    • -
  • Useful metaphor because one can reason about ways to satisfy a query -
    • -
- -

img/BorgFirstContact.jpg -

-
- -
- -
-

10 Dimensions    slide

-
- - - -- - - - - - - -
Day 1Day 2Day 3
Region 1$200$80$600
Region 2$300$90$650
Region 2$400$100$700
- - -
- -
-

10.1 Data… Square    notes

-
- -
    -
  • More of a data square: only 2 dimensions -
    • -
  • Advertising on Yelp -
    • -
  • Now we want to know Product TYpe of things sold (CPC, CPM, National) -
    • -
- - -
- -
- -
-

10.2 Cube: 3rd Dimension    slide

-
- -

img/cube-3d.gif -

-
- -
-

10.2.1 More    notes

-
- -
    -
  • Now we want to know Page Type (Business, Search, Home) -
    • -
  • Hard to draw 4 dimensions, so instead… -
    • -
- - -
-
- -
- -
-

10.3 Multi-Cube    slide

-
- -

img/cube-4d.png -

-
- -
-

10.3.1 More    notes

-
- -
    -
  • Keep adding dimension as necessary -
    • -
- - -
-
- -
- -
-

10.4 Lattice    slide

-
- -

img/cube-lattice.jpg -

-
- -
-

10.4.1 Moving    notes

-
- -
    -
  • Move back and forth from our 2d table -
    • -
  • To our 3d cube, to our 4d multi-cube -
    • -
  • The lower dimensions summarize table -
    • -
  • At the extreme is just the total (ie all money made) -
    • -
- - -
-
-
- -
- -
-

11 Schemas    slide two_col

-
- -
    -
  • Data cube a way of visualizing multi dimensional data -
    • -
  • Star schema is a way store the data in a database -
    • -
- -

img/sun.jpg -

- -
- -
-

11.1 Fact table    slide

-
- -

img/star-1.png -

-
- -
- -
-

11.2 Dimension table    slide

-
- -

img/star-2.png -

-
- -
- -
-

11.3 Dimension tables    slide

-
- -

img/star-3.png -

-
- -
- -
-

11.4 Dimension tables    slide

-
- -

img/star-4.png -

-
- -
- -
-

11.5 Dimension tables    slide

-
- -

img/star-5.png -

-
- -
- -
-

11.6 Star Schema    slide

-
- -

img/star-schema.jpg -

-
- -
- -
-

11.7 Dimensions of Dimensions    slide

-
- -

img/star-6.png -

-
- -
- -
-

11.8 Dimensions of Dimensions    slide

-
- -

img/star-7.png -

-
- -
- -
-

11.9 Dimensions of Dimensions    slide

-
- -

img/star-8.png -

-
- -
- -
-

11.10 Dimensions of Dimensions    slide

-
- -

img/star-9.png -

-
- -
-

11.10.1 Schema Name?    notes

-
- -
    -
  • Any guesses what this fractal looking schema is called? -
    • -
- - -
-
- -
- -
-

11.11 Snowflake Schema    slide

-
- -
    -
  • Schema with radiating dimension tables -
    • -
- -

img/star-snowflake.jpg -

-
- -
- -
-

11.12 Constellation Schema    slide

-
- -
    -
  • Schema with several fact tables and related dimensions -
    • -
- -

img/star-constilation.jpg -

-
-
- -
- -
-

12 Data Warehouse Operations    slide

-
- -
-
Rollup
Summarize data along fewer dimensions -
-
Drill-down
Get details within a particular dimension -
-
Slice
Select a particular value in a dimension -
-
Dice
Consider a subset of the values in a dimension -
-
Pivot
Swap, or rotate dimensions -
-
- - -
- -
-

12.1 Examples    notes

-
- -
-
Rollup
What countries are selling the most ads? -
-
Drill-down
Spike in Q1 ad views. Which month most responsible? -
-
Slice
Chart sales only for CPC -
-
Dice
Only look at sales in US, IT, DE -
-
Pivot
Swap axis on a chart -
-
- - -
-
- -
- -
-

13 Materialized Views    slide

-
- -
-
View
virtual table defined by a query -
-
Full
Pre-compute and store -
-
None
Calculate summaries on the fly -
-
Partial
Variety of strategies: eg. cache results after calculating -
-
- - -
- -
-

13.1 Usefulness    notes

-
- -
    -
  • In DW, often storing different cubes in the lattice -
    • -
  • For the country sample, do we have those summaries stored in another DB - table? On disk? By month? Year? -
    • -
  • Storing all possible summarize expensive when loading data, and requires a - lot more storage -
    • -
- - -
-
- -
- -
-

14 Architecture    slide

-
- -
-
ROLAP
Relational. Implement OLAP on top of a relational database -
-
MOLAP
Multidimensional. Implements data cube as storage paradigm -
-
HOLAP
Hybrid. Data in ROLAP, rollups in MOLAP -
-
Specialized
Often distributed storage, parallel DB technology -
-
NoSQL
Store data as key-value pairs, optimized in different ways -
-
- - -
- -
-

14.1 Details    notes

-
- -
-
ROLAP
MySQL, PostgreSQL -
-
MOLAP
Oracle, Palo -
-
HOLAP
MS SQL -
-
???
Specialized: AsterData, Greenplumb -
-
???
NoSQL: Hive, BigTable, Cassandra -
-
- - -
-
- -
- -
-

15 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-02-15 07:59:48 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-15-Data-Warehouse.org b/slides/2013-02-15-Data-Warehouse.org deleted file mode 100644 index 995fc5c..0000000 --- a/slides/2013-02-15-Data-Warehouse.org +++ /dev/null @@ -1,211 +0,0 @@ -* Data Warehouse :slide: - -* Database Types :slide: - + Data Warehouse :: Database designed for using data to make decisions - + OLAP :: OnLine Analytical Processing - + OLTP :: OnLine Transactional Processing -** Data Mining :notes: - + These databases are often the starting point for data mining in companies - + Most of the data sets from companies typical come from exporting some - portion of their data warehouse - -* Properties :slide: - + Subject Oriented :: Focus on core business objects - + Integrated :: Access to as much data as possible - + Time Variant :: Contains historical data with time parameter - + Non-volatile :: Updated (relatively) infrequently, in bulk -** Examples :notes: - + Yelp users can be directed to a datacenter depending on conditions. This - data probably doesn't need to be in the DW - + Yelp has severl databases: log summaries, user info, salesforce. Most - useful if they are all in the same place - + Operationally, when someone changes their address, we just overwrite it in - the OLTP DB. But DW potentially cares about the old value - + OLTP writes to rows every time someone updates profile, review, etc. Lots of - simultaneous updates. DW: typically once a day, in bulk - -* OLAP or OLTP? :slide:animate: - + Transactional Focus vs. Analytic Focus - + Used by Managers, Executives vs. DBAs, programmers - + Contains current information vs. Historical - + Variety of differently summarized data vs normalized - + Short transactions vs. Long queries - + Full table scans vs. Indexes on for fast lookups - + Simultaneous queries: 100s-1000s vs 1-100 - + Simple updates vs Complex queries - + Guarenteed high performance vs Flexibility & Customization - -* Overview :slide: - [[file:img/olap-overview.png]] -** From the front :notes: - + Analytics team uses charts, reports, et - + Generated from an OLAP server - + Which uses data from a data warehouse (often DW and OLAP server are - integrated) - + Uses a process (ETL) to move the data from other source into DW - -* Types of Data Warehouses :slide: - + Enterprise :: turnkey solution, often expensive, sophisticated but complex - ingestion, integration, security features - + Data Mart :: Smaller, limited in scope. Designed for specific team or - department - + Virtual :: OLAP built on top of an OLTP database - + Cloud :: Google BigQuery, Amazon RedShift -** Vendors :notes: - + Enterprise :: Oracle, Greenplum, AsterData - + Data Mart :: MySQL, PostgreSQL - + Virtual :: MySQL, PostgreSQL views or admin interface - -* Metadata :slide: - + Data about the data being stored - + Overview: schema, languages - + Operational: last update, query latency - + Algorithms: normalization, transformation - + Performance: job dependencies - + Business: ownership, permissions -** Considerations :notes: - + As soon as several people start using the DW, they'll need to know about - how it is put together - + Metadata often comes as an after thought but is an important part of - scaling - -* Overview :slide: - [[file:img/olap-overview.png]] -** Data Cubes :notes: - + Why are there cubes in the OLAP area? - -* Datacube :slide:two_col: - + Way of thinking about multi dimensional data - + Useful metaphor because one can reason about ways to satisfy a query - [[file:img/BorgFirstContact.jpg]] - -* Dimensions :slide: - | | Day 1 | Day 2 | Day 3 | - | Region 1 | $200 | $80 | $600 | - | Region 2 | $300 | $90 | $650 | - | Region 2 | $400 | $100 | $700 | -** Data... Square :notes: - + More of a data square: only 2 dimensions - + Advertising on Yelp - + Now we want to know Product TYpe of things sold (CPC, CPM, National) - -** Cube: 3rd Dimension :slide: -[[file:img/cube-3d.gif]] -*** More :notes: - + Now we want to know Page Type (Business, Search, Home) - + Hard to draw 4 dimensions, so instead... - -** Multi-Cube :slide: -[[file:img/cube-4d.png]] -*** More :notes: - + Keep adding dimension as necessary - -** Lattice :slide: - [[file:img/cube-lattice.jpg]] -*** Moving :notes: - + Move back and forth from our 2d table - + To our 3d cube, to our 4d multi-cube - + The lower dimensions summarize table - + At the extreme is just the total (ie all money made) - -* Schemas :slide:two_col: - + Data cube a way of visualizing multi dimensional data - + Star schema is a way store the data in a database - [[file:img/sun.jpg]] - -** Fact table :slide: - [[file:img/star-1.png]] - -** Dimension table :slide: - [[file:img/star-2.png]] - -** Dimension tables :slide: - [[file:img/star-3.png]] - -** Dimension tables :slide: - [[file:img/star-4.png]] - -** Dimension tables :slide: - [[file:img/star-5.png]] - -** Star Schema :slide: - [[file:img/star-schema.jpg]] - -** Dimensions of Dimensions :slide: - [[file:img/star-6.png]] - -** Dimensions of Dimensions :slide: - [[file:img/star-7.png]] - -** Dimensions of Dimensions :slide: - [[file:img/star-8.png]] - -** Dimensions of Dimensions :slide: - [[file:img/star-9.png]] -*** Schema Name? :notes: - + Any guesses what this fractal looking schema is called? - -** Snowflake Schema :slide: - + Schema with radiating dimension tables - [[file:img/star-snowflake.jpg]] - -** Constellation Schema :slide: - + Schema with several fact tables and related dimensions - [[file:img/star-constilation.jpg]] - -* Data Warehouse Operations :slide: - + Rollup :: Summarize data along fewer dimensions - + Drill-down :: Get details within a particular dimension - + Slice :: Select a particular value in a dimension - + Dice :: Consider a subset of the values in a dimension - + Pivot :: Swap, or rotate dimensions -** Examples :notes: - + Rollup :: What countries are selling the most ads? - + Drill-down :: Spike in Q1 ad views. Which month most responsible? - + Slice :: Chart sales only for CPC - + Dice :: Only look at sales in US, IT, DE - + Pivot :: Swap axis on a chart - -* Materialized Views :slide: - + View :: virtual table defined by a query - + Full :: Pre-compute and store - + None :: Calculate summaries on the fly - + Partial :: Variety of strategies: eg. cache results after calculating -** Usefulness :notes: - + In DW, often storing different cubes in the lattice - + For the country sample, do we have those summaries stored in another DB - table? On disk? By month? Year? - + Storing all possible summarize expensive when loading data, and requires a - lot more storage - -* Architecture :slide: - + ROLAP :: Relational. Implement OLAP on top of a relational database - + MOLAP :: Multidimensional. Implements data cube as storage paradigm - + HOLAP :: Hybrid. Data in ROLAP, rollups in MOLAP - + Specialized :: Often distributed storage, parallel DB technology - + NoSQL :: Store data as key-value pairs, optimized in different ways -** Details :notes: - + ROLAP :: MySQL, PostgreSQL - + MOLAP :: Oracle, Palo - + HOLAP :: MS SQL - + Specialized: AsterData, Greenplumb - + NoSQL: Hive, BigTable, Cassandra - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-15-MapReduce.html b/slides/2013-02-15-MapReduce.html deleted file mode 100644 index 58b216c..0000000 --- a/slides/2013-02-15-MapReduce.html +++ /dev/null @@ -1,1129 +0,0 @@ - - - - -2013-02-15-MapReduce - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-15-MapReduce

- - - - -
-

1 MapReduce    slide

-
- - -
- -
-

1.1 Spoilers    notes

-
- -
    -
  • Don't look ahead in the slides -
    • -
  • If you know MapReduce, try to let others answer and genuinely think about - how you would solve the problem. -
    • -
- - -
-
- -
- -
-

2 Yelp has a problem    slide

-
- -
    -
  • 250+ GB of logs per day -
    • -
  • Each GB takes 10 minutes to process -
    • -
  • How long to handle a day's logs? -
    • -
- -

img/yelp-growth.png -

-
- -
-

2.1 Too long    notes

-
- -
    -
  • On a single machine 40+ hours! -
    • -
  • If we really had only a single machine, we wouldn't be able to keep up! -
    • -
  • Mistake can't be fixed in a day (billing especially important) -
    • -
- - -
-
- -
- -
-

3 Solution?    slide animate

-
- -
    -
  • Don't use one machine! -
    • -
  • What are the new challenges? -
    • -
  • Distributing data -
    • -
  • Calculating overall statistics -
    • -
  • Failures -
    • -
- - -
- -
-

3.1 New Challenges    notes

-
- -
    -
  • With many machines, how to they get access to the 100 GB of logs? -
    • -
  • How do they coordinate who gets which section of logs? -
    • -
  • How do we calculate the average? -
    • -
  • What happens when one of the boxes dies? -
      -
    • Detecting failure (timeout waiting for data? Out of band?) -
      • -
    • Decide who takes over the data -
      • -
    - -
    • -
- - -
-
- -
- -
-

4 Do It Yourself    slide two_col

-
- -
    -
  • There are many ways to deal with these challenges -
    • -
  • Often, people would "roll" their own solutions depending on the problem -
    • -
  • Google implemented a generic solution, shared idea -
    • -
- -

img/mapreduce-paper.png -

-
- -
-

4.1 Dependencies    notes

-
- -
    -
  • Did you have a super-computer? -
    • -
  • What programming language were you using? -
    • -
  • Type of problem being solved (working on graphs, or web logs, …) -
    • -
- - -
-
- -
- -
-

5 Big Idea    slide

-
- -
    -
  • Simplify, limit solution expression -
    • -
  • Enable sophisticated implementation -
    • -
- - - -
    -
  • Interface: Map() Reduce() -
    • -
  • Implementation: Reliably run over 1000s of machines -
    • -
- - -
- -
-

5.1 Really Big Idea    notes

-
- -
    -
  • Limiting yourself to what can be expressed may seem like a loss -
    • -
  • But it enables the implementation to handle the problems we talked about -
    • -
  • And then can be used as understandable building blocks -
    • -
- - -
-
- -
- -
-

6 MapReduce    slide

-
- -
-
Map
Extract a property to summarize over -
-
Reduce
Summarize all items with a particular property -
-
- - - -
    -
  • Simple: Each operation stateless -
    • -
- - -
- -
-

6.1 Reading    notes

-
- -
    -
  • Reading this week includes a video explaining MapReduce much more generally -
    • -
  • This lecture will focus on it from a practical standpoint for homework -
    • -
  • MapReduce's main benefits are for running over many machines, fault - tolerance -
    • -
  • But we'll just practice on one machine -
    • -
- - -
- -
- -
-

6.2 Example    slide

-
- -
    -
  • Web application logs -
    • -
  • How many actions have we seen? -
      -
    • Business views -
      • -
    • User profile views -
      • -
    • Searches -
      • -
    - -
    • -
- - -
- -
-

6.2.1 Details    notes

-
- -
-
Business Views
Triple Rock, Bear Raman -
-
User profile
jimblomo.yelp.com -
-
Searches
query, location -
-
- - -
-
- -
- -
-

6.3 Logs    slide

-
- - - - -
{'page_type': 'search',
- 'user': 'jim', 'query': ...}
-
-{'page_type': 'biz_view',
- 'user': 'shreyas', 'biz_id': 55}
-
-{'page_type': 'user_profile',
- 'user': null, 'profile_id: 123}
-
-...
-
- - -
- -
-

6.3.1 Logs    notes

-
- -
    -
  • JSON logs, various types of information -
    • -
  • entire record on one line (wrapped for slides) -
    • -
- - -
-
- -
- -
-

6.4 Map    slide

-
- -
-
Input
Key, Value -
-
Output
Keys, Values -
-
- - -
- -
- -
-

6.5 Map Example    slide

-
- -
-
Input Key
Log line number -
-
Input Value
Log line text -
-
Output Key
Action -
-
Output Value
times this action has occurred on this line -
-
- - -
- -
-

6.5.1 Counts    notes

-
- -
    -
  • Log line number is not helpful in our specific case -
    • -
  • Log line text: we hope it is machine readable so we can accurately extract - the action -
    • -
  • It has datetime, cookie, action, etc. -
    • -
  • How many times has this action occurred? 1 -
    • -
  • Tunnel vision: all we care about is this line -
    • -
- - -
-
- -
- -
-

6.6 Actions?    slide

-
- - - - -
search       1 
-biz_view     1 
-user_profile 1 
-search       1 
-biz_view     1 
-search       1 
-biz_view     1 
-user_profile 1 
-search       1 
-
- - -
- -
-

6.6.1 Middle Step    notes

-
- -
    -
  • From log lines, we've extracted the information out that we care about -
    • -
  • The counts and the actions -
    • -
  • Next step summarize -
    • -
  • Next step after Map? -
    • -
- - -
-
- -
- -
-

6.7 Reduce    slide

-
- -
-
Input
Key, Values -
-
Output
Keys, Values -
-
- - -
- -
-

6.7.1 Values    notes

-
- -
    -
  • Note: The input is values! Plural -
    • -
  • Because we get a key and all of its associated values -
    • -
  • Remind me: what are we trying to get out of this computation? -
    • -
  • So what do you think the output keys are? -
    • -
  • Values? -
    • -
- - -
-
- -
- -
-

6.8 Reduce Example    slide

-
- -
-
Input Key
Action -
-
Input Values
Counts: [1,1,1,1] -
-
Output Key
Action -
-
Output Value
Total Count -
-
- - -
- -
-

6.8.1 Details    notes

-
- -
    -
  • Action is one of search bizview profileview -
    • -
  • To get total count, sum all of the counts -
    • -
- - -
-
- -
- -
-

6.9 Example Output    slide

-
- -
-
Output Key
Action -
-
Output Value
Total Count -
-
- - - - -
"search"       4
-"user_profile" 2
-"biz_view"     3
-
- - -
-
- -
- -
-

7 Point?    slide

-
- -
    -
  • A lot of work for counting! -
    • -
  • More complex calculations can be done this way, eg. PageRank -
    • -
  • Stateless constraint means it can be used across thousands of computers -
    • -
- - -
- -
-

7.1 Details    notes

-
- -
    -
  • By only looking at keys and values, can optimize a lot of back-end work -
    • -
  • Where to send the results? -
    • -
  • What to do when a computer fails? (Just restart failed part) -
    • -
- - -
- -
- -
-

7.2 Implementation    slide

-
- - - - -
biz_view     1 
-user_profile 1 
-search       1 
-search       1 
-biz_view     1 
-search       1 
-biz_view     1 
-user_profile 1 
-search       1 
-
- -
- -
- -
-

7.3 Intermediate    notes

-
- -
    -
  • This was the situation after map -
    • -
  • Keys all jumbled -
    • -
  • What Hadoop does is sort them and distribute them to computers -
    • -
- - -
- -
- -
-

7.4 "Shuffle"    slide

-
- - - - -
biz_view     1 
-biz_view     1 
-biz_view     1 
-search       1 
-search       1 
-search       1 
-search       1 
-user_profile 1 
-user_profile 1 
-
- -
- -
- -
-

7.5 Distribute    notes

-
- -
    -
  • Now it is easy to distribute, and can handle all the biz_view at once -
    • -
- - -
- -
- -
-

7.6 Inputs    slide

-
- -
    -
  • MapReduce distributes computing power by distributing input -
    • -
  • Input is distributed by splitting on lines (records) -
    • -
  • You cannot depend on lines being "together" in MapReduce -
    • -
- - -
- -
-

7.6.1 Splitting Files    notes

-
- -
    -
  • Image you have a lot of large log files, GB each -
    • -
  • You'd like to let different machines work on the same file -
    • -
  • Split file down the middle, well, at least on a newline -
    • -
  • Enable two separate machines to work on the parts -
    • -
  • You don't know what line came before this one -
    • -
  • You don't know if you will process the next line -
    • -
  • Only view is this line -
    • -
  • Real life slightly more complicated, but mostly hacks around this -
    • -
- - -
-
-
- -
- -
-

8 Word Count    slide

-
- - - - -
{"text": "Greatest pizza ever", "stars": 2, "user": ...}
-
-{"text": "good pizza selection", "stars": 5, "user": ...}
-
- -
    -
  • Total uses of a word in across all reviews -
    • -
- - -
- -
-

8.1 Classic    notes

-
- -
    -
  • This is the traditional MapReduce example, so let's solve it -
    • -
  • No skipping ahead -
    • -
- - -
-
- -
- -
-

9 Steps    slide animate

-
- -
    -
  • Map -
    • -
  • Extract text -
    • -
  • Count words in that review -
    • -
  • Key: word , Value: count -
    • -
  • Reduce -
    • -
  • Key: word , Values: all counts -
    • -
  • sum(values) -
    • -
- - -
- -
-

9.1 Hints    notes

-
- -
    -
  • What's the first step (of MapReduce) -
    • -
  • What part of the record are we interested in? -
    • -
  • What do we want with those words? -
    • -
  • Mapper: Key Value? What are we grouping by? -
    • -
  • Next step (of MapReduce) -
    • -
  • What are the reducer inputs -
    • -
  • with all of these counts, how do we summarize -
    • -
- - -
- -
- -
-

9.2 Examples    slide animate

-
- -
    -
  • "Greatest pizza ever" -
    • -
  • Counts -
      -
    • Greatest: 1 -
      • -
    • pizza: 1 -
      • -
    • ever: 1 -
      • -
    - -
    • -
  • Reducer, Key: pizza -
      -
    • Values: [1, 1] -
      • -
    • Output: ["pizza", 2] -
      • -
    - -
    • -
- - -
-
- -
- -
-

10 Multi-Step    slide

-
- -
    -
  • Not all computations can be done in a single MapReduce step -
    • -
  • Map Input: <key, value> -
    • -
  • Reducer Output: <key, value> -
    • -
  • Compose MapReduce steps! -
    • -
- - -
- -
-

10.1 Output as Input    notes

-
- -
    -
  • The output of one MapReduce job can be used as the input to another -
    • -
- - -
- -
- -
-

10.2 Examples    slide

-
- -
    -
  • PageRank: Multiple steps till solution converges -
    • -
  • Multi-level summaries -
    • -
- -
- -
- -
-

10.3 PageRank    notes

-
- -
    -
  • PageRank is an algorithm for calculating the important of a page -
    • -
  • But it depends on the importance of every page pointing to it! -
    • -
  • So iteratively calculate the important of all pages -
    • -
  • Find average presidential donations by candidate, then normalize averages -
    • -
- - -
-
- -
- -
-

11 Unique Review    slide animate

-
- -
    -
  • Review ID with the most unique words -
    • -
  • Map Input: <line number, text> -
    • -
  • Map Output: <word, review_id> -
    • -
  • Reducer Input: <word, [review_ids]> -
    • -
  • Reducer Output: <review_id, 1> if the word is unique -
    • -
- - -
- -
-

11.1 Questions    notes

-
- -
    -
  • For our purposes, what is always the mapper input? -
    • -
  • What feature do we want to calculate first? -
    • -
  • Given this mapper output, what must the reducer input be? -
    • -
  • What property about a review are we interested in? -
    • -
- - -
- -
- -
-

11.2 Step 2: Count Unique Words    slide animate

-
- -
    -
  • Map Input: <review_id, 1> -
    • -
  • Map Output: <review_id, 1> -
    • -
  • Reducer Input: <review_id, [1,1,…]> -
    • -
  • Reducer Output: <review_id, sum> -
    • -
- -
- -
- -
-

11.3 Questions    notes

-
- -
    -
  • Given the reducer output, what must the mapper input be (for chained - MapReduce steps) -
    • -
  • What do we want to group by? -
    • -
  • Given this mapper output, what must the reducer input be? -
    • -
  • What are we calculating? -
    • -
- - -
- -
- -
-

11.4 Step 3: Max    slide animate

-
- -
    -
  • Map Input: <review_id, sum> -
    • -
  • Map Output: <"MAX", [sum, review_id]> -
    • -
  • Reducer Input: <"MAX", [[sum, review_id],…]> -
    • -
  • Reducer Output: <review_id, sum> of the max(sum) -
    • -
- -
- -
- -
-

11.5 Questions    notes

-
- -
    -
  • Given the reducer output, what must the mapper input be (for chained - MapReduce steps) -
    • -
  • We're calculating a statistic over what portion of the data set? -
    • -
  • What stat are we calculating? -
    • -
- - - - - - - -
-
-
-
- -
-

Date: 2013-02-15 08:29:27 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-15-MapReduce.org b/slides/2013-02-15-MapReduce.org deleted file mode 100644 index 45f36b5..0000000 --- a/slides/2013-02-15-MapReduce.org +++ /dev/null @@ -1,317 +0,0 @@ -* MapReduce :slide: -** Spoilers :notes: - + Don't look ahead in the slides - + If you know MapReduce, try to let others answer and genuinely think about - how *you* would solve the problem. - -* Yelp has a problem :slide: - + 250+ GB of logs per day - + Each GB takes 10 minutes to process - + How long to handle a day's logs? -[[file:img/yelp-growth.png]] -** Too long :notes: - + On a single machine 40+ hours! - + If we really had only a single machine, we wouldn't be able to keep up! - + Mistake can't be fixed in a day (billing especially important) - -* Solution? :slide:animate: - + Don't use one machine! - + What are the new challenges? - + Distributing data - + Calculating overall statistics - + Failures -** New Challenges :notes: - + With many machines, how to they get access to the 100 GB of logs? - + How do they coordinate who gets which section of logs? - + How do we calculate the average? - + What happens when one of the boxes dies? - + Detecting failure (timeout waiting for data? Out of band?) - + Decide who takes over the data - -* Do It Yourself :slide:two_col: - + There are many ways to deal with these challenges - + Often, people would "roll" their own solutions depending on the problem - + Google implemented a generic solution, shared idea -[[file:img/mapreduce-paper.png]] -** Dependencies :notes: - + Did you have a super-computer? - + What programming language were you using? - + Type of problem being solved (working on graphs, or web logs, ...) - -* Big Idea :slide: - + Simplify, limit solution expression - + Enable sophisticated implementation - - - + Interface: Map() Reduce() - + Implementation: Reliably run over 1000s of machines -** Really Big Idea :notes: - + Limiting yourself to what can be expressed may seem like a loss - + But it enables the implementation to handle the problems we talked about - + And then can be used as understandable building blocks - -* MapReduce :slide: - + Map :: Extract a property to summarize over - + Reduce :: Summarize all items with a particular property - - - + Simple: Each operation stateless -** Reading :notes: - + Reading this week includes a video explaining MapReduce much more generally - + This lecture will focus on it from a practical standpoint for homework - + MapReduce's main benefits are for running over many machines, fault - tolerance - + But we'll just practice on one machine - -** Example :slide: - + Web application logs - + How many actions have we seen? - + Business views - + User profile views - + Searches -*** Details :notes: - + Business Views :: Triple Rock, Bear Raman - + User profile :: jimblomo.yelp.com - + Searches :: query, location - -** Logs :slide: -#+begin_src json -{'page_type': 'search', - 'user': 'jim', 'query': ...} - -{'page_type': 'biz_view', - 'user': 'shreyas', 'biz_id': 55} - -{'page_type': 'user_profile', - 'user': null, 'profile_id: 123} - -... -#+end_src -*** Logs :notes: - + JSON logs, various types of information - + entire record on one line (wrapped for slides) - -** Map :slide: - + Input :: Key, Value - + Output :: Keys, Values - -** Map Example :slide: - + Input Key :: Log line number - + Input Value :: Log line text - + Output Key :: Action - + Output Value :: times this action has occurred *on this line* -*** Counts :notes: - + Log line number is not helpful in our specific case - + Log line text: we hope it is machine readable so we can accurately extract - the action - + It has datetime, cookie, action, etc. - + How many times has this action occurred? 1 - + Tunnel vision: all we care about is this line - -** Actions? :slide: -#+begin_src text -search 1 -biz_view 1 -user_profile 1 -search 1 -biz_view 1 -search 1 -biz_view 1 -user_profile 1 -search 1 -#+end_src -*** Middle Step :notes: - + From log lines, we've extracted the information out that we care about - + The counts and the actions - + Next step summarize - + Next step after Map? - -** Reduce :slide: - + Input :: Key, Values - + Output :: Keys, Values -*** Values :notes: - + Note: The input is values! Plural - + Because we get a key and all of its associated values - + Remind me: what are we trying to get out of this computation? - + So what do you think the output keys are? - + Values? - -** Reduce Example :slide: - + Input Key :: Action - + Input Values :: Counts: =[1,1,1,1]= - + Output Key :: Action - + Output Value :: Total Count -*** Details :notes: - + Action is *one of* search biz_view profile_view - + To get total count, sum all of the counts - -** Example Output :slide: - + Output Key :: Action - + Output Value :: Total Count -#+begin_src html -"search" 4 -"user_profile" 2 -"biz_view" 3 -#+end_src - -* Point? :slide: - + A lot of work for counting! - + More complex calculations can be done this way, eg. PageRank - + Stateless constraint means it can be used across thousands of computers -** Details :notes: - + By only looking at keys and values, can optimize a lot of back-end work - + Where to send the results? - + What to do when a computer fails? (Just restart failed part) - -** Implementation :slide: -#+begin_src text -biz_view 1 -user_profile 1 -search 1 -search 1 -biz_view 1 -search 1 -biz_view 1 -user_profile 1 -search 1 -#+end_src -** Intermediate :notes: - + This was the situation after map - + Keys all jumbled - + What Hadoop does is sort them and distribute them to computers - -** "Shuffle" :slide: -#+begin_src text -biz_view 1 -biz_view 1 -biz_view 1 -search 1 -search 1 -search 1 -search 1 -user_profile 1 -user_profile 1 -#+end_src -** Distribute :notes: - + Now it is easy to distribute, and can handle all the =biz_view= at once - -** Inputs :slide: - + MapReduce distributes computing power by distributing input - + Input is distributed by splitting on lines (records) - + You cannot depend on lines being "together" in MapReduce -*** Splitting Files :notes: - + Image you have a lot of large log files, GB each - + You'd like to let different machines work on the same file - + Split file down the middle, well, at least on a newline - + Enable two separate machines to work on the parts - + You don't know what line came before this one - + You don't know if you will process the next line - + Only view is this line - + Real life slightly more complicated, but mostly hacks around this - -* Word Count :slide: -#+begin_src json -{"text": "Greatest pizza ever", "stars": 2, "user": ...} - -{"text": "good pizza selection", "stars": 5, "user": ...} -#+end_src - + Total uses of a word in across all reviews -** Classic :notes: - + This is the traditional MapReduce example, so let's solve it - + No skipping ahead - -* Steps :slide:animate: - + Map - + Extract =text= - + Count words in that review - + Key: word , Value: count - + Reduce - + Key: word , Values: all counts - + sum(values) -** Hints :notes: - + What's the first step (of MapReduce) - + What part of the record are we interested in? - + What do we want with those words? - + Mapper: Key Value? What are we grouping by? - + Next step (of MapReduce) - + What are the reducer inputs - + with all of these counts, how do we summarize - -** Examples :slide:animate: - + "Greatest pizza ever" - + Counts - + Greatest: 1 - + pizza: 1 - + ever: 1 - + Reducer, Key: pizza - + Values: [1, 1] - + Output: ["pizza", 2] - -* Multi-Step :slide: - + Not all computations can be done in a single MapReduce step - + Map Input: - + Reducer Output: - + Compose MapReduce steps! -** Output as Input :notes: - + The output of one MapReduce job can be used as the input to another - -** Examples :slide: - + PageRank: Multiple steps till solution converges - + Multi-level summaries -** PageRank :notes: - + PageRank is an algorithm for calculating the important of a page - + But it depends on the importance of every page pointing to it! - + So iteratively calculate the important of all pages - + Find average presidential donations by candidate, then normalize averages - -* Unique Review :slide:animate: - + Review ID with the most unique words - + Map Input: - + Map Output: - + Reducer Input: - + Reducer Output: if the word is unique -** Questions :notes: - + For our purposes, what is always the mapper input? - + What feature do we want to calculate first? - + Given this mapper output, what *must* the reducer input be? - + What property about a review are we interested in? - -** Step 2: Count Unique Words :slide:animate: - + Map Input: - + Map Output: - + Reducer Input: - + Reducer Output: -** Questions :notes: - + Given the reducer output, what *must* the mapper input be (for chained - MapReduce steps) - + What do we want to group by? - + Given this mapper output, what *must* the reducer input be? - + What are we calculating? - -** Step 3: Max :slide:animate: - + Map Input: - + Map Output: <"MAX", [sum, review\_id]> - + Reducer Input: <"MAX", [[sum, review\_id],...]> - + Reducer Output: of the max(sum) -** Questions :notes: - + Given the reducer output, what *must* the mapper input be (for chained - MapReduce steps) - + We're calculating a statistic over what portion of the data set? - + What stat are we calculating? - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-15-Project.html b/slides/2013-02-15-Project.html deleted file mode 100644 index 36f7371..0000000 --- a/slides/2013-02-15-Project.html +++ /dev/null @@ -1,297 +0,0 @@ - - - - -2013-02-15-Project - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-15-Project

- - - - -
-

1 Final Project    slide

-
- - -
- -
- -
-

2 Deliverables    slide

-
- -
    -
  • Project Proposal -
    • -
  • Written report -
    • -
  • Presentation -
    • -
  • Code -
    • -
  • Data (URL or sampled) -
    • -
  • May 16 midnight -
    • -
- - -
- -
- -
-

3 Groups    slide

-
- -
    -
  • 1-2 people -
    • -
- - -
- -
- -
-

4 Proposal    slide

-
- -
    -
  • Informal, not graded -
    • -
  • A plan for your project -
    • -
  • After the proposal you should just be executing, not brainstorming -
    • -
  • Discussions before proposal encouraged -
    • -
- - -
- -
- -
-

5 Report    slide

-
- -
    -
  • Introduction: problem, insights, solutions -
    • -
  • Problem: motivation, data set -
    • -
  • Solution: techniques, failures, examples -
    • -
  • Details: parameter tuning, software engineering challenges -
    • -
  • Related work: including resources you used -
    • -
  • Further work: any remaining ideas you have -
    • -
- - -
- -
-

5.1 Notes    notes

-
- -
    -
  • Length: around 2 pages, but more important to hit these points -
    • -
  • Introduction: "gosh, if these insights are true, it would be really - exciting" -
    • -
  • Problem: include problems with your data set -
    • -
  • Solution: if you do clustering, give examples of a cluster you found and - individual data points that it contains -
    • -
  • Details: what commands did you use for particular libraries. Can someone - duplicate you work? -
    • -
  • Formats: PDF, Google Doc -
    • -
- - -
-
- -
- -
-

6 Research Paper    slide

-
- - - - -
- -
-

6.1 Skip sections    notes

-
- -
    -
  • No abstract -
    • -
  • Think paragraphs instead of pages -
    • -
- - -
-
- -
- -
-

7 Presentation    slide

-
- -
    -
  • ~10 minutes -
    • -
  • Think: 1 slide per paragraph -
    • -
  • Focus on images, stories, examples -
    • -
  • Motivate people to read your paper, don't read it to them -
    • -
- - -
- -
- -
-

8 Code    slide

-
- -
    -
  • Another GitHub repository -
    • -
  • If private, add Shreyas and me -
    • -
  • Include README with info on how to run algorithms -
    • -
  • Suggestion: include your paper -
    • -
- - -
- -
-

8.1 Reproducible    notes

-
- -
    -
  • Imagine if someone wanted to reproduce your results -
    • -
  • Also great for portfolio (with Paper) -
    • -
- - -
-
- -
- -
-

9 Data    slide

-
- -
    -
  • If large, just point to URL in paper and README -
    • -
  • If very large, talk with me about hosting on Amazon Web Services -
    • -
  • We'll have computing resources available from Amazon -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-05-07 22:49:10 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-15-Project.org b/slides/2013-02-15-Project.org deleted file mode 100644 index 72814ce..0000000 --- a/slides/2013-02-15-Project.org +++ /dev/null @@ -1,80 +0,0 @@ -* Final Project :slide: - -* Deliverables :slide: - + Project Proposal - + Written report - + Presentation - + Code - + Data (URL or sampled) - + *May 16* midnight - -* Groups :slide: - + 1-2 people - -* Proposal :slide: - + Informal, not graded - + A plan for your project - + After the proposal you should just be executing, not brainstorming - + Discussions before proposal encouraged - -* Report :slide: - + Introduction: problem, insights, solutions - + Problem: motivation, data set - + Solution: techniques, failures, examples - + Details: parameter tuning, software engineering challenges - + Related work: including resources you used - + Further work: any remaining ideas you have -** Notes :notes: - + Length: around 2 pages, but more important to hit these points - + Introduction: "gosh, if these insights are true, it would be really - exciting" - + Problem: include problems with your data set - + Solution: if you do clustering, give examples of a cluster you found and - individual data points that it contains - + Details: what commands did you use for particular libraries. Can someone - duplicate you work? - + Formats: PDF, Google Doc - -* Research Paper :slide: - + [[http://research.microsoft.com/en-us/um/people/simonpj/papers/giving-a-talk/writing-a-paper-slides.pdf][How to write a good research paper]] - + But much shorter! -** Skip sections :notes: - + No abstract - + Think paragraphs instead of pages - -* Presentation :slide: - + ~10 minutes - + Think: 1 slide per paragraph - + Focus on images, stories, examples - + Motivate people to read your paper, don't read it to them - -* Code :slide: - + Another GitHub repository - + If private, add Shreyas and me - + Include README with info on how to run algorithms - + Suggestion: include your paper -** Reproducible :notes: - + Imagine if someone wanted to reproduce your results - + Also great for portfolio (with Paper) - -* Data :slide: - + If large, just point to URL in paper and README - + If very large, talk with me about hosting on Amazon Web Services - + We'll have computing resources available from Amazon - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-15-mrjob.html b/slides/2013-02-15-mrjob.html deleted file mode 100644 index 30763ca..0000000 --- a/slides/2013-02-15-mrjob.html +++ /dev/null @@ -1,216 +0,0 @@ - - - - -2013-02-15-mrjob - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-15-mrjob

- - - - -
-

1 Lab: mrjob    slide

-
- -
    -
  • Understand review_word_count.py -
    • -
  • Find review with most unique words -
      -
    • Fill in unique_review.py -
      • -
    - -
    • -
  • Find similar users -
      -
    • Write user_similarity.py -
      • -
    - -
    • -
- - -
- -
-

1.1 mrjob    notes

-
- -
    -
  • Using the Yelp Academic Dataset -
    • -
  • In lecture, we covered the steps for most unique words -
    • -
  • Use Jaccard similarity for user_similarity -
    • -
- - -
-
- -
- -
-

2 Data    slide

-
- -
    -
  • ischool: ~jblomo/yelp_academic_dataset.json -
    • -
  • May copy or use in place -
    • -
- - -
- -
-

2.1 Agreement    notes

-
- -
    -
  • Data set only for use academic purposes -
    • -
  • Yelp Dataset -
    • -
- - -
-
- -
- -
-

3 Understand review_word_count.py    slide

-
- - - - -
$ python review_word_count.py yelp_academic_dataset.json
-
-no configs found; falling back on auto-configuration
-creating tmp directory /tmp/review_word_count.jim.20130215.071901.095847
-reading from file
-> /home/jim/src/datamining290/code/venv/bin/python review_word_count.py --step-num=0 --mapper /tmp/review_word_count.jim.20130215.071901.095847/input_part-00000
-writing to /tmp/review_word_count.jim.20130215.071901.095847/step-0-mapper_part-00000
-Counters from step 1:
-  (no counters found)
-...
-Streaming final output from /tmp/review_word_count.jim.20130215.071901.095847/output
-"4"     2
-"5"     1
-"50"    1
-"6"     2
-"7"     2
-"70s"   1
-"9"     2
-"a"     46
-"abbey" 4
-"able"  1
-"about" 4
-
- - -
- -
- -
-

4 Fill in unique_review.py    slide

-
- -
    -
  • Mutli-step map reduce -
    • -
  • Steps are explained in lecture -
    • -
  • Skeleton in code -
    • -
- - -
- -
- -
-

5 Write user_similarity.py    slide

-
- -
    -
  • Find users >= 0.5 similarity -
    • -
  • User Similarity: Jaccard similarity of businesses reviewed -
    • -
  • {BizA, BizB, BizC} ~ {BizF, BizB, BizG} -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-02-15 13:52:26 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-15-mrjob.org b/slides/2013-02-15-mrjob.org deleted file mode 100644 index b135ca2..0000000 --- a/slides/2013-02-15-mrjob.org +++ /dev/null @@ -1,70 +0,0 @@ -* Lab: mrjob :slide: - + Understand =review_word_count.py= - + Find review with most unique words - + Fill in =unique_review.py= - + Find similar users - + Write =user_similarity.py= -** mrjob :notes: - + Using the Yelp Academic Dataset - + In lecture, we covered the steps for most unique words - + Use Jaccard similarity for user\_similarity - -* Data :slide: - + ischool: =~jblomo/yelp_academic_dataset.json= - + May copy or use in place -** Agreement :notes: - + Data set only for use academic purposes - + [[http://www.yelp.com/academic_dataset][Yelp Dataset]] - -* Understand review\_word\_count.py :slide: -#+begin_src bash -$ python review_word_count.py yelp_academic_dataset.json - -no configs found; falling back on auto-configuration -creating tmp directory /tmp/review_word_count.jim.20130215.071901.095847 -reading from file -> /home/jim/src/datamining290/code/venv/bin/python review_word_count.py --step-num=0 --mapper /tmp/review_word_count.jim.20130215.071901.095847/input_part-00000 -writing to /tmp/review_word_count.jim.20130215.071901.095847/step-0-mapper_part-00000 -Counters from step 1: - (no counters found) -... -Streaming final output from /tmp/review_word_count.jim.20130215.071901.095847/output -"4" 2 -"5" 1 -"50" 1 -"6" 2 -"7" 2 -"70s" 1 -"9" 2 -"a" 46 -"abbey" 4 -"able" 1 -"about" 4 -#+end_src - -* Fill in unique\_review.py :slide: - + Mutli-step map reduce - + Steps are explained in lecture - + Skeleton in code - -* Write user\_similarity.py :slide: - + Find users >= 0.5 similarity - + User Similarity: Jaccard similarity of businesses reviewed - + {BizA, BizB, BizC} ~ {BizF, BizB, BizG} - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-22-Bayes.html b/slides/2013-02-22-Bayes.html deleted file mode 100644 index 1d40f66..0000000 --- a/slides/2013-02-22-Bayes.html +++ /dev/null @@ -1,598 +0,0 @@ - - - - -2013-02-22-Bayes - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-22-Bayes

- - - - -
-

1 Classification: Bayes    slide

-
- - -
- -
- -
-

2 Confusion Matrix    slide

-
- -
    -
  • What are the ways that classification can be wrong? -
    • -
- - - -- - - - - - -
Predict: PositivePredict: Negative
Actual: PositiveTrue PositiveFalse Negative
Actual: NegativeFalse NegativeTrue Negative
- - -
- -
-

2.1 Obtain Data    notes

-
- -
    -
  • How do we obtain this data? -
    • -
- - -
-
- -
- -
-

3 Testing Data    slide two_col

-
- -
    -
  • Data used to test a learned model -
    • -
  • Test data was not used to learn -
    • -
  • Where does test data come from? -
    • -
- -

img/stork.jpg -

-
- -
-

3.1 Not from storks    notes

-
- - - - -
-
- -
- -
-

4 Training Data    slide

-
- -
    -
  • Set aside a portion of training data to test with -
    • -
  • Test data: -
    • -
- -

img/k-fold1.png -

- -
- -
-

4.1 Set Aside Testing    slide

-
- -

img/k-fold2.png -

-

- Testing Data | Training Data -

- -
- -
-

4.2 Colors    notes

-
- -
    -
  • Red: Testing -
    • -
  • Green: Training -
    • -
- - -
- -
- -
-

4.3 Cross Validation    slide

-
- -

img/k-fold3.png -

-

- Train and test model with different subsets of data -

- -
- -
-

4.4 Testing the model    notes

-
- -
    -
  • This is used to test the model -
    • -
  • How well does it perform with a variety of inputs? -
    • -
  • Is it robust against outliers -
    • -
- - -
- -
- -
-

4.5 K-Fold Validation    slide

-
- -

img/k-fold4.png -

-

- Test against K sections of the data -

- -
- -
-

4.6 Statistical Significance    notes

-
- -
    -
  • Similar to the concept in stats: the more distinct samples you have, the - better you know your data -
    • -
- - -
- -
- -
-

4.7 K-Fold Validation    slide

-
- -

img/k-fold5.png -

-
-
- -
- -
-

5 Bayes Theorem    slide

-
- -

img/bayes.png -

-

- Can calculate a posterior given priors -

-
- -
-

5.1 Read    notes

-
- -
    -
  • Probability of A given B equals probability of B given A times prob of A - divided prob of B -
    • -
  • Importance is that we can figure out what future probabilities are based on - what we've already seen -
    • -
- - -
-
- -
- -
-

6 Spam    slide

-
- -

img/bayes-spam.png -

-

- Find the probability of spam given it contains a particular word -

-
- -
-

6.1 Words    notes

-
- -
    -
  • What words would you associate with spam? -
    • -
  • Are these the same across all people? -
    • -
  • Why might you want to train a classifier per person? -
    • -
- - -
-
- -
- -
-

7 Multiple Words    slide animate

-
- -
    -
  • How to calculate probabilities of multiple independent events occurring? -
    • -
  • Model words as independent events -
    • -
  • Multiply probabilities -
    • -
- - -
- -
-

7.1 Naive    notes

-
- -
    -
  • Words are not independent -
    • -
  • San? Francisco is more likely -
    • -
  • But works suprisingly well in practice -
    • -
- - -
-
- -
- -
-

8 Practical concerns    slide animate

-
- -
    -
  • What is the probability of a word we've never seen before? -
    • -
  • Underflow: multiplying numbers still everything is rounded to 0 -
    • -
  • Normalizing words: v1agra -
    • -
- - -
- -
-

8.1 Solutions    notes

-
- -
    -
  • divide by 0. Instead, add 1 to all words -
    • -
  • using log of probabilities -
    • -
  • Rules -
    • -
- - -
-
- -
- -
-

9 Ensemble    slide

-
- -
    -
  • Using multiple models simultaneously -
    • -
  • Run all classifiers over new data, take majority vote -
    • -
  • Netflix Prize won with combination of models from several teams -
    • -
- - -
- -
-

9.1 Requirements    notes

-
- -
    -
  • Nice thing is that the diversity of models is important, and not so much - the accuracy of any single model -
    • -
- - -
-
- -
- -
-

10 Bootstrap Aggregating    slide two_col

-
- -
    -
  • Bagging: training data collected with replacement -
    • -
  • Learn models on different samples -
    • -
  • Run models on new incoming data -
    • -
- -

img/bagging.png -

-
- -
-

10.1 Trade-offs    notes

-
- - - - -
-
- -
- -
-

11 Boosting    slide

-
- -
    -
  • Train classifier to catch what the last one missed -
    • -
  • Train and test first classifier -
    • -
  • Find classification failures -
    • -
  • Weight more heavily those failures in training a new model -
    • -
  • Weight models by their accuracy -
    • -
- - -
- -
-

11.1 Trade-offs    notes

-
- -
    -
  • Boosting can be susceptible to outliers -
    • -
  • Longer to train -
    • -
  • Observed to be more accurate -
    • -
- - -
-
- -
- -
-

12 Many Decision Trees    slide

-
- -
    -
  • Train trees with random selection of attributes, subset of data -
    • -
  • Combine trees using majority or weights -
    • -
  • What to call many arbitrarily picked trees? -
    • -
- - - -
- -
-

12.1 Random Forests    slide two_col

-
- -

img/green-forrest.jpg -

    -
  • Used successfully in many recent competitions -
    • -
  • Carry over robustness properties from individual decision trees -
    • -
  • Can be trained in parallel -
    • -
- -
- -
- -
-

12.2 Parallel    notes

-
- -
    -
  • Potentially good fit for MapReduce paradigms -
    • -
- - - - - - - -
-
-
-
- -
-

Date: 2013-02-22 14:04:03 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-22-Bayes.org b/slides/2013-02-22-Bayes.org deleted file mode 100644 index 33ea758..0000000 --- a/slides/2013-02-22-Bayes.org +++ /dev/null @@ -1,147 +0,0 @@ -* Classification: Bayes :slide: - -* Confusion Matrix :slide: - + What are the ways that classification can be wrong? - | | Predict: Positive | Predict: Negative | - | Actual: Positive | True Positive | False Negative | - | Actual: Negative | False Negative | True Negative | -** Obtain Data :notes: - + How do we obtain this data? - -* Testing Data :slide:two_col: - + Data used to test a learned model - + Test data was not used to learn - + Where does test data come from? - [[file:img/stork.jpg]] -** Not from storks :notes: - + img: http://adamsparkadventures.blogspot.com/2011/09/stork-watch.html - -* Training Data :slide: - + Set aside a portion of training data to test with - + Test data: - [[file:img/k-fold1.png]] - -** Set Aside Testing :slide: - [[file:img/k-fold2.png]] - - Testing Data | Training Data -** Colors :notes: - + Red: Testing - + Green: Training - -** Cross Validation :slide: - [[file:img/k-fold3.png]] - - Train and test model with different subsets of data -** Testing the model :notes: - + This is used to test the *model* - + How well does it perform with a variety of inputs? - + Is it robust against outliers - -** K-Fold Validation :slide: - [[file:img/k-fold4.png]] - - Test against K sections of the data -** Statistical Significance :notes: - + Similar to the concept in stats: the more distinct samples you have, the - better you know your data - -** K-Fold Validation :slide: - [[file:img/k-fold5.png]] - -* Bayes Theorem :slide: - [[file:img/bayes.png]] - - Can calculate a posterior given priors -** Read :notes: - + Probability of A given B equals probability of B given A times prob of A - divided prob of B - + Importance is that we can figure out what future probabilities are based on - what we've already seen - -* Spam :slide: - [[file:img/bayes-spam.png]] - - Find the probability of spam given it contains a particular word -** Words :notes: - + What words would you associate with spam? - + Are these the same across all people? - + Why might you want to train a classifier per person? - -* Multiple Words :slide:animate: - + How to calculate probabilities of multiple independent events occurring? - + Model words as independent events - + Multiply probabilities -** Naive :notes: - + Words are not independent - + San? Francisco is more likely - + But works suprisingly well in practice - -* Practical concerns :slide:animate: - + What is the probability of a word we've never seen before? - + Underflow: multiplying numbers still everything is rounded to 0 - + Normalizing words: v1agra -** Solutions :notes: - + divide by 0. Instead, add 1 to all words - + using log of probabilities - + Rules - -* Ensemble :slide: - + Using multiple models simultaneously - + Run all classifiers over new data, take majority vote - + Netflix Prize won with combination of models from several teams -** Requirements :notes: - + Nice thing is that the diversity of models is important, and not so much - the accuracy of any single model - -* Bootstrap Aggregating :slide:two_col: - + Bagging: training data collected with replacement - + Learn models on different samples - + Run models on new incoming data - [[file:img/bagging.png]] -** Trade-offs :notes: - + Fairly simple: - + Majority vote - + Train models independently - + img: http://cse-wiki.unl.edu/wiki/index.php/Bagging_and_Boosting - -* Boosting :slide: - + Train classifier to catch what the last one missed - + Train and test first classifier - + Find classification failures - + Weight more heavily those failures in training a new model - + Weight models by their accuracy -** Trade-offs :notes: - + Boosting can be susceptible to outliers - + Longer to train - + Observed to be more accurate - -* Many Decision Trees :slide: - + Train trees with random selection of attributes, subset of data - + Combine trees using majority or weights - + What to call many arbitrarily picked trees? - -** Random Forests :slide:two_col: -[[file:img/green-forrest.jpg]] - + Used successfully in many recent competitions - + Carry over robustness properties from individual decision trees - + Can be trained in parallel -** Parallel :notes: - + Potentially good fit for MapReduce paradigms - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-22-Decision-Trees.html b/slides/2013-02-22-Decision-Trees.html deleted file mode 100644 index ed224b5..0000000 --- a/slides/2013-02-22-Decision-Trees.html +++ /dev/null @@ -1,763 +0,0 @@ - - - - -2013-02-22-Decision-Trees - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-22-Decision-Trees

- - - - -
-

1 Classification: Decision Trees    slide

-
- - -
- -
- -
-

2 Types of Models    slide animate

-
- -
    -
  • Classifiers -
    • -
  • Regressions -
    • -
  • Clustering -
    • -
  • Outlier -
    • -
- - -
- -
-

2.1 Details    notes

-
- -
-
Classifiers
describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description -
-
Regressions
Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms -
-
Clustering
find "natural" groups of data without labels -
-
Outlier
find anomalous transactions, eg. finding fraud for credit cards -
-
- - -
-
- -
- -
-

3 Process    slide animate

-
- -
    -
  • Training Set -
    • -
  • Learning -
    • -
  • Model / Classifier -
    • -
  • Testing Set -
    • -
  • Verification / Accuracy -
    • -
  • New Data -
    • -
  • Classification -
    • -
- - -
- -
-

3.1 Steps    notes

-
- -
-
Process
to be able to classify data -
-
Training Set
Cleaned, preprocessed data that has labels. What are - labels? -
-
Learning
Feed the training set to an algorithm. Algorithm associates - some of the features with the labels and generates a model. -
-
Model / Classifier
Process or formula used to predict the label (class) given inputs - (data record) -
-
Testing Set
Data not in training set, with labels. Run through model - to see how the model compares with the real labels. -
-
Verification / Accuracy
Given the matches / mismatches in the testing - set, how can we measure how well the model reflects reality? -
-
Unseen Data
Finally, we're ready to start using our model / classifier to - label new, real, unknown data! So clean and pre-process it the same way. -
-
Classification
Feed the unknown data and get out results! -
-
- - -
-
- -
- -
-

4 Learning    slide

-
- -

img/model.png -

-
- -
-

4.1 Example    notes

-
- -
    -
  • We have training data. What are these column types? -
    • -
  • Feed it into a classification algorithm -
    • -
  • In the case it is generating Rules. -
    • -
  • Models can be as simple as this: just a set of rules to follow. We'll see - how we can extend this idea -
    • -
  • The learning step generates a model: these rules -
    • -
- - -
-
- -
- -
-

5 Classification    slide

-
- -

img/classifying.png -

-
- -
-

5.1 Possibilities    notes

-
- -
    -
  • Now that we have the model / classifier, we can do two things -
    • -
  • 1: Use testing data different from training data -
    • -
  • compare the classifier guesses with reality -
    • -
  • 2: Use the classifier on unknown data -
    • -
  • Why not just jump into classifying unknown data? Why have a test step? -
    • -
- - -
-
- -
- -
-

6 Machine Learning    slide

-
- -
-
Supervised
Given data with a label, predict data without a - label -
-
Unsupervised
Given data without labels, group "similar" items - together -
-
Semi-supervised
Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases -
-
Active
Starting with unlabeled data, select the most helpful cases for a - human to label -
-
- - -
- -
-

6.1 Which is this?    notes

-
- -
    -
  • In the example above, what type of learning? -
    • -
  • Supervised: we have labels, we want to guess unlabeled data -
    • -
- - -
-
- -
- -
-

7 Confusion Matrix    slide

-
- -
    -
  • What are the ways that classification can be wrong? -
    • -
- - - -- - - - - - -
Predict: PositivePredict: Negative
Actual: PositiveTrue PositiveFalse Negative
Actual: NegativeFalse NegativeTrue Negative
- - -
- -
-

7.1 Basis for Evaluation    notes

-
- -
    -
  • Most methods of evaluating results start with the confusion matrix -
    • -
  • Figuring out what different ways you were right or wrong -
    • -
  • Then using different formulas to emphasize the things you care about -
    • -
- - -
-
- -
- -
-

8 Recall & Precision    slide

-
- -
    -
  • Recall: TP / P -
    • -
  • Precision: TP / (TP + FP) -
    • -
  • Sometimes these are in tension; other measurements balance them -
    • -
- - -
- -
-

8.1 Trade-off    notes

-
- -
    -
  • Classic trade-off in search -
    • -
- - -
-
- -
- -
-

9 Example: Search    slide

-
- -

img/burrito-search.png -

-
- -
-

9.1 Searching Yelp    notes

-
- -
    -
  • Searched yelp for a burrito in the Mission -
    • -
  • How good are these search results? -
    • -
  • Let's say we knew this first result was great, and only returned it -
    • -
  • What would our precision be? -
    • -
  • What would the recall be? -
    • -
  • How could we improve recall? -
    • -
  • How can we guarantee 100% recall? -
    • -
  • What will that do to the precision? -
    • -
  • Understand ways of combining these measurements in the book -
    • -
- - -
-
- -
- -
-

10 Decision Trees    slide

-
- -
    -
  • Rules formulated as a tree of decisions -
    • -
  • Choose Your Own Adventure for machine learning -
    • -
  • So how do we build the trees? -
    • -
- - -
- -
-

10.1 Rules expressed trees    notes

-
- -
    -
  • At each node in the tree, pose a question -
    • -
  • Take a branch depending on your answer -
    • -
  • Leaf nodes are labels -
    • -
- - -
-
- -
- -
-

11 Build a Tree    slide

-
- -

img/model.png -

-
- -
-

11.1 Directions    notes

-
- -
    -
  • First node question: is rank=professor? -
    • -
  • If True, what's the label? -
    • -
  • If False, we go to another node -
    • -
  • Second node question: is years > 6? -
    • -
  • If True what's the label? -
    • -
  • If False, what's the label? -
    • -
- - -
-
- -
- -
-

12 Build a Tree    slide

-
- -

img/tree-dataset.png -

-
- -
-

12.1 Next challenge    notes

-
- -
    -
  • How to go from a data set like this -
    • -
- - -
- -
- -
-

12.2 Build a Tree    slide

-
- -

img/tree.png -

-
- -
-

12.2.1 Result    notes

-
- -
    -
  • To a tree like this? -
    • -
- - -
-
-
- -
- -
-

13 Decision Tree Induction    slide

-
- -
    -
  • Start with all the data -
    • -
  • Choose the "best" way to divide it up based on one attribute -
    • -
  • Make a node that asks a question to split the data -
    • -
  • Choose new "best" way to divide based on remaining attributes -
    • -
  • Stop: no attributes left, all records are the same class -
    • -
- - -
- -
-

13.1 Recursive    notes

-
- -
    -
  • Look at all the attributes. What's the best way to split up the data? -
    • -
  • We'll look at way to mathematically evaluate splits -
    • -
  • Now recursively do the same -
    • -
  • If you've split on all the attributes, but still have a mix, use a majority - rule -
    • -
  • If all the records are the same class, you don't have to keep spitting: - your answer is right there! -
    • -
  • For continuous data, must bucket it so you can have a discrete number of - answers -
    • -
- - -
-
- -
- -
-

14 Information Gain    slide

-
- -
    -
  • Comparison of how mixed results are before and after splitting -
    • -
  • Entropy measurement of "mixed" -
    • -
  • Two pure data sets have less entropy on average than one mixed -
    • -
- - -
- -
-

14.1 Information    notes

-
- -
    -
  • Book will go into detail about how to think about entropy -
    • -
  • General idea: how difficult would it be to memorize the data sets? -
    • -
  • Easy if pure: all class A -
    • -
  • Still fairly easy if 2 pure sets: 1 is class A, other is class B -
    • -
  • Now more difficult if they are mixed: first 2 records are A, then one B, - then another A -
    • -
- - -
-
- -
- -
-

15 Gini Index    slide

-
- - - - -
Gini(D) = 1 - sum(frac**2 for frac in classes)
-
- -

- Sum of the squares of the fraction of items in each class -

-
- -
- -
-

16 Splitting    slide

-
- -
    -
  • Discrete values can split per value -
    • -
  • Or discrete values binary split into subsets -
    • -
  • Continuous values can split on range (usually 2) -
    • -
- - -
- -
-

16.1 Different    notes

-
- -
    -
  • If you'd like a binary tree (useful for some algorithms), can split on - subsets -
    • -
  • Can't split 400 different ways on continuous values… what about values - that haven't been seen before? -
    • -
- - -
-
- -
- -
-

17 Decision Tree Advantages    slide

-
- -
    -
  • Models easy to understand and visualize -
    • -
  • Can be faster to construct -
    • -
  • Can encode tree in declarative languages (SQL) -
    • -
  • Robust: outliers generally fit in with normal data -
    • -
- - -
- -
-

17.1 Trees    notes

-
- -
    -
  • Its a tree! Easy to draw -
    • -
  • Greedy algorithm means you're only go over the data so many times -
    • -
  • Models can translate into database statements -
    • -
  • Outliers don't have a numeric pull on the data (similar to difference - between median and mean) -
    • -
- - -
-
- -
- -
-

18 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-02-22 08:41:02 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-22-Decision-Trees.org b/slides/2013-02-22-Decision-Trees.org deleted file mode 100644 index daab6e6..0000000 --- a/slides/2013-02-22-Decision-Trees.org +++ /dev/null @@ -1,225 +0,0 @@ -* Classification: Decision Trees :slide: - -* Types of Models :slide:animate: - + Classifiers - + Regressions - + Clustering - + Outlier -** Details :notes: - + Classifiers :: describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description - + Regressions :: Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms - + Clustering :: find "natural" groups of data *without labels* - + Outlier :: find anomalous transactions, eg. finding fraud for credit cards - -* Process :slide:animate: - + Training Set - + Learning - + Model / Classifier - + Testing Set - + Verification / Accuracy - + New Data - + Classification -** Steps :notes: - + Process :: to be able to classify data - + Training Set :: Cleaned, preprocessed data that has labels. What are - labels? - + Learning :: Feed the training set to an algorithm. Algorithm associates - some of the features with the labels and generates a model. - + Model / Classifier :: Process or formula used to predict the label (class) given inputs - (data record) - + Testing Set :: Data *not in training set*, with labels. Run through model - to see how the model compares with the real labels. - + Verification / Accuracy :: Given the matches / mismatches in the testing - set, how can we measure how well the model reflects reality? - + Unseen Data :: Finally, we're ready to start using our model / classifier to - label new, real, unknown data! So clean and pre-process it the same way. - + Classification :: Feed the unknown data and get out results! - -* Learning :slide: - [[file:img/model.png]] -** Example :notes: - + We have training data. What are these column types? - + Feed it into a classification algorithm - + In the case it is generating Rules. - + Models can be as simple as this: just a set of rules to follow. We'll see - how we can extend this idea - + The learning step generates a model: these rules - -* Classification :slide: - [[file:img/classifying.png]] -** Possibilities :notes: - + Now that we have the model / classifier, we can do two things - + 1: Use testing data *different* from training data - + compare the classifier guesses with reality - + 2: Use the classifier on unknown data - + Why not just jump into classifying unknown data? Why have a test step? - -* Machine Learning :slide: - + Supervised :: Given data with a label, predict data without a - label - + Unsupervised :: Given data without labels, group "similar" items - together - + Semi-supervised :: Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases - + Active :: Starting with unlabeled data, select the most helpful cases for a - human to label -** Which is this? :notes: - + In the example above, what type of learning? - + Supervised: we have labels, we want to guess unlabeled data - -* Confusion Matrix :slide: - + What are the ways that classification can be wrong? - | | Predict: Positive | Predict: Negative | - | Actual: Positive | True Positive | False Negative | - | Actual: Negative | False Negative | True Negative | -** Basis for Evaluation :notes: - + Most methods of evaluating results start with the confusion matrix - + Figuring out what different ways you were right or wrong - + Then using different formulas to emphasize the things you care about - -* Recall & Precision :slide: - + Recall: =TP / P= - + Precision: =TP / (TP + FP)= - + Sometimes these are in tension; other measurements balance them -** Trade-off :notes: - + Classic trade-off in search - -* Example: Search :slide: - [[file:img/burrito-search.png]] -** Searching Yelp :notes: - + Searched yelp for a burrito in the Mission - + How good are these search results? - + Let's say we knew this first result was great, and *only* returned it - + What would our precision be? - + What would the recall be? - + How could we improve recall? - + How can we guarantee 100% recall? - + What will that do to the precision? - + Understand ways of combining these measurements in the book - -* Decision Trees :slide: - + Rules formulated as a tree of decisions - + Choose Your Own Adventure for machine learning - + So how do we build the trees? -** Rules expressed trees :notes: - + At each node in the tree, pose a question - + Take a branch depending on your answer - + Leaf nodes are labels - -* Build a Tree :slide: - [[file:img/model.png]] -** Directions :notes: - + First node question: is rank=professor? - + If True, what's the label? - + If False, we go to another node - + Second node question: is years > 6? - + If True what's the label? - + If False, what's the label? - -* Build a Tree :slide: - [[file:img/tree-dataset.png]] -** Next challenge :notes: - + How to go from a data set like this - -** Build a Tree :slide: - [[file:img/tree.png]] -*** Result :notes: - + To a tree like this? - -* Decision Tree Induction :slide: - + Start with all the data - + Choose the "best" way to divide it up based on one attribute - + Make a node that asks a question to split the data - + Choose new "best" way to divide based on remaining attributes - + Stop: no attributes left, all records are the same class -** Recursive :notes: - + Look at all the attributes. What's the best way to split up the data? - + We'll look at way to mathematically evaluate splits - + Now recursively do the same - + If you've split on all the attributes, but still have a mix, use a majority - rule - + If all the records are the same class, you don't have to keep spitting: - your answer is right there! - + For continuous data, must bucket it so you can have a discrete number of - answers - -* Information Gain :slide: - + Comparison of how mixed results are before and after splitting - + Entropy measurement of "mixed" - + Two pure data sets have less entropy on average than one mixed -** Information :notes: - + Book will go into detail about how to think about entropy - + General idea: how difficult would it be to memorize the data sets? - + Easy if pure: all class A - + Still fairly easy if 2 pure sets: 1 is class A, other is class B - + Now more difficult if they are mixed: first 2 records are A, then one B, - then another A - -* Gini Index :slide: -#+begin_src python - Gini(D) = 1 - sum(frac**2 for frac in classes) -#+end_src - Sum of the squares of the fraction of items in each class - -* Splitting :slide: - + Discrete values can split per value - + Or discrete values binary split into subsets - + Continuous values can split on range (usually 2) -** Different :notes: - + If you'd like a binary tree (useful for some algorithms), can split on - subsets - + Can't split 400 different ways on continuous values... what about values - that haven't been seen before? - -* Continuous Splitting :slide: - + Test every split point to see which is best - + Possible split points: midpoint between every adjacent value pair - + Sort attribute, score midpoints -1 -2 -2 -2 -26 -36 -36 -74 -323 -345 -2234 -** Review :notes: - + What does best mean? - + Can calculate with one pass through the data since you are just moving a - few cases from one class to another - -* Decision Tree Advantages :slide: - + Models easy to understand and visualize - + Can be faster to construct - + Can encode tree in declarative languages (SQL) - + Robust: outliers generally fit in with normal data -** Trees :notes: - + Its a tree! Easy to draw - + Greedy algorithm means you're only go over the data so many times - + Models can translate into database statements - + Outliers don't have a numeric pull on the data (similar to difference - between median and mean) - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-02-22-Gini.html b/slides/2013-02-22-Gini.html deleted file mode 100644 index 5d0084d..0000000 --- a/slides/2013-02-22-Gini.html +++ /dev/null @@ -1,165 +0,0 @@ - - - - -2013-02-22-Gini - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-02-22-Gini

- - - - -
-

1 HW: Gini    slide

-
- -
    -
  • Calculate Gini Index -
    • -
- - -
- -
- -
-

2 Gini Index    slide

-
- - - - -
Gini(D) = 1 - sum(frac**2 for frac in classes)
-
- -

- Sum of the squares of the fraction of items in each class -

-
- -
- -
-

3 Data: Campaign Contributions    slide

-
- -
    -
  • Calculating the Gini Index for the Candidate Names for the enitre data set -
    • -
  • Partition by zip code, calculate the weighted average Gini Index score over - all partitions -
    • -
  • Partitions are weighted by the number of records they contain divided by the - total number of records in the data set -
    • -
- - -
- -
- -
-

4 Extra Credit    slide

-
- -
    -
  • Find a best split of a continuous field -
    • -
- - -
- -
- -
-

5 Python Tips    slide

-
- -
    -
  • collections -
    • -
  • defaultdict autovivifies keys -
    • -
  • Counter autovivifies integer keys -
    • -
- - - - -
zipcodes = defaultdict(Counter)
-
- - -
- -
- -
-

6 Git Tips    slide

-
- - - - -
$ git checkout master
-$ git pull jblomo master
-$ git checkout -b hw-gini
-
- - - - - -
-
-
- -
-

Date: 2013-02-23 16:11:38 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-02-22-Gini.org b/slides/2013-02-22-Gini.org deleted file mode 100644 index 9232e3d..0000000 --- a/slides/2013-02-22-Gini.org +++ /dev/null @@ -1,50 +0,0 @@ -* HW: Gini :slide: - + Calculate Gini Index - -* Gini Index :slide: -#+begin_src python - Gini(D) = 1 - sum(frac**2 for frac in classes) -#+end_src - Sum of the squares of the fraction of items in each class - -* Data: Campaign Contributions :slide: - + Calculating the Gini Index for the Candidate Names for the enitre data set - + Partition by zip code, calculate the weighted average Gini Index score over - all partitions - + Partitions are weighted by the number of records they contain divided by the - total number of records in the data set - -* Extra Credit :slide: - + Find a best split of a continuous field - -* Python Tips :slide: - + [[http://docs.python.org/2/library/collections.html][collections]] - + =defaultdict= autovivifies keys - + =Counter= autovivifies integer keys -#+begin_src python -zipcodes = defaultdict(Counter) -#+end_src - -* Git Tips :slide: -#+begin_src bash -$ git checkout master -$ git pull jblomo master -$ git checkout -b hw-gini -#+end_src - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-01-Lab-NN.html b/slides/2013-03-01-Lab-NN.html deleted file mode 100644 index 9b2dc10..0000000 --- a/slides/2013-03-01-Lab-NN.html +++ /dev/null @@ -1,149 +0,0 @@ - - - - -2013-03-01-Lab-NN - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-01-Lab-NN

- - -
-

Table of Contents

-
- -
-
- -
-

1 Continue Back Propigation    slide

-
- -

img/ann8.png -

-
- -
-

1.1 Expected    notes

-
- -
    -
  • Labeling: Top to bottom, left to right: -
    • -
  • 1 is upper left, 2 is lower left, 3 is top hidden layer, 6 is output layer -
    • -
- - -
-
- -
- -
-

2 Submit    slide

-
- -

nn-train.txt -

- - -
err_6 = 
-err_5 = 
-...
-w_36 = 
-w_46 = 
-...
-w_13 = 
-...
-
- - -
- -
-

2.1 Fill In    notes

-
- -
    -
  • Fill in values -
    • -
  • You may use a calculator or Python -
    • -
  • If you want feedback or partial credit, include any code in another file -
    • -
- - -
-
- -
- -
-

3 Project    slide

-
- -
    -
  • Find a partner (if applicable) -
    • -
  • Find a data set -
    • -
  • Start brainstorming -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-03-01 08:48:39 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-01-Lab-NN.org b/slides/2013-03-01-Lab-NN.org deleted file mode 100644 index 9d2829d..0000000 --- a/slides/2013-03-01-Lab-NN.org +++ /dev/null @@ -1,44 +0,0 @@ -* Continue Back Propigation :slide: - [[file:img/ann8.png]] -** Expected :notes: - + Labeling: Top to bottom, left to right: - + 1 is upper left, 2 is lower left, 3 is top hidden layer, 6 is output layer - -* Submit :slide: - =nn-train.txt= -#+begin_src text -err_6 = -err_5 = -... -w_36 = -w_46 = -... -w_13 = -... -#+end_src -** Fill In :notes: - + Fill in values - + You may use a calculator or Python - + If you want feedback or partial credit, include any code in another file - -* Project :slide: - + Find a partner (if applicable) - + Find a [[http://blog.bigml.com/2013/02/28/data-data-data-thousands-of-public-data-sources/][data set]] - + Start brainstorming - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-01-Neural-Network.html b/slides/2013-03-01-Neural-Network.html deleted file mode 100644 index c8321ba..0000000 --- a/slides/2013-03-01-Neural-Network.html +++ /dev/null @@ -1,959 +0,0 @@ - - - - -2013-03-01-Neural-Network - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-01-Neural-Network

- - - - -
-

1 Bias vs. Variance    slide

-
- - -
- -
- -
-

2 Trade-offs    slide

-
- -
    -
  • Similar to precision, we make trade-offs when training models -
    • -
  • Bias: How far off are the model predictions on average? -
    • -
  • Variance: If we retrained with different data, how different would our - guesses be? -
    • -
- - -
- -
-

2.1 Details    notes

-
- -
    -
  • Bias: difference in "Expected" value from models from the real value -
    • -
  • Variance: difference in "Expected" value from each other -
    • -
  • Variance: Another way to think about it: how specific is our model to our - data? If we were training a tree with k-fold validation, would we get - completely different rule sets for each set of data? -
    • -
  • "Expected": These are model type properties. Train the model multiple - times with different data, then evaluate all models performance -
    • -
- - -
-
- -
- -
-

3 Regression    slide two_col

-
- -
    -
  • Can we do better than linear regression on some data sets? -
    • -
  • Polynomial regression -
    • -
  • How many polynomials? -
    • -
- -

img/overfit1.png -

-
- -
-

3.1 Polynomial    notes

-
- -
    -
  • Sure! Use a polynomial instead: x^2 2x - x^2 + 4x^3 -
    • -
  • If you're not sure what the underlying data model is, have to test -
    • -
  • img: http://cheshmi.tumblr.com/ -
    • -
- - -
- -
- -
-

3.2 One    slide

-
- -

img/overfit1.png -

-
- -
-

3.2.1 So-So    notes

-
- -
    -
  • How is the bias? Not great, fair amount of error -
    • -
  • How is the variance? Pretty good, assuming random sample -
    • -
- - -
-
- -
- -
-

3.3 Two    slide

-
- -

img/overfit2.png -

-
- -
-

3.3.1 Better    notes

-
- -
    -
  • Bias? Better, less error -
    • -
  • Variance? more risky depending on which samples you get, since model - diverges quickly -
    • -
- - -
-
- -
- -
-

3.4 Three    slide

-
- -

img/overfit3.png -

-
- -
-

3.4.1 Worrying    notes

-
- -
    -
  • Now getting a little weird. We're not finding the general pattern, more - like exactly fitting a line over these points -
    • -
  • If we made model with different data, we're going to get a different line -
    • -
- - -
-
- -
- -
-

3.5 Many    slide

-
- -

img/overfit5.png -

-
- -
-

3.5.1 Now kind of ridiculous    notes

-
- -
    -
  • Intuitively we know this is not a description of the data -
    • -
  • If a point was found near the border, completely dependant on the data the - model trained on -
    • -
- - -
-
-
- -
- -
-

4 Over-fitting    slide

-
- -
    -
  • Over-fitting: reflecting the exact data given instead of the general pattern -
    • -
  • High variance is a sign of over-fitting: model guesses vary with the exact - data given -
    • -
  • Avoidance: ensembles average out variance, regularization adds a cost to - model complexity -
    • -
- - -
- -
-

4.1 Avoidance    notes

-
- -
    -
  • Ensembles combine multiple models together. Those multiple models may have - a lot of variance, but as long as they have good Bias, we'll center in on - the correct result -
    • -
  • Remember our cost function? We wanted to minimize the error. If you add in - a way to measure model complexity, you can add that to the cost, so that - you are explicitly trading-off the complexity of your model with the - quality of the solution -
    • -
  • If we wanted to add a complexity cost to the previous model, what would the - cost be dependent on? -
    • -
- - -
-
- -
- -
-

5 Neural Networks    slide

-
- -

img/neuron_culture.jpg -

-
- -
-

5.1 notes    notes

-
- - - - -
-
- -
- -
-

6 Brains    slide

-
- -
    -
  • Neural networks try to model our brains -
    • -
  • Neurons/perceptrons sense input, transform it, send output -
    • -
  • Neurons/perceptrons are connected together -
    • -
  • Connections have different strengths -
    • -
- - -
- -
- -
-

7 Training    slide

-
- -
    -
  • Learn by adjusting the strengths of the connections -
    • -
  • Mathematically, strength is a weight multiplier of the output -
    • -
  • When we've found the right weights -
    • -
- - -
- -
- -
-

8 Nomenclature    slide two_col

-
- -
-
Input layer
neurons whose input is determined by features -
-
Hidden layer
neurons that calculate a combination of features -
-
Output layer
neurons that express the classification -
-
Weights
numeric parameter to adjust input/output -
-
- -

img/nn.png -

-
- -
- -
-

9 Handwriting    slide

-
- -
    -
  • Recognize handwritten digits -
    • -
- -

img/neuron11.gif -

-
- -
-

9.1 Inputs => Outputs    notes

-
- - - - -
-
- -
- -
-

10 Forward Propagation    slide

-
- -
    -
  1. Sum of inputs * weights -
    2. -
  2. Apply sigmoid -
    3. -
  3. Send output to next layer -
    4. -
  4. Repeat -
    5. -
- - - -
- -
-

10.1 Repeat    slide

-
- -
    -
  • Multiple hidden layers used to model complex feature interaction -
    • -
- -

img/2-layer-nn.gif -

-
- -
- -
-

10.2 Sigmoid    slide two_col

-
- -
    -
  • Normalize input to [0,1] -
    • -
  • Makes weak input weaker, strong input stronger -
    • -
  • 1 / (1 + e^-input) -
    • -
- -

img/sigmoid.png] -

-
-
- -
- -
-

11 Example    slide

-
- -

img/nn-fp1.png -

-
- -
-

11.1 Simple    notes

-
- -
    -
  • Simple NN with just one output -
    • -
  • Output can model true/false -
    • -
  • Inputs are numerical -
    • -
- - -
- -
- -
-

11.2 Weights    slide

-
- -

img/ann2.png -

-
- -
-

11.2.1 Later    notes

-
- -
    -
  • We'll discuss how weights are determined later -
    • -
  • Fill in the Hidden layer with sum of inputs * weights -
    • -
- - -
-
- -
- -
-

11.3 Sigmoid    slide

-
- -

img/ann3.png -

-
- -
-

11.3.1 Apply    notes

-
- -
    -
  • Apply the sigmoid to the incoming signals -
    • -
- - -
-
- -
- -
-

11.4 Sigmoid    slide

-
- -

img/ann4.png -

-
- -
-

11.4.1 Apply    notes

-
- -
    -
  • Apply the sigmoid to the incoming signals -
    • -
- - -
-
- -
- -
-

11.5 Sigmoid    slide

-
- -

img/ann5.png -

-
- -
-

11.5.1 Apply    notes

-
- -
    -
  • Apply the sigmoid to the incoming signals -
    • -
- - -
-
- -
- -
-

11.6 Sigmoid    slide

-
- -

img/ann6.png -

-
- -
-

11.6.1 Apply    notes

-
- -
    -
  • Apply the sigmoid to the incoming signals -
    • -
- - -
-
- -
- -
-

11.7 Weights    slide

-
- -

img/ann7.png -

-
- -
-

11.7.1 Repeat    notes

-
- -
    -
  • Take the outputs, apply weights, sum -
    • -
- - -
-
- -
- -
-

11.8 Sigmoid    slide

-
- -

img/ann8.png -

-
- -
-

11.8.1 Apply    notes

-
- -
    -
  • Apply the sigmoid to the incoming signals -
    • -
  • Our result is greater than 0.5, so we can assume true -
    • -
  • If we had multiple outputs, we could choose the highest one -
    • -
- - -
-
-
- -
- -
-

12 Forward Propagation    slide

-
- -
    -
  1. Sum of inputs * weights -
    2. -
  2. Apply sigmoid -
    3. -
  3. Send output to next layer -
    4. -
  4. Repeat -
    5. -
- - -
- -
-

12.1 Get an answer    notes

-
- -
    -
  • Now we have an output, but how do we train to get the right output? -
    • -
- - -
-
- -
- -
-

13 Fitness Function    slide

-
- -
    -
  • Create a fitness function that measures the error -
    • -
  • Take derivative and a step in the right direction -
    • -
  • Try again -
    • -
- - -
- -
-

13.1 Neural Network    notes

-
- -
    -
  • NN training is conceptually similar to gradient descent -
    • -
  • We want to get closer to the answer, so we adjust our weights based on the - amount of incorrectness in the system -
    • -
  • Adjust weights, try again -
    • -
- - -
-
- -
- -
-

14 Back Propagation    slide

-
- -
-
Run forward
Oj is output of node j -
-
Calculate error of output layer
Errj = Oj(1-Oj)(Tj-Oj) -
-
Caclulate error of hidden layer
Errj = Oj(1-Oj)*sum(Errk*wjk) -
-
Find new weights
wij = wij + l*Errj*Oi -
-
Repeat
To move closer to correct weights -
-
- - -
- -
-

14.1 Derivative    notes

-
- -
    -
  • Derivative of the sigmoid is O_j(1-O_j), so we're taking the gradient -
    • -
  • l is the learning rate, similar to a step size in gradient descent -
    • -
- - -
-
- -
- -
-

15 Example    slide

-
- -

img/ann8.png -

-
- -
-

15.1 Expected    notes

-
- - - - -
# Expected Output is 0
-t_6 = 0
-# Actual Output
-o_6 = 0.8387
-# Output Error -0.11346127339699999
-err_6 = o_6*(1-o_6)*(t_6-o_6)
-# Setup hidden node 5
-o_5 = 0.9933 ; w_56 = 1.5
-# Error for node 5 = -0.0011326458827956695
-err_5 = o_5*(1-o_5)*(err_6*w_56)
-# Adjust weight to 0.37298917134759924
-l = 10  # learning rate
-w_56 = w_56 + l*err_6*o_5
-
- - - -
-
- -
- -
-

16 Terminate Learning    slide

-
- -
    -
  • Changes in weights too small -
    • -
  • Accuracy in training models is high -
    • -
  • Maximum number or times for learning -
    • -
- - -
- -
-

16.1 Forward and Back    notes

-
- -
    -
  • Guess, correct, guess, correct -
    • -
  • Stop when you've got a good model -
    • -
  • or you model is not improving -
    • -
  • or when you're out of time -
    • -
- - -
-
- -
- -
-

17 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-03-04 23:20:39 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-01-Neural-Network.org b/slides/2013-03-01-Neural-Network.org deleted file mode 100644 index be35718..0000000 --- a/slides/2013-03-01-Neural-Network.org +++ /dev/null @@ -1,239 +0,0 @@ -* Bias vs. Variance :slide: - -* Trade-offs :slide: - + Similar to precision, we make trade-offs when training models - + Bias: How far off are the model predictions on average? - + Variance: If we retrained with different data, how different would our - guesses be? -** Details :notes: - + Bias: difference in "Expected" value from models from the real value - + Variance: difference in "Expected" value from each other - + Variance: Another way to think about it: how specific is our model to our - data? If we were training a tree with k-fold validation, would we get - completely different rule sets for each set of data? - + "Expected": These are *model type* properties. Train the model multiple - times with different data, then evaluate all models performance - -* Regression :slide:two_col: - + Can we do better than linear regression on some data sets? - + Polynomial regression - + How many polynomials? - [[file:img/overfit1.png]] -** Polynomial :notes: - + Sure! Use a polynomial instead: =x^2= =2x - x^2 + 4x^3= - + If you're not sure what the underlying data model is, have to test - + img: http://cheshmi.tumblr.com/ - -** One :slide: - [[file:img/overfit1.png]] -*** So-So :notes: - + How is the bias? Not great, fair amount of error - + How is the variance? Pretty good, assuming random sample - -** Two :slide: - [[file:img/overfit2.png]] -*** Better :notes: - + Bias? Better, less error - + Variance? more risky depending on which samples you get, since model - diverges quickly - -** Three :slide: - [[file:img/overfit3.png]] -*** Worrying :notes: - + Now getting a little weird. We're not finding the general pattern, more - like exactly fitting a line over these points - + If we made model with different data, we're going to get a different line - -** Many :slide: - [[file:img/overfit5.png]] -*** Now kind of ridiculous :notes: - + Intuitively we know this is not a description of the data - + If a point was found near the border, completely dependant on the data the - model trained on - -* Over-fitting :slide: - + Over-fitting: reflecting the exact data given instead of the general pattern - + High variance is a sign of over-fitting: model guesses vary with the exact - data given - + Avoidance: ensembles average out variance, regularization adds a cost to - model complexity -** Avoidance :notes: - + Ensembles combine multiple models together. Those multiple models may have - a lot of variance, but as long as they have good Bias, we'll center in on - the correct result - + Remember our cost function? We wanted to minimize the error. If you add in - a way to measure model complexity, you can add that to the cost, so that - you are explicitly trading-off the complexity of your model with the - quality of the solution - + If we wanted to add a complexity cost to the previous model, what would the - cost be dependent on? - -* Neural Networks :slide: - [[file:img/neuron_culture.jpg]] -** notes :notes: - + img: http://adrianbowyer.blogspot.com/2010/12/hardwired.html - -* Brains :slide: - + Neural networks try to model our brains - + Neurons/perceptrons sense input, transform it, send output - + Neurons/perceptrons are connected together - + Connections have different strengths - -* Training :slide: - + Learn by adjusting the strengths of the connections - + Mathematically, strength is a weight multiplier of the output - + When we've found the right weights - -* Nomenclature :slide:two_col: - + Input layer :: neurons whose input is determined by features - + Hidden layer :: neurons that calculate a combination of features - + Output layer :: neurons that express the classification - + Weights :: numeric parameter to adjust input/output - [[file:img/nn.png]] - -* Handwriting :slide: - + Recognize handwritten digits - [[file:img/neuron11.gif]] -** Inputs => Outputs :notes: - + Break up drawing cell into pixels - + Input takes pixel=on|off - + Output is highest valued output node, 1 for each digit - + img: http://vv.carleton.ca/~neil/neural/neuron-d.html - -* Forward Propagation :slide: - 1. Sum of inputs * weights - 1. Apply sigmoid - 1. Send output to next layer - 1. Repeat - -** Repeat :slide: - + Multiple hidden layers used to model complex feature interaction - [[file:img/2-layer-nn.gif]] - -** Sigmoid :slide:two_col: - + Normalize input to [0,1] - + Makes weak input weaker, strong input stronger - + =1 / (1 + e^-input)= - [[file:img/sigmoid.png]]] - -* Example :slide: - [[file:img/nn-fp1.png]] -** Simple :notes: - + Simple NN with just one output - + Output can model true/false - + Inputs are numerical - -** Weights :slide: - [[file:img/ann2.png]] -*** Later :notes: - + We'll discuss how weights are determined later - + Fill in the Hidden layer with sum of inputs * weights - -** Sigmoid :slide: - [[file:img/ann3.png]] -*** Apply :notes: - + Apply the sigmoid to the incoming signals - -** Sigmoid :slide: - [[file:img/ann4.png]] -*** Apply :notes: - + Apply the sigmoid to the incoming signals - -** Sigmoid :slide: - [[file:img/ann5.png]] -*** Apply :notes: - + Apply the sigmoid to the incoming signals - -** Sigmoid :slide: - [[file:img/ann6.png]] -*** Apply :notes: - + Apply the sigmoid to the incoming signals - -** Weights :slide: - [[file:img/ann7.png]] -*** Repeat :notes: - + Take the outputs, apply weights, sum - -** Sigmoid :slide: - [[file:img/ann8.png]] -*** Apply :notes: - + Apply the sigmoid to the incoming signals - + Our result is greater than 0.5, so we can assume true - + If we had multiple outputs, we could choose the highest one - -* Forward Propagation :slide: - 1. Sum of inputs * weights - 1. Apply sigmoid - 1. Send output to next layer - 1. Repeat -** Get an answer :notes: - + Now we have *an* output, but how do we train to get the *right* output? - -* Fitness Function :slide: - + Create a fitness function that measures the error - + Take derivative and a step in the right direction - + Try again -** Neural Network :notes: - + NN training is conceptually similar to gradient descent - + We want to get closer to the answer, so we adjust our weights based on the - amount of incorrectness in the system - + Adjust weights, try again - -* Back Propagation :slide: - + Run forward :: O_j is output of node =j= - + Calculate error of output layer :: Err_j = O_j(1-O_j)(T_j-O_j) - + Caclulate error of hidden layer :: Err_j = O_j(1-O_j)*sum(Err_k*w_jk) - + Find new weights :: w_ij = w_ij + l*Err_j*O_i - + Repeat :: To move closer to correct weights -** Derivative :notes: - + Derivative of the sigmoid is =O_j(1-O_j)=, so we're taking the gradient - + =l= is the learning rate, similar to =a= step size in gradient descent - -* Example :slide: - [[file:img/ann8.png]] -** Expected :notes: -#+begin_src python -# Expected Output is 0 -t_6 = 0 -# Actual Output -o_6 = 0.8387 -# Output Error -0.11346127339699999 -err_6 = o_6*(1-o_6)*(t_6-o_6) -# Setup hidden node 5 -o_5 = 0.9933 ; w_56 = 1.5 -# Error for node 5 = -0.0011326458827956695 -err_5 = o_5*(1-o_5)*(err_6*w_56) -# Adjust weight to 0.37298917134759924 -l = 10 # learning rate -w_56 = w_56 + l*err_6*o_5 -#+end_src - - -* Terminate Learning :slide: - + Changes in weights too small - + Accuracy in training models is high - + Maximum number or times for learning -** Forward and Back :notes: - + Guess, correct, guess, correct - + Stop when you've got a good model - + or you model is not improving - + or when you're out of time - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-01-SVM.html b/slides/2013-03-01-SVM.html deleted file mode 100644 index 0636acc..0000000 --- a/slides/2013-03-01-SVM.html +++ /dev/null @@ -1,928 +0,0 @@ - - - - -2013-03-01-SVM - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-01-SVM

- - - - -
-

1 Linear Regression    slide

-
- - -
- -
- -
-

2 Types of Models    slide animate

-
- -
    -
  • Classifiers -
    • -
  • Regressions -
    • -
  • Clustering -
    • -
  • Outlier -
    • -
- - -
- -
-

2.1 Details    notes

-
- -
-
Classifiers
describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description -
-
Regressions
Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms -
-
Clustering
find "natural" groups of data without labels -
-
Outlier
find anomalous transactions, eg. finding fraud for credit cards -
-
- - -
-
- -
- -
-

3 Case Study    slide

-
- -
    -
  • Housing prices: square footage -
    • -
- -

img/housing-regression.gif -

-
- -
-

3.1 Problem    notes

-
- -
    -
  • We'd like to know how to price a house based on the square footage -
    • -
  • Let's pretend this is the data we have -
    • -
  • How would we guess that value for 2500 sq ft? -
    • -
- - -
-
- -
- -
-

4 Solution?    slide animate

-
- -
    -
  • Find a line that represents the data -
    • -
  • y = m*x + b -
    • -
  • A line that is not very far from the points -
    • -
- - -
- -
-

4.1 Prompts    notes

-
- -
    -
  • In English, how would you solve this? -
    • -
  • How to mathematically represent the line? -
    • -
  • What is a good line? -
    • -
- - -
-
- -
- -
-

5 Similarity    slide

-
- -
    -
  • Main challenges in data mining: defining a specific metric for an intuition -
    • -
  • Define distance for an individual point -
    • -
  • Define how to aggregate distances together -
    • -
- - -
- -
-

5.1 Challenge    notes

-
- -
    -
  • This is big problem for engineering and math (stats) in general -
    • -
  • We'll cover some concepts, but if you're ever stuck, try looking in related - fields -
    • -
  • What are some of the ways we can measure distance between points? - Euclidian, Manhattan, Euclidian == L2 norm -
    • -
  • What is a way to aggrgate numbers? sum, sum of squares, sum of logs -
    • -
  • Differences between the last two? -
    • -
- - -
- -
- -
-

5.2 Log & Square    slide two_col

-
- -
-
Log
Useful for de-emphasizing large raw differences -
-
Square
Useful for taking the approximate absolute value -
-
- -

img/logx.gif -

-
-
- -
- -
-

6 Point Distance    slide two_col

-
- -
    -
  • y distance from line -
    • -
  • Intuitively: error in estimate -
    • -
  • h(x) = m*x + b -
    • -
  • err = h(x) - y -
    • -
- -

img/error.gif -

-
- -
-

6.1 Error    notes

-
- -
    -
  • We want the difference from what we estimate to be the value to what the - value actually is -
    • -
- - -
-
- -
- -
-

7 Aggregate    slide animate

-
- -
    -
  • sum -
    • -
  • What about negative error? -
    • -
  • Sum of squares -
    • -
  • err = sum( (h(x) - y)**2 for x,y in dataset) / len(dataset) -
    • -
- - -
- -
-

7.1 Questions    notes

-
- -
    -
  • Now we have info about all the errors from points, how to summarize? -
    • -
  • Some points have negative error, some positive? Do they cancel each other - out? -
    • -
  • Imagine data set of two points: one solutions covers lines, other divides - them. Which is better? -
    • -
  • Use our squaring trick to make sure we don't have any negative values -
    • -
  • Normalize by the number of points -
    • -
- - -
-
- -
- -
-

8 Fitness Function    slide

-
- -
    -
  • Measures the quality or cost of the solution -
    • -
  • Key ingredient for data mining algorithms -
    • -
  • If you can measure it, you can find the best solution -
    • -
- - -
- -
-

8.1 Fitness    notes

-
- -
    -
  • Function spits out a metric. Metric can be thought of as fitness or - cost -
    • -
  • Find the maximum or minimum of that metric -
    • -
  • Depending on your fitness function, this can be easy or difficult -
    • -
  • img: http://onlinestatbook.com -
    • -
- - -
-
- -
- -
-

9 Understanding Error    slide

-
- -

img/Linear_regression.svg.png - Several possible solutions -

-
- -
-

9.1 Error    notes

-
- -
    -
  • What happens to the error as we move line around? -
    • -
  • Decreases until best fit, then increases -
    • -
  • What happens if we plot this error? Say, slope (x) against error (y)? -
    • -
- - -
-
- -
- -
-

10 Solution as Minimization    slide two_col

-
- -
    -
  • Error is a parabola -
    • -
  • Several methods for finding the minimum -
    • -
  • Two categories: analytical, approximations -
    • -
- -

img/parabola.png -

-
- -
- -
-

11 Solution Approximation    slide

-
- -
    -
  • Some fitness functions can be difficult to solve analytically -
    • -
  • Alternative: iteratively get closer to the solution -
    • -
  • Stop when answer is close enough -
    • -
- - -
- -
-

11.1 Analytical    notes

-
- -
    -
  • How to find the minimum of functions in general? -
    • -
  • Take derivative, find 0 -
    • -
  • Taking derivative can be complex or impossible (discontinuities) for some - functions, or solving for 0 is difficult -
    • -
  • Instead, well keep getting closer to the minimum using the function we - already have -
    • -
- - -
-
- -
- -
-

12 Gradient Descent    slide two_col

-
- -
    -
  1. Estimate current gradient (derivative) -
    2. -
  2. Take a step (a * deriv) in the direction of the gradient -
    3. -
  3. Step size is small, stop. Else repeat. -
    4. -
- -

img/parabola.png -

-
- -
-

12.1 Steps    notes

-
- -
    -
  • Take gradient by looking at the local derivative, or perturbating x -
    • -
  • Choose a as step size weight: big a is large step size -
    • -
  • If deriv is large, will also make you step size large. -
    • -
  • If deriv is large, probably means you are far away from minimum -
    • -
  • Keep repeating -
    • -
  • What happens if a is too small? -
    • -
  • What happens if a is too big? -
    • -
- - -
-
- -
- -
-

13 General Case    slide two_col

-
- -
    -
  • Formulate fitness function for your problem -
    • -
  • Use analytics or approximations to find min/max -
    • -
  • Approximations: Newton's Method, Gradient Descent -
    • -
- -

img/error-reduce.png -

-
- -
-

13.1 Approximate visualization    notes

-
- -
    -
  • Desired output of the error as gradient descent runs -
    • -
  • maybe some local problems, as step size is too big, but slowly move down to - a small amount of error -
    • -
- - -
-
- -
- -
-

14 Support Vector Machines    slide

-
- - -
- -
- -
-

15 Decision Trees    slide two_col

-
- -
    -
  • Great for separable attributes -
    • -
  • Rules operate on independent attributes -
    • -
  • Classes separable along an axis/attribute -
    • -
- -

img/tree.png -

- -
- -
-

15.1 Linearly Separable    slide

-
- -
    -
  • How to handle case where separator line is not along an axis? -
    • -
- -

img/dataset_linsep.png -

- -
- -
-

15.2 Details    notes

-
- - - - -
-
- -
- -
-

16 Possibilities    slide

-
- -
    -
  • Many lines could separate these classes -
    • -
- -

img/dataset_linsep.png -

-
- -
-

16.1 Best?    notes

-
- -
    -
  • Which is the best? -
    • -
  • Why? -
    • -
- - -
- -
- -
-

16.2 Best Separator    slide two_col

-
- -
    -
  • Best line gives the most distance between the two classes -
    • -
  • Measure distance between closest points -
    • -
  • Closest points == support vectors -
    • -
- -

img/separable.jpg -

- -
- -
-

16.3 Points, Vectors    notes

-
- - - - -
-
- -
- -
-

17 Dimensions    slide

-
- -
    -
  • When separating two dimensions, we need a line -
    • -
  • When separating 3 dimensions? -
    • -
  • 4 dimensions? -
    • -
- - -
- -
-

17.1 Vocabulary    notes

-
- -
    -
  • Plane -
    • -
  • Hyperplane -
    • -
- - -
-
- -
- -
-

18 Expressing the Hyperplane    slide animate

-
- -
    -
  • y = m*x + b -
    • -
  • x_2 = m*x_1 + b -
    • -
  • 0 = m*x_1 + b - x_2 -
    • -
  • 0 = [m -1] * [x_1, x_2] + b -
    • -
  • 0 = w * x + b -
    • -
- - -
- -
-

18.1 Questions    notes

-
- -
    -
  • How do you mathematically represent a line? -
    • -
  • Now, we're not going to think of a new letter for every dimension, we're - just going to say x1 , x2 , x3 … -
    • -
  • Rewrite mathematically -
    • -
  • How to add more dimensions? x22? Express x as a vector of all attributes -
    • -
  • Again, don't want to come up with a bunch more letters after m, so use - w as the matrix representing all the m slopes -
    • -
- - -
-
- -
- -
-

19 Challenge    slide two_col

-
- -
    -
  • Find w, b such that w * x + b maximizes the distance between the - support vectors -
    • -
- -

img/svm.png -

-
- -
- -
-

20 Maximizing Fitness Function    slide two_col

-
- -
    -
  • Now we have a fitness function and parameters we're trying to optimize -
    • -
  • Sound familiar? -
    • -
- -

img/svm.png -

-
- -
- -
-

21 Kernel Tricks    slide two_col

-
- -
    -
  • SVM good for linearly separable data -
    • -
  • How to handle other data? -
    • -
- -

img/svm-circular.jpg -

- -
- -
-

21.1 Polynomial Kernel    slide

-
- -
    -
  • Transform it into linearly separable -
    • -
  • What function can we apply to these data points to make them separable? -
    • -
- -

img/svm-circular.jpg -

-
- -
-

21.1.1 Square    notes

-
- -
    -
  • Square all of them -
    • -
- - -
-
- -
- -
-

21.2 Polynomial Kernel    slide

-
- -

img/kernel-trick.jpg -

-

- Now apply SVM -

- -
- -
-

21.3 Details    notes

-
- - - - -
-
- -
- -
-

22 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-03-15 10:19:25 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-01-SVM.org b/slides/2013-03-01-SVM.org deleted file mode 100644 index a420310..0000000 --- a/slides/2013-03-01-SVM.org +++ /dev/null @@ -1,239 +0,0 @@ -* Linear Regression :slide: - -* Types of Models :slide:animate: - + Classifiers - + Regressions - + Clustering - + Outlier -** Details :notes: - + Classifiers :: describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description - + Regressions :: Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms - + Clustering :: find "natural" groups of data *without labels* - + Outlier :: find anomalous transactions, eg. finding fraud for credit cards - -* Case Study :slide: - + Housing prices: square footage - [[file:img/housing-regression.gif]] -** Problem :notes: - + We'd like to know how to price a house based on the square footage - + Let's pretend this is the data we have - + How would we guess that value for 2500 sq ft? - -* Solution? :slide:animate: - + Find a line that represents the data - + =y = m*x + b= - + A line that is not very far from the points -** Prompts :notes: - + In English, how would you solve this? - + How to mathematically represent the line? - + What is a good line? - -* Similarity :slide: - + Main challenges in data mining: defining a specific metric for an intuition - + Define distance for an individual point - + Define how to aggregate distances together -** Challenge :notes: - + This is big problem for engineering and math (stats) in general - + We'll cover some concepts, but if you're ever stuck, try looking in related - fields - + What are some of the ways we can measure distance between points? - Euclidian, Manhattan, Euclidian == L_2 norm - + What is a way to aggrgate numbers? sum, sum of squares, sum of logs - + Differences between the last two? - -** Log & Square :slide:two_col: - + Log :: Useful for de-emphasizing large raw differences - + Square :: Useful for taking the approximate absolute value - [[file:img/logx.gif]] - -* Point Distance :slide:two_col: - + =y= distance from line - + Intuitively: error in estimate - + =h(x) = m*x + b= - + =err = h(x) - y= - [[file:img/error.gif]] -** Error :notes: - + We want the difference from what we estimate to be the value to what the - value actually is - -* Aggregate :slide:animate: - + =sum= - + What about negative error? - + Sum of squares - + =err = sum( (h(x) - y)**2 for x,y in dataset) / len(dataset)= -** Questions :notes: - + Now we have info about all the errors from points, how to summarize? - + Some points have negative error, some positive? Do they cancel each other - out? - + Imagine data set of two points: one solutions covers lines, other divides - them. Which is better? - + Use our squaring trick to make sure we don't have any negative values - + Normalize by the number of points - -* Fitness Function :slide: - + Measures the quality or cost of the solution - + *Key* ingredient for data mining algorithms - + If you can measure it, you can find the best solution -** Fitness :notes: - + Function spits out a metric. Metric can be thought of as *fitness* or - *cost* - + Find the maximum or minimum of that metric - + Depending on your fitness function, this can be easy or difficult - + img: http://onlinestatbook.com - -* Understanding Error :slide: - [[file:img/Linear_regression.svg.png]] - Several possible solutions -** Error :notes: - + What happens to the error as we move line around? - + Decreases until best fit, then increases - + What happens if we plot this error? Say, slope (x) against error (y)? - -* Solution as Minimization :slide:two_col: - + Error is a parabola - + Several methods for finding the minimum - + Two categories: analytical, approximations -[[file:img/parabola.png]] - -* Solution Approximation :slide: - + Some fitness functions can be difficult to solve analytically - + Alternative: iteratively get closer to the solution - + Stop when answer is close enough -** Analytical :notes: - + How to find the minimum of functions in general? - + Take derivative, find 0 - + Taking derivative can be complex or impossible (discontinuities) for some - functions, or solving for 0 is difficult - + Instead, well keep getting closer to the minimum using the function we - already have - -* Gradient Descent :slide:two_col: - 1. Estimate current gradient (derivative) - 1. Take a step (=a * deriv=) in the direction of the gradient - 1. Step size is small, stop. Else repeat. - [[file:img/parabola.png]] -** Steps :notes: - + Take gradient by looking at the local derivative, or perturbating x - + Choose =a= as step size weight: big =a= is large step size - + If =deriv= is large, will also make you step size large. - + If =deriv= is large, probably means you are far away from minimum - + Keep repeating - + What happens if =a= is too small? - + What happens if =a= is too big? - -* General Case :slide:two_col: - + Formulate fitness function for your problem - + Use analytics or approximations to find min/max - + Approximations: Newton's Method, Gradient Descent - [[file:img/error-reduce.png]] -** Approximate visualization :notes: - + Desired output of the error as gradient descent runs - + maybe some local problems, as step size is too big, but slowly move down to - a small amount of error - -* Support Vector Machines :slide: - -* Decision Trees :slide:two_col: - + Great for separable attributes - + Rules operate on independent attributes - + Classes separable along an axis/attribute - [[file:img/tree.png]] - -** Linearly Separable :slide: - + How to handle case where separator line is not along an axis? - [[file:img/dataset_linsep.png]] -** Details :notes: - + Could say if =x>2= and =y>2=, but not a great intuitive fit - + Draw a line that takes both into account - + =y = m*x + b= - + img: http://www.eric-kim.net/eric-kim-net/posts/1/kernel_trick.html - -* Possibilities :slide: - + Many lines *could* separate these classes - [[file:img/dataset_linsep.png]] -** Best? :notes: - + Which is the best? - + Why? - -** Best Separator :slide:two_col: - + Best line gives the most distance between the two classes - + Measure distance between closest points - + Closest points == support vectors - [[file:img/separable.jpg]] -** Points, Vectors :notes: - + Points can be represented as vectors - + Vector math can be easier to express succinctly - + img: http://www.sciencedirect.com/science/article/pii/S1072751511001918 - -* Dimensions :slide: - + When separating two dimensions, we need a line - + When separating 3 dimensions? - + 4 dimensions? -** Vocabulary :notes: - + Plane - + Hyperplane - -* Expressing the Hyperplane :slide:animate: - + =y = m*x + b= - + =x_2 = m*x_1 + b= - + =0 = m*x_1 + b - x_2= - + =0 = [m -1] * [x_1, x_2] + b= - + =0 = w * x + b= -** Questions :notes: - + How do you mathematically represent a line? - + Now, we're not going to think of a new letter for every dimension, we're - just going to say x_1 , x_2 , x_3 ... - + Rewrite mathematically - + How to add more dimensions? x_22? Express x as a vector of all attributes - + Again, don't want to come up with a bunch more letters after =m=, so use - =w= as the matrix representing all the =m= slopes - -* Challenge :slide:two_col: - + Find =w=, =b= such that =w * x + b= maximizes the distance between the - support vectors - [[file:img/svm.png]] - -* Maximizing Fitness Function :slide:two_col: - + Now we have a fitness function and parameters we're trying to optimize - + Sound familiar? - [[file:img/svm.png]] - -* Kernel Tricks :slide:two_col: - + SVM good for linearly separable data - + How to handle other data? - [[file:img/svm-circular.jpg]] - -** Polynomial Kernel :slide: - + Transform it into linearly separable - + What function can we apply to these data points to make them separable? - [[file:img/svm-circular.jpg]] -*** Square :notes: - + Square all of them - -** Polynomial Kernel :slide: - [[file:img/kernel-trick.jpg]] - - Now apply SVM -** Details :notes: - + img: http://www.sciencedirect.com/science/article/pii/S1072751511001918 - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-07-Clustering.html b/slides/2013-03-07-Clustering.html deleted file mode 100644 index a5e776e..0000000 --- a/slides/2013-03-07-Clustering.html +++ /dev/null @@ -1,771 +0,0 @@ - - - - -2013-03-07-Clustering - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-07-Clustering

- - - - -
-

1 Clustering    slide

-
- - -
- -
- -
-

2 Types of Models    slide animate

-
- -
    -
  • Classifiers -
    • -
  • Regressions -
    • -
  • Clustering -
    • -
  • Outlier -
    • -
- - -
- -
-

2.1 Details    notes

-
- -
-
Classifiers
describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description -
-
Regressions
Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms -
-
Clustering
find "natural" groups of data without labels -
-
Outlier
find anomalous transactions, eg. finding fraud for credit cards -
-
- - -
-
- -
- -
-

3 Clustering    slide

-
- -
    -
  • Group together similar items -
    • -
  • Separate dissimilar items -
    • -
  • Automatically discover groups without providing labels -
    • -
- - -
- -
-

3.1 Perspectives    notes

-
- -
    -
  • Similar items: again, metrics of similarity critical in defining these - groups -
    • -
  • Marking boundaries between different classes -
    • -
  • Type of groups unknown before hand. Out of many attributes, what tend to be - shared? -
    • -
- - -
-
- -
- -
-

4 Machine Learning    slide

-
- -
    -
  • Supervised -
    • -
  • Unsupervised -
    • -
  • Semi-supervised -
    • -
  • Active -
    • -
- - -
- -
-

4.1 Definitions    notes

-
- -
-
Supervised
Given data with a label, predict data without a - label -
-
Unsupervised
Given data without labels, group "similar" items - together -
-
Semi-supervised
Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases -
-
Active
Starting with unlabeled data, select the most helpful cases for a - human to label -
-
- - -
-
- -
- -
-

5 Clustering Applications    slide

-
- -
    -
  • Gain insight into how data is distributed -
    • -
  • Preprocessing step to bootstrap labeling -
    • -
  • Discover outliers -
    • -
- - -
- -
-

5.1 Apps    notes

-
- -
    -
  • Closest we have to "magic box": put structured data in, see what groups may - exist -
    • -
  • You want labeled data, but where to start? How many classes? What to name - them? -
      -
    • Cluster data, investigate examples. -
      • -
    • Hand label exemplary cases -
      • -
    • Choose names that distinguish groups -
      • -
    • Run classifier on labeled data, compare with clustering, examine errors, - repeat -
      • -
    - -
    • -
- - -
-
- -
- -
-

6 Yelp Examples    slide

-
- -
    -
  • User groups based on usage, reviewing habits, feature adoption -
    • -
  • Businesses: when should a new category be created, what should it be called? -
    • -
  • Reviews: for a particular business, are there common themes. Show better - variety? -
    • -
- - -
- -
-

6.1 Examples    notes

-
- -
    -
  • User groups may be trend spotters, "lurkers", travelers, early adopters -
    • -
  • Do we need a New American and American category? How similar are these - categories? -
    • -
  • Does a reviewer need to read 10 reviews about great food, so-so service? - Maybe providing different view points helps give a better picture -
    • -
- - -
-
- -
- -
-

7 Intuition    slide

-
- -
    -
  • Intuition => Mathematical Expression => Solution => Evaluation -
    • -
  • High intra-class similarity -
    • -
  • Low inter-class similarity -
    • -
  • Interpretable -
    • -
- - -
- -
-

7.1 Good Clusters    notes

-
- -
    -
  • Just like all data mining, needs to be used to take action -
    • -
  • Can't take action if you don't understand the results -
    • -
  • Trade-offs: testing shows it works, but you don't understand it -
    • -
- - -
-
- -
- -
-

8 Methods    slide

-
- -
-
Partitioning
Construct k groups, evaluate fitness, improve groups -
-
Hierarchical
Agglomerate items into groups, creating "bottom-up" clusters; or divide set into ever smaller groups, creating "top-down" clusters -
-
Density
Find groups by examining continuous density within a potential - group -
-
Grid
Chunk space into units, cluster units instead of individual records -
-
- - -
- -
-

8.1 Algorithms    notes

-
- -
-
Partitioning
Method similar to gradient descent: find some grouping, - evaluate it, improve it somehow, repeat. k-means. -
-
Hierarchical
Build groups 1 "join" at a time, examining distance between - two things that can be joined together, if close, combine groups. Reverse: - divisive. -
-
Density
Many of the above methods just look for distance. This method - tries to find groups that might be strung out, but maintain a density. Think - about an asteroid belt. It is one group, but not clustered together in a way - you typically think. -
-
Grid
Can speed up clustering and provide similar results -
-
- - -
-
- -
- -
-

9 k-means    slide

-
- -
    -
  • Start: Randomly pick k centers for clusters -
    • -
  • Repeat: -
      -
    • Assign all other points to their closest cluster -
      • -
    • Recalculate the center of the cluster -
      • -
    - -
    • -
- - -
- -
-

9.1 Iterative    notes

-
- -
    -
  • Start at a random point, find step in right direction, take step, - re-evaluate -
    • -
- - -
-
- -
- -
-

10 Example    slide

-
- -

img/kmeansclustering.jpg -

-
- -
-

10.1 Process    notes

-
- - - - -
-
- -
- -
-

11 Distance    slide

-
- -
    -
  • Centroid is the average of all points in a cluster; the center -
    • -
  • Different distance metrics for real numbers -
    • -
  • But how to find "average" of binary or normative data? -
    • -
- - -
- -
-

11.1 You Can't    notes

-
- -
    -
  • k-means is used for numerical data -
    • -
- - -
-
- -
- -
-

12 Normalization    slide

-
- -
    -
  • Cluster cities by average temperature and population attributes -
    • -
  • <x,y> = <temp, pop> -
    • -
  • Using Euclidean distance, which attribute will affect similarity more? -
    • -
- - -
- -
-

12.1 Un-normalized    notes

-
- -
    -
  • Population: it is a much bigger number, will contribute much more to - distance -
    • -
  • Artificially inflating importance just because units are different -
    • -
- - -
-
- -
- -
-

13 Normalization Techniques    slide

-
- -
-
Z-score
(v - mean) / stddev -
-
Min-max
(v - min) / (max - min) -
-
Decimal
* 10 / 10 -
-
Square
x**2 -
-
Log
log(x) -
-
- - -
- -
-

13.1 Useful for?    notes

-
- -
-
Z-score
1-pass normalization, retaining information about stdev -
-
Min-max
keep within expected range, usually [0-1] -
-
Decimal
easy to apply -
-
Square
keep inputs positive -
-
Log
de-emphasize differences between large numbers -
-
- - -
-
- -
- -
-

14 Local Optima    slide

-
- -

img/k-means-local.png -

-
- -
-

14.1 No Guarantee    notes

-
- -
    -
  • Since there are many possible stable centers, we may not end up at the best - one -
    • -
  • How can we improve our odds of finding a good separation? -
      -
    • Why did we end up here? starting points -
      • -
    • Choose different starting points -
      • -
    • Compare results -
      • -
    - -
    • -
  • Other problems? Mouse -
    • -
- - -
-
- -
- -
-

15 Uneven Groups    slide

-
- -

img/k-means-mouse.png -

-
- -
-

15.1 k-means    notes

-
- -
    -
  • k-means is good for similarly sized groups, or at least groups that are - similar distance between other members -
    • -
  • Other problems that would pull the centroid away from the real groups? -
    • -
  • Outliers -
    • -
  • img: http://en.wikipedia.org/wiki/K-means_clustering -
    • -
- - -
-
- -
- -
-

16 Medoids    slide

-
- -
    -
  • Instead of finding a centroid find a medoid -
    • -
  • Medoid: actual data point that represents median of the cluster -
    • -
  • PAM: Partitioning Around Medoids -
    • -
- - -
- -
-

16.1 Trade-offs    notes

-
- -
    -
  • PAM more expensive to evaluate -
    • -
  • Scales poorly, since we need to evaluate many more medoids with many more - points -
    • -
- - -
-
- -
- -
-

17 Example    slide

-
- -

img/k-medoids.png -

-
- -
-

17.1 Stability    notes

-
- - - - -
-
- -
- -
-

18 Break    slide

-
- -

img/screenshot_metroid2.jpg -

    -
  • Do not confuse Medoid with Metroid -
    • -
- - -
- -
-

18.1 Note    notes

-
- - - - - - - - - -
-
-
-
- -
-

Date: 2013-03-08 08:57:18 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-07-Clustering.org b/slides/2013-03-07-Clustering.org deleted file mode 100644 index b067f61..0000000 --- a/slides/2013-03-07-Clustering.org +++ /dev/null @@ -1,201 +0,0 @@ -* Clustering :slide: - -* Types of Models :slide:animate: - + Classifiers - + Regressions - + Clustering - + Outlier -** Details :notes: - + Classifiers :: describes and distinguishes cases. Yelp may want to find a - category for a business based on the reviews and business description - + Regressions :: Predict a continuous value. Eg. predict a home's selling - price given sq footage, # of bedrooms - + Clustering :: find "natural" groups of data *without labels* - + Outlier :: find anomalous transactions, eg. finding fraud for credit cards - -* Clustering :slide: - + Group together similar items - + Separate dissimilar items - + Automatically discover groups without providing labels -** Perspectives :notes: - + Similar items: again, metrics of similarity critical in defining these - groups - + Marking boundaries between different classes - + Type of groups unknown before hand. Out of many attributes, what tend to be - shared? - -* Machine Learning :slide: - + Supervised - + Unsupervised - + Semi-supervised - + Active -** Definitions :notes: - + Supervised :: Given data with a label, predict data without a - label - + Unsupervised :: Given data without labels, group "similar" items - together - + Semi-supervised :: Mix of the above: eg. unsupervised to find groups, - supervised to label and distinguish borderline cases - + Active :: Starting with unlabeled data, select the most helpful cases for a - human to label - -* Clustering Applications :slide: - + Gain insight into how data is distributed - + Preprocessing step to bootstrap labeling - + Discover outliers -** Apps :notes: - + Closest we have to "magic box": put structured data in, see what groups may - exist - + You want labeled data, but where to start? How many classes? What to name - them? - + Cluster data, investigate examples. - + Hand label exemplary cases - + Choose names that distinguish groups - + Run classifier on labeled data, compare with clustering, examine errors, - repeat - -* Yelp Examples :slide: - + User groups based on usage, reviewing habits, feature adoption - + Businesses: when should a new category be created, what should it be called? - + Reviews: for a particular business, are there common themes. Show better - variety? -** Examples :notes: - + User groups may be trend spotters, "lurkers", travelers, early adopters - + Do we need a New American and American category? How similar are these - categories? - + Does a reviewer need to read 10 reviews about great food, so-so service? - Maybe providing different view points helps give a better picture - -* Intuition :slide: - + Intuition => Mathematical Expression => Solution => Evaluation - + High intra-class similarity - + Low inter-class similarity - + Interpretable -** Good Clusters :notes: - + Just like all data mining, needs to be used to take action - + Can't take action if you don't understand the results - + Trade-offs: testing shows it works, but you don't understand it - -* Methods :slide: - + Partitioning :: Construct =k= groups, evaluate fitness, improve groups - + Hierarchical :: Agglomerate items into groups, creating "bottom-up" clusters; or divide set into ever smaller groups, creating "top-down" clusters - + Density :: Find groups by examining continuous density within a potential - group - + Grid :: Chunk space into units, cluster units instead of individual records -** Algorithms :notes: - + Partitioning :: Method similar to gradient descent: find some grouping, - evaluate it, improve it somehow, repeat. k-means. - + Hierarchical :: Build groups 1 "join" at a time, examining distance between - two things that can be joined together, if close, combine groups. Reverse: - divisive. - + Density :: Many of the above methods just look for distance. This method - tries to find groups that might be strung out, but maintain a density. Think - about an asteroid belt. It is one group, but not clustered together in a way - you typically think. - + Grid :: Can speed up clustering and provide similar results - -* k-means :slide: - + Start: Randomly pick =k= centers for clusters - + Repeat: - + Assign all other points to their closest cluster - + Recalculate the center of the cluster -** Iterative :notes: - + Start at a random point, find step in right direction, take step, - re-evaluate - -* Example :slide: - [[file:img/kmeansclustering.jpg]] -** Process :notes: - + We pick some nodes at random, mark with a cross - + Find other points that are closest to the crosses - + Find new *centroid* based on the average of all points - + Start again - + img: http://apandre.wordpress.com/visible-data/cluster-analysis/ - -* Distance :slide: - + *Centroid* is the average of all points in a cluster; the center - + Different distance metrics for real numbers - + But how to find "average" of binary or nominal data? -** You Can't :notes: - + k-means is used for numerical data - -* Normalization :slide: - + Cluster cities by average temperature and population attributes - + = - + Using Euclidean distance, which attribute will affect similarity more? -** Un-normalized :notes: - + Population: it is a much bigger number, will contribute much more to - distance - + Artificially inflating importance just because units are different - -* Normalization Techniques :slide: - + Z-score :: =(v - mean) / stddev= - + Min-max :: =(v - min) / (max - min)= - + Decimal :: =* 10= =/ 10= - + Square :: =x**2= - + Log :: =log(x)= -** Useful for? :notes: - + Z-score :: 1-pass normalization, retaining information about stdev - + Min-max :: keep within expected range, usually [0-1] - + Decimal :: easy to apply - + Square :: keep inputs positive - + Log :: de-emphasize differences between large numbers - -* Local Optima :slide: - [[file:img/k-means-local.png]] -** No Guarantee :notes: - + Since there are many possible stable centers, we may not end up at the best - one - + How can we improve our odds of finding a good separation? - + Why did we end up here? starting points - + Choose different starting points - + Compare results - + Other problems? Mouse - -* Uneven Groups :slide: - [[file:img/k-means-mouse.png]] -** k-means :notes: - + k-means is good for similarly sized groups, or at least groups that are - similar distance between other members - + Other problems that would pull the centroid away from the real groups? - + Outliers - + img: http://en.wikipedia.org/wiki/K-means_clustering - -* Medoids :slide: - + Instead of finding a *centroid* find a *medoid* - + Medoid: actual data point that represents median of the cluster - + PAM: Partitioning Around Medoids -** Trade-offs :notes: - + PAM more expensive to evaluate - + Scales poorly, since we need to evaluate many more medoids with many more - points - -* Example :slide: - [[file:img/k-medoids.png]] -** Stability :notes: - + No stability between real clusters - + Outliers can't pull centroid far out of actual cluster - + img: http://en.wikipedia.org/wiki/K-medoids - -* *Break* :slide: -[[file:img/screenshot_metroid2.jpg]] - + Do not confuse Medoid with Metroid -** Note :notes: - + img: http://stealthboy.com/~msherman/metroid.html - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-07-Hierarchical.html b/slides/2013-03-07-Hierarchical.html deleted file mode 100644 index 3c559bd..0000000 --- a/slides/2013-03-07-Hierarchical.html +++ /dev/null @@ -1,590 +0,0 @@ - - - - -2013-03-07-Hierarchical - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-07-Hierarchical

- - - - -
-

1 Hierarchical & Density    slide

-
- - -
- -
- -
-

2 Methods    slide

-
- -
-
Partitioning
Construct k groups, evaluate fitness, improve groups -
-
Hierarchical
Agglomerate items into groups, creating "bottom-up" - clusters; or divide set into ever smaller groups, creating "top-down" - clusters -
-
Density
Find groups by examining continuous density within a potential - group -
-
Grid
Chunk space into units, cluster units instead of individual records -
-
- - -
- -
-

2.1 Algorithms    notes

-
- -
-
Partitioning
k-means, k-medoid -
-
Hierarchical
Build groups 1 "join" at a time, examining distance between - two things that can be joined together, if close, combine groups. Reverse: - divisive. -
-
Density
Many of the above methods just look for distance. This method - tries to find groups that might be strung out, but maintain a density. Think - about an asteroid belt. It is one group, but not clustered together in a way - you typically think. -
-
Grid
Read the book -
-
- - -
-
- -
- -
-

3 k-means Input    slide animate

-
- -
    -
  • What did we supply to k-means before we ran it? -
    • -
  • k: number of clusters -
    • -
  • Clusters disjoint* -
    • -
  • Hierarchical clustering builds up clusters incrementally -
    • -
- - -
- -
-

3.1 Difference    notes

-
- -
    -
  • Hierarchical can find cluster of clusters -
    • -
  • Can illustrate clusters at many levels, let human intemperate what makes - sense without guess-and-check -
    • -
  • Clusters are built 1 cluster at a time, starting with all points being - their own cluster -
    • -
  • *We'll learn about "fuzzy" clustering next time, where cluster membership - is a probability -
    • -
- - -
-
- -
- -
-

4 Agglomerative    slide

-
- -
    -
  • All points are separate clusters -
    • -
  • Find closest clusters: Join them -
    • -
  • Repeat -
    • -
- -

img/agglomerative.png -

-
- -
-

4.1 Bottom-up    notes

-
- -
    -
  • Any questions about this? -
    • -
  • What does "close" mean? -
    • -
- - -
-
- -
- -
-

5 Cluster Distance    slide

-
- -
-
Minimum
Use the two closest points -
-
Maximum
Use the two farthest points -
-
Mean
Use the mean of the two clusters -
-
Average
Sum of the distances of all pairs, divided by number of pairs -
-
- - -
- -
-

5.1 Meta Distance    notes

-
- -
    -
  • These are actually distance metrics for clusters that translate down to - distance metrics for points. -
    • -
  • Still need to decide distance measures for points: Euclidean, Manhattan, - etc. And that's just for numerical distance -
    • -
  • Choose based on expected cluster topology, cross validation testing using - human observers -
    • -
- - -
-
- -
- -
-

6 Termination    slide

-
- -
    -
  • Define have k clusters -
    • -
  • Distance between clusters exceeds threshold -
    • -
  • Fitness function for cluster -
    • -
- - -
- -
-

6.1 Details    notes

-
- -
    -
  • If you wanted to look at all potential k, set k to 1, then look at sub - clusters -
    • -
  • Distance or fitness function (eg. density or minimum intra-cluster - similarity score) can help define k automatically -
    • -
- - -
-
- -
- -
-

7 Dendrogram    slide two_col

-
- -

img/dendrogram1.jpg -

- -
    -
  • Display of clustered groups -
    • -
  • Concise visualization: groups do not need to be identified or named -
    • -
  • Y axis can represent iteration -
    • -
- - -
- -
-

7.1 Usefulness    notes

-
- -
    -
  • Can move up and down clustering to make sense of individual clusters -
    • -
- - -
-
- -
- -
-

8 CHAMELEON    slide two_col

-
- -
    -
  • Discover large number of small clusters -
    • -
  • Group together small clusters -
    • -
  • Join clusters with a high interconnectedness relative to their existing - interconnectedness -
    • -
- -

img/chameleon.png -

-
- -
-

8.1 Details    notes

-
- -
    -
  • Mix of partition & agglomerative -
    • -
  • Partition by finding groups of k-nearest neighbors: A, B in the same group - if A is a k-nearest neighbor of B. -
    • -
  • Interconnectedness measured by aggregate proximity in the group, or using a - network model the book provides details on (10.3.4) -
    • -
- - -
-
- -
- -
-

9 Results    slide

-
- -

img/chameleon-cluster.png -

-
- -
-

9.1 Properties    notes

-
- -
    -
  • Tends to "follow" clusters as long as interconnectedness stays high -
    • -
- - -
-
- -
- -
-

10 Density: DBSCAN    slide two_col

-
- -
    -
  • Find "paths" of points that are in "dense" regions -
    • -
  • Paths: points within a distance e -
    • -
  • Density: surrounded by MinPts within region of radius e -
    • -
- -

img/density-connected.png -

-
- -
-

10.1 Details    notes

-
- -
    -
  • Can find non linear "paths" to follow as long as they stay dense -
    • -
- - -
-
- -
- -
-

11 Density Trade-offs    slide two_col

-
- -

img/DBSCAN.png -

    -
  • Finds clusters of different sizes, shapes -
    • -
  • DBSCAN is sensitive to the parameters used. How big is e? How many - points is "dense"? -
    • -
- - -
- -
-

11.1 Details    notes

-
- - - - -
-
- -
- -
-

12 Algorithm Choice    slide

-
- -
    -
  • Simple techniques often work surprisingly well -
    • -
  • Choose other algorithms to tackle specific problems -
    • -
  • Evaluation metrics -
    • -
- - -
- -
-

12.1 Lessons    notes

-
- -
    -
  • Just like Naive Bayes, we make assumptions about our data that turn out - to be right enough: clusters are uniformly sized, don't wander around our - dimensioned space -
    • -
  • Topic drift: tendency for a cluster to change its properties slowly over - time: eg. articles on politics might use different words -
    • -
  • Performance: many of these algos are computationally expensive, hard to - distribute. Book goes into run times and where to make compromises on the - algo -
    • -
  • Figure out a fitness function for your metric. If you used these clusters - to take action, what would be the result? -
    • -
- - -
-
- -
- -
-

13 Elbow Method    slide two_col

-
- -
    -
  • Calculate intra-cluster variance -
    • -
  • Compare to data set variance (F-test) -
    • -
  • Find point where marginal gain of explicative power decreases -
    • -
- -

img/elbow.JPG -

-
- -
- -
-

14 Labels    slide

-
- -
    -
  • Clustering is an example of unsupervised learning -
    • -
  • But after clustering, humans can label clusters, and their contents -
    • -
  • Now one can use homogeneity metrics to evaluate clusters -
    • -
- - -
- -
-

14.1 Homogeneity    notes

-
- -
    -
  • Gini Index -
    • -
  • Entropy -
    • -
  • Precision / Recall -
    • -
- - -
-
- -
- -
-

15 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-03-08 09:42:20 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-07-Hierarchical.org b/slides/2013-03-07-Hierarchical.org deleted file mode 100644 index b8b004a..0000000 --- a/slides/2013-03-07-Hierarchical.org +++ /dev/null @@ -1,160 +0,0 @@ -* Hierarchical & Density :slide: - -* Methods :slide: - + Partitioning :: Construct =k= groups, evaluate fitness, improve groups - + Hierarchical :: Agglomerate items into groups, creating "bottom-up" - clusters; or divide set into ever smaller groups, creating "top-down" - clusters - + Density :: Find groups by examining continuous density within a potential - group - + Grid :: Chunk space into units, cluster units instead of individual records -** Algorithms :notes: - + Partitioning :: k-means, k-medoid - + Hierarchical :: Build groups 1 "join" at a time, examining distance between - two things that can be joined together, if close, combine groups. Reverse: - divisive. - + Density :: Many of the above methods just look for distance. This method - tries to find groups that might be strung out, but maintain a density. Think - about an asteroid belt. It is one group, but not clustered together in a way - you typically think. - + Grid :: Read the book - -* k-means Input :slide:animate: - + What did we supply to k-means before we ran it? - + =k=: number of clusters - + Clusters disjoint* - + Hierarchical clustering builds up clusters incrementally -** Difference :notes: - + Hierarchical can find cluster of clusters - + Can illustrate clusters at many levels, let human intemperate what makes - sense without guess-and-check - + Clusters are built 1 cluster at a time, starting with all points being - their own cluster - + *We'll learn about "fuzzy" clustering next time, where cluster membership - is a probability - -* Agglomerative :slide: - + All points are separate clusters - + Find closest clusters: Join them - + Repeat - [[file:img/agglomerative.png]] -** Bottom-up :notes: - + Any questions about this? - + What does "close" mean? - -* Cluster Distance :slide: - + Minimum :: Use the two closest points - + Maximum :: Use the two farthest points - + Mean :: Use the mean of the two clusters - + Average :: Sum of the distances of all pairs, divided by number of pairs -** Meta Distance :notes: - + These are actually distance metrics for clusters that translate down to - distance metrics for points. - + Still need to decide distance measures for points: Euclidean, Manhattan, - etc. And that's just for numerical distance - + Choose based on expected cluster topology, cross validation testing using - human observers - -* Termination :slide: - + Define have =k= clusters - + Distance between clusters exceeds threshold - + Fitness function for cluster -** Details :notes: - + If you wanted to look at all potential =k=, set =k= to 1, then look at sub - clusters - + Distance or fitness function (eg. density or minimum intra-cluster - similarity score) can help define =k= automatically - -* Dendrogram :slide:two_col: - [[file:img/dendrogram1.jpg]] - - - + Display of clustered groups - + Concise visualization: groups do not need to be identified or named - + Y axis can represent iteration -** Usefulness :notes: - + Can move up and down clustering to make sense of individual clusters - -* CHAMELEON :slide:two_col: - + Discover large number of small clusters - + Group together small clusters - + Join clusters with a high interconnectedness relative to their existing - interconnectedness - [[file:img/chameleon.png]] -** Details :notes: - + Mix of partition & agglomerative - + Partition by finding groups of k-nearest neighbors: A, B in the same group - if A is a k-nearest neighbor of B. - + Interconnectedness measured by aggregate proximity in the group, or using a - network model the book provides details on (10.3.4) - -* Results :slide: - [[file:img/chameleon-cluster.png]] -** Properties :notes: - + Tends to "follow" clusters as long as interconnectedness stays high - -* Density: DBSCAN :slide:two_col: - + Find "paths" of points that are in "dense" regions - + Paths: points within a distance =e= - + Density: surrounded by =MinPts= within region of radius =e= - [[file:img/density-connected.png]] -** Details :notes: - + Can find non linear "paths" to follow as long as they stay dense - -* Density Trade-offs :slide:two_col: - [[file:img/DBSCAN.png]] - + Finds clusters of different sizes, shapes - + DBSCAN is sensitive to the parameters used. How big is =e=? How many - points is "dense"? -** Details :notes: - + img: http://en.wikipedia.org/wiki/DBSCAN - -* Algorithm Choice :slide: - + Simple techniques often work surprisingly well - + Choose other algorithms to tackle specific problems - + Evaluation metrics -** Lessons :notes: - + Just like Naive Bayes, we make assumptions about our data that turn out - to be right enough: clusters are uniformly sized, don't wander around our - dimensioned space - + Topic drift: tendency for a cluster to change its properties slowly over - time: eg. articles on politics might use different words - + Performance: many of these algos are computationally expensive, hard to - distribute. Book goes into run times and where to make compromises on the - algo - + Figure out a fitness function for your metric. If you used these clusters - to take action, what would be the result? - -* Elbow Method :slide:two_col: - + Calculate intra-cluster variance - + Compare to data set variance (F-test) - + Find point where marginal gain of explicative power decreases - [[file:img/elbow.JPG]] - -* Labels :slide: - + Clustering is an example of unsupervised learning - + But after clustering, humans can label clusters, and their contents - + Now one can use homogeneity metrics to evaluate clusters -** Homogeneity :notes: - + Gini Index - + Entropy - + Precision / Recall - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-07-k-means.html b/slides/2013-03-07-k-means.html deleted file mode 100644 index 59b24b4..0000000 --- a/slides/2013-03-07-k-means.html +++ /dev/null @@ -1,98 +0,0 @@ - - - - -2013-03-07-k-means - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-07-k-means

- - -
-

Table of Contents

-
- -
-
- -
-

1 Implement k-means Clustering    slide

-
- -
    -
  • Very small data set -
    • -
  • Select starting centroids -
    • -
  • Find new centroids -
    • -
  • Stop after centroid moves less than err -
    • -
- - -
- -
- -
-

2 Code    slide

-
- -
    -
  • code/k_means.py -
    • -
  • zip combine lists by alternating members -
    • -
- - - - -
zip([1,2,3], ['one', 'two', 'three'])
-[(1, 'one'), (2, 'two'), (3, 'three')]
-
- - - - - -
-
-
- -
-

Date: 2013-03-08 00:36:16 PST

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-07-k-means.org b/slides/2013-03-07-k-means.org deleted file mode 100644 index ecc8ecb..0000000 --- a/slides/2013-03-07-k-means.org +++ /dev/null @@ -1,30 +0,0 @@ -* Implement k-means Clustering :slide: - + Very small data set - + Select starting centroids - + Find new centroids - + Stop after centroid moves less than =err= - -* Code :slide: - + =code/k_means.py= - + =zip= combine lists by alternating members -#+begin_src python -zip([1,2,3], ['one', 'two', 'three']) -[(1, 'one'), (2, 'two'), (3, 'three')] -#+end_src - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-15-Advanced-Cluster.html b/slides/2013-03-15-Advanced-Cluster.html deleted file mode 100644 index 4a763f8..0000000 --- a/slides/2013-03-15-Advanced-Cluster.html +++ /dev/null @@ -1,716 +0,0 @@ - - - - -2013-03-15-Advanced-Cluster - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-15-Advanced-Cluster

- - - - -
-

1 Advanced Clustering    slide

-
- - -
- -
-

1.1 Questions from last week    notes

-
- -
    -
  • Heuristic for choosing # of clusters? root(n/2) -
    • -
  • k-means: how to calculate the centroid? -
    • -
  • k-medoids: how to calculate the medoid? -
    • -
- - -
-
- -
- -
-

2 Review    slide

-
- -
    -
  • Clustering groups points by using similarity -
    • -
  • Build up, or break down groups -
    • -
  • Each point belongs to 1 cluster -
    • -
- - -
- -
-

2.1 Types    notes

-
- -
    -
  • Agglomerative, Divisive -
    • -
  • Assign each point to 1 centroid (k-means) -
    • -
  • or assign group clusters together, starting with every point as a cluster - (hierarchical) -
    • -
- - -
-
- -
- -
-

3 Topics?    slide

-
- -
    -
  • "This place is great. I've been here for meetings, to get work done, to hang out with friends, and on dates, and it's fit the bill every time. In the summer, there's a wonderful patio, and in the winter it's cozy and warm. All the food I've had is delicious – especially the salad dressing!" -
    • -
- - -
- -
-

3.1 Model by topic    notes

-
- -
    -
  • ambiance, -
    • -
  • romantic -
    • -
  • good for work -
    • -
  • food quality -
    • -
  • Many -
    • -
  • So how do we model? -
    • -
- - -
-
- -
- -
-

4 Fuzzy Clusters    slide

-
- -
    -
  • Membership is a degree in [0-1] -
    • -
  • 1 = sum(membership(v,c) for c in clusters) -
    • -
  • Every point belongs to at least 1 cluster -
    • -
- - -
- -
-

4.1 Restrictions    notes

-
- -
    -
  • Every point still must be in a cluster -
    • -
  • Think of the degree as a probability -
    • -
  • Probabilities must add up to 100% (1) -
    • -
- - -
-
- -
- -
-

5 Generative Model    slide animate

-
- -
    -
  • "Real" model that produced original data points -
    • -
  • Our mission is to reproduce the original model -
    • -
  • Thus we have different techniques that can model different behavior -
    • -
- - -
- -
-

5.1 Questions    notes

-
- -
    -
  • What is a "generative model"? -
    • -
  • What is data mining trying to discover? What is machine learning hoping to - reproduce? -
    • -
  • Why have different classifiers? Decision tree, Naive Bayes, etc? -
    • -
- - -
-
- -
- -
-

6 HW: 1-D Clustering    slide animate

-
- -
    -
  • Imagine plotting the points on the number line -
    • -
  • I generated these points with a process -
    • -
  • Two Gaussian arrays, concatenated -
    • -
- - -
- -
-

6.1 Questions    notes

-
- -
    -
  • Draw number line -
    • -
  • How did I come up with these numbers? I wanted two clusters. -
    • -
  • What parameters did I use, in the code to generate this specific set of N - numbers? -
    • -
- - -
-
- -
- -
-

7 Parameters    slide two_col

-
- -
    -
  • Median -
    • -
  • Standard Deviation -
    • -
  • How many points to generate from each -
    • -
- -

img/gaussian-simple.png -

-
- -
-

7.1 Translation    notes

-
- -
    -
  • I had two distributions (median, stddev) -
    • -
  • Now I picked one or the other with a certain probability -
    • -
  • Then generated a number from it -
    • -
  • In reality, just generated 10 from A, 10 from B, but you can imagine that - being 50% 50% -
    • -
- - -
-
- -
- -
-

8 Generative Model    slide

-
- -

img/gaussian.png -

-
- -
-

8.1 3 Clusters    notes

-
- -
    -
  • We have all three parameters: -
    • -
  • median -
    • -
  • stddev -
    • -
  • probability of choosing distribution (height) -
    • -
- - -
-
- -
- -
-

9 Best Fit?    slide two_col

-
- -

img/gaussian-badfit.png -

-

- img/gaussian-goodfit.png -

-
- -
-

9.1 Choose    notes

-
- -
    -
  • These letters are points on our number line -
    • -
  • Which is more likely to be generated by our real model? -
    • -
  • But we don't know the generative model, so how do we discover it? -
    • -
- - -
-
- -
- -
-

10 Revisit k-means    slide animate

-
- -
    -
  • Each object assigned to closest cluster -
    • -
  • Reset center of the cluster to average -
    • -
  • Repeat until steady -
    • -
- - -
- -
-

10.1 Questions    notes

-
- -
    -
  • What are the steps of k-means? -
    • -
- - -
-
- -
- -
-

11 Expectation-Maximization    slide

-
- -
-
Expectation
Given current state, create a solution that fits our - expectations -
-
Maximization
Adjust the state to maximize the likelihood of the solution - being true -
-
Terminate
When adjustments do not change -
-
- - -
- -
-

11.1 k-means translation    notes

-
- -
    -
  • Our expectation in k-means is that points belong to the cluster closest to - them -
    • -
  • Our state or parameters for our model are the locations of the centers of - those clusters -
    • -
  • The maximization step therefore moves the centers to maximize the - likelihood of their being the true center -
    • -
- - -
-
- -
- -
-

12 Revisit k-means    slide animate

-
- -
    -
  • Each object assigned to closest cluster -
    • -
  • Reset center of the cluster to average -
    • -
  • Repeat until steady -
    • -
- - -
- -
-

12.1 Questions    notes

-
- -
    -
  • What are the steps of k-means? -
    • -
- - -
-
- -
- -
-

13 Fuzzy Clustering    slide

-
- -
    -
  • Each object assigned to closest cluster -
    • -
  • Each object assigned probability of cluster -
    • -
  • Reset center of the cluster to average -
    • -
  • Reset center of the cluster to weighted average -
    • -
  • Repeat until steady -
    • -
- - -
- -
-

13.1 Change    notes

-
- -
    -
  • Only difference here is that we're calculating the probability of a point - belonging to a cluster -
    • -
  • What should we base that probability off of? distance -
    • -
  • If point A has a high probability of belonging to cluster C, what can you - say about A and C? Close -
    • -
- - -
-
- -
- -
-

14 Distance    slide

-
- -
    -
  • dist(o,C) / sum(dist(o,c) for c in clusters)) -
    • -
- - -
- -
-

14.1 Similarity    notes

-
- -
    -
  • Our old friend distance -
    • -
- - -
-
- -
- -
-

15 Distance2    slide

-
- -
    -
  • dist(o,C)**2 / sum(dist(o,c)**2 for c in clusters)) -
    • -
  • Squared distance to primary cluster, divided by squared distance to all - clusters -
    • -
- - -
- -
-

15.1 Similarity    notes

-
- -
    -
  • Our old friend distance -
    • -
  • And squared, to make sure we stay positive -
    • -
- - -
-
- -
- -
-

16 Reset Center    slide

-
- -
    -
  • sum(weight[c][p]**2 * p for p in points) -
    • -
  • Squared weight for this point in this cluster, multiplied by point - coordinates -
    • -
- - -
- -
-

16.1 Weighting by distance    notes

-
- -
    -
  • Distance of point affects how much a cluster center is pulled toward it -
    • -
- - -
-
- -
- -
-

17 Stability    slide

-
- -
    -
  • When our centroids stabilize, we can estimate the parameters of our - distributions -
    • -
  • Or use probabilities of points directly -
    • -
- - -
- -
-

17.1 Uses    notes

-
- -
    -
  • Sometimes you may not need original parameters -
    • -
- - -
-
- -
- -
-

18 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-03-15 13:43:10 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-15-Advanced-Cluster.org b/slides/2013-03-15-Advanced-Cluster.org deleted file mode 100644 index c03d491..0000000 --- a/slides/2013-03-15-Advanced-Cluster.org +++ /dev/null @@ -1,170 +0,0 @@ -* Advanced Clustering :slide: -** Questions from last week :notes: - + Heuristic for choosing # of clusters? root(n/2) - + k-means: how to calculate the centroid? - + k-medoids: how to calculate the medoid? - -* Review :slide: - + Clustering groups points by using similarity - + Build up, or break down groups - + Each point belongs to 1 cluster -** Types :notes: - + Agglomerative, Divisive - + Assign each point to 1 centroid (k-means) - + or assign group clusters together, starting with every point as a cluster - (hierarchical) - -* Topics? :slide: - + "This place is great. I've been here for meetings, to get work done, to hang out with friends, and on dates, and it's fit the bill every time. In the summer, there's a wonderful patio, and in the winter it's cozy and warm. All the food I've had is delicious -- especially the salad dressing!" -** Model by topic :notes: - + ambiance, - + romantic - + good for work - + food quality - + *Many* - + So how do we model? - -* Fuzzy Clusters :slide: - + Membership is a degree in [0-1] - + =1 = sum(membership(v,c) for c in clusters)= - + Every point belongs to at least 1 cluster -** Restrictions :notes: - + Every point still must be in a cluster - + Think of the degree as a probability - + Probabilities must add up to 100% (1) - -* Generative Model :slide:animate: - + "Real" model that produced original data points - + Our mission is to reproduce the original model - + Thus we have different techniques that can model different behavior -** Questions :notes: - + What is a "generative model"? - + What is data mining trying to discover? What is machine learning hoping to - reproduce? - + Why have different classifiers? Decision tree, Naive Bayes, etc? - -* HW: 1-D Clustering :slide:animate: - + Imagine plotting the points on the number line - + I generated these points with a process - + Two Gaussian arrays, concatenated -** Questions :notes: - + Draw number line - + How did I come up with these numbers? I wanted two clusters. - + What parameters did I use, in the code to generate this specific set of N - numbers? - -* Parameters :slide:two_col: - + Median - + Standard Deviation - + How many points to generate from each - [[file:img/gaussian-simple.png]] -** Translation :notes: - + I had two distributions (median, stddev) - + Now I picked one or the other with a certain probability - + Then generated a number from it - + In reality, just generated 10 from A, 10 from B, but you can imagine that - being 50% 50% - -* Generative Model :slide: - [[file:img/gaussian.png]] -** 3 Clusters :notes: - + We have all three parameters: - + median - + stddev - + probability of choosing distribution (height) - -* Best Fit? :slide:two_col: - [[file:img/gaussian-badfit.png]] - - [[file:img/gaussian-goodfit.png]] -** Choose :notes: - + These letters are points on our number line - + Which is more likely to be generated by our real model? - + But we don't know the generative model, so how do we discover it? - -* Revisit k-means :slide:animate: - + Each object assigned to closest cluster - + Reset center of the cluster to average - + Repeat until steady -** Questions :notes: - + What are the steps of k-means? - -* Expectation-Maximization :slide: - + Expectation :: Given current state, create a solution that fits our - expectations - + Maximization :: Adjust the state to maximize the likelihood of the solution - being true - + Terminate :: When adjustments do not change -** k-means translation :notes: - + Our expectation in k-means is that points belong to the cluster closest to - them - + Our state or parameters for our model are the locations of the centers of - those clusters - + The maximization step therefore moves the centers to maximize the - likelihood of their being the true center - -* Revisit k-means :slide:animate: - + Each object assigned to closest cluster - + Reset center of the cluster to average - + Repeat until steady -** Questions :notes: - + What are the steps of k-means? - -* Fuzzy Clustering :slide: - + +Each object assigned to closest cluster+ - + Each object assigned *probability* of cluster - + +Reset center of the cluster to average+ - + Reset center of the cluster to *weighted* average - + Repeat until steady -** Change :notes: - + Only difference here is that we're calculating the probability of a point - belonging to a cluster - + What should we base that probability off of? distance - + If point A has a high probability of belonging to cluster C, what can you - say about A and C? Close - -* Distance :slide: - + =dist(o,C) / sum(dist(o,c) for c in clusters))= -** Similarity :notes: - + Our old friend distance - -* Distance^2 :slide: - + =dist(o,C)**2 / sum(dist(o,c)**2 for c in clusters))= - + Squared distance to primary cluster, divided by squared distance to all - clusters -** Similarity :notes: - + Our old friend distance - + And squared, to make sure we stay positive - -* Reset Center :slide: - + =sum(weight[c][p]**2 * p for p in points)= - + Squared weight for this point in this cluster, multiplied by point - coordinates -** Weighting by distance :notes: - + Distance of point affects how much a cluster center is pulled toward it - -* Stability :slide: - + When our centroids stabilize, we can estimate the parameters of our - distributions - + Or use probabilities of points directly -** Uses :notes: - + Sometimes you may not need original parameters - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-03-15-Review.html b/slides/2013-03-15-Review.html deleted file mode 100644 index 536c96a..0000000 --- a/slides/2013-03-15-Review.html +++ /dev/null @@ -1,386 +0,0 @@ - - - - -2013-03-14-Review - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-03-14-Review

- - - - -
-

1 Review    slide

-
- -
    -
  • Midterm target length: 1.5 hours -
    • -
  • Time limit: 3 hours -
    • -
  • 1 cheat sheet, 8.5x11 -
    • -
  • Calculators OK, not required -
    • -
  • Questions from slides have a higher probability of appearing -
    • -
  • Questions from the reading are fair game -
    • -
- - -
- -
- -
-

2 Case Studies    slide

-
- -
    -
  • Example of "transactional data" -
    • -
  • Example of non-transactional data -
    • -
- - -
- -
- -
-

3 Obtaining Data    slide

-
- -
    -
  • Tradeoffs of dataset vs API? -
    • -
  • Tradeoffs of operational database vs data warehouse -
    • -
  • Unix commands to explore data? -
    • -
- - -
- -
- -
-

4 Probability    slide

-
- -
    -
  • Other names for a Feature -
    • -
  • Difference between Discrete and Continuous feature -
    • -
  • What type of feature is day of the week? -
    • -
  • Ways to measure central tendency? -
    • -
  • What is skew? -
    • -
  • When is asymmetric binary dissimilarity useful? -
    • -
  • How to calculate L2 norm of two points? -
    • -
  • What is cosign similarity? -
    • -
- - -
- -
- -
-

5 Preprocessing    slide

-
- -
    -
  • When storing the same fact in different ways, what type of problem is - likely? -
    • -
  • Can data corruption happen with no mistakes and no bugs? -
    • -
  • What are some options to deal with missing values? -
    • -
  • What are some options to deal with outliers? -
    • -
  • What does correlation imply? -
    • -
- - -
- -
- -
-

6 Data Warehouse    slide

-
- -
    -
  • OLAP vs OLTP -
    • -
  • Examples of database metadata? -
    • -
  • What is a data multi-cube? -
    • -
  • What is at the center of a star schema? -
    • -
  • What is the tradeoff being made in dimension tables? -
    • -
  • Define: -
      -
    • Rollup -
      • -
    • Drill-down -
      • -
    • Slice -
      • -
    • Dice -
      • -
    • Pivot -
      • -
    - -
    • -
- - -
- -
- -
-

7 MapReduce    slide

-
- -
    -
  • What tradeoff are we making with MapReduce? -
    • -
  • Why is log processing a typical use of MapReduce? -
    • -
  • What types of processing is not well suited? -
    • -
  • For a multi-step job, the output of a reducer is fed into what? -
    • -
- - -
- -
- -
-

8 Decision Tree    slide

-
- -
    -
  • What table can we create from the verification results to understand - performance of our model? -
    • -
  • For supervised learning, what is required to train a model? -
    • -
  • What is a naive way to optimize precision? -
    • -
  • Recall? -
    • -
  • Assuming we use all attributes to classify, what is the height of - our tree? -
    • -
  • What are we optimizing for in the leaf nodes? -
    • -
- - -
- -
- -
-

9 Naive Bays    slide

-
- -
    -
  • Where does the testing set come from? -
    • -
  • What is the k in k-fold cross-validation? -
    • -
  • Bayes theorem finds P(A|B). In email spam detection, what are A and B? -
    • -
  • What is the Naive assumption we make in Naive Bayes? -
    • -
  • Why can training many models be useful? -
    • -
  • What is bootstrap sampling? -
    • -
  • What is a random forest? -
    • -
- - -
- -
- -
-

10 SVM    slide

-
- -
    -
  • When finding a linear fit for home prices, what is our fitness function? -
    • -
  • What is the gradient in gradient descent? -
    • -
  • In the general case, are you guaranteed to find the globally optimal - solution when using gradient descent? -
    • -
  • Why does SVM work so well in practice, even though it requires linear - separability? -
    • -
  • If your data is not linearly separable, can you use SVM? -
    • -
- - -
- -
- -
-

11 Neural Networks    slide

-
- -
    -
  • What is model variance? -
    • -
  • What problem does high model variance indicate? -
    • -
  • What is an activation function? -
    • -
  • What types of problems are neural networks especially suited for? -
    • -
  • What are we improve during backward propagation? -
    • -
- - -
- -
- -
-

12 Partitioning Clusters    slide

-
- -
    -
  • What is the difference between k-means and k-nearest-neighbor? -
    • -
  • What are some of the problems with k-means? -
    • -
  • Why is normalization especially useful in clustering? -
    • -
  • What are the tradeoffs for using k-medoid clustering? -
    • -
- - -
- -
- -
-

13 Hierarchical Clustering    slide

-
- -
    -
  • What are the options to calculate cluster distance? -
    • -
  • Describe how to draw a dendrogram -
    • -
  • What are the drawbacks to density clustering with DBSCAN? -
    • -
  • If you had movie description data, but no genres, would you use Fuzzy - Clustering or Partitioned Clustering? -
    • -
  • How can we evaluate a clustering algorithm if our data is already labeled - with clusters? -
    • -
- - -
- -
- -
-

14 Good Luck!    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-03-15 13:36:52 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-03-15-Review.org b/slides/2013-03-15-Review.org deleted file mode 100644 index 2e497d5..0000000 --- a/slides/2013-03-15-Review.org +++ /dev/null @@ -1,122 +0,0 @@ -* Review :slide: - + Midterm target length: 1.5 hours - + Time limit: 3 hours - + 1 cheat sheet, 8.5x11 - + Calculators OK, not required - + Questions from slides have a higher probability of appearing - + Questions from the reading are fair game - -* Case Studies :slide: - + Example of "transactional data" - + Example of non-transactional data - -* Obtaining Data :slide: - + Tradeoffs of dataset vs API? - + Tradeoffs of operational database vs data warehouse - + Unix commands to explore data? - -* Probability :slide: - + Other names for a Feature - + Difference between Discrete and Continuous feature - + What type of feature is day of the week? - + Ways to measure central tendency? - + What is skew? - + When is asymmetric binary dissimilarity useful? - + How to calculate L_2 norm of two points? - + What is cosign similarity? - -* Preprocessing :slide: - + When storing the same fact in different ways, what type of problem is - likely? - + Can data corruption happen with no mistakes and no bugs? - + What are some options to deal with missing values? - + What are some options to deal with outliers? - + What does correlation imply? - -* Data Warehouse :slide: - + OLAP vs OLTP - + Examples of database metadata? - + What is a data multi-cube? - + What is at the center of a star schema? - + What is the tradeoff being made in dimension tables? - + Define: - + Rollup - + Drill-down - + Slice - + Dice - + Pivot - -* MapReduce :slide: - + What tradeoff are we making with MapReduce? - + Why is log processing a typical use of MapReduce? - + What types of processing is not well suited? - + For a multi-step job, the output of a reducer is fed into what? - -* Decision Tree :slide: - + What table can we create from the verification results to understand - performance of our model? - + For supervised learning, what is required to train a model? - + What is a naive way to optimize precision? - + Recall? - + Assuming we use all attributes to classify, what is the height of - our tree? - + What are we optimizing for in the leaf nodes? - -* Naive Bays :slide: - + Where does the testing set come from? - + What is the =k= in k-fold cross-validation? - + Bayes theorem finds P(A|B). In email spam detection, what are A and B? - + What is the Naive assumption we make in Naive Bayes? - + Why can training many models be useful? - + What is bootstrap sampling? - + What is a random forest? - -* SVM :slide: - + When finding a linear fit for home prices, what is our fitness function? - + What is the gradient in gradient descent? - + In the general case, are you guaranteed to find the globally optimal - solution when using gradient descent? - + Why does SVM work so well in practice, even though it requires linear - separability? - + If your data is not linearly separable, can you use SVM? - -* Neural Networks :slide: - + What is model variance? - + What problem does high model variance indicate? - + What is an activation function? - + What types of problems are neural networks especially suited for? - + What are we improve during backward propagation? - -* Partitioning Clusters :slide: - + What is the difference between k-means and k-nearest-neighbor? - + What are some of the problems with k-means? - + Why is normalization especially useful in clustering? - + What are the tradeoffs for using k-medoid clustering? - -* Hierarchical Clustering :slide: - + What are the options to calculate cluster distance? - + Describe how to draw a dendrogram - + What are the drawbacks to density clustering with DBSCAN? - + If you had movie description data, but no genres, would you use Fuzzy - Clustering or Partitioned Clustering? - + How can we evaluate a clustering algorithm if our data is already labeled - with clusters? - -* *Good Luck!* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-05-AWS.html b/slides/2013-04-05-AWS.html deleted file mode 100644 index ee42c6f..0000000 --- a/slides/2013-04-05-AWS.html +++ /dev/null @@ -1,286 +0,0 @@ - - - - -2013-04-05-AWS - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-05-AWS

- - - - -
-

1 Amazon Web Services    slide

-
- - -
- -
- -
-

2 Grant    slide

-
- -
    -
  • $3900 for this class -
    • -
  • Store data in S3 -
    • -
  • Use mrjob on EMR -
    • -
- - -
- -
-

2.1 No crazy    notes

-
- -
    -
  • Overcharges go on my credit card! -
    • -
  • Ask me before doing anything not discussed here -
    • -
- - -
-
- -
- -
-

3 S3    slide

-
- -
    -
  • S3: Simple Storage Service -
    • -
  • Can store and share large data files -
    • -
  • Free bandwidth when processing with EMR -
    • -
- - -
- -
- -
-

4 s3cmd    slide

-
- -
    -
  • Upload / Download files -
    • -
  • s3cmd --configure -
    • -
  • s3cmd mb s3://i290-group-name -
    • -
  • s3cmd put localfile s3://i290-group-name/data/ -
    • -
- - -
- -
-

4.1 Configuration    notes

-
- -
    -
  • check ~jblomo/mrjob.conf -
    • -
- - -
-
- -
- -
-

5 Elastic MapReduce    slide

-
- -
    -
  • python job.py -r emr -c ~jblomo/mrjob.conf s3://i290-group-name/data/file -
    • -
  • -r emr : run on EMR instead of localhost -
    • -
  • -c ~jblomo/mrjob.conf : use a configuration file -
      -
    • 5 machines, can change with command line options -
      • -
    - -
    • -
- - -
- -
- -
-

6 Copying Keys    slide

-
- -
    -
  • SSH keys are used to connect to server to check status -
    • -
  • Copy my keys, set correct permissions -
    • -
- - - - -
~$ cp ~jblomo/mrjob-common.conf ~/mrjob.conf
-~$ chmod 0600 ~/mrjob.conf
-
- - -
- -
- -
-

7 All Together Now    slide

-
- - - - -
~$ s3cmd put yelp_academic_dataset.json.gz s3://i290-jblomo/data/
-# yelp_academic_dataset.json.gz -> s3://i290-jblomo/data/yelp_academic_dataset.json.gz  [1 of 1]
-# 127506871 of 127506871   100% in    8s    13.60 MB/s  done
-
-~$ cd datamining290/code/
-~/datamining290/code$ python unique_review.py -v -r emr -c ~jblomo/mrjob.conf --output-dir s3://i290-jblomo/output/unique_review/ --no-output s3://i290-jblomo/data/yelp_academic_dataset.json.gz
-# ...
-# Creating Elastic MapReduce job flow
-# ...
-# Job flow created with ID: j-EFG48CIR1APW
-# ...
-# Job launched 60.8s ago, status STARTING: Provisioning Amazon EC2 capacity
-# ...
-# Job launched 334.4s ago, status RUNNING: Running step (unique_review.jblomo.20130406.171258.267523: Step 1 of 3)
-# ...
-# map 73% reduce  42%
-# ...
-# Counters from step 1:
-# ...
-
-~/datamining290/code$ s3cmd ls -r s3://i290-jblomo/output/
-# ...
-# 2013-04-06 18:05        29   s3://i290-jblomo/output/unique_review/part-00005
-# ...
-
- - -
- -
-

7.1 Trade-offs    notes

-
- -
    -
  • Jobs may take 5 minutes to spin up -
    • -
  • Errors are harder to debug because they are mixed in with Hadoop and EMR - errors -
    • -
- - -
-
- -
- -
-

8 Extra Notes    slide

-
- -
    -
  • Cannot overwrite output directory: choose a new one for each run -
    • -
  • Errors may be hard to debug. Run locally with a sample of your data -
    • -
  • Output from job will be split into files, recall the Hadoop video lecture -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-04-06 11:19:43 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-05-AWS.org b/slides/2013-04-05-AWS.org deleted file mode 100644 index fc18dde..0000000 --- a/slides/2013-04-05-AWS.org +++ /dev/null @@ -1,90 +0,0 @@ -* Amazon Web Services :slide: - -* Grant :slide: - + $3900 for this class - + Store data in S3 - + Use mrjob on EMR -** No crazy :notes: - + Overcharges go on my credit card! - + Ask me before doing anything not discussed here - -* S3 :slide: - + [[http://aws.amazon.com/s3/][S3]]: Simple Storage Service - + Can store and share large data files - + Free bandwidth when processing with EMR - -* =s3cmd= :slide: - + Upload / Download files - + =s3cmd --configure= - + =s3cmd mb s3://i290-group-name= - + =s3cmd put localfile s3://i290-group-name/data/= -** Configuration :notes: - + check =~jblomo/mrjob.conf= - -* Elastic MapReduce :slide: - + =python job.py -r emr -c ~jblomo/mrjob.conf s3://i290-group-name/data/file= - + =-r emr= : run on EMR instead of localhost - + =-c ~jblomo/mrjob.conf= : use a configuration file - + 5 machines, can change with command line options - -* Copying Keys :slide: - + SSH keys are used to connect to server to check status - + Copy my keys, set correct permissions -#+begin_src bash -~$ cp ~jblomo/mrjob-common.conf ~/mrjob.conf -~$ chmod 0600 ~/mrjob.conf -#+end_src - -* All Together Now :slide: -#+begin_src bash -~$ s3cmd put yelp_academic_dataset.json.gz s3://i290-jblomo/data/ -# yelp_academic_dataset.json.gz -> s3://i290-jblomo/data/yelp_academic_dataset.json.gz [1 of 1] -# 127506871 of 127506871 100% in 8s 13.60 MB/s done - -~$ cd datamining290/code/ -~/datamining290/code$ python unique_review.py -v -r emr -c ~jblomo/mrjob.conf --output-dir s3://i290-jblomo/output/unique_review/ --no-output s3://i290-jblomo/data/yelp_academic_dataset.json.gz -# ... -# Creating Elastic MapReduce job flow -# ... -# Job flow created with ID: j-EFG48CIR1APW -# ... -# Job launched 60.8s ago, status STARTING: Provisioning Amazon EC2 capacity -# ... -# Job launched 334.4s ago, status RUNNING: Running step (unique_review.jblomo.20130406.171258.267523: Step 1 of 3) -# ... -# map 73% reduce 42% -# ... -# Counters from step 1: -# ... - -~/datamining290/code$ s3cmd ls -r s3://i290-jblomo/output/ -# ... -# 2013-04-06 18:05 29 s3://i290-jblomo/output/unique_review/part-00005 -# ... -#+end_src -** Trade-offs :notes: - + Jobs may take 5 minutes to spin up - + Errors are harder to debug because they are mixed in with Hadoop and EMR - errors - -* Extra Notes :slide: - + Cannot overwrite output directory: choose a new one for each run - + Errors may be hard to debug. Run locally with a sample of your data - + Output from job will be split into files, recall the Hadoop video lecture - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-05-Frequent-Pattern.html b/slides/2013-04-05-Frequent-Pattern.html deleted file mode 100644 index cea6067..0000000 --- a/slides/2013-04-05-Frequent-Pattern.html +++ /dev/null @@ -1,480 +0,0 @@ - - - - -2013-04-05-Frequent-Pattern - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-05-Frequent-Pattern

- - - - -
-

1 Frequent Patterns    slide

-
- - -
- -
- -
-

2 Finding Patterns    slide

-
- -
    -
  • Cookies frequently purchased with milk -
    • -
  • Website signups frequently occurring after reading FAQ -
    • -
  • DNA sections frequently seen with a drug reaction -
    • -
- - -
- -
-

2.1 Patterns    notes

-
- -
    -
  • set of items -
    • -
  • subsequences of actions -
    • -
  • substructures -
    • -
  • Generalized to any kind of pattern that occurs "frequently" in the dataset -
    • -
- - -
-
- -
- -
-

3 Market Basket    slide

-
- -
    -
  • What things are frequently purchased together? -
    • -
  • Apocryphal example: beer and diapers -
    • -
  • Can be used for any natural grouping -
    • -
- - -
- -
-

3.1 Details    notes

-
- -
    -
  • Example of how patterns are discovered is to look at groups of actions -
    • -
  • One natural group is the shopping basket: what items are in it? -
    • -
  • But can also be applied to anytime there is a natural grouping -
      -
    • Eg. web session logs group naturally around a person and time window -
      • -
    - -
    • -
- - -
-
- -
- -
-

4 Define "Frequently"    slide

-
- -
-
Action
A and B -
-
Support
probability that a transaction contains A ∪ B -
-
Confidence
conditional probability that a transaction having A also - contains B -
-
- - -
- -
-

4.1 Probabilities    notes

-
- -
    -
  • We have two actions A and B -
    • -
  • Out of all the groupings, how many had both items? -
    • -
  • Out of all the groupings with A, how many had B? -
    • -
- - -
-
- -
- -
-

5 Minimums    slide two_col

-
- -
-
Min Support
lower bound on support probability -
-
Min Confidence
lower bound on confidence probability -
-
Strong
Rule that satisfies both minimums -
-
- -

img/strawberry-milk.jpg -

-
- -
-

5.1 "Frequently"    notes

-
- -
    -
  • Now we can talk about what frequently means -
    • -
  • It doesn't matter if two very unpopular items were purchased together: car - battery and smoke detector -
    • -
  • Also don't care if A happens a lot: everybody buys milk, so not a big - deal if some bought milk and strawberries -
    • -
  • Also important to note confidence is not symmetric: buying strawberries may be - frequent with buying milk, but not visa versa -
    • -
- - -
-
- -
- -
-

6 Too Many Rules    slide

-
- -
    -
  • Patterns not limited to 2 events -
    • -
  • But looking for all patterns leads to combinatorial number of options -
    • -
- - - -- - - - - - - - - - -
a,b,c,d,e
a,b
a,c
…
a,b,c
a,b,e
…
- - -
- -
- -
-

7 Subset Patterns    slide

-
- -
-
Max-Pattern
X rule is frequent and there exists no frequent - super-pattern Y -
-
Closed
X rule is frequent and there exists no super-pattern Y with the same support -
-
Shortcut
Find only max-pattern or closed patterns, let other patterns be - subsets -
-
- - -
- -
-

7.1 Shortcut    notes

-
- -
    -
  • So how can we calculate all the potentially frequently occurring patterns? -
    • -
  • We can find either the max or closed pattern that encompasses all of the - patterns we're looking for -
    • -
  • These are more easily tracked, and we can still derive all of the - frequently occurring sub-patterns -
    • -
  • We can use the reverse: if a rule or item is not frequent enough alone, its - super-set will not be frequent enough: -
      -
    • If A is does not meet min support, there's no way for A,B to make - support -
      • -
    - -
    • -
- - -
-
- -
- -
-

8 Apriori    slide

-
- -
    -
  1. Find supported single event rules -
    2. -
  2. Combine to make 2-event rules, check DB for support -
    3. -
  3. Combine to make 3-event rules, check DB… -
    4. -
  4. Stop when no N-event rules -
    5. -
- - -
- -
- -
-

9    slide

-
- -

img/apriori.png -

-
- -
-

9.1 Speed    notes

-
- -
    -
  • Isn't that slow? Yes! -
    • -
  • Book has some techniques to speed it up, mostly around grouping -
    • -
  • Can group together sets and if the group does not meet the support - threshold, then none of the members do -
    • -
- - -
-
- -
- -
-

10 Interesting Patterns    slide two_col

-
- -
    -
  • Strong rules may not always be interesting rules -
    • -
  • Basketball => eat cereal [40%, 66.7%] is strong -
    • -
  • But "not cereal" has a bigger effect on if you play basketball -
    • -
- - - -- - - - - - - -
BasketballNot basketballSum
Cereal200017503750
Not cereal10002501250
Sum300020005000
- - -
- -
-

10.1 Details    notes

-
- -
    -
  • Not cereal column: has a huge effect on if someone plays basketball -
    • -
  • cereal + basketball… sure it happens frequently, but you'd actually - expect to see a bigger effect -
    • -
- - -
-
- -
- -
-

11 Lift    slide

-
- -
    -
  • P(A ∪ B) / P(A)*P(B) -
    • -
  • If A and B independent, what is likelihood of A and B? -
    • -
- - -
- -
-

11.1 Correlation    notes

-
- -
    -
  • 1 -
    • -
  • so if lift > 1, you're seeing something that is happening more often than - random -
    • -
  • < 1 means they negatively correlated -
    • -
  • X2, cosine, others in book -
    • -
- - -
-
- -
- -
-

12 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-04-05 00:56:49 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-05-Frequent-Pattern.org b/slides/2013-04-05-Frequent-Pattern.org deleted file mode 100644 index 1e97d05..0000000 --- a/slides/2013-04-05-Frequent-Pattern.org +++ /dev/null @@ -1,129 +0,0 @@ -* Frequent Patterns :slide: - -* Finding Patterns :slide: - + Cookies frequently purchased with milk - + Website signups frequently occurring after reading FAQ - + DNA sections frequently seen with a drug reaction -** Patterns :notes: - + set of items - + subsequences of actions - + substructures - + Generalized to any kind of pattern that occurs "frequently" in the dataset - -* Market Basket :slide: - + What things are frequently purchased together? - + Apocryphal example: beer and diapers - + Can be used for any natural grouping -** Details :notes: - + Example of how patterns are discovered is to look at groups of actions - + One natural group is the shopping basket: what items are in it? - + But can also be applied to anytime there is a natural grouping - + Eg. web session logs group naturally around a person and time window - -* Define "Frequently" :slide: - + Action :: =A= and =B= - + Support :: probability that a transaction contains =A ∪ B= - + Confidence :: conditional probability that a transaction having =A= also - contains =B= -** Probabilities :notes: - + We have two actions =A= and =B= - + Out of all the groupings, how many had both items? - + Out of all the groupings with =A=, how many had =B=? - -* Minimums :slide:two_col: - + Min Support :: lower bound on support probability - + Min Confidence :: lower bound on confidence probability - + Strong :: Rule that satisfies both minimums - [[file:img/strawberry-milk.jpg]] -** "Frequently" :notes: - + Now we can talk about what frequently means - + It doesn't matter if two very unpopular items were purchased together: car - battery and smoke detector - + Also don't care if =A= happens a lot: everybody buys milk, so not a big - deal if some bought milk and strawberries - + Also important to note confidence is not symmetric: buying strawberries may be - frequent with buying milk, but not visa versa - -* Too Many Rules :slide: - + Patterns not limited to 2 events - + But looking for all patterns leads to combinatorial number of options - | a,b,c,d,e | - | a,b | - | a,c | - | ... | - | a,b,c | - | a,b,e | - |...| - -* Subset Patterns :slide: - + Max-Pattern :: =X= rule is frequent and there exists no frequent - super-pattern =Y= - + Closed :: =X= rule is frequent and there exists no super-pattern =Y= *with the same support* - + Shortcut :: Find only max-pattern or closed patterns, let other patterns be - subsets -** Shortcut :notes: - + So how can we calculate all the potentially frequently occurring patterns? - + We can find either the max or closed pattern that encompasses all of the - patterns we're looking for - + These are more easily tracked, and we can still derive all of the - frequently occurring sub-patterns - + We can use the reverse: if a rule or item is not frequent enough alone, its - super-set will not be frequent enough: - + If =A= is does not meet min support, there's no way for =A,B= to make - support - -* Apriori :slide: - 1. Find supported single event rules - 1. Combine to make 2-event rules, check DB for support - 1. Combine to make 3-event rules, check DB... - 1. Stop when no N-event rules - -* :slide: - [[file:img/apriori.png]] -** Speed :notes: - + Isn't that slow? Yes! - + Book has some techniques to speed it up, mostly around grouping - + Can group together sets and if the group does not meet the support - threshold, then none of the members do - -* Interesting Patterns :slide:two_col: - + Strong rules may not always be interesting rules - + Basketball => eat cereal [40%, 66.7%] is strong - + But "not cereal" has a bigger effect on if you play basketball - | | Basketball | Not basketball | Sum | - | Cereal | 2000 | 1750 | 3750 | - | Not cereal | 1000 | 250 | 1250 | - | Sum | 3000 | 2000 | 5000 | -** Details :notes: - + Not cereal column: has a huge effect on if someone plays basketball - + cereal + basketball... sure it happens frequently, but you'd actually - expect to see a bigger effect - -* Lift :slide: - + =P(A ∪ B) / P(A)*P(B)= - + If =A= and =B= independent, what is likelihood of =A= and =B=? -** Correlation :notes: - + 1 - + so if lift > 1, you're seeing something that is happening more often than - random - + < 1 means they negatively correlated - + X^2, cosine, others in book - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-12-AdjacencyRepresentations.html b/slides/2013-04-12-AdjacencyRepresentations.html deleted file mode 100644 index 88a484c..0000000 --- a/slides/2013-04-12-AdjacencyRepresentations.html +++ /dev/null @@ -1,171 +0,0 @@ - - - - -2013-04-12-AdjacencyRepresentations - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-12-AdjacencyRepresentations

- - -
-

Table of Contents

-
- -
-
- -
-

1 Homework    slide

-
- -
    -
  • Represent graphs as adjacency matrix, adjacency list -
    • -
  • Work on your projects -
    • -
  • Review your midterm -
    • -
- - -
- -
- -
-

2 Graphs    slide

-
- -
    -
  1. img/Directed_acyclic_graph.png -
    2. -
  2. img/6n-graf.svg.png -
    3. -
- - -
- -
- -
-

3 Output    slide

-
- -
    -
  • File in Github pull request -
    • -
  • Represent Matrix and list in some sort of organized way -
    • -
- - - - -
[[0 1 1 1]
- [1 0 0 1]
- ...]
-
-{1: [2,3,4],
- 2: [1, 4]}
-
- - - -
0,1,1,1
-1,0,0,1
-...
-
-1,2,3,4
-2,1,4
-
- - -
- -
- -
-

4 NetworkX    slide

-
- -
    -
  • Python library for manipulating graphs -
    • -
  • Potentially useful for your projects -
    • -
  • Not homework -
    • -
- - -
- -
- -
-

5 VirtualEnv    slide

-
- -
    -
  • Install and manage libraries -
    • -
  • activate each time you start a new session -
    • -
- - - - -
$ virtualenv venv
-$ source venv/bin/activate
-$ pip install <python package>
-
- - - - - -
-
-
- -
-

Date: 2013-04-12 13:38:59 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-12-AdjacencyRepresentations.org b/slides/2013-04-12-AdjacencyRepresentations.org deleted file mode 100644 index 5ddd68b..0000000 --- a/slides/2013-04-12-AdjacencyRepresentations.org +++ /dev/null @@ -1,60 +0,0 @@ -* Homework :slide: - + Represent graphs as adjacency matrix, adjacency list - + Work on your projects - + Review your midterm - -* Graphs :slide: - 1. [[file:img/Directed_acyclic_graph.png]] - 2. [[file:img/6n-graf.svg.png]] - -* Output :slide: - + File in Github pull request - + Represent Matrix and list in some sort of organized way -#+begin_src python -[[0 1 1 1] - [1 0 0 1] - ...] - -{1: [2,3,4], - 2: [1, 4]} -#+end_src - -#+begin_src csv -0,1,1,1 -1,0,0,1 -... - -1,2,3,4 -2,1,4 -#+end_src - -* [[http://networkx.github.io/][NetworkX]] :slide: - + Python library for manipulating graphs - + Potentially useful for your projects - + Not homework - -* [[https://pypi.python.org/pypi/virtualenv][VirtualEnv]] :slide: - + Install and manage libraries - + =activate= each time you start a new session -#+begin_src bash -$ virtualenv venv -$ source venv/bin/activate -$ pip install -#+end_src - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-12-Graphs.html b/slides/2013-04-12-Graphs.html deleted file mode 100644 index e068f86..0000000 --- a/slides/2013-04-12-Graphs.html +++ /dev/null @@ -1,591 +0,0 @@ - - - - -2013-04-12-Graphs - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-12-Graphs

- - - - -
-

1 Graphs & Networks    slide

-
- - -
- -
-

1.1 Midterm Stats    notes

-
- -
    -
  • min 53.00 -
    • -
  • max 87.00 -
    • -
  • avg 75.53 -
    • -
  • med 77.00 -
    • -
  • stdev 9.28 -
    • -
- - -
-
- -
- -
-

2 Graphs    slide

-
- -
    -
  • Can model a surprising number of domains -
    • -
  • Modeling with network opens up large number of algorithms -
    • -
  • Linear math has many connection to graphs -
    • -
- - -
- -
-

2.1 Math    notes

-
- -
    -
  • Data mining theme: get your problem stated as a math problem, whole slew of - solutions present themselves -
    • -
  • Linear math really useful for running equations of all nodes, are simulate - moving across network -
    • -
- - -
-
- -
- -
-

3 Vertices & Edges    slide two_col

-
- -
-
Vertex
the interconnected objects, or nodes -
-
Edge
the lines or curves that connect vertices -
-
Graph
Collection of vertices and edges G = (V,E) -
-
- -

img/GraphNodesEdges.gif -

-
- -
-

3.1 Definitions    notes

-
- -
    -
  • These are the abstract terms, how do they relate to the real world? -
    • -
- - -
-
- -
- -
-

4 Examples    slide

-
- -
-
Vertex
User, building, router, product -
-
Edge
Relationship, road, network cable, purchased -
-
Graph
Social Network, physical infrastructure, internet, purchasing - history -
-
- - -
- -
-

4.1 Examples    notes

-
- -
    -
  • Many graphs have assumed edge labels: they edges represent something - consistent -
    • -
  • Some graphs have multiple times of edges: relationship is one of family, - friend, co-worker, etc. -
    • -
  • Edge can be anything that ties two things together: purchase history, eg. - is not a physical thing connecting, but an idea -
    • -
- - -
-
- -
- -
-

5 Social Networks    slide two_col

-
- -
    -
  • Edge connecting two people -
    • -
  • If this is just a line, what information are we missing about how the link - was formed? -
    • -
- -

img/jblomo-linkedin.gif -

-
- -
-

5.1 Symmetric    notes

-
- -
    -
  • "Just a line" is symmetric, ie "undirected" -
    • -
  • We're missing information about who invited whom. Asymmetric and directed -
    • -
- - -
-
- -
- -
-

6 Definitions    slide two_col

-
- -
-
Directed
Connections have a direction. Invitations, water pipes, email -
-
Undirected
Connections have no direction. "Friends," walkways on campus, - physical wires -
-
Cycle
Set of nodes and edges in which you can travel back to a vertex -
-
Acyclic
A graph without any cycles -
-
- -

img/Directed_acyclic_graph.png -

-
- -
-

6.1 Modeling    notes

-
- -
    -
  • Can always model undirected graph as a directed one by having two - connections between nodes always -
    • -
- - -
-
- -
- -
-

7 Acyclic?    slide

-
- -
    -
  • Social network (undirected) -
    • -
  • Product purchases (directed) -
    • -
  • Internet links (directed) -
    • -
  • Class prerequisites (directed) -
    • -
- - -
- -
-

7.1 Answers    notes

-
- -
    -
  • Social network: cyclic -
    • -
  • Product purchases: acyclic -
    • -
  • Internet links: cyclic -
    • -
  • Class prerequisites: acyclic -
    • -
- - -
-
- -
- -
-

8 Bipartite    slide two_col

-
- -
    -
  • Graph whose vertices can be divided into two distinct sets -
    • -
  • Vertices in U are only connected to those in V, vice versa -
    • -
  • Product purchases: users U, products V -
    • -
- -

img/Simple-bipartite-graph.svg.png -

-
- -
-

8.1 Recommendations    notes

-
- -
    -
  • Can model recommendations as link following: -
    • -
  • From a user, follow to products -
    • -
  • From products, follow back to other users -
    • -
  • From other users, follow back to products -
    • -
- - -
-
- -
- -
-

9 Measurements    slide

-
- -
-
Geodesic distance
Number of edges to connect to vertices -
-
Eccentricity
Largest geodesic distance from v to another -
-
Radius
Minimum eccentricity -
-
Diameter
Maximum eccentricity -
-
Peripheral vertex
Vertex with eccentricity == diameter -
-
Incoming/Outgoing edge count
Number of edges point to or from an edge -
-
- - -
- -
-

9.1 Data Stats    notes

-
- -
    -
  • Similar to getting distribution stats from initial datasets, these - measurements can help you understand graphs as a summary -
    • -
  • Once you have the incoming/outgoing edge counts, can use regular stats: - what is the distribution of counts? -
    • -
- - -
- -
- -
-

9.2 Examples    slide center

-
- -

img/6n-graf.svg.png -

-
- -
-

9.2.1 Answers    notes

-
- -
    -
  • Distance 6, 5: 2 -
    • -
  • Eccentricity 2: 3 (disconnected graph is infinity) -
    • -
  • Radius: 2 -
    • -
  • Diameter: 3 -
    • -
  • Pericheral Verticies: 1, 2, 6 -
    • -
- - -
-
-
- -
- -
-

10 SimRank    slide

-
- -
    -
  • Vertices are similar if they share similar neighbors -
    • -
  • SimRank between two vertices is the average of the SimRank of its neighbors -
    • -
- -

img/simrank.png - img/simrank-iterative.png -

-
- -
-

10.1 Recursive    notes

-
- -
    -
  • This is an iterative and recursive definition -
    • -
  • Iterative because neighbors are influenced by each other -
      -
    • What is your simrank? Well, what is your simrank? -
      • -
    • Converges -
      • -
    - -
    • -
  • Recursive because you're figuring out simrank for all neighbors -
    • -
- - -
- -
- -
-

10.2 Example    slide

-
- -

img/6n-graf.svg.png -

-

- 2 SimRank 4 -

-
- -
-

10.2.1 Calculations    notes

-
- -
    -
  • I(u) = 5,3,1 -
    • -
  • I(v) = 5,3,6 -
    • -
  • C = 0.6 daming factor.. similarity fades over time -
    • -
  • s0(5,5) = 1 -
    • -
  • s0(3,3) = 1 -
    • -
  • s0(5,3) = 0 -
    • -
  • s0(3,5) = 0 -
    • -
  • s0(1,5) = s0(1,3) = 0, s0(6,1)… = 0 -
    • -
  • s1= 0.6/(2*2) * sum(1,1,0,0,0,0) -
    • -
  • 0.3 -
    • -
  • Next round, we'll need to figure out s1 of 5,3 to calculate update -
    • -
- - -
-
-
- -
- -
-

11 Random Walk    slide

-
- -
    -
  • Many algorithms based on concept of randomly deciding: -
      -
    • Follow link or not -
      • -
    • Which link to follow -
      • -
    - -
    • -
  • Simulate the decision many times -
    • -
  • What is the probability you will wind up on u from v? -
    • -
- - -
- -
- -
-

12 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-04-12 13:44:58 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-12-Graphs.org b/slides/2013-04-12-Graphs.org deleted file mode 100644 index ef31ef7..0000000 --- a/slides/2013-04-12-Graphs.org +++ /dev/null @@ -1,157 +0,0 @@ -* Graphs & Networks :slide: -** Midterm Stats :notes: - + min 53.00 - + max 87.00 - + avg 75.53 - + med 77.00 - + stdev 9.28 - -* Graphs :slide: - + Can model a surprising number of domains - + Modeling with network opens up large number of algorithms - + Linear math has many connection to graphs -** Math :notes: - + Data mining theme: get your problem stated as a math problem, whole slew of - solutions present themselves - + Linear math really useful for running equations of all nodes, are simulate - moving across network - -* Vertices & Edges :slide:two_col: - + Vertex :: the interconnected objects, or nodes - + Edge :: the lines or curves that connect vertices - + Graph :: Collection of vertices and edges =G = (V,E)= - [[file:img/GraphNodesEdges.gif]] -** Definitions :notes: - + These are the abstract terms, how do they relate to the real world? - -* Examples :slide: - + Vertex :: User, building, router, product - + Edge :: Relationship, road, network cable, purchased - + Graph :: Social Network, physical infrastructure, internet, purchasing - history -** Examples :notes: - + Many graphs have assumed edge labels: they edges represent something - consistent - + Some graphs have multiple times of edges: relationship is one of family, - friend, co-worker, etc. - + Edge can be anything that ties two things together: purchase history, eg. - is not a physical thing connecting, but an idea - -* Social Networks :slide:two_col: - + Edge connecting two people - + If this is just a line, what information are we missing about how the link - was formed? - [[file:img/jblomo-linkedin.gif]] -** Symmetric :notes: - + "Just a line" is symmetric, ie "undirected" - + We're missing information about who invited whom. Asymmetric and *directed* - -* Definitions :slide:two_col: - + Directed :: Connections have a direction. Invitations, water pipes, email - + Undirected :: Connections have no direction. "Friends," walkways on campus, - physical wires - + Cycle :: Set of nodes and edges in which you can travel back to a vertex - + Acyclic :: A graph without any cycles - [[file:img/Directed_acyclic_graph.png]] -** Modeling :notes: - + Can always model undirected graph as a directed one by having two - connections between nodes always - -* Acyclic? :slide: - + Social network (undirected) - + Product purchases (directed) - + Internet links (directed) - + Class prerequisites (directed) -** Answers :notes: - + Social network: cyclic - + Product purchases: acyclic - + Internet links: cyclic - + Class prerequisites: acyclic - -* Bipartite :slide:two_col: - + Graph whose vertices can be divided into two distinct sets - + Vertices in =U= are only connected to those in =V=, vice versa - + Product purchases: users =U=, products =V= - [[file:img/Simple-bipartite-graph.svg.png]] -** Recommendations :notes: - + Can model recommendations as link following: - + From a user, follow to products - + From products, follow back to other users - + From other users, follow back to products - -* Measurements :slide: - + Geodesic distance :: Number of edges to connect to vertices - + Eccentricity :: Largest geodesic distance from =v= to another - + Radius :: Minimum eccentricity - + Diameter :: Maximum eccentricity - + Peripheral vertex :: Vertex with eccentricity == diameter - + Incoming/Outgoing edge count :: Number of edges point to or from an edge -** Data Stats :notes: - + Similar to getting distribution stats from initial datasets, these - measurements can help you understand graphs as a summary - + Once you have the incoming/outgoing edge counts, can use regular stats: - what is the distribution of counts? - -** Examples :slide:center: - [[file:img/6n-graf.svg.png]] -*** Answers :notes: - + Distance 6, 5: 2 - + Eccentricity 2: 3 (disconnected graph is infinity) - + Radius: 2 - + Diameter: 3 - + Pericheral Verticies: 1, 2, 6 - -* SimRank :slide: - + Vertices are similar if they share similar neighbors - + SimRank between two vertices is the average of the SimRank of its neighbors - [[file:img/simrank.png]] - [[file:img/simrank-iterative.png]] -** Recursive :notes: - + This is an iterative and recursive definition - + Iterative because neighbors are influenced by each other - + What is your simrank? Well, what is your simrank? - + Converges - + Recursive because you're figuring out simrank for all neighbors - -** Example :slide: - [[file:img/6n-graf.svg.png]] - - 2 SimRank 4 -*** Calculations :notes: - + I(u) = 5,3,1 - + I(v) = 5,3,6 - + C = 0.6 daming factor.. similarity fades over time - + s0(5,5) = 1 - + s0(3,3) = 1 - + s0(5,3) = 0 - + s0(3,5) = 0 - + s0(1,5) = s0(1,3) = 0, s0(6,1)... = 0 - + s1= 0.6/(2*2) * sum(1,1,0,0,0,0) - + 0.3 - + Next round, we'll need to figure out s1 of 5,3 to calculate update - -* Random Walk :slide: - + Many algorithms based on concept of randomly deciding: - + Follow link or not - + Which link to follow - + Simulate the decision many times - + What is the probability you will wind up on =u= from =v=? - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-12-PageRank.html b/slides/2013-04-12-PageRank.html deleted file mode 100644 index f31ff33..0000000 --- a/slides/2013-04-12-PageRank.html +++ /dev/null @@ -1,631 +0,0 @@ - - - - -2013-04-12-PageRank - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-12-PageRank

- - - - -
-

1 PageRank    slide center

-
- -

img/PageRanks-Example.svg.png -

-
- -
-

1.1 Web page importance    notes

-
- -
    -
  • Also used to model importance of people, places… anything that has a - reputation -
    • -
  • inbound links are important, but scaled by the importance of the source -
    • -
  • C still important, even though it only has one inbound -
    • -
- - -
-
- -
- -
-

2 Random Walks    slide

-
- -
    -
  • Starting from a random page, what is the likelihood of winding up on a target - page? -
    • -
  • Starting point captured by initial constant -
    • -
  • Stopping captured by "damping factor" (0.85) -
    • -
- -

img/pagerank.png -

-
- -
-

2.1 Original    notes

-
- -
    -
  • Original paper did not divide by N -
    • -
  • This gives relative weights of pages, but not a formal probability because - sum will not add up to N -
    • -
  • Either way is fine for our purposes -
    • -
- - -
-
- -
- -
-

3 Example    slide

-
- -
    -
  • B, C, D, all link to A -
    • -
  • B has PageRank of 0.5, 4 links -
    • -
  • C has PageRank of 0.7, 4 links -
    • -
  • D has PageRank of 0.2, 1 link -
    • -
- - -
- -
-

3.1 Calculations    notes

-
- -
    -
  • 0.15 + 0.85 * sum(PR/links for (pr,links) in pages) -
    • -
  • 0.15 + 0.85 * sum(0.5/4, 0.7/4, 0.2/1) -
    • -
  • 0.15 + 0.85 * 0.465 -
    • -
  • 0.554525 -
    • -
  • From Programming Collective Intelligence -
    • -
- - -
-
- -
- -
-

4 Other Pages    slide

-
- -
    -
  • But how did we know the PageRank of other pages? -
    • -
  • Similar to SimRank, we calculate iteratively until convergence -
    • -
- - -
- -
- -
-

5 Representing Graphs    slide

-
- -
-
Adjacency Matrix
Represent graph edges in a matrix -
-
- - - - -- - - - - - - - -
VABCD
A0000
B1010
C1001
D1100
- - -
- -
-

5.1 Diversion    notes

-
- -
    -
  • Take a step back so we can motivate how to express these calculations as - linear algebra -
    • -
  • Using linear algebra can help us translate graph concepts to fairly elegant - code, as well as realize some optimizations -
    • -
  • Draw -
    • -
  • Symmetric? When? -
    • -
- - -
-
- -
- -
-

6 Representing Graphs    slide

-
- -
-
Adjacency List
For a vertex, list all connections -
-
- - - - - -
A []
-B [A,C]
-C [A,D]
-D [A,B]
-
- - -
- -
-

6.1 Diversion    notes

-
- -
    -
  • You can think of this as keys (vertex) and values (list of vertices) -
    • -
  • When would thinking in key-values be useful? MapReduce -
    • -
  • Back to matrix representation -
    • -
- - -
-
- -
- -
-

7 Eigenvector    slide

-
- -
    -
  • PageRank formula divides by number of links -
    • -
  • Modify adjacency matrix typically also normalized such that all rows sum to 1 -
    • -
  • PageRank scores are entries in the largest eigenvector of the matrix - representation -
    • -
- -

img/pagerank-eigen.png -

-
- -
- -
-

8 Eigenvector centrality    slide

-
- -
    -
  • Another measurement for graphs, using the simple adjacency matrix -
    • -
  • Relative influence of a node (no normalization) -
    • -
- - -
- -
- -
-

9 Adversarial    slide

-
- -
    -
  • Source does not want to be discovered -
    • -
  • Patterns are purposefully hidden: so discover the patterns of hiding -
    • -
  • If adversary knows your techniques, they can take advantage of weakness -
    • -
- - -
- -
-

9.1 Weakness    notes

-
- -
    -
  • Reading: paper discovering hiding patterns -
    • -
  • Weakness of pagerank? -
    • -
  • We assume that these links are legitimate. -
    • -
  • What happens if the links are not conveying authority? -
    • -
- - -
-
- -
- -
-

10 Google Bomb :slide:center

-
- -
    -
  • Milder forms of adversarial work -
    • -
- -

img/Google_Bomb_Miserable_Failure.png -

-
- -
-

10.1 Link farms    notes

-
- - - - -
-
- -
- -
-

11 Hubs & Authorities    slide two_col

-
- -
    -
  • Earlier in the web, more structure -
    • -
  • Hubs: collected links to different resources -
    • -
  • Authorities: Gave out specific information -
    • -
  • Score separately? -
    • -
- -

img/early-yahoo.jpg -

-
- -
-

11.1 Alternatives    notes

-
- -
    -
  • Some other interesting network analysis tools -
    • -
- - -
-
- -
- -
-

12 HITS    slide

-
- -
-
Authority score
sum(hub(i) for i in inbound_links) -
-
Hub score
sum(authority(i) for i in outbound_links) -
-
Normalize
to ensure convergence, square root sum of squares of scores -
-
- - -
- -
-

12.1 Iterative    notes

-
- -
    -
  • Sill iterative, but now using inbound and outbound links to judge -
    • -
  • Hubs have outbound links to authoritive pages -
    • -
  • Authorities have inbound links from good hubs -
    • -
- - -
-
- -
- -
-

13 Connections    slide

-
- -
-
Connected
there exists a path from one vertex to another -
-
Connectivity
minimum number of vertices to remove to disconnect remaining - vertices -
-
Clustering Coefficient
Measure of how connected a vertex or group of - vertices are -
-
- - -
- -
-

13.1 Robustness    notes

-
- -
    -
  • This is used to understand robustness of a system: if an earthquake - damaged the Bay Bridge, could we still travel from one point to another? -
    • -
  • What is the connectedness of Oakland and SF? -
    • -
  • Closely related to min-cuts, which is discussed in the book -
    • -
  • Network topology: what happens if a router fails? -
    • -
- - -
-
- -
- -
-

14 Clustering Coefficient    slide animate

-
- -
    -
  • How many directed edges are possible between 3 vertices? -
    • -
  • 4 vertices? -
    • -
  • v*(v-1) -
    • -
  • Undirected? -
    • -
  • v*(v-1)/2 -
    • -
  • Clustering Coefficient: Ratio of actual edges to possible edges -
    • -
- - -
- -
-

14.1 Reading    notes

-
- -
    -
  • Used in Reading this week -
    • -
  • v*(v-1) connection to every other node but yourself -
    • -
  • /2 undirected, don't double count connections -
    • -
- - -
- -
- -
-

14.2 Example    slide

-
- -

img/Directed_acyclic_graph.png -

-

- Connectivity Coefficient of 1; 4 -

-
- -
-

14.2.1 Answer    notes

-
- -
    -
  • Neighbors of 1: 5 2 -
    • -
  • 2*(2-1) / 2 = 1 -
    • -
  • Actual links = 1 -
    • -
  • Neighbors of 4: 3,5,6 -
    • -
  • 3*(3-1) / 2 = 3 -
    • -
  • Actual: 0 -
    • -
  • If 3-5 connected? 1/3 -
    • -
- - -
-
-
- -
- -
-

15 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-04-12 13:50:16 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-12-PageRank.org b/slides/2013-04-12-PageRank.org deleted file mode 100644 index 8944529..0000000 --- a/slides/2013-04-12-PageRank.org +++ /dev/null @@ -1,168 +0,0 @@ -* PageRank :slide:center: - [[file:img/PageRanks-Example.svg.png]] -** Web page importance :notes: - + Also used to model importance of people, places... anything that has a - reputation - + inbound links are important, but scaled by the importance of the source - + C still important, even though it only has one inbound - -* Random Walks :slide: - + Starting from a random page, what is the likelihood of winding up on a target - page? - + Starting point captured by initial constant - + Stopping captured by "damping factor" (0.85) - [[file:img/pagerank.png]] -** Original :notes: - + Original paper did not divide by N - + This gives relative weights of pages, but not a formal probability because - sum will not add up to N - + Either way is fine for our purposes - -* Example :slide: - + =B=, =C=, =D=, all link to =A= - + =B= has PageRank of 0.5, 4 links - + =C= has PageRank of 0.7, 4 links - + =D= has PageRank of 0.2, 1 link -** Calculations :notes: - + 0.15 + 0.85 * sum(PR/links for (pr,links) in pages) - + 0.15 + 0.85 * sum(0.5/4, 0.7/4, 0.2/1) - + 0.15 + 0.85 * 0.465 - + 0.554525 - + From _Programming Collective Intelligence_ - -* Other Pages :slide: - + But how did we know the PageRank of other pages? - + Similar to SimRank, we calculate iteratively until convergence - -* Representing Graphs :slide: - + Adjacency Matrix :: Represent graph edges in a matrix - - | V | A | B | C | D | - | A | 0 | 0 | 0 | 0 | - | B | 1 | 0 | 1 | 0 | - | C | 1 | 0 | 0 | 1 | - | D | 1 | 1 | 0 | 0 | -** Diversion :notes: - + Take a step back so we can motivate how to express these calculations as - linear algebra - + Using linear algebra can help us translate graph concepts to fairly elegant - code, as well as realize some optimizations - + Draw - + Symmetric? When? - -* Representing Graphs :slide: - + Adjacency List :: For a vertex, list all connections - -#+begin_src csv -A [] -B [A,C] -C [A,D] -D [A,B] -#+end_src -** Diversion :notes: - + You can think of this as keys (vertex) and values (list of vertices) - + When would thinking in key-values be useful? MapReduce - + Back to matrix representation - -* Eigenvector :slide: - + PageRank formula divides by number of links - + Modify adjacency matrix typically also normalized such that all rows sum to 1 - + PageRank scores are entries in the largest eigenvector of the matrix - representation - [[file:img/pagerank-eigen.png]] - -* Eigenvector centrality :slide: - + Another measurement for graphs, using the simple adjacency matrix - + Relative influence of a node (no normalization) - -* Adversarial :slide: - + Source does not want to be discovered - + Patterns are purposefully hidden: so discover the patterns of hiding - + If adversary knows your techniques, they can take advantage of weakness -** Weakness :notes: - + Reading: paper discovering hiding patterns - + Weakness of pagerank? - + We assume that these links are legitimate. - + What happens if the links are not conveying authority? - -* Google Bomb :slide:center - + Milder forms of adversarial work - [[file:img/Google_Bomb_Miserable_Failure.png]] -** Link farms :notes: - + Link farms try to create fake links to pages, - + [[http://www.nytimes.com/2011/02/13/business/13search.html?pagewanted=all][JC Penny's link farm]] - -* Hubs & Authorities :slide:two_col: - + Earlier in the web, more structure - + Hubs: collected links to different resources - + Authorities: Gave out specific information - + Score separately? -[[file:img/early-yahoo.jpg]] -** Alternatives :notes: - + Some other interesting network analysis tools - -* HITS :slide: - + Authority score :: sum(hub(i) for i in inbound\_links) - + Hub score :: sum(authority(i) for i in outbound\_links) - + Normalize :: to ensure convergence, square root sum of squares of scores -** Iterative :notes: - + Sill iterative, but now using inbound and outbound links to judge - + Hubs have outbound links to authoritive pages - + Authorities have inbound links from good hubs - -* Connections :slide: - + Connected :: there exists a path from one vertex to another - + Connectivity :: minimum number of vertices to remove to disconnect remaining - vertices - + Clustering Coefficient :: Measure of how connected a vertex or group of - vertices are -** Robustness :notes: - + This is used to understand robustness of a system: if an earthquake - damaged the Bay Bridge, could we still travel from one point to another? - + What is the connectedness of Oakland and SF? - + Closely related to min-cuts, which is discussed in the book - + Network topology: what happens if a router fails? - -* Clustering Coefficient :slide:animate: - + How many directed edges are possible between 3 vertices? - + 4 vertices? - + =v*(v-1)= - + Undirected? - + =v*(v-1)/2= - + Clustering Coefficient: Ratio of actual edges to possible edges -** Reading :notes: - + Used in Reading this week - + =v*(v-1)= connection to every other node but yourself - + =/2= undirected, don't double count connections - -** Example :slide: - [[file:img/Directed_acyclic_graph.png]] - - Connectivity Coefficient of 1; 4 -*** Answer :notes: - + Neighbors of 1: 5 2 - + 2*(2-1) / 2 = 1 - + Actual links = 1 - + Neighbors of 4: 3,5,6 - + 3*(3-1) / 2 = 3 - + Actual: 0 - + If 3-5 connected? 1/3 - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-19-Elasticity.html b/slides/2013-04-19-Elasticity.html deleted file mode 100644 index dfcbfb2..0000000 --- a/slides/2013-04-19-Elasticity.html +++ /dev/null @@ -1,181 +0,0 @@ - - - - -2013-04-19-Elasticity - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-19-Elasticity

- - -
-

Table of Contents

-
- -
-
- -
-

1 Homework    slide

-
- -
    -
  • Find price elasticity -
    • -
  • Data in code/price-elasticity.csv -
    • -
  • Your choice of solving technology -
    • -
- - -
- -
- -
-

2 Data    slide

-
- - - -- - - - - - - - - - - - - -
DOWRoomsRate
1700$216.79
11020$201.64
11327$136.60
12087$118.10
1757$179.12
190$258.73
11489$136.37
1781$165.35
1209$287.84
- - -
- -
- -
-

3 Data meanings    slide

-
- -
    -
  • DOW: 1 == Sunday -
    • -
  • Weekends are nights going into a weekend, ie Friday, Saturday, ie 6 & 7 -
    • -
- - -
- -
- -
-

4 Result    slide

-
- -
    -
  1. What is the price elasticity of weekday prices? -
    2. -
  2. What is the price elasticity of weekend prices? -
    3. -
  3. If we are currently forecast to be at 1000 rooms at $200 rate with 100 - available capacity, what price should we set to optimize max revenue for - both -
      -
    • Weekday -
      • -
    • Weekend -
      • -
    - -
    4. -
- - -
- -
- -
-

5 Excel    slide

-
- - - - -
LN()
-Options > Add-Ins > Analysis ToolPak > Go > ToolPak
-Data Analysis > Regression > Y Range, X Range
-
- - -
- -
- -
-

6 Python    slide

-
- - - - -
from math import log, exp
-from scipy.stats import linregress
-slope, intercept, r_value, p_value, std_err = linregress(indep, depend)
-
- - - - - -
-
-
- -
-

Date: 2013-05-17 00:53:42 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-19-Elasticity.org b/slides/2013-04-19-Elasticity.org deleted file mode 100644 index 9e0281d..0000000 --- a/slides/2013-04-19-Elasticity.org +++ /dev/null @@ -1,60 +0,0 @@ -* Homework :slide: - + Find price elasticity - + Data in =code/price-elasticity.csv= - + Your choice of solving technology - -* Data :slide: -|DOW|Rooms|Rate| -|1|700|$216.79 | -|1|1020|$201.64 | -|1|1327|$136.60 | -|1|2087|$118.10 | -|1|757|$179.12 | -|1|90|$258.73 | -|1|1489|$136.37 | -|1|781|$165.35 | -|1|209|$287.84 | - -* Data meanings :slide: - + DOW: 1 == Sunday - + Weekends are nights going into a weekend, ie Friday, Saturday, ie 6 & 7 - -* Result :slide: - 1. What is the price elasticity of weekday prices? - 1. What is the price elasticity of weekend prices? - 1. If we are currently forecast to be at 1000 rooms at $200 rate with 100 - available capacity, what price should we set to optimize max revenue for - both - + Weekday - + Weekend - -* Excel :slide: -#+begin_src Excel -LN() -Options > Add-Ins > Analysis ToolPak > Go > ToolPak -Data Analysis > Regression > Y Range, X Range -#+end_src - -* Python :slide: -#+begin_src python -from math import log, exp -from scipy.stats import linregress -slope, intercept, r_value, p_value, std_err = linregress(indep, depend) -#+end_src - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-19-Nonlinear.pdf b/slides/2013-04-19-Nonlinear.pdf deleted file mode 100644 index 8cfb39c..0000000 Binary files a/slides/2013-04-19-Nonlinear.pdf and /dev/null differ diff --git a/slides/2013-04-26-Midterm-HW.html b/slides/2013-04-26-Midterm-HW.html deleted file mode 100644 index fef2c9e..0000000 --- a/slides/2013-04-26-Midterm-HW.html +++ /dev/null @@ -1,136 +0,0 @@ - - - - -2013-04-26-Midterm-HW - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-26-Midterm-HW

- - -
-

Table of Contents

-
- -
-
- -
-

1 Homework: Midterm Corrections    slide

-
- -
    -
  • Midterm should not be cram-and-forget -
    • -
  • Correct the mistakes on your midterms -
    • -
  • Ask questions for problems you don't understand -
    • -
- - -
- -
- -
-

2 Deliverable    slide

-
- -
    -
  • GitHub pull request: -
    • -
  • Question # : Correct answer -
    • -
  • Open book, notes, everything. Cite sources -
    • -
  • Have slightly more exposition than the midterm requires -
    • -
- - -
- -
-

2.1 Exposition    notes

-
- -
    -
  • Eg. if the midterm asks 2 sentences, maybe write 3-4 -
    • -
  • If midterm asks for psudo code, consider writing Python (though syntax will - not be graded) -
    • -
- - -
-
- -
- -
-

3 Why?    slide

-
- -
    -
  • These questions frequently come up on interviews, conversation. -
    • -
  • "How can you describe a distribution?" -
    • -
  • "Write out a MapReduce job to calculate click through rates." -
    • -
  • "How can we tell if a review is duplicated?" -
    • -
- - - - - - - -
-
-
- -
-

Date: 2013-04-26 09:56:50 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-26-Midterm-HW.org b/slides/2013-04-26-Midterm-HW.org deleted file mode 100644 index 9477f71..0000000 --- a/slides/2013-04-26-Midterm-HW.org +++ /dev/null @@ -1,37 +0,0 @@ -* Homework: Midterm Corrections :slide: - + Midterm should not be cram-and-forget - + Correct the mistakes on your midterms - + Ask questions for problems you don't understand - -* Deliverable :slide: - + GitHub pull request: - + Question # : Correct answer - + Open book, notes, everything. Cite sources - + Have slightly more exposition than the midterm requires -** Exposition :notes: - + Eg. if the midterm asks 2 sentences, maybe write 3-4 - + If midterm asks for psudo code, consider writing Python (though syntax will - not be graded) - -* Why? :slide: - + These questions frequently come up on interviews, conversation. - + "How can you describe a distribution?" - + "Write out a MapReduce job to calculate click through rates." - + "How can we tell if a review is duplicated?" - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-26-Multimedia.html b/slides/2013-04-26-Multimedia.html deleted file mode 100644 index e32732e..0000000 --- a/slides/2013-04-26-Multimedia.html +++ /dev/null @@ -1,686 +0,0 @@ - - - - -2013-04-26-Multimedia - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-26-Multimedia

- - - - -
-

1 Multimedia Data Mining    slide

-
- - -
- -
- -
-

2 Features    slide two_col

-
- -
    -
  • Core algorithms similar to "traditional" data mining -
    • -
  • Difference lies in feature engineering -
    • -
  • How to translate intuitions to numbers and formulas? -
    • -
- -

img/face-recognition.jpg -

-
- -
- -
-

3 Types    slide

-
- -
-
Spatial
geographic points and features, including natural and man-made - phenomenon -
-
Images
Size, color, shape, curves, relative positions -
-
Music
Tone, tempo, beat, rhythm -
-
Voice
Speed, accent, word pauses, background noise -
-
- - -
- -
-

3.1 Covering    notes

-
- -
    -
  • We'll cover these areas briefly to get an overview of techniques used in - these fields -
    • -
  • All of these things have embedded information in them, and we are trying - to extract it -
    • -
  • One of the reasons data mining is not a black box: some one has to be on - the outside interpreting results. Results inform technique -
    • -
- - -
-
- -
- -
-

4 Generalization    slide two_col

-
- -
    -
  • Many of these areas have digital representations -
    • -
  • Can we use the raw bit representations? -
    • -
  • Usually not: must generalize patterns -
    • -
- -

img/digits.png -

-
- -
-

4.1 Density    notes

-
- -
    -
  • The data we get from digital representations is generally too sparse -
    • -
  • Key component of good learning is data, but you need fairly dense data - to learn a pattern -
    • -
  • Hypothetically, a neural network could extract general features from raw - data, but you'd need a really large amount of data in order to get the - density needed -
    • -
  • Example: for NLP, perhaps your corpus is too sparse: not many words are - shared between documents. So instead generalize: what parts of speech or - patterns show up across documents? -
    • -
- - -
-
- -
- -
-

5 Generalized Features    slide

-
- -
    -
  • Derivative / Slope of behavior -
    • -
  • Min / Max of groups of points -
    • -
  • Bucketing / Blurring -
    • -
  • Relative positions / angles -
    • -
- - -
- -
-

5.1 Techniques    notes

-
- -
    -
  • How can you strip some of the non-essential information, keep important - patterns? -
    • -
  • Many times we care about relative change, like in pricing -
    • -
  • Or group data points together (clustering is an advanced form of this) -
    • -
  • OK, let's get into some specifics: -
    • -
- - -
-
- -
- -
-

6 GIS    slide two_col

-
- -
    -
  • Geographic Information Systems -
    • -
  • Analysis and visualization of geographic data -
    • -
  • Search, terrain, object detection, flow calculations -
    • -
- -

img/gis.jpg -

-
- -
- -
-

7 Spatial Databases    slide

-
- -
    -
  • Integrates spatial information with traditional DBMS operations -
    • -
  • Spatial indexing, distance metrics, polygon definitions, layering -
    • -
  • Eg: Oracle Spatial Data Cartridge, ESRI Spatial Engine -
    • -
- - -
- -
- -
-

8 Discovery    slide

-
- -
    -
  • What are examples of efficient city layouts? -
    • -
  • What influences successful business centers? -
    • -
  • Deforestation rates -
    • -
- - -
- -
-

8.1 Ideas    notes

-
- -
    -
  • City layouts: Understanding home->work distances, not Euclidean, but - traffic on streets or by public transportation, recognizing traffic jams -
    • -
  • Business centers: analyzing network flow based on roads: industrial - supply centers nearby? Creative centers, restaurants, nightlife? -
    • -
  • Deforestation: nearby cities' effect? Recognizing forested areas vs - clear cut. Time series -
    • -
- - -
-
- -
- -
-

9 ATM Locations given obstacles    slide center

-
- -

img/obstacle-clustering.png -

-
- -
-

9.1 Yelp    notes

-
- -
    -
  • This is a current area we could improve at Yelp: -
    • -
  • Just because you're a mile from a restaurant doesn't mean it is "close" -
    • -
  • Maybe across the Bay, or maybe in between metro stops -
    • -
  • How can you calculate efficiently? -
    • -
- - -
-
- -
- -
-

10 Images    slide two_col

-
- -
    -
  • General Feature Extraction -
    • -
  • Sketch Recognition -
    • -
  • Image Recognition -
    • -
- -

img/Sift_keypoints_filtering.jpg -

-
- -
-

10.1 Covering    notes

-
- -
    -
  • We'll cover some interesting ways to extract dimensions -
    • -
  • ML/data mining combine these dimensions to do recognition with, eg. - labeled data -
    • -
  • Image on the right is using an algorithm to pick out, then filter - "interesting" points on the image -
    • -
  • img: http://en.wikipedia.org/wiki/Scale-invariant_feature_transform -
    • -
- - -
-
- -
- -
-

11 SIFT    slide

-
- -

img/Sift_keypoints_filtering.jpg -

-
- -
-

11.1 Process    notes

-
- -
    -
  • Successively apply Gaussian blur to image -
    • -
  • Find points which "stand out" between blurs (ie big differences) -
    • -
  • You can connect these keypoints to make a kind of fingerprint -
    • -
  • These fingerprints can be used, scaled, etc. to match against other images -
    • -
- - -
-
- -
- -
-

12 Sketch Recognition    slide center

-
- -

img/sketch-1.png -

    -
  • Find (x,y) points along a sketch -
    • -
- - -
- -
-

12.1 Why?    notes

-
- -
    -
  • Sketch recognition can be used to see if you're drawing shapes -
    • -
  • Be nice to be able to snap a picture of your diagram on a napkin and have - it come out nicely formatted? -
    • -
  • But how to recognize a circle, assuming you can't draw a perfect circle? -
    • -
  • Start with (x,y) points, but as we mentioned, very sparse -
    • -
  • Images by Marty Field -
    • -
- - -
-
- -
- -
-

13 Direction    slide center

-
- -

img/sketch-2.png -

    -
  • Find angles along a sketch -
    • -
- - -
- -
-

13.1 Angles?    notes

-
- -
    -
  • Instead of points, measure the angle at each turn -
    • -
  • You'll notice something peculiar about these angles. What? -
    • -
  • They're more than +/- 180 because we want to continue a "trend" if - they're turning the same way. Help identify changes in direction vs - spirals -
    • -
- - -
-
- -
- -
-

14 Direction Plot    slide center

-
- -

img/sketch-3.png -

    -
  • Plot angles vs time -
    • -
- - -
- -
-

14.1 Why?    notes

-
- -
    -
  • Becomes even more generalized: -
      -
    • What is the derivative? -
      • -
    • How many times to we change derivatives? -
      • -
    - -
    • -
- - -
-
- -
- -
-

15 Direction Plot    slide center

-
- -

img/sketch-4.png -

    -
  • Plot angles vs time -
    • -
- - -
- -
-

15.1 Why?    notes

-
- -
    -
  • Example where we change directions -
    • -
- - -
-
- -
- -
-

16 Features    slide center

-
- -

img/sketch-4.png -

-
NDDE
Normalized Distance between Direction Extremes -
-
DCR
Direction Change Ratio -
-
- - -
- -
-

16.1 Why?    notes

-
- -
-
NDDE
Are the discontinuous changes in direction, or is the line -
    -
  • generally curvy, and follows a similar path? -
    • -
- -
-
DCR
Total amount of angle change in the sketch. Low for first, high - for second -
-
Others?
bounding box size/ratio, stroke length, distance between endpoints, - length, width, height, speed, direction, acceleration -
-
- - -
-
- -
- -
-

17 All Together Now    slide

-
- - -

Sketch2Photo: Internet Image Montage from Tao Chen on Vimeo.

-

-Sketch2Photo -

-
- -
- -
-

18 Music    slide

-
- -
    -
  • Generate a finger print: time, frequency, amplitude -
    • -
  • Filter most intense (largest) amplitudes -
    • -
  • Create a hash of connections between points -
    • -
  • Match, in time, the hash between songs -
    • -
- -

img/music_match.png -

-
- -
-

18.1 Relation to Images    notes

-
- -
    -
  • Interesting to note: we transformed one media type (music) into another - (image), then started using some techniques we've seen in image - fingerprinting -
    • -
  • More in reading -
    • -
- - -
-
- -
- -
-

19 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-04-26 09:55:15 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-26-Multimedia.org b/slides/2013-04-26-Multimedia.org deleted file mode 100644 index 5a3cde6..0000000 --- a/slides/2013-04-26-Multimedia.org +++ /dev/null @@ -1,185 +0,0 @@ -* Multimedia Data Mining :slide: - -* Features :slide:two_col: - + Core algorithms similar to "traditional" data mining - + Difference lies in feature engineering - + How to translate intuitions to numbers and formulas? - [[file:img/face-recognition.jpg]] - -* Types :slide: - + Spatial :: geographic points and features, including natural and man-made - phenomenon - + Images :: Size, color, shape, curves, relative positions - + Music :: Tone, tempo, beat, rhythm - + Voice :: Speed, accent, word pauses, background noise -** Covering :notes: - + We'll cover these areas briefly to get an overview of techniques used in - these fields - + All of these things *have* embedded information in them, and we are trying - to extract it - + One of the reasons data mining is not a black box: some one has to be on - the outside interpreting results. Results inform technique - -* Generalization :slide:two_col: - + Many of these areas have digital representations - + Can we use the raw bit representations? - + Usually not: must generalize patterns - [[file:img/digits.png]] -** Density :notes: - + The data we get from digital representations is generally too sparse - + Key component of good learning is *data*, but you need fairly *dense* data - to learn a pattern - + Hypothetically, a neural network could extract general features from raw - data, but you'd need a really large amount of data in order to get the - density needed - + Example: for NLP, perhaps your corpus is too sparse: not many words are - shared between documents. So instead generalize: what parts of speech or - patterns show up across documents? - -* Generalized Features :slide: - + Derivative / Slope of behavior - + Min / Max of groups of points - + Bucketing / Blurring - + Relative positions / angles -** Techniques :notes: - + How can you strip some of the non-essential information, keep important - patterns? - + Many times we care about relative change, like in pricing - + Or group data points together (clustering is an advanced form of this) - + OK, let's get into some specifics: - -* GIS :slide:two_col: - + Geographic Information Systems - + Analysis and visualization of geographic data - + Search, terrain, object detection, flow calculations - [[file:img/gis.jpg]] - -* Spatial Databases :slide: - + Integrates spatial information with traditional DBMS operations - + Spatial indexing, distance metrics, polygon definitions, layering - + Eg: Oracle Spatial Data Cartridge, ESRI Spatial Engine - -* Discovery :slide: - + What are examples of efficient city layouts? - + What influences successful business centers? - + Deforestation rates -** Ideas :notes: - + City layouts: Understanding home->work distances, not Euclidean, but - traffic on streets or by public transportation, recognizing traffic jams - + Business centers: analyzing network flow based on roads: industrial - supply centers nearby? Creative centers, restaurants, nightlife? - + Deforestation: nearby cities' effect? Recognizing forested areas vs - clear cut. Time series - -* ATM Locations given obstacles :slide:center: - [[file:img/obstacle-clustering.png]] -** Yelp :notes: - + This is a current area we could improve at Yelp: - + Just because you're a mile from a restaurant doesn't mean it is "close" - + Maybe across the Bay, or maybe in between metro stops - + How can you calculate efficiently? - -* Images :slide:two_col: - + General Feature Extraction - + Sketch Recognition - + Image Recognition - [[file:img/Sift_keypoints_filtering.jpg]] -** Covering :notes: - + We'll cover some interesting ways to extract dimensions - + ML/data mining combine these dimensions to do recognition with, eg. - labeled data - + Image on the right is using an algorithm to pick out, then filter - "interesting" points on the image - + img: http://en.wikipedia.org/wiki/Scale-invariant_feature_transform - -* SIFT :slide: - [[file:img/Sift_keypoints_filtering.jpg]] -** Process :notes: - + Successively apply Gaussian blur to image - + Find points which "stand out" between blurs (ie big differences) - + You can connect these keypoints to make a kind of fingerprint - + These fingerprints can be used, scaled, etc. to match against other images - -* Sketch Recognition :slide:center: - [[file:img/sketch-1.png]] - + Find (x,y) points along a sketch -** Why? :notes: - + Sketch recognition can be used to see if you're drawing shapes - + Be nice to be able to snap a picture of your diagram on a napkin and have - it come out nicely formatted? - + But how to recognize a circle, assuming you can't draw a perfect circle? - + Start with (x,y) points, but as we mentioned, very sparse - + Images by Marty Field - -* Direction :slide:center: - [[file:img/sketch-2.png]] - + Find angles along a sketch -** Angles? :notes: - + Instead of points, measure the angle at each turn - + You'll notice something peculiar about these angles. What? - + They're more than +/- 180 because we want to continue a "trend" if - they're turning the same way. Help identify changes in direction vs - spirals - -* Direction Plot :slide:center: - [[file:img/sketch-3.png]] - + Plot angles vs time -** Why? :notes: - + Becomes even more generalized: - + What is the derivative? - + How many times to we change derivatives? - -* Direction Plot :slide:center: - [[file:img/sketch-4.png]] - + Plot angles vs time -** Why? :notes: - + Example where we change directions - -* Features :slide:center: - [[file:img/sketch-4.png]] - + NDDE :: Normalized Distance between Direction Extremes - + DCR :: Direction Change Ratio -** Why? :notes: - + NDDE :: Are the discontinuous changes in direction, or is the line - + generally curvy, and follows a similar path? - + DCR :: Total amount of angle change in the sketch. Low for first, high - for second - + Others? :: bounding box size/ratio, stroke length, distance between endpoints, - length, width, height, speed, direction, acceleration - -* All Together Now :slide: -#+BEGIN_HTML -

Sketch2Photo: Internet Image Montage from Tao Chen on Vimeo.

-#+END_HTML -[[http://vimeo.com/6496886][Sketch2Photo]] - -* Music :slide: - + Generate a finger print: time, frequency, amplitude - + Filter most intense (largest) amplitudes - + Create a hash of connections between points - + Match, in time, the hash between songs - [[file:img/music_match.png]] -** Relation to Images :notes: - + Interesting to note: we transformed one media type (music) into another - (image), then started using some techniques we've seen in image - fingerprinting - + More in reading - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-04-26-Outliers.html b/slides/2013-04-26-Outliers.html deleted file mode 100644 index 780b47f..0000000 --- a/slides/2013-04-26-Outliers.html +++ /dev/null @@ -1,594 +0,0 @@ - - - - -2013-04-26-Outliers - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-04-26-Outliers

- - - - -
-

1 Outliers    slide

-
- - -
- -
- -
-

2 Generative Model    slide animate

-
- -
    -
  • "Real" model that produced original data points -
    • -
  • Our mission is to reproduce the original model -
    • -
  • Thus we have different techniques that can model different behavior -
    • -
- - -
- -
-

2.1 Questions    notes

-
- -
    -
  • What is a "generative model"? -
    • -
  • What is data mining trying to discover? What is machine learning hoping to - reproduce? -
    • -
  • Why have different classifiers? Decision tree, Naive Bayes, etc? -
    • -
- - -
-
- -
- -
-

3 Outliers    slide two_col

-
- -
    -
  • Significant deviation -
    • -
  • Probably generated through a different model than the rest of the data -
    • -
  • Normal / Abnormal -
    • -
- -

img/outlier.jpg -

-
- -
-

3.1 Intuitive    notes

-
- - - - -
-
- -
- -
-

4 Outlier Types    slide

-
- -
-
Global
points which deviate from the rest of the entire data set. Point - anomalies -
-
Contextual
points which deviate from their peers. Conditional outliers -
-
Collective
points which deviate as a group, even though individual - points may not be considered outliers. -
-
- - -
- -
- -
-

5 Which Type?    slide

-
- -
    -
  • Given class sizes at Berkeley: -
      -
    • A day with 10 people in class -
      • -
    • A day with 7000 people in class -
      • -
    • 3 weeks of 15 people in this class -
      • -
    - -
    • -
  • Given Earth's temperatures: -
      -
    • A day at 100°C -
      • -
    • 30 straight days of rain in Berkeley -
      • -
    • A day at 100°F -
      • -
    - -
    • -
- - -
- -
- -
-

6 Types of Learning    slide two_col

-
- -
    -
  • Supervised -
    • -
  • Unsupervised -
    • -
  • Semi-Supervised -
    • -
- - -

- img/ml-large-icon.png -

-
- -
-

6.1    notes

-
- -
-
Supervised
learning from "gold standard" labels -
-
Unsupervised
learning without labels -
-
Semi-Supervised
infer more labels from a few, learn based on inferred + - labeled -
-
img
https://www.coursera.org/course/ml -
-
- - -
-
- -
- -
-

7 Outlier Methods    slide

-
- -
-
Supervised
Label outliers, treat as classification problem -
-
Unsupervised
Cluster data, find points not clustered well -
-
Semi-Supervised
Manually label few, find point nearby to automatically - label, then treat as classification -
-
Statistical
Decide on a generative model / distribution, find points - which have a low probability of belonging -
-
Proximity
Use relative distance to neighbors -
-
- - -
- -
-

7.1 Features    notes

-
- -
    -
  • Some methods may be overlapping -
    • -
  • When developing features for classification, using relative features can - be helpful: eg. distance from mean -
    • -
  • Eg. Agglomerative clustering, find lone/small groups that are last to - glom together -
    • -
  • Eg. k-means find points which are "far" out from centroids -
      -
    • Determining "far", "last" can be application specific, part of the - challenge -
      • -
    - -
    • -
  • What algorithm could we use to automatically label nearby points? k-nearest - neighbor -
    • -
  • Statistical: Again, must define "low" in your domain -
    • -
  • Proximity: basically translating features into another, relative, space, - then applying a different type of outlier detection (eg. statistical) -
    • -
- - -
-
- -
- -
-

8 Statistical    slide

-
- -
    -
  • Assume a distribution -
    • -
  • Determine parameters -
    • -
  • Calculate probability of a point be generated by distribution -
    • -
- - -
- -
-

8.1 Why Statistical    notes

-
- -
    -
  • We've covered supervised, clustering, so let's skip to statistical methods -
    • -
  • Most straight forward way is to use distributions -
    • -
- - -
- -
- -
-

8.2 Statistical Example    slide two_col

-
- -
    -
  • Assume normal distribution -
    • -
  • Determine mean and standard distribution -
    • -
  • If (point-mean)/stddev > 3, consider outlier -
    • -
- -

img/gaussian-simple.png -

-
- -
-

8.2.1 Pros/Cons    notes

-
- -
    -
  • Straight forward -
    • -
  • Can use % to intuitively motivate (3 stdevs is outside 99.7%) -
    • -
  • But must manually determine cut-off -
    • -
  • How do we know we got the parameters right? -
    • -
- - -
-
-
- -
- -
-

9 Grubb's Test    slide

-
- -
    -
  • Takes into account sample size; reliability of mean/stddev measurements -
    • -
  • Take Z-score of a point, assign to G -
    • -
  • Student t-test: used to measure the distribution of actual mean from a - sample -
    • -
- -

img/grubbs.png -

-
- -
-

9.1 Pros/Cons    notes

-
- -
    -
  • Z-score: abs(x-u)/s -
    • -
  • This isn't actually that different from measuring stddev -
    • -
  • But accounts for sample size, can express your confidence with alpha 95% (0.05) -
    • -
  • Not going to go into t-test/t-distribution here, but basically it helps - show where the mean likely is, given a set of sample data. -
    • -
- - -
-
- -
- -
-

10 Outlier Distance    slide animate

-
- -
    -
  • How to take outliers in > 1 dimension? -
    • -
  • Translate distance to 1 dimension, find outliers -
    • -
  • How to measure distance? -
    • -
- - -
- -
-

10.1 Limitations    notes

-
- -
    -
  • What are the limitations of the techniques we've seen? -
    • -
  • Limited to one dimension! Taking mean, stddev, etc. applies to 1 - dimension -
    • -
  • Euclidean: doesn't take into account dependent variables -
    • -
- - -
-
- -
- -
-

11 Mahalanobis Distance    slide two_col

-
- -
    -
  • y depends somewhat on x -
    • -
  • Euclidean distance measures all dimensions equally -
    • -
  • Use covariance matrix to normalize distances in each dimension -
    • -
  • Matrix in which E_i,j is the covariance of i, j dimensions -
    • -
- -

img/GaussianScatterPCA.png -

-
- -
-

11.1 Mahalanobis    notes

-
- -
    -
  • How to capture intuition that a distance along major axis is different than - along this minor axis? -
    • -
  • Expand this drawing into 3 dimensions -
    • -
  • Euclidean distance will equally weight something that is out in the z - direction as something that is along this primary scatter area -
    • -
- - -
- -
- -
-

11.2 Mahalanobis Definition    slide

-
- -
    -
  • Find mean vector -
    • -
  • Normalize by covariance -
    • -
- -

img/mahalanobis.png -

- -
- -
-

11.3 Some Math    notes

-
- -
    -
  • Some extra math tricks to make the units work out: -
    • -
  • We're taking the squared distance, then taking the square root -
    • -
  • DM has squared Mahalanobis distance defined -
    • -
  • What happens if we have no covariance? S is the Identity matrix -
    • -
- - -
-
- -
- -
-

12 Contextual Outliers    slide

-
- -
    -
  • Typically reduce scope to context, use global techniques -
    • -
  • Example: Calculate normal distribution for Berkeley weather -
    • -
  • Collective outliers: find collections, use as context -
    • -
- - -
- -
- -
-

13 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-04-26 09:40:08 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-04-26-Outliers.org b/slides/2013-04-26-Outliers.org deleted file mode 100644 index 15bd891..0000000 --- a/slides/2013-04-26-Outliers.org +++ /dev/null @@ -1,165 +0,0 @@ -* Outliers :slide: - -* Generative Model :slide:animate: - + "Real" model that produced original data points - + Our mission is to reproduce the original model - + Thus we have different techniques that can model different behavior -** Questions :notes: - + What is a "generative model"? - + What is data mining trying to discover? What is machine learning hoping to - reproduce? - + Why have different classifiers? Decision tree, Naive Bayes, etc? - -* Outliers :slide:two_col: - + Significant deviation - + Probably generated through a *different model* than the rest of the data - + Normal / Abnormal - [[file:img/outlier.jpg]] -** Intuitive :notes: - + We all have a pretty good intuitive understand of what outliers are - + Mathematically, you can express the variation as a different generative - model - + Normal / Abnormal data (be careful about using it in human contexts) - + img: http://enriquegortiz.com/wordpress/enriquegortiz/research/undergraduate/ - -* Outlier Types :slide: - + Global :: points which deviate from the rest of the *entire* data set. Point - anomalies - + Contextual :: points which deviate from their *peers*. Conditional outliers - + Collective :: points which deviate as a *group*, even though individual - points may not be considered outliers. - -* Which Type? :slide: - + Given class sizes at Berkeley: - + A day with 10 people in class - + A day with 7000 people in class - + 3 weeks of 15 people in *this* class - + Given Earth's temperatures: - + A day at 100°C - + 30 straight days of rain in Berkeley - + A day at 100°F - -* Types of Learning :slide:two_col: - + Supervised - + Unsupervised - + Semi-Supervised - - [[file:img/ml-large-icon.png]] -** :notes: - + Supervised :: learning from "gold standard" labels - + Unsupervised :: learning without labels - + Semi-Supervised :: infer more labels from a few, learn based on inferred + - labeled - + img :: https://www.coursera.org/course/ml - -* Outlier Methods :slide: - + Supervised :: Label outliers, treat as classification problem - + Unsupervised :: Cluster data, find points not clustered well - + Semi-Supervised :: Manually label few, find point nearby to automatically - label, then treat as classification - + Statistical :: Decide on a generative model / distribution, find points - which have a low probability of belonging - + Proximity :: Use relative distance to neighbors -** Features :notes: - + Some methods may be overlapping - + When developing features for classification, using relative features can - be helpful: eg. distance from mean - + Eg. Agglomerative clustering, find lone/small groups that are last to - glom together - + Eg. k-means find points which are "far" out from centroids - + Determining "far", "last" can be application specific, part of the - challenge - + What algorithm could we use to automatically label nearby points? k-nearest - neighbor - + Statistical: Again, must define "low" in your domain - + Proximity: basically translating features into another, relative, space, - then applying a different type of outlier detection (eg. statistical) - -* Statistical :slide: - + Assume a distribution - + Determine parameters - + Calculate probability of a point be generated by distribution -** Why Statistical :notes: - + We've covered supervised, clustering, so let's skip to statistical methods - + Most straight forward way is to use distributions - -** Statistical Example :slide:two_col: - + Assume normal distribution - + Determine mean and standard distribution - + If =(point-mean)/stddev > 3=, consider outlier - [[file:img/gaussian-simple.png]] -*** Pros/Cons :notes: - + Straight forward - + Can use % to intuitively motivate (3 stdevs is outside 99.7%) - + But must manually determine cut-off - + How do we know we got the parameters right? - -* Grubb's Test :slide: - + Takes into account sample size; reliability of mean/stddev measurements - + Take Z-score of a point, assign to =G= - + Student t-test: used to measure the distribution of *actual* mean from a - sample - [[file:img/grubbs.png]] -** Pros/Cons :notes: - + Z-score: =abs(x-u)/s= - + This isn't actually *that* different from measuring stddev - + But accounts for sample size, can express your confidence with alpha 95% (0.05) - + Not going to go into t-test/t-distribution here, but basically it helps - show where the mean likely is, given a set of sample data. - -* Outlier Distance :slide:animate: - + How to take outliers in > 1 dimension? - + Translate distance to 1 dimension, find outliers - + How to measure distance? -** Limitations :notes: - + What are the limitations of the techniques we've seen? - + Limited to one dimension! Taking mean, stddev, etc. applies to 1 - dimension - + Euclidean: doesn't take into account dependent variables - -* Mahalanobis Distance :slide:two_col: - + =y= depends somewhat on =x= - + Euclidean distance measures all dimensions equally - + Use *covariance matrix* to normalize distances in each dimension - + Matrix in which =E_i,j= is the covariance of =i=, =j= dimensions -[[file:img/GaussianScatterPCA.png]] -** Mahalanobis :notes: - + How to capture intuition that a distance along major axis is different than - along this minor axis? - + Expand this drawing into 3 dimensions - + Euclidean distance will equally weight something that is out in the =z= - direction as something that is along this primary scatter area - -** Mahalanobis Definition :slide: - + Find mean vector - + Normalize by covariance - [[file:img/mahalanobis.png]] -** Some Math :notes: - + Some extra math tricks to make the units work out: - + We're taking the squared distance, then taking the square root - + DM has *squared* Mahalanobis distance defined - + What happens if we have no covariance? S is the Identity matrix - -* Contextual Outliers :slide: - + Typically reduce scope to context, use global techniques - + Example: Calculate normal distribution for Berkeley weather - + Collective outliers: find collections, use as context - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-05-03-D3.html b/slides/2013-05-03-D3.html deleted file mode 100644 index af3e5ba..0000000 --- a/slides/2013-05-03-D3.html +++ /dev/null @@ -1,237 +0,0 @@ - - - - -2013-05-03-D3 - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-05-03-D3

- - - - -
-

1 Develop a Visualization    slide

-
- -
    -
  • Learn D3 -
    • -
  • Extract data from your project -
    • -
  • Build a visualization of it -
    • -
- - -
- -
- -
-

2 Deliverable    slide

-
- -
    -
  • HTML with visualization -
    • -
  • Dependencies -
    • -
- - -
- -
- -
-

3 Example    slide

-
- -
    -
  • code/histogram.html -
    • -
- - - - -
# extract 2000 reviews
-grep 'type": "review' ../yelp_phoenix_academic_dataset/yelp_academic_dataset_review.json | head -n 200 > reviews-200.json
-
- - -
import json
-with open("reviews-200.json") as f:
-    r = map(json.loads, f)
-# extract star ratings
-[rev['stars'] for rev in r]
-
- - -
- -
-

3.1 Notes    notes

-
- -
    -
  • Include notes on how you extracted your data -
    • -
  • Notes can be in a separate files, or comments in the code -
    • -
  • My notes might be these shell/python commands -
    • -
- - -
-
- -
- -
-

4 Partners    slide

-
- -
    -
  • If you've never written JS on your own: -
    • -
  • Find someone familiar with D3 -
    • -
  • Still must turn in separate homeworks -
    • -
  • Cite your sources -
    • -
- - -
- -
-

4.1 Javascript    notes

-
- -
    -
  • Since this class is not teaching Javascript, you'll need to learn on your - own -
    • -
  • Special case: find someone to help you learn -
    • -
  • Folks who know D3: this assignment is not a challenge, so please find - someone to help. Teaching is a great way to learn -
    • -
- - -
-
- -
- -
-

5 Extra Credit    slide

-
- -
    -
  • [[http://trifacta.github.io/vega/][Vega] is a JS visualization Grammer -
    • -
  • Write homework in Vega instead -
    • -
- - -
- -
-

5.1 Closer to Grammar    notes

-
- -
    -
  • Vega is a declarative way of specifying a graphic -
    • -
  • Uses D3 underneath -
    • -
- - -
-
- -
- -
-

6 D3 Intro    slide

-
- - - - - - - - - -
-
-
- -
-

Date: 2013-05-03 09:41:43 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-05-03-D3.org b/slides/2013-05-03-D3.org deleted file mode 100644 index a847baf..0000000 --- a/slides/2013-05-03-D3.org +++ /dev/null @@ -1,65 +0,0 @@ -* Develop a Visualization :slide: - + Learn D3 - + Extract data from your project - + Build a visualization of it - -* Deliverable :slide: - + HTML with visualization - + Dependencies - -* Example :slide: - + =code/histogram.html= -#+begin_src shell -# extract 2000 reviews -grep 'type": "review' ../yelp_phoenix_academic_dataset/yelp_academic_dataset_review.json | head -n 200 > reviews-200.json -#+end_src -#+begin_src python -import json -with open("reviews-200.json") as f: - r = map(json.loads, f) -# extract star ratings -[rev['stars'] for rev in r] -#+end_src -** Notes :notes: - + Include notes on how you extracted your data - + Notes can be in a separate files, or comments in the code - + My notes might be these shell/python commands - -* Partners :slide: - + If you've *never* written JS on your own: - + Find someone familiar with D3 - + Still must turn in separate homeworks - + Cite your sources -** Javascript :notes: - + Since this class is not teaching Javascript, you'll need to learn on your - own - + Special case: find someone to help you *learn* - + Folks who know D3: this assignment is not a challenge, so please find - someone to help. Teaching is a great way to learn - -* Extra Credit :slide: - + [[http://trifacta.github.io/vega/][Vega] is a JS visualization Grammer - + Write homework in Vega instead -** Closer to Grammar :notes: - + Vega is a declarative way of specifying a graphic - + Uses D3 underneath - -* D3 Intro :slide: - + [[http://vogievetsky.github.io/IntroD3][D3 Intro]] - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-05-03-Visualization.html b/slides/2013-05-03-Visualization.html deleted file mode 100644 index a4448dd..0000000 --- a/slides/2013-05-03-Visualization.html +++ /dev/null @@ -1,811 +0,0 @@ - - - - -2013-05-03-Visualization - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-05-03-Visualization

- - - - -
-

1 Visualization in Data Mining    slide

-
- - -
- -
- -
-

2 Your Brain    slide two_col

-
- -
    -
  • Pattern detector -
    • -
  • Visualizations help you search for possible models -
    • -
  • Help intuitively understand the data -
    • -
- -

img/memory-recall.png -

-
- -
-

2.1 Visual    notes

-
- -
    -
  • Most people, vision is the strongest sense -
    • -
  • Recall improves 55% (10%=>65%) with addition of a picture -
    • -
  • We've talked about the need to understand the data before using - algorithms on it. Visualization can speed that process up. -
    • -
- - -
-
- -
- -
-

3 Patterns    slide

-
- -
    -
  • Use visualizations that surface patterns and relationships -
    • -
  • Know the context for the visualization -
    • -
  • Verify results -
    • -
- - -
- -
-

3.1 Steps    notes

-
- -
    -
  • For gaining intuition, focus on simple visualizations that help you see - relationships in the data. -
    • -
  • At this time, labels, titles, etc. not very important. Multiple dimension - in multiple windows? Fine! -
    • -
  • We'll discuss, but the context a visualization is going to be used in - matters a lot. Don't feel like you have to import every cool infographic - into your project -
    • -
  • Clustering, classification, outlier selection can be verified visually, eg. - highlighting points. Use it to gut check conclusions, even if you have to - drastically reduce dimensionality -
    • -
- - -
-
- -
- -
-

4 Scatter    slide

-
- -
    -
  • Great for multidimensional data -
    • -
  • Just plot > 2 dimensions in different plots -
    • -
  • Reveals correlation, clustering, distribution, … -
    • -
- - -
- -
-

4.1 Data Mining    notes

-
- -
    -
  • DM bread and butter. Often deal with high dimensionality, so scatter is one - of the best ways to visualize -
    • -
  • Wide variety of patterns can be searched -
    • -
- - -
- -
- -
-

4.2 Multiple Dimensions    slide center

-
- -

img/vp-sample.png -

- -
- -
-

4.3 vp    notes

-
- -
    -
  • This data is for body positions over time -
    • -
  • Dimensions are the different angles for different body parts, like hip - ankle, knee, over time -
    • -
  • We can see some strong patterns. Maybe we'll need to kernelize them to - make them learnable, but we have a good understanding that there are, or - are not relationships between the data -
    • -
- - -
-
- -
- -
-

5 Geographic    slide

-
- -

img/cancer-county.jpg -

-
- -
-

5.1 Trade-offs    notes

-
- -
    -
  • Coordinates intuitively understandable -
    • -
  • Lots of ways to bucket/aggregate -
    • -
  • Dependence on geographical area (eg. when you'd like to depend - on human impact instead) -
    • -
- - -
-
- -
- -
-

6 Other Chart Types    slide

-
- -
-
Box plot
aggregate data -
-
Bar charts
simple summaries -
-
Pie charts
compound proportions -
-
- - -
- -
-

6.1 Types    notes

-
- -
    -
  • Box plots, for real data, still carry a lot of data -
    • -
  • Bar charts nice for summarizing, not great for exploring -
    • -
  • Same for pie charts. Pie charts are mostly bad, but can use in particular - circumstances -
    • -
- - -
-
- -
- -
-

7 Aesthetics    slide

-
- -
    -
  • The visual aesthetics you use should be tied to the data -
    • -
- -

img/graphics-aesthetics.png -

-
- -
-

7.1 Aesthetics    notes

-
- -
    -
  • What are some of the techniques we can use to tie data to a visual - representation? -
    • -
  • img: Kevin Lynagh, http://keminglabs.com/talks/ -
    • -
- - -
- -
- -
-

7.2 Larger Value?    slide

-
- -
    -
  • Position -
    • -
  • Length / Angle -
    • -
  • Area / Volume -
    • -
  • Color: Chroma Luminance -
    • -
- -
- -
- -
-

7.3 Slide Switch    notes

-
- -
    -
  • Hadley Wickham slides, OSCON -
    • -
- - -
-
- -
- -
-

8 Color: HCL    slide two_col

-
- -
-
Hue
color type, relative to RGBY -
-
Chroma
colorfulness, perceived color intensity -
-
Luminosity
brightness, light-dark -
-
- -

img/Munsell.png -

-
- -
-

8.1 Color Spaces    notes

-
- - - - -
- -
- -
-

8.2 ColorBrewer    slide

-
- - - - -
-
- -
- -
-

9 Careful    slide

-
- - - - -
- -
-

9.1 Line Lengths    notes

-
- -
    -
  • Line lengths can appear to look smaller when extended instead of right - next to each other -
    • -
- - -
- -
- -
-

9.2 Careful    slide

-
- - - - -
- -
- -
-

9.3 Careful    slide

-
- - -

Motion silences awareness of color changes from Jordan Suchow on Vimeo.

- -
-
- -
- -
-

10 Grammar of Graphics    slide

-
- -
-
Geom
Graphic element -
-
Aesthetics
appearance of a geom -
-
Data
raw, context, statistical aggregations of data -
-
Mapping
functions which map data to geom properties or aesthetics -
-
- - -
- -
-

10.1 Bringing Together    notes

-
- -
    -
  • We've talked about different aesthetics of showing data, we've talked about - data, all that's needed is to bring them together -
    • -
  • Wilkinson, L. (2005), The Grammar of Graphics (2nd ed.). Statistics and Computing, New York: Springer. -
    • -
  • Rigorous way of describing graphics beyond "scatter plot" or "bar chart" -
    • -
- - -
-
- -
- -
-

11 Scatter Plot    slide animate

-
- -

img/scatter-ice-cream.gif -

-
    -
  • Geoms? -
      -
    • points, tick marks -
      • -
    - -
    • -
  • Data? -
      -
    • temperature, sales -
      • -
    - -
    • -
  • Mapping? -
      -
    • sales -> y, temp -> x -
      • -
    • Note, not a simple 1:1 mapping, we must map to something visual, like - pixels -
      • -
    - -
    • -
- - -
- -
-

11.1 Ice Cream    notes

-
- -
    -
  • Plot shows hypothetical sales of ice cream vs temperature -
    • -
  • Geoms: points (actually, ticks are geoms, too) -
    • -
  • Data: sales, temperature (and context: how large is the potential plot - size) -
    • -
  • Mapping: sales -
    • -
  • img: http://www.mathsisfun.com/data/scatter-xy-plots.html -
    • -
- - -
-
- -
- -
-

12 Bar Plot    slide animate

-
- -

img/bar-graph-fruit.gif -

-
    -
  • Geoms? -
      -
    • rectangles (ticks, text) -
      • -
    - -
    • -
  • Data? -
      -
    • Fruit to popularity -
      • -
    - -
    • -
  • Mapping? -
      -
    • popularity -> height, fruit type -> x, color -
      • -
    - -
    • -
- - -
- -
-

12.1 Fruit    notes

-
- - - - -
-
- -
- -
-

13 Hipmonk    slide

-
- -

img/hipmonk.png -

-
    -
  • Geoms? -
      -
    • rectangles, text, ticks, -
      • -
    - -
    • -
  • Data? -
      -
    • Carrier, flight time, layover time, cost, wifi available, airports -
      • -
    - -
    • -
  • Mapping? -
      -
    • travel time -> bar length, flight times -> sub-bars, "agony" -> y, airline -> color -
      • -
    - -
    • -
- - -
- -
-

13.1 Fruit    notes

-
- -
    -
  • Shows travel options from SFO to Ithica, connecting flights, airports, etc. -
    • -
  • More complex, but still expressible via Grammar -
    • -
  • img: http://www.hipmonk.com -
    • -
- - -
-
- -
- -
-

14 Recursive    slide

-
- -

img/grammar-af.png -

-
    -
  • Geoms? -
    • -
- - -
- -
-

14.1 Complex    notes

-
- -
    -
  • Reading will go a further extension of this, where the geoms are themselves - other plots -
    • -
- - -
-
- -
- -
-

15 Tufte    slide

-
- -
    -
  • Clarity from data -
    • -
  • Avoid chart junk -
    • -
  • Techniques for displaying many types -
    • -
- -

img/tufte-books.jpg -

-
- -
-

15.1 Tufte    notes

-
- -
    -
  • No talk on visualization would be complete without mentioning Tufte -
    • -
  • Great examples -
    • -
- - -
-
- -
- -
-

16 Break    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-05-03 12:26:13 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-05-03-Visualization.org b/slides/2013-05-03-Visualization.org deleted file mode 100644 index 4bcb739..0000000 --- a/slides/2013-05-03-Visualization.org +++ /dev/null @@ -1,214 +0,0 @@ -* Visualization in Data Mining :slide: - -* Your Brain :slide:two_col: - + Pattern detector - + Visualizations help you search for possible models - + Help intuitively understand the data - [[file:img/memory-recall.png]] -** Visual :notes: - + Most people, vision is the strongest sense - + Recall improves 55% (10%=>65%) with addition of a picture - + We've talked about the need to understand the data before using - algorithms on it. Visualization can speed that process up. - -* Patterns :slide: - + Use visualizations that surface patterns and relationships - + Know the context for the visualization - + Verify results -** Steps :notes: - + For gaining intuition, focus on simple visualizations that help you see - relationships in the data. - + At this time, labels, titles, etc. not very important. Multiple dimension - in multiple windows? Fine! - + We'll discuss, but the context a visualization is going to be used in - matters a lot. Don't feel like you have to import every cool infographic - into your project - + Clustering, classification, outlier selection can be verified visually, eg. - highlighting points. Use it to gut check conclusions, even if you have to - drastically reduce dimensionality - -* Scatter :slide: - + Great for multidimensional data - + Just plot > 2 dimensions in different plots - + Reveals correlation, clustering, distribution, ... -** Data Mining :notes: - + DM bread and butter. Often deal with high dimensionality, so scatter is one - of the best ways to visualize - + Wide variety of patterns can be searched - -** Multiple Dimensions :slide:center: - [[file:img/vp-sample.png]] -** vp :notes: - + This data is for body positions over time - + Dimensions are the different angles for different body parts, like hip - ankle, knee, over time - + We can see some strong patterns. Maybe we'll need to kernelize them to - make them learnable, but we have a good understanding that there are, or - are not relationships between the data - -* Geographic :slide: - [[file:img/cancer-county.jpg]] -** Trade-offs :notes: - + Coordinates intuitively understandable - + Lots of ways to bucket/aggregate - + Dependence on geographical area (eg. when you'd like to depend - on human impact instead) - -* Other Chart Types :slide: - + Box plot :: aggregate data - + Bar charts :: simple summaries - + Pie charts :: compound proportions -** Types :notes: - + Box plots, for real data, still carry a lot of data - + Bar charts nice for summarizing, not great for exploring - + Same for pie charts. Pie charts are mostly bad, but can use in particular - circumstances - -* Aesthetics :slide: - + The visual aesthetics you use should be tied to the data - [[file:img/graphics-aesthetics.png]] -** Aesthetics :notes: - + What are some of the techniques we can use to tie data to a visual - representation? - + img: Kevin Lynagh, http://keminglabs.com/talks/ - -** Larger Value? :slide: - + Position - + Length / Angle - + Area / Volume - + Color: Chroma Luminance -** Slide Switch :notes: - + Hadley Wickham slides, OSCON - -* Color: HCL :slide:two_col: - + Hue :: color type, relative to RGBY - + Chroma :: colorfulness, perceived color intensity - + Luminosity :: brightness, light-dark - [[file:img/Munsell.png]] -** Color Spaces :notes: - + Many other color spaces, probably most familiar with RGB - + HCL is useful because it separates the properties of a color into ones - that can be mapped to data - + Hue: nominal, can't compare - + Chroma, Luminosity: numerical / comparable value - + Chroma vs Saturation: chroma *perception* relative to white, saturation - measure of color intensity - + http://rourkevisualart.com/wordpress/2008/02/22/the-difference-between-chroma-and-saturation/ - -** ColorBrewer :slide: - + http://colorbrewer2.org/ - + Type of comparison => type of color difference - + Lots of other practical features - -* Careful :slide: - + Some aesthetics can combine to form illusions - + http://www.michaelbach.de/ot/sze_sineIllusion/ -** Line Lengths :notes: - + Line lengths can appear to look smaller when extended instead of right - next to each other - -** Careful :slide: -#+BEGIN_HTML - -#+END_HTML - -** Careful :slide: -#+BEGIN_HTML -

Motion silences awareness of color changes from Jordan Suchow on Vimeo.

-#+END_HTML - -* Grammar of Graphics :slide: - + Geom :: Graphic element - + Aesthetics :: appearance of a geom - + Data :: raw, context, statistical aggregations of data - + Mapping :: functions which map data to geom properties or aesthetics -** Bringing Together :notes: - + We've talked about different aesthetics of showing data, we've talked about - data, all that's needed is to bring them together - + Wilkinson, L. (2005), The Grammar of Graphics (2nd ed.). Statistics and Computing, New York: Springer. - + Rigorous way of describing graphics beyond "scatter plot" or "bar chart" - -* Scatter Plot :slide:animate: - [[file:img/scatter-ice-cream.gif]] - - + Geoms? - + points, tick marks - + Data? - + temperature, sales - + Mapping? - + sales -> y, temp -> x - + Note, not a simple 1:1 mapping, we must map to something visual, like - pixels -** Ice Cream :notes: - + Plot shows hypothetical sales of ice cream vs temperature - + Geoms: points (actually, ticks are geoms, too) - + Data: sales, temperature (and context: how large is the potential plot - size) - + Mapping: sales - + img: http://www.mathsisfun.com/data/scatter-xy-plots.html - -* Bar Plot :slide:animate: - [[file:img/bar-graph-fruit.gif]] - - + Geoms? - + rectangles (ticks, text) - + Data? - + Fruit to popularity - + Mapping? - + popularity -> height, fruit type -> x, color -** Fruit :notes: - + Plot shows fruit popularity - + Geoms: bars (and ticket, text) - + Data: - + Mapping: sales - + img: http://www.mathsisfun.com/data/bar-graphs.html - -* Hipmonk :slide: - [[file:img/hipmonk.png]] - - + Geoms? - + rectangles, text, ticks, - + Data? - + Carrier, flight time, layover time, cost, wifi available, airports - + Mapping? - + travel time -> bar length, flight times -> sub-bars, "agony" -> y, airline -> color -** Fruit :notes: - + Shows travel options from SFO to Ithica, connecting flights, airports, etc. - + More complex, but still expressible via Grammar - + img: http://www.hipmonk.com - -* Recursive :slide: - [[file:img/grammar-af.png]] - - + Geoms? -** Complex :notes: - + Reading will go a further extension of this, where the geoms are themselves - other plots - -* Tufte :slide: - + Clarity from data - + Avoid chart junk - + Techniques for displaying many types - [[file:img/tufte-books.jpg]] -** Tufte :notes: - + No talk on visualization would be complete without mentioning Tufte - + Great examples - -* *Break* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-05-03-Yelp-Visualization.html b/slides/2013-05-03-Yelp-Visualization.html deleted file mode 100644 index 3348100..0000000 --- a/slides/2013-05-03-Yelp-Visualization.html +++ /dev/null @@ -1,1019 +0,0 @@ - - - - -2013-05-03-Yelp-Visualizaiton - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-05-03-Yelp-Visualizaiton

- - - - -
-

1 Visualizing Data at Yelp    slide center

-
- - -
- -
- -
-

2 Visualizing Data is Important    slide

-
- -
    -
  • Effectively summarizes data -
    • -
  • Highlights patterns -
    • -
  • Improves recall -
    • -
- - -
- -
-

2.1 Metrics    notes

-
- -
    -
  • Can't improve something till you measure -
    • -
  • True, but have to look at and understand the data! -
    • -
  • Often best way to understand data is visually -
    • -
  • Having metrics you care about evident will make you focus on improving - them -
    • -
  • More sophisticated your visualizations, more sophisticated your goals -
    • -
- - -
-
- -
- -
-

3 Visualizing Data is Difficult    slide

-
- -
    -
  • Requires investment -
    • -
  • Dimensions of success -
    • -
  • Successful visualizations in Yelp -
    • -
- - -
- -
-

3.1 Role in Yelp    notes

-
- -
    -
  • Often requires specific domain knowledge of both the data and the tools -
    • -
  • Moving to a new office -
    • -
  • Ideally have 2 big screens per pod -
    • -
  • "That's a lot of TVs!" -
    • -
  • Get motivated everyday -
    • -
  • Show what you care about -
    • -
  • Don't want a sterile office, decorate with the results of your work -
    • -
- - -
-
- -
- -
-

4 Birth of a City    slide

-
- - - - -
- -
-

4.1 Review activity over time on a map    notes

-
- -
    -
  • Written as part of Yelp's quarterly Hackathons -
    • -
  • A lot of feedback from our Community Managers on understanding their city -
    • -
  • Demonstrable value to advertisers -
    • -
  • But main feature is… Cool -
    • -
- - -
-
- -
- -
-

5 Cool    slide

-
- -
    -
  • Good looking is a dimension of any visualization -
    • -
  • We want to be Neo or John Anderton, not Milton -
    • -
- -

img/minority-report-int2.jpg -

-
- -
-

5.1 Cool is OK    notes

-
- -
    -
  • Engineers need to come to grips that to be visually compelling, a visualization needs to look nice -
    • -
  • Just like the most compelling novels need to be well written -
    • -
  • We realize this, we just don't like to admit it -
    • -
- - -
-
- -
- -
-

6 Avoid Chart Junk    slide two_col

-
- -
    -
  • Edward Tufte rightfully suspicious of cool -
    • -
  • Worry about data/ink ratio -
    • -
  • But remember tradeoffs: memorability, fun -
    • -
- -

img/usefuljunk-monster.jpg -

-
- -
-

6.1 Useful Junk?    notes

-
- -
    -
  • Data/ink ratio describes the amount of information displayed per ink/pixel -
    • -
  • If you remove a pixel, will you remove information? -
    • -
  • Best Paper by Scott Bateman HCI: some useful Junk -
    • -
  • Noted the context of the chart -
    • -
  • Bad ratio limits richness, especially important on mobile -
    • -
- - -
-
- -
- -
-

7 Grapperr    slide

-
- - - - -
- -
-

7.1 Shows errors live from log    notes

-
- -
    -
  • Error activity -
    • -
  • Highlight error type UnicodeDecodeError -
    • -
  • Text details available -
    • -
  • Still Cool! -
      -
    • Colors slick, modern -
      • -
    • But used for differentiation (data) -
      • -
    - -
    • -
- -
- -
- -
-

7.2 Grapperr Snapshot    slide

-
- -

img/graperr.png -

-
-
- -
- -
-

8 Actionable    slide two_col

-
- -
    -
  • Realtime* -
    • -
  • Context -
    • -
  • Connections -
    • -
- -

img/dr-who.png -

-
- -
-

8.1 Definitions    notes

-
- -
    -
  • As realtime as problem domain requires -
      -
    • Seconds matter when fixing site problems, so should be up to the second -
      • -
    • Days or weeks might matter when deciding budget issues -
      • -
    - -
    • -
  • Context: Is this a normal amount of errors? -
    • -
  • Connections: Ability to drill down to specific instance -
    • -
- - -
-
- -
- -
-

9 Dimensions    slide

-
- -
-
Fun
cool, pretty, engaging -
-
Actionable
realtime, contextual, connecting -
-
- - -
- -
-

9.1 Agenda    notes

-
- -
    -
  • Dimensions important to visualizations -
    • -
  • Axis on which you can evaluate them -
    • -
  • Tradeoffs in developing them -
    • -
- - -
-
- -
- -
-

10 A Tale of Two Datacenters    slide

-
- -
    -
  • Testing datacenter failover -
    • -
  • Tracking metrics in a new way -
    • -
  • Did we spend a week preparing a dashboard? -
    • -
- - -
- -
-

10.1 How?    notes

-
- -
    -
  • Yelp used to be in only one datacenter -
    • -
  • Moving to two datacenters is a huge undertaking, but worth it for - reliability reasons -
    • -
  • Don't want to bring down a worldwide site when freak electrical storms hit your datacenter -
    • -
  • After months of work, how did watch over our site when we finally flipped - the switch? -
    • -
  • This was the first time Yelp had done this: we didn't have a premade - dashboard so everyone could track the important metrics -
    • -
- - -
-
- -
- -
-

11 Firefly    slide

-
- - - -

-Github: Yelp/firefly -

-
- -
-

11.1 Demo    notes

-
- -
    -
  • One of our many open source projects -
    • -
  • Hosted on Github -
    • -
  • Existing extension to Ganglia -
    • -
- - -
-
- -
- -
-

12 Easy    slide

-
- -
    -
  • Make repeated operations fast and within reach -
    • -
  • Must understand problem domain -
    • -
  • Accessible -
    • -
- - -
- -
-

12.1 Definitions    notes

-
- -
    -
  • Sophisticated Tool: Data discovery, stacking options, coloring, layout -
    • -
  • But all of the steps are repeated, formulaic: we're making similar things over and over -
    • -
  • So make it easy! -
    • -
  • Not much more accessible than Web: share links, etc. -
    • -
- - -
-
- -
- -
-

13 Easy from Simple    slide

-
- -
    -
  • Avoid temptation to make visualizations easy from the start -
    • -
  • Easy systems are designed for non-experts -
    • -
  • Long term investment in the system to manage complexity -
    • -
- - -
- -
-

13.1 Non-experts    notes

-
- -
    -
  • Simple Made Easy, Rich Hickey -
    • -
  • Still potentially technical users -
    • -
  • Just don't know the details of how metrics are collected, or how to display - across browsers -
    • -
  • Always will require experts to make changes -
    • -
  • Always are going to want new features -
    • -
  • Make sure you have the ability to add them -
    • -
  • Not extensible -
    • -
- - -
-
- -
- -
-

14 Search Maps    slide

-
- -

img/yelp-beer.png -Mo' Map -

-
- -
-

14.1 Times Change    notes

-
- -
    -
  • 2005, 8 years ago -
    • -
  • May not seem like important visualization, but times have changed -
    • -
  • Full page refresh for each map square -
    • -
  • Now we take zoom in, panning for granted -
    • -
  • Sign of a great visualization: don't think about it: it's a tool -
    • -
  • What else are we not plotting on maps that we should be? -
    • -
- - -
-
- -
- -
-

15 Interactive    slide two_col

-
- -
    -
  • Fast -
    • -
  • Explorable -
    • -
  • Feedback -
    • -
- -

img/yelp-mobile-map.png -

-
- -
-

15.1 Definitions    notes

-
- -
-
Fast
One of the reasons its a fairly recent technology, hard to get fast -
    -
  • Speed gives the UI illusion that you are interacting with a physical - thing, something we're much more comfortable with -
    • -
- -
-
Explorable
Multiple levels of detail that can be discovered by user -
-
Feedback
Update all other dependent displays (search results) -
-
- - -
-
- -
- -
-

16 Creation    slide

-
- -
    -
  • Michael Bostock had a problem -
    • -
  • Protovis useful, but not flexible -
    • -
  • How to provide coherent description for visualizing data? -
    • -
- - -
- -
-

16.1 D3 Intro    notes

-
- -
    -
  • Mike Bostock professor at Stanford -
    • -
  • Protivis was a declarative Javascript charting library -
    • -
  • But hard to keep up with changes in technology -
    • -
  • Wasn't quite flexible enough for new visualizations -
    • -
- - -
-
- -
- -
-

17 D3: Data-Driven Documents    slide center

-
- - -

D3 Show Reel from Mike Bostock on Vimeo.

- -
- -
- -
-

18 Flexible    slide

-
- -
    -
  • Language level -
    • -
  • Access to medium -
    • -
  • Access to data -
    • -
- - - - -
d3.selectAll("p")
-  .data([4, 8, 15, 16, 23, 42])
-  .style("font-size",
-    function(d) { return d + "px"; });
-
- - -
- -
-

18.1 Why?    notes

-
- -
    -
  • Metaphor natural language -
    • -
  • General language most flexible tool humans have to describe new things -
    • -
  • Full access to medium to be able to create take advantage of all possibilities -
      -
    • and new tech -
      • -
    - -
    • -
  • Not D3 specific, but need full data to find new ways to summarize, explore, - drill -
    • -
  • Need to understand where data came from to clean, normalize -
    • -
- - -
-
- -
- -
-

19 Dimensions    slide

-
- -
-
Fun
cool, pretty, engaging -
-
Actionable
realtime, contextual, connecting -
-
Easy
available for non-experts, remove repetition -
-
Interactive
fast, explorable -
-
Flexible
expressive, full access to lowest level -
-
- - -
- -
-

19.1 Tension    notes

-
- -
    -
  • Obvious: Flexible vs Easy. Too many options is confusing. -
    • -
  • Less obvious: Interactive vs Actionable. Spend too long playing, not enough fixing -
    • -
  • In fact: All in contention for your time -
    • -
- - -
-
- -
- -
-

20 Understand Usage Context    slide

-
- - - -
- -
-

20.1 Press: Fun    slide

-
- -

img/yelp20m.jpg -

-
- -
- -
-

20.2 Alerting: Actionable    slide

-
- -

img/snoopy.png -

- -
- -
-

20.3 Search Metrics    notes

-
- -
    -
  • This is a visualization of the status of our search cluster -
    • -
- - -
- -
- -
-

20.4 Product Managers: Easy    slide

-
- -

img/admin-metrics.png -

-
- -
- -
-

20.5 Investigation: Interactive    slide

-
- -

img/ipy_0.13.png -img/IPy_header.png -

-
- -
- -
-

20.6 Explorable: Interactive    slide

-
- -

img/maptivity.png -

- -
- -
-

20.7 Another Case    notes

-
- -
    -
  • Another case for Interactivity is geographical data -
    • -
- - -
- -
- -
-

20.8 New tools: Simple    slide

-
- -

img/nvd3.png -

-
- -
- -
-

20.9 New tools: Flexible    slide

-
- -

img/tron-db.png -

- -
- -
-

20.10 Unique    notes

-
- -
    -
  • You can see this is not a standard visualization -
    • -
  • It is one that is customized to its purpose -
    • -
  • Made possible by flexible tools -
    • -
- - -
-
- -
- -
-

21 Dimensions    slide

-
- -
-
Fun
cool, pretty, engaging -
-
Actionable
realtime, contextual, connecting -
-
Easy
available for non-experts, remove repetition -
-
Interactive
fast, explorable -
-
Flexible
expressive, full access to lowest level -
-
- - -
- -
-

21.1 Consider Tradeoffs    notes

-
- -
    -
  • Visualization is just part of making an effective biz, team -
    • -
  • Interested in working at Yelp? -
    • -
- - - - - - - -
-
-
-
- -
-

Date: 2013-05-03 09:46:19 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-05-03-Yelp-Visualization.org b/slides/2013-05-03-Yelp-Visualization.org deleted file mode 100644 index 59be981..0000000 --- a/slides/2013-05-03-Yelp-Visualization.org +++ /dev/null @@ -1,275 +0,0 @@ -* Visualizing Data at *Yelp* :slide:center: - -* Visualizing Data is Important :slide: - + Effectively summarizes data - + Highlights patterns - + Improves recall -** Metrics :notes: - + Can't improve something till you measure - + True, but have to look at and understand the data! - + Often best way to understand data is visually - + Having metrics you care about evident will make you focus on improving - them - + More sophisticated your visualizations, more sophisticated your goals - -* Visualizing Data is Difficult :slide: - + Requires investment - + Dimensions of success - + Successful visualizations in Yelp -** Role in Yelp :notes: - + Often requires specific domain knowledge of both the data and the tools - + Moving to a new office - + Ideally have 2 big screens per pod - + "That's a lot of TVs!" - + Get motivated everyday - + Show what you care about - + Don't want a sterile office, decorate with the results of your work - -* Birth of a City :slide: -#+BEGIN_HTML - -#+END_HTML -** Review activity over time on a map :notes: - + Written as part of Yelp's quarterly Hackathons - + A lot of feedback from our Community Managers on understanding their city - + Demonstrable value to advertisers - + But main feature is... Cool - -* Cool :slide: - + Good looking is a dimension of any visualization - + We want to be Neo or John Anderton, not Milton -[[file:img/minority-report-int2.jpg]] -** Cool is OK :notes: - + Engineers need to come to grips that to be visually compelling, a visualization needs to look nice - + Just like the most compelling novels need to be well written - + We realize this, we just don't like to admit it - -* Avoid Chart Junk :slide:two_col: - + Edward Tufte rightfully suspicious of cool - + Worry about =data/ink= ratio - + But remember tradeoffs: memorability, fun - [[file:img/usefuljunk-monster.jpg]] -** Useful Junk? :notes: - + Data/ink ratio describes the amount of information displayed per ink/pixel - + If you remove a pixel, will you remove information? - + Best Paper by Scott Bateman HCI: some useful Junk - + Noted the context of the chart - + Bad ratio limits richness, especially important on mobile - -* Grapperr :slide: -#+BEGIN_HTML - -#+END_HTML -** Shows errors live from log :notes: - + Error activity - + Highlight error type UnicodeDecodeError - + Text details available - + Still Cool! - + Colors slick, modern - + But used for differentiation (data) -** Grapperr Snapshot :slide: -[[file:img/graperr.png]] - -* Actionable :slide:two_col: - + Realtime* - + Context - + Connections - [[file:img/dr-who.png]] -** Definitions :notes: - + As realtime as problem domain requires - + Seconds matter when fixing site problems, so should be up to the second - + Days or weeks might matter when deciding budget issues - + Context: Is this a normal amount of errors? - + Connections: Ability to drill down to specific instance - -* Dimensions :slide: - + Fun :: cool, pretty, engaging - + Actionable :: realtime, contextual, connecting -** Agenda :notes: - + Dimensions important to visualizations - + Axis on which you can evaluate them - + Tradeoffs in developing them - -* A Tale of Two Datacenters :slide: - + Testing datacenter failover - + Tracking metrics in a new way - + Did we spend a week preparing a dashboard? -** How? :notes: - + Yelp used to be in only one datacenter - + Moving to two datacenters is a huge undertaking, but worth it for - reliability reasons - + Don't want to bring down a worldwide site when freak electrical storms hit your datacenter - + After months of work, how did watch over our site when we finally flipped - the switch? - + This was the first time Yelp had done this: we didn't have a premade - dashboard so everyone could track the important metrics - -* Firefly :slide: -#+BEGIN_HTML - -#+END_HTML -[[https://github.com/Yelp/firefly][Github: Yelp/firefly]] -** Demo :notes: - + One of our many open source projects - + Hosted on Github - + Existing extension to Ganglia - -* Easy :slide: - + Make repeated operations fast and within reach - + Must understand problem domain - + Accessible -** Definitions :notes: - + Sophisticated Tool: Data discovery, stacking options, coloring, layout - + But all of the steps are repeated, formulaic: we're making similar things over and over - + So make it easy! - + Not much more accessible than Web: share links, etc. - -* Easy from Simple :slide: - + Avoid temptation to make visualizations easy from the start - + Easy systems are designed for non-experts - + Long term investment in the system to manage complexity -** Non-experts :notes: - + Simple Made Easy, Rich Hickey - + Still potentially technical users - + Just don't know the details of how metrics are collected, or how to display - across browsers - + Always will require experts to make changes - + Always are going to want new features - + Make sure you have the ability to add them - + Not extensible - -* Search Maps :slide: -[[file:img/yelp-beer.png]] -Mo' Map -** Times Change :notes: - + 2005, 8 years ago - + May not seem like important visualization, but times have changed - + Full page refresh for each map square - + Now we take zoom in, panning for granted - + Sign of a great visualization: don't think about it: it's a tool - + What else are we not plotting on maps that we should be? - -* Interactive :slide:two_col: - + Fast - + Explorable - + Feedback - [[file:img/yelp-mobile-map.png]] -** Definitions :notes: - + Fast :: One of the reasons its a fairly recent technology, hard to get fast - + Speed gives the UI illusion that you are interacting with a physical - thing, something we're much more comfortable with - + Explorable :: Multiple levels of detail that can be discovered by user - + Feedback :: Update all other dependent displays (search results) - -* Creation :slide: - + Michael Bostock had a problem - + Protovis useful, but not flexible - + How to provide coherent description for visualizing data? -** D3 Intro :notes: - + Mike Bostock professor at Stanford - + Protivis was a declarative Javascript charting library - + But hard to keep up with changes in technology - + Wasn't quite flexible enough for new visualizations - -* D3: Data-Driven Documents :slide:center: -#+BEGIN_HTML -

D3 Show Reel from Mike Bostock on Vimeo.

-#+END_HTML - -* Flexible :slide: - + Language level - + Access to medium - + Access to data -#+begin_src javascript -d3.selectAll("p") - .data([4, 8, 15, 16, 23, 42]) - .style("font-size", - function(d) { return d + "px"; }); -#+end_src -** Why? :notes: - + Metaphor natural language - + General language most flexible tool humans have to describe new things - + Full access to medium to be able to create take advantage of all possibilities - + and new tech - + Not D3 specific, but need full data to find new ways to summarize, explore, - drill - + Need to understand where data came from to clean, normalize - -* Dimensions :slide: - + Fun :: cool, pretty, engaging - + Actionable :: realtime, contextual, connecting - + Easy :: available for non-experts, remove repetition - + Interactive :: fast, explorable - + Flexible :: expressive, full access to lowest level -** Tension :notes: - + Obvious: Flexible vs Easy. Too many options is confusing. - + Less obvious: Interactive vs Actionable. Spend too long playing, not enough fixing - + In fact: All in contention for your time - -* Understand Usage Context :slide: - -** Press: Fun :slide: -[[file:img/yelp20m.jpg]] - -** Alerting: Actionable :slide: -[[file:img/snoopy.png]] -** Search Metrics :notes: - + This is a visualization of the status of our search cluster - -** Product Managers: Easy :slide: -[[file:img/admin-metrics.png]] - -** Investigation: Interactive :slide: -[[file:img/ipy_0.13.png]] -[[file:img/IPy_header.png]] - -** Explorable: Interactive :slide: -[[file:img/maptivity.png]] -** Another Case :notes: - + Another case for Interactivity is geographical data - -** New tools: Simple :slide: -[[file:img/nvd3.png]] - -** New tools: Flexible :slide: -[[file:img/tron-db.png]] -** Unique :notes: - + You can see this is not a standard visualization - + It is one that is customized to its purpose - + Made possible by flexible tools - -* Dimensions :slide: - + Fun :: cool, pretty, engaging - + Actionable :: realtime, contextual, connecting - + Easy :: available for non-experts, remove repetition - + Interactive :: fast, explorable - + Flexible :: expressive, full access to lowest level -** Consider Tradeoffs :notes: - + Visualization is just part of making an effective biz, team - + Interested in working at Yelp? - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2013-05-10-Real-World.html b/slides/2013-05-10-Real-World.html deleted file mode 100644 index 00cbc65..0000000 --- a/slides/2013-05-10-Real-World.html +++ /dev/null @@ -1,1497 +0,0 @@ - - - - -2013-05-10-Real-World - - - - - - - - - - - - - - - - - - - -
- -
- -
-

2013-05-10-Real-World

- - - - -
-

1 Datamining IRL    slide

-
- - -
- -
- -
-

2 General vs. Practical    slide

-
- -
    -
  • "the language in which you'll spend most of your working life hasn't been - invented yet, so we can't teach it to you. Instead we have to give you the - skills you need to learn new languages as they appear." -
    • -
  • Brian Harvey, Why SCIP matters -
    • -
- - -
- -
-

2.1 Important Parts    notes

-
- -
    -
  • Important: understanding your domain, asking interesting questions, - answering them with data, fitting questions to mathematical concepts -
    • -
  • Not as important: scipy lingregress, d3 -
    • -
- - -
-
- -
- -
-

3 But    slide

-
- -
    -
  • Who is doing interviews? -
    • -
  • Who is starting their own company? -
    • -
  • Who wants to build their portfolio? -
    • -
- - -
- -
-

3.1 Real World    notes

-
- -
    -
  • I realize many of you may be needing to apply this stuff very soon -
    • -
  • So here's the lecture where I try to tell you what I would do when - practicing data mining -
    • -
- - -
-
- -
- -
-

4 Product or Company Ideas    slide

-
- -
    -
  • Understand exponential growth -
    • -
  • Get the timing right -
    • -
  • Execute, execute, execute -
    • -
- - -
- -
-

4.1 Idea required but not sufficient    notes

-
- -
    -
  • Wide variation of thoughts on ideas -
    • -
  • One of the biggest road blocks for wanna be entrepreneurs, but most derided -
    • -
  • Timing is mostly luck, is the world ready for your ideas? -
    • -
  • I think the vision is important, it's what drives the company, keeps - people working -
    • -
  • The plan for the idea less so -
    • -
  • The concrete product most important -
    • -
  • Let's talk about these elements -
    • -
- - -
-
- -
- -
-

5 Toys    slide

-
- -
    -
  • "the next big thing always starts out being dismissed as a “toy.”" -
    • -
  • Chris Dixon, Blog -
    • -
- -

img/cdixon.jpg -

-
- -
-

5.1 Why?    notes

-
- - - - -
- -
- -
-

5.2 Exponential Improvements    slide two_col

-
- -
    -
  • The future is changing at a faster rate than ever before -
    • -
  • Every field is being quantized, instrumented -
    • -
  • "You're not analyzing your data?" -
    • -
- - -

- img/cost_per_megabase.jpg -

-
- -
-

5.2.1 Change to Data    notes

-
- -
    -
  • You think your parents are bad at not operating devices? Your habits will - become out of date > twice as fast. -
    • -
  • So what happens when we combine exponential improvements and instrumented - fields? Overwhelming amounts of data -
    • -
  • Winners will be the ones -
    • -
  • Obviously web recommendations: data, but also -
    • -
- - -
-
- -
- -
-

5.3 Exercise    slide two_col

-
- -

img/strava.jpg -

-

- img/strava-ride.png -

-
- -
-

5.3.1 Strava    notes

-
- -
    -
  • No device, just data -
    • -
- - -
-
- -
- -
-

5.4 Thermostats    slide center

-
- -

img/nest.jpg -

-
- -
-

5.4.1 Nest    notes

-
- -
    -
  • OK, they have a device… but what separates them is data -
    • -
- - -
-
- -
- -
-

5.5 Biology    slide center

-
- -

img/cost_per_megabase.jpg -

-
- -
-

5.5.1 Gene sequencing    notes

-
- - - - -
-
-
- -
- -
-

6 Execute    slide

-
- -
    -
  • Users are hiring you to do a job: what is it? -
    • -
  • "Institutions will try to preserve the problem to which they are the solution." – Clay Shirky -
    • -
  • Make your product so easy to use, people do it by accident. -
    • -
- - -
- -
-

6.1 Do the job    notes

-
- -
    -
  • All of these company examples, you're typically not thinking of them as - "data processors"… they are solving a specific problem for you -
    • -
  • Strava isn't doing any crazy SVM analytics (at least on the consumer facing - side): they're showing you min/max, avg speed. Simple, but effective, - stuff. -
    • -
  • Disruption most often comes from using established technologies in new ways - or areas -
    • -
  • Can disrupt by completely simplified, often crappy at first, solutions to - an even more fundamental problem -
    • -
  • Dell did a great job selling cheap computers, then more expensive computers -
    • -
  • But now Amazon is saying: "you don't even need to own computers!" (Cloud) -
    • -
  • More info: Clayton Christensen -
    • -
  • Focus on that one thing that is important and do it very, very well -
    • -
- - -
-
- -
- -
-

7 Specifics (The Joel Test)    slide

-
- -
    -
  • Do you use source control? -
    • -
  • Can you make a build in one step? -
    • -
  • Do you make daily builds? -
    • -
  • Do you have a bug database? -
    • -
  • Do you fix bugs before writing new code? -
    • -
  • Do you have an up-to-date schedule? -
    • -
  • Do you have a spec? -
    • -
  • Do programmers have quiet working conditions? -
    • -
  • Do you use the best tools money can buy? -
    • -
  • Do you have testers? -
    • -
  • Do new candidates write code during their interview? -
    • -
  • Do you do hallway usability testing? -
    • -
- - -
- -
-

7.1 Joel on Software    notes

-
- -
    -
  • When developing software, please follow as many of these as reasonable -
    • -
  • Joel Spolksy wrote this in 2000! Still a great guide! -
    • -
  • This is what I'd suggest to quickly get moving on the right foot -
    • -
  • If you're managing a team, make sure these are happening -
    • -
- - -
- -
- -
-

7.2 Source Control    slide center

-
- -

img/git.png -

-
- -
-

7.2.1 Surprised?    notes

-
- -
    -
  • Github will solve a few problems on this list, just use it, even if you're - developing alone -
    • -
- - -
-
- -
- -
-

7.3 One step build    slide

-
- -
    -
  • Data mining exploration often involves manual commands -
    • -
  • Don't do that in production -
    • -
  • Should have scripts which extract features, build model, verify, deploy -
    • -
- - -
- -
-

7.3.1 Area for Improvement    notes

-
- -
    -
  • This is actually a big area solutions -
    • -
  • Deploying websites has solutions like Heroku, but no equivilant for - storing, processing, serving data -
    • -
- - -
-
- -
- -
-

7.4 Bug Database    slide

-
- -
    -
  • Easy to loose track of problems -
    • -
  • Also good way to prioritize issues -
    • -
  • Use Github Issues -
    • -
- - -
- -
-

7.4.1 Managing Up    notes

-
- -
    -
  • Good defense -
    • -
- - -
-
- -
- -
-

7.5 Write a Spec    slide

-
- -
    -
  • Alternatively, write the press release -
    • -
  • Don't write a novel -
    • -
  • Disagreements can be solved with code, but after talking -
    • -
- - -
- -
-

7.5.1 Bad rap    notes

-
- -
    -
  • Developers don't like writing them much -
    • -
  • But it helps nail down issues -
    • -
  • Yelp uses CEP process -
    • -
  • If you get to the "agree to disagree" point, data or code can solve - differences -
    • -
- - -
-
- -
- -
-

7.6 Testers    slide

-
- -
    -
  • Use unit tests to test code (eg. unittest2 in Python) -
    • -
  • Use cross-validation to test models -
    • -
  • Very easy to skip, will bite you within 6 months -
    • -
- -
- -
- -
-

7.7 Differences    notes

-
- -
    -
  • Joel suggests having and paying testers -
    • -
  • I don't think this is best use of resources for small companies -
    • -
  • Economics change when developers can effectively write tests -
    • -
  • Must allocate time to this -
    • -
  • Add tests when you fix bugs -
    • -
  • Helps if developers use product daily -
    • -
- - -
- -
- -
-

7.8 Tools    slide

-
- -
    -
  • Right tool for the job -
    • -
  • Text Editor: Use vim or emacs -
    • -
  • virtualenv (Python); RVM (Ruby) -
    • -
  • Learn the command line -
    • -
- - -
- -
-

7.8.1 Woodworker    notes

-
- -
    -
  • (slightly off topic from Joel's list) -
    • -
  • Woodworkers don't hammer stuff in with their shoe -
    • -
  • Make their own tools as first part of job -
    • -
  • When a custom problem comes up, make a custom tool -
    • -
  • These slides, written with mappings in vim -
    • -
  • Text Editor -
      -
    • Syntax Highlighting -
      • -
    • Macros -
      • -
    • Interact with other tools -
      • -
    • Find across files -
      • -
    - -
    • -
- - -
-
-
- -
- -
-

8 How to Use Recommendations    slide two_col

-
- -
    -
  • Start with them as default -
    • -
  • If you understand why something is better for your case, use it -
    • -
  • Understand trade-offs -
    • -
- -

img/grain-of-salt.jpg -

-
- -
-

8.1 Trade-offs    notes

-
- -
    -
  • One of the themes of this course -
    • -
  • Trying to provide you with a starting point -
    • -
  • My point of view: user driven behavior, engineers implementing solutions -
    • -
- - -
-
- -
- -
-

9 Data Storage    slide

-
- -
    -
  • S3 for unstructured data -
    • -
  • PostgreSQL for structured -
    • -
  • Hive on S3 for very large structured data -
    • -
- - -
- -
-

9.1 Data most important asset    notes

-
- -
    -
  • S3 is a pay-as-you go model, opens up many data processing possibilities -
    • -
  • Don't have to worry about how to connect -
    • -
  • PostgreSQL solid database, but also offers many improvements like storing - geo data -
    • -
  • Once you get beyond PostgreSQL limits, use Hive to structure data in S3 -
    • -
- - -
-
- -
- -
-

10 Exploration    slide two_col

-
- -
    -
  • Python -
    • -
  • IPython Notebook, matplotlib -
    • -
- -

img/ipython-notebook.jpg -

-
- -
-

10.1 Py    notes

-
- -
    -
  • Main reason: it is convenient and practical to stay in the same language as - production -
    • -
  • Using production libraries, settings, to extract data -
    • -
  • R, matlab/octive, Tableau are typically not used in large production code -
    • -
  • SAS also effective for exploration, can be used in production, but skill - set not as transferable for smaller companies -
    • -
- - -
-
- -
- -
-

11 Public Visualizations    slide

-
- -
    -
  • D3 for visualizations -
    • -
  • HTML is sharable, universal -
    • -
  • (Adventurous: Vega) -
    • -
- - -
- -
-

11.1 Visualization    notes

-
- -
    -
  • Vega more directly maps to grammar of graphics, but is very new library -
    • -
- - -
-
- -
- -
-

12 Processing    slide two_col

-
- -
    -
  • Hadoop + mrjob -
    • -
  • Elastic MapReduce -
    • -
  • (Adventurous: Spark -
    • -
- -

img/hadoop.png -

-
- -
-

12.1 Scaling    notes

-
- -
    -
  • Hadoop scales up and down fairly well, especially with mrjob -
    • -
  • Constraints are going to be on your time, not necessary to eek out every - bit of computing poser -
    • -
  • Spark is a new model out of Berkeley that does a better job of keeping data - in memory, but doesn't have the maturity of Hadoop -
    • -
- - -
-
- -
- -
-

13 Models    slide

-
- -
    -
  • Text: Naive Bayes -
    • -
  • Numeric Classification: SVMlight -
    • -
  • General: sklearn/RandomForrestClassifier -
    • -
- - -
- -
-

13.1 Even then    notes

-
- -
    -
  • Start with simple stats to understand your data -
    • -
  • Next: use heuristics, they are easy to understand and change -
    • -
  • Next: use third party models that you can drop in -
    • -
  • Often heuristics with understanding of false postive/negative costs will - get you far -
    • -
- - -
-
- -
- -
-

14 Practice    slide

-
- - - - -
- -
-

14.1 Other services    notes

-
- -
    -
  • Dataset challenge is open ended, so it lets you practice all elements -
    • -
  • Kaggle has many great competitions -
    • -
  • Collective Intelligence has many good examples -
    • -
  • Keep in mind trade-offs: that's what interviewers will ask -
    • -
- - -
-
- -
- -
-

15 Work    slide

-
- - -
- -
-

15.1 Topic Change    notes

-
- -
    -
  • Jumping topics a bit, what if you'd like to work at a web company instead - of build one? -
    • -
- - -
-
- -
- -
-

16 Hiring    slide two_col

-
- -
    -
  • Learn about the company -
    • -
  • Ask questions to learn about their problems -
    • -
  • Provide solutions -
    • -
- -

img/briefcase.jpg -

-
- -
-

16.1 Experience    notes

-
- -
    -
  • Use experience to answer questions -
    • -
  • Make sure you continue asking questions in the interview -
    • -
  • Ramit Sethi calls this the Briefcase Technique -
    • -
  • Know what's on your resume (Why is it applicable? Why is it interesting?) -
    • -
  • Think of the "interview" as a conversation, what would you say if you met - in a coffee shop? -
    • -
- - -
-
- -
- -
-

17 Resume    slide

-
- -
    -
  • Use quantitative data -
    • -
  • Describe the difference you made in a company/project, not what you did -
    • -
  • Include your side projects! -
    • -
- - -
- -
-

17.1 Unique    notes

-
- -
    -
  • What makes you a unique candidate? -
    • -
  • Your side projects set you apart. All students here have made a mobile - page. How is yours different? -
    • -
- - -
-
- -
- -
-

18 Resume is a Formality    slide

-
- -
    -
  • Be recognized independently of being in the resume pile -
    • -
  • Present at meetup -
    • -
  • Use their product in a cool way -
    • -
- - -
- -
-

18.1 Recognition    notes

-
- -
    -
  • Catch their attention, then start process -
    • -
  • Also makes you think "Do I want to work for this company?" -
    • -
  • Stories -
    • -
- - -
-
- -
- -
-

19 Negotiation    slide

-
- -
    -
  • Always try to have > 2 offers on the table -
    • -
  • Once a company decides, they've already sunk a lot of resources into you -
    • -
  • "That would make me comfortable" -
    • -
- - -
- -
-

19.1 Timing    notes

-
- -
    -
  • Pace interviews so you can make the decision together -
    • -
- - -
-
- -
- -
-

20 Do What it Takes    slide two_col

-
- -
    -
  • Most essential attribute: asking great questions -
    • -
  • > 50% of the work will be finding, formatting data -
    • -
  • Data product must be reliable to be effective -
    • -
  • Learn about distributed computing, software engineering -
    • -
- -

img/scrumtshape.jpg -

-
- -
-

20.1 The Job    notes

-
- -
    -
  • As Gene said, the thing that can't be taught is to think creatively about - all the cool stuff you can do with this data, frame it in a way that is - specific, actionable -
    • -
  • Most jobs require a combination of DM and coding skills -
    • -
  • Companies don't need just "idea people", need "idea + execution" -
    • -
  • Don't expect to just put on you DM lab coat and work with Kaggle-style data - all day -
    • -
  • Remember, biggest impact comes from putting together existing technology - in a useful way -
    • -
- - -
-
- -
- -
-

21 Managing upward    slide

-
- -
    -
  • Ideal email: "I've done the analysis below and recommend we do X. Sound good?" -
    • -
  • If no one is in charge, you're in charge -
    • -
  • Say "yes" but prioritize -
    • -
- - -
- -
-

21.1 Busy    notes

-
- -
    -
  • Your boss is busy, you do the work, make sure you're on the right track -
    • -
  • You shouldn't take on everything, but also shouldn't just start rejecting - things. -
    • -
  • Be a positive person: yes, we can do that after X, Z -
    • -
- - -
-
- -
- -
-

22 Engineering Career Paths    slide

-
- -
-
Hacker
Very broad, up-to-date. Best suited in very early startups. -
-
Individual Contributor
Reasonably skilled in areas of interest. Best - suited in mid-sized to large companies. -
-
Principal Engineer
Company or industry wide recognition for contributions - in specific areas. Very strong T-shaped skills. -
-
Manager
Ability and desire to solve people challenges, verify technical - solutions. -
-
- - -
- -
-

22.1 Gross Simplification    notes

-
- -
    -
  • Hacker: just get things done long enough to find a business model -
    • -
  • IC: majority of engineers, doing solid day-to-day work. -
    • -
  • Principal: Can include CTO at some companies, "tech leads." Go to person - for leading up projects. Must have a history of success, -
    • -
  • Management: If you like working with people, coaching, growing a team. - People are more complex than machines, so are solutions. -
    • -
  • Big themes: ownership, focus, excellence -
    • -
  • Joel's Ladder -
    • -
- - -
-
- -
- -
-

23 Stay Sharp    slide two_col

-
- -
    -
  • Long term, expected to combine the best of both: -
      -
    • Skills -
      • -
    • Wisdom -
      • -
    - -
    • -
  • So keep building skills -
    • -
- -

img/stay-sharp.png -

-
- -
-

23.1 Dig    notes

-
- - - - -
-
- -
- -
-

24 Networking    slide

-
- -
    -
  • Ask questions -
    • -
  • Learn from others -
    • -
  • Help others -
    • -
  • Don't skip stuff because you're lazy or scared -
    • -
- -

img/shy-connector.png -

- - -
- -
-

24.1 Skipping Stuff    notes

-
- -
    -
  • There are many good reasons not to go to an event, but being lazy is not - one of them -
    • -
  • Best opportunities are when you do stuff that pushes your boundaries -
    • -
- - -
-
- -
- -
-

25 Just Do It    slide

-
- -
    -
  • Practice -
    • -
  • Start with any idea -
    • -
  • Make a website you're proud to show friends -
    • -
  • Improve it -
    • -
- - -
- -
-

25.1 Doing is best for learning    notes

-
- -
    -
  • Employers look for engagement in these areas -
    • -
  • Almost any are you want to focus in, your website can be your medium -
    • -
- - -
-
- -
- -
-

26 Thank You!    slide

-
- - - - - - - -
-
-
- -
-

Date: 2013-05-10 01:11:00 PDT

-

Author: Jim Blomo

-

Org version 7.8.02 with Emacs version 23

-Validate XHTML 1.0 - -
- - diff --git a/slides/2013-05-10-Real-World.org b/slides/2013-05-10-Real-World.org deleted file mode 100644 index 2b8777e..0000000 --- a/slides/2013-05-10-Real-World.org +++ /dev/null @@ -1,389 +0,0 @@ -* *Datamining IRL* :slide: - -* General vs. Practical :slide: - + "the language in which you'll spend most of your working life hasn't been - invented yet, so we can't teach it to you. Instead we have to give you the - skills you need to learn new languages as they appear." - + Brian Harvey, [[http://www.eecs.berkeley.edu/~bh/sicp.html][Why SCIP matters]] -** Important Parts :notes: - + Important: understanding your domain, asking interesting questions, - answering them with data, fitting questions to mathematical concepts - + Not as important: scipy lingregress, d3 - -* But :slide: - + Who is doing interviews? - + Who is starting their own company? - + Who wants to build their portfolio? -** Real World :notes: - + I realize many of you may be needing to apply this stuff very soon - + So here's the lecture where I try to tell you what I would do when - practicing data mining - -* Product or Company Ideas :slide: - + Understand exponential growth - + Get the timing right - + Execute, execute, execute -** Idea required but not sufficient :notes: - + Wide variation of thoughts on ideas - + One of the biggest road blocks for wanna be entrepreneurs, but most derided - + Timing is mostly luck, is the world ready for your ideas? - + I think the *vision* is important, it's what drives the company, keeps - people working - + The plan for the idea less so - + The concrete product most important - + Let's talk about these elements - -* Toys :slide: - + "the next big thing always starts out being dismissed as a “toy.”" - + Chris Dixon, [[http://cdixon.org/2010/01/03/the-next-big-thing-will-start-out-looking-like-a-toy/][Blog]] -[[file:img/cdixon.jpg]] -** Why? :notes: - + Also, [[http://dcurt.is/what-a-stupid-idea][Stupid Ideas]] - + A few reasons for this - -** Exponential Improvements :slide:two_col: - + The future is changing at a faster rate than ever before - + Every field is being quantized, instrumented - + "You're not on the Internet?" :: "You're not analyzing your data?" - - [[file:img/cost_per_megabase.jpg]] -*** Change to Data :notes: - + You think your parents are bad at not operating devices? Your habits will - become out of date > twice as fast. - + So what happens when we combine exponential improvements and instrumented - fields? Overwhelming amounts of data - + Winners will be the ones - + Obviously web recommendations: data, but also - -** Exercise :slide:two_col: - [[file:img/strava.jpg]] - - [[file:img/strava-ride.png]] -*** Strava :notes: - + No device, just data - -** Thermostats :slide:center: - [[file:img/nest.jpg]] -*** Nest :notes: - + OK, they have a device... but what separates them is data - -** Biology :slide:center: - [[file:img/cost_per_megabase.jpg]] -*** Gene sequencing :notes: - + Biology: data, and data growing so fast a single computer can't keep up - + img: http://www.genome.gov/sequencingcosts/ - -* Execute :slide: - + Users are hiring you to do a job: what is it? - + "Institutions will try to preserve the problem to which they are the solution." -- [[http://www.shirky.com/][Clay Shirky]] - + Make your product so easy to use, people do it by accident. -** Do the job :notes: - + All of these company examples, you're typically not thinking of them as - "data processors"... they are solving a specific problem for you - + Strava isn't doing any crazy SVM analytics (at least on the consumer facing - side): they're showing you min/max, avg speed. Simple, but effective, - stuff. - + Disruption most often comes from using established technologies in new ways - or areas - + Can disrupt by completely simplified, often crappy at first, solutions to - an even more fundamental problem - + Dell did a great job selling cheap computers, then more expensive computers - + But now Amazon is saying: "you don't even need to own computers!" (Cloud) - + More info: [[http://www.claytonchristensen.com/][Clayton Christensen]] - + Focus on that one thing that is important and do it very, very well - -* Specifics ([[http://www.joelonsoftware.com/articles/fog0000000043.html][The Joel Test]]) :slide: - + Do you use source control? - + Can you make a build in one step? - + Do you make daily builds? - + Do you have a bug database? - + Do you fix bugs before writing new code? - + Do you have an up-to-date schedule? - + Do you have a spec? - + Do programmers have quiet working conditions? - + Do you use the best tools money can buy? - + Do you have testers? - + Do new candidates write code during their interview? - + Do you do hallway usability testing? -** Joel on Software :notes: - + When developing software, please follow as many of these as reasonable - + Joel Spolksy wrote this in 2000! Still a great guide! - + This is what I'd suggest to quickly get moving on the right foot - + If you're managing a team, make sure these are happening - -** Source Control :slide:center: - [[file:img/git.png]] -*** Surprised? :notes: - + Github will solve a few problems on this list, just use it, even if you're - developing alone - -** One step build :slide: - + Data mining exploration often involves manual commands - + *Don't* do that in production - + Should have scripts which extract features, build model, verify, deploy -*** Area for Improvement :notes: - + This is actually a big area solutions - + Deploying websites has solutions like Heroku, but no equivilant for - storing, processing, serving data - -** Bug Database :slide: - + Easy to loose track of problems - + Also good way to prioritize issues - + Use [[http://github.com][Github]] Issues -*** Managing Up :notes: - + Good defense - -** Write a Spec :slide: - + Alternatively, write the press release - + Don't write a novel - + Disagreements can be solved with code, but after talking -*** Bad rap :notes: - + Developers don't like writing them much - + But it helps nail down issues - + Yelp uses CEP process - + If you get to the "agree to disagree" point, data or code can solve - differences - -** Testers :slide: - + Use *unit tests* to test code (eg. =unittest2= in Python) - + Use cross-validation to test models - + Very easy to skip, will bite you within 6 months -** Differences :notes: - + Joel suggests having and paying testers - + I don't think this is best use of resources for small companies - + Economics change when developers can effectively write tests - + *Must* allocate time to this - + Add tests when you fix bugs - + Helps if developers use product daily - -** Tools :slide: - + Right tool for the job - + Text Editor: Use =vim= or =emacs= - + =virtualenv= (Python); =RVM= (Ruby) - + Learn the command line -*** Woodworker :notes: - + (slightly off topic from Joel's list) - + Woodworkers don't hammer stuff in with their shoe - + Make their own tools as first part of job - + When a custom problem comes up, make a custom tool - + These slides, written with mappings in =vim= - + Text Editor - + Syntax Highlighting - + Macros - + Interact with other tools - + Find across files - -* How to Use Recommendations :slide:two_col: - + Start with them as default - + If you understand why something is better for your case, use it - + Understand trade-offs -[[file:img/grain-of-salt.jpg]] -** Trade-offs :notes: - + One of the themes of this course - + Trying to provide you with a starting point - + My point of view: user driven behavior, engineers implementing solutions - -* Data Storage :slide: - + S3 for unstructured data - + PostgreSQL for structured - + Hive on S3 for very large structured data -** Data most important asset :notes: - + S3 is a pay-as-you go model, opens up many data processing possibilities - + Don't have to worry about how to connect - + PostgreSQL solid database, but also offers many improvements like storing - geo data - + Once you get beyond PostgreSQL limits, use Hive to structure data in S3 - -* Exploration :slide:two_col: - + Python - + IPython Notebook, matplotlib - [[file:img/ipython-notebook.jpg]] -** Py :notes: - + Main reason: it is convenient and practical to stay in the same language as - production - + Using production libraries, settings, to extract data - + R, matlab/octive, Tableau are typically not used in large production code - + SAS also effective for exploration, can be used in production, but skill - set not as transferable for smaller companies - -* Public Visualizations :slide: - + D3 for visualizations - + HTML is sharable, universal - + (Adventurous: Vega) -** Visualization :notes: - + Vega more directly maps to grammar of graphics, but is very new library - -* Processing :slide:two_col: - + Hadoop + mrjob - + Elastic MapReduce - + (Adventurous: [[http://spark-project.org][Spark]] - [[file:img/hadoop.png]] -** Scaling :notes: - + Hadoop scales up and down fairly well, especially with mrjob - + Constraints are going to be on *your* time, not necessary to eek out every - bit of computing poser - + Spark is a new model out of Berkeley that does a better job of keeping data - in memory, but doesn't have the maturity of Hadoop - -* Models :slide: - + Text: Naive Bayes - + Numeric Classification: SVM^light - + General: sklearn/RandomForrestClassifier -** Even then :notes: - + Start with simple stats to understand your data - + Next: use heuristics, they are easy to understand and change - + Next: use third party models that you can drop in - + Often heuristics with understanding of false postive/negative costs will - get you far - -* Practice :slide: - + [[http://www.yelp.com/dataset_challenge/][Yelp Dataset Challenge]] :) - + [[http://www.kaggle.com/][kaggle]] - + [[http://www.amazon.com/Programming-Collective-Intelligence-Building-Applications/dp/0596529325][Programming Collective Intelligence]] - + Ask around Berkeley -** Other services :notes: - + Dataset challenge is open ended, so it lets you practice all elements - + Kaggle has many great competitions - + Collective Intelligence has many good examples - + Keep in mind trade-offs: that's what interviewers will ask - -* *Work* :slide: -** Topic Change :notes: - + Jumping topics a bit, what if you'd like to work at a web company instead - of build one? - -* Hiring :slide:two_col: - + Learn about the company - + Ask questions to learn about their problems - + Provide solutions -[[file:img/briefcase.jpg]] -** Experience :notes: - + Use experience to answer questions - + Make sure you continue asking questions in the interview - + Ramit Sethi calls this the [[http://www.iwillteachyoutoberich.com/the-briefcase-technique/][Briefcase Technique]] - + Know what's on your resume (Why is it applicable? Why is it interesting?) - + Think of the "interview" as a conversation, what would you say if you met - in a coffee shop? - -* Resume is a Formality :slide: - + Be recognized independently of being in the resume pile - + Present at meetup - + Use their product in a cool way -** Recognition :notes: - + Catch their attention, then start process - + Also makes you think "Do I *want* to work for this company?" - + Stories - -* Resume :slide: - + Use quantitative data - + Describe the difference you made in a company/project, not what you did - + Include your side projects! -** Unique :notes: - + What makes you a unique candidate? - + Your side projects set you apart. All students here have made a mobile - page. How is yours different? - -* Negotiation :slide: - + Always try to have > 2 offers on the table - + Once a company decides, they've already sunk a lot of resources into you - + "That would make me comfortable" -** Timing :notes: - + Pace interviews so you can make the decision together - -* Do What it Takes :slide: - + Most essential attribute: asking great questions - + > 50% of the work will be finding, formatting data - + Data product must be reliable to be effective - + Learn about distributed computing, software engineering -** The Job :notes: - + As Gene said, the thing that can't be taught is to think creatively about - all the cool stuff you can do with this data, frame it in a way that is - specific, actionable - + Most jobs require a combination of DM and coding skills - + Companies don't need just "idea people", need "idea + execution" - + Don't expect to just put on you DM lab coat and work with Kaggle-style data - all day - + Remember, biggest impact comes from putting together *existing* technology - in a useful way - -* T-shape Skills :slide:center: - [[file:img/scrumtshape.jpg]] - -* Managing upward :slide: - + Ideal email: "I've done the analysis below and recommend we do X. Sound good?" - + If no one is in charge, you're in charge - + Say "yes" but prioritize -** Busy :notes: - + Your boss is busy, you do the work, make sure you're on the right track - + You shouldn't take on everything, but also shouldn't just start rejecting - things. - + Be a positive person: yes, we can do that after X, Z - -* Engineering Career Paths :slide: - + Hacker :: Very broad, up-to-date. Best suited in very early startups. - + Individual Contributor :: Reasonably skilled in areas of interest. Best - suited in mid-sized to large companies. - + Principal Engineer :: Company or industry wide recognition for contributions - in specific areas. Very strong T-shaped skills. - + Manager :: Ability and desire to solve people challenges, verify technical - solutions. -** Gross Simplification :notes: - + Hacker: just get things done long enough to find a business model - + IC: majority of engineers, doing solid day-to-day work. - + Principal: Can include CTO at some companies, "tech leads." Go to person - for leading up projects. Must have a history of success, - + Management: If you like working with people, coaching, growing a team. - People are more complex than machines, so are solutions. - + Big themes: ownership, focus, excellence - + [[http://www.joelonsoftware.com/articles/Ladder.html][Joel's Ladder]] - -* Stay Sharp :slide:two_col: - + Long term, expected to combine the best of both: - + Skills - + Wisdom - + So keep building skills - [[file:img/stay-sharp.png]] -** Dig :notes: - + Dig into areas you're not familiar - + Talk to people, help solve their problems, learn how it turned out - + img: http://shirt.woot.com/blog/post/stay-sharp - -* Networking :slide: - + Ask questions - + Learn from others - + Help others - + Don't skip stuff because you're lazy or scared -[[file:img/shy-connector.png]] - + [[http://www.slideshare.net/sachac/the-shy-connector][Shy Connector]] -** Skipping Stuff :notes: - + There are many good reasons not to go to an event, but being lazy is not - one of them - + Best opportunities are when you do stuff that pushes your boundaries - -* Just Do It :slide: - + Practice - + Start with any idea - + Make a website you're proud to show friends - + Improve it -** Doing is best for learning :notes: - + Employers look for engagement in these areas - + Almost any are you want to focus in, your website can be your medium - -* *Thank You!* :slide: - -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: -#+STYLE: - -#+BEGIN_HTML - -#+END_HTML - -# Local Variables: -# org-export-html-style-include-default: nil -# org-export-html-style-include-scripts: nil -# buffer-file-coding-system: utf-8-unix -# End: diff --git a/slides/2014-01-23-Intro.html b/slides/2014-01-23-Intro.html new file mode 100644 index 0000000..da44346 --- /dev/null +++ b/slides/2014-01-23-Intro.html @@ -0,0 +1,503 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-01-23-Intro.markdown b/slides/2014-01-23-Intro.markdown new file mode 100644 index 0000000..796bb1a --- /dev/null +++ b/slides/2014-01-23-Intro.markdown @@ -0,0 +1,334 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +class: center + +# Data Mining 290T-03 +## Jimmy Retzlaff & Shreyas + +--- + +## Course Goals + + Extract *information* from *data* + + Understand techniques to find patterns + + Apply algorithms to real data sets + +--- + +## We'll Do Stuff + + 30%: 10 Homework Assignments + + 30%: 1 Midterm + + 40%: 1 Project: Find, Mine, Report on Data + +??? + +## Homework Details + + Homework due at midnight Wednesday before class + + Each 24 hours late is 10% off + + Homework will be turned in by GitHub pull request + + Project will be submitted by email & presentation + +--- + +## But Don't Worry + + This isn't a programming class + + Grades are based on understanding of the concepts, not the craziest project + + Shreyas & I are here to help + +??? + +## Help + + We realize there's a wide range of technical skill + + We will help get anyone up to speed in these technical areas + +--- + +## This is a Graduate Course + + Perform well without supervision + + Readings from both book and online documentation + + TMTOWTDI + + Getting frameworks working on your computer + +??? + +## Style + + More fire hose than spoon fed, you'll need to follow up for understanding + + Honor system: No copying code or answers. Helping each other with + concepts is encouraged, but document it. + + Everybody has a different work flow. We'll be covering the most basic. + Great if you want to do something different, but realize we may not be + able to help you as much. + + Non ISchool students should email student ID from EDU account to shreyas + and jretz and we will get you an ischool accounts. + + You may want to use other frameworks for your projects. Great! But again, + we may not be familiar with them. + +--- + +## Prerequisites + + Basic probability: P(A), P(A or B), P(A and B), P(A | B) + + Basic programming: Python + + Basic command line: SSH, downloading, copying large files, running programs + against data + + Textbook: Han, J., Kamber, M., & Pei, J. (2011). _Data Mining: Concepts and Techniques_, Third Edition *(3rd ed.)*. Morgan Kaufmann. + + Technology will be available on ```ischool.berkeley.edu``` + +??? + +## Basics + + "Probability of A", "Probability of A or B" "A and B" "A given B" + + Most assignments filling in algorithm code + + Project you may use any language, though we suggest Python. + + We'll introduce any specific frameworks + + Command line: cp, mv, less... Imagine you have a 10GB file, how are you + going to inspect the contents? + +--- + +## Material + + Process: from finding data to mining it to visualizing results + + Algorithms: all intuitively motivated, some rigorously studied + + Programming: using algorithms against data sets + + Discovery: finding information in a self-defined project + +??? + +## What will we learn? + + Data mining is not just about algorithms. We'll learn how to obtain, clean, + and store data. + + In real life, this is 70% of the job! + + We'll cover many different algorithms, and dive deep on several of + them. But we're not going to get into any hairy math proofs. + + Programming is the best way to precisely describe an algorithm. It is also + the way data mining is used in the real world. + + Your own project should emphasize your passion. Again, the real world requires + you to grab data and squeeze information out of it without external help. + +--- + +## Lectures & Labs + + Start with Q&A for at least 10 minutes + + Expect to be asked a question + + Breaks + + Lab: Stick around and get the first question of homework done + + Slides on http://jretz.github.io/datamining290/ + + Source for everything on https://github.com/jretz/datamining290/ + +??? + +## Helpful tips + + Helpful to me if you say your name + + Sorry, I tend to forget names + + If I am not calling on you, check to make sure you are on the class list! + + I'm not taking attendance, but let me know if you can't make it so I + won't call on you + +--- + +## Office Hours + + We'll stay after class + + or schedule a Skype call + + [Piazza](https://piazza.com/berkeley/spring2014/info290t03/home) for + questions and announcements + + Wait list will be processed normally until 3rd week... then I'll accept + everyone who's participated in class if we have physical room + +--- + +## *Questions?* + +--- + +## Schedule +Available at [GitHub Syllabus page](http://jretz.github.io/datamining290/) + + + Jan 23 Class Intro ; Tools Intro by GUEST: Shreyas + + lab: Git Intro + + Jan 30 Case Studies ; Obtaining Data + + Feb 6 Probability ; Preprocessing + + Feb 13 MapReduce, Data Warehouse + + Feb 20 Decision Trees; Naive Bayes + + Feb 27 SVM ; Neural Networks + + Mar 6 Clustering + + Mar 13 Advanced Clustering ; Review + + Mar 20 *Midterm* + + lab: - + + Mar 27 HOLIDAY + + Apr 3 Patterns ; Evaluations + + lab: Project Proposal Due + + Apr 10 Graphs; PageRank + + Apr 17 Feature Extraction ; Evaluation + + Apr 24 Outliers ; Images ; Audio + + May 1 Visualization ; HTML + + May 8 In Real Life ; Review + + lab: - + + May 15 Final Presentation + + lab: Bye! + +--- + +## Hi, I'm Jimmy Retzlaff + + Yelp - ads engineer and now ads engineering manager + + Amazon / Lab126 - Kindle on-device content search + + Aver - sales visualizations for the investment industry + + Career Central - allow employers to search for job seekers + + Animatrix - executive information systems + + Xerox - printer drivers for high volume printers + + Harvey Mudd College, B.S. Math + +--- + +## Hi, I'm Shreyas + + Second year graduate student at ISchool + + Also TA'd Analyzing Big Data class and this class last year + + I can be reached at [shreyas@ischool](mailto:shreyas@ischool) + +--- + +## Data is Important + + Making decisions is a core part of humanity + + Data can help you make better decisions + + Challenge: extract information from data to improve decisions + +??? + +## Decisions + + From big to small; from planning to execution + + Business questions: what is the ROI of this feature? Where to concentrate + development? + + Personal questions: Where to eat dinner tonight? What movie to see? + + Improving decisions means improving quality of life + +--- + +## Data is Important +[![Reviews, Reputation, and Revenue: The Case for Yelp.com](http://img.youtube.com/vi/y7een27u1GM/0.jpg)](http://www.youtube.com/embed/y7een27u1GM) + +??? +[![Reviews, Reputation, and Revenue: The Case for Yelp.com](http://img.youtube.com/vi/y7een27u1GM/0.jpg)](http://www.youtube.com/watch?v=y7een27u1GM) + +## Nice example of data mining + + Stop at 3:51 + + Had to work with external parties to get data (Yelp, city of Seattle) + + Had to clean data (literally, sometimes he was just handed paper receipts) + + Used regression analysis to discover patterns + + created follow up questions + + Used result to understand the meaning behind the data + +--- + +## Data Mining ecosystem + + Data mining is part of a process to make decisions from data + + Intersection between statistics, computer science, data management, and + machine learning + + Analysis & visualization often required + +??? + +## Ecosystem + + We'll talk about several ways to think about the process from data to + knowledge + + No universally agreed process, or black-and-white boundaries + + Analysis: used at the beginning of investigations to understand data + characteristics + + Visualization: better understanding of the results of analysis or data + mining + +--- + +## Analysis vs. Data Mining + +### Analysis + + Manually investigating data. No algorithms. + + Statistical qualities: mean, median, standard deviation + + Histograms (manually set buckets) + + Counts / Percentages + +### Data Mining + + Discovering patterns though automated algorithms + + Regressions: fitting data to a model + + Clustering: grouping data without manually set descriptions + + Classification: identifying divisive features + +??? + +## Pedantic + + + Difference is subtle, but important for both the project and your resume + +--- + +## Machine Learning + + + Programs that can learn from data + + Focus on prediction, based on verified training data + + Used in two ways: during Data Mining and after Data Mining + +[![Robot reading](http://distillery.s3.amazonaws.com/media/2011/06/15/04cfeacce2f4404483d96a4428f9adbd_7.jpg)](http://photopile.me/user/hartanta/?max_id=100596871_2487764#) + +??? + +## Uses + + + During - assume we have training data, train on it, see how useful trained + program is or find outliers + + After - Discover clusters, verify and label clusters. Use labeled clusters + to train a program to recognize new data points + +--- + +## Probability & Statistics + + + Data describes real world events + + Probability can describe real world *expected* events + + Distributions can be used to summarize data, understand the factors behind + its creation + +
+ +??? + +## Uses + + + Can "fit" data to a distribution, find outliers that are unexpected + + An example: Poisson distribution describes the expectation of a particular + number of events occurring. + + e.g., pieces of mail. Average is 4, but it can vary. Is getting 7 or more + pieces of mail really an outlier? + +--- + +## Process + +### [Knowledge Discovery in Databases (KDD)](http://en.wikipedia.org/wiki/Data_mining#Process) + + Selection + + Pre-processing + + Transformation + + Data Mining + + Interpretation/Evaluation + +### [Cross Industry Standard Process for Data Mining](http://en.wikipedia.org/wiki/Cross_Industry_Standard_Process_for_Data_Mining) + + Business Understanding + + Data Understanding + + Data Preparation + + Modeling + + Evaluation + + Deployment + +??? + +## Common Themes + + Figure out what you want to do + + Get the data + + Make sure it's OK + + Understanding + + Make a decision, test its effectiveness + + Reading will cover another process, aimed at "Data Science", but basically + applies to Data Mining + +--- + +## *Break* diff --git a/slides/2014-01-23-Lab.html b/slides/2014-01-23-Lab.html new file mode 100644 index 0000000..1fa6ec3 --- /dev/null +++ b/slides/2014-01-23-Lab.html @@ -0,0 +1,260 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-01-23-Lab.markdown b/slides/2014-01-23-Lab.markdown new file mode 100644 index 0000000..150c5a9 --- /dev/null +++ b/slides/2014-01-23-Lab.markdown @@ -0,0 +1,91 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Lab: Github + + + Setup GitHub account + + Submit assignment via GitHub + +--- + +## Why GitHub? + + + ```git``` tool is standard in industry + + GitHub provides best tools for sharing, commenting code + + This assignment will not have code, just practice submitting + +--- + +## Setup GitHub account + + + Create a [GitHub Account](https://github.com/signup/free), making sure to + use your .edu address + + Use [GitHub/Edu](https://github.com/edu) to request a free micro plan: + these let us use private accounts + + Setup a [GitHub SSH Key](https://help.github.com/articles/generating-ssh-keys) + +--- + +## Setup git repository on ischool server (can also use your own laptop) + + + On the server ischool.berkeley.edu +```bash +$ git clone git://github.com/jretz/datamining290.git +``` + + On the server, in the datamining290 directory run +```bash +$ git remote rename origin jretz +``` + +--- + +## Connect it to GitHub + + + After you receive your free micro account on GitHub, create a private repository called datamining290 + + It will provide you with an SSH git path, let's call it PATH + + You must use the *SSH* PATH starting with ```git://``` + + On the server, in the datamining290 directory, run +```bash +$ git remote add origin PATH +$ git push origin master +``` + +--- + +## Share with us + + + Hopefully you now have a private copy of my repository + + Add Shreyas and me (users: seekshreyas, jretz) as a contributor to your private repository + +--- + +## Submit Homework + + + On the ischool server, create a branch called ```hw1``` + + Create a new text file with your favorite editor (a simple one is + ```pico```) telling us what you hope to get out of the course + + ```git add``` the file + + ```git commit``` the change + + ```git push origin hw1``` to put it on GitHub + + On github, submit a "pull request" from the ```hw1``` branch to your master branch + +??? + +## Pull Requests + + + Pull requests are a way of showing updates in a way that lets me provide + comments, get notifications + +--- + +## Going Forward + + + Other homework assignments will be completing code + + General work-flow: + + Start a new branch + + Add required files + + Push to GitHub + + Submit Pull Request diff --git a/slides/2014-01-30-CaseStudies.html b/slides/2014-01-30-CaseStudies.html new file mode 100644 index 0000000..1b76f3f --- /dev/null +++ b/slides/2014-01-30-CaseStudies.html @@ -0,0 +1,515 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-01-30-CaseStudies.markdown b/slides/2014-01-30-CaseStudies.markdown new file mode 100644 index 0000000..f32e756 --- /dev/null +++ b/slides/2014-01-30-CaseStudies.markdown @@ -0,0 +1,346 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Case Studies + +--- + +## Process + +### [Knowledge Discovery in Databases (KDD)](http://en.wikipedia.org/wiki/Data_mining#Process) + + Selection + + Pre-processing + + Transformation + + Data Mining + + Interpretation/Evaluation + +### [Cross Industry Standard Process for Data Mining](http://en.wikipedia.org/wiki/Cross_Industry_Standard_Process_for_Data_Mining) + + Business Understanding + + Data Understanding + + Data Preparation + + Modeling + + Evaluation + + Deployment + +??? + +## Data to Knowledge + + + We learned last week that the goal of data mining is to turn raw data into + knowledge + +--- + +## Search Engine Logs + +.tight-code[ +```log +193.139.1 jimmy [10/Oct/2013:13:55:36] "GET /search?q=headache HTTP/1.1" 200 9288 +282.482.3 shreyas [10/Oct/2013:13:56:36] "GET /search?q=bananas HTTP/1.1" 200 2929 +345.114.1 steven [10/Oct/2013:13:56:37] "GET /search?q=cold HTTP/1.1" 200 8232 +10.328.52 anne [10/Oct/2013:13:56:39] "GET /search?q=flu+shot HTTP/1.1" 200 2342 +10.328.52 lily [10/Oct/2013:13:57:40] "GET /search?q=i290 HTTP/1.1" 200 2342 +``` +] + +What is a common theme in these queries? + +??? + +## Raw Data + + raw data comes in many forms + + often we'll use tech examples: e.g., search engine logs + + these have information like user, IP, date-time, HTTP version, query + + can we extract actionable information from it? + +--- + +## Flu Trends +.left-column[ + + + Use dates to plot trends over time + + Use IPs to show activity per state or city + + Other ideas? +] + +.right-column[ + ![Flu Trends](img/flu-trends.png) +] + +??? + +## Other ideas + + + What other information could you extract from log data? + + Spread of flu over countries, cities? + + Time of day? Do people notice in the morning? + + Correlated with any other activity? (e.g., travel) + + Best day of the week to call in sick (and get away with it)? + +--- + +## Asking Questions + + + Many potential discoveries within search logs + + Asking meaningful questions is a difficult but essential part of data + mining + + Algorithms can answer questions for you, but it can't ask them + +??? + +## No magic + + + Data mining is not a magical machine into which one throws data and gets + out interesting facts + + Data + question + algorithm suited for question => potential insights + +--- + +## Data Mining Process + + + Data cleaning + + Data integration + + Data selection + + Data transformation + + Data mining* + + Pattern evaluation + + Knowledge presentation + +??? + +## We cover the full process + +### Cleaning + remove abuse requests, "Estimates for Connecticut for weeks + 2012-12-16 to 2013-01-06 were affected by a software glitch" + +### Integration + Collecting logs from different data centers, maybe from + different formats (over the years) + +### Selection + IPs, dates, queries + +### Transformation + IP to location. Dates to local time. + +### Mining + what words are associated with the flu? cold? fever? other + languages? + +### Evaluation + This year worse than last, peaking later. + +### Presentation + plotting, cartograms + +--- + +## Data Preparation + + + Collecting, cleaning, integrating takes > 50% of the time in real world + situations + + Explains difficulty in finding good candidates for Data Scientist roles + +??? + +## Data Scientist + + + In industry, most companies are hiring engineers to interact with the full + stack, so that they can collect data + + If preparation is > 50% and they hire you just for algorithms, they need to + hire > 1 other person just to support you + + How many of you like just preparing data? + +--- + +## Transactional Data + + + Discrete history of events, containing some minimum amount of data: + + Subject: Who initiated action? + + Verb: What was done? + + Object: What was it done to? + + Timestamp: When? + +??? + +## Storage + + + Most common example is purchase history + + Subject: user ID, or name + + Verb: In logs, can vary. In databases, you'll have a purchases table, so + verb is assumed to be "purchased" + + Object: product IDs (or in web logs, web pages) + + Timestamp: Make sure you account for timezones + + Other Data: previous page, extra info about action (purchase with CC? + Cash?) + +--- + +## Other Data + + + Often does not contain timestamps + + Spatial Data + + Multimedia + +![Moonlight Sonata](img/moonlight_sonata.jpg) + +??? + +## Data + + + Maps in general can be used to find interesting information: where are + cities typically located? What are properties of well planned cities? + + Videos have a time component, but are not transactional. + + Music can be seen non-linearly and analyzed + + image: http://flyingpudding.com/projects/viz_music/ + +--- + +## Purpose of Data Mining + +Purpose + + + Obtaining *actionable knowledge* + +Descriptive + + + Explains data already seen + +Predictive + + + Immediately understand new data + +??? + +## Tasks + + At Amazon, dashboards for different countries + + Americans shopped at work; Germans shopped early morning, early evening; Japanese shopped late at night + + Can help with capacity planning, ideas for discounts, warehouse staffing + + Predictive: at Yelp, what business are you most likely to want to review + next? As you have activity, instantly understand what is the best + recommendation + +--- + +## Types of Models + + + Classifiers + + Regressions + + Clustering + + Outlier + +??? + +## Details + +Classifiers + + + describes and distinguishes cases. Yelp may want to find a + category for a business based on the reviews and business description + +Regressions + + + Predict a continuous value. e.g., predict a home's selling + price given sq footage, # of bedrooms + +Clustering + + + find "natural" groups of data *without labels* + +Outlier + + + find anomalous transactions, e.g., finding fraud for credit cards + +--- + +## Tip of the Iceberg + + + Thousands of ways to calculate a model + + Combinatorially more ways to combine them + + In technique, large amount of overlap between purpose + + + +??? + +## Survey + + + Machine Learning and Data Mining fields churn these models out + + Newest methods combine multiple models (boosting & bagging) + + We're going to cover these in much greater detail in the course + +--- + +## Your own examples + + + Classifiers + + Regressions + + Clustering + + Outlier + +??? + +## Examples + +Classifiers + + + Newly opened business + +Regressions + + + Revenue estimates for a franchise store + +Clustering + + + Movie genres + +Outlier + + + Bot vs human web traffic + +--- + +## Machine Learning + +Supervised + + + Given data with a label, predict data without a + label + +Unsupervised + + + Given data without labels, group "similar" items + together + +Semi-supervised + + + Mix of the above: e.g., unsupervised to find groups, + supervised to label and distinguish borderline cases + +Active + + + Starting with unlabeled data, select the most helpful cases + for a human to label + +--- + +## Matching + + + Categories for businesses, where some business have correct labels, but not sure how precise categories should be + + Comparing search results algorithms: some queries return the same results, some return very different businesses + + Spam filter with existing corpus + + Demographic information about customers + +??? + +## Details + + + Matching with the type of learning + +--- + +## *Break* diff --git a/slides/2014-01-30-Lab.html b/slides/2014-01-30-Lab.html new file mode 100644 index 0000000..44ed4fe --- /dev/null +++ b/slides/2014-01-30-Lab.html @@ -0,0 +1,229 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-01-30-Lab.markdown b/slides/2014-01-30-Lab.markdown new file mode 100644 index 0000000..d7261f6 --- /dev/null +++ b/slides/2014-01-30-Lab.markdown @@ -0,0 +1,60 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Lab: Obtain and Explore Data + + + Find a data set or external API + + Superficially examine it + + Summarize findings + + Submit assignment via GitHub + +--- + +## Obtain Data + + + Look through the links in slides for interesting data sets, or find your own + + Or find a service API, like NYTimes + + Explore the data available to answer the following questions + +--- + +## Questions + + + What are the types of data available to you? + + For data sets: how many records are in the data set? + + For API: what are the limits on fetching data? + + Provide an "interesting" record, explain its properties and why it is + interesting + + What are 3 questions you could answer using your data? + +--- + +## Submit Homework + + + On the ischool server, create a branch called ```hw-obtain-data``` + + Create a text file to write the solution, a simple editor to use is ```pico``` + + ```git add``` the file + + ```git commit``` the change + + ```git push origin hw-obtain-data``` to put it on GitHub + + On github, submit a "pull request" from the ```hw-obtain-data``` branch to your master branch + +??? + +## Pull Requests + + + Pull requests are a way of showing updates in a way that lets me provide + comments, get notifications + +--- + +## Going Forward + + + Other homework assignments will be completing code + + General work-flow: + + Start a new branch + + Add required files + + Push to GitHub + + Submit Pull Request diff --git a/slides/2014-01-30-Obtaining-Data.html b/slides/2014-01-30-Obtaining-Data.html new file mode 100644 index 0000000..7721cf4 --- /dev/null +++ b/slides/2014-01-30-Obtaining-Data.html @@ -0,0 +1,624 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-01-30-Obtaining-Data.markdown b/slides/2014-01-30-Obtaining-Data.markdown new file mode 100644 index 0000000..14dfd7c --- /dev/null +++ b/slides/2014-01-30-Obtaining-Data.markdown @@ -0,0 +1,455 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Obtaining Data + +--- + +## Ways to Collect + +.left-column[ + + + Operational Data + + Data Warehouse + + Unstructured Data + + External API + + Data Sets +] + +.right-column[ + +] + +??? + +image: http://woodwarddesign.ca/blog/2009/03/06/bottle-caps/ + +--- + +## Operational Data + + + Most frequent in industry + + Usually stored in databases best suited for transactional use + + Challenge is reorganizing data to suit question + +??? + +## Data from production + + Most frequently you'll have data that is being used by the application, + and you'll want to find insights in it + + We'll go into more detail in another class, but online use is + optimized for small queries and small updates + + Frequently just accessing the data in bulk is a software engineering + problem: + + ensuring long queries don't hold up production usage + + joining across databases via software + + understanding esoteric columns, like "flags" + + Often will want to reorganize data to look like transactional + +--- + +## Example + +Find the user names with most "liked" reviews on Yelp + +Users + +| user_id | name | flags | +|---------|------|-------| +| 25234 | Bob | 0x200 | + +Reviews + +| review_id | business_id | user_id | stars | text | flags | +|-----------|-------------|---------|-------|--------------|-------| +| 282 | 52432 | 25234 | 4 | great place! | 0x1 | + +Feedback + +| review_id | source_user_id | ufc_flags | flags | +|-----------|----------------|-----------|-------| +| 282 | 8205 | 0x1 | 0x0 | + +??? + +## Distributed Data + + + At Yelp we have a variety of database tables, and those tables can be + spread across different databases + + At a minimum we frequently need to ```JOIN``` across tables to answer queries + + e.g., matching up user names with reviews from separate tables + + It is possible the review table is only indexed on business_id, and so + finding all reviews by a user is really disk intensive: make sure you're + not slowing down the whole site! + + An additional challenge is when the "feedback" tables are in a separate + database: can no longer issue normal SQL queries + + What are these "flag" columns for? + + Exactly: no one knows. Often must look into code, or compare data to + production representation to guess meaning. In Yelp, ```0x1``` often means + "inactive", so we probably don't want to count that feedback + +--- + +## Data Warehouse + +.left-column[ + + + Data located on same system + + Organized for analytics queries + + Requires extra maintenance and understanding of construction +] +.right-column[ + +] +??? + +## No free lunch + + + A strong data warehouse can be a big improvement over operational data + + Hopefully, someone has already cleaned, joined data in a way that makes + sense! + + Optimized for long running queries: less fear of brining down website! + + But you must learn how that process was accomplished in order to understand + potential problems + + How to handle missing data? + + We'll go into more detail about how data warehouse schemas compare to + online ones later in the course + +--- + +## Unstructured + + + Haphazard collection of data + + Unclear what structure should be + + Examples: Web logs, text, multimedia + + Must extract structure eventually + +??? + +## Yelp JSON logs + + + When developing a web application, new context or details become + important: how long did certain requests take? What link did a user follow + to a website? + + Relational Databases aren't well suited for these wide varieties of + potential attributes that don't apply to all items + + So the current work around is just to write all useful information down in + a log, and extract what is needed later + + Text, like business reviews, another example: desired structure changes + radically between questions: How many words? Characters? What is the sentiment? + + Pictures can contain attributes like color depth, length, width + + First step of data mining is often imposing structure on data: the data is + not inherently unstructured, it just is unclear what the structure *should be* + until query time + +--- + +## Search Logs Example + +.tight-code[ +``` +193.139.1 jimmy [10/Oct/2013:13:55:36] "GET /search?q=headache HTTP/1.1" 200 9288 +282.482.3 shreyas [10/Oct/2013:13:56:36] "GET /search?q=bananas HTTP/1.1" 200 2929 +345.114.1 steven [10/Oct/2013:13:56:37] "GET /search?q=cold HTTP/1.1" 200 8232 +10.328.52 anne [10/Oct/2013:13:56:39] "GET /search?q=flu+shot HTTP/1.1" 200 2342 +10.328.52 lily [10/Oct/2013:13:57:40] "GET /search?q=i290 HTTP/1.1" 200 2342 +``` +] + +| user_name | date | query | +|-----------|----------------------|----------| +| jimmy | 10/Oct/2013:13:55:36 | headache | +| shreyas | 10/Oct/2013:13:56:36 | bananas | +| steven | 10/Oct/2013:13:56:37 | cold | +| anne | 10/Oct/2013:13:56:39 | flu shot | +| lily | 10/Oct/2013:13:57:40 | i290 | + +??? + +## Imposing Structure + + + Extract only the rows we know follow a format + + Format queries from some encoding (e.g., URL) to standardized format + +--- + +## External APIs + + + Better documented than internal data! + + More limited in amount and detail + + Commonly HTTP/REST based + +??? + +## Motivation + + + Companies are often searching for other ways to leverage their data + + Both for immediate business purposes, and for brand recognition + + Twitter more (in)famous example + + NYTimes another good option + +--- + +## NYTimes API Example + + + [Article Search API](http://developer.nytimes.com/docs/read/article_search_api_v2) + + http://api.nytimes.com/svc/search/v2/articlesearch.json?fq=berkeley&begin_date=20140101&end_date=20140131&api-key=d394cd6a13605351d187e3864dfcea30:8:68746734 + +??? + +## Accessing these + + More info on how to access these APIs is in the Web Architecture class, + but feel free to ask Shreyas or I about how best to access them + +--- + +## Data Sets + + + Download large, curated set of data all at once + + Formats vary, but usually documented + + Can be useful to combine with other datasets or APIs + + + +??? + +## Research + + + Data sets commonly used in research: can compare different techniques on + same data to understand advantages + + Sizes can range to a few MB to GB + + JSON, CSV, XML all potential formats. Cleaning, organization for your + question again becomes an important aspect + +--- + +## Data Set Examples + +.left-column[ + + + [MovieLens Data Sets](http://www.grouplens.org/node/73) + + [Kaggle Digit Recognizer](https://www.kaggle.com/c/digit-recognizer) + + [Hilary Mason's Data Sets](https://bitly.com/bundles/hmason/1) +] +.right-column[ + +] + +--- + +## Exploring Data + + + Data sets are frequently too large to fit in standard tools like Excel + or Word + + Simplest to explore on the command line + + Homework will be exploring a data set of your choice + +??? + +## Size + + + Some formats will not be easily parsed into Excel: e.g., JSON, XML + + Word will be slow, or unworkable for GB size data + + CLI provides many composable tools for text manipulation + +--- + +## Yelp Academic Dataset + + + [Yelp Dataset Challenge](http://www.yelp.com/dataset_challenge/) data covers reviews, users, + businesses, and check-ins + + To download, you'll need to sign up, but it's instant + + Use .edu email + +??? + +## Example + + + We'll use this as an example, you can use any data set of your choice + + Just for homework, don't need to use for project + +--- + +## CLI introduction + + + Standard commands available in [Learn CLI the hard way](http://cli.learncodethehardway.com) + + All example will be run on ```ischool.berkeley.edu``` + + Shreyas and I are available for more help + +??? + +## Help + + + If you're new, don't be intimidated. + + Security policies ensure you can't break anything besides your own files + + Keep backups of important stuff anyway + +--- + +## ```wget``` + + + Used for downloading files + + Downloading with the browser is fine, but sometimes nice to use a faster + connection, or download it directly to the machine you're working on +.tight-code[ +```bash +$ wget 'http://www.grouplens.org/system/files/ml-100k.zip' +``` +] + +??? + +## Command + + + Just ```wget "URL"``` + + I like to use quotes in case there are special characters in the URL, eg + ```?``` + + Will download to current directory with the same name as the remote file + +--- + +## ```scp``` + + + Copy a file to or from a remote machine + + Uses same connection as SSH, but copies data instead + + Example: Copy data you've downloaded in your browser + +.tight-code[ +```bash +$ scp ~/Downloads/ml-100k.zip jretz@ischool.berkeley.edu: +``` +] + +OR + +.tight-code[ +``` +$ scp ~/Downloads/ml-100k.zip \ +jretz@ischool.berkeley.edu:i290/movielens-100k.zip +``` +] + +??? + +## Command + + + Trailing ```:``` is important: signifies remote machine + + If you don't specify path or filename, will copy the file with the same + name into your home directory + +--- + +## ```gunzip``` / ```unzip``` + + + Uncompress data sets for simpler, faster manipulation + ``` + $ unzip ml-100k.zip + ``` + + OR + + ``` + $ gunzip dataset.json.gz + ``` + +??? + +## Commands + +### unzip + expand potentially many files, leave original alone + +### gunzip + expand original file, leaving only the uncompressed version + +--- + +## ```less``` + + + View a file + + History: original command was called ```more``` to see a file a page at a time + + "Less is more" + ```bash + less yelp_academic_dataset_user.json + ``` + +--- + +## Searching in ```less``` + + + ```/``` (forward slash) lets you input search text, `````` performs the search + + After finding the first occurrence, ```/``` will find the next occurrence + + ```?``` will find the previous occurrence + + ```q``` will quit +``` +/"name"": "Bob" +/"name"": "Cindy" +``` + +??? + +## Command + + + Useful for finding specific instances to investigate + +--- + +## ```grep``` + + + Find and print lines matching a "regular expression" + + [Regular expressions](http://www.regular-expressions.info/quickstart.html) + are "find" on steroids, but you can use simple strings + +.tight-code[ +```bash +$ grep '"name": "Cindy"' yelp_academic_dataset_user.json +``` +] + +--- + +## ```wc``` + + + "wordcount" counts characters, words, lines + + Most useful in data sets for lines: ```-l``` +```bash +$ wc -l yelp_academic_dataset_user.json +43873 yelp_academic_dataset_user.json +``` + +--- + +## Composability + + + Genius of Unix: do one thing well, compose commands to get what you want + + ```|``` pipe characters "sends" output of one program to the input of another + + How many people named Cindy in the dataset? + +.tight-code[ +```bash +$ grep '"name": "Cindy"' yelp_academic_dataset_user.json | wc -l +91 +``` +] + ++ What are the most common names in the dataset? + +.tight-code[ +```bash +$ egrep -o '"name": "([^"]*)"' yelp_academic_dataset_user.json | \ + sort | uniq -c | sort -nr | head +465 "name": "David" +447 "name": "John" +418 "name": "Michael" +417 "name": "Chris" +383 "name": "Mike" +365 "name": "Jennifer" +298 "name": "Brian" +267 "name": "Scott" +265 "name": "Jason" +261 "name": "Mark" +``` +] diff --git a/slides/2014-02-06-Lab.html b/slides/2014-02-06-Lab.html new file mode 100644 index 0000000..1f01171 --- /dev/null +++ b/slides/2014-02-06-Lab.html @@ -0,0 +1,270 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-06-Lab.markdown b/slides/2014-02-06-Lab.markdown new file mode 100644 index 0000000..41ea02d --- /dev/null +++ b/slides/2014-02-06-Lab.markdown @@ -0,0 +1,101 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Lab: Data Stats + + + Obtain federal [campaign finance data](ftp://ftp.fec.gov/FEC/2012/pas212.zip) + + Reference the [data dictionary](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml) + + Decompress + + Manually check + + Run ```code/stats.py``` + + Edit ```code/stats.py``` to add functionality + + Pull Request submission + +--- + +## Data + + + [Campaign finance data](ftp://ftp.fec.gov/FEC/2012/pas212.zip) + + The [data dictionary](http://www.fec.gov/finance/disclosure/metadata/DataDictionaryContributionstoCandidates.shtml) tells you the format of the data + + Decompress and investigate using the tools we discussed + +--- + +## Code + +```bash +$ git checkout master +$ git pull jretz master +$ git checkout -b hw-stats +``` + + Run and edit ```code/stats.py``` + +--- + +## Stats + + + Minimum + + Maximum + + Mean + + Median + + Standard Deviation + + Candidates + + Normalized sample contributions + +--- + +## Extra credit + + + Extra credit is used to get you *up to 100%* + + On the *current assignment* + + Also helpful for learning topics more in depth + + You may do partial extra credit + +??? + +## Overall Extra Credit + + + Extra credit that applies to overall grade will not be assigned + +--- + +## Extra Credit + + Stats per candidate + + z-score + +--- + +## Git usage + + + All edits, commits, pushes should happen on a ```hw-``` or ```project``` branch + + ```git status``` + + All pulls (typically from ```jretz```) should happen on ```master``` branch + + If you use an editor connected to ischool server, make sure you are + *either* editing *or* using git + +??? + +## Exceptions + + + There are exceptions but know what you're trying to do + + External editors can write back files *after* you've changed git branches + +--- + +## Submission + + + GitHub pull request + + If something is going wrong, submit by email: jretz@ischool, + shreyas@ischool + + We'll help you submit the pull request, but you'll get full credit + +??? + +## Submission + + + The pull request is a way for Shreyas and I to easily see changes, grade + + It'll give you good experience, but it is not a fundamental skill of the + class, so I'm not too worried about it diff --git a/slides/2014-02-06-Preprocessing.html b/slides/2014-02-06-Preprocessing.html new file mode 100644 index 0000000..26bb060 --- /dev/null +++ b/slides/2014-02-06-Preprocessing.html @@ -0,0 +1,698 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-06-Preprocessing.markdown b/slides/2014-02-06-Preprocessing.markdown new file mode 100644 index 0000000..8045cd5 --- /dev/null +++ b/slides/2014-02-06-Preprocessing.markdown @@ -0,0 +1,529 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Preprocessing + +--- + +## Real World is Dirty + +### Incomplete +missing timestamps for actions + +### Noisy +salary = -10 + +### Inconsistent +age: 42, birthday: 1997-03-07 + +??? + +## Types of dirty + +### Incomplete +lacking some attribute values, containing only aggregate data. +e.g., We often regret not including timestamps on different actions +like UFCing, instead of tracking total votes (aggregation) + +### Noisy +Containing errors, like impossible salary data, or decimals in the +wrong place + +### Inconsistent +If two fields depend on each other in a large dataset, you'll find them +disagreeing. Errors often come from failures: processes failing halfway +into updating + +--- + +## Causes of Problems + + + Humans + + Software + + Hardware + +??? + +## Problems + + + Berkeley experiment to measure temperature across campus + + Turned out average on campus much warmer than external weather services + predicted + + But sample data looked in line with predictions + + Problem: one monitoring station right next to air conditioning unit! + + Hardware failure rare, but with large numbers of machines, probable. e.g., + RAM can suffer ~1 bit/hour/gigabyte (ECC can help) + +--- + +## Inconsistent Different Sources + + + Great value in combining data sources + + Challenge is merging them together, removing duplicates + + Example: Business names + +??? + +## Business names + + + Starbucks vs. Starbucks Coffee Shop + + Buck's vs Bucks + + Trying to use address? Stackbucks vs. Starbucks across the street + + Best strategy here is to use DM/ML techniques on the *combination* of + features to determine likelihood of match. We'll discuss specific + algorithms later in the course + +--- + +## Preprocessing + +### Cleaning +fill missing values, smooth noisy data, identify or remove +outliers, resolve inconsistencies + +### Integration +merging data from multiple sources + +### Reduction +obtain a smaller data set that can sufficiently answer +important questions + +### Transformation +change data to a form that is easier to mine or analyze + +??? + +## Flu Trend Problems (Questions) + + + We have millisecond search resolution, but will only be plotting on a per day basis + + We have the exact text of each query, but just care if it is about the flu or not + + Flu Trends, we sometimes see out of control search bots doing 100,000s of searches per day + + Mobile phone searches and web searches hit different machines, software, logs + + We have IPs in the logs, but will be plotting against geographical areas + +--- + +## Missing Values + +.left-column[ +  + +| Person | Height | +|--------|--------| +| Bob | 6'0 | +| Ashley | - | +| Sam | 5'11 | +| Alice | 5'9 | +| Kate | - | +] +.right-column[ + +] + +??? + +## What to do? + + (Heights are made up) + + We want to get an average class height + + Q: What to do with missing rows? + + ignore, fill, constant, average, average wrt gender + +--- + +## Fill Missing Values + +??? + +## Details + + + Trade-offs + + core to engineering + +--- + +## Fill Missing Values + + + Ignore the record + +??? + +## Details + + + Ignore + + simply drop from data set. Hope there are not too many to affect + answer. Drawbacks? When missing values are all same class (skew data) + +--- + +## Fill Missing Values + + + Ignore the record + + Find value manually + +??? + +## Details + + + Find value manually + + Even for a small class, might be difficult. Get + ruler, measure them. For historical data, impossible. + +--- + +## Fill Missing Values + + + Ignore the record + + Find value manually + + Global constant + +??? + +## Details + + + Global constant + + replace with "N/A" or "6 foot". Can skew data, or cause + data to pop in other analysis (all grouped together) + +--- + +## Fill Missing Values + + + Ignore the record + + Find value manually + + Global constant + + Average + +??? + +## Details + + + Average + + Mean or median. Either one has potential problems. + +--- + +## Fill Missing Values + + + Ignore the record + + Find value manually + + Global constant + + Average + + Average with respect to class + +??? + +## Details + + + Average with respect to class + + gender. Average female/male height to fill + in values + +--- + +## Fill Missing Values + + + Ignore the record + + Find value manually + + Global constant + + Average + + Average with respect to class + + "Most probable" + +??? + +## Details + + + "Most probable" + + Think of as another step from avg -> class avg. Now + throw in other details: age, family history, shoe size. Then weight + depending on how much those factors are correlated. Pretty soon you have a + regression or Bayesian model, which will cover later + +--- + +## Normalization + + + Type of data transformation to make reasoning and comparison easier + + Is 6' tall? + + Coefficients on attributes in regressions understandable + +??? + +## Context, Comparison + + + 6' Might be tall for this class, but not on a basketball team + + How to know when a data point is "average" or towards the top of a range? + + For our housing model, we wanted to use sq. footage and # of bedrooms. But + the sq. footage number is huge compared to bedrooms. If we didn't + normalize, a formula for determine house price might seem to indicate that # + of bedrooms was way more important + +--- + +## Min-max + +.white-background[ + +] + +??? + +## New Range + + Typically new range is + + [0-1] (thought of as %) + + [-1-1] (though of as bad->good + +--- + +## Z-score + +.white-background[ + +] + +??? + +## Uses + + + When you want a relative measure of deviation + + When you have a distribution estimate, but are unsure of absolute min-max + +--- + +## Comparison + + + + +??? + +## Min-max vs Z-score + + + Min-max: Known range + + Z-score: more expressive range + + Min-max: requires knowing min-max + + Z-score: can estimate with sampling or informed guess + +--- + +## Removing Noise + +### Binning + +create B bins << N data samples, use aggregate statistic of bin +for value + +### Regression + +fit data to a function, use function value + +### Outlier analysis + +find outlying points, understand and/or ignore them + +??? + +## Monitoring Problem + + + For the problem encountered in temperature monitoring, which makes the most + sense? + +--- + +## Trade-offs + +### Binning + +Simple way to remove outliers, but difficult to pick buckets +correctly + +### Regression + +If one metric is a direct function of another, what extra +information does the value provide? + +### Outlier analysis + +Manual process of understanding outliers, ignoring them +can obscure some analysis (e.g., income disparity) + +??? + +## Trade-offs again + + + Remember: this class is exposing you to potential tools, it's up to you + to be asking the right questions, selecting the appropriate algorithms, + interpreting results + +--- + +## Data integration + + + Merging two data sources + + Problem: uniquely identify a concept in both sources + + Find data points that are very "close" to each other, call them the same + with some probability + + Example: [Yelp Menu Data](http://www.yelp.com/menu/tartine-bakery-san-francisco) + +??? + +## Yelp Menu Data + + + Launched menu data in 2012 + + Takes data about the restaurant menu, find reviews & pictures referring to + the menu item + + Joins them together + + Many different metrics for "close": remember them? + +--- + +## Other measures of "close" + +Are ```A``` and ```B``` close? + +| A | B | +|----|-----| +| 2 | 60 | +| 5 | 150 | +| 6 | 180 | +| 10 | 300 | +| 13 | 390 | + +??? + +## Correlation + + + Imagine ```A``` and ```B``` have several different dimensions, maybe things like + length, height, width, radius + + Are they similar? + + On one hand no: clearly different order of magnitude + + Another way to think about similarity is correlation + + All of ```B``` dimensions are 30x of ```A``` + + Maybe just using different units! + + If I plotted ```A``` and ```B``` as x,y, what would the result look like? + +--- + +## χ2 Correlation Test + + + + +??? + +## Motivation + + + Answer: a straight line + + So a correlation coefficient gives a sense of how closely *linearly* + related two data sets are + + Note, besides positive & negative, the slope does not affect the correlation + score, just how well fit the data is + + Also note I said linear: patterns may still be exhibited, but they are not + linearly related, eg 30x + + Details of test are in book, you are expected to understand it + + Motivation: how different are the observed values from the expected? + + Expected is calculated using probability with the assumptions that the sets + are *independent* + +--- + +## Covariance & Correlation + + + Correlation is "normalized" covariance + + Covariance describes the degree to which two data sets track each other in + units of the two data sets + + Correlations describes the degree of similarity without units + +??? + +## Use in industry + + + χ2 used most commonly, handy to have an expected [0-1] range + + "Correlation does not imply causation" + + A->B, B->A, C->A,B, A->B->A..., coincidence + +--- + +## Data Reduction + +### Dimensionality + +remove attributes that are the same or similar to other attributes + +### Numerosity + +represent or aggregate the data, sometimes with precision loss + +### Compression + +generalized techniques to decrease the number of bytes needed +to store data + +??? + +## Deep Dive + + + We're only going to cover selected topics in these areas. + + When reading, make sure to understand the intuition behind the other + techniques, but if we don't cover it in lecture, you won't need to + calculate it in midterm + + Ask questions about the concepts you don't understand! That's what + separates this class from a book :) + + But still potentially useful for your projects! + +--- + +## Subset Selection + + + Two many attributes? + + *Ignore some* + + Tricky part: which to ignore? + + height x width = area + +??? + +## Simple to Sophisticated + + + Ignore the ones that are not helpful + + Ignore an attribute highly correlated with another (cm, in) + + Ignore an attribute that can be built from others + +--- + +## Principal Component Analysis + + + Map data to a location along a few vectors + + + +??? + +## Higher dimensions + + + Remember, 2 dimensions might not make much sense, but becomes useful in + higher number of dimensions + + These points described by two attributes, + + What if we wanted to describe them in just 1 dimension? + + Pick some good vectors (in our case 1) + + Describe where a point is located using only those vectors + +--- + +## Netflix and PCA + + + A user may have many preferences: Mission Impossible, Love Actually, Man + from Nowhere, ... + + Instead of keeping track of every preference, we can summarize + + Action, RomCom, Foreign + +??? + +## Summarize in discovered dimensions + + + With 3 or more "categories", we can reconstruct the user's likely + preferences + + Dimensions don't necessarily fit into human notions: probably is not a + "foreign" dimension, but a subtle combination of other aspects diff --git a/slides/2014-02-06-Probability.html b/slides/2014-02-06-Probability.html new file mode 100644 index 0000000..61ea1eb --- /dev/null +++ b/slides/2014-02-06-Probability.html @@ -0,0 +1,535 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-06-Probability.markdown b/slides/2014-02-06-Probability.markdown new file mode 100644 index 0000000..c993b2a --- /dev/null +++ b/slides/2014-02-06-Probability.markdown @@ -0,0 +1,366 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Probability + +--- + +## Nomenclature + +### Record + +a single entity or concept. Also: data object, sample, example, +instance, data point + +### Feature + +a characteristic or way of describing a record. Also: attribute, +dimension, variable, signal + +??? + +## Slightly different from book + + + The meanings do carry different connotations, but are generally + transferable + + e.g., dimensions is usually used in the math domain + + Feature is usually used in the ML domain + +--- + +## Feature Types + + + Binary: True/False. Also: 0/1 + + Numeric: Involving numbers. Also: integer, float, double + + Ordinal: Feature with sortable values. + + Discrete: countable, finite set. Also: classes + + Continuous: unbounded numeric number. Also: integer, float, double + + Enumerated: feature named, discrete values. Also: nominal, classed + +??? + +## Rain data set + + + Stored did/dot not rain + + Stored how many inches it rained + + Stored the day as an integer offset from Jan 1 1970 + + Stored weather information: Sunny, Partly Sunny, Cloudy, Rainy + + Stored barometer reading + + Stored day of the week + +--- + +## Central Tendency + + + +??? + +## Define + + + Mean: "average" all data points divided by size of set + + Median: middle value + + Mode: The value most likely to be picked + + discrete: most common value + + continuous: max probability density function + +--- + +## Skew Positive + + + +??? + +## Skew + + + Think about ```mean - mode``` + + Or think about where the "tail" is + +--- + +## Skew Negative + + + +--- + +## The Long Tail + + + Most popular are *very* popular + + Everything else, not so much + + But there's a lot of everything else + + + +??? + +## Movies + + + Current releases: millions of people watching + + Older movies are rented by < 1 person a week + + What is the skew? + + Power law distribution (please follow up on Wikipedia or a stats class) + + Distributions are important, but will only be covered as necessary + +--- + +## Dispersion + + + Centrality not the whole story + +.white-background[ + +] + +??? + +## Differences + + + Wildly different data sets can still share many of these characteristics + +--- + +## Quartiles + +.limit-size[.white-background[ + +]] + +??? + +## Parts + + + Go back to our unskewed normal distribution + + Quartiles divide the data into quarters + + InterQuartile Range is the distance of the middle two quartiles + + BoxPlot is one of the most useful tools for data. For public results, I + almost never want to see scatter plot or bar charts. I want to see box + plots. + + Bottom, we spit it up into standard deviations + + Variance measures, on average, how far points are away from the mean + + Standard deviation is the square root of the variance + +--- + +## Standard Deviation + + + Within 1: 68% + + Within 2: 95% + + Within 3: 99.7% + +.white-background[ + +] + +??? + +## Standard Deviation + + + Useful for thinking about what % of outliers you'd like to catch + + We use it for alerting: let us know when we're 2 stddev away from the + median, there's a very small likelihood of that happening + +--- + +## Visualization Tools + + + Python: Matplotlib + + R: builtin + + Matlab: builtin + + Octave: builtin (gnuplot) + + HTML: D3.js + +??? + +## Covered later + + + Chapter 2 is going to cover some visualization stuff + + We're going to cover visualization a bit later in the course, and more of a + "how its done in industry" + + There is another class on visualization in general + +--- + +## Mathmatical Representation +  + +| | Bad Boys | Robin Hood | Waterworld | +|--------|----------|------------|------------| +| Prabha | 1 | 3 | 2 | +| AJ | 5 | 4 | 3 | +| Victor | 4 | 4 | 1 | + +  + +```octave +[ 1 3 2 + 5 4 3 + 4 4 1 ] +``` + +??? + +## Matrix + + + Matrix representations very powerful, as we'll see later in class + + Usually rows are records, columns are attributes + + Sometimes you can think of data in different ways, can take the transpose + of the matrix to get attributes about movies + +--- + +## Similarity | Distance + + + Two sides of the same coin + + ```similarity = 1 - distance``` + + We'll use these metrics for many other algorithms + +??? + +## Core Concept + + + Many data mining techniques rely on finding a way to quantify similarity + + When you think about questions like "how similar are two users?" "is this + text plagiarism?" "are these products likely to be purchased together?" + + All are ways of thinking about similarity + +--- + +## Nominal Distance + + + Ratio of mismatches to potential matches + + Why can't we take the sum of the mismatches? + +??? + +## Nominal + + + Nominal means we can't compare two values: there is no ordering + + All we can do is take ratio of the ones that are exactly the same + + The book describes how to think about this in terms of matrices + +--- + +## Binary Distance + + + Could use Nominal Distance: count all exact matches or mismatches + + Could use Numeric Distance: just treat values as 0/1 + + asymmetric binary dissimilarity: don't care about *negative matches* + + ```mismatches / (positive_matches + mismatches)``` + + asymmetric binary similarity: care more about *positive matches* than mismatches + + ```positive_matches / (positive_matches + mismatches)``` + +??? + +## Binary + + + Nominal problem: for rare attributes, like a disease, two people who + *don't* have the disease, aren't necessarily very similar + +--- + +## Jaccard Coefficient + + + Asymmetric binary similarity + + More commonly used for calculating set similarity + + ```|intersection| / |union|``` + + "Jimmy likes pizza" | "Shreyas likes pizza" + +??? + +## Jaccard + + 1. Break up into a set + 1. calculate # in intersection + 1. calculate # in union + 1. divide + +--- + +## Euclidean distance + + + Straight line between two points + + Again: usually considered with just (x,y), but can calculate for any number + of dimensions + +.white-background[ + +] + +??? + +## Ordinary + + + Distance as you probably learned in grade school + +--- + + +## Manhattan distance + + + How many blocks would you need to walk between two points? + +.white-background[ + +] + +??? + +## Usefulness + + Obviously useful for maps/directions + + But haven't seen it used much beyond that + +--- + +## Lp norm + + + Euclidean distance and Manhattan can be generalized + + Euclidean distance referred to as L2 norm + + Chebyshev distance is L∞ + +.white-background[ + +] + +??? + +## Lp space + + + Important for signal processing, math, other applications + + You may want to study these distances for comparing wave forms, like audio + +--- + +## Ordinal Distance + + + Normalize the ordinal rankings + + Use a numerical distance metric + +--- + +## Cosine Similarity + + + Jaccard similarity can work well for sets of roughly equal size + + How to compare sets with a large difference in magnitude? + + Model them as vectors, take the cosign of the angle between + +.white-background[ + +] + +??? + +## Cosine + + + Why cosine? Hint: normalization + + img: http://cs.carleton.edu/cs_comps/0910/netflixprize/final_results/knn/index.html + +--- + +## Cosine Example + + + "Jimmy likes pizza" | "Shreyas likes pizza" diff --git a/slides/2014-02-13-Data-Warehouse.html b/slides/2014-02-13-Data-Warehouse.html new file mode 100644 index 0000000..c44bc7b --- /dev/null +++ b/slides/2014-02-13-Data-Warehouse.html @@ -0,0 +1,676 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-13-Data-Warehouse.markdown b/slides/2014-02-13-Data-Warehouse.markdown new file mode 100644 index 0000000..89eeb40 --- /dev/null +++ b/slides/2014-02-13-Data-Warehouse.markdown @@ -0,0 +1,507 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Data Warehouse + +--- + +## Database Types + + + Data Warehouse + + Database designed for using data to make decisions + + OLAP + + OnLine Analytical Processing + + OLTP + + OnLine Transactional Processing + +??? + +## Data Mining + + + These databases are often the starting point for data mining in companies + + Most of the data sets from companies typically come from exporting some + portion of their data warehouse + +--- + +## Properties + + + Subject Oriented + + Focus on core business objects + + Integrated + + Access to as much data as possible + + Time Variant + + Contains historical data with time parameter + + Non-volatile + + Updated (relatively) infrequently, in bulk + +??? + +## Examples + + + Yelp users can be directed to a datacenter depending on conditions. This + data probably doesn't need to be in the DW + + Yelp has several databases: log summaries, user info, salesforce. Most + useful if they are all in the same place + + Operationally, when someone changes their address, we just overwrite it in + the OLTP DB. But DW potentially cares about the old value + + OLTP writes to rows every time someone updates profile, review, etc. Lots of + simultaneous updates. DW: typically once a day, in bulk + +--- + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + + Short transactions vs. Long queries + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + + Short transactions vs. Long queries + + Indexes on strategic fields for fast lookups vs. Full table scans + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + + Short transactions vs. Long queries + + Indexes on strategic fields for fast lookups vs. Full table scans + + Simultaneous queries: 1-100 vs 100s-1000s + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + + Short transactions vs. Long queries + + Indexes on strategic fields for fast lookups vs. Full table scans + + Simultaneous queries: 1-100 vs 100s-1000s + + Simple updates vs Complex queries + +--- + + +## OLAP or OLTP? + + + Transactional Focus vs. Analytic Focus + + Used by Managers, Executives vs. DBAs, programmers + + Contains current information vs. historical + + Variety of differently summarized data vs normalized + + Short transactions vs. Long queries + + Indexes on strategic fields for fast lookups vs. Full table scans + + Simultaneous queries: 1-100 vs 100s-1000s + + Simple updates vs Complex queries + + Guaranteed high performance vs Flexibility & Customization + +--- + +## Overview + + + +??? + +## From the front + + + Analytics team uses charts, reports, etc. + + Generated from an OLAP server + + Which uses data from a data warehouse (often DW and OLAP server are + integrated) + + Uses a process (ETL) to move the data from other source into DW + +--- + +## Types of Data Warehouses + + + Enterprise + + turnkey solution, often expensive, sophisticated but complex + ingestion, integration, security features + + Data Mart + + Smaller, limited in scope. Designed for specific team or + department + + Virtual + + OLAP built on top of an OLTP database + + Cloud + + Google BigQuery + + Amazon Redshift + +??? + +## Vendors + + + Enterprise + + Oracle + + Greenplum + + AsterData + + Data Mart + + MySQL + + PostgreSQL + + Virtual + + MySQL + + PostgreSQL + + views or admin interface + +--- + +## Metadata + + + Data about the data being stored + + Overview: schema, languages + + Operational: last update, query latency + + Algorithms: normalization, transformation + + Performance: job dependencies + + Business: ownership, permissions + +??? + +## Considerations + + + As soon as several people start using the DW, they'll need to know about + how it is put together + + Metadata often comes as an after thought but is an important part of + scaling + +--- + +## Overview + + + +??? + +## Data Cubes + + + What are those cubes in the OLAP area? + +--- + +## Data Cubes + +.left-column[ + + + Way of thinking about multi dimensional data + + Useful metaphor because one can reason about ways to satisfy a query +] + +.right-column[ + +] + +--- + +## Dimensions + +| | Day 1 | Day 2 | Day 3 | +|----------|-------|-------|-------| +| Region 1 | $200 | $80 | $600 | +| Region 2 | $300 | $90 | $650 | +| Region 3 | $400 | $100 | $700 | + +??? + +## Data... Square + + + More of a data square: only 2 dimensions + + Advertising on Yelp + + Now we want to know Product Type (CPC, CPM, National) + +--- + +## Cube: 3rd Dimension + + + +??? + +## More + + + Now we want to know Page Type (Business, Search, Home) + + Hard to draw 4 dimensions, so instead... + +--- + +## Multi-Cube + + + +??? + +## More + + + Keep adding dimension as necessary + +--- + +## Lattice + + + +??? + +## Moving + + + Move back and forth from our 2d table + + To our 3d cube, to our 4d multi-cube + + The lower dimensional parts are summaries + + At the extreme is just the total (i.e., all money made) + +--- + +## Schemas + +.left-column[ + + + A *Data Cube* is a way of visualizing multi dimensional data +] + +--- + +## Schemas + +.left-column[ + + + A *Data Cube* is a way of visualizing multi dimensional data + + A *Star Schema* is a way to store the data in a database +] + +.right-column[ + +] + +--- + +## Fact table + +.white-background[ + +] + +--- + +## Dimension table + +.white-background[ + +] + +--- + +## Dimension tables + +.white-background[ + +] + +--- + +## Dimension tables + +.white-background[ + +] + +--- + +## Dimension tables + +.white-background[ + +] + +--- + +## Star Schema + + + +--- + +## Dimensions of Dimensions + +.white-background[ + +] + +--- + +## Dimensions of Dimensions + +.white-background[ + +] + +--- + +## Dimensions of Dimensions + +.white-background[ + +] + +--- + +## Dimensions of Dimensions + +.white-background[ + +] + +??? + +## Schema Name? + + + Any guesses what this fractal looking schema is called? + +--- + +## Snowflake Schema + + + Schema with radiating dimension tables + + + +--- + +## Constellation Schema + + + Schema with several fact tables and related dimensions + + + +--- + +## Data Warehouse Operations + + + Rollup + + Summarize data along fewer dimensions + + Drill-down + + Get details within a particular dimension + + Slice + + Select a particular value in a dimension + + Dice + + Consider a subset of the values in a dimension + + Pivot + + Swap, or rotate dimensions + +??? + +## Examples + + + Rollup + + What countries are selling the most ads? + + Drill-down + + Spike in Q1 ad views. Which month most responsible? + + Slice + + Chart sales only for CPC + + Dice + + Only look at sales in US, IT, DE + + Pivot + + Swap axis on a chart + +--- + +## Materialized Views + + + View + + virtual table defined by a query + + Non-materialized + + Calculate summaries on the fly + + Fully materialized + + Pre-compute and store + + Partially materialized + + Variety of strategies: e.g., cache results after calculating + +??? + +## Usefulness + + + In DW, we're often storing different cubes in the lattice + + For the country sample, do we have those summaries stored in another DB + table? On disk? By month? Year? + + Storing all possible summaries is expensive when loading data, + and requires a lot more storage + +--- + +## Architecture + + + ROLAP + + Relational. Implement OLAP on top of a relational database + + MOLAP + + Multidimensional. Implements data cube as storage paradigm + + HOLAP + + Hybrid. Data in ROLAP, rollups in MOLAP + + Specialized + + Often distributed storage, parallel DB technology + + NoSQL + + Store data as key-value pairs, optimized in different ways + +??? + +## Details + + + ROLAP: MySQL, PostgreSQL + + MOLAP: Oracle, Palo + + HOLAP: Microsoft SQL Server + + Specialized: AsterData, Greenplumb + + NoSQL: Hive, BigTable, Cassandra + +--- + +## *Break* diff --git a/slides/2014-02-13-MapReduce.html b/slides/2014-02-13-MapReduce.html new file mode 100644 index 0000000..bf14b38 --- /dev/null +++ b/slides/2014-02-13-MapReduce.html @@ -0,0 +1,922 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-13-MapReduce.markdown b/slides/2014-02-13-MapReduce.markdown new file mode 100644 index 0000000..55d3eee --- /dev/null +++ b/slides/2014-02-13-MapReduce.markdown @@ -0,0 +1,753 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# MapReduce + +??? + +## Spoilers + + + Don't look ahead in the slides + + If you know MapReduce, try to let others answer and genuinely think about + how *you* would solve the problem. + +--- + +## Yelp has a problem + + + 2+ TB of logs per day + + Each GB takes at least 2 minutes to process + + How long to handle a day's logs? + + + +??? + +## Too long + + + On a single machine 65+ hours! + + If we really had only a single machine, we wouldn't be able to keep up! + + Mistake can't be fixed in a day (billing especially important) + +--- + +## Solution? + + + Don't use one machine! + + What are the new challenges? + +--- + + +## Solution? + + + Don't use one machine! + + What are the new challenges? + + Distributing data + + Calculating overall statistics + + Failures + +??? + +## New Challenges + + + With many machines, how do they get access to the 2 TB of logs? + + How do they coordinate who gets which section of logs? + + How do we calculate the average? + + What happens when one of the boxes dies? + + Detecting failure (timeout waiting for data? Out of band?) + + Decide who takes over the data + +--- + +## Do It Yourself + + + There are many ways to deal with these challenges + + Often, people would "roll" their own solutions depending on the problem + +??? + +## Dependencies + + + Did you have a super-computer? + + What programming language were you using? + + Type of problem being solved (working on graphs, or web logs, ...) + +--- + +## MapReduce + + + Google implemented a generic solution and shared the idea + + + +??? + +http://research.google.com/archive/mapreduce.html + +--- + +## Big Idea + + + Simplify, limit solution expression + + Enable sophisticated implementation + +### Interface + + + mapper() + + reducer() + +### Implementation + + + Reliably use 1,000s of machines for each job + +??? + +## Really Big Idea + + + Limiting yourself to what can be expressed may seem like a loss + + But it enables the implementation to handle the problems we talked about + + And then can be used as understandable building blocks + +--- + +## MapReduce + + + Mapper + + Extracts a property to summarize over + + Reducer + + Summarizes all items with a particular property + +### Simple Constraint + ++ Each operation is stateless + +??? + +## Reading + + + Reading this week includes a video explaining MapReduce much more generally + + This lecture will focus on it from a practical standpoint for homework + + MapReduce's main benefits are for running on many machines, with fault + tolerance + + But we'll just practice on one machine + +--- + +## Example + + + Web application logs + + Question: How many instance of each of these actions have we seen? + + Business views + + User profile views + + Searches + +??? + +## Details + + + Business Views: [I.B.'s Hoagies](http://www.yelp.com/biz/i-b-s-hoagies-berkeley) or [Gypsy's Trattoria Italiano](http://www.yelp.com/biz/gypsys-trattoria-italiano-berkeley) + + User profile: [Jimmy](http://jretz.yelp.com) + + Searches: [cheese near Downtown Berkeley](http://www.yelp.com/search?find_desc=cheese&find_loc=Downtown+Berkeley%2C+Berkeley%2C+CA&ns=1) + +--- + +## Logs + +```json +{'page_type': 'search', +'user': 'jimmy', 'query': ...} + +{'page_type': 'biz_view', +'user': 'shreyas', 'biz_id': 55} + +{'page_type': 'user_profile', +'user': null, 'profile_id: 123} + +... +``` + +??? + +## Logs + + + JSON logs, various types of information + + entire record on one line (wrapped for slides) + +--- + +## Mapper + + + Input: Key, Value + + Output: Keys, Values + +--- + +## Mapper Example + + + Input Key: Log line number + + Input Value: Log line text + + Output Key: Action + + Output Value: times this action occurred *on this line* + +??? + +## Counts + + + Log line number is not helpful in our specific case + + Log line text: we hope it is machine readable so we can accurately extract + the action + + It has datetime, cookie, action, etc. + + How many times has this action occurred? 1 + + Tunnel vision: all we care about is this line + +--- + +## Actions? + +```text +search 1 +biz_view 1 +user_profile 1 +search 1 +biz_view 1 +search 1 +biz_view 1 +user_profile 1 +search 1 +``` + +??? + +## Middle Step + + + From log lines, we've extracted the information out that we care about + + The counts and the actions + + Next step summarize + + Next step after Mapper? + +--- + +## Reducer + + + Input: Key, Values + + Output: Keys, Values + +??? + +## Values + + + Note: The input is value*s* (plural) + + Because we get a key and all of its associated values + + Remind me: what are we trying to get out of this computation? + + So what do you think the output keys are? + + Values? + +--- + +## Reducer Example + + + Input Key: Action + + Input Values: Counts: ```[1, 1, 1, 1]``` + + Output Key: Action + + Output Value: Total Count + +??? + +## Details + + + Action is *one of* search/biz_view/profile_view + + To get total count, sum all of the counts + +--- + +## Example Output + + + Output Key: Action + + Output Value: Total Count +```html +"search" 4 +"user_profile" 2 +"biz_view" 3 +``` + +--- + +## Point? + + + A lot of work for counting! + + More complex calculations can be done this way, eg. PageRank + + Stateless constraint means it can be used across thousands of computers + +??? + +## Details + + + By only looking at keys and values, can optimize a lot of back-end work + + Where to send the results? + + What to do when a computer fails? (Just restart failed part) + +--- + +## Implementation + +```text +biz_view 1 +user_profile 1 +search 1 +search 1 +biz_view 1 +search 1 +biz_view 1 +user_profile 1 +search 1 +``` + +??? + +## Intermediate + + + This was the situation after map + + Keys all jumbled + + What Hadoop does is sort them and distribute them to computers + +--- + +## "Shuffle" + +```text +biz_view 1 +biz_view 1 +biz_view 1 +search 1 +search 1 +search 1 +search 1 +user_profile 1 +user_profile 1 +``` + +??? + +## Distribute + + + Now it is easy to distribute, and can handle all the ```biz_view``` at once + +--- + +## Inputs + + + MapReduce distributes computing power by distributing input + + Input is distributed by splitting on lines (records) + + You cannot depend on lines being "together" in MapReduce + +??? + +## Splitting Files + + + Imagine you have a lot of large log files, GBs each + + You'd like to let different machines work on the same file + + Split file down the middle, well, at least on a newline + + Enable two separate machines to work on the parts + + You don't know what line came before this one + + You don't know if you will process the next line + + Only view is this line + + Real life slightly more complicated, but mostly hacks around this + +--- + +## Word Count + +.tight-code[ +```json +{"text": "Greatest pizza ever", "stars": 2, "user": ...} +{"text": "good pizza selection", "stars": 5, "user": ...} +``` +] + + Total uses of a word across all reviews + +??? + +## Classic + + + This is the traditional MapReduce example, so let's solve it + + No skipping ahead + +--- + +## Steps + +??? + +## Hints + + + What's the first step of MapReduce? + +--- + +## Steps + + + Mapper + +??? + +## Hints + + + What part of the record are we interested in? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + +??? + +## Hints + + + What do we want to do with the text? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + + Split text up into words + +??? + +## Hints + + + Mapper: Key / Value? What are we grouping by? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + + Split text up into words + + Key: word ; Value: count + +??? + +## Hints + + + Next step of MapReduce? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + + Split text up into words + + Key: word ; Value: count + + Reducer + +??? + +## Hints + + + What are the reducer inputs? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + + Split text up into words + + Key: word ; Value: count + + Reducer + + Key: word ; Values: all counts for that word + +??? + +## Hints + + + With all of these counts, how do we summarize? + +--- + +## Steps + + + Mapper + + Extract the ```text``` of the review + + Split text up into words + + Key: word ; Value: count + + Reducer + + Key: word ; Values: all counts for that word + + ```sum(values)``` + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + + Greatest: ```1``` + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + + Greatest: ```1``` + + pizza: ```1``` + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + + Greatest: ```1``` + + pizza: ```1``` + + ever: ```1``` + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + + Greatest: ```1``` + + pizza: ```1``` + + ever: ```1``` + + Reducer + + Key: ```pizza``` + + Values: ```[1, 1]``` + +--- + +## Example + + + ```"Greatest pizza ever"``` + + Mapper + + Greatest: ```1``` + + pizza: ```1``` + + ever: ```1``` + + Reducer + + Key: ```pizza``` + + Values: ```[1, 1]``` + + Output: ```["pizza", 2]``` + +--- + +## Multi-Step + + + Not all computations can be done in a single MapReduce step + + Map Input: `````` + + Reducer Output: `````` + + Compose MapReduce steps! + +??? + +## Output as Input + + + The output of one MapReduce job can be used as the input to another + +--- + +## Examples + + + PageRank: Multiple steps till solution converges + + Multi-level summaries + +??? + +## PageRank + + + PageRank is an algorithm for calculating the importance of a page + + But it depends on the importance of every page pointing to it! + + So iteratively calculate the importance of all pages + + Find average presidential donations by candidate, then normalize averages + +--- + +## Unique Review, Step 1 + + + Determine the Review ID with the most unique words + +??? + +## Questions + + + For our purposes, what is always the mapper input? + +--- + +## Unique Review, Step 1 + + + Determine the Review ID with the most unique words + + Mapper Input: `````` + +??? + +## Questions + + + What feature do we want to calculate first? + +--- + +## Unique Review, Step 1 + + + Determine the Review ID with the most unique words + + Mapper Input: `````` + + Mapper Output: `````` + +??? + +## Questions + + + Given this mapper output, what *must* the reducer input be? + +--- + +## Unique Review, Step 1 + + + Determine the Review ID with the most unique words + + Mapper Input: `````` + + Mapper Output: `````` + + Reducer Input: `````` + +??? + +## Questions + + + What property about a review are we interested in? + +--- + +## Unique Review, Step 1 + + + Determine the Review ID with the most unique words + + Mapper Input: `````` + + Mapper Output: `````` + + Reducer Input: `````` + + Reducer Output: `````` if the word is unique + +??? + +## Questions + + + Given the reducer output, what *must* the mapper input be (for chained + MapReduce steps) + +--- + +## Step 2: Count Unique Words in Each Review + + + Mapper Input: `````` + +??? + +## Questions + + + What do we want to group by? + +--- + +## Step 2: Count Unique Words in Each Review + + + Mapper Input: `````` + + Mapper Output: `````` + +??? + +## Questions + + + Given this mapper output, what *must* the reducer input be? + +--- + +## Step 2: Count Unique Words in Each Review + + + Mapper Input: `````` + + Mapper Output: `````` + + Reducer Input: `````` + +??? + +## Questions + + + What are we calculating? + +--- + +## Step 2: Count Unique Words in Each Review + + + Mapper Input: `````` + + Mapper Output: `````` + + Reducer Input: `````` + + Reducer Output: `````` + +??? + +## Questions + + + Given the reducer output, what *must* the mapper input be (for chained + MapReduce steps) + +--- + +## Step 3: Max + + + Mapper Input: `````` + +??? + +## Questions + + + We're calculating a statistic over what portion of the data set? + + How do we get all of the data to one reducer? + +--- + +## Step 3: Max + + + Mapper Input: `````` + + Mapper Output: ```<"MAX", [sum, review_id]>``` + +??? + +## Questions + + + Given this mapper output, what *must* the reducer input be? + +--- + +## Step 3: Max + + + Mapper Input: `````` + + Mapper Output: ```<"MAX", [sum, review_id]>``` + + Reducer Input: ```<"MAX", [[sum, review_id],...]>``` + +??? + +## Questions + + + What stat are we calculating? + +--- + +## Step 3: Max + + + Mapper Input: `````` + + Mapper Output: ```<"MAX", [sum, review_id]>``` + + Reducer Input: ```<"MAX", [[sum, review_id],...]>``` + + Reducer Output: `````` of the ```max(sum)``` diff --git a/slides/2014-02-13-Project.html b/slides/2014-02-13-Project.html new file mode 100644 index 0000000..0a2d669 --- /dev/null +++ b/slides/2014-02-13-Project.html @@ -0,0 +1,287 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-13-Project.markdown b/slides/2014-02-13-Project.markdown new file mode 100644 index 0000000..bc11c32 --- /dev/null +++ b/slides/2014-02-13-Project.markdown @@ -0,0 +1,118 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Final Project + +--- + +## Deliverables + + + Project Proposal (due April 9) + + Code (due April 30) + + Data (URL or sampled, due April 30) + + Presentation (May 8) + + Written Report (due May 14) + +--- + +## Groups + + + 1-3 people + +--- + +## Proposal + + + Informal, not graded + + A plan for your project + + After the proposal you should just be executing, not brainstorming + + Discussions before proposal encouraged + +--- + +## Report + + + Introduction + + problem, insights, solutions + + Problem + + motivation, data set + + Solution + + techniques, failures, examples + + Details + + parameter tuning, software engineering challenges + + Related work + + including resources you used + + Further work + + any remaining ideas you have + +??? + +## Notes + + + Length + + around 2 pages, but more important to hit these points + + Introduction + + "gosh, if these insights are true, it would be really + exciting" + + Problem + + include problems with your data set + + Solution + + if you do clustering, give examples of a cluster you found and + individual data points that it contains + + Details + + what commands did you use for particular libraries. Can someone + duplicate your work? + + Formats + + PDF, Google Doc + +--- + +## Research Paper + + + [How to write a good research paper](http://research.microsoft.com/en-us/um/people/simonpj/papers/giving-a-talk/writing-a-paper-slides.pdf) + + But much shorter! + +??? + +## Skip sections + + + No abstract + + Think paragraphs instead of pages + +--- + +## Presentation + + + ~10 minutes + + Think: 1 slide per paragraph + + Focus on images, stories, examples + + Motivate people to read your paper, don't read it to them + +--- + +## Code + + + Another GitHub repository + + Public is best, but if private then add Shreyas and me + + Include README with info on how to run algorithms + + Suggestion: include your paper + +??? + +## Reproducible + + + Imagine if someone wanted to reproduce your results + + Also great for portfolio (with Paper) + +--- + +## Data + + + If large, just point to URL in paper and README + + git is not very good at handling large data files, don't commit large + datasets + + If very large, talk with me about hosting on Amazon Web Services + + We'll have computing resources available from Amazon diff --git a/slides/2014-02-13-mrjob.html b/slides/2014-02-13-mrjob.html new file mode 100644 index 0000000..a525dc9 --- /dev/null +++ b/slides/2014-02-13-mrjob.html @@ -0,0 +1,272 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-13-mrjob.markdown b/slides/2014-02-13-mrjob.markdown new file mode 100644 index 0000000..63030a6 --- /dev/null +++ b/slides/2014-02-13-mrjob.markdown @@ -0,0 +1,103 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Lab: mrjob + + + Understand ```review_word_count.py``` + + Find review with most unique words + + Fill in ```unique_review.py``` + + Find similar users + + Write ```user_similarity.py``` + +??? + +## mrjob + + + Using the Yelp Academic Dataset + + In lecture, we covered the steps for most unique words + + Use Jaccard similarity for user_similarity + +--- + +## Data + + + May copy or use in place on ischool machine + + ischool: + +```bash +~jretz/ + yelp_phoenix_academic_dataset/ + yelp_academic_dataset_review.json +``` + +## Agreement + + + Dataset can only be used for academic purposes + + You can download it yourself from the [Yelp Dataset Challenge](http://www.yelp.com/dataset_challenge/) + +--- + +## Understand review_word_count.py + + + Note, this will take ~8 minutes on the ischool machine + + Don't run it just yet! + +.tight-code[ +```bash +$ python review_word_count.py yelp_academic_dataset_review.json + +no configs found; falling back on auto-configuration +creating tmp directory /tmp/review_word_count.jretz.20130215.071901.095847 +reading from file +> /home/jretz/src/datamining290/code/venv/bin/python review_word_count.py --step-num=0 --mapper /tmp/review_word_count.jretz.20130215.071901.095847/input_part-00000 +writing to /tmp/review_word_count.jretz.20130215.071901.095847/step-0-mapper_part-00000 +Counters from step 1: + (no counters found) +... +Streaming final output from /tmp/review_word_count.jretz.20130215.071901.095847/output +"4" 2 +"5" 1 +"50" 1 +"6" 2 +"7" 2 +"70s" 1 +"9" 2 +"a" 46 +"abbey" 4 +"able" 1 +"about" 4 +``` +] + +--- + +## A trick for running quickly while developing... + +.tight-code[ +```bash +$ head -n 1000 yelp_academic_dataset_review.json | \ + python review_word_count.py +``` +] + ++ That runs over the first 1,000 lines ++ When things start looking good, try 10,000, then the entire file + +--- + +## Fill in unique_review.py + + + Mutli-step map reduce + + Steps are explained in lecture + + Skeleton in code + +--- + +## Write user_similarity.py + + + Find users >= 0.5 similarity + + User Similarity: Jaccard similarity of businesses reviewed + + {BizA, BizB, BizC} ~ {BizF, BizB, BizG} diff --git a/slides/2014-02-20-Bayes.html b/slides/2014-02-20-Bayes.html new file mode 100644 index 0000000..875e736 --- /dev/null +++ b/slides/2014-02-20-Bayes.html @@ -0,0 +1,453 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-20-Bayes.markdown b/slides/2014-02-20-Bayes.markdown new file mode 100644 index 0000000..a86394d --- /dev/null +++ b/slides/2014-02-20-Bayes.markdown @@ -0,0 +1,284 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Classification: Bayes + +--- + +## Confusion Matrix + + + What are the ways that classification can be wrong? + +  + +| | Predict: Positive | Predict: Negative | +|------------------|-------------------|-------------------| +| Actual: Positive | True Positive | False Negative | +| Actual: Negative | False Positive | True Negative | + +??? + +## Obtain Data + + + How do we obtain this data? + +--- + +## Testing Data + + + Data used to test a learned model + + Test data was not used to learn + + Where does test data come from? + + + +??? + +## Not from storks + + + img: http://adamsparkadventures.blogspot.com/2011/09/stork-watch.html + +--- + +## Training Data + + + Set aside a portion of training data to test with + + Test data: + + + +--- + +## Set Aside Testing + + + + Testing Data | Training Data + +??? + +## Colors + + + Red: Testing + + Green: Training + +--- + +## Cross Validation + + + +Train and test model with different subsets of data + +??? + +## Testing the model + + + This is used to test the *model* + + How well does it perform with a variety of inputs? + + Is it robust against outliers + +--- + +## K-Fold Validation + + + +Test against K sections of the data + +??? + +## Statistical Significance + + + Similar to the concept in stats: the more distinct samples you have, the + better you know your data + +--- + +## K-Fold Validation + + + +--- + +## Bayes Theorem + +.white-background[ + +] + +Can calculate a posterior given priors + +??? + +## Read + + + Probability of A given B equals probability of B given A times prob of A + divided prob of B + + Importance is that we can figure out what future probabilities are based on + what we've already seen + +--- + +## Spam + +.white-background[ + +] + +Find the probability of spam given it contains a particular word + +??? + +## Words + + + What words would you associate with spam? + + Are these the same across all people? + + Why might you want to train a classifier per person? + +--- + +## Multiple Words + + + How to calculate probabilities of multiple independent events occurring? + +??? + +## Naive + + + Words are not independent + + San? Francisco is more likely + +--- + +## Multiple Words + + + How to calculate probabilities of multiple independent events occurring? + + Model words as independent events + + Multiply probabilities + +??? + +## Naive + + + Words are not independent + + San? Francisco is more likely + + But works surprisingly well in practice + +--- + +## Practical concerns + + + What is the probability of a word we've never seen before? + +??? + +## Solutions + + + divide by 0. Instead, add 1 to all words + +--- +## Practical concerns + + + What is the probability of a word we've never seen before? + + Underflow: multiplying small numbers eventually causes rounding to 0 + +??? + +## Solutions + + + use log of probabilities + +--- +## Practical concerns + + + What is the probability of a word we've never seen before? + + Underflow: multiplying small numbers eventually causes rounding to 0 + + Normalizing words: v1agra + +??? + +## Solutions + + + come up with rules + +--- + +## Ensemble + + + Using multiple models simultaneously + + Run all classifiers over new data, take majority vote + + Netflix Prize won with combination of models from several teams + +??? + +## Requirements + + + Nice thing is that the diversity of models is important, and not so much + the accuracy of any single model + +--- + +## Bootstrap Aggregating + + + Bagging: training data collected with replacement + + Learn models on different samples + + Run models on new incoming data + + + +??? + +*TODO should both be indented?* + +## Trade-offs + + + Fairly simple: + + Majority vote + + Train models independently + + img: http://cse-wiki.unl.edu/wiki/index.php/Bagging_and_Boosting + +--- + +## Boosting + + + Train classifier to catch what the last one missed + + Train and test first classifier + + Find classification failures + + Weight those failures more heavily in training a new model + + Weight models by their accuracy + +??? + +## Trade-offs + + + Boosting can be susceptible to outliers + + Takes longer to train + + Observed to be more accurate + +--- + +## Many Decision Trees + + + Train trees with random selection of attributes and a subset of the data + + Combine trees using majority or weights + + What to call many arbitrarily picked trees? + +--- + +## Random Forests + + + Used successfully in many recent competitions + + Carry over robustness properties from individual decision trees + + Can be trained in parallel + + + +??? + +## Parallel + + + Potentially good fit for MapReduce paradigms diff --git a/slides/2014-02-20-Decision-Trees.html b/slides/2014-02-20-Decision-Trees.html new file mode 100644 index 0000000..2350bac --- /dev/null +++ b/slides/2014-02-20-Decision-Trees.html @@ -0,0 +1,658 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-20-Decision-Trees.markdown b/slides/2014-02-20-Decision-Trees.markdown new file mode 100644 index 0000000..304c086 --- /dev/null +++ b/slides/2014-02-20-Decision-Trees.markdown @@ -0,0 +1,489 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Classification: Decision Trees + +--- + +## Types of Models + +??? + +## Details + +--- + +## Types of Models + + + Classifiers + +??? + +## Details + + + Classifiers + + describes and distinguishes cases. Yelp may want to find a category for + a business based on the reviews and business description + +--- + +## Types of Models + + + Classifiers + + Regressions + +??? + +## Details + + + Regressions + + Predict a continuous value. e.g., predict a home's selling price given + square footage and # of bedrooms + +--- + +## Types of Models + + + Classifiers + + Regressions + + Clustering + +??? + +## Details + + + Clustering + + find "natural" groups of data *without labels* + +--- + +## Types of Models + + + Classifiers + + Regressions + + Clustering + + Outlier + +??? + +## Details + + + Outlier + + find anomalous transactions, e.g., finding fraud for credit cards + +--- + +## Process + +??? + +## Steps + + + to be able to classify data + +--- + +## Process + + + Training Set + +??? + +## Steps + + + Cleaned, preprocessed data that has labels. What are labels? + +--- + +## Process + + + Training Set + + Learning + +??? + +## Steps + + + Feed the training set into an algorithm. Algorithm associates some of the + features with the labels and generates a model. + +--- + +## Process + + + Training Set + + Learning + + Model / Classifier + +??? + +## Steps + + + Process or formula used to predict the label (class) given inputs (data + records) + +--- + +## Process + + + Training Set + + Learning + + Model / Classifier + + Testing Set + +??? + +## Steps + + + Data *not in training set*, with labels. Run through model to see how the + model compares with the real labels. + +--- + +## Process + + + Training Set + + Learning + + Model / Classifier + + Testing Set + + Verification / Accuracy + +??? + +## Steps + + + Given the matches / mismatches in the testing set, how can we measure how + well the model reflects reality? + +--- + +## Process + + + Training Set + + Learning + + Model / Classifier + + Testing Set + + Verification / Accuracy + + New Data + +??? + +## Steps + + + Unseen Data + + Finally, we're ready to start using our model / classifier to label new, + real, unknown data! So clean and pre-process it the same way. + +--- + +## Process + + + Training Set + + Learning + + Model / Classifier + + Testing Set + + Verification / Accuracy + + New Data + + Classification + +??? + +## Steps + + + Feed in the unknown data and get out results! + +--- + +## Learning + + + +??? + +## Example + + + We have training data. What are these column types? + + Feed it into a classification algorithm + + In the case it is generating Rules. + + Models can be as simple as this: just a set of rules to follow. We'll see + how we can extend this idea + + The learning step generates a model: these rules + +--- + +## Classification + + + +??? + +## Possibilities + + + Now that we have the model / classifier, we can do two things + + 1: Use testing data *different* from training data + + compare the classifier guesses with reality + + 2: Use the classifier on unknown data + + Why not just jump into classifying unknown data? Why have a test step? + +--- + +## Machine Learning + + + Supervised + + Given data with a label, predict data without a + label + + Unsupervised + + Given data without labels, group "similar" items + together + + Semi-supervised + + Mix of the above: eg. unsupervised to find groups, + supervised to label and distinguish borderline cases + + Active + + Starting with unlabeled data, select the most helpful cases for a + human to label + +??? + +## Which is this? + + + In the example above, what type of learning? + + Supervised: we have labels, we want to guess unlabeled data + +--- + +## Confusion Matrix + + + What are the ways that classification can be wrong? + +  + +| | Predict: Positive | Predict: Negative | +|------------------|-------------------|-------------------| +| Actual: Positive | True Positive | False Negative | +| Actual: Negative | False Positive | True Negative | + +??? + +## Basis for Evaluation + + + Most methods of evaluating results start with the confusion matrix + + Figuring out what different ways you were right or wrong + + Then using different formulas to emphasize the things you care about + +--- + +## Recall & Precision + + + Recall: ```TP / P``` + + Precision: ```TP / (TP + FP)``` + + Sometimes these are in tension; other measurements balance them + +??? + +## Trade-off + + + Classic trade-off in search + +--- + +## Example: Search + + + +??? + +## Searching Yelp + + + Searched yelp for a burrito in the Mission + + How good are these search results? + + Let's say we knew this first result was great, and *only* returned it + + What would our precision be? + + What would the recall be? + + How could we improve recall? + + How can we guarantee 100% recall? + + What will that do to the precision? + + Understand ways of combining these measurements in the book + +--- + +## Decision Trees + + + Rules formulated as a tree of decisions + + Choose Your Own Adventure for machine learning + + So how do we build the trees? + +??? + +## Rules expressed as trees + + + At each node in the tree, pose a question + + Take a branch depending on your answer + + Leaf nodes are labels + +--- + +## Build a Tree + + + +??? + +## Directions + + + First node question: is rank=professor? + + If True, what's the label? + + If False, we go to another node + + Second node question: is years > 6? + + If True what's the label? + + If False, what's the label? + +--- + +## Build a Tree + + + +??? + +## Next challenge + + + How to go from a data set like this + +--- + +## Build a Tree + + + +??? + +## Result + + + To a tree like this? + +--- + +## Decision Tree Induction + + + Start with all the data + + Choose the "best" way to divide it up based on one attribute + + Make a node that asks a question to split the data + + Choose new "best" way to divide based on remaining attributes + + Stop: no attributes left, all records are the same class + +??? + +## Recursive + + + Look at all the attributes. What's the best way to split up the data? + + We'll look at way to mathematically evaluate splits + + Now recursively do the same + + If you've split on all the attributes, but still have a mix, use a majority + rule + + If all the records are the same class, you don't have to keep spitting: + your answer is right there! + + For continuous data, must bucket it so you can have a discrete number of + answers + +--- + +## Information Gain + + + Comparison of how mixed results are before and after splitting + + Entropy measurement of "mixed" + + Two pure data sets have less entropy on average than one mixed + +??? + +## Information + + + Book will go into detail about how to think about entropy + + General idea: how difficult would it be to memorize the data sets? + + Easy if pure: all class A + + Still fairly easy if 2 pure sets: 1 is class A, other is class B + + Now more difficult if they are mixed: first 2 records are A, then one B, + then another A + +--- + +## Gini Index + +```python + Gini(D) = 1 - sum(frac**2 for frac in classes) +``` + Sum of the squares of the fraction of items in each class + +??? + + + D is a given partitioning of the data + +--- + +## Splitting + + + Discrete values can split per value + + Or discrete values binary split into subsets + + Continuous values can split on range (usually 2) + +??? + +## Different + + + If you'd like a binary tree (useful for some algorithms), can split on + subsets + + Can't split 400 different ways on continuous values... what about values + that haven't been seen before? + +--- + +## Continuous Splitting + + + Test every split point to see which is best + + Possible split points: midpoint between every adjacent value pair + + Sort attribute, score midpoints + +1 +2 +2 +2 +26 +36 +36 +74 +323 +345 +2234 + +??? + +## Review + + + What does best mean? + + Can calculate with one pass through the data since you are just moving a + few cases from one class to another + +--- + +## Decision Tree Advantages + + + Models easy to understand and visualize + + Can be faster to construct + + Can encode tree in declarative languages (SQL) + + Robust: outliers generally fit in with normal data + +??? + +## Trees + + + Its a tree! Easy to draw + + Greedy algorithm means you're only go over the data so many times + + Models can translate into database statements + + Outliers don't have a numeric pull on the data (similar to difference + between median and mean) + +--- + +## *Break* diff --git a/slides/2014-02-20-Gini.html b/slides/2014-02-20-Gini.html new file mode 100644 index 0000000..fae8551 --- /dev/null +++ b/slides/2014-02-20-Gini.html @@ -0,0 +1,238 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-20-Gini.markdown b/slides/2014-02-20-Gini.markdown new file mode 100644 index 0000000..3032876 --- /dev/null +++ b/slides/2014-02-20-Gini.markdown @@ -0,0 +1,69 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## HW: Gini + + + Calculate Gini Index + +--- + +## Gini Index + +```python + Gini(D) = 1 - sum(frac**2 for frac in classes) +``` + Sum of the squares of the fraction of items in each class + +--- + +## Data: Campaign Contributions + + + Federal [campaign finance data](ftp://ftp.fec.gov/FEC/2012/pas212.zip) (same as before) + + Calculating the Gini Index for the Candidate Names for the entire data set + + Partition by zip code, calculate the weighted average Gini Index score + over all partitions + + Partitions are weighted by the number of records they contain divided by + the total number of records in the data set + +--- + +## Extra Credit + + + Find a best split of a continuous field + +--- + +## Python Tips + + + [collections](http://docs.python.org/2/library/collections.html) module + + ```defaultdict``` auto creates keys + + ```Counter``` counts hashable objects + +```python +>>> x = defaultdict(int) +>>> x['a'] += 3 +>>> x['a'] +3 +>>> x['b'] +0 +>>> x +defaultdict(, {'a': 3, 'b': 0}) + +>>> y = Counter() +>>> y.update(['a', 'b', 'a', 'c']) +>>> y +Counter({'a': 2, 'c': 1, 'b': 1}) +``` + +--- + +## Git Tips + +```bash +$ git checkout master +$ git pull jimmy master +$ git checkout -b hw-gini +``` diff --git a/slides/2014-02-27-Lab-NN.html b/slides/2014-02-27-Lab-NN.html new file mode 100644 index 0000000..d461423 --- /dev/null +++ b/slides/2014-02-27-Lab-NN.html @@ -0,0 +1,228 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-27-Lab-NN.markdown b/slides/2014-02-27-Lab-NN.markdown new file mode 100644 index 0000000..5677d09 --- /dev/null +++ b/slides/2014-02-27-Lab-NN.markdown @@ -0,0 +1,59 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Continue Back Propagation from Slides + +.white-background[ + +] + +??? + +## Expected + + + Labeling: Top to bottom, left to right: + + 1 is upper left, 2 is lower left, 3 is top hidden layer, 6 is output layer + +--- + +## Submit + + ```nn-train.txt``` +```text +err_6 = -0.11346127339699999 +err_5 = -0.0011326458827956695 +err_4 = +err_3 = + +w_56 = 0.37298917134759924 +w_46 = +w_36 = + +err_2 = +err_1 = +w_25 = +w_24 = +w_23 = +w_15 = +w_14 = +w_13 = +``` + +??? + +## Fill In + + + Fill in values + + You may use a calculator or Python + + If you want feedback or partial credit, include any code in another file + +--- + +## Project + + + Find a partner (if applicable) + + Find a [data set](http://blog.bigml.com/2013/02/28/data-data-data-thousands-of-public-data-sources/) + + Start brainstorming diff --git a/slides/2014-02-27-Neural-Network.html b/slides/2014-02-27-Neural-Network.html new file mode 100644 index 0000000..eadf36f --- /dev/null +++ b/slides/2014-02-27-Neural-Network.html @@ -0,0 +1,629 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-27-Neural-Network.markdown b/slides/2014-02-27-Neural-Network.markdown new file mode 100644 index 0000000..086c107 --- /dev/null +++ b/slides/2014-02-27-Neural-Network.markdown @@ -0,0 +1,460 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Bias vs. Variance + +--- + +## Trade-offs + + + Similar to precision, we make trade-offs when training models + + Bias: How far off are the model predictions on average? + + Variance: If we retrained with different data, how different would our + guesses be? + +??? + +## Details + + + Bias: difference in "Expected" value from models from the real value + + Variance: difference in "Expected" value from each other + + Variance: Another way to think about it: how specific is our model to our + data? If we were training a tree with k-fold validation, would we get + completely different rule sets for each set of data? + + "Expected": These are *model type* properties. Train the model multiple + times with different data, then evaluate all models performance + +--- + +## Regression + + + Can we do better than linear regression on some data sets? + + Polynomial regression + + How many polynomials? + + + +??? + +## Polynomial + + + Sure! Use a polynomial instead: x2, 2x - x2 + 4x3, ... + + If you're not sure what the underlying data model is, have to test + + img: http://cheshmi.tumblr.com/ + +--- + +## One + + + +??? + +## So-So + + + How is the bias? Not great, fair amount of error + + How is the variance? Pretty good, assuming random sample + +--- + +## Two + + + +??? + +## Better + + + Bias? Better, less error + + Variance? more risky depending on which samples you get, since model + diverges quickly + +--- + +## Three + + + +??? + +## Worrying + + + Now getting a little weird. We're not finding the general pattern, more + like exactly fitting a line over these points + + If we made model with different data, we're going to get a different line + +--- + +## Many + + + +??? + +## Now kind of ridiculous + + + Intuitively we know this is not a description of the data + + If a point was found near the border, completely dependant on the data the + model trained on + +--- + +## Over-fitting + + + Over-fitting + + reflecting the exact data given instead of the general pattern + + High variance is a sign of over-fitting + + model guesses vary with the exact data given + + Avoidance + + ensembles average out variance, regularization adds a cost to model complexity + +??? + +## Avoidance + + + Ensembles combine multiple models together. Those multiple models may have + a lot of variance, but as long as they have good Bias, we'll center in on + the correct result + + Remember our cost function? We wanted to minimize the error. If you add in + a way to measure model complexity, you can add that to the cost, so that + you are explicitly trading-off the complexity of your model with the + quality of the solution + + If we wanted to add a complexity cost to the previous model, what would the + cost be dependent on? + +--- + +## Neural Networks + + + +??? + + + img: http://adrianbowyer.blogspot.com/2010/12/hardwired.html + +--- + +## Brains + + + Neural networks try to model our brains + + Neurons/perceptrons sense input, transform it, send output + + Neurons/perceptrons are connected together + + Connections have different strengths + +--- + +## Training + + + Learn by adjusting the strengths of the connections + + Mathematically, strength is a weight multiplier of the output + + Training is complete when we've found good weights + +--- + +## Nomenclature + +.left-column[ + +Input layer + + + neurons whose input is determined by features + +Hidden layer + + + neurons that calculate a combination of features + +Output layer + + + neurons that express the classification + +Weights + + + numeric parameter to adjust input/output + +] + +.right-column[ +.white-background[ + +] +] + +--- + +## Handwriting + + + Recognize handwritten digits + +.white-background[ + +] + +??? + +## Inputs => Outputs + + + Break up drawing cell into pixels + + Input takes pixel=on|off + + Output is highest valued output node, 1 for each digit + + img: http://vv.carleton.ca/~neil/neural/neuron-d.html + +--- + +## Forward Propagation + + 1. Sum of inputs * weights + 1. Apply sigmoid + 1. Send output to next layer + 1. Repeat + +--- + +## Repeat + + + Multiple hidden layers used to model complex feature interaction + + + +--- + +## Sigmoid + + + Normalize input to [0,1] + + Makes weak input weaker, strong input stronger + + ```1 / (1 + e^-input)``` + +.white-background[ + +] + +--- + +## Example + +.white-background[ + +] + +??? + +## Simple + + + Simple NN with just one output + + Output can model true/false + + Inputs are numerical + +--- + +## Weights + +.white-background[ + +] + +??? + +## Later + + + We'll discuss how weights are determined later + + Fill in the Hidden layer with sum of inputs * weights + +--- + +## Sigmoid + +.white-background[ + +] + +??? + +## Apply + + + Apply the sigmoid to the incoming signals + +--- + +## Sigmoid + +.white-background[ + +] + +??? + +## Apply + + + Apply the sigmoid to the incoming signals + +--- + +## Sigmoid + +.white-background[ + +] + +??? + +## Apply + + + Apply the sigmoid to the incoming signals + +--- + +## Sigmoid + +.white-background[ + +] + +??? + +## Apply + + + Apply the sigmoid to the incoming signals + +--- + +## Weights + +.white-background[ + +] + +??? + +## Repeat + + + Take the outputs, apply weights, sum + +--- + +## Sigmoid + +.white-background[ + +] + +??? + +## Apply + + + Apply the sigmoid to the incoming signals + + Our result is greater than 0.5, so we can assume true + + If we had multiple outputs, we could choose the highest one + +--- + +## Forward Propagation + + 1. Sum of inputs * weights + 1. Apply sigmoid + 1. Send output to next layer + 1. Repeat + +??? + +## Get an answer + + + Now we have *an* output, but how do we train to get the *right* output? + +--- + +## Fitness Function + + + Create a fitness function that measures the error + + Take the derivative and a step in the right direction + + Try again + +??? + +## Neural Network + + + NN training is conceptually similar to gradient descent + + We want to get closer to the answer, so we adjust our weights based on the + amount of incorrectness in the system + + Adjust weights, try again + +--- + +## Back Propagation + + + Run forward + + Oj is output of node j + + Calculate error of output layer + + Errj = Oj(1 - Oj)(Tj-Oj) + + Caclulate error of hidden layer + + Errj = Oj(1 - Oj) sum(Errk wjk) + + Find new weights + + wij = wij + l Errj Oi + + Repeat + + To move closer to correct weights + +??? + +## Derivative + + + Derivative of the sigmoid is Oj(1 - Oj), so we're taking the gradient + + ```l``` is the learning rate, similar to ```a``` step size in gradient descent + +--- + +### Example + +.left-column[ + + + Expected Output is 0 + + t6 = 0 + + Actual Output + + o6 = 0.8387 + + Output Error: + + err6 = + + o6\*(1-o6)\*(t6-o6) = + + -0.11346127339699999 + + Setup hidden node 5 + + o5 = 0.9933 + + w56 = 1.5 +] + +.right-column[ + + + Error for node 5 + + err5 = + + o5\*(1-o5)\*(err6*w56) + + -0.0011326458827956695 + + Adjust weight w56 + + l = 10 # learn rate + + w56 = + + w56 + l\*err6\*o5 = + + 0.37298917134759924 + +.white-background[ + +] +] + +--- + +## Terminate Learning + + + Changes in weights too small + + Accuracy in training models is high + + Maximum number iterations + + Maximum time for learning + +??? + +## Forward and Back + + + Guess, correct, guess, correct + + Stop when you've got a good model + + or you model is not improving + + or when you're out of time + +--- + +## *Break* diff --git a/slides/2014-02-27-SVM.html b/slides/2014-02-27-SVM.html new file mode 100644 index 0000000..9ee92b6 --- /dev/null +++ b/slides/2014-02-27-SVM.html @@ -0,0 +1,653 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-02-27-SVM.markdown b/slides/2014-02-27-SVM.markdown new file mode 100644 index 0000000..d0ca132 --- /dev/null +++ b/slides/2014-02-27-SVM.markdown @@ -0,0 +1,484 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Linear Regression + +--- + +## Types of Models + + + Classifiers + + *Regressions* + + Clustering + + Outlier + +??? + +## Details + + + Classifiers + + describes and distinguishes cases. Yelp may want to find a category for a + business based on the reviews and business description + + Regressions + + *Predict a continuous value. e.g., predict a home's selling price given + square footage and # of bedrooms* + + Clustering + + find "natural" groups of data *without labels* + + Outlier + + find anomalous transactions, e.g., finding fraud for credit cards + +--- + +## Case Study + + + Housing prices: square footage + + + +??? + +## Problem + + + We'd like to know how to price a house based on the square footage + + Let's pretend this is the data we have + + How would we guess that value for 2500 sq ft? + + img: http://realestatemetrics.blogspot.com/2013/04/starting-out-square-footage-vs-price.html + +--- + +## Solution? + +??? + +## Prompts + + + In English, how would you solve this? + +--- + +## Solution? + + + Find a line that represents the data + +??? + +## Prompts + + + How to mathematically represent the line? + +--- + +## Solution? + + + Find a line that represents the data + + ```y = m*x + b``` + +??? + +## Prompts + + + What is a good line? + +--- + +## Solution? + + + Find a line that represents the data + + ```y = m*x + b``` + + A line that is not very far from the points + +--- + +## Similarity + + + Main challenges in data mining: defining a specific metric for an intuition + + Define distance for an individual point + + Define how to aggregate distances together + +??? + +## Challenge + + + This is big problem for engineering and math (stats) in general + + We'll cover some concepts, but if you're ever stuck, try looking in related + fields + + What are some of the ways we can measure distance between points? + Euclidean, Manhattan, Euclidean == L2 norm + + What is a way to aggregate numbers? sum, sum of squares, sum of logs + + Differences between the last two? + +--- + +## Log & Square + + + Log: Useful for deemphasizing large raw differences + + Square: Useful for taking the approximate absolute value + + + +--- + +## Point Distance + + + ```y``` distance from line + + Intuitively: error in estimate + + ```h(x) = m*x + b``` + + ```err = h(x) - y``` + + + +??? + +## Error + + + We want the difference from what we estimate to be the value to what the + value actually is + +--- + +## Aggregate + + + ```sum``` + + What about negative error? + + Sum of squares + +.tight-code[ +``` +err = sum((h(x) - y)**2 for x,y in dataset) / len(dataset) +``` +] + +??? + +## Questions + + + Now we have info about all the errors from points, how to summarize? + + Some points have negative error, some positive? Do they cancel each other + out? + + Imagine data set of two points: one solutions covers lines, other divides + them. Which is better? + + Use our squaring trick to make sure we don't have any negative values + + Normalize by the number of points + +--- + +## Fitness Function + + + Measures the quality or cost of the solution + + *Key* ingredient for data mining algorithms + + If you can't measure it, you can't find the best solution + +??? + +## Fitness + + + Function spits out a metric. Metric can be thought of as *fitness* or + *cost* + + Find the maximum or minimum of that metric + + Depending on your fitness function, this can be easy or difficult + +--- + +## Understanding Error + +Several possible solutions + + + +??? + +## Error + + + What happens to the error as we move line around? + + Decreases until best fit, then increases + + What happens if we plot this error? Say, slope (x) against error (y)? + +--- + +## Solution as Minimization + + + Error is a parabola + + Several methods for finding the minimum + + Two categories: analytical, approximations + + + +--- + +## Solution Approximation + + + Some fitness functions can be difficult to solve analytically + + Alternative: iteratively get closer to the solution + + Stop when answer is close enough + +??? + +## Analytical + + + How to find the minimum of functions in general? + + Take derivative, find 0 + + Taking derivative can be complex or impossible (discontinuities) for some + functions, or solving for 0 is difficult + + Instead, well keep getting closer to the minimum using the function we + already have + +--- + +## Gradient Descent + + 1. Estimate current gradient (derivative) + 1. Take a step (```a * deriv```) in the direction of the gradient + 1. Step size is small, stop. Else repeat. + + + +??? + +## Steps + + + Take gradient by looking at the local derivative, or perturbing x + + Choose ```a``` as step size weight: big ```a``` is large step size + + If ```deriv``` is large, will also make your step size large. + + If ```deriv``` is large, probably means you are far away from minimum + + Keep repeating + + What happens if ```a``` is too small? + + What happens if ```a``` is too big? + +--- + +## General Case + + + Formulate a fitness function for your problem + + Use analytics or approximations to find min/max + + Approximations: Newton's Method, Gradient Descent + + + +??? + +## Approximate visualization + + + Graph of error vs. gradient descent iteration number + + Maybe some local problems, as step size is too big, but slowly move down to + a small amount of error + +--- + +# Support Vector Machines + +--- + +## Decision Trees + + + Great for separable attributes + + Rules operate on independent attributes + + Classes separable along an axis/attribute + + + +--- + +## Linearly Separable + + + How to handle case where separator line is not along an axis? + + + +??? + +## Details + + + Could say if ```x>2``` and ```y>2```, but not a great intuitive fit + + Draw a line that takes both into account + + ```y = m*x + b``` + + img: http://www.eric-kim.net/eric-kim-net/posts/1/kernel_trick.html + +--- + +## Possibilities + + + Many lines *could* separate these classes + + + +??? + +## Best? + + + Which is the best? + + Why? + +--- + +## Best Separator + + + Best line gives the most distance between the two classes + + Measure distance between closest points + + Closest points == support vectors + + + +??? + +## Points, Vectors + + + Points can be represented as vectors + + Vector math can be easier to express succinctly + + img: http://www.sciencedirect.com/science/article/pii/S1072751511001918 + +--- + +# Dimensions + + + When separating two dimensions, we need a line + + When separating 3 dimensions? + + 4 dimensions? + +??? + +## Vocabulary + + + Plane + + Hyperplane + +--- + +## Expressing the Hyperplane + +??? + +## Questions + + + How do you mathematically represent a line? + +--- + +## Expressing the Hyperplane + + + y = m*x + b + +??? + +## Questions + + + Now, we're not going to think of a new letter for every dimension, we're + just going to say x1, x2, x3 ... + +--- + +## Expressing the Hyperplane + + + y = m*x + b + + x2 = m*x1 + b + +??? + +## Questions + + + Rewrite mathematically + +--- + +## Expressing the Hyperplane + + + y = m*x + b + + x2 = m*x1 + b + + 0 = m*x1 - x2 + b +??? + +## Questions + + + How to add more dimensions? x22? Express x as a vector of all attributes + +--- + +## Expressing the Hyperplane + + + y = m*x + b + + x2 = m*x1 + b + + 0 = m*x1 - x2 + b + + 0 = [m -1] * [x1, x2] + b + +??? + +## Questions + + + Again, don't want to come up with a bunch more letters after ```m```, so use + ```w``` as the matrix representing all the ```m``` slopes + +--- + +## Expressing the Hyperplane + + + y = m*x + b + + x2 = m*x1 + b + + 0 = m*x1 - x2 + b + + 0 = [m -1] * [x1, x2] + b + + 0 = w * x + b + +--- + +## Challenge + + + Find ```w```, ```b``` such that ```w * x + b``` maximizes the distance between the + support vectors + +.white-background[ + +] + +--- + +## Maximizing Fitness Function + + + Now we have a fitness function and parameters we're trying to optimize + + Sound familiar? + +.white-background[ + +] + +--- + +## Kernel Tricks + + + SVMs are good for linearly separable data + + How to handle other data? + + + +--- + +## Polynomial Kernel + + + Transform it into linearly separable + + What function can we apply to these data points to make them separable? + + + +??? + +## Square + + + Square all of them + +--- + +## Polynomial Kernel + + + Now apply SVM + + + +??? + +## Details + + + img: http://www.sciencedirect.com/science/article/pii/S1072751511001918 + +--- + +## *Break* diff --git a/slides/2014-03-06-Clustering.html b/slides/2014-03-06-Clustering.html new file mode 100644 index 0000000..5a4190f --- /dev/null +++ b/slides/2014-03-06-Clustering.html @@ -0,0 +1,506 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-03-06-Clustering.markdown b/slides/2014-03-06-Clustering.markdown new file mode 100644 index 0000000..71b649d --- /dev/null +++ b/slides/2014-03-06-Clustering.markdown @@ -0,0 +1,337 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Clustering + +--- + +## Types of Models + + + Classifiers + + Regressions + + *Clustering* + + Outlier + +??? + +## Details + + + Classifiers + + describes and distinguishes cases. Yelp may want to find a + category for a business based on the reviews and business description + + Regressions + + Predict a continuous value. Eg. predict a home's selling + price given sq footage, # of bedrooms + + Clustering + + find "natural" groups of data *without labels* + + Outlier + + find anomalous transactions, eg. finding fraud for credit cards + +--- + +## Clustering + + + Group together similar items + + Separate dissimilar items + + Automatically discover groups without providing labels + +??? + +## Perspectives + + + Similar items: again, metrics of similarity critical in defining these + groups + + Marking boundaries between different classes + + Type of groups unknown before hand. Out of many attributes, what tend to be + shared? + +--- + +## Machine Learning + + + Supervised + + Unsupervised + + Semi-supervised + + Active + +??? + +## Definitions + + + Supervised + + Given data with a label, predict data without a + label + + Unsupervised + + Given data without labels, group "similar" items + together + + Semi-supervised + + Mix of the above: e.g., unsupervised to find groups, + supervised to label and distinguish borderline cases + + Active + + Starting with unlabeled data, select the most helpful cases for a + human to label + +--- + +## Clustering Applications + + + Gain insight into how data is distributed + + Discover outliers + + Preprocessing step to bootstrap labeling + +??? + +## Apps + + + Closest we have to "magic box": put structured data in, see what groups may + exist + + You want labeled data, but where to start? How many classes? What to name + them? + + Cluster data, investigate examples. + + Hand label exemplary cases + + Choose names that distinguish groups + + Run classifier on labeled data, compare with clustering, examine errors, + repeat + +--- + +## Yelp Examples + + + User groups based on usage, reviewing habits, feature adoption + + Businesses: when should a new category be created, what should it be called? + + Reviews: for a particular business, are there common themes. Show better + variety? + +??? + +## Examples + + + User groups may be trend spotters, "lurkers", travelers, early adopters + + Do we need a New American and American category? How similar are these + categories? + + Does a reviewer need to read 10 reviews about great food, so-so service? + Maybe providing different view points helps give a better picture + +--- + +## Intuition + + + Intuition => Mathematical Expression => Solution => Evaluation + + High intra-class similarity + + Low inter-class similarity + + Interpretable + +??? + +## Good Clusters + + + Just like all data mining, needs to be used to take action + + Can't take action if you don't understand the results + + Trade-offs: testing shows it works, but you don't understand it + +--- + +## Methods + + + Partitioning + + Construct ```k``` groups, evaluate fitness, improve groups + + Hierarchical + + Agglomerate items into groups, creating "bottom-up" clusters; or divide set into ever smaller groups, creating "top-down" clusters + + Density + + Find groups by examining continuous density within a potential + group + + Grid + + Chunk space into units, cluster units instead of individual records + +??? + +## Algorithms + + + Partitioning + + Method similar to gradient descent: find some grouping, + evaluate it, improve it somehow, repeat. k-means. + + Hierarchical + + Build groups 1 "join" at a time, examining distance between + two things that can be joined together, if close, combine groups. Reverse: + divisive. + + Density + + Many of the above methods just look for distance. This method + tries to find groups that might be strung out, but maintain a density. Think + about an asteroid belt. It is one group, but not clustered together in a way + you typically think. + + Grid + + Can speed up clustering and provide similar results + +--- + +## k-means + + + Start: Randomly pick ```k``` centers for clusters + + Repeat: + + Assign all other points to their closest cluster + + Recalculate the center of the cluster + +??? + +## Iterative + + + Start at a random point, find step in right direction, take step, + re-evaluate + +--- + +## Example + + + +??? + +## Process + + + We pick some nodes at random, mark with a cross + + Find other points that are closest to the crosses + + Find new *centroid* based on the average of all points + + Start again + + img: http://apandre.wordpress.com/visible-data/cluster-analysis/ + +--- + +## Distance + + + *Centroid* is the average of all points in a cluster; the center + + Different distance metrics for real numbers + + But how to find "average" of binary or nominal data? + +??? + +## You Can't + + + k-means is used for numerical data + +--- + +## Normalization + + + Cluster cities by average temperature and population attributes + + ``` = ``` + + Using Euclidean distance, which attribute will affect similarity more? + +??? + +## Un-normalized + + + Population: it is a much bigger number, will contribute much more to + distance + + Artificially inflating importance just because units are different + +--- + +## Normalization Techniques + + + Z-score + + ```(v - mean) / stddev``` + + Min-max + + ```(v - min) / (max - min)``` + + Decimal + + ```* 10^n``` or ```/ 10^n``` + + Square + + ```x**2``` + + Log + + ```log(x)``` + +??? + +## Useful for? + + + Z-score + + 1-pass normalization, retaining information about stdev + + Min-max + + keep within expected range, usually [0-1] + + Decimal + + easy to apply + + Square + + keep inputs positive + + Log + + de-emphasize differences between large numbers + +--- + +## Local Optima + + + +??? + +## No Guarantee + + + Since there are many possible stable centers, we may not end up at the best + one + + How can we improve our odds of finding a good separation? + + Why did we end up here? starting points + + Choose different starting points + + Compare results + + Other problems? Mouse + +--- + +## Uneven Groups + + + +??? + +## k-means + + + k-means is good for similarly sized groups, or at least groups that are + similar distance between other members + + Other problems that would pull the centroid away from the real groups? + + Outliers + + img: http://en.wikipedia.org/wiki/K-means_clustering + +--- + +## Medoids + + + Instead of finding a *centroid* find a *medoid* + + Medoid: actual data point that represents median of the cluster + + PAM: Partitioning Around Medoids + +??? + +## Trade-offs + + + PAM more expensive to evaluate + + Scales poorly, since we need to evaluate many more medoids with many more + points + +--- + +## Example + + + +??? + +## Stability + + + No stability between real clusters + + Outliers can't pull centroid far out of actual cluster + + img: http://en.wikipedia.org/wiki/K-medoids + +--- + +# *Break* + + + Do not confuse Medoid with Metroid + + + + +??? + +## Note + + + img: http://stealthboy.com/~msherman/metroid.html diff --git a/slides/2014-03-06-Hierarchical.html b/slides/2014-03-06-Hierarchical.html new file mode 100644 index 0000000..90374aa --- /dev/null +++ b/slides/2014-03-06-Hierarchical.html @@ -0,0 +1,445 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-03-06-Hierarchical.markdown b/slides/2014-03-06-Hierarchical.markdown new file mode 100644 index 0000000..71ea0ed --- /dev/null +++ b/slides/2014-03-06-Hierarchical.markdown @@ -0,0 +1,276 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Hierarchical & Density + +--- + +## Methods + + + Partitioning + + Construct ```k``` groups, evaluate fitness, improve groups + + Hierarchical + + Agglomerate items into groups, creating "bottom-up" + clusters; or divide set into ever smaller groups, creating "top-down" + clusters + + Density + + Find groups by examining continuous density within a potential + group + + Grid + + Chunk space into units, cluster units instead of individual records + +??? + +## Algorithms + + + Partitioning + + k-means, k-medoid + + Hierarchical + + Build groups 1 "join" at a time, examining distance between + two things that can be joined together, if close, combine groups. Reverse: + divisive. + + Density + + Many of the above methods just look for distance. This method + tries to find groups that might be strung out, but maintain a density. Think + about an asteroid belt. It is one group, but not clustered together in a way + you typically think. + + Grid + + Read the book + +--- + +## k-means Limitations + + + Must supply ```k```, the number of clusters + + Clusters must be disjoint* + +??? + +## Difference + + + *We'll learn about "fuzzy" clustering next time, where cluster membership + is a probability + +--- + +## k-means Limitations + + + Must supply ```k```, the number of clusters + + Clusters must be disjoint* + +## Another Approach + + + Hierarchical clustering builds up clusters incrementally + +??? + +## Difference + + + Hierarchical can find cluster of clusters + + Can illustrate clusters at many levels, let human interpret what makes + sense without guess-and-check + + Clusters are built 1 cluster at a time, starting with all points being + their own cluster + +--- + +## Agglomerative + + + All points are separate clusters + + Find closest clusters: Join them + + Repeat + + + +??? + +## Bottom-up + + + Any questions about this? + + What does "close" mean? + +--- + +## Cluster Distance + + + Minimum + + Use the two closest points + + Maximum + + Use the two farthest points + + Mean + + Use the mean of the two clusters + + Average + + Sum of the distances of all pairs, divided by number of pairs + +??? + +## Meta Distance + + + These are actually distance metrics for clusters that translate down to + distance metrics for points. + + Still need to decide distance measures for points: Euclidean, Manhattan, + etc. And that's just for numerical distance + + Choose based on expected cluster topology, cross validation testing using + human observers + +--- + +## Termination + + + Define have ```k``` clusters + + Distance between clusters exceeds threshold + + Fitness function for cluster + +??? + +## Details + + + If you wanted to look at all potential ```k```, set ```k = 1```, then look at sub + clusters + + Distance or fitness function (eg. density or minimum intra-cluster + similarity score) can help define ```k``` automatically + +--- + +## Dendrogram + + + Display of clustered groups + + Concise visualization: groups do not need to be identified or named + + Y axis can represent iteration + + + + +??? + +## Usefulness + + + Can move up and down clustering to make sense of individual clusters + +--- + +## Chameleon + + + Discover large number of small clusters + + Group together small clusters + + Join clusters with a high interconnectedness relative to their existing + interconnectedness + + + +??? + +## Details + + + Mix of partition & agglomerative + + Partition by finding groups of k-nearest neighbors: A, B in the same group + if A is a k-nearest neighbor of B. + + Interconnectedness measured by aggregate proximity in the group, or using a + network model the book provides details on (10.3.4) + +--- + +## Results + + + +??? + +## Properties + + + Tends to "follow" clusters as long as interconnectedness stays high + +--- + +## Density: DBSCAN + + + Find "paths" of points that are in "dense" regions + + Paths: points within a distance ```e``` + + Density: surrounded by ```MinPts``` within region of radius ```e``` + + + +??? + +## Details + + + Can find non linear "paths" to follow as long as they stay dense + +--- + +## Density Trade-offs + +.left-column[ + + + Finds clusters of different sizes, shapes + + DBSCAN is sensitive to the parameters used. How big is ```e```? How many + points is "dense"? +] + +.right-column[ +.white-background[ + +] +] + +??? + +## Details + + + img: http://en.wikipedia.org/wiki/DBSCAN + +--- + +## Algorithm Choice + + + Simple techniques often work surprisingly well + + Choose other algorithms to tackle specific problems + + Evaluation metrics + +??? + +## Lessons + + + Just like Naive Bayes, we make assumptions about our data that turn out + to be right enough: clusters are uniformly sized, don't wander around our + dimensioned space + + Topic drift: tendency for a cluster to change its properties slowly over + time: e.g., articles on politics might use different words + + Performance: many of these algorithms are computationally expensive, hard to + distribute. Book goes into run times and where to make compromises on the + algorithm + + Figure out a fitness function for your metric. If you used these clusters + to take action, what would be the result? + +--- + +## Elbow Method + + + Calculate intra-cluster variance + + Compare to data set variance (F-test) + + Find point where marginal gain of explicative power decreases + + + +--- + +## Labels + + + Clustering is an example of unsupervised learning + + But after clustering, humans can label clusters, and their contents + + Now one can use homogeneity metrics to evaluate clusters + +??? + +## Homogeneity + + + Gini Index + + Entropy + + Precision / Recall + +--- + +# *Break* diff --git a/slides/2014-03-06-k-means.html b/slides/2014-03-06-k-means.html new file mode 100644 index 0000000..6a2d5ee --- /dev/null +++ b/slides/2014-03-06-k-means.html @@ -0,0 +1,193 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-03-06-k-means.markdown b/slides/2014-03-06-k-means.markdown new file mode 100644 index 0000000..298321e --- /dev/null +++ b/slides/2014-03-06-k-means.markdown @@ -0,0 +1,24 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Implement k-means Clustering + + + Very small data set + + Select starting centroids + + Find new centroids + + Stop after centroid moves less than ```err``` + +--- + +## Code + + + ```code/k_means.py``` + + ```zip``` combine lists by alternating members + +```python +>>> zip([1,2,3], ['one', 'two', 'three']) +[(1, 'one'), (2, 'two'), (3, 'three')] +``` diff --git a/slides/2014-03-13-Advanced-Cluster.html b/slides/2014-03-13-Advanced-Cluster.html new file mode 100644 index 0000000..411b1ed --- /dev/null +++ b/slides/2014-03-13-Advanced-Cluster.html @@ -0,0 +1,513 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-03-13-Advanced-Cluster.markdown b/slides/2014-03-13-Advanced-Cluster.markdown new file mode 100644 index 0000000..63ae726 --- /dev/null +++ b/slides/2014-03-13-Advanced-Cluster.markdown @@ -0,0 +1,344 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Advanced Clustering + +--- + +## Review + + + Clustering groups points by using similarity + + Build up, or break down groups + + Each point belongs to 1 cluster + +??? + +## Types + + + Agglomerative, Divisive + + Assign each point to 1 centroid (k-means) + + or group clusters together, starting with every point as a cluster + (hierarchical) + +--- + +## Topics? + +"This place is great. I've been here for meetings, to get work done, to hang out with friends, and on dates, and it's fit the bill every time. In the summer, there's a wonderful patio, and in the winter it's cozy and warm. All the food I've had is delicious -- especially the salad dressing!" + +??? + +## Model by topic + + + ambiance + + romantic + + good for work + + food quality + + *Many* + + So how do we model? + +--- + +## Fuzzy Clusters + + + Membership is a degree in [0-1] + + ```1 = sum(membership(v,c) for c in clusters)``` + + Every point belongs to at least 1 cluster + +??? + +## Restrictions + + + Every point still must be in a cluster + + Think of the degree as a probability + + Probabilities must add up to 100% (1) + +--- + +## Generative Model + +??? + +## Questions + + + What is a "generative model"? + +--- + +## Generative Model + + + "Real" model that produced original data points + +??? + +## Questions + + + What is data mining trying to discover? What is machine learning hoping to + reproduce? + +--- + +## Generative Model + + + "Real" model that produced original data points + + Our mission is to reproduce the original model + +??? + +## Questions + + + Why have different classifiers? Decision tree, Naive Bayes, etc? + +--- + +## Generative Model + + + "Real" model that produced original data points + + Our mission is to reproduce the original model + + Thus we have different techniques that can model different behavior + +--- + +## Homework: 1-D Clustering + +??? + +## Questions + + + Draw number line + +--- + +## Homework: 1-D Clustering + + + Imagine plotting the points on the number line + +??? + +## Questions + + + Where did these numbers come from? We wanted two clusters. + +--- + +## Homework: 1-D Clustering + + + Imagine plotting the points on the number line + + Points were generated with a process: + + Two Gaussian arrays, concatenated + +??? + +## Questions + + + What parameters were used in the code to generate this specific set of N + numbers? + +--- + +## Parameters + + + Median + + Standard Deviation + + How many points to generate from each + + + +??? + +## Translation + + + Two distributions (median, stddev) + + Picked one or the other with a certain probability + + Then generated a number from it + + In reality, just generated 10 from A, 10 from B, but you can imagine that + being 50% and 50% + +--- + +## Generative Model + +.white-background[ + +] + +??? + +## 3 Clusters + + + We have all three parameters: + + median + + stddev + + probability of choosing distribution (height) + +--- + +## Best Fit? + +.left-column[ +.white-background[ + +] +] + +.right-column[ +.white-background[ + +] +] + +??? + +## Choose + + + These letters are points on our number line + + Which is more likely to be generated by our real model? + + But we don't know the generative model, so how do we discover it? + +--- + +## Revisit k-means + +??? + +## Questions + + + What are the steps of k-means? + +--- + +## Revisit k-means + + + Each object assigned to closest cluster + +--- + +## Revisit k-means + + + Each object assigned to closest cluster + + Reset center of the cluster to average + +--- + +## Revisit k-means + + + Each object assigned to closest cluster + + Reset center of the cluster to average + + Repeat until steady + +--- + +## Expectation-Maximization + + + Expectation + + Given current state, create a solution that fits our expectations + + Maximization + + Adjust the state to maximize the likelihood of the solution + being true + + Terminate + + When adjustments do not change + +??? + +## k-means translation + + + Our expectation in k-means is that points belong to the cluster closest to + them + + Our state or parameters for our model are the locations of the centers of + those clusters + + The maximization step therefore moves the centers to maximize the + likelihood of their being the true center + +--- + +## Revisit k-means + + + Each object assigned to closest cluster + + Reset center of the cluster to average + + Repeat until steady + +--- + +## Fuzzy Clustering + + + ~~Each object assigned to closest cluster~~ + + Each object assigned *probability* of cluster + + ~~Reset center of the cluster to average~~ + + Reset center of the cluster to *weighted* average + + Repeat until steady + +??? + +## Change + + + Only difference here is that we're calculating the probability of a point + belonging to a cluster + + What should we base that probability off of? distance + + If point A has a high probability of belonging to cluster C, what can you + say about A and C? Close + +--- + +## Distance + +```dist(o,C) / sum(dist(o,c) for c in clusters))``` + +??? + +## Similarity + + + Our old friend distance + +--- + +## Distance² + +.tight-code[ +```dist(o,C)**2 / sum(dist(o,c)**2 for c in clusters))``` +] + + + Squared distance to primary cluster, divided by squared distance to all + clusters + +??? + +## Similarity + + + Our old friend distance + + And squared, to make sure we stay positive + +--- + +## Reset Center + +```sum(weight[c][p]**2 * p for p in points)``` + + + Squared weight for this point in this cluster, multiplied by point + coordinates + +??? + +## Weighting by distance + + + Distance of point affects how much a cluster center is pulled toward it + +--- + +## Stability + + + When our centroids stabilize, we can estimate the parameters of our + distributions + + Or use probabilities of points directly + +??? + +## Uses + + + Sometimes you may not need original parameters + +--- + +# *Break* diff --git a/slides/2014-03-13-Review.html b/slides/2014-03-13-Review.html new file mode 100644 index 0000000..72bcec1 --- /dev/null +++ b/slides/2014-03-13-Review.html @@ -0,0 +1,371 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-03-13-Review.markdown b/slides/2014-03-13-Review.markdown new file mode 100644 index 0000000..3da8cba --- /dev/null +++ b/slides/2014-03-13-Review.markdown @@ -0,0 +1,202 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Review + +--- + +## Review + + + Midterm target length: 1.5 hours + + Time limit: 3 hours + + 1 cheat sheet, 8.5 x 11 + + Calculators OK, not required + + Questions from slides have a higher probability of appearing + + Questions from the reading are fair game + +--- + +## Case Studies + + 1. Example of "transactional data" + 1. Example of non-transactional data + +??? + +1, 2 + +--- + +## Obtaining Data + + 1. Trade-offs of dataset vs API? + 1. Trade-offs of operational database vs data warehouse + 1. Unix commands to explore data? + +??? + +1, 2, 3 + +--- + +## Probability + + 1. Other names for a Feature + 1. Difference between Discrete and Continuous feature + 1. What type of feature is day of the week? + 1. Ways to measure central tendency? + 1. What is skew? + 1. When is asymmetric binary dissimilarity useful? + 1. How to calculate L2 norm of two points? + 1. What is cosine similarity? + +??? + +1, 2, 3, 4, 5, 7 + +--- + +## Preprocessing + + 1. When storing the same fact in different ways, what type of problem is + likely? + 1. Can data corruption happen with no mistakes and no bugs? + 1. What are some options to deal with missing values? + 1. What are some options to deal with outliers? + 1. What does correlation imply? + +??? + +2, 4, 5 + +--- + +## Data Warehouse + + 1. OLAP vs OLTP + 1. Examples of database metadata? + 1. What is a data multi-cube? + 1. What is at the center of a star schema? + 1. What is the tradeoff being made in dimension tables? + 1. Define: + 1. Rollup + 1. Drill-down + 1. Slice + 1. Dice + 1. Pivot + +??? + +1, 4, 6.1 + +--- + +## MapReduce + + 1. What tradeoff are we making with MapReduce? + 1. Why is log processing a typical use of MapReduce? + 1. What types of processing are not well suited? + 1. For a multi-step job, the output of a reducer is fed into what? + +??? + +1, 2, 3, 4 + +--- + +## Decision Tree + + 1. What table can we create from the verification results to understand + performance of our model? + 1. For supervised learning, what is required to train a model? + 1. What is a naive way to optimize precision? + 1. Recall? + 1. Assuming we use all attributes to classify, what is the height of + our tree? + 1. What are we optimizing for in the leaf nodes? + +??? + +2, 3, 4 + +--- + +## Naive Bayes + + 1. Where does the testing set come from? + 1. What is the ```k``` in k-fold cross-validation? + 1. Bayes theorem finds P(A|B). In email spam detection, what are A and B? + 1. What is the Naive assumption we make in Naive Bayes? + 1. Why can training many models be useful? + 1. What is bootstrap sampling? + 1. What is a random forest? + +??? + +1, 2, 3, 4, 7 + +--- + +## SVM + + 1. When finding a linear fit for home prices, what is our fitness function? + 1. What is the gradient in gradient descent? + 1. In the general case, are you guaranteed to find the globally optimal + solution when using gradient descent? + 1. Why does SVM work so well in practice, even though it requires linear + separability? + 1. If your data is not linearly separable, can you use SVM? + +??? + +2, 3 + +--- + +## Neural Networks + + 1. What is model variance? + 1. What problem does high model variance indicate? + 1. What is an activation function? + 1. What types of problems are neural networks especially suited for? + 1. What are we improving during backward propagation? + +??? + +1, 2, 4 + +--- + +## Partitioning Clusters + + 1. What is the difference between k-means and k-medoids? + 1. What are some of the problems with k-means? + 1. Why is normalization especially useful in clustering? + 1. What are the tradeoffs for using k-medoid clustering? + +??? + +1, 2 + +--- + +## Hierarchical Clustering + + 1. What are the options to calculate cluster distance? + 1. Describe how to draw a dendrogram + 1. What are the drawbacks to density clustering with DBSCAN? + 1. If you had movie description data, but no genres, would you use Fuzzy + Clustering or Partitioned Clustering? + 1. How can we evaluate a clustering algorithm if our data is already labeled + with clusters? + +??? + +1, 2 + +--- + +# *Good Luck!* diff --git a/slides/2014-04-03-AWS.html b/slides/2014-04-03-AWS.html new file mode 100644 index 0000000..1cee72a --- /dev/null +++ b/slides/2014-04-03-AWS.html @@ -0,0 +1,286 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-03-AWS.markdown b/slides/2014-04-03-AWS.markdown new file mode 100644 index 0000000..c918b00 --- /dev/null +++ b/slides/2014-04-03-AWS.markdown @@ -0,0 +1,117 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Amazon Web Services + +--- + +## Grant + + + $5,600 for this class + + Store data in S3 + + Use mrjob on EMR + +??? + +## No crazy + + + Overcharges go on my credit card! + + Ask me before doing anything not discussed here + +--- + +## S3 + + + [S3](http://aws.amazon.com/s3/): Simple Storage Service + + Can store and share large data files + + Free bandwidth when processing with EMR + +--- + +## ```s3cmd``` + + + Upload / Download files + + ```s3cmd --configure``` + + ```cat ~jretz/mrjob.conf``` to get answers to first two questions, defaults should work for the rest + + ```s3cmd mb s3://i290-name``` ← your name or team name here + + ```s3cmd put localfile s3://i290-name/data/``` + +??? + +## Configuration + + + check ```~jretz/mrjob.conf``` + +--- + +## Elastic MapReduce + + + ```python job.py -r emr -c ~jretz/mrjob.conf s3://i290-name/data/file``` + + ```-r emr``` : run on EMR instead of localhost + + ```-c ~jretz/mrjob.conf``` : use a configuration file + + 5 machines, can change with command line options + +--- + +## Copying Keys + + + SSH keys are used to connect to server to check status + + Copy my keys, set correct permissions +```bash +$ cp ~jretz/i290T-03.pem.shared ~/i290T-03.pem +$ chmod 0600 ~/i290T-03.pem +``` + +--- + +.tight-code[ +```bash +$ s3cmd put yelp_academic_dataset.json.gz s3://i290-jretz/data/ +# yelp_academic_dataset.json.gz -> s3://i290-jretz/data/yelp_academic_dataset.json.gz [1 of 1] +# 127506871 of 127506871 100% in 8s 13.60 MB/s done + +$ cd datamining290/code/ +~/datamining290/code$ python unique_review.py -v -r emr \ + -c ~jretz/mrjob.conf \ + --output-dir s3://i290-jretz/output/unique_review/ \ + --no-output \ + s3://i290-jretz/data/yelp_academic_dataset.json.gz +# ... +# Creating Elastic MapReduce job flow +# ... +# Job flow created with ID: j-EFG48CIR1APW +# ... +# Job launched 60.8s ago, status STARTING: Provisioning Amazon EC2 capacity +# ... +# Job launched 334.4s ago, status RUNNING: Running step (unique_review.jretz.20130406.171258.267523: Step 1 of 3) +# ... +# map 73% reduce 42% +# ... +# Counters from step 1: +# ... + +~/datamining290/code$ s3cmd ls -r s3://i290-jretz/output/ +# ... +# 2013-04-06 18:05 29 s3://i290-jretz/output/unique_review/part-00005 +# ... +``` +] + +??? + +## Trade-offs + + + Jobs may take 5 minutes to spin up + + Errors are harder to debug because they are mixed in with Hadoop and EMR + errors + +--- + +## Extra Notes + + + Cannot overwrite output directory: choose a new one for each run + + Errors may be hard to debug. Run locally with a sample of your data + + Output from job will be split into files, recall the Hadoop video lecture diff --git a/slides/2014-04-03-Frequent-Pattern.html b/slides/2014-04-03-Frequent-Pattern.html new file mode 100644 index 0000000..bea2fa8 --- /dev/null +++ b/slides/2014-04-03-Frequent-Pattern.html @@ -0,0 +1,364 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-03-Frequent-Pattern.markdown b/slides/2014-04-03-Frequent-Pattern.markdown new file mode 100644 index 0000000..7209f7e --- /dev/null +++ b/slides/2014-04-03-Frequent-Pattern.markdown @@ -0,0 +1,195 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Frequent Patterns + +--- + +## Finding Patterns + + + Cookies frequently purchased with milk + + Website sign-ups frequently occurring after reading FAQ + + DNA sections frequently seen with a drug reaction + +??? + +## Patterns + + + set of items + + subsequences of actions + + substructures + + Generalized to any kind of pattern that occurs "frequently" in the dataset + +--- + +## Market Basket + + + What things are frequently purchased together? + + Apocryphal example: beer and diapers + + Can be used for any natural grouping + +??? + +## Details + + + Example of how patterns are discovered is to look at groups of actions + + One natural group is the shopping basket: what items are in it? + + But can also be applied to anytime there is a natural grouping + + Eg. web session logs group naturally around a person and time window + +--- + +## Define "Frequently" + + + Action + + ```A``` and ```B``` + + Support + + probability that a transaction contains ```A ∪ B``` + + Confidence + + conditional probability that a transaction having ```A``` also + contains ```B``` + +??? + +## Probabilities + + + We have two actions ```A``` and ```B``` + + Out of all the groupings, how many had both items? + + Out of all the groupings with ```A```, how many had ```B```? + +--- + +## Minimums + + + Min Support + + lower bound on support probability + + Min Confidence + + lower bound on confidence probability + + Strong + + Rule that satisfies both minimums + + + +??? + +## "Frequently" + + + Now we can talk about what frequently means + + It doesn't matter if two very unpopular items were purchased together: car + battery and smoke detector + + Also don't care if ```A``` happens a lot: everybody buys milk, so not a big + deal if some bought milk and strawberries + + Also important to note confidence is not symmetric: buying strawberries may be + frequent with buying milk, but not vice versa + +--- + +## Too Many Rules + + + Patterns not limited to 2 events + + But looking for all patterns leads to combinatorial number of options + +a,b,c,d,e
+
+a,b
+a,c
+...
+
+a,b,c
+a,b,d
+...
+ +--- + +## Subset Patterns + + + Max-Pattern + + ```X``` rule is frequent and there exists no frequent + super-pattern ```Y``` + + Closed + + ```X``` rule is frequent and there exists no super-pattern ```Y``` *with the same support* + + Shortcut + + Find only max-pattern or closed patterns, let other patterns be + subsets + +??? + +## Shortcut + + + So how can we calculate all the potentially frequently occurring patterns? + + We can find either the max or closed pattern that encompasses all of the + patterns we're looking for + + These are more easily tracked, and we can still derive all of the + frequently occurring sub-patterns + + We can use the reverse: if a rule or item is not frequent enough alone, its + super-set will not be frequent enough: + + If ```A``` is does not meet min support, there's no way for ```A,B``` to make + support + +--- + +## Apriori + + 1. Find supported single event rules + 1. Combine to make 2-event rules, check DB for support + 1. Combine to make 3-event rules, check DB... + 1. Stop when no N-event rules + +--- + + + +??? + +## Speed + + + Isn't that slow? Yes! + + Book has some techniques to speed it up, mostly around grouping + + Can group together sets and if the group does not meet the support + threshold, then none of the members do + +--- + +## Interesting Patterns + + + Strong rules may not always be interesting rules + + Basketball → eat cereal [40%, 66.7%] is strong + + But "not cereal" has a bigger effect on if you play basketball + +| | Basketball | Not basketball | Sum | +|------------|------------|----------------|------| +| Cereal | 2000 | 1750 | 3750 | +| Not cereal | 1000 | 250 | 1250 | +| Sum | 3000 | 2000 | 5000 | + +??? + +## Details + + + Not cereal row: has a huge effect on if someone plays basketball + + cereal + basketball... sure it happens frequently, but you'd actually + expect to see a bigger effect + +--- + +## Lift + + + ```P(A ∪ B) / P(A)*P(B)``` + + If ```A``` and ```B``` independent, what is likelihood of ```A``` and ```B```? + +??? + +## Correlation + + + 1 + + so if lift > 1, you're seeing something that is happening more often than + random + + < 1 means they negatively correlated + + χ², cosine, others in book + +--- + +# *Break* diff --git a/slides/2014-04-10-AdjacencyRepresentations.html b/slides/2014-04-10-AdjacencyRepresentations.html new file mode 100644 index 0000000..3d53820 --- /dev/null +++ b/slides/2014-04-10-AdjacencyRepresentations.html @@ -0,0 +1,235 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-10-AdjacencyRepresentations.markdown b/slides/2014-04-10-AdjacencyRepresentations.markdown new file mode 100644 index 0000000..4d62b73 --- /dev/null +++ b/slides/2014-04-10-AdjacencyRepresentations.markdown @@ -0,0 +1,66 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Homework + + + Represent each of 2 graphs as both an adjacency matrix and an adjacency list + + Work on your projects + +--- + +## Graphs + +.left-column[ +1. +] +.right-column[ +2. .white-background[] +] + +--- + +## Output + + + File in Github pull request + + Represent Matrix and list in some sort of organized way + +```python +[[0 1 1 1] +[1 0 0 1] +...] + +{1: [2,3,4], +2: [1, 4]} +``` + +```csv +0,1,1,1 +1,0,0,1 +... + +1,2,3,4 +2,1,4 +``` + +--- + +## [NetworkX](http://networkx.github.io/) + + + Python library for manipulating graphs + + Potentially useful for your projects + + Not homework + +--- + +## [VirtualEnv](https://pypi.python.org/pypi/virtualenv) + + + Install and manage libraries + + ```activate``` each time you start a new session +```bash +$ virtualenv venv +$ source venv/bin/activate +$ pip install +``` diff --git a/slides/2014-04-10-Graphs.html b/slides/2014-04-10-Graphs.html new file mode 100644 index 0000000..f520495 --- /dev/null +++ b/slides/2014-04-10-Graphs.html @@ -0,0 +1,445 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-10-Graphs.markdown b/slides/2014-04-10-Graphs.markdown new file mode 100644 index 0000000..eee3a1c --- /dev/null +++ b/slides/2014-04-10-Graphs.markdown @@ -0,0 +1,276 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Graphs & Networks + +--- + +## Graphs + + + Graphs (or networks) can model a surprising number of domains + + Modeling with graphs opens up a large number of algorithms + + Linear Algebra has many connections to graphs + +??? + +## Math + + + Data mining theme: get your problem stated as a math problem, whole slew of + solutions present themselves + + Linear Algebra is really useful for running equations of all nodes, or + simulate moving across network + +--- + +## Vertices & Edges + + + Vertex + + the interconnected objects, or nodes + + Edge + + the lines or curves that connect vertices + + Graph + + Collection of vertices and edges ```G = (V,E)``` + +.white-background[ + +] + +??? + +## Definitions + + + These are the abstract terms, how do they relate to the real world? + +--- + +## Examples + + + Vertex + + User, building, router, product + + Edge + + Relationship, road, network cable, purchased + + Graph + + Social Network, physical infrastructure, Internet, purchasing + history + +??? + +## Examples + + + Many graphs have assumed edge labels + + the edges represent something + consistent + + Some graphs have multiple types of edges + + relationship is one of family, + friend, co-worker, etc. + + Edge can be anything that ties two things together + + purchase history, e.g., + is not a physical thing connecting, but an idea + +--- + +## Social Networks + + + An edge connects two people + + If this is just a line, what information are we missing about how the link + was formed? + + + +??? + +## Symmetric vs. Asymmetric Edges + + + "Just a line" is called *undirected* + + We're missing information about who invited whom. That would be expressed + as an arrow and is called *directed*. + +--- + +## Definitions + + + Directed: Connections have a direction. Invitations, water pipes, email. + + Undirected: Connections have no direction. Friends, walkways on campus, + physical wires. + + Cycle: Set of nodes and edges in which you can travel back to a vertex + + Acyclic: A graph without any cycles + + + +??? + +## Modeling + + + Can always model an undirected graph as a directed graph by having two + directed edges going in opposite directions in place of each undirected + edge. + +--- + +## Acyclic? + + + Social network (undirected) + + Product purchases (directed) + + Internet links (directed) + + Class prerequisites (directed) + +??? + +## Answers + + + Social network: cyclic + + Product purchases: acyclic + + Internet links: cyclic + + Class prerequisites: acyclic + +--- + +## Bipartite + +.left-column[ + + + Graph whose vertices can be divided into two distinct sets + + Vertices in ```U``` are only connected to those in ```V```, vice versa + + Product purchases: users ```U```, products ```V``` +] + +.right-column[ +.white-background[ + +] +] + +??? + +## Recommendations + + + Can model recommendations as link following: + + From a user, follow to products + + From products, follow back to other users + + From other users, follow back to products + +--- + +## Measurements + + + Geodesic distance + + Number of edges to connect two vertices + + Eccentricity of ```v``` + + Largest geodesic distance from ```v``` to the most distant vertex + + Radius + + Minimum eccentricity of any vertex in the graph + + Diameter + + Maximum eccentricity of any vertex in the graph + + Peripheral vertex + + Vertex with eccentricity == diameter + + Incoming/Outgoing edge count of a vertex + + Number of edges coming in/out of a vertex + +??? + +## Data Stats + + + Similar to getting distribution stats from initial datasets, these + measurements can help you understand graphs as a summary + + Once you have the incoming/outgoing edge counts, you can use regular stats: + what is the distribution of counts? + +--- + +## Examples + +.white-background[ + +] + +??? + +## Answers + + + Distance 6, 5: 2 + + Eccentricity 2: 3 (disconnected graph is infinity) + + Radius: 2 + + Diameter: 3 + + Peripheral Verticies: 1, 2, 6 + +--- + +## Connections + + + Connected + + there exists a path from one vertex to another + + Connectivity + + minimum number of vertices to remove to disconnect remaining + vertices + + Clustering Coefficient + + Measure of how connected a vertex or group of vertices are + +??? + +## Robustness + + + This is used to understand robustness of a system: if an earthquake + damaged the Bay Bridge, could we still travel from one point to another? + + What is the connectedness of Oakland and SF? + + Closely related to min-cuts, which is discussed in the book + + Network topology: what happens if a router fails? + +--- + +## Clustering Coefficient + + + How many directed edges are possible between 3 vertices? + + 4 vertices? + + ```v*(v-1)``` + + Undirected? + + ```v*(v-1)/2``` + + Clustering Coefficient: Ratio of actual edges to possible edges amongst neighbors + +??? + +## Reading + + + Used in Reading this week + + ```v*(v-1)``` connection to every other node but yourself + + ```/2``` undirected, don't double count connections + +--- + +## Example + +.center[ +.white-background[ + +] +] + +??? + +## Answer + + + Clustering Coefficient of 1: + + Neighbors of 1: 5 2 + + 2*(2-1) / 2 = 1 + + Actual links = 1 + + Clustering Coefficient of 4: + + Neighbors of 4: 3,5,6 + + 3*(3-1) / 2 = 3 + + Actual: 0 + + If 3-5 connected? 1/3 + +--- + +## Random Walk + + + Many algorithms based on concept of randomly deciding: + + Follow link or not + + Which link to follow + + Simulate the decision many times + + What is the probability you will wind up on ```u``` from ```v```? + +--- + +# *Break* diff --git a/slides/2014-04-10-PageRank.html b/slides/2014-04-10-PageRank.html new file mode 100644 index 0000000..701bb25 --- /dev/null +++ b/slides/2014-04-10-PageRank.html @@ -0,0 +1,387 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-10-PageRank.markdown b/slides/2014-04-10-PageRank.markdown new file mode 100644 index 0000000..53b8ca6 --- /dev/null +++ b/slides/2014-04-10-PageRank.markdown @@ -0,0 +1,218 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# PageRank + +.center[ +.white-background[ + +] +] + +??? + +## Web page importance + + + Also used to model importance of people, places... anything that has a + reputation + + inbound links are important, but scaled by the importance of the source + + C is still important, even though it only has one inbound edge + +--- + +## Random Walks + + + Starting from a random page, what is the likelihood of winding up on a + target page? + + Starting point captured by initial constant + + Stopping captured by "damping factor" (0.85) + +.white-background[ + +] + +??? + +## Original + + + Original paper did not divide by N + + This gives relative weights of pages, but not a formal probability because + sum will not add up to N + + Either way is fine for our purposes + +--- + +## Example + + + ```B```, ```C```, ```D```, all link to ```A``` + + ```B``` has PageRank of 0.5, 4 links + + ```C``` has PageRank of 0.7, 4 links + + ```D``` has PageRank of 0.2, 1 link + +.white-background[ + +] + + + 0.15/4 + 0.85 * sum(PR/links for (pr,links) in pages) + + 0.15/4 + 0.85 * sum(0.5/4, 0.7/4, 0.2/1) + + 0.15/4 + 0.85 * 0.465 + + .43275 + +--- + +## Other Pages + + + But how did we know the PageRank of other pages? + + Start with something and calculate iteratively until convergence (sound familiar?) + +--- + +## Representing Graphs + + + Adjacency Matrix - represent graph edges in a matrix + +
+ +| V | A | B | C | D | +|---|---|---|---|---| +| A | 0 | 0 | 0 | 0 | +| B | 1 | 0 | 1 | 0 | +| C | 1 | 0 | 0 | 1 | +| D | 1 | 1 | 0 | 0 | + +??? + +## Diversion + + + Take a step back so we can motivate how to express these calculations as + linear algebra + + Using linear algebra can help us translate graph concepts to fairly elegant + code, as well as realize some optimizations + + Draw the graph! + + Symmetric? When? + +--- + +## Representing Graphs + +.left-column[ + + + Adjacency List - for each vertex, list all connections + +] + +.right-column[ +```csv +A [] +B [A,C] +C [A,D] +D [A,B] +``` +] + +??? + +## Diversion + + + You can think of this as keys (vertex) and values (list of vertices) + + When would thinking in key-values be useful? MapReduce + + Back to matrix representation + +--- + +## Eigenvector + + + PageRank formula divides by number of links + + Adjacency matrix typically normalized such that all columns sum to 1 + + PageRank scores are entries in the largest eigenvector of the matrix + representation + +.white-background[ + +] + +--- + +## Eigenvector centrality + + + Another measurement for graphs, using the simple adjacency matrix + + Relative influence of a node (no normalization) + +--- + +## Adversarial + + + Source does not want to be discovered + + Patterns are purposefully hidden: so discover the patterns of hiding + + If adversary knows your techniques, they can take advantage of weakness + +??? + +## Weakness + + + Reading: paper discovering hiding patterns + + Weakness of pagerank? + + We assume that these links are legitimate. + + What happens if the links are not conveying authority? + +--- + +## Google Bomb + + + Milder forms of adversarial work + + + +??? + +## Link farms + + + Link farms try to create fake links to pages, + + [JC Penny's link farm](http://www.nytimes.com/2011/02/13/business/13search.html?pagewanted=all) + +--- + +## Hubs & Authorities + +.left-column[ + + + Earlier in the web, there was more structure + + Hubs: collected links to different resources + + Authorities: Gave out specific information + + Score separately? +] +.right-column[ + +] + +??? + +## Alternatives + + + Some other interesting network analysis tools + +--- + +## HITS + + + Authority score + + sum(hub(i) for i in inbound_links) + + Hub score + + sum(authority(i) for i in outbound_links) + + Normalize + + to ensure convergence, square root sum of squares of scores + +??? + +## Iterative + + + Sill iterative, but now using inbound and outbound links to judge + + Hubs have outbound links to authoritive pages + + Authorities have inbound links from good hubs + +--- + +# *Break* diff --git a/slides/2014-04-17-Midterm-HW.html b/slides/2014-04-17-Midterm-HW.html new file mode 100644 index 0000000..e71993b --- /dev/null +++ b/slides/2014-04-17-Midterm-HW.html @@ -0,0 +1,227 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-17-Midterm-HW.markdown b/slides/2014-04-17-Midterm-HW.markdown new file mode 100644 index 0000000..119bb41 --- /dev/null +++ b/slides/2014-04-17-Midterm-HW.markdown @@ -0,0 +1,58 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Midterm + ++ Min 43 ++ Max 90 ++ Mean 75.3 ++ Median 77 ++ Mode 77 ++ Standard Deviation 10.2 + +--- + +## Homework: Midterm Corrections + + + Midterm should not be cram-and-forget + + Correct the mistakes on your midterms + + Ask questions for problems you don't understand + +??? + ++ Min 43 ++ Max 90 ++ Mean 75.3 ++ Median 77 ++ Mode 77 ++ Standard Deviation 10.2 + +--- + +## Deliverable + + + GitHub pull request: + + Question # : Correct answer + + Open book, notes, everything. Cite sources + + Have slightly more exposition than the midterm requires + +??? + +## Exposition + + + e.g., if the midterm asks 2 sentences, maybe write 3-4 + + If midterm asks for pseudo code, consider writing Python (though syntax + will not be graded) + +--- + +## Why? + + + These questions frequently come up on interviews and in discussions around + data. + + "How can you describe a distribution?" + + "Write out a MapReduce job to calculate click through rates." + + "How can we tell if a review is duplicated?" diff --git a/slides/2014-04-17-Multimedia.html b/slides/2014-04-17-Multimedia.html new file mode 100644 index 0000000..b5b68b5 --- /dev/null +++ b/slides/2014-04-17-Multimedia.html @@ -0,0 +1,484 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-17-Multimedia.markdown b/slides/2014-04-17-Multimedia.markdown new file mode 100644 index 0000000..0ba64b4 --- /dev/null +++ b/slides/2014-04-17-Multimedia.markdown @@ -0,0 +1,315 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Multimedia Data Mining + +--- + +## Features + +.left-column[ + + + Core algorithms similar to "traditional" data mining + + Difference lies in feature engineering + + How to translate intuitions to numbers and formulas? +] + +.right-column[ + +] + +--- + +## Types + + + Spatial + + geographic points and features, including natural and man-made + phenomenon + + Images + + Size, color, shape, curves, relative positions + + Music + + Tone, tempo, beat, rhythm + + Voice + + Speed, accent, word pauses, background noise + +??? + +## Covering + + + We'll cover these areas briefly to get an overview of techniques used in + these fields + + All of these things *have* embedded information in them, and we are trying + to extract it + + One of the reasons data mining is not a black box: some one has to be on + the outside interpreting results. Results inform technique + +--- + +## Generalization + +.left-column[ + + + Many of these areas have digital representations + + Can we use the raw bit representations? + + Usually not: must generalize patterns +] + +.right-column[ + +] + +??? + +## Density + + + The data we get from digital representations is generally too sparse + + Key component of good learning is *data*, but you need fairly *dense* data + to learn a pattern + + Hypothetically, a neural network could extract general features from raw + data, but you'd need a really large amount of data in order to get the + density needed + + Example: for NLP, perhaps your corpus is too sparse: not many words are + shared between documents. So instead generalize: what parts of speech or + patterns show up across documents? + +--- + +## Generalized Features + + + Derivative / Slope of behavior + + Min / Max of groups of points + + Bucketing / Blurring + + Relative positions / angles + +??? + +## Techniques + + + How can you strip some of the non-essential information, keep important + patterns? + + Many times we care about relative change, like in pricing + + Or group data points together (clustering is an advanced form of this) + + OK, let's get into some specifics: + +--- + +## GIS + +.left-column[ + + + Geographic Information Systems + + Analysis and visualization of geographic data + + Search, terrain, object detection, flow calculations +] + +.right-column[ + +] + +--- + +## Spatial Databases + + + Integrates spatial information with traditional DBMS operations + + Spatial indexing, distance metrics, polygon definitions, layering + + e.g., Oracle Spatial Data Cartridge, ESRI Spatial Engine, PostreSQL + + PostGIS + +--- + +## Discovery + + + What are examples of efficient city layouts? + + What influences successful business centers? + + Deforestation rates + +??? + +## Ideas + + + City layouts + + Understanding home->work distances, not Euclidean, but + traffic on streets or by public transportation, recognizing traffic jams + + Business centers + + analyzing network flow based on roads: industrial + supply centers nearby? Creative centers, restaurants, nightlife? + + Deforestation + + nearby cities' effect? Recognizing forested areas vs + clear cut. Time series + +--- + +## ATM Locations given obstacles + + + +??? + +## Yelp + + + This is a current area we could improve at Yelp + + Just because you're a mile from a restaurant doesn't mean it is "close" + + Maybe across the Bay, or maybe in between metro stops + + How can you calculate efficiently? + +--- + +## Images + +.left-column[ + + + General Feature Extraction + + Sketch Recognition + + Image Recognition +] + +.right-column[ + +] + +??? + +## Covering + + + We'll cover some interesting ways to extract dimensions + + ML/data mining combine these dimensions to do recognition with, e.g., + labeled data + + Image on the right is using an algorithm to pick out, then filter + "interesting" points on the image + + img: http://en.wikipedia.org/wiki/Scale-invariant_feature_transform + +--- + +## [SIFT](https://www.google.com/maps/place/San+Francisco,+CA/@37.8023623,-122.4055517,79a,24.4y,152.43h,85.12t,358.65r/data=!3m5!1e4!3m3!1s1547597185919489823!2e3!3e9!4m2!3m1!1s0x80859a6d00690021:0x4a501367f076adff) + +.left-column[ + + + Successively apply Gaussian blur to image + + Find points which "stand out" between blurs (i.e., big differences) + + You can connect these key points to make a kind of fingerprint + + These fingerprints can be used, scaled, etc. to match against other images +] + +.right-column[ + +] + +--- + +## Sketch Recognition + ++ Find (x,y) points along a sketch + + + +??? + +## Why? + + + Sketch recognition can be used to see if you're drawing shapes + + Be nice to be able to snap a picture of your diagram on a napkin and have + it come out nicely formatted? + + But how to recognize a circle, assuming you can't draw a perfect circle? + + Start with (x,y) points, but as we mentioned, very sparse + + Images by Marty Field + +--- + +## Direction + ++ Find angles along a sketch + + + +??? + +## Angles? + + + Instead of points, measure the angle at each turn + + You'll notice something peculiar about these angles. What? + + They're more than +/- 180 because we want to continue a "trend" if + they're turning the same way. Help identify changes in direction vs + spirals + +--- + +## Direction Plot + ++ Plot angles vs time + + + +??? + +## Why? + + + Becomes even more generalized: + + What is the derivative? + + How many times do we change derivatives? + +--- + +## Direction Plot + ++ Plot angles vs time + + + +??? + +## Why? + + + Example where we change directions + +--- + +## Features + ++ NDDE: Normalized Distance between Direction Extremes ++ DCR: Direction Change Ratio + + + +??? + +## Why? + + + NDDE: Are there discontinuous changes in direction, or is the line + generally curvy and follows a similar path? + + DCR: Total amount of angle change in the sketch. Low for first, high + for second + + Others?: bounding box size/ratio, stroke length, distance between endpoints, + length, width, height, speed, direction, acceleration + +--- + +## All Together Now + +http://player.vimeo.com/video/6496886 + +--- + +## Music + + + Generate a finger print: time, frequency, amplitude + + Filter most intense (largest) amplitudes + + Create a hash of connections between points + + Match, in time, the hash between songs + + + +??? + +## Relation to Images + + + Interesting to note: we transformed one media type (music) into another + (image), then started using some techniques we've seen in image + fingerprinting + + More in reading + +--- + +# *Break* diff --git a/slides/2014-04-17-Outliers.html b/slides/2014-04-17-Outliers.html new file mode 100644 index 0000000..5a83f16 --- /dev/null +++ b/slides/2014-04-17-Outliers.html @@ -0,0 +1,437 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-17-Outliers.markdown b/slides/2014-04-17-Outliers.markdown new file mode 100644 index 0000000..0401718 --- /dev/null +++ b/slides/2014-04-17-Outliers.markdown @@ -0,0 +1,268 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Outliers + +--- + +## Generative Model + + + "Real" model that produced original data points + + Our mission is to reproduce the original model + + Thus we have different techniques that can model different behavior + +??? + +## Questions + + + What is a "generative model"? + + What is data mining trying to discover? What is machine learning hoping to + reproduce? + + Why have different classifiers? Decision tree, Naive Bayes, etc? + +--- + +## Outliers + + + Significant deviation + + Probably generated through a *different model* than the rest of the data + + Normal / Abnormal + +.center[ + + +] + +??? + +## Intuitive + + + We all have a pretty good intuitive understand of what outliers are + + Mathematically, you can express the variation as a different generative + model + + Normal / Abnormal data (be careful about using it in human contexts) + + img: http://enriquegortiz.com/wordpress/enriquegortiz/research/undergraduate/ + +--- + +## Outlier Types + + + Global + + points which deviate from the rest of the *entire* data set - point + anomalies + + Contextual + + points which deviate from their *peers* - conditional outliers + + Collective + + points which deviate as a *group*, even though individual + points may not be considered outliers. + +--- + +## Which Type? + + + Given class sizes at Berkeley: + + A day with 10 people in class + + A day with 7000 people in class + + 3 weeks of 15 people in *this* class + + Given Earth's temperatures: + + A day at 100°C + + 30 straight days of rain in Berkeley + + A day at 100°F + +--- + +## Types of Learning + + + Supervised + + Unsupervised + + Semi-Supervised + + + +??? + +## Types of Learning + + + Supervised: learning from "gold standard" labels + + Unsupervised: learning without labels + + Semi-Supervised: infer more labels from a few, learn based on inferred + + labeled + + img: https://www.coursera.org/course/ml + +--- + +## Outlier Methods + + + Supervised + + Label outliers, treat as classification problem + + Unsupervised + + Cluster data, find points not clustered well + + Semi-Supervised + + Manually label a few, find points nearby to automatically + label, then treat as classification + + Statistical + + Decide on a generative model / distribution, find points + which have a low probability of belonging + + Proximity + + Use relative distance to neighbors + +??? + +## Features + + + Some methods may be overlapping + + When developing features for classification, using relative features can + be helpful: e.g., distance from mean + + e.g., Agglomerative clustering, find lone/small groups that are last to + glom together + + e.g. k-means find points which are "far" out from centroids + + Determining "far", "last" can be application specific, part of the + challenge + + What algorithm could we use to automatically label nearby points? k-nearest + neighbor + + Statistical: Again, must define "low" in your domain + + Proximity: basically translating features into another, relative space, + then applying a different type of outlier detection (e.g., statistical) + +--- + +## Statistical + + + Assume a distribution + + Determine parameters + + Calculate probability of a point be generated by distribution + +??? + +## Why Statistical + + + We've covered supervised and clustering, so let's skip to statistical methods + + Most straight forward way is to use distributions + +--- + +## Statistical Example + + + Assume a normal distribution + + Determine mean and standard distribution + + If ```(point-mean)/stddev > 3```, consider it an outlier + + + +??? + +## Pros/Cons + + + Straightforward + + Can use % to intuitively motivate (3 stdevs is outside 99.7%) + + But must manually determine cut-off + + How do we know we got the parameters right? + +--- + +## Grubb's Test + + + Takes into account sample size; reliability of mean/stddev measurements + + Take Z-score of a point, assign to ```G``` + + Student t-test: used to measure the distribution of *actual* mean from a + sample + +.white-background[ + +] + +??? + +## Pros/Cons + + + Z-score: ```abs(x-u)/s``` + + This isn't actually *that* different from measuring stddev + + But accounts for sample size, can express your confidence with alpha 95% (0.05) + + Not going to go into t-test/t-distribution here, but basically it helps + show where the mean likely is, given a set of sample data. + +--- + +## Outlier Distance + + + How to find outliers in > 1 dimension? + +??? + +## Limitations + + + What are the limitations of the techniques we've seen? + + Limited to one dimension! Taking mean, stddev, etc. applies to 1 + dimension + +--- + +## Outlier Distance + + + How to find outliers in > 1 dimension? + + Translate distance to 1 dimension, find outliers + + How to measure distance? + +??? + +## Limitations + + + Euclidean: doesn't take into account dependent variables + +--- + +## Mahalanobis Distance + + + ```y``` depends somewhat on ```x``` + + Euclidean distance measures all dimensions equally + + Use *covariance matrix* to normalize distances in each dimension + + Matrix in which ```E_i,j``` is the covariance of ```i```, ```j``` dimensions + +.center[ + +] + +??? + +## Mahalanobis + + + How to capture intuition that a distance along major axis is different than + along this minor axis? + + Expand this drawing into 3 dimensions + + Euclidean distance will equally weight something that is out in the ```z``` + direction as something that is along this primary scatter area + +--- + +## Mahalanobis Definition + + + Find the mean vector + + Normalize by covariance + +.white-background[ + +] + +??? + +## Some Math + + + Some extra math tricks to make the units work out: + + We're taking the squared distance, then taking the square root + + DM has *squared* Mahalanobis distance defined + + What happens if we have no covariance? S is the Identity matrix + +--- + +## Contextual Outliers + + + Typically reduce scope to context, use global techniques + + Example: Calculate normal distribution for Berkeley weather + + Collective outliers: find collections, use as context + +--- + +# *Break* diff --git a/slides/2014-04-24-D3.html b/slides/2014-04-24-D3.html new file mode 100644 index 0000000..0fcdeb1 --- /dev/null +++ b/slides/2014-04-24-D3.html @@ -0,0 +1,256 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-24-D3.markdown b/slides/2014-04-24-D3.markdown new file mode 100644 index 0000000..014ddf6 --- /dev/null +++ b/slides/2014-04-24-D3.markdown @@ -0,0 +1,87 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +## Develop a Visualization + + + Learn D3 + + Extract data from your project + + Build a visualization of it + +--- + +## Deliverable + + + HTML with visualization + + Dependencies + +--- + +## Example + +.tight-code[ +```code/histogram.html``` + +```shell +# extract 2000 reviews +grep 'type": "review' yelp_academic_dataset_review.json | \ + head -n 200 > reviews-200.json +``` + +```python +import json +with open("reviews-200.json") as f: + r = map(json.loads, f) +# extract star ratings +[rev['stars'] for rev in r] +``` +] + +??? + +## Notes + + + Include notes on how you extracted your data + + Notes can be in a separate files, or comments in the code + + My notes might be these shell/python commands + +--- + +## Partners + + + If you've *never* written JS on your own: + + Find someone familiar with D3 + + Still must turn in separate homeworks + + Cite your sources + +??? + +## Javascript + + + Since this class is not teaching Javascript, you'll need to learn on your + own + + Special case: find someone to help you *learn* + + Folks who know D3: this assignment is not a challenge, so please find + someone to help. Teaching is a great way to learn + +--- + +## Extra Credit + + + [Vega](http://trifacta.github.io/vega/) is a JS visualization Grammer + + Write homework in Vega instead + +??? + +## Closer to Grammar + + + Vega is a declarative way of specifying a graphic + + Uses D3 underneath + +--- + +## D3 Intro + + + [D3 Intro](http://vogievetsky.github.io/IntroD3) diff --git a/slides/2014-04-24-Visualization.html b/slides/2014-04-24-Visualization.html new file mode 100644 index 0000000..60e6a39 --- /dev/null +++ b/slides/2014-04-24-Visualization.html @@ -0,0 +1,528 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-24-Visualization.markdown b/slides/2014-04-24-Visualization.markdown new file mode 100644 index 0000000..8ee4339 --- /dev/null +++ b/slides/2014-04-24-Visualization.markdown @@ -0,0 +1,359 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Visualization in Data Mining + +--- + +## Your Brain + +.left-column[ + + + Pattern detector + + Visualizations help you search for possible models + + Help intuitively understand the data +] +.right-column[ + +] + +??? + +## Visual + + + For most people, vision is the strongest sense + + Recall improves 55% (10% => 65%) with the addition of a picture + + We've talked about the need to understand the data before using + algorithms on it. Visualization can speed that process up. + +--- + +## Patterns + + + Use visualizations that surface patterns and relationships + + Know the context for the visualization + + Verify results + +??? + +## Steps + + + For gaining intuition, focus on simple visualizations that help you see + relationships in the data. + + At this time, labels, titles, etc. not very important. Multiple dimension + in multiple windows? Fine! + + We'll discuss, but the context a visualization is going to be used in + matters a lot. Don't feel like you have to import every cool infographic + into your project + + Clustering, classification, outlier selection can be verified visually, + e.g., highlighting points. Use it to gut check conclusions, even if you + have to drastically reduce dimensionality + +--- + +## Scatter + + + Great for multidimensional data + + Just plot > 2 dimensions in different plots + + Reveals correlation, clustering, distribution, ... + +??? + +## Data Mining + + + DM bread and butter. Often deal with high dimensionality, so scatter is one + of the best ways to visualize + + Wide variety of patterns can be searched + +--- + +## Multiple Dimensions + + + +??? + +## vp + + + This data is for body positions over time + + Dimensions are the different angles for different body parts, like hip + ankle, knee, over time + + We can see some strong patterns. Maybe we'll need to kernelize them to + make them learnable, but we have a good understanding that there are, or + are not relationships between the data + +--- + +## Geographic + +.center[ + +] + +??? + +## Trade-offs + + + Coordinates intuitively understandable + + Dependence on geographical area (e.g., when you'd like to depend + on human impact instead) + + Lots of ways to bucket/aggregate + + 2004 Presidential election - Bush won 50.3% of the popular vote and + Kerry 48.3%, does it looks like that here? + + img: http://www-personal.umich.edu/~mejn/election/2004/ + +--- + +## Geographic + +.center[ + +] + +??? + ++ Don't be afraid to "bend" things to get different insights ++ Each pixel represents 1,000 votes in the 2004 Presidential election + + Red ==> Bush + + Blue ==> Kerry + + Green ==> Nader ++ Because some parts of the country have far more than 1000 votes per pixel, + draw the pixel on the closest part of the map that isn't already used ++ You lose precise vote locations, but you see how mixed the results actually + are and how population density is involved + +--- + +## Other Chart Types + + + Box plot + + aggregate data + + Bar charts + + simple summaries + + Pie charts + + compound proportions + +??? + +## Types + + + Box plots, for real data, still carry a lot of data + + Bar charts nice for summarizing, not great for exploring + + Same for pie charts. Pie charts are mostly bad, but can use in particular + circumstances + +--- + +## Aesthetics + + + The visual aesthetics you use should be tied to the data + + + +??? + +## Aesthetics + + + What are some of the techniques we can use to tie data to a visual + representation? + + img: Kevin Lynagh, http://keminglabs.com/talks/ + +--- + +## Larger Value? + + + Position + + Length / Angle + + Area / Volume + + Color: Chroma Luminance + +??? + +## Slide Switch + + + Hadley Wickham slides, OSCON: http://cdn.oreillystatic.com/en/assets/1/event/80/Designing%20effective%20visualisations_%20matching%20data%20problems%20to%20our%20perceptual%20strengths%20%20Presentation.pdf + +--- + +## Color: HCL + +.left-column[ + + + Hue + + color type + + Chroma + + colorfulness, perceived color intensity + + Luminosity + + brightness, light-dark +] +.right-column[ + +] + +??? + +## Color Spaces + + + Many other color spaces, probably most familiar with RGB + + HCL is useful because it separates the properties of a color into ones + that can be mapped to data + + Hue: nominal, can't compare + + Chroma, Luminosity: numerical / comparable value + + Chroma vs Saturation: chroma *perception* relative to white, saturation + measure of color intensity + + http://rourkevisualart.com/wordpress/2008/02/22/the-difference-between-chroma-and-saturation/ + +--- + +## ColorBrewer + + + http://colorbrewer2.org/ + + Type of comparison => type of color difference + + Lots of other practical features + +--- + +## Careful + + + Some aesthetics can combine to form illusions + + http://www.michaelbach.de/ot/sze_sineIllusion/ + +??? + +## Line Lengths + + + Line lengths can appear to look smaller when extended instead of right + next to each other + +--- + +## Careful + +http://www.youtube.com/embed/FWSxSQsspiQ + +??? + +## Comparisons + + + We're good at comparing things side by side. + + We're bad at comparing things from memory. + +--- + +## Grammar of Graphics + + + Geom + + Graphic element + + Aesthetics + + appearance of a geom + + Data + + raw, context, statistical aggregations of data + + Mapping + + functions which map data to geom properties or aesthetics + +??? + +## Bringing Together + + + We've talked about different aesthetics of showing data, we've talked about + data, all that's needed is to bring them together + + Wilkinson, L. (2005), The Grammar of Graphics (2nd ed.). Statistics and Computing, New York: Springer. + + Rigorous way of describing graphics beyond "scatter plot" or "bar chart" + +--- + +## Scatter Plot + + + +??? + +## Ice Cream + + + Plot shows hypothetical sales of ice cream vs temperature + + Geoms + + points (actually, ticks are geoms, too) + + Data + + sales, temperature (and context: how large is the potential plot + size) + + Mapping + + sales ==> y, temp ==> x + + img: http://www.mathsisfun.com/data/scatter-xy-plots.html + +--- + +## Bar Plot + +.white-background[ + +] + +??? + +## Fruit + + + Plot shows fruit popularity + + Geoms + + bars (and ticks, text) + + Data + + fruit to popularity + + Mapping + + popularity ==> height, fruit type ==> x, color + + img: http://www.mathsisfun.com/data/bar-graphs.html + +--- + +## Hipmonk + + + +??? + +## Fruit + + + Shows travel options from SFO to Ithica, connecting flights, airports, etc. + + More complex, but still expressible via Grammar + + Geoms? + + rectangles, text, ticks + + Data? + + Carrier, flight time, layover time, cost, wifi available, airports + + Mapping? + + travel time ==> bar length, flight times ==> sub-bars, airline ==> color + + img: http://www.hipmonk.com + +--- + +## Recursive + + + +??? + +## Complex + + + Reading will go into a further extension of this, where the geoms are + themselves other plots + +--- + +## Tufte + + + Clarity from data + + Avoid chart junk + + Techniques for displaying many types of data + + + +??? + +## Tufte + + + No talk on visualization would be complete without mentioning Tufte + + Great examples + +--- + +# *Break* diff --git a/slides/2014-04-24-Yelp-Visualization.html b/slides/2014-04-24-Yelp-Visualization.html new file mode 100644 index 0000000..edb6499 --- /dev/null +++ b/slides/2014-04-24-Yelp-Visualization.html @@ -0,0 +1,625 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-04-24-Yelp-Visualization.markdown b/slides/2014-04-24-Yelp-Visualization.markdown new file mode 100644 index 0000000..83a2cf2 --- /dev/null +++ b/slides/2014-04-24-Yelp-Visualization.markdown @@ -0,0 +1,456 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# Visualizing Data at *Yelp* + +--- + +## Visualizing Data is Important + + + Effectively summarizes data + + Highlights patterns + + Improves recall + +??? + +## Metrics + + + Can't improve something until you measure it + + True, but have to look at and understand the data! + + Often best way to understand data is visually + + Having metrics you care about evident will make you focus on improving + them + + The more sophisticated your visualizations, the more sophisticated your + goals + +--- + +## Visualizing Data is Difficult + + + Requires investment + + Dimensions of success + + Successful visualizations in Yelp + +??? + +## Role in Yelp + + + Often requires specific domain knowledge of both the data and the tools + + Ideally have 2 big screens per pod + + "That's a lot of TVs!" + + Get motivated everyday + + Show what you care about + + Don't want a sterile office, decorate with the results of your work + +--- + +## Where are the hipsters? + +http://www.yelp.com/wordmap + +??? + +## How many reviews include certain words, on a map + + + Can help you find dim sum outside Chinatown + + Helps you understand neighborhoods + + But main feature is... Cool + +--- + +## Cool + + + Good looking is a dimension of any visualization + + People keep looking for a "Minority Report UI" + + + +??? + +## Cool is OK + + + Engineers need to come to grips that to be visually compelling, a visualization needs to look nice + + Just like the most compelling novels need to be well written + + We realize this, we just don't like to admit it + +--- + +## Avoid Chart Junk + + + Edward Tufte rightfully suspicious of cool + + Worry about ```data/ink``` ratio + + But remember tradeoffs: memorability, fun + + + +??? + +## Useful Junk? + + + Data/ink ratio describes the amount of information displayed per ink/pixel + + If you remove a pixel, will you remove information? + + Best Paper by Scott Bateman HCI: some useful Junk + + Noted the context of the chart + + Bad ratio limits richness, especially important on mobile + +--- + +## Graperr + + + + +??? + +## Shows errors live from log + + + Error activity + + Highlight error type UnicodeDecodeError + + Text details available + + Still Cool! + + Colors slick, modern + + But used for differentiation (data) +--- + +## Actionable + +.left-column[ + + + Realtime* + + Context + + Connections +] +.right-column[ + +] + +??? + +## Definitions + + + As realtime as problem domain requires + + Seconds matter when fixing site problems, so should be up to the second + + Days or weeks might matter when deciding budget issues + + Context: Is this a normal amount of errors? + + Connections: Ability to drill down to specific instance + +--- + +## Dimensions + + + Fun + + cool, pretty, engaging + + Actionable + + realtime, contextual, connecting + +??? + +## Agenda + + + Dimensions important to visualizations + + Axis on which you can evaluate them + + Tradeoffs in developing them + +--- + +## Mario + +.center[ + +] + +??? + +## Ads Visualizations + + + Realtime tracking + + Clicks + +--- + +## A Tale of Two Datacenters + + + Testing datacenter failover + + Tracking metrics in a new way + + Did we spend a week preparing a dashboard? + +??? + +## How? + + + Yelp used to be in only one datacenter + + Moving to two datacenters is a huge undertaking, but worth it for + reliability reasons + + Don't want to bring down a worldwide site when freak electrical storms hit your datacenter + + After months of work, how did watch over our site when we finally flipped + the switch? + + This was the first time Yelp had done this: we didn't have a premade + dashboard so everyone could track the important metrics + +--- + +## Firefly + + + +??? + +## Demo + + + One of our many open source projects + + Hosted on Github + + Existing extension to Ganglia + +--- + +## Easy + + + Make repeated operations fast and within reach + + Must understand problem domain + + Accessible + +??? + +## Definitions + + + Sophisticated Tool: Data discovery, stacking options, coloring, layout + + But all of the steps are repeated, formulaic: we're making similar things over and over + + So make it easy! + + Not much more accessible than Web: share links, etc. + +--- + +## Easy from Simple + + + Avoid temptation to make visualizations easy from the start + + Easy systems are designed for non-experts + + Long term investment in the system to manage complexity + +??? + +## Non-experts + + + Simple Made Easy, Rich Hickey + + Still potentially technical users + + Just don't know the details of how metrics are collected, or how to display + across browsers + + Always will require experts to make changes + + Always are going to want new features + + Make sure you have the ability to add them + + Not extensible + +--- + +## Search Maps + + + +??? + +## Times Change + + + 2005, 9 years ago + + May not seem like important visualization, but times have changed + + Full page refresh for each map square + + Now we take zoom in, panning for granted + + Sign of a great visualization: don't think about it: it's a tool + + What else are we not plotting on maps that we should be? + +--- + +## Interactive + +.left-column[ + + + Fast + + Explorable + + Feedback +] +.right-column[ + +] + +??? + +## Definitions + + + Fast + + One of the reasons its a fairly recent technology, hard to get fast + + Speed gives the UI illusion that you are interacting with a physical + thing, something we're much more comfortable with + + Explorable + + Multiple levels of detail that can be discovered by user + + Feedback + + Update all other dependent displays (search results) + +--- + +## Where did D3 come from? + + + Michael Bostock had a problem + + Protovis useful, but not flexible + + How to provide coherent description for visualizing data? + +??? + +## D3 Intro + + + Mike Bostock professor at Stanford + + Protivis was a declarative Javascript charting library + + But hard to keep up with changes in technology + + Wasn't quite flexible enough for new visualizations + +--- + +## D3: Data-Driven Documents + +http://bl.ocks.org/mbostock/raw/1256572/ + +--- + +## Flexible + + + Language level + + Access to medium + + Access to data +```javascript +d3.selectAll("p") + .data([4, 8, 15, 16, 23, 42]) + .style("font-size", + function(d) { return d + "px"; }); +``` + +??? + +## Why? + + + Metaphor natural language + + General language most flexible tool humans have to describe new things + + Full access to medium to be able to create take advantage of all possibilities + + and new tech + + Not D3 specific, but need full data to find new ways to summarize, explore, + drill + + Need to understand where data came from to clean, normalize + +--- + +## Dimensions + + + Fun + + cool, pretty, engaging + + Actionable + + realtime, contextual, connecting + + Easy + + available for non-experts, remove repetition + + Interactive + + fast, explorable + + Flexible + + expressive, full access to lowest level + +??? + +## Tension + + + Obvious: Flexible vs Easy. Too many options is confusing. + + Less obvious: Interactive vs Actionable. Spend too long playing, not enough fixing + + In fact: All in contention for your time + +--- + +## Understand Usage Context + +--- + +## Press: Fun + +.center[ + +] + +--- + +## Alerting: Actionable + + + +??? + +## Search Metrics + + + This is a visualization of the status of our search cluster + +--- + +## Product Managers: Easy + + + +--- + +## Investigation: Interactive + +.white-background[ + + +] + +--- + +## Explorable: Interactive + +.center[ + +] + +??? + +## Another Case + + + Another case for Interactivity is geographical data + +--- + +## New tools: Simple + + + +--- + +## New tools: Flexible + +.center[ + +] + +??? + +## Unique + + + You can see this is not a standard visualization + + It is one that is customized to its purpose + + Made possible by flexible tools + +--- + +## Dimensions + + + Fun + + cool, pretty, engaging + + Actionable + + realtime, contextual, connecting + + Easy + + available for non-experts, remove repetition + + Interactive + + fast, explorable + + Flexible + + expressive, full access to lowest level + +??? + +## Consider Tradeoffs + + + Visualization is just part of making an effective biz, team + + Interested in working at Yelp? diff --git a/slides/2014-05-01-Real-World.html b/slides/2014-05-01-Real-World.html new file mode 100644 index 0000000..dbee80a --- /dev/null +++ b/slides/2014-05-01-Real-World.html @@ -0,0 +1,815 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/2014-05-01-Real-World.markdown b/slides/2014-05-01-Real-World.markdown new file mode 100644 index 0000000..1937d40 --- /dev/null +++ b/slides/2014-05-01-Real-World.markdown @@ -0,0 +1,646 @@ +name: inverse +layout: true +class: left, top, inverse + +--- + +# *Datamining IRL* + +--- + +## General vs. Practical + + + "the language in which you'll spend most of your working life hasn't been + invented yet, so we can't teach it to you. Instead we have to give you the + skills you need to learn new languages as they appear." + + -- Brian Harvey, [Why SICP matters](http://www.eecs.berkeley.edu/~bh/sicp.html) + +??? + +## What's Really Important? + + + Important: understanding your domain, asking interesting questions, + answering them with data, fitting questions to mathematical concepts + + Not as important: scipy, mrjob, d3 + +--- + +## But + + + Are you interviewing? + + Are you starting your own company? + + Want to build your portfolio? + +??? + +## Real World + + + Many of you may be needing to apply this stuff very soon + + So here's the lecture where I try to tell you what I would do when + practicing data mining + +--- + +## Product or Company Ideas + + + Understand exponential growth + + Get the timing right + + Execute, execute, execute + +??? + +## Idea required but not sufficient + + + Wide variation of thoughts on ideas + + One of the biggest road blocks for wannabe entrepreneurs, but most derided + + Timing is mostly luck, is the world ready for your ideas? + + I think the *vision* is important, it's what drives the company, keeps + people working + + The plan for the idea less so + + The concrete product most important + + Let's talk about these elements + +--- + +## Toys + + + "the next big thing always starts out being dismissed as a “toy.”" + + Chris Dixon, [Blog](http://cdixon.org/2010/01/03/the-next-big-thing-will-start-out-looking-like-a-toy/) + + + +??? + +## Why? + + + Also, [Stupid Ideas](http://dcurt.is/what-a-stupid-idea) + + A few reasons for this + +--- + +## Exponential Improvements + + + The future is changing at a faster rate than ever before + + Every field is being quantized, instrumented + + "You're not on the Internet?": "You're not analyzing your data?" + +.center[ + +] + +??? + +## Change to Data + + + You think your parents are bad at not operating devices? Your habits will + become out of date > twice as fast. + + So what happens when we combine exponential improvements and instrumented + fields? Overwhelming amounts of data + + Winners will be the ones who use this to their advantage, for example... + +--- + +## Exercise + +.left-column[ + +] +.right-column[ + +] + +??? + +## Strava + + + No device, just data + +--- + +## Thermostats + + + +??? + +## Nest + + + OK, they have a device... but what separates them is data + +--- + +## Biology + + + +??? + +## Gene sequencing + + + Biology: data growing much faster than hardware is advancing + + img: http://www.genome.gov/sequencingcosts/ + +--- + +## Execute + + + Users are hiring you to do a job: what is it? + + "Institutions will try to preserve the problem to which they are the solution." -- [Clay Shirky](http://www.shirky.com/) + + Make your product so easy to use, people do it by accident. + +??? + +## Do the job + + + All of these company examples, you're typically not thinking of them as + "data processors"... they are solving a specific problem for you + + Strava isn't doing any crazy SVM analytics (at least on the consumer facing + side): they're showing you min/max, avg speed. Simple, but effective, + stuff. + + Disruption most often comes from using established technologies in new ways + or areas + + Can disrupt by completely simplified, often crappy at first, solutions to + an even more fundamental problem + + Dell did a great job selling cheap computers, then more expensive computers + + But now Amazon is saying: "you don't even need to own computers!" (Cloud) + + More info: [Clayton Christensen](http://www.claytonchristensen.com/) + + Focus on that one thing that is important and do it very, very well + +--- + +## Specifics ([The Joel Test](http://www.joelonsoftware.com/articles/fog0000000043.html)) + + + Do you use source control? + + Can you make a build in one step? + + Do you make daily builds? + + Do you have a bug database? + + Do you fix bugs before writing new code? + + Do you have an up-to-date schedule? + + Do you have a spec? + + Do programmers have quiet working conditions? + + Do you use the best tools money can buy? + + Do you have testers? + + Do new candidates write code during their interview? + + Do you do hallway usability testing? + +??? + +## Joel on Software + + + When developing software, please follow as many of these as reasonable + + Joel Spolksy wrote this in 2000! Still a great guide! + + This is what I'd suggest to quickly get moving on the right foot + + If you're managing a team, make sure these are happening + +--- + +## Source Control + + + +??? + +## Surprised? + + + Github will solve a few problems on this list, just use it, even if you're + developing alone + +--- + +## One step build + + + Data mining exploration often involves manual commands + + *Don't* do that in production + + Should have scripts which extract features, build model, verify, deploy + +??? + +## Area for Improvement + + + This is actually a big area needing solutions + + Deploying websites has solutions like Heroku, but no equivilant for + storing, processing, serving data + +--- + +## Bug Database + + + Easy to loose track of problems + + Also good way to prioritize issues + + Use [Github](http://github.com) Issues + +??? + +## Managing Up + + + Good defense + +--- + +## Write a Spec + + + Alternatively, write the press release + + Don't write a novel + + Disagreements can be solved with code, but after talking + +??? + +## Bad rap + + + Developers don't usually like writing them + + But it helps nail down issues + + Yelp uses CEP process + + If you get to the "agree to disagree" point, data or code can solve + differences + +--- + +## Testers + + + Use *unit tests* to test code (eg. ```unittest2``` in Python) + + Use cross-validation to test models + + Very easy to skip, will bite you within 6 months + +??? + +## Differences + + + Joel suggests having and paying testers + + I don't think this is best use of resources for small companies + + Economics change when developers can effectively write tests + + *Must* allocate time to this + + Add tests when you fix bugs + + Helps if developers use product daily + +--- + +## Tools + + + Right tool for the job + + Text Editor: Use vim, emacs, Sublime Text, etc. for Python, Eclipse for Java, ... + + virtualenv (Python); RVM (Ruby) + + Learn the command line + +??? + +## Woodworker + + + (slightly off topic from Joel's list) + + Woodworkers don't hammer stuff in with their shoe + + Make their own tools as first part of job + + When a custom problem comes up, make a custom tool + + These slides, written in Markdown in Sublime Text, HTML built with a + Makefile + + Text Editor + + Syntax Highlighting + + Macros + + Interact with other tools + + Find across files + +--- + +## How to Use Recommendations + + + Start with them as default + + If you understand why something is better for your case, use it + + Understand trade-offs + +.center[ + + +] + +??? + +## Trade-offs + + + One of the themes of this course + + Trying to provide you with a starting point + + My point of view: user driven behavior, engineers implementing solutions + +--- + +## Data Storage + + + S3 for unstructured data + + PostgreSQL for structured + + Hive on S3 for very large structured data + +??? + +## Data most important asset + + + S3 is a pay-as-you go model, opens up many data processing possibilities, + you get price drops without doing any work + + Don't have to worry about how to connect + + PostgreSQL solid database, but also offers many improvements like storing + geo data + + Once you get beyond PostgreSQL limits, use Hive to structure data in S3 + +--- + +## Exploration + + + Python + + IPython Notebook, matplotlib + +.center[ + + +] + +??? + +## Python + + + Main reason: it is convenient and practical to stay in the same language as + production + + Using production libraries, settings, to extract data + + R, matlab/octive, Tableau are typically not used in large production code + + SAS also effective for exploration, can be used in production, but skill + set not as transferable for smaller companies + +--- + +## Public Visualizations + + + D3 for visualizations + + HTML is sharable, universal + +--- + +## Processing + + + Hadoop + mrjob + + Elastic MapReduce + + (Adventurous: [Spark](http://spark-project.org) and + [Storm](http://storm.incubator.apache.org/)) + + + +??? + +## Scaling + + + Hadoop scales up and down fairly well, especially with mrjob + + Constraints are going to be on *your* time, not necessary to eek out every + bit of computing poser + + Spark is a new model out of Berkeley that does a better job of keeping data + in memory, but doesn't have the maturity of Hadoop + + Storm is much like MapReduce but designed for doing computation in real + time + +--- + +## Models + + + Text: Naive Bayes + + Numeric Classification: SVMlight + + General: sklearn/RandomForrestClassifier + +??? + +## Even then + + + Start with simple stats to understand your data + + Next: use heuristics, they are easy to understand and change + + Next: use third party models that you can drop in + + Often heuristics with understanding of false postive/negative costs will + get you far + +--- + +## Practice + + + [Yelp Dataset Challenge](http://www.yelp.com/dataset_challenge/) :) + + [kaggle](http://www.kaggle.com/) + + [Programming Collective Intelligence](http://www.amazon.com/Programming-Collective-Intelligence-Building-Applications/dp/0596529325) + + Ask around Berkeley + +??? + +## Other services + + + Dataset challenge is open ended, so it lets you practice all elements + + Kaggle has many great competitions + + Collective Intelligence has many good examples + + Keep in mind trade-offs: that's what interviewers will ask + +--- + +# *Work* + +??? + +## Topic Change + + + Jumping topics a bit, what if you'd like to work at a web company instead + of build one? + +--- + +## Hiring + + + Learn about the company + + Ask questions to learn about their problems + + Provide solutions + + + +??? + +## Experience + + + Use experience to answer questions + + Make sure you continue asking questions in the interview + + Ramit Sethi calls this the [Briefcase Technique](http://www.iwillteachyoutoberich.com/the-briefcase-technique/) + + Know what's on your resume (Why is it applicable? Why is it interesting?) + + Think of the "interview" as a conversation, what would you say if you met + in a coffee shop? + +--- + +## Resume is a Formality + + + Be recognized independently of being in the resume pile + + Present at meetup + + Use their product in a cool way + +??? + +## Recognition + + + Catch their attention, then start process + + Also makes you think "Do I *want* to work for this company?" + + Stories + +--- + +## Resume + + + Use quantitative data + + Describe the difference you made in a company/project, not what you did + + Include your side projects! + +??? + +## Unique + + + What makes you a unique candidate? + + Your side projects set you apart. All students here have made a web + page. How is yours different? + +--- + +## Negotiation + + + Always try to have > 2 offers on the table + + Once a company decides, they've already sunk a lot of resources into you + +??? + +## Timing + + + Pace interviews so you can make the decision together + +--- + +## Do What it Takes + + + Most essential attribute: asking great questions + + > 50% of the work will be finding, formatting data + + Data product must be reliable to be effective + + Learn about distributed computing, software engineering + +??? + +## The Job + + + The thing that can't be taught is to think creatively about + all the cool stuff you can do with this data, frame it in a way that is + specific, actionable + + Most jobs require a combination of DM and coding skills + + Companies don't need just "idea people", need "idea + execution" + + Don't expect to just put on you DM lab coat and work with Kaggle-style data + all day + + Remember, biggest impact comes from putting together *existing* technology + in a useful way + +--- + +## T-shaped Skills + + + +--- + +## Managing upward + + + Ideal email: "I've done the analysis below and recommend we do X. Sound good?" + + If no one is in charge, you're in charge + + Say "yes" but prioritize + +??? + +## Busy + + + Your boss is busy, you do the work, make sure you're on the right track + + You shouldn't take on everything, but also shouldn't just start rejecting + things. + + Be a positive person: yes, we can do that after X, Z + +--- + +## Engineering Career Paths + + + Hacker + + Very broad, up-to-date. Best suited in very early startups. + + Individual Contributor + + Reasonably skilled in areas of interest. Best + suited in mid-sized to large companies. + + Principal Engineer + + Company or industry wide recognition for contributions + in specific areas. Very strong T-shaped skills. + + Manager + + Ability and desire to solve people challenges, verify technical + solutions. + +??? + +## Gross Simplification + + + Hacker: just get things done long enough to find a business model + + IC: majority of engineers, doing solid day-to-day work. + + Principal: Can include CTO at some companies, "tech leads." Go to person + for leading up projects. Must have a history of success, + + Management: If you like working with people, coaching, growing a team. + People are more complex than machines, so are solutions. + + Big themes: ownership, focus, excellence + + [Joel's Ladder](http://www.joelonsoftware.com/articles/Ladder.html) + +--- + +## Stay Sharp + +.left-column[ + + + Long term, expected to combine the best of both... + + Skills + + Wisdom + + So keep building skills +] + +.right-column[ + + +] + +??? + +## Dig + + + Dig into areas you're not familiar + + Talk to people, help solve their problems, learn how it turned out + + img: http://shirt.woot.com/blog/post/stay-sharp + +--- + +## Networking + + + Ask questions + + Learn from others + + Help others + + Don't skip stuff because you're lazy or scared + + + + + [Shy Connector](http://www.slideshare.net/sachac/the-shy-connector) + +??? + +## Skipping Stuff + + + There are many good reasons not to go to an event, but being lazy is not + one of them + + Best opportunities are when you do stuff that pushes your boundaries + +--- + +## Just Do It + + + Practice + + Start with any idea + + Make a website you're proud to show friends + + Improve it + +??? + +## Doing is best for learning + + + Employers look for engagement in these areas + + Almost any area you want to focus in, your website can be your medium + +--- + +# *Thank You!* diff --git a/slides/Campus Recruiting Deck_2012_UC Berkeley.ppt b/slides/Campus Recruiting Deck_2012_UC Berkeley.ppt deleted file mode 100644 index 9573638..0000000 Binary files a/slides/Campus Recruiting Deck_2012_UC Berkeley.ppt and /dev/null differ diff --git a/slides/RM Pricing Strategy.ppt b/slides/RM Pricing Strategy.ppt deleted file mode 100644 index ca9ba50..0000000 Binary files a/slides/RM Pricing Strategy.ppt and /dev/null differ diff --git a/slides/img/GraphNodesEdges.gif b/slides/img/GraphNodesEdges.gif deleted file mode 100644 index bec0d3a..0000000 Binary files a/slides/img/GraphNodesEdges.gif and /dev/null differ diff --git a/slides/img/GraphNodesEdges.png b/slides/img/GraphNodesEdges.png new file mode 100644 index 0000000..92ef44d Binary files /dev/null and b/slides/img/GraphNodesEdges.png differ diff --git a/slides/img/cancer-county.jpg b/slides/img/cancer-county.jpg deleted file mode 100644 index e7a3ce3..0000000 Binary files a/slides/img/cancer-county.jpg and /dev/null differ diff --git a/slides/img/chameleon.png b/slides/img/chameleon.png index f0c40ad..a0f5f5e 100644 Binary files a/slides/img/chameleon.png and b/slides/img/chameleon.png differ diff --git a/slides/img/countymapredbluelarge.png b/slides/img/countymapredbluelarge.png new file mode 100644 index 0000000..c7bcd36 Binary files /dev/null and b/slides/img/countymapredbluelarge.png differ diff --git a/slides/img/dendrogram.png b/slides/img/dendrogram.png new file mode 100644 index 0000000..6fb5952 Binary files /dev/null and b/slides/img/dendrogram.png differ diff --git a/slides/img/dendrogram1.jpg b/slides/img/dendrogram1.jpg deleted file mode 100644 index ff94d9d..0000000 Binary files a/slides/img/dendrogram1.jpg and /dev/null differ diff --git a/slides/img/election1000.png b/slides/img/election1000.png new file mode 100644 index 0000000..f80409f Binary files /dev/null and b/slides/img/election1000.png differ diff --git a/slides/img/firefly.png b/slides/img/firefly.png new file mode 100644 index 0000000..bd7864c Binary files /dev/null and b/slides/img/firefly.png differ diff --git a/slides/img/graperr.png b/slides/img/graperr.png index 60ca7b8..9377e0e 100644 Binary files a/slides/img/graperr.png and b/slides/img/graperr.png differ diff --git a/slides/img/housing-regression.gif b/slides/img/housing-regression.gif deleted file mode 100644 index 1d89e09..0000000 Binary files a/slides/img/housing-regression.gif and /dev/null differ diff --git a/slides/img/housing-regression.jpg b/slides/img/housing-regression.jpg new file mode 100644 index 0000000..c508ec8 Binary files /dev/null and b/slides/img/housing-regression.jpg differ diff --git a/slides/img/jblomo-linkedin.gif b/slides/img/jblomo-linkedin.gif deleted file mode 100644 index 88c1f7d..0000000 Binary files a/slides/img/jblomo-linkedin.gif and /dev/null differ diff --git a/slides/img/jretz-linkedin.png b/slides/img/jretz-linkedin.png new file mode 100644 index 0000000..3669e2b Binary files /dev/null and b/slides/img/jretz-linkedin.png differ diff --git a/slides/img/mario.jpg b/slides/img/mario.jpg new file mode 100644 index 0000000..661e124 Binary files /dev/null and b/slides/img/mario.jpg differ diff --git a/slides/img/yelp-growth.png b/slides/img/yelp-growth.png index 85ee5a2..ba1d6d4 100644 Binary files a/slides/img/yelp-growth.png and b/slides/img/yelp-growth.png differ diff --git a/slides/production/remark-0.5.9.min.js b/slides/production/remark-0.5.9.min.js new file mode 100644 index 0000000..7e5f7db --- /dev/null +++ b/slides/production/remark-0.5.9.min.js @@ -0,0 +1,5 @@ +!function(e,t,n){function i(n,s){if(!t[n]){if(!e[n]){var o=typeof require=="function"&&require;if(!s&&o)return o(n,!0);if(r)return r(n,!0);throw new Error("Cannot find module '"+n+"'")}var u=t[n]={exports:{}};e[n][0].call(u.exports,function(t){var r=e[n][1][t];return i(r?r:t)},u,u.exports)}return t[n].exports}var r=typeof require=="function"&&require;for(var s=0;s/gm,">")}function findCode(pre){for(var node=pre.firstChild;node;node=node.nextSibling){if(node.nodeName=="CODE")return node;if(!(node.nodeType==3&&node.nodeValue.match(/\s+/)))break}}function blockText(block,ignoreNewLines){return Array.prototype.map.call(block.childNodes,function(node){if(node.nodeType==3){return ignoreNewLines?node.nodeValue.replace(/\n/g,""):node.nodeValue}if(node.nodeName=="BR"){return"\n"}return blockText(node,ignoreNewLines)}).join("")}function blockLanguage(block){var classes=(block.className+" "+(block.parentNode?block.parentNode.className:"")).split(/\s+/);classes=classes.map(function(c){return c.replace(/^language-/,"")});for(var i=0;i"}while(stream1.length||stream2.length){var current=selectStream().splice(0,1)[0];result+=escape(value.substr(processed,current.offset-processed));processed=current.offset;if(current.event=="start"){result+=open(current.node);nodeStack.push(current.node)}else if(current.event=="stop"){var node,i=nodeStack.length;do{i--;node=nodeStack[i];result+=""}while(node!=current.node);nodeStack.splice(i,1);while(i'+match[0]+""}else{result+=match[0]}last_index=top.lexemsRe.lastIndex;match=top.lexemsRe.exec(buffer)}return result+buffer.substr(last_index)}function processSubLanguage(){if(top.subLanguage&&!languages[top.subLanguage]){return escape(mode_buffer)}var result=top.subLanguage?highlight(top.subLanguage,mode_buffer):highlightAuto(mode_buffer);if(top.relevance>0){keyword_count+=result.keyword_count;relevance+=result.relevance}return''+result.value+""}function processBuffer(){return top.subLanguage!==undefined?processSubLanguage():processKeywords()}function startNewMode(mode,lexem){var markup=mode.className?'':"";if(mode.returnBegin){result+=markup;mode_buffer=""}else if(mode.excludeBegin){result+=escape(lexem)+markup;mode_buffer=""}else{result+=markup;mode_buffer=lexem}top=Object.create(mode,{parent:{value:top}})}function processLexem(buffer,lexem){mode_buffer+=buffer;if(lexem===undefined){result+=processBuffer();return 0}var new_mode=subMode(lexem,top);if(new_mode){result+=processBuffer();startNewMode(new_mode,lexem);return new_mode.returnBegin?0:lexem.length}var end_mode=endOfMode(top,lexem);if(end_mode){var origin=top;if(!(origin.returnEnd||origin.excludeEnd)){mode_buffer+=lexem}result+=processBuffer();do{if(top.className){result+=""}relevance+=top.relevance;top=top.parent}while(top!=end_mode.parent);if(origin.excludeEnd){result+=escape(lexem)}mode_buffer="";if(end_mode.starts){startNewMode(end_mode.starts,"")}return origin.returnEnd?0:lexem.length}if(isIllegal(lexem,top))throw new Error('Illegal lexem "'+lexem+'" for mode "'+(top.className||"")+'"');mode_buffer+=lexem;return lexem.length||1}var language=languages[language_name];compileLanguage(language);var top=language;var mode_buffer="";var relevance=0;var keyword_count=0;var result="";try{var match,count,index=0;while(true){top.terminators.lastIndex=index;match=top.terminators.exec(value);if(!match)break;count=processLexem(value.substr(index,match.index-index),match[0]);index=match.index+count}processLexem(value.substr(index));return{relevance:relevance,keyword_count:keyword_count,value:result,language:language_name}}catch(e){if(e.message.indexOf("Illegal")!=-1){return{relevance:0,keyword_count:0,value:escape(value)}}else{throw e}}}function highlightAuto(text){var result={keyword_count:0,relevance:0,value:escape(text)};var second_best=result;for(var key in languages){if(!languages.hasOwnProperty(key))continue;var current=highlight(key,text,false);current.language=key;if(current.keyword_count+current.relevance>second_best.keyword_count+second_best.relevance){second_best=current}if(current.keyword_count+current.relevance>result.keyword_count+result.relevance){second_best=result;result=current}}if(second_best.language){result.second_best=second_best}return result}function fixMarkup(value,tabReplace,useBR){if(tabReplace){value=value.replace(/^((<[^>]+>|\t)+)/gm,function(match,p1,offset,s){return p1.replace(/\t/g,tabReplace)})}if(useBR){value=value.replace(/\n/g,"
")}return value}function highlightBlock(block,tabReplace,useBR){var text=blockText(block,useBR);var language=blockLanguage(block);if(language=="no-highlight")return;var result=language?highlight(language,text,true):highlightAuto(text);language=result.language;var original=nodeStream(block);if(original.length){var pre=document.createElement("pre");pre.innerHTML=result.value;result.value=mergeStreams(original,nodeStream(pre),text)}result.value=fixMarkup(result.value,tabReplace,useBR);var class_name=block.className;if(!class_name.match("(\\s|^)(language-)?"+language+"(\\s|$)")){class_name=class_name?class_name+" "+language:language}block.innerHTML=result.value;block.className=class_name;block.result={language:language,kw:result.keyword_count,re:result.relevance};if(result.second_best){block.second_best={language:result.second_best.language,kw:result.second_best.keyword_count,re:result.second_best.relevance}}}function initHighlighting(){if(initHighlighting.called)return;initHighlighting.called=true;Array.prototype.map.call(document.getElementsByTagName("pre"),findCode).filter(Boolean).forEach(function(code){highlightBlock(code,hljs.tabReplace)})}function initHighlightingOnLoad(){window.addEventListener("DOMContentLoaded",initHighlighting,false);window.addEventListener("load",initHighlighting,false)}var languages={};this.LANGUAGES=languages;this.highlight=highlight;this.highlightAuto=highlightAuto;this.fixMarkup=fixMarkup;this.highlightBlock=highlightBlock;this.initHighlighting=initHighlighting;this.initHighlightingOnLoad=initHighlightingOnLoad;this.IDENT_RE="[a-zA-Z][a-zA-Z0-9_]*";this.UNDERSCORE_IDENT_RE="[a-zA-Z_][a-zA-Z0-9_]*";this.NUMBER_RE="\\b\\d+(\\.\\d+)?";this.C_NUMBER_RE="(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)";this.BINARY_NUMBER_RE="\\b(0b[01]+)";this.RE_STARTERS_RE="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|\\.|-|-=|/|/=|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~";this.BACKSLASH_ESCAPE={begin:"\\\\[\\s\\S]",relevance:0};this.APOS_STRING_MODE={className:"string",begin:"'",end:"'",illegal:"\\n",contains:[this.BACKSLASH_ESCAPE],relevance:0};this.QUOTE_STRING_MODE={className:"string",begin:'"',end:'"',illegal:"\\n",contains:[this.BACKSLASH_ESCAPE],relevance:0};this.C_LINE_COMMENT_MODE={className:"comment",begin:"//",end:"$"};this.C_BLOCK_COMMENT_MODE={className:"comment",begin:"/\\*",end:"\\*/"};this.HASH_COMMENT_MODE={className:"comment",begin:"#",end:"$"};this.NUMBER_MODE={className:"number",begin:this.NUMBER_RE,relevance:0};this.C_NUMBER_MODE={className:"number",begin:this.C_NUMBER_RE,relevance:0};this.BINARY_NUMBER_MODE={className:"number",begin:this.BINARY_NUMBER_RE,relevance:0};this.REGEXP_MODE={className:"regexp",begin:/\//,end:/\/[gim]*/,illegal:/\n/,contains:[this.BACKSLASH_ESCAPE,{begin:/\[/,end:/\]/,relevance:0,contains:[this.BACKSLASH_ESCAPE]}]};this.inherit=function(parent,obj){var result={};for(var key in parent)result[key]=parent[key];if(obj)for(var key in obj)result[key]=obj[key];return result}},languages=[{name:"javascript",create:function(hljs){return{keywords:{keyword:"in if for while finally var new function do return void else break catch "+"instanceof with throw case default try this switch continue typeof delete "+"let yield const",literal:"true false null undefined NaN Infinity"},contains:[hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE,hljs.C_NUMBER_MODE,{begin:"("+hljs.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",keywords:"return throw case",contains:[hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE,hljs.REGEXP_MODE,{begin:/;/,subLanguage:"xml"}],relevance:0},{className:"function",beginWithKeyword:true,end:/{/,keywords:"function",contains:[{className:"title",begin:/[A-Za-z$_][0-9A-Za-z$_]*/},{className:"params",begin:/\(/,end:/\)/,contains:[hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE],illegal:/["'\(]/}],illegal:/\[|%/}]}}},{name:"ruby",create:function(hljs){var RUBY_IDENT_RE="[a-zA-Z_][a-zA-Z0-9_]*(\\!|\\?)?";var RUBY_METHOD_RE="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?";var RUBY_KEYWORDS={keyword:"and false then defined module in return redo if BEGIN retry end for true self when "+"next until do begin unless END rescue nil else break undef not super class case "+"require yield alias while ensure elsif or include"};var YARDOCTAG={className:"yardoctag",begin:"@[A-Za-z]+"};var COMMENTS=[{className:"comment",begin:"#",end:"$",contains:[YARDOCTAG]},{className:"comment",begin:"^\\=begin",end:"^\\=end",contains:[YARDOCTAG],relevance:10},{className:"comment",begin:"^__END__",end:"\\n$"}];var SUBST={className:"subst",begin:"#\\{",end:"}",lexems:RUBY_IDENT_RE,keywords:RUBY_KEYWORDS};var STR_CONTAINS=[hljs.BACKSLASH_ESCAPE,SUBST];var STRINGS=[{className:"string",begin:"'",end:"'",contains:STR_CONTAINS,relevance:0},{className:"string",begin:'"',end:'"',contains:STR_CONTAINS,relevance:0},{className:"string",begin:"%[qw]?\\(",end:"\\)",contains:STR_CONTAINS},{className:"string",begin:"%[qw]?\\[",end:"\\]",contains:STR_CONTAINS},{className:"string",begin:"%[qw]?{",end:"}",contains:STR_CONTAINS},{className:"string",begin:"%[qw]?<",end:">",contains:STR_CONTAINS,relevance:10},{className:"string",begin:"%[qw]?/",end:"/",contains:STR_CONTAINS,relevance:10},{className:"string",begin:"%[qw]?%",end:"%",contains:STR_CONTAINS,relevance:10},{className:"string",begin:"%[qw]?-",end:"-",contains:STR_CONTAINS,relevance:10},{className:"string",begin:"%[qw]?\\|",end:"\\|",contains:STR_CONTAINS,relevance:10}];var FUNCTION={className:"function",beginWithKeyword:true,end:" |$|;",keywords:"def",contains:[{className:"title",begin:RUBY_METHOD_RE,lexems:RUBY_IDENT_RE,keywords:RUBY_KEYWORDS},{className:"params",begin:"\\(",end:"\\)",lexems:RUBY_IDENT_RE,keywords:RUBY_KEYWORDS}].concat(COMMENTS)};var RUBY_DEFAULT_CONTAINS=COMMENTS.concat(STRINGS.concat([{className:"class",beginWithKeyword:true,end:"$|;",keywords:"class module",contains:[{className:"title",begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?",relevance:0},{className:"inheritance",begin:"<\\s*",contains:[{className:"parent",begin:"("+hljs.IDENT_RE+"::)?"+hljs.IDENT_RE}]}].concat(COMMENTS)},FUNCTION,{className:"constant",begin:"(::)?(\\b[A-Z]\\w*(::)?)+",relevance:0},{className:"symbol",begin:":",contains:STRINGS.concat([{begin:RUBY_METHOD_RE}]),relevance:0},{className:"symbol",begin:RUBY_IDENT_RE+":",relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{className:"number",begin:"\\?\\w"},{className:"variable",begin:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{begin:"("+hljs.RE_STARTERS_RE+")\\s*",contains:COMMENTS.concat([{className:"regexp",begin:"/",end:"/[a-z]*",illegal:"\\n",contains:[hljs.BACKSLASH_ESCAPE,SUBST]}]),relevance:0}]));SUBST.contains=RUBY_DEFAULT_CONTAINS;FUNCTION.contains[1].contains=RUBY_DEFAULT_CONTAINS;return{lexems:RUBY_IDENT_RE,keywords:RUBY_KEYWORDS,contains:RUBY_DEFAULT_CONTAINS}}},{name:"python",create:function(hljs){var PROMPT={className:"prompt",begin:/^(>>>|\.\.\.) /};var STRINGS=[{className:"string",begin:/(u|b)?r?'''/,end:/'''/,contains:[PROMPT],relevance:10},{className:"string",begin:/(u|b)?r?"""/,end:/"""/,contains:[PROMPT],relevance:10},{className:"string",begin:/(u|r|ur)'/,end:/'/,contains:[hljs.BACKSLASH_ESCAPE],relevance:10},{className:"string",begin:/(u|r|ur)"/,end:/"/,contains:[hljs.BACKSLASH_ESCAPE],relevance:10},{className:"string",begin:/(b|br)'/,end:/'/,contains:[hljs.BACKSLASH_ESCAPE]},{className:"string",begin:/(b|br)"/,end:/"/,contains:[hljs.BACKSLASH_ESCAPE]}].concat([hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE]);var TITLE={className:"title",begin:hljs.UNDERSCORE_IDENT_RE};var PARAMS={className:"params",begin:/\(/,end:/\)/,contains:["self",hljs.C_NUMBER_MODE,PROMPT].concat(STRINGS)};var FUNC_CLASS_PROTO={beginWithKeyword:true,end:/:/,illegal:/[${=;\n]/,contains:[TITLE,PARAMS],relevance:10};return{keywords:{keyword:"and elif is global as in if from raise for except finally print import pass return "+"exec else break not with class assert yield try while continue del or def lambda "+"nonlocal|10",built_in:"None True False Ellipsis NotImplemented"},illegal:/(<\/|->|\?)/,contains:STRINGS.concat([PROMPT,hljs.HASH_COMMENT_MODE,hljs.inherit(FUNC_CLASS_PROTO,{className:"function",keywords:"def"}),hljs.inherit(FUNC_CLASS_PROTO,{className:"class",keywords:"class"}),hljs.C_NUMBER_MODE,{className:"decorator",begin:/@/,end:/$/},{begin:/\b(print|exec)\(/}])}}},{name:"bash",create:function(hljs){var VAR1={className:"variable",begin:/\$[\w\d#@][\w\d_]*/};var VAR2={className:"variable",begin:/\$\{(.*?)\}/};var QUOTE_STRING={className:"string",begin:/"/,end:/"/,contains:[hljs.BACKSLASH_ESCAPE,VAR1,VAR2,{className:"variable",begin:/\$\(/,end:/\)/,contains:hljs.BACKSLASH_ESCAPE}],relevance:0};var APOS_STRING={className:"string",begin:/'/,end:/'/,relevance:0};return{lexems:/-?[a-z]+/,keywords:{keyword:"if then else elif fi for break continue while in do done exit return set "+"declare case esac export exec",literal:"true false",built_in:"printf echo read cd pwd pushd popd dirs let eval unset typeset readonly "+"getopts source shopt caller type hash bind help sudo",operator:"-ne -eq -lt -gt -f -d -e -s -l -a"},contains:[{className:"shebang",begin:/^#![^\n]+sh\s*$/,relevance:10},{className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:true,contains:[{className:"title",begin:/\w[\w\d_]*/}],relevance:0},hljs.HASH_COMMENT_MODE,hljs.NUMBER_MODE,QUOTE_STRING,APOS_STRING,VAR1,VAR2]}}},{name:"java",create:function(hljs){return{keywords:"false synchronized int abstract float private char boolean static null if const "+"for true while long throw strictfp finally protected import native final return void "+"enum else break transient new catch instanceof byte super volatile case assert short "+"package default double public try this switch continue throws",contains:[{className:"javadoc",begin:"/\\*\\*",end:"\\*/",contains:[{className:"javadoctag",begin:"(^|\\s)@[A-Za-z]+"}],relevance:10},hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE,hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,{className:"class",beginWithKeyword:true,end:"{",keywords:"class interface",excludeEnd:true,illegal:":",contains:[{beginWithKeyword:true,keywords:"extends implements",relevance:10},{className:"title",begin:hljs.UNDERSCORE_IDENT_RE}]},hljs.C_NUMBER_MODE,{className:"annotation",begin:"@[A-Za-z]+"}]}}},{name:"php",create:function(hljs){var VARIABLE={className:"variable",begin:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"};var STRINGS=[hljs.inherit(hljs.APOS_STRING_MODE,{illegal:null}),hljs.inherit(hljs.QUOTE_STRING_MODE,{illegal:null}),{className:"string",begin:'b"',end:'"',contains:[hljs.BACKSLASH_ESCAPE]},{className:"string",begin:"b'",end:"'",contains:[hljs.BACKSLASH_ESCAPE]}];var NUMBERS=[hljs.BINARY_NUMBER_MODE,hljs.C_NUMBER_MODE];var TITLE={className:"title",begin:hljs.UNDERSCORE_IDENT_RE};return{case_insensitive:true,keywords:"and include_once list abstract global private echo interface as static endswitch "+"array null if endwhile or const for endforeach self var while isset public "+"protected exit foreach throw elseif include __FILE__ empty require_once do xor "+"return implements parent clone use __CLASS__ __LINE__ else break print eval new "+"catch __METHOD__ case exception php_user_filter default die require __FUNCTION__ "+"enddeclare final try this switch continue endfor endif declare unset true false "+"namespace trait goto instanceof insteadof __DIR__ __NAMESPACE__ __halt_compiler",contains:[hljs.C_LINE_COMMENT_MODE,hljs.HASH_COMMENT_MODE,{className:"comment",begin:"/\\*",end:"\\*/",contains:[{className:"phpdoc",begin:"\\s@[A-Za-z]+"}]},{className:"comment",excludeBegin:true,begin:"__halt_compiler.+?;",endsWithParent:true},{className:"string",begin:"<<<['\"]?\\w+['\"]?$",end:"^\\w+;",contains:[hljs.BACKSLASH_ESCAPE]},{className:"preprocessor",begin:"<\\?php",relevance:10},{className:"preprocessor",begin:"\\?>"},VARIABLE,{className:"function",beginWithKeyword:true,end:"{",keywords:"function",illegal:"\\$|\\[|%",contains:[TITLE,{className:"params",begin:"\\(",end:"\\)",contains:["self",VARIABLE,hljs.C_BLOCK_COMMENT_MODE].concat(STRINGS).concat(NUMBERS)}]},{className:"class",beginWithKeyword:true,end:"{",keywords:"class",illegal:"[:\\(\\$]",contains:[{beginWithKeyword:true,endsWithParent:true,keywords:"extends",contains:[TITLE]},TITLE]},{begin:"=>"}].concat(STRINGS).concat(NUMBERS)}}},{name:"perl",create:function(hljs){var PERL_KEYWORDS="getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc "+"ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime "+"readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qq"+"fileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent "+"shutdown dump chomp connect getsockname die socketpair close flock exists index shmget"+"sub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr "+"unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 "+"getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline "+"endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand "+"mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink "+"getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr "+"untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link "+"getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller "+"lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and "+"sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 "+"chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach "+"tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedir"+"ioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe "+"atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when";var SUBST={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:PERL_KEYWORDS,relevance:10};var VAR1={className:"variable",begin:"\\$\\d"};var VAR2={className:"variable",begin:"[\\$\\%\\@\\*](\\^\\w\\b|#\\w+(\\:\\:\\w+)*|[^\\s\\w{]|{\\w+}|\\w+(\\:\\:\\w*)*)"};var STRING_CONTAINS=[hljs.BACKSLASH_ESCAPE,SUBST,VAR1,VAR2];var METHOD={begin:"->",contains:[{begin:hljs.IDENT_RE},{begin:"{",end:"}"}]};var COMMENT={className:"comment",begin:"^(__END__|__DATA__)",end:"\\n$",relevance:5};var PERL_DEFAULT_CONTAINS=[VAR1,VAR2,hljs.HASH_COMMENT_MODE,COMMENT,{className:"comment",begin:"^\\=\\w",end:"\\=cut",endsWithParent:true},METHOD,{className:"string",begin:"q[qwxr]?\\s*\\(",end:"\\)",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"q[qwxr]?\\s*\\[",end:"\\]",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"q[qwxr]?\\s*\\{",end:"\\}",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"q[qwxr]?\\s*\\|",end:"\\|",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"q[qwxr]?\\s*\\<",end:"\\>",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"qw\\s+q",end:"q",contains:STRING_CONTAINS,relevance:5},{className:"string",begin:"'",end:"'",contains:[hljs.BACKSLASH_ESCAPE],relevance:0},{className:"string",begin:'"',end:'"',contains:STRING_CONTAINS,relevance:0},{className:"string",begin:"`",end:"`",contains:[hljs.BACKSLASH_ESCAPE]},{className:"string",begin:"{\\w+}",relevance:0},{className:"string",begin:"-?\\w+\\s*\\=\\>",relevance:0},{className:"number",begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",relevance:0},{begin:"("+hljs.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",keywords:"split return print reverse grep",relevance:0,contains:[hljs.HASH_COMMENT_MODE,COMMENT,{className:"regexp",begin:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",relevance:10},{className:"regexp",begin:"(m|qr)?/",end:"/[a-z]*",contains:[hljs.BACKSLASH_ESCAPE],relevance:0}]},{className:"sub",beginWithKeyword:true,end:"(\\s*\\(.*?\\))?[;{]",keywords:"sub",relevance:5},{className:"operator",begin:"-\\w\\b",relevance:0}];SUBST.contains=PERL_DEFAULT_CONTAINS;METHOD.contains[1].contains=PERL_DEFAULT_CONTAINS;return{keywords:PERL_KEYWORDS,contains:PERL_DEFAULT_CONTAINS}}},{name:"cpp",create:function(hljs){var CPP_KEYWORDS={keyword:"false int float while private char catch export virtual operator sizeof "+"dynamic_cast|10 typedef const_cast|10 const struct for static_cast|10 union namespace "+"unsigned long throw volatile static protected bool template mutable if public friend "+"do return goto auto void enum else break new extern using true class asm case typeid "+"short reinterpret_cast|10 default double register explicit signed typename try this "+"switch continue wchar_t inline delete alignof char16_t char32_t constexpr decltype "+"noexcept nullptr static_assert thread_local restrict _Bool complex",built_in:"std string cin cout cerr clog stringstream istringstream ostringstream "+"auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set "+"unordered_map unordered_multiset unordered_multimap array shared_ptr"};return{keywords:CPP_KEYWORDS,illegal:"",illegal:"\\n"},hljs.C_LINE_COMMENT_MODE]},{className:"stl_container",begin:"\\b(deque|list|queue|stack|vector|map|set|bitset|multiset|multimap|unordered_map|unordered_set|unordered_multiset|unordered_multimap|array)\\s*<",end:">",keywords:CPP_KEYWORDS,relevance:10,contains:["self"]}]}}},{name:"objectivec",create:function(hljs){var OBJC_KEYWORDS={keyword:"int float while private char catch export sizeof typedef const struct for union "+"unsigned long volatile static protected bool mutable if public do return goto void "+"enum else break extern asm case short default double throw register explicit "+"signed typename try this switch continue wchar_t inline readonly assign property "+"self synchronized end synthesize id optional required "+"nonatomic super unichar finally dynamic IBOutlet IBAction selector strong "+"weak readonly",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"NSString NSDictionary CGRect CGPoint UIButton UILabel UITextView UIWebView MKMapView "+"UISegmentedControl NSObject UITableViewDelegate UITableViewDataSource NSThread "+"UIActivityIndicator UITabbar UIToolBar UIBarButtonItem UIImageView NSAutoreleasePool "+"UITableView BOOL NSInteger CGFloat NSException NSLog NSMutableString NSMutableArray "+"NSMutableDictionary NSURL NSIndexPath CGSize UITableViewCell UIView UIViewController "+"UINavigationBar UINavigationController UITabBarController UIPopoverController "+"UIPopoverControllerDelegate UIImage NSNumber UISearchBar NSFetchedResultsController "+"NSFetchedResultsChangeType UIScrollView UIScrollViewDelegate UIEdgeInsets UIColor "+"UIFont UIApplication NSNotFound NSNotificationCenter NSNotification "+"UILocalNotification NSBundle NSFileManager NSTimeInterval NSDate NSCalendar "+"NSUserDefaults UIWindow NSRange NSArray NSError NSURLRequest NSURLConnection "+"UIInterfaceOrientation MPMoviePlayerController dispatch_once_t "+"dispatch_queue_t dispatch_sync dispatch_async dispatch_once"};return{keywords:OBJC_KEYWORDS,illegal:""}]},{className:"preprocessor",begin:"#",end:"$"},{className:"class",beginWithKeyword:true,end:"({|$)",keywords:"interface class protocol implementation",contains:[{className:"id",begin:hljs.UNDERSCORE_IDENT_RE}]},{className:"variable",begin:"\\."+hljs.UNDERSCORE_IDENT_RE,relevance:0}]}}},{name:"cs",create:function(hljs){return{keywords:"abstract as base bool break byte case catch char checked class const continue decimal "+"default delegate do double else enum event explicit extern false finally fixed float "+"for foreach goto if implicit in int interface internal is lock long namespace new null "+"object operator out override params private protected public readonly ref return sbyte "+"sealed short sizeof stackalloc static string struct switch this throw true try typeof "+"uint ulong unchecked unsafe ushort using virtual volatile void while async await "+"ascending descending from get group into join let orderby partial select set value var "+"where yield",contains:[{className:"comment",begin:"///",end:"$",returnBegin:true,contains:[{className:"xmlDocTag",begin:"///|"},{className:"xmlDocTag",begin:""}]},hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE,{className:"preprocessor",begin:"#",end:"$",keywords:"if else elif endif define undef warning error line region endregion pragma checksum"},{className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]},hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,hljs.C_NUMBER_MODE]}}},{name:"sql",create:function(hljs){return{case_insensitive:true,contains:[{className:"operator",begin:"(begin|end|start|commit|rollback|savepoint|lock|alter|create|drop|rename|call|delete|do|handler|insert|load|replace|select|truncate|update|set|show|pragma|grant)\\b(?!:)",end:";",endsWithParent:true,keywords:{keyword:"all partial global month current_timestamp using go revoke smallint "+"indicator end-exec disconnect zone with character assertion to add current_user "+"usage input local alter match collate real then rollback get read timestamp "+"session_user not integer bit unique day minute desc insert execute like ilike|2 "+"level decimal drop continue isolation found where constraints domain right "+"national some module transaction relative second connect escape close system_user "+"for deferred section cast current sqlstate allocate intersect deallocate numeric "+"public preserve full goto initially asc no key output collation group by union "+"session both last language constraint column of space foreign deferrable prior "+"connection unknown action commit view or first into float year primary cascaded "+"except restrict set references names table outer open select size are rows from "+"prepare distinct leading create only next inner authorization schema "+"corresponding option declare precision immediate else timezone_minute external "+"varying translation true case exception join hour default double scroll value "+"cursor descriptor values dec fetch procedure delete and false int is describe "+"char as at in varchar null trailing any absolute current_time end grant "+"privileges when cross check write current_date pad begin temporary exec time "+"update catalog user sql date on identity timezone_hour natural whenever interval "+"work order cascade diagnostics nchar having left call do handler load replace "+"truncate start lock show pragma exists number trigger if before after each row",aggregate:"count sum min max avg"},contains:[{className:"string",begin:"'",end:"'",contains:[hljs.BACKSLASH_ESCAPE,{begin:"''"}],relevance:0},{className:"string",begin:'"',end:'"',contains:[hljs.BACKSLASH_ESCAPE,{begin:'""'}],relevance:0},{className:"string",begin:"`",end:"`",contains:[hljs.BACKSLASH_ESCAPE]},hljs.C_NUMBER_MODE]},hljs.C_BLOCK_COMMENT_MODE,{className:"comment",begin:"--",end:"$"}]} +}},{name:"xml",create:function(hljs){var XML_IDENT_RE="[A-Za-z0-9\\._:-]+";var TAG_INTERNALS={endsWithParent:true,relevance:0,contains:[{className:"attribute",begin:XML_IDENT_RE,relevance:0},{begin:'="',returnBegin:true,end:'"',contains:[{className:"value",begin:'"',endsWithParent:true}]},{begin:"='",returnBegin:true,end:"'",contains:[{className:"value",begin:"'",endsWithParent:true}]},{begin:"=",contains:[{className:"value",begin:"[^\\s/>]+"}]}]};return{case_insensitive:true,contains:[{className:"pi",begin:"<\\?",end:"\\?>",relevance:10},{className:"doctype",begin:"",relevance:10,contains:[{begin:"\\[",end:"\\]"}]},{className:"comment",begin:"",relevance:10},{className:"cdata",begin:"<\\!\\[CDATA\\[",end:"\\]\\]>",relevance:10},{className:"tag",begin:"|$)",end:">",keywords:{title:"style"},contains:[TAG_INTERNALS],starts:{end:"",returnEnd:true,subLanguage:"css"}},{className:"tag",begin:"|$)",end:">",keywords:{title:"script"},contains:[TAG_INTERNALS],starts:{end:"",returnEnd:true,subLanguage:"javascript"}},{begin:"<%",end:"%>",subLanguage:"vbscript"},{className:"tag",begin:"",relevance:0,contains:[{className:"title",begin:"[^ /><]+"},TAG_INTERNALS]}]}}},{name:"css",create:function(hljs){var IDENT_RE="[a-zA-Z-][a-zA-Z0-9_-]*";var FUNCTION={className:"function",begin:IDENT_RE+"\\(",end:"\\)",contains:["self",hljs.NUMBER_MODE,hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE]};return{case_insensitive:true,illegal:"[=/|']",contains:[hljs.C_BLOCK_COMMENT_MODE,{className:"id",begin:"\\#[A-Za-z0-9_-]+"},{className:"class",begin:"\\.[A-Za-z0-9_-]+",relevance:0},{className:"attr_selector",begin:"\\[",end:"\\]",illegal:"$"},{className:"pseudo",begin:":(:)?[a-zA-Z0-9\\_\\-\\+\\(\\)\\\"\\']+"},{className:"at_rule",begin:"@(font-face|page)",lexems:"[a-z-]+",keywords:"font-face page"},{className:"at_rule",begin:"@",end:"[{;]",contains:[{className:"keyword",begin:/\S+/},{begin:/\s/,endsWithParent:true,excludeEnd:true,relevance:0,contains:[FUNCTION,hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,hljs.NUMBER_MODE]}]},{className:"tag",begin:IDENT_RE,relevance:0},{className:"rules",begin:"{",end:"}",illegal:"[^\\s]",relevance:0,contains:[hljs.C_BLOCK_COMMENT_MODE,{className:"rule",begin:"[^\\s]",returnBegin:true,end:";",endsWithParent:true,contains:[{className:"attribute",begin:"[A-Z\\_\\.\\-]+",end:":",excludeEnd:true,illegal:"[^\\s]",starts:{className:"value",endsWithParent:true,excludeEnd:true,contains:[FUNCTION,hljs.NUMBER_MODE,hljs.QUOTE_STRING_MODE,hljs.APOS_STRING_MODE,hljs.C_BLOCK_COMMENT_MODE,{className:"hexcolor",begin:"#[0-9A-Fa-f]+"},{className:"important",begin:"!important"}]}}]}]}]}}},{name:"scala",create:function(hljs){var ANNOTATION={className:"annotation",begin:"@[A-Za-z]+"};var STRING={className:"string",begin:'u?r?"""',end:'"""',relevance:10};return{keywords:"type yield lazy override def with val var false true sealed abstract private trait "+"object null if for while throw finally protected extends import final return else "+"break new catch super class case package default try this match continue throws",contains:[{className:"javadoc",begin:"/\\*\\*",end:"\\*/",contains:[{className:"javadoctag",begin:"@[A-Za-z]+"}],relevance:10},hljs.C_LINE_COMMENT_MODE,hljs.C_BLOCK_COMMENT_MODE,STRING,hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,{className:"class",begin:"((case )?class |object |trait )",end:"({|$)",illegal:":",keywords:"case class trait object",contains:[{beginWithKeyword:true,keywords:"extends with",relevance:10},{className:"title",begin:hljs.UNDERSCORE_IDENT_RE},{className:"params",begin:"\\(",end:"\\)",contains:[hljs.APOS_STRING_MODE,hljs.QUOTE_STRING_MODE,STRING,ANNOTATION]}]},hljs.C_NUMBER_MODE,ANNOTATION]}}},{name:"coffeescript",create:function(hljs){var KEYWORDS={keyword:"in if for while finally new do return else break catch instanceof throw try this "+"switch continue typeof delete debugger super "+"then unless until loop of by when and or is isnt not",literal:"true false null undefined "+"yes no on off",reserved:"case default function var void with const let enum export import native "+"__hasProp __extends __slice __bind __indexOf",built_in:"npm require console print module exports global window document"};var JS_IDENT_RE="[A-Za-z$_][0-9A-Za-z$_]*";var TITLE={className:"title",begin:JS_IDENT_RE};var SUBST={className:"subst",begin:"#\\{",end:"}",keywords:KEYWORDS};var EXPRESSIONS=[hljs.BINARY_NUMBER_MODE,hljs.inherit(hljs.C_NUMBER_MODE,{starts:{end:"(\\s*/)?",relevance:0}}),{className:"string",begin:"'''",end:"'''",contains:[hljs.BACKSLASH_ESCAPE]},{className:"string",begin:"'",end:"'",contains:[hljs.BACKSLASH_ESCAPE],relevance:0},{className:"string",begin:'"""',end:'"""',contains:[hljs.BACKSLASH_ESCAPE,SUBST]},{className:"string",begin:'"',end:'"',contains:[hljs.BACKSLASH_ESCAPE,SUBST],relevance:0},{className:"regexp",begin:"///",end:"///",contains:[hljs.HASH_COMMENT_MODE]},{className:"regexp",begin:"//[gim]*",relevance:0},{className:"regexp",begin:"/\\S(\\\\.|[^\\n])*?/[gim]*(?=\\s|\\W|$)"},{className:"property",begin:"@"+JS_IDENT_RE},{begin:"`",end:"`",excludeBegin:true,excludeEnd:true,subLanguage:"javascript"}];SUBST.contains=EXPRESSIONS;return{keywords:KEYWORDS,contains:EXPRESSIONS.concat([{className:"comment",begin:"###",end:"###"},hljs.HASH_COMMENT_MODE,{className:"function",begin:"("+JS_IDENT_RE+"\\s*=\\s*)?(\\(.*\\))?\\s*[-=]>",end:"[-=]>",returnBegin:true,contains:[TITLE,{className:"params",begin:"\\(",returnBegin:true,contains:[{begin:/\(/,end:/\)/,keywords:KEYWORDS,contains:["self"].concat(EXPRESSIONS)}]}]},{className:"class",beginWithKeyword:true,keywords:"class",end:"$",illegal:"[:\\[\\]]",contains:[{beginWithKeyword:true,keywords:"extends",endsWithParent:true,illegal:":",contains:[TITLE]},TITLE]},{className:"attribute",begin:JS_IDENT_RE+":",end:":",returnBegin:true,excludeEnd:true}])}}},{name:"lisp",create:function(hljs){var LISP_IDENT_RE="[a-zA-Z_\\-\\+\\*\\/\\<\\=\\>\\&\\#][a-zA-Z0-9_\\-\\+\\*\\/\\<\\=\\>\\&\\#!]*";var LISP_SIMPLE_NUMBER_RE="(\\-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s)(\\+|\\-)?\\d+)?";var SHEBANG={className:"shebang",begin:"^#!",end:"$"};var LITERAL={className:"literal",begin:"\\b(t{1}|nil)\\b"};var NUMBERS=[{className:"number",begin:LISP_SIMPLE_NUMBER_RE},{className:"number",begin:"#b[0-1]+(/[0-1]+)?"},{className:"number",begin:"#o[0-7]+(/[0-7]+)?"},{className:"number",begin:"#x[0-9a-f]+(/[0-9a-f]+)?"},{className:"number",begin:"#c\\("+LISP_SIMPLE_NUMBER_RE+" +"+LISP_SIMPLE_NUMBER_RE,end:"\\)"}];var STRING={className:"string",begin:'"',end:'"',contains:[hljs.BACKSLASH_ESCAPE],relevance:0};var COMMENT={className:"comment",begin:";",end:"$"};var VARIABLE={className:"variable",begin:"\\*",end:"\\*"};var KEYWORD={className:"keyword",begin:"[:&]"+LISP_IDENT_RE};var QUOTED_LIST={begin:"\\(",end:"\\)",contains:["self",LITERAL,STRING].concat(NUMBERS)};var QUOTED1={className:"quoted",begin:"['`]\\(",end:"\\)",contains:NUMBERS.concat([STRING,VARIABLE,KEYWORD,QUOTED_LIST])};var QUOTED2={className:"quoted",begin:"\\(quote ",end:"\\)",keywords:{title:"quote"},contains:NUMBERS.concat([STRING,VARIABLE,KEYWORD,QUOTED_LIST])};var LIST={className:"list",begin:"\\(",end:"\\)"};var BODY={endsWithParent:true,relevance:0};LIST.contains=[{className:"title",begin:LISP_IDENT_RE},BODY];BODY.contains=[QUOTED1,QUOTED2,LIST,LITERAL].concat(NUMBERS).concat([STRING,COMMENT,VARIABLE,KEYWORD]);return{illegal:"[^\\s]",contains:NUMBERS.concat([SHEBANG,LITERAL,STRING,COMMENT,QUOTED1,QUOTED2,LIST])}}},{name:"clojure",create:function(hljs){var keywords={built_in:"def cond apply if-not if-let if not not= = < < > <= <= >= == + / * - rem "+"quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? "+"set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? "+"class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? "+"string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . "+"inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last "+"drop-while while intern condp case reduced cycle split-at split-with repeat replicate "+"iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext "+"nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends "+"add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler "+"set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter "+"monitor-exit defmacro defn defn- macroexpand macroexpand-1 for doseq dosync dotimes and or "+"when when-not when-let comp juxt partial sequence memoize constantly complement identity assert "+"peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast "+"sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import "+"intern refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! "+"assoc! dissoc! pop! disj! import use class type num float double short byte boolean bigint biginteger "+"bigdec print-method print-dup throw-if throw printf format load compile get-in update-in pr pr-on newline "+"flush read slurp read-line subvec with-open memfn time ns assert re-find re-groups rand-int rand mod locking "+"assert-valid-fdecl alias namespace resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! "+"reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! "+"new next conj set! memfn to-array future future-call into-array aset gen-class reduce merge map filter find empty "+"hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list "+"disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer "+"chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate "+"unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta "+"lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"};var CLJ_IDENT_RE="[a-zA-Z_0-9\\!\\.\\?\\-\\+\\*\\/\\<\\=\\>\\&\\#\\$';]+";var SIMPLE_NUMBER_RE="[\\s:\\(\\{]+\\d+(\\.\\d+)?";var NUMBER={className:"number",begin:SIMPLE_NUMBER_RE,relevance:0};var STRING={className:"string",begin:'"',end:'"',contains:[hljs.BACKSLASH_ESCAPE],relevance:0};var COMMENT={className:"comment",begin:";",end:"$",relevance:0};var COLLECTION={className:"collection",begin:"[\\[\\{]",end:"[\\]\\}]"};var HINT={className:"comment",begin:"\\^"+CLJ_IDENT_RE};var HINT_COL={className:"comment",begin:"\\^\\{",end:"\\}"};var KEY={className:"attribute",begin:"[:]"+CLJ_IDENT_RE};var LIST={className:"list",begin:"\\(",end:"\\)"};var BODY={endsWithParent:true,keywords:{literal:"true false nil"},relevance:0};var TITLE={keywords:keywords,lexems:CLJ_IDENT_RE,className:"title",begin:CLJ_IDENT_RE,starts:BODY};LIST.contains=[{className:"comment",begin:"comment"},TITLE];BODY.contains=[LIST,STRING,HINT,HINT_COL,COMMENT,KEY,COLLECTION,NUMBER];COLLECTION.contains=[LIST,STRING,HINT,COMMENT,KEY,COLLECTION,NUMBER];return{illegal:"\\S",contains:[COMMENT,LIST]}}},{name:"http",create:function(hljs){return{illegal:"\\S",contains:[{className:"status",begin:"^HTTP/[0-9\\.]+",end:"$",contains:[{className:"number",begin:"\\b\\d{3}\\b"}]},{className:"request",begin:"^[A-Z]+ (.*?) HTTP/[0-9\\.]+$",returnBegin:true,end:"$",contains:[{className:"string",begin:" ",end:" ",excludeBegin:true,excludeEnd:true}]},{className:"attribute",begin:"^\\w",end:": ",excludeEnd:true,illegal:"\\n|\\s|=",starts:{className:"string",end:"$"}},{begin:"\\n\\n",starts:{subLanguage:"",endsWithParent:true}}]}}}];for(var i=0;i.left{text-align:left;}.remark-slide>.center{text-align:center;}.remark-slide>.right{text-align:right;}.remark-slide>.top{vertical-align:top;}.remark-slide>.middle{vertical-align:middle;}.remark-slide>.bottom{vertical-align:bottom;}.remark-slide .remark-slide-content{background-position:center;background-repeat:no-repeat;display:table-cell;padding:1em 4em 1em 4em;}.remark-slide .remark-slide-content .left{display:block;text-align:left;}.remark-slide .remark-slide-content .center{display:block;text-align:center;}.remark-slide .remark-slide-content .right{display:block;text-align:right;}.remark-slide .remark-slide-number{bottom:12px;opacity:0.5;position:absolute;right:20px;}.remark-visible{display:block;}.remark-backdrop{position:absolute;top:0;bottom:0;left:0;right:0;display:none;opacity:0.95;background:#000;}.remark-help{bottom:0;top:0;right:0;left:0;display:none;position:absolute;z-index:1000;-webkit-transform-origin:top left;-moz-transform-origin:top left;transform-origin:top-left;}.remark-help .remark-help-content{color:white;font-family:Helvetica,arial,freesans,clean,sans-serif;font-size:12pt;position:absolute;top:10%;bottom:10%;left:10%;height:10%;}.remark-help .remark-help-content td{color:white;font-size:12pt;padding:10px;}.remark-help .remark-help-content td:first-child{padding-left:0;}.remark-help .remark-help-content .key{background:white;color:black;min-width:1em;display:inline-block;padding:3px 6px;text-align:center;border-radius:4px;}.remark-help .dismiss{top:85%;}.remark-container.remark-help-mode .remark-help{display:block;}.remark-container.remark-help-mode .remark-backdrop{display:block;}.remark-preview-area{bottom:2%;left:2%;display:none;opacity:0.5;position:absolute;height:47.25%;width:48%;}.remark-preview-area .remark-slide-container{display:block;}.remark-notes-area{background:#e7e8e2;bottom:0;display:none;left:52%;overflow:hidden;padding:1.5em;position:absolute;right:0;top:0;}.remark-toolbar{color:#979892;padding-bottom:1em;}.remark-toolbar .remark-toolbar-link{border:2px solid #d7d8d2;color:#979892;display:inline-block;padding:2px 2px;text-decoration:none;text-align:center;min-width:20px;}.remark-toolbar .remark-toolbar-link:hover{border-color:#979892;color:#676862;}.remark-container.remark-presenter-mode .remark-slides-area{top:2%;left:2%;height:47.25%;width:48%;}.remark-container.remark-presenter-mode .remark-preview-area{display:block;}.remark-container.remark-presenter-mode .remark-notes-area{display:block;}@media print{.remark-container{overflow:visible;} .remark-slide-container{display:block;}}@page {size:908px 681px;margin:0;}",containerLayout:'
\n
\n +\n -\n
\n
\n
\n
\n\n
\n
\n
\n
\n
\n
\n

Help

\n

Keyboard shortcuts

\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n ↑,\n ←,\n Pg Up,\n K\n Go to previous slide
\n ↓,\n →,\n Pg Dn,\n Space,\n J\n Go to next slide
\n Home\n Go to first slide
\n End\n Go to last slide
\n F\n Toggle fullscreen mode
\n C\n Clone slideshow
\n P\n Toggle presenter mode
\n ?\n Toggle this help
\n
\n
\n \n \n \n \n \n
\n Esc\n Back to slideshow
\n
\n
\n'}},{}],5:[function(require,module,exports){var process=module.exports={};process.nextTick=function(){var canSetImmediate=typeof window!=="undefined"&&window.setImmediate;var canPost=typeof window!=="undefined"&&window.postMessage&&window.addEventListener;if(canSetImmediate){return function(f){return window.setImmediate(f)}}if(canPost){var queue=[];window.addEventListener("message",function(ev){if(ev.source===window&&ev.data==="process-tick"){ev.stopPropagation();if(queue.length>0){var fn=queue.shift();fn()}}},true);return function nextTick(fn){queue.push(fn);window.postMessage("process-tick","*")}}return function nextTick(fn){setTimeout(fn,0)}}();process.title="browser";process.browser=true;process.env={};process.argv=[];process.binding=function(name){throw new Error("process.binding is not supported")};process.cwd=function(){return"/"};process.chdir=function(dir){throw new Error("process.chdir is not supported")}},{}],6:[function(require,module,exports){!function(process){if(!process.EventEmitter)process.EventEmitter=function(){};var EventEmitter=exports.EventEmitter=process.EventEmitter;var isArray=typeof Array.isArray==="function"?Array.isArray:function(xs){return Object.prototype.toString.call(xs)==="[object Array]"};function indexOf(xs,x){if(xs.indexOf)return xs.indexOf(x);for(var i=0;i0&&this._events[type].length>m){this._events[type].warned=true;console.error("(node) warning: possible EventEmitter memory "+"leak detected. %d listeners added. "+"Use emitter.setMaxListeners() to increase limit.",this._events[type].length);console.trace()}}this._events[type].push(listener)}else{this._events[type]=[this._events[type],listener]}return this};EventEmitter.prototype.on=EventEmitter.prototype.addListener;EventEmitter.prototype.once=function(type,listener){var self=this;self.on(type,function g(){self.removeListener(type,g);listener.apply(this,arguments)});return this};EventEmitter.prototype.removeListener=function(type,listener){if("function"!==typeof listener){throw new Error("removeListener only takes instances of Function")}if(!this._events||!this._events[type])return this;var list=this._events[type];if(isArray(list)){var i=indexOf(list,listener);if(i<0)return this;list.splice(i,1);if(list.length==0)delete this._events[type]}else if(this._events[type]===listener){delete this._events[type]}return this};EventEmitter.prototype.removeAllListeners=function(type){if(arguments.length===0){this._events={};return this}if(type&&this._events&&this._events[type])this._events[type]=null;return this};EventEmitter.prototype.listeners=function(type){if(!this._events)this._events={};if(!this._events[type])this._events[type]=[];if(!isArray(this._events[type])){this._events[type]=[this._events[type]]}return this._events[type]}}(require("__browserify_process"))},{__browserify_process:5}],2:[function(require,module,exports){var EventEmitter=require("events").EventEmitter,highlighter=require("./highlighter"),Slideshow=require("./models/slideshow"),SlideshowView=require("./views/slideshowView"),Controller=require("./controller");module.exports.highlighter=highlighter;module.exports.create=function(options){var events,slideshow,slideshowView,controller;options=applyDefaults(options);events=new EventEmitter;events.setMaxListeners(0);slideshow=new Slideshow(events,options);slideshowView=new SlideshowView(events,options.container,slideshow);controller=new Controller(events,slideshowView);return slideshow};function applyDefaults(options){var sourceElement;options=options||{};if(!options.hasOwnProperty("source")){sourceElement=document.getElementById("source");if(sourceElement){options.source=sourceElement.innerHTML;sourceElement.style.display="none"}}if(!(options.container instanceof window.HTMLElement)){options.container=document.body}return options}},{events:6,"./highlighter":3,"./models/slideshow":7,"./views/slideshowView":8,"./controller":9}],9:[function(require,module,exports){module.exports=Controller;function Controller(events,slideshowView){addApiEventListeners(events,slideshowView);addNavigationEventListeners(events,slideshowView);addKeyboardEventListeners(events);addMouseEventListeners(events);addTouchEventListeners(events)}function addApiEventListeners(events,slideshowView){events.on("pause",function(event){removeKeyboardEventListeners(events);removeMouseEventListeners(events);removeTouchEventListeners(events)});events.on("resume",function(event){addKeyboardEventListeners(events);addMouseEventListeners(events);addTouchEventListeners(events)})}function addNavigationEventListeners(events,slideshowView){if(slideshowView.isEmbedded()){events.emit("gotoSlide",1)}else{events.on("hashchange",navigateByHash);events.on("slideChanged",updateHash);navigateByHash()}events.on("message",navigateByMessage);function navigateByHash(){var slideNoOrName=(window.location.hash||"").substr(1);events.emit("gotoSlide",slideNoOrName)}function updateHash(slideNoOrName){window.location.hash="#"+slideNoOrName}function navigateByMessage(message){var cap;if((cap=/^gotoSlide:(\d+)$/.exec(message.data))!==null){events.emit("gotoSlide",parseInt(cap[1],10))}}}function removeKeyboardEventListeners(events){events.removeAllListeners("keydown");events.removeAllListeners("keypress")}function addKeyboardEventListeners(events){events.on("keydown",function(event){switch(event.keyCode){case 33:case 37:case 38:events.emit("gotoPreviousSlide");break;case 32:case 34:case 39:case 40:events.emit("gotoNextSlide");break;case 36:events.emit("gotoFirstSlide");break;case 35:events.emit("gotoLastSlide");break;case 27:events.emit("hideOverlay");break}});events.on("keypress",function(event){if(event.metaKey||event.ctrlKey){return}switch(String.fromCharCode(event.which)){case"j":events.emit("gotoNextSlide");break;case"k":events.emit("gotoPreviousSlide");break;case"c":events.emit("createClone");break;case"p":events.emit("togglePresenterMode");break;case"f":events.emit("toggleFullscreen");break;case"?":events.emit("toggleHelp");break}})}function removeMouseEventListeners(events){events.removeAllListeners("mousewheel")}function addMouseEventListeners(events){events.on("mousewheel",function(event){if(event.wheelDeltaY>0){events.emit("gotoPreviousSlide")}else if(event.wheelDeltaY<0){events.emit("gotoNextSlide")}})}function removeTouchEventListeners(events){events.removeAllListeners("touchstart");events.removeAllListeners("touchend");events.removeAllListeners("touchmove")}function addTouchEventListeners(events){var touch,startX,endX;var isTap=function(){return Math.abs(startX-endX)<10};var handleTap=function(){events.emit("tap",endX)};var handleSwipe=function(){if(startX>endX){events.emit("gotoNextSlide")}else{events.emit("gotoPreviousSlide")}};events.on("touchstart",function(event){touch=event.touches[0];startX=touch.clientX});events.on("touchend",function(event){if(event.target.nodeName.toUpperCase()==="A"){return}touch=event.changedTouches[0];endX=touch.clientX;if(isTap()){handleTap()}else{handleSwipe()}});events.on("touchmove",function(event){event.preventDefault()})}},{}],7:[function(require,module,exports){var Navigation=require("./slideshow/navigation"),Events=require("./slideshow/events"),utils=require("../utils"),Slide=require("./slide"),Parser=require("../parser");module.exports=Slideshow;function Slideshow(events,options){var self=this,slides=[];options=options||{};Events.call(self,events);Navigation.call(self,events);self.loadFromString=loadFromString;self.getSlides=getSlides;self.getSlideCount=getSlideCount;self.getSlideByName=getSlideByName;self.getRatio=getOrDefault("ratio","4:3");self.getHighlightStyle=getOrDefault("highlightStyle","default");self.getHighlightLanguage=getOrDefault("highlightLanguage","");loadFromString(options.source);function loadFromString(source){source=source||"";slides=createSlides(source);expandVariables(slides);events.emit("slidesChanged")}function getSlides(){return slides.map(function(slide){return slide})}function getSlideCount(){return slides.length}function getSlideByName(name){return slides.byName[name]}function getOrDefault(key,defaultValue){return function(){if(options[key]===undefined){return defaultValue}return options[key]}}}function createSlides(slideshowSource){var parser=new Parser,parsedSlides=parser.parse(slideshowSource),slides=[],byName={},layoutSlide;slides.byName={};parsedSlides.forEach(function(slide,i){var template,slideViewModel;if(slide.properties.continued==="true"&&i>0){template=slides[slides.length-1]}else if(byName[slide.properties.template]){template=byName[slide.properties.template]}else if(slide.properties.layout==="false"){layoutSlide=undefined}else if(layoutSlide&&slide.properties.layout!=="true"){template=layoutSlide}slideViewModel=new Slide(slides.length+1,slide,template);if(slide.properties.layout==="true"){layoutSlide=slideViewModel}if(slide.properties.name){byName[slide.properties.name]=slideViewModel}if(slide.properties.layout!=="true"){slides.push(slideViewModel);if(slide.properties.name){slides.byName[slide.properties.name]=slideViewModel}}});return slides}function expandVariables(slides){slides.forEach(function(slide){slide.expandVariables()})}},{"./slideshow/navigation":10,"./slideshow/events":11,"../utils":12,"./slide":13,"../parser":14}],8:[function(require,module,exports){var SlideView=require("./slideView"),Scaler=require("../scaler"),resources=require("../resources"),addClass=require("../utils").addClass,toggleClass=require("../utils").toggleClass,getPrefixedProperty=require("../utils").getPrefixedProperty;module.exports=SlideshowView;function SlideshowView(events,containerElement,slideshow){var self=this;self.events=events;self.slideshow=slideshow;self.scaler=new Scaler(events,slideshow);self.slideViews=[];self.configureContainerElement(containerElement);self.configureChildElements();self.updateDimensions();self.scaleElements();self.updateSlideViews();events.on("slidesChanged",function(){self.updateSlideViews()});events.on("hideSlide",function(slideIndex){self.hideSlide(slideIndex)});events.on("showSlide",function(slideIndex){self.showSlide(slideIndex)});events.on("togglePresenterMode",function(){toggleClass(self.containerElement,"remark-presenter-mode");self.scaleElements()});events.on("toggleHelp",function(){toggleClass(self.containerElement,"remark-help-mode")});handleFullscreen(self)}function handleFullscreen(self){var requestFullscreen=getPrefixedProperty(self.containerElement,"requestFullScreen"),cancelFullscreen=getPrefixedProperty(document,"cancelFullScreen");self.events.on("toggleFullscreen",function(){var fullscreenElement=getPrefixedProperty(document,"fullscreenElement")||getPrefixedProperty(document,"fullScreenElement");if(!fullscreenElement&&requestFullscreen){requestFullscreen.call(self.containerElement,Element.ALLOW_KEYBOARD_INPUT)}else if(cancelFullscreen){cancelFullscreen.call(document)}self.scaleElements()})}SlideshowView.prototype.isEmbedded=function(){return this.containerElement!==document.body};SlideshowView.prototype.configureContainerElement=function(element){var self=this;self.containerElement=element;addClass(element,"remark-container");if(element===document.body){addClass(document.getElementsByTagName("html")[0],"remark-container");forwardEvents(self.events,window,["hashchange","resize","keydown","keypress","mousewheel","message"]);forwardEvents(self.events,document,["touchstart","touchmove","touchend"])}else{element.style.position="absolute";element.tabIndex=-1;forwardEvents(self.events,window,["resize"]);forwardEvents(self.events,element,["keydown","keypress","mousewheel","touchstart","touchmove","touchend"])}self.events.on("tap",function(endX){if(endX0){self.showSlide(self.slideshow.getCurrentSlideNo()-1)}};SlideshowView.prototype.scaleSlideBackgroundImages=function(dimensions){var self=this;self.slideViews.forEach(function(slideView){slideView.scaleBackgroundImage(dimensions)})};SlideshowView.prototype.showSlide=function(slideIndex){var self=this,slideView=self.slideViews[slideIndex],nextSlideView=self.slideViews[slideIndex+1];self.events.emit("beforeShowSlide",slideIndex);slideView.show();self.notesElement.innerHTML=slideView.notesMarkup;if(nextSlideView){self.previewArea.innerHTML=nextSlideView.containerElement.outerHTML}else{self.previewArea.innerHTML=""}self.events.emit("afterShowSlide",slideIndex)};SlideshowView.prototype.hideSlide=function(slideIndex){var self=this,slideView=self.slideViews[slideIndex];self.events.emit("beforeHideSlide",slideIndex);slideView.hide();self.events.emit("afterHideSlide",slideIndex)};SlideshowView.prototype.updateDimensions=function(){var self=this,dimensions=self.scaler.dimensions;self.helpElement.style.width=dimensions.width+"px";self.helpElement.style.height=dimensions.height+"px";self.scaleSlideBackgroundImages(dimensions);self.scaleElements()};SlideshowView.prototype.scaleElements=function(){var self=this;self.slideViews.forEach(function(slideView){slideView.scale(self.elementArea)});if(self.previewArea.children.length){self.scaler.scaleToFit(self.previewArea.children[0].children[0],self.previewArea)}self.scaler.scaleToFit(self.helpElement,self.containerElement)}},{"./slideView":15,"../scaler":16,"../resources":4,"../utils":12}],10:[function(require,module,exports){module.exports=Navigation;function Navigation(events){var self=this,currentSlideNo=0;self.getCurrentSlideNo=getCurrentSlideNo;self.gotoSlide=gotoSlide;self.gotoPreviousSlide=gotoPreviousSlide;self.gotoNextSlide=gotoNextSlide;self.gotoFirstSlide=gotoFirstSlide;self.gotoLastSlide=gotoLastSlide;self.pause=pause;self.resume=resume;events.on("gotoSlide",gotoSlide);events.on("gotoPreviousSlide",gotoPreviousSlide);events.on("gotoNextSlide",gotoNextSlide);events.on("gotoFirstSlide",gotoFirstSlide);events.on("gotoLastSlide",gotoLastSlide);events.on("slidesChanged",function(){if(currentSlideNo>self.getSlideCount()){currentSlideNo=self.getSlideCount()}});events.on("createClone",function(){if(!self.clone||self.clone.closed){self.clone=window.open(location.href,"_blank","location=no")}else{self.clone.focus()}});function pause(){events.emit("pause")}function resume(){events.emit("resume")}function getCurrentSlideNo(){return currentSlideNo}function gotoSlide(slideNoOrName){var slideNo=getSlideNo(slideNoOrName),alreadyOnSlide=slideNo===currentSlideNo,slideOutOfRange=slideNo<1||slideNo>self.getSlideCount();if(alreadyOnSlide||slideOutOfRange){return}if(currentSlideNo!==0){events.emit("hideSlide",currentSlideNo-1)}events.emit("showSlide",slideNo-1);currentSlideNo=slideNo;events.emit("slideChanged",slideNoOrName||slideNo);if(self.clone&&!self.clone.closed){self.clone.postMessage("gotoSlide:"+currentSlideNo,"*")}if(window.opener){window.opener.postMessage("gotoSlide:"+currentSlideNo,"*")}}function gotoPreviousSlide(){self.gotoSlide(currentSlideNo-1)}function gotoNextSlide(){self.gotoSlide(currentSlideNo+1)}function gotoFirstSlide(){self.gotoSlide(1)}function gotoLastSlide(){self.gotoSlide(self.getSlideCount())}function getSlideNo(slideNoOrName){var slideNo,slide;if(typeof slideNoOrName==="number"){return slideNoOrName}slideNo=parseInt(slideNoOrName,10);if(slideNo.toString()===slideNoOrName){return slideNo}slide=self.getSlideByName(slideNoOrName);if(slide){return slide.getSlideNo()}return 1}}},{}],11:[function(require,module,exports){var EventEmitter=require("events").EventEmitter;module.exports=Events;function Events(events){var self=this,externalEvents=new EventEmitter;externalEvents.setMaxListeners(0);self.on=function(){externalEvents.on.apply(externalEvents,arguments);return self};["showSlide","hideSlide","beforeShowSlide","afterShowSlide","beforeHideSlide","afterHideSlide"].map(function(eventName){events.on(eventName,function(slideIndex){var slide=self.getSlides()[slideIndex];externalEvents.emit(eventName,slide)})})}},{events:6}],12:[function(require,module,exports){exports.addClass=function(element,className){element.className=exports.getClasses(element).concat([className]).join(" ")};exports.removeClass=function(element,className){element.className=exports.getClasses(element).filter(function(klass){return klass!==className}).join(" ")};exports.toggleClass=function(element,className){var classes=exports.getClasses(element),index=classes.indexOf(className);if(index!==-1){classes.splice(index,1)}else{classes.push(className)}element.className=classes.join(" ")};exports.getClasses=function(element){return element.className.split(" ").filter(function(s){return s!==""})};exports.getPrefixedProperty=function(element,propertyName){var capitalizedPropertName=propertyName[0].toUpperCase()+propertyName.slice(1);return element[propertyName]||element["moz"+capitalizedPropertName]||element["webkit"+capitalizedPropertName]};forEach([Array,window.NodeList,window.HTMLCollection],extend);function extend(object){var prototype=object&&object.prototype;if(!prototype){return}prototype.forEach=prototype.forEach||function(f){forEach(this,f)};prototype.filter=prototype.filter||function(f){var result=[];this.forEach(function(element){if(f(element,result.length)){result.push(element)}});return result};prototype.map=prototype.map||function(f){var result=[];this.forEach(function(element){result.push(f(element,result.length))});return result}}function forEach(list,f){var i;for(i=0;icontainerHeight/ratio.height){scale=containerHeight/dimensions.height}else{scale=containerWidth/dimensions.width}scaledWidth=dimensions.width*scale;scaledHeight=dimensions.height*scale;left=(containerWidth-scaledWidth)/2;top=(containerHeight-scaledHeight)/2;element.style["-webkit-transform"]="scale("+scale+")";element.style.MozTransform="scale("+scale+")";element.style.left=Math.max(left,0)+"px";element.style.top=Math.max(top,0)+"px"};function getRatio(slideshow){var ratioComponents=slideshow.getRatio().split(":"),ratio;ratio={width:parseInt(ratioComponents[0],10),height:parseInt(ratioComponents[1],10)};ratio.ratio=ratio.width/ratio.height;return ratio}function getDimensions(ratio){return{width:Math.floor(referenceWidth/referenceRatio*ratio.ratio),height:referenceHeight}}},{}],14:[function(require,module,exports){var Lexer=require("./lexer"),converter=require("./converter");module.exports=Parser;function Parser(){}Parser.prototype.parse=function(src){var lexer=new Lexer,tokens=lexer.lex(src),slides=[],slide=createSlide(),tag,classes;tokens.forEach(function(token){switch(token.type){case"text":case"code":case"fences":appendTo(slide,token.text);break;case"content_start":tag=token.block?"div":"span";classes=token.classes.join(" ");appendTo(slide,"<"+tag+' class="'+classes+'">');break;case"content_end":tag=token.block?"div":"span";appendTo(slide,"</"+tag+">");break;case"separator":slides.push(slide);slide=createSlide();slide.properties.continued=(token.text==="--").toString();break;case"notes_separator":slide.notes="";break}});slides.push(slide);slides.forEach(function(slide){slide.source=extractProperties(slide.source,slide.properties)});return slides};function createSlide(){return{source:"",properties:{continued:"false"}}}function appendTo(slide,content){if(slide.notes!==undefined){slide.notes+=content}else{slide.source+=content}}function extractProperties(source,properties){var propertyFinder=/^\n*([-\w]+):([^$\n]*)/i,match;while((match=propertyFinder.exec(source))!==null){source=source.substr(0,match.index)+source.substr(match.index+match[0].length);properties[match[1].trim()]=match[2].trim();propertyFinder.lastIndex=match.index}return source}},{"./lexer":17,"./converter":18}],15:[function(require,module,exports){var converter=require("../converter"),highlighter=require("../highlighter"),utils=require("../utils");module.exports=SlideView;function SlideView(events,slideshow,scaler,slide){var self=this;self.events=events;self.slideshow=slideshow;self.scaler=scaler;self.slide=slide;self.notesMarkup=createNotesMarkup(slideshow,slide.notes);self.configureElements();self.updateDimensions();self.events.on("propertiesChanged",function(changes){if(changes.hasOwnProperty("ratio")){self.updateDimensions() +}})}SlideView.prototype.updateDimensions=function(){var self=this,dimensions=self.scaler.dimensions;self.scalingElement.style.width=dimensions.width+"px";self.scalingElement.style.height=dimensions.height+"px"};SlideView.prototype.scale=function(containerElement){var self=this;self.scaler.scaleToFit(self.scalingElement,containerElement)};SlideView.prototype.show=function(){utils.addClass(this.containerElement,"remark-visible")};SlideView.prototype.hide=function(){utils.removeClass(this.containerElement,"remark-visible")};SlideView.prototype.configureElements=function(){var self=this;self.containerElement=document.createElement("div");self.containerElement.className="remark-slide-container";self.scalingElement=document.createElement("div");self.scalingElement.className="remark-slide-scaler";self.element=document.createElement("div");self.element.className="remark-slide";self.contentElement=createContentElement(self.events,self.slideshow,self.slide);self.numberElement=document.createElement("div");self.numberElement.className="remark-slide-number";self.numberElement.innerHTML=self.slide.number+" / "+self.slideshow.getSlides().length;self.contentElement.appendChild(self.numberElement);self.element.appendChild(self.contentElement);self.scalingElement.appendChild(self.element);self.containerElement.appendChild(self.scalingElement)};SlideView.prototype.scaleBackgroundImage=function(dimensions){var self=this,styles=window.getComputedStyle(this.contentElement),backgroundImage=styles.backgroundImage,match,image;if((match=/^url\(("?)([^\)]+?)\1\)/.exec(backgroundImage))!==null){image=new Image;image.onload=function(){if(image.width>dimensions.width||image.height>dimensions.height){if(!self.originalBackgroundSize){self.originalBackgroundSize=self.contentElement.style.backgroundSize;self.backgroundSizeSet=true;self.contentElement.style.backgroundSize="contain"}}else{if(self.backgroundSizeSet){self.contentElement.style.backgroundSize=self.originalBackgroundSize;self.backgroundSizeSet=false}}};image.src=match[2]}};function createContentElement(events,slideshow,slide){var element=document.createElement("div");if(slide.properties.name){element.id="slide-"+slide.properties.name}styleContentElement(slideshow,element,slide.properties);element.innerHTML=converter.convertMarkdown(slide.source);element.innerHTML=element.innerHTML.replace(/

\s*<\/p>/g,"");highlightCodeBlocks(element,slideshow);return element}function styleContentElement(slideshow,element,properties){element.className="";setClassFromProperties(element,properties);setHighlightStyleFromProperties(element,properties,slideshow);setBackgroundFromProperties(element,properties)}function createNotesMarkup(slideshow,notes){var element=document.createElement("div");element.innerHTML=converter.convertMarkdown(notes);element.innerHTML=element.innerHTML.replace(/

\s*<\/p>/g,"");highlightCodeBlocks(element,slideshow);return element.innerHTML}function setBackgroundFromProperties(element,properties){var backgroundImage=properties["background-image"];if(backgroundImage){element.style.backgroundImage=backgroundImage}}function setHighlightStyleFromProperties(element,properties,slideshow){var highlightStyle=properties["highlight-style"]||slideshow.getHighlightStyle();if(highlightStyle){utils.addClass(element,"hljs-"+highlightStyle)}}function setClassFromProperties(element,properties){utils.addClass(element,"remark-slide-content");(properties["class"]||"").split(/,| /).filter(function(s){return s!==""}).forEach(function(c){utils.addClass(element,c)})}function highlightCodeBlocks(content,slideshow){var codeBlocks=content.getElementsByTagName("code");codeBlocks.forEach(function(block){if(block.className===""){block.className=slideshow.getHighlightLanguage()}if(block.className!==""){highlighter.engine.highlightBlock(block," ")}utils.addClass(block,"remark-code")})}},{"../converter":18,"../highlighter":3,"../utils":12}],17:[function(require,module,exports){module.exports=Lexer;var CODE=1,CONTENT=2,FENCES=3,SEPARATOR=4,NOTES_SEPARATOR=5;var regexByName={CODE:/(?:^|\n)( {4}[^\n]+\n*)+/,CONTENT:/(?:\\)?((?:\.[a-zA-Z_\-][a-zA-Z\-_0-9]*)+)\[/,FENCES:/(?:^|\n) *(`{3,}|~{3,}) *(?:\S+)? *\n(?:[\s\S]+?)\s*\3 *(?:\n+|$)/,SEPARATOR:/(?:^|\n)(---?)(?:\n|$)/,NOTES_SEPARATOR:/(?:^|\n)(\?{3})(?:\n|$)/};var block=replace(/CODE|CONTENT|FENCES|SEPARATOR|NOTES_SEPARATOR/,regexByName),inline=replace(/CODE|CONTENT|FENCES/,regexByName);function Lexer(){}Lexer.prototype.lex=function(src){var tokens=lex(src,block),i;for(i=tokens.length-2;i>=0;i--){if(tokens[i].type==="text"&&tokens[i+1].type==="text"){tokens[i].text+=tokens[i+1].text;tokens.splice(i+1,1)}}return tokens};function lex(src,regex,tokens){var cap,text;tokens=tokens||[];while((cap=regex.exec(src))!==null){if(cap.index>0){tokens.push({type:"text",text:src.substring(0,cap.index)})}if(cap[CODE]){tokens.push({type:"code",text:cap[0]})}else if(cap[FENCES]){tokens.push({type:"fences",text:cap[0]})}else if(cap[SEPARATOR]){tokens.push({type:"separator",text:cap[SEPARATOR]})}else if(cap[NOTES_SEPARATOR]){tokens.push({type:"notes_separator",text:cap[NOTES_SEPARATOR]})}else if(cap[CONTENT]){text=getTextInBrackets(src,cap.index+cap[0].length);if(text!==undefined){src=src.substring(text.length+1);tokens.push({type:"content_start",classes:cap[CONTENT].substring(1).split("."),block:text.indexOf("\n")!==-1});lex(text,inline,tokens);tokens.push({type:"content_end",block:text.indexOf("\n")!==-1})}else{tokens.push({type:"text",text:cap[0]})}}src=src.substring(cap.index+cap[0].length)}if(src||!src&&tokens.length===0){tokens.push({type:"text",text:src})}return tokens}function replace(regex,replacements){return new RegExp(regex.source.replace(/\w{2,}/g,function(key){return replacements[key].source}))}function getTextInBrackets(src,offset){var depth=1,pos=offset,chr;while(depth>0&&pos");source=marked(source.replace(/^\s+/,""));source=source.replace(/&[l|g]t;/g,function(match){return match==="<"?"<":">"});source=source.replace(/&/g,"&");source=source.replace(/"/g,'"');return source}},{marked:19}],19:[function(require,module,exports){!function(global){!function(){var block={newline:/^\n+/,code:/^( {4}[^\n]+\n*)+/,fences:noop,hr:/^( *[-*_]){3,} *(?:\n+|$)/,heading:/^ *(#{1,6}) *([^\n]+?) *#* *(?:\n+|$)/,nptable:noop,lheading:/^([^\n]+)\n *(=|-){3,} *\n*/,blockquote:/^( *>[^\n]+(\n[^\n]+)*\n*)+/,list:/^( *)(bull) [\s\S]+?(?:hr|\n{2,}(?! )(?!\1bull )\n*|\s*$)/,html:/^ *(?:comment|closed|closing) *(?:\n{2,}|\s*$)/,def:/^ *\[([^\]]+)\]: *]+)>?(?: +["(]([^\n]+)[")])? *(?:\n+|$)/,table:noop,paragraph:/^((?:[^\n]+\n?(?!hr|heading|lheading|blockquote|tag|def))+)\n*/,text:/^[^\n]+/};block.bullet=/(?:[*+-]|\d+\.)/;block.item=/^( *)(bull) [^\n]*(?:\n(?!\1bull )[^\n]*)*/;block.item=replace(block.item,"gm")(/bull/g,block.bullet)();block.list=replace(block.list)(/bull/g,block.bullet)("hr",/\n+(?=(?: *[-*_]){3,} *(?:\n+|$))/)();block._tag="(?!(?:"+"a|em|strong|small|s|cite|q|dfn|abbr|data|time|code"+"|var|samp|kbd|sub|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo"+"|span|br|wbr|ins|del|img)\\b)\\w+(?!:/|@)\\b";block.html=replace(block.html)("comment",//)("closed",/<(tag)[\s\S]+?<\/\1>/)("closing",/])*?>/)(/tag/g,block._tag)();block.paragraph=replace(block.paragraph)("hr",block.hr)("heading",block.heading)("lheading",block.lheading)("blockquote",block.blockquote)("tag","<"+block._tag)("def",block.def)();block.normal=merge({},block);block.gfm=merge({},block.normal,{fences:/^ *(`{3,}|~{3,}) *(\w+)? *\n([\s\S]+?)\s*\1 *(?:\n+|$)/,paragraph:/^/});block.gfm.paragraph=replace(block.paragraph)("(?!","(?!"+block.gfm.fences.source.replace("\\1","\\2")+"|")();block.tables=merge({},block.gfm,{nptable:/^ *(\S.*\|.*)\n *([-:]+ *\|[-| :]*)\n((?:.*\|.*(?:\n|$))*)\n*/,table:/^ *\|(.+)\n *\|( *[-:]+[-| :]*)\n((?: *\|.*(?:\n|$))*)\n*/});function Lexer(options){this.tokens=[];this.tokens.links={};this.options=options||marked.defaults;this.rules=block.normal;if(this.options.gfm){if(this.options.tables){this.rules=block.tables}else{this.rules=block.gfm}}}Lexer.rules=block;Lexer.lex=function(src,options){var lexer=new Lexer(options);return lexer.lex(src)};Lexer.prototype.lex=function(src){src=src.replace(/\r\n|\r/g,"\n").replace(/\t/g," ").replace(/\u00a0/g," ").replace(/\u2424/g,"\n");return this.token(src,true)};Lexer.prototype.token=function(src,top){var src=src.replace(/^ +$/gm,""),next,loose,cap,bull,b,item,space,i,l;while(src){if(cap=this.rules.newline.exec(src)){src=src.substring(cap[0].length);if(cap[0].length>1){this.tokens.push({type:"space"})}}if(cap=this.rules.code.exec(src)){src=src.substring(cap[0].length);cap=cap[0].replace(/^ {4}/gm,"");this.tokens.push({type:"code",text:!this.options.pedantic?cap.replace(/\n+$/,""):cap});continue}if(cap=this.rules.fences.exec(src)){src=src.substring(cap[0].length);this.tokens.push({type:"code",lang:cap[2],text:cap[3]});continue}if(cap=this.rules.heading.exec(src)){src=src.substring(cap[0].length);this.tokens.push({type:"heading",depth:cap[1].length,text:cap[2]});continue}if(top&&(cap=this.rules.nptable.exec(src))){src=src.substring(cap[0].length);item={type:"table",header:cap[1].replace(/^ *| *\| *$/g,"").split(/ *\| */),align:cap[2].replace(/^ *|\| *$/g,"").split(/ *\| */),cells:cap[3].replace(/\n$/,"").split("\n")};for(i=0;i ?/gm,"");this.token(cap,top);this.tokens.push({type:"blockquote_end"});continue}if(cap=this.rules.list.exec(src)){src=src.substring(cap[0].length);this.tokens.push({type:"list_start",ordered:isFinite(cap[2])});cap=cap[0].match(this.rules.item);if(this.options.smartLists){bull=block.bullet.exec(cap[0])[0]}next=false;l=cap.length;i=0;for(;i])/,autolink:/^<([^ >]+(@|:\/)[^ >]+)>/,url:noop,tag:/^|^<\/?\w+(?:"[^"]*"|'[^']*'|[^'">])*?>/,link:/^!?\[(inside)\]\(href\)/,reflink:/^!?\[(inside)\]\s*\[([^\]]*)\]/,nolink:/^!?\[((?:\[[^\]]*\]|[^\[\]])*)\]/,strong:/^__([\s\S]+?)__(?!_)|^\*\*([\s\S]+?)\*\*(?!\*)/,em:/^\b_((?:__|[\s\S])+?)_\b|^\*((?:\*\*|[\s\S])+?)\*(?!\*)/,code:/^(`+)\s*([\s\S]*?[^`])\s*\1(?!`)/,br:/^ {2,}\n(?!\s*$)/,del:noop,text:/^[\s\S]+?(?=[\\?(?:\s+['"]([\s\S]*?)['"])?\s*/;inline.link=replace(inline.link)("inside",inline._inside)("href",inline._href)();inline.reflink=replace(inline.reflink)("inside",inline._inside)();inline.normal=merge({},inline);inline.pedantic=merge({},inline.normal,{strong:/^__(?=\S)([\s\S]*?\S)__(?!_)|^\*\*(?=\S)([\s\S]*?\S)\*\*(?!\*)/,em:/^_(?=\S)([\s\S]*?\S)_(?!_)|^\*(?=\S)([\s\S]*?\S)\*(?!\*)/});inline.gfm=merge({},inline.normal,{escape:replace(inline.escape)("])","~|])")(),url:/^(https?:\/\/[^\s<]+[^<.,:;"')\]\s])/,del:/^~~(?=\S)([\s\S]*?\S)~~/,text:replace(inline.text)("]|","~]|")("|","|https?://|")()});inline.breaks=merge({},inline.gfm,{br:replace(inline.br)("{2,}","*")(),text:replace(inline.gfm.text)("{2,}","*")()});function InlineLexer(links,options){this.options=options||marked.defaults;this.links=links;this.rules=inline.normal;if(!this.links){throw new Error("Tokens array requires a `links` property.")}if(this.options.gfm){if(this.options.breaks){this.rules=inline.breaks}else{this.rules=inline.gfm}}else if(this.options.pedantic){this.rules=inline.pedantic}}InlineLexer.rules=inline;InlineLexer.output=function(src,links,options){var inline=new InlineLexer(links,options);return inline.output(src)};InlineLexer.prototype.output=function(src){var out="",link,text,href,cap;while(src){if(cap=this.rules.escape.exec(src)){src=src.substring(cap[0].length);out+=cap[1];continue}if(cap=this.rules.autolink.exec(src)){src=src.substring(cap[0].length);if(cap[2]==="@"){text=cap[1][6]===":"?this.mangle(cap[1].substring(7)):this.mangle(cap[1]);href=this.mangle("mailto:")+text}else{text=escape(cap[1]);href=text}out+=''+text+"";continue}if(cap=this.rules.url.exec(src)){src=src.substring(cap[0].length);text=escape(cap[1]);href=text;out+=''+text+"";continue}if(cap=this.rules.tag.exec(src)){src=src.substring(cap[0].length);out+=this.options.sanitize?escape(cap[0]):cap[0];continue}if(cap=this.rules.link.exec(src)){src=src.substring(cap[0].length);out+=this.outputLink(cap,{href:cap[2],title:cap[3]});continue}if((cap=this.rules.reflink.exec(src))||(cap=this.rules.nolink.exec(src))){src=src.substring(cap[0].length);link=(cap[2]||cap[1]).replace(/\s+/g," ");link=this.links[link.toLowerCase()];if(!link||!link.href){out+=cap[0][0];src=cap[0].substring(1)+src;continue}out+=this.outputLink(cap,link);continue}if(cap=this.rules.strong.exec(src)){src=src.substring(cap[0].length);out+=""+this.output(cap[2]||cap[1])+"";continue}if(cap=this.rules.em.exec(src)){src=src.substring(cap[0].length);out+=""+this.output(cap[2]||cap[1])+"";continue}if(cap=this.rules.code.exec(src)){src=src.substring(cap[0].length);out+=""+escape(cap[2],true)+"";continue}if(cap=this.rules.br.exec(src)){src=src.substring(cap[0].length);out+="
";continue}if(cap=this.rules.del.exec(src)){src=src.substring(cap[0].length);out+=""+this.output(cap[1])+"";continue}if(cap=this.rules.text.exec(src)){src=src.substring(cap[0].length);out+=escape(cap[0]);continue}if(src){throw new Error("Infinite loop on byte: "+src.charCodeAt(0))}}return out};InlineLexer.prototype.outputLink=function(cap,link){if(cap[0][0]!=="!"){return'"+this.output(cap[1])+""}else{return''+escape(cap[1])+'"}};InlineLexer.prototype.mangle=function(text){var out="",l=text.length,i=0,ch;for(;i.5){ch="x"+ch.toString(16)}out+="&#"+ch+";"}return out};function Parser(options){this.tokens=[];this.token=null;this.options=options||marked.defaults}Parser.parse=function(src,options){var parser=new Parser(options);return parser.parse(src)};Parser.prototype.parse=function(src){this.inline=new InlineLexer(src.links,this.options);this.tokens=src.reverse();var out="";while(this.next()){out+=this.tok()}return out};Parser.prototype.next=function(){return this.token=this.tokens.pop()};Parser.prototype.peek=function(){return this.tokens[this.tokens.length-1]||0};Parser.prototype.parseText=function(){var body=this.token.text;while(this.peek().type==="text"){body+="\n"+this.next().text}return this.inline.output(body)};Parser.prototype.tok=function(){switch(this.token.type){case"space":{return""}case"hr":{return"

\n"}case"heading":{return""+this.inline.output(this.token.text)+"\n"}case"code":{if(this.options.highlight){var code=this.options.highlight(this.token.text,this.token.lang);if(code!=null&&code!==this.token.text){this.token.escaped=true;this.token.text=code}}if(!this.token.escaped){this.token.text=escape(this.token.text,true)}return"
"+this.token.text+"
\n"}case"table":{var body="",heading,i,row,cell,j;body+="\n\n";for(i=0;i'+heading+"\n":""+heading+"\n"}body+="\n\n";body+="\n";for(i=0;i\n";for(j=0;j'+cell+"\n":""+cell+"\n"}body+="\n"}body+="\n";return"\n"+body+"
\n"}case"blockquote_start":{var body="";while(this.next().type!=="blockquote_end"){body+=this.tok()}return"
\n"+body+"
\n"}case"list_start":{var type=this.token.ordered?"ol":"ul",body="";while(this.next().type!=="list_end"){body+=this.tok()}return"<"+type+">\n"+body+"\n"}case"list_item_start":{var body="";while(this.next().type!=="list_item_end"){body+=this.token.type==="text"?this.parseText():this.tok()}return"
  • "+body+"
    • \n"}case"loose_item_start":{var body="";while(this.next().type!=="list_item_end"){body+=this.tok()}return"
  • "+body+"
    • \n"}case"html":{return!this.token.pre&&!this.options.pedantic?this.inline.output(this.token.text):this.token.text}case"paragraph":{return"

    "+this.inline.output(this.token.text)+"

    \n"}case"text":{return"

    "+this.parseText()+"

    \n"}}};function escape(html,encode){return html.replace(!encode?/&(?!#?\w+;)/g:/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}function replace(regex,opt){regex=regex.source;opt=opt||"";return function self(name,val){if(!name)return new RegExp(regex,opt);val=val.source||val;val=val.replace(/(^|[^\[])\^/g,"$1");regex=regex.replace(name,val);return self}}function noop(){}noop.exec=noop;function merge(obj){var i=1,target,key;for(;iAn error occured:

    "+escape(e.message+"",true)+"
    "}throw e}}marked.options=marked.setOptions=function(opt){merge(marked.defaults,opt);return marked};marked.defaults={gfm:true,tables:true,breaks:false,pedantic:false,sanitize:false,smartLists:false,silent:false,highlight:null,langPrefix:"lang-"};marked.Parser=Parser;marked.parser=Parser.parse;marked.Lexer=Lexer;marked.lexer=Lexer.lex;marked.InlineLexer=InlineLexer;marked.inlineLexer=InlineLexer.output;marked.parse=marked;if(typeof exports==="object"){module.exports=marked}else if(typeof define==="function"&&define.amd){define(function(){return marked})}else{this.marked=marked}}.call(function(){return this||(typeof window!=="undefined"?window:global)}())}(window)},{}]},{},[1]); + diff --git a/slides/slide_template.html b/slides/slide_template.html new file mode 100644 index 0000000..9ce5339 --- /dev/null +++ b/slides/slide_template.html @@ -0,0 +1,169 @@ + + + + Data Mining + + + + + + + + + diff --git a/slides/unconverted/org2md.py b/slides/unconverted/org2md.py new file mode 100644 index 0000000..4a9c78e --- /dev/null +++ b/slides/unconverted/org2md.py @@ -0,0 +1,153 @@ +""" +python org2md.py test.org test.markdown && sed -e "/<\!--markdown goes here-->/r test.markdown" < slide_template.html | sed -e "s/<\!--markdown goes here-->//" > test.html && fmdiff test.markdown 2014-02-06-Preprocessing.markdown +python org2md.py test.org ../test.markdown +""" +import re +import sys + + +HEADER = '''name: inverse +layout: true +class: left, top, inverse + +''' + +LINES_TO_ELIMINATE = set([ + r'''#+STYLE: ''', + r'''#+STYLE: ''', + r'''#+STYLE: ''', + r'''#+STYLE: ''', + r'''#+STYLE: ''', + r'''#+STYLE: ''', + r'''#+BEGIN_HTML''', + r'''''', + r'''#+END_HTML''', + r'''# Local Variables:''', + r'''# org-export-html-style-include-default: nil''', + r'''# org-export-html-style-include-scripts: nil''', + r'''# buffer-file-coding-system: utf-8-unix''', + r'''# End:''', +]) + + +def count_of_leading_char(s, leading_char): + index = 0 + while index < len(s) and s[index] == leading_char: + index += 1 + return index + + +def main(input_filename, output_filename): + todo = [] + slide_number = 0 + under_heading = False + + with open(output_filename, 'wb') as output_file: + output_file.write(HEADER) + with open(input_filename, 'rb') as input_file: + for line in input_file: + line = line[:-1] # remove \n + + if line in LINES_TO_ELIMINATE: + continue + + if under_heading and line.strip(): + output_file.write('\n') + + # Checks for upcoming slide + if ':animate:' in line: + todo.append('next slide is animated') + if ':two_col:' in line: + todo.append('next slide is two column') + + # Is this a new slide? + if ':slide:' in line: + slide_number += 1 + todo.append('Slide %d' % slide_number) + todo.append(' main') + output_file.write('---\n\n') + line = line.replace(':slide:', '') + + # Is this notes? + if ':notes:' in line: + todo.append(' notes') + output_file.write('\n???\n\n') + line = line.replace(':notes:', '') + + # Is this a heading? + if line.startswith('*'): + level = count_of_leading_char(line, '*') + line = (level * '#') + line[level:] + under_heading = True + else: + under_heading = False + + # Normalize indentation (should be an even number of spaces) + if count_of_leading_char(line, ' ') % 2: + line = line[1:] + + # Definitions + if ' :: ' in line: + todo.append(' a series of sections will work better for some definitions') + line = line.replace(' :: ', ': ') + + # Images + line = re.sub( + r'''\[\[(file:)?([^\]]+)\]\]''', + r'''''', + line + ) + + # Links + old_line = line + line = re.sub( + r'''\[\[([^]]+)\]\[([^]]+)\]\]''', + r'''[\2](\1)''', + line + ) + if old_line != line: + todo.append(' check link') + + # Block code + line = re.sub( + r'''#\+begin_src *''', + '```', + line + ) + line = line.replace('#+end_src', '```') + + # Inline code + line = re.sub( + r''' =([^ =].*?[^ ])=''', + r''' ```\1```''', + line + ) + + # Jim --> Jimmy + line = line.replace('Jim', 'Jimmy') + line = line.replace('jim', 'jimmy') + line = line.replace('Blomo', 'Retzlaff') + line = line.replace('jblomo', 'jretz') + + # Table + first_bar = line.find('|') + if first_bar >= 0: + if line.find('|', first_bar + 1): + todo.append(' table') + + # Unknown construct + if line.startswith('#+'): + print 'Unknown construct:', line + + output_file.write(line.rstrip()) + output_file.write('\n') + + todo.append('Headings are the right level?') + output_file.write('\n---\n\n') + for line in todo: + output_file.write(line) + output_file.write('\n') + + +if __name__ == '__main__': + main(*sys.argv[1:])