Initial commit

{\rtf1\ansi\ansicpg1252\cocoartf1038\cocoasubrtf350

{\fonttbl\f0\fswiss\fcharset0 Helvetica;}

{\colortbl;\red255\green255\blue255;}

\margl1440\margr1440\vieww9000\viewh8400\viewkind0

\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural\pardirnatural

\f0\fs24 \cf0 No work files for this lesson...}

 No newline at end of file

//

// RDD and DataFrame.scala

//

// 

// Using RDDs 

//

// Initialize

val mobiletxt = sc.textFile("file:///Users/dennylee/spark/data/MobileSampleData.csv")

val tabs = mobiletxt.map(_.split(","))

// Show column header

tabs.first

// Count of the number of Apple Devices 

tabs.filter(_(4) == "Apple").count

//

// Using DataFrames 

//

// Read table and print Schema

val mobile = spark.read.option("header", "true").csv("file:///Users/dennylee/spark/data/MobileSampleData.csv")

mobile.printSchema()

// Count of number of Apple Devices

mobile.filter(mobile("DeviceMake")==="Apple").count

// Using SQL 

mobile.createOrReplaceTempView("mobile")

spark.sql("SELECT count(1) FROM mobile WHERE DeviceMake = 'Apple'").show()

// Explain physical plan

spark.sql("SELECT count(1) FROM mobile WHERE DeviceMake = 'Apple'").explain()

# ./bin/pyspark --master spark://kintaro.f1xwivc2xm4uporkjsjoixul2c.dx.internal.cloudapp.net:7077 --packages graphframes:graphframes:0.4.0-spark2.0-s_2.11 --executor-memory 8G 

# 

# Example 1: Extract data and run Count

#

# Row Count

mobiletxt = sc.textFile("file:///home/dennylee/data/HiveSampleData.txt")

mobiletxt.count()

# Word Count

from operator import add

wc = mobiletxt.flatMap(lambda x: x.split(' ')).map(lambda x: (x, 1)).reduceByKey(add)

wc.count()

 No newline at end of file

// ./bin/spark-shell --master spark://kintaro.f1xwivc2xm4uporkjsjoixul2c.dx.internal.cloudapp.net:7077 --packages graphframes:graphframes:0.4.0-spark2.0-s_2.11 --executor-memory 8G 

// 

// Example 1: Extract data and run Count

//

// Row Count

val mobiletxt = sc.textFile("file:///home/dennylee/data/HiveSampleData.txt")

mobiletxt.count

// Word Count

var wc = mobiletxt.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)

wc.count

//

// Rerun example 1 using SQL

//

val df = spark.read.option("inferSchema", "true").csv("file:///home/dennylee/data/HiveSampleData.csv")

val colNames = Seq("ClientID", "QueryTime", "Market", "DevicePlatform", "DeviceMake", "DeviceModel", "State", "Country", "SessionId", "SessionPageViewOrder")		

val mobile = df.toDF(colNames: _*)

mobile.printSchema()

mobile.createOrReplaceTempView("mobile")

spark.sql("select count(1) from mobile").show()