Initial commit

{\rtf1\ansi\ansicpg1252\cocoartf1038\cocoasubrtf350

{\fonttbl\f0\fswiss\fcharset0 Helvetica;}

{\colortbl;\red255\green255\blue255;}

\margl1440\margr1440\vieww9000\viewh8400\viewkind0

\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural\pardirnatural

\f0\fs24 \cf0 No work files for this lesson...}

 No newline at end of file

## Creating two tables with 

## same data but different rowkeys in HBase to avoid scans

create 'users','f1', SPLITS => ['0250000','0500000','0750000']

create 'users_name','f1'

## Create table with id as rowkey

hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=HBASE_ROW_KEY,f1:name,f1:address,f1:dob,f1:phone,f1:state -Dimporttsv.separator=$'\x01' users data_padded.hive

## Create table with name as Rowkey

hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.columns=f1:id,HBASE_ROW_KEY,f1:address,f1:dob,f1:phone,f1:state -Dimporttsv.separator=$'\x01' users_name data_padded.hive

## sample queries

scan 'users', LIMIT=>1

scan 'users_name', LIMIT=>1

scan 'users_name', {STARTROW => 'Cassie', ENDROW => 'Cassif'}

get 'users_name', 'Cassie Wilkinson'

## Prejoining Data in Hive

## to avoid a shuffle

create table prejoined as

select users.id, users.name, users.address, users.dob,users.phone,users.state, collect_set(orders.total) as orderlist from users left join orders on users.id=orders.id group by users.id,users.name,users.address, users.dob,users.phone,users.state ;

## sample queries

select * from prejoined where size(orderlist) > 0 limit 1;

select * from prejoined where size(orderlist) > 1 limit 1;

### point lookup => scan => scan with shuffle=>scan with multiple shuffles

### single record point lookup 0.0220 seconds

get 'users','0000001'

### Table Scan

scan 'users', { COLUMNS => 'f1:name', FILTER => "ValueFilter( =, 'regexstring:Cassie' )" }

scan 'users', { COLUMNS => 'f1:name', FILTER => "ValueFilter( =, 'regexstring:Roberts MD' )" }

## Hive commands

set mapreduce.job.reduces=2;

set hive.auto.convert.join = false;

## Full Table Scan in hive 16.419 seconds

select * from users where id=234567;

## Full Table Scan=>small Shuffle=>aggregate

select count(*) from users;

## Table Scan=>larger Shuffle

select avg(id) from users where state < 'LA' group by state;

##Join that leads to scan + shuffle + hdfs write

insert overwrite directory '/user/cloudera/hive_output' select * from users join orders on users.id=orders.id ;

## show output 

hadoop fs -cat hive_output/0* | grep 669410 | less

## Scan=>Shuffle=>rescan=>shuffle

insert overwrite directory '/user/cloudera/hive_output' select * from users join orders on users.id=orders.id sort by state;