Initial commit

{\rtf1\ansi\ansicpg1252\cocoartf1038\cocoasubrtf350

{\fonttbl\f0\fswiss\fcharset0 Helvetica;}

{\colortbl;\red255\green255\blue255;}

\margl1440\margr1440\vieww9000\viewh8400\viewkind0

\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\ql\qnatural\pardirnatural

\f0\fs24 \cf0 No work files for this lesson...}

 No newline at end of file

Amazon AWS setup for this course. This video discusses steps that only

need to be done once.

1. Log in to AWS using Amazon / AWS credentials

2. Select your data center (I suggest US East (N. Virginia)

3. Select VPC under Networking

4. Select Start VPC wizard

5. After creating the VPC, select "Modify Auto-Assign Public IP" to enable

6. Create a free-for-all security group associated with the VPC

   allowing all traffic inbound and outbound

7. Generate security credentials. Cloudera Director supports IAM users

   and Amazon prefers them but this video doesn't use them. You're welcome

   to try them.

8. Note the access key and secret access key in your workspace

9. Create a keypair

10. Download the keypair and give it 600 permissions

For more information:

http://aws.amazon.com/what-is-cloud-computing/?sc_channel=PS&sc_campaign=acquisition_US&sc_publisher=google&sc_medium=cloud_computing_b&sc_content=sitelink&sc_detail=%2Baws%20%2Bamazon&sc_category=cloud_computing&sc_segment=what_is_cloud_computing&sc_matchtype=b&sc_country=US&s_kwcid=AL!4422!3!78924598722!b!!g!!%2Baws%20%2Bamazon&ef_id=VhDB6AAABX9I0js-:20160120232207:s

https://aws.amazon.com/training/intro_series/

http://docs.aws.amazon.com/gettingstarted/latest/awsgsg-intro/gsg-aws-intro.html

https://en.wikipedia.org/wiki/Amazon_Virtual_Private_Cloud

#

# Copyright (c) 2014 Cloudera, Inc. All rights reserved.

#

#

# Simple AWS Cloudera Director configuration file with automatic role assignments

# that works as expected if you use a single instance type for all cluster nodes

#

#

# Cluster name

#

name: Hadoop-Security

#

# Cloud provider configuration (credentials, region or zone and optional default image)

#

provider {

    type: aws

    #

    # Get AWS credentials from the OS environment

    # See http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html

    #

    # If specifying the access keys directly and not through variables, make sure to enclose

    # them in double quotes.

    #

    # Not needed when running on an instance launched with an IAM role.

    accessKeyId: ${?AWS_ACCESS_KEY_ID}

    secretAccessKey: ${?AWS_SECRET_ACCESS_KEY}

    #

    # Whether to publish access keys. Default is false.

    # Publishing access keys to clients is necessary for those clients to have access to S3.

    # These keys are published in the configuration files of all HDFS clients.

    #

    publishAccessKeys: true

    #

    # ID of the Amazon AWS region to use

    # See: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html

    #

    region: us-east-1

    #

    # Region endpoint (if you are using one of the Gov. regions)

    #

    # regionEndpoint: ec2.us-gov-west-1.amazonaws.com

    #

    # ID of the VPC subnet

    # See: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html

    #

    subnetId: subnet-e2440ec9 

    #

    # Comma separated list of security group IDs

    # See: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_SecurityGroups.html

    #

    securityGroupsIds: sg-57296430 

    #

    # A prefix that Cloudera Director should use when naming the instances (this is not part of the hostname)

    #

    instanceNamePrefix: or-1

    #

    # Specify a size for the root volume (in GBs). Cloudera Director will automatically expand the

    # filesystem so that you can use all the available disk space for your application

    # See: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/storage_expand_partition.html

    #

    # rootVolumeSizeGB: 100 # defaults to 50 GB if not specified

    #

    # Specify the type of the EBS volume used for the root partition. Defaults to gp2

    # See: http://aws.amazon.com/ebs/details/

    #

    # rootVolumeType: gp2 # OR standard (for EBS magnetic)

    #

    # Whether to associate a public IP address with instances or not. If this is false

    # we expect instances to be able to access the internet using a NAT instance

    #

    # Currently the only way to get optimal S3 data transfer performance is to assign

    # public IP addresses to your instances and not use NAT (public subnet type of setup)

    #

    # See: http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-ip-addressing.html

    #

    # associatePublicIpAddresses: true

}

#

# SSH credentials to use to connect to the instances

#

ssh {

    username: ec2-user # for RHEL image

    privateKey: /home/ec2-user/hadoopsecurity.pem # with an absolute path to .pem file

}

#

# A list of instance types to use for group of nodes or management services

#

instances {

    m3x {

        type: m3.xlarge   

        #

        # Amazon Machine Image (AMI)

        #

        # See: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html

        # Compatibility matrix: https://aws.amazon.com/amazon-linux-ami/instance-type-matrix/

        #

        # Red Hat Enterprise Linux AMI IDs: http://aws.amazon.com/partners/redhat/

        # We support Red Hat Enterprise Linux 6.4 (64bit) 64bit PV or HVM

        #

        image: ami-8767d1ec

        #

        # Name of the IAM Role to use for this instance type

        # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html

        #

        # iamProfileName: iam-profile-REPLACE-ME

        tags {

            owner: ${?USER}

        }

        bootstrapScript: """#!/bin/sh

# This is an embedded bootstrap script that runs as root and can be used to customize

# the instances immediately after boot and before any other Cloudera Director action

# If the exit code is not zero Cloudera Director will automatically retry

echo 'Hello World!'

exit 0

"""

        #

        # Flag indicating whether to normalize the instance. Not setting normalization here implies that your

        # bootstrap script will take care of normalization.

        #

        # Normalization includes:

        #   downloading and installing packages

        #   minimizing swappiness

        #   increasing the maximun number of open files

        #   mounting ephemeral disks

        #   resizing the root partition.

        #

        # Defaults to true

        #

        # normalizeInstance: true

    }

}

#

# Configuration for Cloudera Manager. Cloudera Director can use an existing instance

# or bootstrap everything from scratch for a new cluster

#

cloudera-manager {

    instance: ${instances.m3x} {

        tags {

            application: "Cloudera Manager 5"

        }

    }

    #

    # Automatically activate 60-Day Cloudera Enterprise Trial

    #

    enableEnterpriseTrial: true

}

#

# Cluster description

#

cluster {

    # List the products and their versions that need to be installed.

    # These products must have a corresponding parcel in the parcelRepositories

    # configured above. The specified version will be used to find a suitable

    # parcel. Specifying a version that points to more than one parcel among

    # those available will result in a configuration error. Specify more granular

    # versions to avoid conflicts.

    products {

      CDH: 5 # includes Impala and Spark

    }

    #

    # OR for CDH4

    #

    # products {

    #   CDH: 4

    #   IMPALA: 1.3

    #   SPARK: 0.9

    # }

    parcelRepositories: ["http://archive.cloudera.com/cdh5/parcels/5.4.4/"] 

    services: [HDFS, YARN, ZOOKEEPER, HBASE, HIVE, IMPALA, SPARK_ON_YARN]

    nodes {

        count: 5

        instance: ${instances.m3x}

    }

    postCreateScripts: ["""#!/bin/sh

# This is an embedded post creation script script that runs as root and can be used to

# customize the cluster after it has been created.

# If the exit code is not zero Cloudera Director will fail

echo 'Hello World!'

exit 0

    """,

    """#!/bin/sh

# Additionally, multiple post creation scripts can be supplied.  They will run in the

# order they are listed here.

echo 'Hello again!'

exit 0

    """]

}

export AWS_ACCESS_KEY_ID=AKIAIF3SJAN7C3LVBVRQ

export AWS_SECRET_ACCESS_KEY=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

cloudera-director bootstrap-remote ./aws.simple.conf --lp.remote.username=admin --lp.remote.password=admin --lp.remote.hostAndPort=localhost:7189

export AWS_ACCESS_KEY_ID=REPLACE-ME

export AWS_SECRET_ACCESS_KEY=REPLACE-ME

cloudera-director bootstrap-remote ./dev.conf --lp.remote.username=admin --lp.remote.password=admin --lp.remote.hostAndPort=localhost:7189