add 'Deep Learning with TensorFlow' NBs and data (2e2d4ab3) · Commits · live-training / deep-learning

data/tf-mnist/mnist.pkl

0 → 100644

+215 MiB

File added.

No diff preview for this file type.

View file

notebooks/deep_net_in_tensorflow.ipynb

0 → 100644

+222 −0

Changes for notebooks/deep_net_in_tensorflow.ipynb: 222 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		%% Cell type:markdown id: tags:

		# Deep Neural Network in TensorFlow

		%% Cell type:markdown id: tags:

		This notebook adds a dense hidden layer to the [Intermediate Net in TensorFlow](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/intermediate_net_in_tensorflow.ipynb) model architecture.

		%% Cell type:markdown id: tags:

		#### Load dependencies

		%% Cell type:code id: tags:

		``` python
		import numpy as np
		np.random.seed(42)
		import tensorflow as tf
		tf.set_random_seed(42)
		```

		%% Cell type:markdown id: tags:

		#### Load data

		%% Cell type:code id: tags:

		``` python
		# from tensorflow.examples.tutorials.mnist import input_data
		# mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

		import pickle
		mnist = pickle.load(open('../data/tf-mnist/mnist.pkl', 'rb'))
		```

		%% Cell type:markdown id: tags:

		#### Set neural network hyperparameters (tidier at top of file!)

		%% Cell type:code id: tags:

		``` python
		lr = 0.1
		epochs = 20
		batch_size = 128
		weight_initializer = tf.contrib.layers.xavier_initializer()
		```

		%% Cell type:markdown id: tags:

		#### Set number of neurons for each layer

		%% Cell type:code id: tags:

		``` python
		n_input = 784
		n_dense_1 = # CODE HERE
		n_dense_2 = # CODE HERE
		n_dense_3 = # CODE HERE
		n_classes = 10
		```

		%% Cell type:markdown id: tags:

		#### Define placeholders Tensors for inputs and labels

		%% Cell type:code id: tags:

		``` python
		x = tf.placeholder(tf.float32, [None, n_input])
		y = tf.placeholder(tf.float32, [None, n_classes])
		```

		%% Cell type:markdown id: tags:

		#### Define types of layers

		%% Cell type:code id: tags:

		``` python
		# dense layer with ReLU activation:
		def dense(x, W, b):

		# CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Design neural network architecture

		%% Cell type:code id: tags:

		``` python
		def network(x, weights, biases):

		# two dense hidden layers:
		dense_1 = dense(x, weights['W1'], biases['b1'])
		dense_2 = dense(dense_1, weights['W2'], biases['b2'])
		dense_3 = dense(dense_2, weights['W3'], biases['b3'])

		# linear output layer (softmax)
		out_layer_z = tf.add(tf.matmul(dense_3, weights['W_out']), biases['b_out'])

		return out_layer_z
		```

		%% Cell type:markdown id: tags:

		#### Define dictionaries for storing weights and biases for each layer -- and initialize

		%% Cell type:code id: tags:

		``` python
		bias_dict = {
		'b1': tf.Variable(tf.zeros([n_dense_1])),
		'b2': tf.Variable(tf.zeros([n_dense_2])),
		'b3': tf.Variable(tf.zeros([n_dense_3])),
		'b_out': tf.Variable(tf.zeros([n_classes]))
		}

		weight_dict = {
		'W1': tf.get_variable('W1', [n_input, n_dense_1], initializer=weight_initializer),
		'W2': tf.get_variable('W2', [n_dense_1, n_dense_2], initializer=weight_initializer),
		'W3': tf.get_variable('W3', [n_dense_3, n_dense_3], initializer=weight_initializer),
		'W_out': tf.get_variable('W_out', [n_dense_3, n_classes], initializer=weight_initializer)
		}
		```

		%% Cell type:markdown id: tags:

		#### Build model

		%% Cell type:code id: tags:

		``` python
		predictions = network(x, weights=weight_dict, biases=bias_dict)
		```

		%% Cell type:markdown id: tags:

		#### Define model's loss and its optimizer

		%% Cell type:code id: tags:

		``` python
		cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predictions, labels=y))
		optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr).minimize(cost)
		```

		%% Cell type:markdown id: tags:

		#### Define evaluation metrics

		%% Cell type:code id: tags:

		``` python
		# calculate accuracy by identifying test cases where the model's highest-probability class matches the true y label:
		correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(y, 1))
		accuracy_pct = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) * 100
		```

		%% Cell type:markdown id: tags:

		#### Create op for variable initialization

		%% Cell type:code id: tags:

		``` python
		initializer_op = tf.global_variables_initializer()
		```

		%% Cell type:markdown id: tags:

		#### Train the network in a session

		%% Cell type:code id: tags:

		``` python
		with tf.Session() as session:
		session.run(initializer_op)

		print("Training for", epochs, "epochs.")

		# loop over epochs:
		for epoch in range(epochs):

		avg_cost = 0.0 # track cost to monitor performance during training
		avg_accuracy_pct = 0.0

		# loop over all batches of the epoch:
		n_batches = int(mnist.train.num_examples / batch_size)
		for i in range(n_batches):

		batch_x, batch_y = mnist.train.next_batch(batch_size)

		# feed batch data to run optimization and fetching cost and accuracy:
		_, batch_cost, batch_acc = session.run([optimizer, cost, accuracy_pct],
		feed_dict={x: batch_x, y: batch_y})

		# accumulate mean loss and accuracy over epoch:
		avg_cost += batch_cost / n_batches
		avg_accuracy_pct += batch_acc / n_batches

		# output logs at end of each epoch of training:
		print("Epoch ", '%03d' % (epoch+1),
		": cost = ", '{:.3f}'.format(avg_cost),
		", accuracy = ", '{:.2f}'.format(avg_accuracy_pct), "%",
		sep='')

		print("Training Complete. Testing Model.\n")

		test_cost = cost.eval({x: mnist.test.images, y: mnist.test.labels})
		test_accuracy_pct = accuracy_pct.eval({x: mnist.test.images, y: mnist.test.labels})

		print("Test Cost:", '{:.3f}'.format(test_cost))
		print("Test Accuracy: ", '{:.2f}'.format(test_accuracy_pct), "%", sep='')
		```

		%% Cell type:code id: tags:

		``` python
		```

notebooks/first_tensorflow_graphs.ipynb

0 → 100644

+61 −0

Changes for notebooks/first_tensorflow_graphs.ipynb: 61 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		%% Cell type:markdown id: tags:

		# First TensorFlow Graphs

		%% Cell type:markdown id: tags:

		In this notebook, we execute elementary TensorFlow computational graphs.

		%% Cell type:markdown id: tags:

		#### Load dependencies

		%% Cell type:code id: tags:

		``` python
		import numpy as np
		import tensorflow as tf
		```

		%% Cell type:markdown id: tags:

		#### Simple arithmetic

		%% Cell type:code id: tags:

		``` python
		x1 = # CODE HERE
		x2 = # CODE HERE
		```

		%% Cell type:code id: tags:

		``` python
		sum_op = # CODE HERE
		product_op = # CODE HERE
		```

		%% Cell type:code id: tags:

		``` python
		with tf.Session() as session:
		sum_result = # CODE HERE
		product_result = # CODE HERE
		```

		%% Cell type:code id: tags:

		``` python
		sum_result
		```

		%% Cell type:code id: tags:

		``` python
		product_result
		```

		%% Cell type:code id: tags:

		``` python
		```

notebooks/first_tensorflow_neurons.ipynb

0 → 100644

+101 −0

Changes for notebooks/first_tensorflow_neurons.ipynb: 101 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		%% Cell type:markdown id: tags:

		# First TensorFlow Neurons

		%% Cell type:markdown id: tags:

		#### Load dependencies

		%% Cell type:code id: tags:

		``` python
		import numpy as np
		np.random.seed(42)
		import tensorflow as tf
		tf.set_random_seed(42)
		import matplotlib.pyplot as plt
		%matplotlib inline
		```

		%% Cell type:markdown id: tags:

		#### Set number of neurons

		%% Cell type:code id: tags:

		``` python
		n_input = # CODE HERE
		n_dense = # CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Define placeholder Tensor for simulated MNIST digits

		%% Cell type:code id: tags:

		``` python
		x = # CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Create Variable Tensors for neuron biases `b` and weight matrix `W`

		%% Cell type:code id: tags:

		``` python
		b = # CODE HERE
		W = # CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Design the computational graph

		%% Cell type:code id: tags:

		``` python
		z = # CODE HERE
		a = # CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Create op for variable initialization

		%% Cell type:code id: tags:

		``` python
		initializer_op = tf.global_variables_initializer()
		```

		%% Cell type:markdown id: tags:

		#### Execute the graph in a session

		%% Cell type:code id: tags:

		``` python
		with tf.Session() as session:
		session.run(initializer_op)

		layer_output = session.run(a, {x: np.random.random([1, n_input])})
		```

		%% Cell type:code id: tags:

		``` python
		layer_output
		```

		%% Cell type:code id: tags:

		``` python
		_ = plt.hist(np.transpose(layer_output))
		```

		%% Cell type:code id: tags:

		``` python
		```

notebooks/intro_to_tensorflow_times_a_million.ipynb

0 → 100644

+121 −0

Changes for notebooks/intro_to_tensorflow_times_a_million.ipynb: 121 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		%% Cell type:markdown id: tags:

		# (Introduction to Tensorflow) * 10^6

		%% Cell type:markdown id: tags:

		In this notebook, we modify the [tensor-fied intro to TensorFlow notebook](https://github.com/the-deep-learners/TensorFlow-LiveLessons/blob/master/notebooks/tensor-fied_intro_to_tensorflow.ipynb) to use placeholder tensors and feed in data from a data set of millions of points. This is a derivation of Jared Ostmeyer's [Naked Tensor](https://github.com/jostmey/NakedTensor/) code.

		%% Cell type:code id: tags:

		``` python
		import numpy as np
		np.random.seed(42)
		import pandas as pd
		import matplotlib.pyplot as plt
		%matplotlib inline
		import tensorflow as tf
		tf.set_random_seed(42)
		```

		%% Cell type:code id: tags:

		``` python
		xs = np.linspace(0., 8., 8000000) # eight million points spaced evenly over the interval zero to eight
		ys = 0.3*xs-0.8+np.random.normal(scale=0.25, size=len(xs)) # eight million labels given xs, m=0.3, b=-0.8, plus normally-distributed noise
		```

		%% Cell type:code id: tags:

		``` python
		fig, ax = plt.subplots()
		data_subset = pd.DataFrame(list(zip(xs, ys)), columns=['x', 'y']).sample(n=1000)
		_ = ax.scatter(data_subset.x, data_subset.y)
		```

		%% Cell type:code id: tags:

		``` python
		m = tf.Variable(-0.5)
		b = tf.Variable(1.0)
		```

		%% Cell type:code id: tags:

		``` python
		batch_size = # CODE HERE
		```

		%% Cell type:markdown id: tags:

		#### Define placeholder tensors of length batch_size whose values will be filled in during graph execution

		%% Cell type:code id: tags:

		``` python
		xs_placeholder = tf.placeholder(tf.float32, [batch_size])
		ys_placeholder = tf.placeholder(tf.float32, [batch_size])
		```

		%% Cell type:markdown id: tags:

		#### Define graph that incorporates placeholders

		%% Cell type:code id: tags:

		``` python
		ys_model = # CODE HERE
		total_error = # CODE HERE
		```

		%% Cell type:code id: tags:

		``` python
		optimizer_operation = tf.train.GradientDescentOptimizer(learning_rate=0.001).minimize(total_error) # demo 0.01, 0.0001
		```

		%% Cell type:code id: tags:

		``` python
		initializer_operation = tf.global_variables_initializer()
		```

		%% Cell type:markdown id: tags:

		#### Sample from the full data set while running the session

		%% Cell type:code id: tags:

		``` python
		with tf.Session() as session:

		session.run(initializer_operation)

		n_batches = 1000 # 10, then 1000
		for iteration in range(n_batches):
		random_indices = np.random.randint(len(xs), size=batch_size) # sample the batch by random selection
		feed = { # feeds are dictionaries
		xs_placeholder: xs[random_indices],
		ys_placeholder: ys[random_indices]
		}
		session.run(optimizer_operation, feed_dict=feed) # minimize cost with the mini-batch

		slope, intercept = session.run([m, b])
		```

		%% Cell type:code id: tags:

		``` python
		slope
		```

		%% Cell type:code id: tags:

		``` python
		intercept
		```

		%% Cell type:code id: tags:

		``` python
		```