diff --git a/DataSet.py b/DataSet.py
new file mode 100644
index 00000000..dd888e1d
--- /dev/null
+++ b/DataSet.py
@@ -0,0 +1,36 @@
+import numpy as np
+
+class DataSet(object):
+
+ def __init__(self,
+ images,
+ labels):
+
+ assert images.shape[0] == labels.shape[0], (
+ 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))
+ self._num_examples = images.shape[0]
+
+ self._images = images
+ self._labels = labels
+ self._epochs_completed = 0
+ self._index_in_epoch = 0
+
+ def next_batch(self, batch_size):
+ """Return the next `batch_size` examples from this data set."""
+
+ start = self._index_in_epoch
+ self._index_in_epoch += batch_size
+ if self._index_in_epoch > self._num_examples:
+ # Finished epoch
+ self._epochs_completed += 1
+ # Shuffle the data
+ perm = np.arange(self._num_examples)
+ np.random.shuffle(perm)
+ self._images = self._images[perm]
+ self._labels = self._labels[perm]
+ # Start next epoch
+ start = 0
+ self._index_in_epoch = batch_size
+ assert batch_size <= self._num_examples
+ end = self._index_in_epoch
+ return self._images[start:end], self._labels[start:end]
\ No newline at end of file
diff --git a/dataprep.py b/dataprep.py
new file mode 100644
index 00000000..ad93c76c
--- /dev/null
+++ b/dataprep.py
@@ -0,0 +1,121 @@
+import numpy as np
+import cv2
+import re
+import os
+from datetime import datetime
+
+
+def process_session(session_path):
+
+ cap = cv2.VideoCapture(session_path + "/output.mov")
+ video_timestamps = []
+ with open(session_path + '/video_timestamps.txt') as video_timestamps_reader:
+ for line in video_timestamps_reader:
+ line = line.replace("\n", "")
+ ts = datetime.strptime(line, '%Y-%m-%d %H:%M:%S.%f')
+ video_timestamps.append(ts)
+
+ commands = []
+ with open(session_path + '/clean_session.txt') as clean_session_reader:
+ for line in clean_session_reader:
+ line = line.replace("\n", "")
+ match = re.match(r"^.*\['(.*)'\].*$", line)
+ if match is not None:
+ command = match.group(1)
+ else:
+ command = 'no command'
+ raw_ts = line[line.index(" ") + 1:]
+ ts = datetime.strptime(raw_ts, '%Y-%m-%d %H:%M:%S.%f')
+ commands.append([command, ts])
+
+ # time after which no other data is relevant because driving session has ended
+ end_time = commands[len(commands) - 1][1]
+
+ # cleanup to track only command transitions
+ compact_commands = []
+ prev_command = None
+ for item in commands:
+ command, ts = item[0], item[1]
+ if command != prev_command and command != 'no command':
+ compact_commands.append([command, ts])
+ prev_command = command
+ commands = compact_commands
+
+ # time before which no other data is relevant because driving session just started
+ start_time = commands[0][1]
+
+ current_command = commands[0][0]
+ command_counter = 1
+ future_command = commands[command_counter][0]
+ future_command_ts = commands[command_counter][1]
+
+ predictors = []
+ targets = []
+
+ frame_counter = -1
+ while (cap.isOpened()):
+ frame_counter = frame_counter + 1
+ ret, frame = cap.read()
+ if cv2.waitKey(1) & 0xFF == ord('q'): # don't remove this if statement or video feed will die
+ break
+ video_timestamp = video_timestamps[frame_counter]
+ if video_timestamp > start_time:
+ if video_timestamp < end_time:
+ if video_timestamp > future_command_ts:
+ current_command = future_command
+ command_counter = command_counter + 1
+ if command_counter < len(commands):
+ future_command = commands[command_counter][0]
+ future_command_ts = commands[command_counter][1]
+ else:
+ future_command = "END"
+ future_command_ts = end_time
+ print(current_command)
+ cv2.imshow('frame', frame)
+ predictors.append(frame)
+ target = [0, 0, 0] # in order: left, up, right
+ if current_command == 'left':
+ target[0] = 1
+ elif current_command == 'up':
+ target[1] = 1
+ elif current_command == 'right':
+ target[2] = 1
+ targets.append(target)
+ else:
+ cap.release()
+ cv2.destroyAllWindows()
+
+ return predictors, targets
+
+
+def data_prep(data_path):
+
+ data_folders = os.listdir(data_path)
+ train_folder_size = int(len(data_folders) * 0.8)
+
+ train_predictors = []
+ train_targets = []
+ for folder in data_folders[:train_folder_size]:
+ predictors, targets = process_session(data_path+'/'+folder)
+ train_predictors.append(predictors)
+ train_targets.append(targets)
+ train_predictors_np = np.array(predictors)
+ train_targets_np = np.array(targets)
+
+ validation_predictors = []
+ validation_targets = []
+ for folder in data_folders[train_folder_size:]:
+ predictors, targets = process_session(data_path + '/' + folder)
+ validation_predictors.append(predictors)
+ validation_targets.append(targets)
+ validation_predictors_np = np.array(predictors)
+ validation_targets_np = np.array(targets)
+
+ np.savez(data_path+'/final_processed_data', train_predictors=train_predictors_np,
+ train_targets=train_targets_np,validation_predictors = validation_predictors_np,
+ validation_targets = validation_targets_np)
+
+if __name__ == '__main__':
+ data_path = str(os.path.dirname(os.path.realpath(__file__))) + "/data"
+ data_prep(data_path)
+ print("Finished.")
\ No newline at end of file
diff --git a/drive_api.py b/drive_api.py
new file mode 100644
index 00000000..645a8a13
--- /dev/null
+++ b/drive_api.py
@@ -0,0 +1,222 @@
+import tornado.ioloop
+import tornado.web
+from datetime import datetime
+import os
+from operator import itemgetter
+import RPi.GPIO as GPIO
+import requests
+from time import sleep
+
+class PostHandler(tornado.web.RequestHandler):
+
+ def post(self):
+ timestamp = datetime.now()
+ data_json = tornado.escape.json_decode(self.request.body)
+ allowed_commands = set(['37','38','39','40'])
+ command = data_json['command']
+ command = list(command.keys())
+ command = set(command)
+ command = allowed_commands & command
+ file_path = str(os.path.dirname(os.path.realpath(__file__)))+"/session.txt"
+ log_entry = str(command)+" "+str(timestamp)
+ log_entries.append((command,timestamp))
+ with open(file_path,"a") as writer:
+ writer.write(log_entry+"\n")
+ print(log_entry)
+ command_duration = 0.1
+
+ if '37' in command:
+ motor.forward_left(100)
+ elif '38' in command:
+ motor.forward(100)
+ elif '39' in command:
+ motor.forward_right(100)
+ elif '40' in command:
+ motor.backward(100)
+ else:
+ motor.stop()
+
+# This only works on data from the same live python process. It doesn't
+# read from the session.txt file. It only sorts data from the active
+# python process. This is required because it reads from a list instead
+# of a file on data from the same live python process. It doesn't
+# read from the session.txt file. It only sorts data from the active
+# log_entries python list
+class StoreLogEntriesHandler(tornado.web.RequestHandler):
+ def get(self):
+ file_path = str(os.path.dirname(os.path.realpath(__file__)))+"/clean_session.txt"
+ sorted_log_entries = sorted(log_entries,key=itemgetter(1))
+ prev_command = set()
+ allowed_commands = set(['38','37','39','40'])
+ for log_entry in sorted_log_entries:
+ command = log_entry[0]
+ timestamp = log_entry[1]
+ if len(command ^ prev_command) > 0:
+ prev_command = command
+ with open(file_path,"a") as writer:
+ readable_command = []
+ for element in list(command):
+ if element == '37':
+ readable_command.append("left")
+ if element == '38':
+ readable_command.append("up")
+ if element == '39':
+ readable_command.append("right")
+ if element == '40':
+ readable_command.append("down")
+ log_entry = str(list(readable_command))+" "+str(timestamp)
+ writer.write(log_entry+"\n")
+ print(log_entry)
+ self.write("Finished")
+
+class MultipleKeysHandler(tornado.web.RequestHandler):
+
+ def get(self):
+ print("HelloWorld")
+ self.write('''
+
+
+
+
+
+
+
+ Click in this frame, then try holding down some keys
+
+
+
+ ''')
+
+
+class Motor:
+
+ def __init__(self, pinForward, pinBackward, pinControlStraight,pinLeft, pinRight, pinControlSteering):
+ """ Initialize the motor with its control pins and start pulse-width
+ modulation """
+
+ self.pinForward = pinForward
+ self.pinBackward = pinBackward
+ self.pinControlStraight = pinControlStraight
+ self.pinLeft = pinLeft
+ self.pinRight = pinRight
+ self.pinControlSteering = pinControlSteering
+ GPIO.setup(self.pinForward, GPIO.OUT)
+ GPIO.setup(self.pinBackward, GPIO.OUT)
+ GPIO.setup(self.pinControlStraight, GPIO.OUT)
+
+ GPIO.setup(self.pinLeft, GPIO.OUT)
+ GPIO.setup(self.pinRight, GPIO.OUT)
+ GPIO.setup(self.pinControlSteering, GPIO.OUT)
+
+ self.pwm_forward = GPIO.PWM(self.pinForward, 100)
+ self.pwm_backward = GPIO.PWM(self.pinBackward, 100)
+ self.pwm_forward.start(0)
+ self.pwm_backward.start(0)
+
+ self.pwm_left = GPIO.PWM(self.pinLeft, 100)
+ self.pwm_right = GPIO.PWM(self.pinRight, 100)
+ self.pwm_left.start(0)
+ self.pwm_right.start(0)
+
+ GPIO.output(self.pinControlStraight,GPIO.HIGH)
+ GPIO.output(self.pinControlSteering,GPIO.HIGH)
+
+ def forward(self, speed):
+ """ pinForward is the forward Pin, so we change its duty
+ cycle according to speed. """
+ self.pwm_backward.ChangeDutyCycle(0)
+ self.pwm_forward.ChangeDutyCycle(speed)
+
+ def forward_left(self, speed):
+ """ pinForward is the forward Pin, so we change its duty
+ cycle according to speed. """
+ self.pwm_backward.ChangeDutyCycle(0)
+ self.pwm_forward.ChangeDutyCycle(speed)
+ self.pwm_right.ChangeDutyCycle(0)
+ self.pwm_left.ChangeDutyCycle(100)
+
+ def forward_right(self, speed):
+ """ pinForward is the forward Pin, so we change its duty
+ cycle according to speed. """
+ self.pwm_backward.ChangeDutyCycle(0)
+ self.pwm_forward.ChangeDutyCycle(speed)
+ self.pwm_left.ChangeDutyCycle(0)
+ self.pwm_right.ChangeDutyCycle(100)
+
+ def backward(self, speed):
+ """ pinBackward is the forward Pin, so we change its duty
+ cycle according to speed. """
+
+ self.pwm_forward.ChangeDutyCycle(0)
+ self.pwm_backward.ChangeDutyCycle(speed)
+
+ def left(self, speed):
+ """ pinForward is the forward Pin, so we change its duty
+ cycle according to speed. """
+ self.pwm_right.ChangeDutyCycle(0)
+ self.pwm_left.ChangeDutyCycle(speed)
+
+ def right(self, speed):
+ """ pinForward is the forward Pin, so we change its duty
+ cycle according to speed. """
+ self.pwm_left.ChangeDutyCycle(0)
+ self.pwm_right.ChangeDutyCycle(speed)
+
+ def stop(self):
+ """ Set the duty cycle of both control pins to zero to stop the motor. """
+
+ self.pwm_forward.ChangeDutyCycle(0)
+ self.pwm_backward.ChangeDutyCycle(0)
+ self.pwm_left.ChangeDutyCycle(0)
+ self.pwm_right.ChangeDutyCycle(0)
+
+def make_app():
+ return tornado.web.Application([
+ (r"/drive",MultipleKeysHandler),(r"/post", PostHandler),
+ (r"/StoreLogEntries",StoreLogEntriesHandler)
+ ])
+
+if __name__ == "__main__":
+ GPIO.setmode(GPIO.BOARD)
+ command_duration = 0.1
+ motor = Motor(16, 18, 22, 19, 21, 23)
+ log_entries = []
+ app = make_app()
+ app.listen(81)
+ tornado.ioloop.IOLoop.current().start()
diff --git a/read_camera_file.py b/read_camera_file.py
new file mode 100644
index 00000000..cff605ed
--- /dev/null
+++ b/read_camera_file.py
@@ -0,0 +1,23 @@
+import numpy as np
+import cv2
+import argparse
+
+# example: python read_camera_file.py -f /Users/ryanzotti/Documents/repos/OpenCV_examples/output.mov
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-f", "--file", required = True,
+ help = "path to where the face cascade resides")
+args = vars(ap.parse_args())
+mov_file = args["file"]
+
+cap = cv2.VideoCapture(mov_file)
+
+while(cap.isOpened()):
+ ret, frame = cap.read()
+
+ cv2.imshow('frame',frame)
+ if cv2.waitKey(1) & 0xFF == ord('q'):
+ break
+
+cap.release()
+cv2.destroyAllWindows()
\ No newline at end of file
diff --git a/run_dataprep.py b/run_dataprep.py
new file mode 100644
index 00000000..52540adb
--- /dev/null
+++ b/run_dataprep.py
@@ -0,0 +1,10 @@
+from dataprep import data_prep
+import argparse
+
+ap = argparse.ArgumentParser()
+ap.add_argument("-d", "--datapath", required = True,
+ help = "path to where the face cascade resides")
+args = vars(ap.parse_args())
+data_path = args["datapath"]
+
+data_prep(data_path)
\ No newline at end of file
diff --git a/save_streaming_video_data.py b/save_streaming_video_data.py
new file mode 100644
index 00000000..daeef7e8
--- /dev/null
+++ b/save_streaming_video_data.py
@@ -0,0 +1,42 @@
+import cv2
+import urllib.request
+import numpy as np
+from datetime import datetime
+import os
+
+# First log into the raspberry pi, and then do these two things:
+# cd /usr/src/ffmpeg
+# sudo ffserver -f /etc/ff.conf_original & ffmpeg -v quiet -r 5 -s 320x240 -f video4linux2 -i /dev/video0 http://localhost/webcam.ffm
+
+fourcc = cv2.VideoWriter_fourcc(*'jpeg')
+out = cv2.VideoWriter('output.mov',fourcc, 20.0, (320,240))
+file_path = str(os.path.dirname(os.path.realpath(__file__)))+"/video_timestamps.txt"
+stream = urllib.request.urlopen('http://192.168.0.35/webcam.mjpeg')
+bytes = bytes()
+while True:
+ bytes += stream.read(1024)
+ a = bytes.find(b'\xff\xd8')
+ b = bytes.find(b'\xff\xd9')
+ if a != -1 and b != -1:
+ jpg = bytes[a:b+2]
+ bytes = bytes[b+2:]
+ frame = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR)
+ #cv2.imshow('Car Camera', frame)
+ now = datetime.now()
+ print(now)
+ if frame is not None:
+ cv2.imshow("car camera", frame)
+
+ # Use the code below if I need find the dimensions of the video
+ '''
+ height, width, channels = frame.shape
+ print(height)
+ print(width)
+ '''
+ out.write(frame)
+ timestamp = datetime.now()
+ with open(file_path,"a") as writer:
+ writer.write(str(timestamp)+"\n")
+
+ if cv2.waitKey(1) == 27:
+ exit(0)
\ No newline at end of file
diff --git a/stream_mjpeg_video.py b/stream_mjpeg_video.py
new file mode 100644
index 00000000..e30ca596
--- /dev/null
+++ b/stream_mjpeg_video.py
@@ -0,0 +1,21 @@
+import cv2
+import urllib.request
+import numpy as np
+
+# First log into the raspberry pi, and then do these two things:
+# cd /usr/src/ffmpeg
+# sudo ffserver -f /etc/ff.conf_original & ffmpeg -v quiet -r 5 -s 320x240 -f video4linux2 -i /dev/video0 http://localhost/webcam.ffm
+
+stream = urllib.request.urlopen('http://192.168.0.35/webcam.mjpeg')
+bytes = bytes()
+while True:
+ bytes += stream.read(1024)
+ a = bytes.find(b'\xff\xd8')
+ b = bytes.find(b'\xff\xd9')
+ if a != -1 and b != -1:
+ jpg = bytes[a:b+2]
+ bytes = bytes[b+2:]
+ i = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_COLOR)
+ cv2.imshow('i', i)
+ if cv2.waitKey(1) == 27:
+ exit(0)
\ No newline at end of file
diff --git a/train_conv_net.py b/train_conv_net.py
new file mode 100644
index 00000000..9617ee27
--- /dev/null
+++ b/train_conv_net.py
@@ -0,0 +1,110 @@
+import tensorflow as tf
+import numpy as np
+
+'''
+
+Helpful notes
+
+- Excellent source explaining convoluted neural networks:
+ http://cs231n.github.io/convolutional-networks/
+- Output size of a conv layer is computed by (W−F+2P)/S+1
+ W = input volumne size
+ F = field size of conv neuron
+ S = stride size
+ P = zero padding size
+
+(240-6+2)/2=118
+(320-6+2)/2=158
+
+(28-5+2)/2
+
+'''
+
+input_file_path = '/Users/ryanzotti/Documents/repos/Self_Driving_RC_Car/data/final_processed_data.npz'
+npzfile = np.load(input_file_path)
+
+# training data
+train_predictors = npzfile['train_predictors']
+train_targets = npzfile['train_targets']
+
+# validation/test data
+validation_predictors = npzfile['validation_predictors']
+validation_targets = npzfile['validation_targets']
+
+sess = tf.InteractiveSession()
+
+def next_batch(size, predictors, targets):
+ record_count = predictors.shape[0]
+ shuffle_index = np.arange(record_count)
+ np.random.shuffle(shuffle_index)
+ predictors = predictors[shuffle_index]
+ targets = targets[shuffle_index]
+ return predictors[:size], targets[:size]
+
+def weight_variable(shape):
+ initial = tf.truncated_normal(shape, stddev=0.1)
+ return tf.Variable(initial)
+
+def bias_variable(shape):
+ initial = tf.constant(0.1, shape=shape)
+ return tf.Variable(initial)
+
+def conv2d(x, W):
+ return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
+
+def max_pool_2x2(x):
+ return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
+ strides=[1, 2, 2, 1], padding='SAME')
+
+x = tf.placeholder(tf.float32, shape=[None, 240, 320, 3])
+y_ = tf.placeholder(tf.float32, shape=[None, 3])
+
+W_conv1 = weight_variable([6, 6, 3, 4])
+b_conv1 = bias_variable([4])
+h_conv1 = tf.nn.relu(conv2d(x, W_conv1) + b_conv1)
+h_pool1 = max_pool_2x2(h_conv1)
+
+W_conv2 = weight_variable([6, 6, 4, 4])
+b_conv2 = bias_variable([4])
+h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
+h_pool2 = max_pool_2x2(h_conv2)
+
+W_conv3 = weight_variable([6, 6, 4, 4])
+b_conv3 = bias_variable([4])
+h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)
+h_pool3 = max_pool_2x2(h_conv3)
+
+W_conv4 = weight_variable([6, 6, 4, 4])
+b_conv4 = bias_variable([4])
+h_conv4 = tf.nn.relu(conv2d(h_pool3, W_conv4) + b_conv4)
+h_pool4 = max_pool_2x2(h_conv4)
+
+W_fc1 = weight_variable([15 * 20 * 4, 4])
+b_fc1 = bias_variable([4])
+
+h_pool4_flat = tf.reshape(h_pool4, [-1, 15 * 20 * 4])
+h_fc1 = tf.nn.relu(tf.matmul(h_pool4_flat, W_fc1) + b_fc1)
+
+keep_prob = tf.placeholder(tf.float32)
+h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
+
+W_fc2 = weight_variable([4, 3])
+b_fc2 = bias_variable([3])
+
+y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
+
+cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1]))
+train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
+correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
+accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+sess.run(tf.initialize_all_variables())
+for i in range(5):
+ predictors, target = next_batch(50, train_predictors, train_targets)
+ if i%100 == 0:
+ train_accuracy = accuracy.eval(feed_dict={
+ x:predictors, y_: target, keep_prob: 1.0})
+ print("step %d, training accuracy %g"%(i, train_accuracy))
+ train_step.run(feed_dict={x: predictors, y_: target, keep_prob: 0.5})
+
+print("test accuracy %g"%accuracy.eval(feed_dict={
+ x: validation_predictors, y_: validation_targets, keep_prob: 1.0}))
\ No newline at end of file