From cefc6f78939404f9d3323d89c67fc0ace5375906 Mon Sep 17 00:00:00 2001 From: Felix Martin Date: Thu, 15 Oct 2020 16:44:21 -0400 Subject: [PATCH] Add files for qlearning assignment --- qlearning_robot/QLearner.py | 72 ++++ qlearning_robot/grade_robot_qlearning.py | 387 ++++++++++++++++++ qlearning_robot/testqlearner.py | 223 ++++++++++ qlearning_robot/testworlds/world01.csv | 10 + qlearning_robot/testworlds/world02.csv | 10 + qlearning_robot/testworlds/world03.csv | 10 + qlearning_robot/testworlds/world04.csv | 10 + qlearning_robot/testworlds/world05.csv | 10 + qlearning_robot/testworlds/world06.csv | 10 + qlearning_robot/testworlds/world07.csv | 10 + qlearning_robot/testworlds/world08.csv | 10 + qlearning_robot/testworlds/world09.csv | 10 + qlearning_robot/testworlds/world10.csv | 10 + ...hing.zip => 19Fall_optimize_something.zip} | Bin zips/20Spring_qlearning_robot.zip | Bin 0 -> 11498 bytes 15 files changed, 782 insertions(+) create mode 100644 qlearning_robot/QLearner.py create mode 100644 qlearning_robot/grade_robot_qlearning.py create mode 100644 qlearning_robot/testqlearner.py create mode 100644 qlearning_robot/testworlds/world01.csv create mode 100644 qlearning_robot/testworlds/world02.csv create mode 100644 qlearning_robot/testworlds/world03.csv create mode 100644 qlearning_robot/testworlds/world04.csv create mode 100644 qlearning_robot/testworlds/world05.csv create mode 100644 qlearning_robot/testworlds/world06.csv create mode 100644 qlearning_robot/testworlds/world07.csv create mode 100644 qlearning_robot/testworlds/world08.csv create mode 100644 qlearning_robot/testworlds/world09.csv create mode 100644 qlearning_robot/testworlds/world10.csv rename zips/{19fall_optimize_something.zip => 19Fall_optimize_something.zip} (100%) create mode 100644 zips/20Spring_qlearning_robot.zip diff --git a/qlearning_robot/QLearner.py b/qlearning_robot/QLearner.py new file mode 100644 index 0000000..dbae5f8 --- /dev/null +++ b/qlearning_robot/QLearner.py @@ -0,0 +1,72 @@ +""" +Template for implementing QLearner (c) 2015 Tucker Balch + +Copyright 2018, Georgia Institute of Technology (Georgia Tech) +Atlanta, Georgia 30332 +All Rights Reserved + +Template code for CS 4646/7646 + +Georgia Tech asserts copyright ownership of this template and all derivative +works, including solutions to the projects assigned in this course. Students +and other users of this template code are advised not to share it with others +or to make it available on publicly viewable websites including repositories +such as github and gitlab. This copyright statement should not be removed +or edited. + +We do grant permission to share solutions privately with non-students such +as potential employers. However, sharing with other current or future +students of CS 7646 is prohibited and subject to being investigated as a +GT honor code violation. + +-----do not edit anything above this line--- + +Student Name: Tucker Balch (replace with your name) +GT User ID: tb34 (replace with your User ID) +GT ID: 900897987 (replace with your GT ID) +""" + +import numpy as np +import random as rand + +class QLearner(object): + + def __init__(self, \ + num_states=100, \ + num_actions = 4, \ + alpha = 0.2, \ + gamma = 0.9, \ + rar = 0.5, \ + radr = 0.99, \ + dyna = 0, \ + verbose = False): + + self.verbose = verbose + self.num_actions = num_actions + self.s = 0 + self.a = 0 + + def querysetstate(self, s): + """ + @summary: Update the state without updating the Q-table + @param s: The new state + @returns: The selected action + """ + self.s = s + action = rand.randint(0, self.num_actions-1) + if self.verbose: print(f"s = {s}, a = {action}") + return action + + def query(self,s_prime,r): + """ + @summary: Update the Q table and return an action + @param s_prime: The new state + @param r: The reward + @returns: The selected action + """ + action = rand.randint(0, self.num_actions-1) + if self.verbose: print(f"s = {s_prime}, a = {action}, r={r}") + return action + +if __name__=="__main__": + print("Remember Q from Star Trek? Well, this isn't him") diff --git a/qlearning_robot/grade_robot_qlearning.py b/qlearning_robot/grade_robot_qlearning.py new file mode 100644 index 0000000..cb6c6ae --- /dev/null +++ b/qlearning_robot/grade_robot_qlearning.py @@ -0,0 +1,387 @@ +"""MC3-P2: Q-learning & Dyna - grading script. + +Usage: +- Switch to a student feedback directory first (will write "points.txt" and "comments.txt" in pwd). +- Run this script with both ml4t/ and student solution in PYTHONPATH, e.g.: + PYTHONPATH=ml4t:MC1-P2/jdoe7 python ml4t/mc2_p1_grading/grade_marketsim.py + +Copyright 2018, Georgia Institute of Technology (Georgia Tech) +Atlanta, Georgia 30332 +All Rights Reserved + +Template code for CS 4646/7646 + +Georgia Tech asserts copyright ownership of this template and all derivative +works, including solutions to the projects assigned in this course. Students +and other users of this template code are advised not to share it with others +or to make it available on publicly viewable websites including repositories +such as github and gitlab. This copyright statement should not be removed +or edited. + +We do grant permission to share solutions privately with non-students such +as potential employers. However, sharing with other current or future +students of CS 7646 is prohibited and subject to being investigated as a +GT honor code violation. + +-----do not edit anything above this line--- + +Student Name: Tucker Balch (replace with your name) +GT User ID: tb34 (replace with your User ID) +GT ID: 900897987 (replace with your GT ID) + +""" + +import pytest +from grading.grading import grader, GradeResult, run_with_timeout, IncorrectOutput + +import os +import sys +import traceback as tb + +import datetime as dt + +import random + +import numpy as np +import pandas as pd +from collections import namedtuple + +import util + +# Student modules to import +main_code = "QLearner" # module name to import + +robot_qlearning_testing_seed=1490652871 +QLearningTestCase = namedtuple('QLearning', ['description', 'group','world_file','best_reward','median_reward','max_time','points']) +qlearning_test_cases = [ + QLearningTestCase( + description="World 1", + group='nodyna', + world_file='world01.csv', + best_reward=-17, + median_reward=-29.5, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 2", + group='nodyna', + world_file='world02.csv', + best_reward=-14, + median_reward=-19, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 4", + group='nodyna', + world_file='world04.csv', + best_reward=-24, + median_reward=-33, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 6", + group='nodyna', + world_file='world06.csv', + best_reward=-16, + median_reward=-23.5, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 7", + group='nodyna', + world_file='world07.csv', + best_reward=-14, + median_reward=-26, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 8", + group='nodyna', + world_file='world08.csv', + best_reward=-14, + median_reward=-19, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 9", + group='nodyna', + world_file='world09.csv', + best_reward=-15, + median_reward=-20, + max_time=2, + points=9.5 + ), + QLearningTestCase( + description="World 10", + group='nodyna', + world_file='world10.csv', + best_reward=-28, + median_reward=-42, + max_time=2, + points=9.5 + ), + # Dyna test cases + QLearningTestCase( + description="World 1, dyna=200", + group='dyna', + world_file='world01.csv', + best_reward=-12, + median_reward=-29.5, + max_time=10, + points=2.5 + ), + QLearningTestCase( + description="World 2, dyna=200", + group='dyna', + world_file='world02.csv', + best_reward=-14, + median_reward=-19, + max_time=10, + points=2.5 + ), + QLearningTestCase( + description="Author check", + group='author', + world_file='world01.csv', + best_reward=0, + median_reward=0, + max_time=10, + points=0 + ), +] + +max_points = 100.0 +html_pre_block = True # surround comments with HTML 
 tag (for T-Square comments field)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# Test functon(s)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+@pytest.mark.parametrize("description,group,world_file,best_reward,median_reward,max_time,points", qlearning_test_cases)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def test_qlearning(description, group, world_file, best_reward, median_reward, max_time, points, grader):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    points_earned = 0.0  # initialize points for this test case  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    try:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        incorrect = True  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if not 'QLearner' in globals():  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            import importlib  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            m = importlib.import_module('QLearner')  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            globals()['QLearner'] = m  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        # Unpack test case  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        world = np.array([list(map(float,s.strip().split(','))) for s in util.get_robot_world_file(world_file).readlines()])  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        student_reward = None  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        student_author = None  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        msgs = []  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if group=='nodyna':  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            def timeoutwrapper_nodyna():  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # Note: the following will NOT be commented durring final grading  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # np.random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                learner = QLearner.QLearner(num_states=100,\
+                                            num_actions = 4, \
+                                            alpha = 0.2, \
+                                            gamma = 0.9, \
+                                            rar = 0.98, \
+                                            radr = 0.999, \
+                                            dyna = 0, \
+                                            verbose=False)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                return qltest(worldmap=world,iterations=500,max_steps=10000,learner=learner,verbose=False)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            student_reward = run_with_timeout(timeoutwrapper_nodyna,max_time,(),{})  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            incorrect = False  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if student_reward < 1.5*median_reward:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                incorrect = True  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                msgs.append("   Reward too low, expected %s, found %s"%(median_reward,student_reward))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        elif group=='dyna':  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            def timeoutwrapper_dyna():  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # Note: the following will NOT be commented durring final grading  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # np.random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                learner = QLearner.QLearner(num_states=100,\
+                                            num_actions = 4, \
+                                            alpha = 0.2, \
+                                            gamma = 0.9, \
+                                            rar = 0.5, \
+                                            radr = 0.99, \
+                                            dyna = 200, \
+                                            verbose=False)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                return qltest(worldmap=world,iterations=50,max_steps=10000,learner=learner,verbose=False)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            student_reward = run_with_timeout(timeoutwrapper_dyna,max_time,(),{})  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            incorrect = False  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if student_reward < 1.5*median_reward:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                incorrect = True  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                msgs.append("   Reward too low, expected %s, found %s"%(median_reward,student_reward))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        elif group=='author':  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            points_earned = -20  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            def timeoutwrapper_author():  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # Note: the following will NOT be commented durring final grading  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                # np.random.seed(robot_qlearning_testing_seed)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                learner = QLearner.QLearner(num_states=100,\
+                                            num_actions = 4, \
+                                            alpha = 0.2, \
+                                            gamma = 0.9, \
+                                            rar = 0.98, \
+                                            radr = 0.999, \
+                                            dyna = 0, \
+                                            verbose=False)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                return learner.author()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            student_author = run_with_timeout(timeoutwrapper_author,max_time,(),{})  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            student_reward = best_reward+1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            incorrect = False  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if (student_author is None) or (student_author=='tb34'):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                incorrect = True  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                msgs.append("   author() method not implemented correctly. Found {}".format(student_author))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            else:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                points_earned = points  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if (not incorrect):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            points_earned += points  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if incorrect:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            inputs_str = "    group: {}\n" \
+                         "    world_file: {}\n"\
+                         "    median_reward: {}\n".format(group, world_file, median_reward)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            raise IncorrectOutput("Test failed on one or more output criteria.\n  Inputs:\n{}\n  Failures:\n{}".format(inputs_str, "\n".join(msgs)))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    except Exception as e:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        # Test result: failed  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        msg = "Test case description: {}\n".format(description)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        # Generate a filtered stacktrace, only showing erroneous lines in student file(s)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        tb_list = tb.extract_tb(sys.exc_info()[2])  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        for i in range(len(tb_list)):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            row = tb_list[i]  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            tb_list[i] = (os.path.basename(row[0]), row[1], row[2], row[3])  # show only filename instead of long absolute path  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        tb_list = [row for row in tb_list if row[0] in ['QLearner.py','StrategyLearner.py']]  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if tb_list:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            msg += "Traceback:\n"  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            msg += ''.join(tb.format_list(tb_list))  # contains newlines  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        elif 'grading_traceback' in dir(e):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            msg += "Traceback:\n"  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            msg += ''.join(tb.format_list(e.grading_traceback))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        msg += "{}: {}".format(e.__class__.__name__, str(e))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        # Report failure result to grader, with stacktrace  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        grader.add_result(GradeResult(outcome='failed', points=points_earned, msg=msg))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        raise  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    else:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        # Test result: passed (no exceptions)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        grader.add_result(GradeResult(outcome='passed', points=points_earned, msg=None))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def getrobotpos(data):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    R = -999  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    C = -999  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    for row in range(0, data.shape[0]):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        for col in range(0, data.shape[1]):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if data[row,col] == 2:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                C = col  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                R = row  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if (R+C)<0:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        print("warning: start location not defined")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return R, C  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# find where the goal is in the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def getgoalpos(data):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    R = -999  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    C = -999  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    for row in range(0, data.shape[0]):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        for col in range(0, data.shape[1]):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if data[row,col] == 3:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                C = col  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                R = row  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if (R+C)<0:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        print("warning: goal location not defined")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return (R, C)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# move the robot and report reward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def movebot(data,oldpos,a):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    randomrate = 0.20 # how often do we move randomly  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    quicksandreward = -100 # penalty for stepping on quicksand  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # decide if we're going to ignore the action and  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # choose a random one instead  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if random.uniform(0.0, 1.0) <= randomrate: # going rogue  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        a = random.randint(0,3) # choose the random direction  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # update the test location  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if a == 0: #north  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr = testr - 1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif a == 1: #east  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testc = testc + 1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif a == 2: #south  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr = testr + 1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif a == 3: #west  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testc = testc - 1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    reward = -1 # default reward is negative one  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # see if it is legal. if not, revert  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if testr < 0: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif testr >= data.shape[0]: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif testc < 0: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif testc >= data.shape[1]: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 1: # it is an obstacle  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 5: # it is quicksand  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = quicksandreward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data[testr, testc] = 6 # mark the event  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 6: # it is still quicksand  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = quicksandreward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data[testr, testc] = 6 # mark the event  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 3:  # it is the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = 1 # for reaching the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return (testr, testc), reward #return the new, legal location  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# convert the location to a single integer  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def discretize(pos):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return pos[0]*10 + pos[1]  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def qltest(worldmap, iterations, max_steps, learner, verbose):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# each iteration involves one trip to the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    startpos = getrobotpos(worldmap) #find where the robot starts  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    goalpos = getgoalpos(worldmap) #find where the goal is  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # max_reward = -float('inf')  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    all_rewards = list()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    for iteration in range(1,iterations+1):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        total_reward = 0  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data = worldmap.copy()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        robopos = startpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        state = discretize(robopos) #convert the location to a state  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        action = learner.querysetstate(state) #set the state and get first action  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        count = 0  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        while (robopos != goalpos) & (count= data.shape[0]: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif testc < 0: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif testc >= data.shape[1]: # off the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 1: # it is an obstacle  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        testr, testc = oldpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 5: # it is quicksand  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = quicksandreward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data[testr, testc] = 6 # mark the event  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 6: # it is still quicksand  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = quicksandreward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data[testr, testc] = 6 # mark the event  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    elif data[testr, testc] == 3:  # it is the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        reward = 1 # for reaching the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return (testr, testc), reward #return the new, legal location  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# convert the location to a single integer  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def discretize(pos):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return pos[0]*10 + pos[1]  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def test(map, epochs, learner, verbose):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# each epoch involves one trip to the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    startpos = getrobotpos(map) #find where the robot starts  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    goalpos = getgoalpos(map) #find where the goal is  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    scores = np.zeros((epochs,1))  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    for epoch in range(1,epochs+1):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        total_reward = 0  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        data = map.copy()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        robopos = startpos  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        state = discretize(robopos) #convert the location to a state  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        action = learner.querysetstate(state) #set the state and get first action  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        count = 0  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        while (robopos != goalpos) & (count<10000):  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            #move to new location according to action and then get a new action  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            newpos, stepreward = movebot(data,robopos,action)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if newpos == goalpos:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                r = 1 # reward for reaching the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            else:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                r = stepreward # negative reward for not being at the goal  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            state = discretize(newpos)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            action = learner.query(state,r)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if data[robopos] != 6:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                data[robopos] = 4 # mark where we've been for map printing  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            if data[newpos] != 6:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+                data[newpos] = 2 # move to new location  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            robopos = newpos # update the location  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            #if verbose: time.sleep(1)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            total_reward += stepreward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            count = count + 1  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if count == 100000:  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+            print("timeout")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if verbose: printmap(data)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        if verbose: print(f"{epoch}, {total_reward}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+        scores[epoch-1,0] = total_reward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    return np.median(scores)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+# run the code to test a learner  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+def test_code():  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    verbose = True # print lots of debug stuff if True  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    # read in the map  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    filename = 'testworlds/world01.csv'  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    inf = open(filename)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    data = np.array([list(map(float,s.strip().split(','))) for s in inf.readlines()])  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    originalmap = data.copy() #make a copy so we can revert to the original map later  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    if verbose: printmap(data)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    rand.seed(5)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    ######## run non-dyna test ########  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    learner = ql.QLearner(num_states=100,\
+        num_actions = 4, \
+        alpha = 0.2, \
+        gamma = 0.9, \
+        rar = 0.98, \
+        radr = 0.999, \
+        dyna = 0, \
+        verbose=False) #initialize the learner  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    epochs = 500  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    total_reward = test(data, epochs, learner, verbose)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print(f"{epochs}, median total_reward {total_reward}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    non_dyna_score = total_reward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    ######## run dyna test ########  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    learner = ql.QLearner(num_states=100,\
+        num_actions = 4, \
+        alpha = 0.2, \
+        gamma = 0.9, \
+        rar = 0.5, \
+        radr = 0.99, \
+        dyna = 200, \
+        verbose=False) #initialize the learner  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    epochs = 50  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    data = originalmap.copy()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    total_reward = test(data, epochs, learner, verbose)  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print(f"{epochs}, median total_reward {total_reward}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    dyna_score = total_reward  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print(f"results for {filename}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print(f"non_dyna_score: {non_dyna_score}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    print(f"dyna_score    : {dyna_score}")  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+if __name__=="__main__":  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
+    test_code()  		  	   		     			  		 			     			  	  		 	  	 		 			  		  			
diff --git a/qlearning_robot/testworlds/world01.csv b/qlearning_robot/testworlds/world01.csv
new file mode 100644
index 0000000..d56d367
--- /dev/null
+++ b/qlearning_robot/testworlds/world01.csv
@@ -0,0 +1,10 @@
+3,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,1,1,1,1,1,0,0,0
+0,5,1,0,0,0,1,0,0,0
+0,5,1,0,0,0,1,0,0,0
+0,0,1,0,0,0,1,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,2,0,0,0,0,0
diff --git a/qlearning_robot/testworlds/world02.csv b/qlearning_robot/testworlds/world02.csv
new file mode 100644
index 0000000..fc2833f
--- /dev/null
+++ b/qlearning_robot/testworlds/world02.csv
@@ -0,0 +1,10 @@
+0,1,0,1,0,0,0,0,0,0
+0,1,0,1,0,0,0,0,0,0
+0,1,0,0,0,0,0,0,0,0
+0,1,0,1,1,1,1,1,1,1
+2,1,0,1,0,0,0,0,0,0
+0,1,0,1,0,0,1,0,0,3
+0,0,0,1,0,0,1,0,0,0
+0,1,0,0,0,0,1,1,1,1
+0,1,0,1,0,0,0,0,0,0
+0,0,0,1,0,0,0,0,0,0
diff --git a/qlearning_robot/testworlds/world03.csv b/qlearning_robot/testworlds/world03.csv
new file mode 100644
index 0000000..924da5f
--- /dev/null
+++ b/qlearning_robot/testworlds/world03.csv
@@ -0,0 +1,10 @@
+0,0,0,1,0,0,0,1,0,3
+0,1,0,1,0,1,0,1,0,0
+0,1,0,1,0,1,0,1,0,1
+0,1,0,1,0,1,0,1,0,0
+0,1,0,1,0,1,0,1,1,0
+0,1,0,1,0,1,0,1,0,0
+0,1,0,1,0,1,0,1,0,1
+0,1,0,1,0,1,0,1,0,0
+0,1,0,1,0,1,0,1,1,0
+2,1,0,0,0,1,0,0,0,0
diff --git a/qlearning_robot/testworlds/world04.csv b/qlearning_robot/testworlds/world04.csv
new file mode 100644
index 0000000..88c108f
--- /dev/null
+++ b/qlearning_robot/testworlds/world04.csv
@@ -0,0 +1,10 @@
+0,0,0,0,0,1,0,1,0,3
+0,0,0,0,0,1,0,1,0,0
+0,0,0,1,0,1,0,1,0,1
+0,0,0,1,0,1,0,1,0,0
+0,0,0,1,0,0,0,1,1,0
+2,0,0,1,1,1,0,1,0,0
+0,0,0,1,0,1,0,0,0,1
+0,0,5,0,0,1,0,1,0,0
+0,0,1,1,1,1,0,1,1,0
+0,0,0,0,0,1,0,0,0,0
diff --git a/qlearning_robot/testworlds/world05.csv b/qlearning_robot/testworlds/world05.csv
new file mode 100644
index 0000000..da714ff
--- /dev/null
+++ b/qlearning_robot/testworlds/world05.csv
@@ -0,0 +1,10 @@
+0,1,0,0,0,1,0,1,0,3
+1,0,0,0,0,0,1,0,0,0
+0,0,1,1,0,1,0,1,0,1
+1,1,0,0,0,0,1,0,0,0
+0,0,0,0,0,0,0,0,1,0
+0,1,0,1,1,0,0,1,0,0
+1,0,0,0,0,1,0,0,0,1
+0,0,0,0,0,0,0,1,0,0
+1,0,1,0,0,1,0,0,1,0
+2,0,0,0,0,0,1,0,0,0
diff --git a/qlearning_robot/testworlds/world06.csv b/qlearning_robot/testworlds/world06.csv
new file mode 100644
index 0000000..e299798
--- /dev/null
+++ b/qlearning_robot/testworlds/world06.csv
@@ -0,0 +1,10 @@
+0,1,0,0,0,1,0,1,0,2
+1,0,0,0,0,0,1,0,0,0
+0,0,1,1,0,1,0,1,0,1
+1,1,0,0,0,0,1,0,0,0
+0,0,0,0,0,0,0,0,1,0
+0,1,0,1,1,0,0,1,0,0
+1,0,0,0,0,1,0,0,0,1
+0,0,0,0,0,0,0,1,0,0
+1,0,1,0,0,1,0,0,1,0
+3,0,0,0,0,0,1,0,0,0
diff --git a/qlearning_robot/testworlds/world07.csv b/qlearning_robot/testworlds/world07.csv
new file mode 100644
index 0000000..394f4e3
--- /dev/null
+++ b/qlearning_robot/testworlds/world07.csv
@@ -0,0 +1,10 @@
+0,0,0,0,2,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,1,1,1,1,1,0,0,0
+0,0,1,0,3,0,1,0,0,0
+0,0,1,0,0,0,1,0,0,0
+0,0,1,0,0,0,1,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
+0,0,0,0,0,0,0,0,0,0
diff --git a/qlearning_robot/testworlds/world08.csv b/qlearning_robot/testworlds/world08.csv
new file mode 100644
index 0000000..936c865
--- /dev/null
+++ b/qlearning_robot/testworlds/world08.csv
@@ -0,0 +1,10 @@
+0,1,0,1,0,0,0,0,0,0
+0,1,0,1,0,0,0,0,0,0
+0,1,0,0,0,0,0,0,0,0
+0,1,0,1,1,1,1,1,1,1
+3,1,0,1,0,0,0,0,0,0
+0,1,0,1,0,0,1,0,0,2
+0,0,0,1,0,0,1,0,0,0
+0,1,0,0,0,0,1,1,1,1
+0,1,0,1,0,0,0,0,0,0
+0,0,0,1,0,0,0,0,0,0
diff --git a/qlearning_robot/testworlds/world09.csv b/qlearning_robot/testworlds/world09.csv
new file mode 100644
index 0000000..1595036
--- /dev/null
+++ b/qlearning_robot/testworlds/world09.csv
@@ -0,0 +1,10 @@
+0,0,0,0,0,2,0,0,0,0
+0,0,0,1,0,0,1,0,0,0
+0,1,0,1,0,0,1,0,1,0
+0,1,0,1,1,1,1,0,1,0
+0,1,0,0,1,0,0,0,1,0
+0,1,1,1,1,1,1,1,1,0
+0,0,0,0,1,0,0,0,0,0
+0,0,0,0,1,0,0,1,0,0
+0,0,0,0,1,0,0,1,0,0
+0,0,0,0,1,3,0,1,0,0
diff --git a/qlearning_robot/testworlds/world10.csv b/qlearning_robot/testworlds/world10.csv
new file mode 100644
index 0000000..69bd052
--- /dev/null
+++ b/qlearning_robot/testworlds/world10.csv
@@ -0,0 +1,10 @@
+0,0,0,0,0,0,0,0,0,0
+0,0,0,1,0,0,1,0,0,0
+0,1,0,1,0,0,1,0,1,0
+0,1,0,1,1,1,1,0,1,0
+0,1,0,0,1,0,0,0,1,0
+0,1,1,1,1,0,1,1,1,0
+0,0,0,0,1,0,0,0,0,0
+0,0,0,0,1,0,0,1,0,0
+0,0,0,0,1,0,0,1,0,0
+0,0,0,2,1,3,0,1,0,0
diff --git a/zips/19fall_optimize_something.zip b/zips/19Fall_optimize_something.zip
similarity index 100%
rename from zips/19fall_optimize_something.zip
rename to zips/19Fall_optimize_something.zip
diff --git a/zips/20Spring_qlearning_robot.zip b/zips/20Spring_qlearning_robot.zip
new file mode 100644
index 0000000000000000000000000000000000000000..02054f18aa83df39ed06726656068567cc4dc903
GIT binary patch
literal 11498
zcmb7~WmFx>)`kxnG`PD42=1;2cXxN!zyU&nySqC9g1ZwuSn%NP8azO-FPWL|-em6F
zxlH%!THQa+e%@WHPOZJ_-3rnWkZ^z>*RU*^!f!8s49Ea{fTNA6p_84Zow>e~y^*~O
zgQ^NF0AkD#N$-Jsri2oeDXae(O*PzF?DwN
zJ-VqAy@UJT1wPWTwO{B*c}RZu`URF-LJ3>wK1j-#elUGaF0ovAi>vqN&uThqT%Wap!Z5t4HX6h4lO(vT#@S
zR`JgLqy=w}7Y6kFZCsa?rI6*Fa#(^EOhtI$V5%m`5Q$(c;;Id6yLTG5r=R+RCi+};
z4M5`b%!QYm53iHMo!)u3txWZ6A8f`I;H^*@i)9;e6LV}oBG37u%xX$8yBb|$Cc+tc
zz;S*RJjLp>qt>&!B!g`SvYG<+08JZWONMM`zmdZarvYPnhu4`=Bcr3uaddDXAuTQt
zNk;9*Ojt0qk
zD=7x(X36Lf?Lxf_IcP+oLV`T&OK)BEi3O%I*0=;q$XR?Knb0^rv$*cLC!pjpJ{nI$
z1CpaCpAMss>YF(pYKuj=@SwznW-yBFMX~h;J9dNf2`SP>OeQLnLPsj1*cx&>hjmjd
zJ0hy0&Y=-{^}Lo7da8@w?Y!&gPMKUX4~!?+DY~V7qLr8&e9p!`&KB#OC8Z=~fqG-<
zB%GYOLAjfF6P^ZcL});|F151%{wiV5Ff{=-TRbLuzeNmU7lM}1FryIZCc2b#>*1FI
z+YXEedMS4=4GD^1#(;j4@FK)qrbNR6e242VL@*sE-TLL7qa#F1eZ@`Q0XU~kxDlB;
zeC-}**X(9%sb5Mx5Vq{i!QYpLUuGXv&nvv&3k@UBHL#P)l{{`N@=$dRDhO1ge(jcR
zYr$uKi!^m*>v;9%XcARKL+@6BBO$lN*qJLzB}O9_o&kCWAyha&U@VAF3Ho5`4uqn!T^loy%~fqqYJimJfh>N;DkU33#$2FN
zxmj?|j>v!mt)2hoDiLwyd7h-SCkPE2aa3$pn3f4=cu`tbo6{QSh=_rfomQtILvlk9
za{e?}LtQDX%O2dHYv07;n>|Py2m0nV*(qg^V(V5bQ$veVQpsMBuGTVf1QZA1z-Hjp
zpV-HlzwkUhLJXF5Uh>32&^Gw2avF|6fijPz9DifY0wDjwGe+3fY-%P|^b^NhROBXv
zJuJgzDn6paspl}r7^4XZ@4=%gsrLvAO-o0#K~%uJ2gl$T|_5R5sWHx8)7
zHK~Sj6Oy;%m9qIbp+<0hSU~;sj1eRm#ZhKPZ$c~w8xzG@nB-*`gXN
z1Ius0#aDW&2>q)p3^%0ACx~PsruD6n;E|HtYoo!(g?QFZOWP&_$ZEV8%yUM(I8yiq
z52k9P=(>!6Lv8cu`t_@{C`mKA8B1DJ&alTw$VxY6Zgd**bv1e$Whl;CSE2j7BzQ}n
znp_;Ia-Bd*MS%fGusBQ5#4I+fk>u%TkUtx18_|b$&-2(3bW~XSBI^D@6KQcFWV^Rg
zV2A?UeF>e_1wctSQ*+ci*XsEmXs^r)zJXXT3~N^Xpv8gz8|D3rf4U`Zk>jQQBjv;kbkV+h^D
zG!VD(w@eZA-3=vKn#?>&rpV{)Q*gOqKWmf-^86BWLdeRt$e76w9=W#C=x+h-6NiZ#
zXm4%FA@{9=683{4J}(L|WVkyH5@c3R%Qw@@_9ax@%&L@_Gtan9r7^X(FJMJ}9!S_h
zL=1bE87U5`^yiKnbGg{~>YIb{8Vib4M6f?C<;yZL)_tCzHn1^sj?RTxbKVnXgG7T7q$&H``
zz(47B+0l!3Gu1}e3B0eS3xiD;YGJ!2bagsgN`OYBB{;0Z4=^Pvn~VQ4vqln@^D@6Q
zmdT!*V9F8+$=g3pwq@d@dii}2D9=KXgN#loejh9NPAS0L8ZfFKUxCK@t6xSuNq)MrFhG?srY;pW%T?9{h!s2s4ZD
zO~rR;Tssi~E}!rhkUeHmKRF7wYvE~cemOW{#zkUj;5J;#yGXBxrYVj~q>v}&uw~y)
zs#LRyRSyV7duO7Dy{63J!g?VIEuxMw_~OOhfB|E$+i*0KVkHSqB4PZifmOCBmA>}@
z6S>43Y&_DRkh6&hd$88H+#J>db7Xd1{c3S!>mCUgWmd4q_;B1?LBj>!LhTMb;c?9J
z^j|`N_uDWU(ntq8cyy#=b^`0GlSW@5modA)#!LWbwpdy5?R+W3za1YB}q
zGSJ4fR(sbQTpqF^g1&0h!!qD*55f*yFx);_+hd1c>Bb!}j6n|z6Y+^u;&kM<_rGB*
zh)xbn4gT3eJ&xZ~ZzPM$u9pxg_yTTr=ff#4WpM!4(uv60RsC
z$nGqpEI$H%6t{m}Cgc5=;?~XH$;QO_A5_fQC|VG--|Of8?+@kQ2mLKJBNx4~v+Lg#
z&Mae0-oj+qVerCuq%p)yBIryar0vK~{GIvm$d8-njZZeY^5c;!jGYhWc7*#=-5xO>D8w7*>NeG1cl`!c)5yJ8W
zApl_!M%bK)CSq1*_6|}riFj_2QV(jDQJCH~gMO$#DZt)`HUCFb!TAIQf1V1;A5*dZ
z_frv3xJJqKPo{$HX+lIqsMxoCLpp#y(ePdDKqd2j(<0+w)eNxrZ|IW=Mf|1c5i$hN
z8C`@wlk7pHnfv#8{b`28M+ZMK{}W-XPY~uU{6iLV2Qtg5@RlDFCX>l4WZeFBX5Sje
z6mtN_D0*01po9nFnn%nL(<=C%6j`TLc`*C~MNEuOP$U?ngFKHM0vUpn8;nC5B1Bw`
ztwW@Pkkf}uW9FSZ0uULF_D)8LHb=mM_>&x&c%&49e;|jE>3_HGTqv250#zHGw(W7!Dep2bch0|2A3
z007lr?78Mnh9;)JpMB{6>n!Ad9ar$($1T)X3EbXefF|;Dtq95XM$A_tK3nd$Q>Iv^
zPqoVwI-fcaqsB((!9YXGtMNUow7`3#Qs^)3IBfB-tq&7Wgf221VpRfW_gvaN9X-yv
zcVq95$kB9mn6s)@sst*WM&xE?&e0j-LlOf*ustte9+G($CdqDZ#;4ufU
zPU3b%QDlV-O&iF~9u%++q~r8*zLZS}U92od-t&LKL7SjOF1Ck}CZ&suNo$%|RgfrF
zDVJknH`w3GM$mU2{|dEEBlwacrN$`b>%!2!f7^Q}v+e4Ob#uY9bXoK}Z4>it>9DqU
zZ7Q0k`GY8rNRN%*wr`UJUcb7O?FX`k%@4sJ&j}KceH%K~LlClWzvEI}r+%qs_Pn1?
zH_QYbt9$lfBngCh>U)4^DAg1k`uV|
z>X$LY&I!eHp-`8Ipt2R*dv(fH=5CVY*H-Q)XKXb4q>=H088i=qw6=wloHet>G>lo^0o6C3pFym(y?!b!XtuO#zvkI3fY6FUPuqz48**)po~}{5`716%$poj75AT3KNQ5
zJU#W!A?qRRMm=~#7E7Aa4f2|>^<>B1DaxVwunyd*2@Q+{C5~?9J&W-yxFsBaikPoE)z2_0|-+kFmC{kK)HZBuTmnqbP&3~oFs0DW_hBczk!3=O2a}jsY+-()FVdk
z|9*9_!9ADttA2ZhjAGr35GmpJUbC0cgBUnec|vuzSxAeeDZJQ*1*_kxL}N-X)@n(B
zU(wMoa-!qZ;df!tZOahWjT5Rrc^|y`Ma~%@y@Cb
zA42w~+py{tEE8ZCj!x0*1KL)Zqinyim}Pvp$N1Rn9|)MZY_ussUbjXRED2#gWwJ5Ea2ESgS`
zFPeTdNh)?*6FfPYC(7#*#wj2X#<^l0&bgw$?BX$;M%XUP%O{Zr^2m0d44znw#H_*!
z1Gz?Ho*oL)t@_&vlwM0El!Z78o1j21uB0s}x95dDpZQQ6z>Y=ad{C8kAQl)`ZV&@juRg
z9B2J%fUB=+7CEW0L)w(n0B4s&WHXk1sAZ$s%m0nmvofSpq$-#L(!Unc+sW0)(0F$Y
zA@^%nN}4DpmR3)k3q9|T9pFH_P|3vd@d0Yubhml7b`UM^EZatYf@mp!6=tSg6l&%*NqFKTx=`WTgz$k!
zS418R9wvcZilN4W+%7KW-HPsH<~&+vXz`J{4P_RTY-mOU0^SInO>>UJg1f(U98z0G
z;2Rp}EHlGg2;2o&FKUYd)l2%grk-9RV*9s2y05p@8E^Q}VBh4avxcvd9Lz0PpXY+}
z)=ATPTn>LEq{mM)wd$&2CbkM;qv!CO$Y7RaD)Vrr6=b&VkGVx}p-%4%6O(Gqqr_0-4!>D
zkwm-b()+N|unRn(qw81?>7r}#I3N@Y)rM;Cs(8IT@9K99E
z;;Mk&*Wzg`dO+U%OjURH_VqTB7*($dPz
zInh*WcT)FVR*m7s=9TsyN88G>cBmE+HV2ta9xo#I$$f(H2vsk!guhHH9e@WIz>xit
zsg7WjVQ>Vi)3w1^o7Jm15JQs;;cX`IYlO~^$ai>%o$^}1Rec<9aldZF&Sw}tQKwfk
zRNYJu?CTX3vtv^3ZopF$pt5FB6c1_Z#s>CA6w?F8_i@RlKJrvgBzK-9di}x5@$yXCLR#?2ref#Vp>vAq}eM$=97;&+&ck
zE4bC7m1K#>LguQHe+nP4+TU|t*YDUP$oM?u4fH;V1CeFs;_T9xutJ_qKuf}Wr0*E+
zl$5F+nwU0{-f>a{g@NxLC7Y=kss#!wQGhv@kRye{s6_$9mjeR
z)!moAQwB1nA3|OqP)x|rP9M@>DX{(Izxv;^5P+_)mOPu@HV+TZP$2BlTgid2$RB*li>OLEwK&AJQkBwrtXhG+@
z9Ph9^vP$`o0IgUJUN-tGmNbQj-
zD7@(twUz4AMZ|~IEX_jAIbAGP4oPNx8&;NPJ<0uFyGT41Jm0(6jV{_*
zcC24rUuAna_q=1y_W*Ea%X7`lM*0zJmo^~oLkm+LNFUeug9>vKHQ0Ldh$}Nb@xJgT
zFU!k@#qQwT4n={nvr7mh5i4EBCg3~pBQI(X+-`EEP`vt4r!A4fmp&}G?~i8tb&65N
zrS40R9D>^jvGO{+jkWFAg%hL?$DaAY7=2{Ky_h?96j7)5!TwmK#?qBanApV!1rq+i
z0V98lH(iG1Fx+xQn(IvzETsa~PSqW2F(X^1%c?l(5MF|;-kfa^FOu1nP`P;HKd#br
zWRO+>f4wR8+BG+D`=*4^c1(PX44mo-{7;_p*
zo-10ze3v5XxI6xTSVXoetxzC1(T-Q>7}r_9ja(Mor3IKA<41(1k3rp5ZtN0f@9
zZPkRD5X}#9iz+ka2&QWqVDl$UEm-mVubcG}mO>}CI~wD}?yK#X7wVu;5CS*KnvFj3
zT}-=4!Pw1?@Qa}dd2Mx|C?ugg@I5s93;8r#FE~%4RHY?C@}DWa^2Q+zz=CHSBsTp?vho7(a4%xTq^qL-wBApwhXNGa(Zw2-J`
z?)$~)_x(=vB&DCbOdom5b;YvoBcU)?&g=9k$K6T0PIsF8NPfpURh{jLVk7CaM7+mLipVzJ*1Vspn+|R$N55&qr3X
z+MGAe*C{w}QfS66!ffT}2~Zb_#z*4ji9!wL-5b8ZwVEHN?w@2vH_+$Wx#VWrQxk5r
z8uHeASZm|T1r&}OmL%n{pN_$lYa{uvUgvlpPQybZ+EG#2!va_f^nl*3iy<3O
zN$&V=!R!ZR4ZuWqhb68~zHsA~*UXWyiLdVAf9>udxm@FBg#-YKumOO-IsjLc`O!D>
zKRY{?)NSoaIgo#JcA)6Pq2_FE(SyNWZoRL6p_&lu(9H0nthEK=pwHUgmH|feYK6xb
z!e`M-TYa+`wKBZS-D~@_apnLlJ!AaYy8Sg5{IR+CV_vYntMsY!*CqS|G1#a|MojWH
zQcilj`IX-&!Rkn8!#gl)ogT
z2%siYS?24wTsUYDSR9+#yXPsTg%P*RyTI^#oy@j2t6FUcyq4lVY4P@_7g89Mu63el
z)iE?TpYG#tCXvuBs^64z606h0l{G_55w|G4!Gm{-(Mgb&#Yp7-rkXs>jo;~KFTF1^
zGlvs78Z8S?XWHwP;;o=)xTjp6Po`dbdRSPZLB&^EP>~z@Wf|E);uRgCpIH^EQizUX
zo|VPC&VaD30fi%hzx8m&#qM)v%PJ;v7BwR`a~-@6aA~;8PY|fP3HZ2R!^TXm0H;i)
z-W~}{rm|4U!I-;Z*-+?$Rs+eZ#JtI?33304jtSHzA;0=Wf5rEsgs7tycEUr%pJ@$%
zcwS#z8xaSN5cH}Id9%cY!62q>U;@alyGtDdT(NnLba;)?L#MUq(FjEDTqd4%&>-)MR+MID@|Z#X_W8RK+p%8OsZFO}lb31~I5UZFh=Gi0^
zTabVqCl52!*LVBhtP|_;VzG^R;!dw!omiUMZc?i%3tNiyd~s@Pz*Ds&F0&V!=NwA*
zC=;ZNIwz(|<5uK|EORVOW}~*f&^XN*`Pz(I3wxkUQl=)%HCJ-95Z076R)a$00*t3V
z(f4g6z03_3A4DieO7gd&5&FJEohh`Uzar?36RE;rhgrdgAEL^@KceH{w&08(kBP+#
zB7cyj3aX$eyK`U{AALAlj(sms-o5rA-%R6%vrH;R!Zergx=^&maTj8-A!0Q)o0{K}
zlY?KNDbZuV?%hca^$mZ=Rop6w^QFu|&K4B^ko(@3#h^9blzTA^MZY}faX^C%D)l62%_rBTT9ns<&kv+uDUxnQ2eZ9ZwB~DRn;xsCw*2n7K
z&;3}%*EA}!TJIH_C~!4_6Nw?Oz8+tSJlA0wABG$BRhu+`Cpcr&d5pdnwx+eNwZJbq
z`!O6n7X$B7TL(VPsD6RwaT|MwZ9)XH13nhpH-m5Pkx5;GF#Dz}#k59~VaST)RpRAD
zB~ZrUn9#b^x!H2Lvuk2$yK{1ux8eGl>@~0;@U)sf<6&8Z;sA2s0ZTy|44e?+ue~^A
z!1w*=w+k6?FZAQ(Ux&8;YG8xUw6%=UMX
zzm|^oi}V-p|0(?kG|}
z{b~*V%nE-B>kp&wX{=w}zn@tch)>At4=?a(tY3Y;pIJ#yVg2FyJ&pCNmG?7?`1up^
z`oq|J8tYeg?q^ohQ&@j^bx&jc>bw2Sl1BR9dHuf&_cYe8rrOV}`6sac-(q_j>sJr$
zXO{YR1MAQG+)tau@Bgnq9JQyheznDZW}W;6>%W_1Pow=>O#V!x`fj`bIkR6%%l`&|
Tg8s4jBYnTgzQ0vb{`mSo`;t*4

literal 0
HcmV?d00001