Merge pull request #17 from rufo123/experiment_minimap

Added Dynamic Reward/State Configuration

Author: rufo123

action_translator_enum.py

@@ -46,6 +46,7 @@ def take_action(self, par_controls: Controls,
         Returns:
           - The integer value of the action taken.
           """
+        print(par_sleep_time)
         executed_correctly: bool = False
         action = self.value
         par_controls.release_all_keys()

agents/ppo.py

@@ -17,9 +17,7 @@
 from torch.multiprocessing import Process, Pipe
 
 import graph.make_graph
-from car_states.enabled_game_api_values import EnabledGameApiValues
-from envs.strategy.reward.a_reward_strategy import ARewardStrategy
-from envs.strategy.state_calc.a_state_calc_strategy import AStateCalculationStrategy
+from configuration.i_configuration import IConfiguration
 from game_inputs import GameInputs
 from utils.print_utils.printer import Printer
 from utils.stats import MovingAverageScore, write_to_file, append_to_file
@@ -82,7 +80,7 @@ def scalar_to_support(par_x, par_support_size):
 # pylint: disable=too-many-statements
 # noinspection DuplicatedCode
 def worker(connection, env_param, env_func, count_of_iterations, count_of_envs,
-           count_of_steps, gamma, gae_lambda) -> None:
+           count_of_steps, gamma, gae_lambda, start_iteration_number) -> None:
     """
     worker function for Proximal Policy Optimization (PPO) agent training.
 
@@ -100,7 +98,9 @@ def worker(connection, env_param, env_func, count_of_iterations, count_of_envs,
     None.
     """
     envs = [env_func(*env_param) for _ in range(count_of_envs)]
-    observations = torch.stack([torch.from_numpy(env.reset()) for env in envs])
+    observations = torch.stack([torch.from_numpy(
+        env.reset(start_iteration_number)
+    ) for env in envs])
     game_score = np.zeros(count_of_envs)
     steps_taken_storage = np.zeros(count_of_steps)
 
@@ -141,7 +141,7 @@ def worker(connection, env_param, env_func, count_of_iterations, count_of_envs,
                     game_score[idx] = 0
                     steps_taken_storage[idx] = steps_took_to_complete
                     steps_taken_list.append(steps_taken_storage[idx])
-                    observation = envs[idx].reset()
+                    observation = envs[idx].reset(start_iteration_number + iteration)
                 # observations[idx] = observation.clone().detach()
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore")
@@ -219,9 +219,7 @@ def __init__(self, model, optimizer, gamma=0.997, epsilon=0.1,
     # pylint: disable=too-many-locals
     # pylint: disable=too-many-statements
     def train(self,
-              env_param: tuple[GameInputs, ARewardStrategy,
-              AStateCalculationStrategy, EnabledGameApiValues
-              ],
+              env_param: tuple[GameInputs, IConfiguration],
               env_func, count_of_actions,
               count_of_iterations=10000, count_of_processes=2,
               count_of_envs=16, count_of_steps=128, count_of_epochs=4,
@@ -267,7 +265,8 @@ def train(self,
             parr_connection, child_connection = Pipe()
             process = Process(target=worker, args=(
                 child_connection, env_param, env_func, count_of_iterations,
-                count_of_envs, count_of_steps, self.gamma, self.gae_lambda))
+                count_of_envs, count_of_steps, self.gamma, self.gae_lambda,
+                self.start_iteration_value))
             connections.append(parr_connection)
             processes.append(process)
             process.start()

car_states/car_state_in_environment.py

@@ -81,7 +81,7 @@ def assign_values(self, par_lap_progress_difference: float = -1,
              car.
             par_revolutions_per_minute (float): The number of revolutions of the car's engine per
              minute.
-            par_wrong_way_indicator (int): A binary indicator (0 or 1) that shows if the car is
+            par_wrong_way_indicator (float): A binary indicator (0 or 1) that shows if the car is
              going the wrong way.
             par_mini_map (numpy.ndarray): A 2D array that represents the mini-map of the track.
             par_car_state (CarState): An instance of the parent class `CarState` to copy values

configuration/experiments/fifth_experiment_removed_progress_reward.py

@@ -44,7 +44,7 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Return a reward strategy object for the fifth experiment.
 
@@ -55,7 +55,7 @@ def return_reward_strategy(self) -> ARewardStrategy:
             RewardStrategyEnum.THIRD_REWARD_STRATEGY
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Return a state calculation strategy object for the fifth experiment.
 
@@ -94,3 +94,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_partial_terminal_lap_smaller_reward"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 3
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/experiments/first_experiment_small_state.py

@@ -44,7 +44,7 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
          Return a reward strategy object for the first experiment.
 
@@ -55,7 +55,7 @@ def return_reward_strategy(self) -> ARewardStrategy:
             RewardStrategyEnum.FIRST_REWARD_STRATEGY
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Return a state calculation strategy object for the first experiment.
 
@@ -94,3 +94,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_first_small_state"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 3
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/experiments/fourth_experiment_lap_terminal_scaled_reward.py

@@ -44,7 +44,7 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Returns an instance of `ARewardStrategy` that is used to calculate the rewards in the
          reinforcement learning algorithm for the fourth experiment.
@@ -56,7 +56,7 @@ def return_reward_strategy(self) -> ARewardStrategy:
             RewardStrategyEnum.SECOND_REWARD_STRATEGY
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Returns an instance of `AStateCalculationStrategy` that is used to calculate the state
          representation for the reinforcement learning algorithm for the fourth experiment.
@@ -96,3 +96,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_partial_terminal_lap"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 3
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/experiments/second_experiment_bigger_state.py

@@ -45,7 +45,7 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Returns an instance of `ARewardStrategy` that is used to calculate the rewards in the
          reinforcement learning algorithm for the second experiment.
@@ -57,7 +57,7 @@ def return_reward_strategy(self) -> ARewardStrategy:
             RewardStrategyEnum.FIRST_REWARD_STRATEGY
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Returns an instance of `AStateCalculationStrategy` that is used to calculate the state
          representation for the reinforcement learning algorithm for the second experiment.
@@ -97,3 +97,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_second_bigger_state"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 3
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/experiments/sixth_experiment_minimap.py

@@ -48,28 +48,34 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Returns an instance of `ARewardStrategy` that is used to calculate the rewards in the
          reinforcement learning algorithm for the sixth experiment.
 
         Returns:
             ARewardStrategy: An instance of `ARewardStrategy`.
         """
-        return self.a_configuration_factory.create_reward_strategy(
-            RewardStrategyEnum.FIFTH_REWARD_STRATEGY
+        return self.a_configuration_factory.create_dynamic_reward_strategy(
+            [1680, 5110, 10770],
+            [RewardStrategyEnum.FOURTH_MINIMAP_10_PERCENT,
+             RewardStrategyEnum.FOURTH_MINIMAP_20_PERCENT,
+             RewardStrategyEnum.FOURTH_MINIMAP_40_PERCENT],
+            par_iteration_number
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Returns an instance of `AStateCalculationStrategy` that is used to calculate the state
          representation for the reinforcement learning algorithm for the sixth experiment.
 
         Returns:
             AStateCalculationStrategy: An instance of `AStateCalculationStrategy`.
         """
-        return self.a_configuration_factory.create_state_calc_strategy(
-            StateStrategyEnum.MINIMAP_STATE_STRATEGY
+        return self.a_configuration_factory.create_dynamic_state_calc_strategy(
+            [1680, 5110],
+            [StateStrategyEnum.MINIMAP_STATE_STRATEGY,
+             StateStrategyEnum.MINIMAP_STATE_NORMALIZED_STRATEGY]
         )
 
     def return_enabled_game_api_values(self) -> EnabledGameApiValues:
@@ -100,3 +106,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_mini_map"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 6
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/experiments/third_experiment_bigger_state_normalized.py

@@ -44,7 +44,7 @@ def return_model(self) -> AShortRaceFactory:
             )
         )
 
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Returns an instance of `ARewardStrategy` that is used to calculate the rewards in the
          reinforcement learning algorithm for the third experiment.
@@ -56,7 +56,7 @@ def return_reward_strategy(self) -> ARewardStrategy:
             RewardStrategyEnum.FIRST_REWARD_STRATEGY
         )
 
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Returns an instance of `AStateCalculationStrategy` that is used to calculate the state
          representation for the reinforcement learning algorithm for the third experiment.
@@ -96,3 +96,21 @@ def return_dimensional_input(self) -> tuple:
 
     def return_name(self) -> str:
         return "experiment_third_bigger_state_normalized"
+
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment - 3.
+        """
+        return 3
+
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment - 3.
+
+        Returns:
+            int: The maximum speed for the visualized experiment - 3.
+        """
+        return 3

configuration/factory/configuration_factory.py

@@ -36,6 +36,42 @@ def create_reward_strategy(self, par_reward_strategy: RewardStrategyEnum) -> ARe
         """
         return par_reward_strategy.return_strategy()
 
+    def create_dynamic_reward_strategy(self,
+                                       par_border_iteration_numbers: list[int],
+                                       par_reward_strategies: list[RewardStrategyEnum],
+                                       par_iteration_number: int = 0):
+        """
+        Creates a dynamic reward strategy for the reinforcement learning agent based on the
+         position of iteration number
+        in the par_border_iteration_numbers list.
+
+        Args:
+        - par_border_iteration_numbers (list[int]): A list of border iteration numbers.
+        - par_reward_strategies (list[RewardStrategyEnum]): A list of reward strategies.
+
+        Returns:
+        - ARewardStrategy: An object that implements the dynamic reward strategy.
+        """
+        # Check if the lengths of the input lists match
+        if len(par_border_iteration_numbers) != len(par_reward_strategies):
+            raise ValueError(
+                "Length of border iteration numbers list does"
+                " not match length of reward strategies list.")
+
+        # Get the current iteration number
+        current_iteration = par_iteration_number
+
+        # Iterate through the border iteration numbers list and compare with current iteration
+        # number
+        for border_iteration, reward_strategy in zip(par_border_iteration_numbers,
+                                                     par_reward_strategies):
+            if current_iteration < border_iteration:
+                return reward_strategy.return_strategy()
+
+        # If current iteration number is greater than all border iteration numbers,
+        # return the last reward strategy
+        return par_reward_strategies[-1].return_strategy()
+
     def create_state_calc_strategy(
             self,
             par_state_calc_strategy: StateStrategyEnum
@@ -49,6 +85,42 @@ def create_state_calc_strategy(
         """
         return par_state_calc_strategy.return_strategy()
 
+    def create_dynamic_state_calc_strategy(self,
+                                           par_border_iteration_numbers: list[int],
+                                           par_state_calc_strategies: list[StateStrategyEnum],
+                                           par_iteration_number: int = 0):
+        """
+        Creates a dynamic state clac strategy for the reinforcement learning agent based on the
+         position of iteration number
+        in the par_state_calc_strategies list.
+
+        Args:
+        - par_border_iteration_numbers (list[int]): A list of border iteration numbers.
+        - par_state_calc_strategies (list[StateStrategyEnum]): A list of state calc strategies.
+
+        Returns:
+        - ARewardStrategy: An object that implements the dynamic reward strategy.
+        """
+        # Check if the lengths of the input lists match
+        if len(par_border_iteration_numbers) != len(par_state_calc_strategies):
+            raise ValueError(
+                "Length of border iteration numbers list does"
+                " not match length of reward strategies list.")
+
+        # Get the current iteration number
+        current_iteration = par_iteration_number
+
+        # Iterate through the border iteration numbers list and compare with current iteration
+        # number
+        for border_iteration, reward_strategy in zip(par_border_iteration_numbers,
+                                                     par_state_calc_strategies):
+            if current_iteration < border_iteration:
+                return reward_strategy.return_strategy()
+
+        # If current iteration number is greater than all border iteration numbers,
+        # return the last reward strategy
+        return par_state_calc_strategies[-1].return_strategy()
+
     def create_dimensional_input(self, par_input_dim: Union[tuple, int] = 4) -> tuple:
         """
         Creates a dimensional input tuple.

configuration/i_configuration.py

@@ -25,21 +25,25 @@ def return_model(self) -> AShortRaceFactory:
         """
 
     @abstractmethod
-    def return_reward_strategy(self) -> ARewardStrategy:
+    def return_reward_strategy(self, par_iteration_number=0) -> ARewardStrategy:
         """
         Return an instance of ARewardStrategy.
 
         Returns:
             ARewardStrategy: An instance of ARewardStrategy.
+            par_iteration_number: A optional value to set dynamic reward strategies based on
+                current iteration number
         """
 
     @abstractmethod
-    def return_state_calc_strategy(self) -> AStateCalculationStrategy:
+    def return_state_calc_strategy(self, par_iteration_number=0) -> AStateCalculationStrategy:
         """
         Return an instance of AStateCalculationStrategy.
 
         Returns:
             AStateCalculationStrategy: An instance of AStateCalculationStrategy.
+            par_iteration_number: A optional value to set dynamic state calc strategies based on
+                current iteration number
         """
 
     @abstractmethod
@@ -68,3 +72,21 @@ def return_dimensional_input(self) -> tuple:
           Returns:
               tuple: a dimensional input for the experiment.
         """
+
+    @abstractmethod
+    def return_max_speed_non_visualised(self) -> int:
+        """
+        Return the maximum speed for the non-visualized experiment.
+
+        Returns:
+            int: The maximum speed for the non-visualized experiment.
+        """
+
+    @abstractmethod
+    def return_max_speed_visualised(self) -> int:
+        """
+        Return the maximum speed for the visualized experiment.
+
+        Returns:
+            int: The maximum speed for the visualized experiment.
+        """

envs/short_race_env.py

@@ -11,6 +11,7 @@
 from car_states.car_state import CarState
 from car_states.car_state_in_environment import CarStateInEnvironment
 from car_states.enabled_game_api_values import EnabledGameApiValues
+from configuration.i_configuration import IConfiguration
 from envs.strategy.reward.a_reward_strategy import ARewardStrategy
 from envs.strategy.state_calc.a_state_calc_strategy import AStateCalculationStrategy
 from game_inputs import GameInputs
@@ -38,6 +39,7 @@ class Env:
     a_state_calculation_strategy: AStateCalculationStrategy
     a_enabled_game_api_values: EnabledGameApiValues
     a_state_matrix: np.ndarray
+    a_configuration: IConfiguration
 
     a_edited_car_state: CarStateInEnvironment
 
@@ -49,9 +51,7 @@ class Env:
     }
 
     def __init__(self, par_game_inputs: GameInputs,
-                 par_reward_strategy: ARewardStrategy,
-                 par_state_calc_strategy: AStateCalculationStrategy,
-                 par_enabled_game_api_values: EnabledGameApiValues
+                 par_configuration: IConfiguration
                  ):
         """
         Initializes an instance of the Env class.
@@ -60,14 +60,17 @@ def __init__(self, par_game_inputs: GameInputs,
         par_game_inputs: A GameInputs object representing the input to the game.
         """
         super().__init__()
+        print("KEK")
         self.a_game_speed: int = 1
         self.env = None
         self.action_counter = 0
         self.controls = Controls()
         self.game_steps_per_episode: int = self.default_settings['game_steps_per_episode']
-        self.a_reward_strategy = par_reward_strategy
-        self.a_state_calculation_strategy = par_state_calc_strategy
-        self.a_enabled_game_api_values = par_enabled_game_api_values
+        self.a_reward_strategy = par_configuration.return_reward_strategy()
+        self.a_state_calculation_strategy = par_configuration.return_state_calc_strategy()
+        self.a_enabled_game_api_values = par_configuration.return_enabled_game_api_values()
+
+        self.a_configuration = par_configuration
 
         self.a_game_inputs: GameInputs = par_game_inputs
         self.a_lap_percent_curr = 0.00
@@ -77,6 +80,13 @@ def __init__(self, par_game_inputs: GameInputs,
 
         self.a_state_matrix = np.zeros((5, 5), dtype=float) - 1
 
+        par_game_inputs.agent_settings_to_game.put((
+            self.default_settings['visualize'],
+            self.default_settings['realtime']
+        ))
+
+        print("Values")
+
     def make_state(self):
         """
         Generates the state tuple to be used in the next step of the environment.
@@ -106,13 +116,19 @@ def make_state(self):
 
         return state, tmp_reward, terminal
 
-    def reset(self):
+    def reset(self, iteration_number: int):
         """
         Resets the environment to its initial state.
 
         Returns:
         A numpy array representing the initial state of the environment.
         """
+        Printer.print_error(str(iteration_number))
+        self.a_reward_strategy = \
+            self.a_configuration.return_reward_strategy(iteration_number)
+        self.a_state_calculation_strategy = \
+            self.a_configuration.return_state_calc_strategy(iteration_number)
+
         self.a_lap_percent_curr = 0.00
         self.controls.release_all_keys()
         self.controls.reset_directional_controls()
@@ -122,7 +138,7 @@ def reset(self):
         tmp_queue_game_inputs: multiprocessing.Queue = \
             self.a_game_inputs.game_initialization_inputs.get()
         # noinspection PyUnresolvedReferences
-        self.a_game_speed = tmp_queue_game_inputs[1]
+        self.a_game_speed = tmp_queue_game_inputs[1] if not self.default_settings['realtime'] else 1
         self.a_game_inputs.game_initialization_inputs.put(tmp_queue_game_inputs)
 
         self.controls.a_is_executing_critical_action = True
@@ -184,6 +200,8 @@ def step(self, action):
         while not tmp_car_state_from_game.has_non_default_values():
             tmp_car_state_from_game: CarState = self.a_game_inputs.agent_inputs_state.get()
 
+        Printer.print_info("SPEED " + str(tmp_car_state_from_game.speed_mph), "ENV")
+
         tmp_lap_progress_diff: float = \
             self.get_lap_progress_dif(tmp_car_state_from_game.lap_progress)
 
@@ -244,21 +262,15 @@ def close(self):
 
 
 def create_env(par_game_inputs: GameInputs,
-               par_reward_strategy: ARewardStrategy,
-               par_state_calc_strategy: AStateCalculationStrategy,
-               par_enabled_game_api_values: EnabledGameApiValues) -> Env:
+               par_configuration: IConfiguration) -> Env:
     """
     Creates a game environment for playing the game.
 
     :param par_game_inputs: The game inputs for the environment.
-    :param par_reward_strategy: The reward strategy.
-    :param par_state_calc_strategy: The state calculation strategy.
-    :param par_enabled_game_api_values: The enabled game api values.
+    :param par_configuration: The specified configuration.
     :return: An instance of the game environment.
     """
     return Env(
         par_game_inputs,
-        par_reward_strategy,
-        par_state_calc_strategy,
-        par_enabled_game_api_values
+        par_configuration
     )

envs/strategy/reward/fouth_minimap/__init__.py

@@ -15,7 +15,7 @@
 
 # pylint: disable=too-few-public-methods
 # pylint: disable=R0801
-class FourthRewardStrategy(ARewardStrategy):
+class FourthMinimap10PercentRewardStrategy(ARewardStrategy):
     """
     This class is an implementation of the ARewardStrategy abstract class.
     This implementation gives:

envs/strategy/reward/fourth_reward_strategy.py envs/strategy/reward/fouth_minimap/fourth_minimap_10_percent_strateg.py

@@ -15,19 +15,16 @@
 
 # pylint: disable=too-few-public-methods
 # pylint: disable=R0801
-class FifthRewardStrategy(ARewardStrategy):
+class FourthMinimap20PercentRewardStrategy(ARewardStrategy):
     """
     This class is an implementation of the ARewardStrategy abstract class.
     This implementation gives:
      - positive reward for:
-        Offset (Distance From Road Centre): <0, 1> and <-1, 0>
-
-        Completing The Race (Partially - 10%)
+        Completing The Race (Partially - 20%)
 
      - negative reward for:
-        Offset (Distance From Road Centre): (-10, -1> and (-inf, -10> 
-
         Not Completing The Race in specified count_of_steps
+        
     """
 
     # noinspection DuplicatedCode

envs/strategy/reward/fifth_reward_strategy.py envs/strategy/reward/fouth_minimap/fourth_minimap_20_percent_strategy.py

@@ -0,0 +1,102 @@
+"""
+Module: first_reward_strategy
+
+This module contains the FirstRewardStrategy class which is an implementation of
+ the ARewardStrategy abstract class.
+
+Classes:
+    FirstRewardStrategy
+
+"""
+from car_states.car_state_in_environment import CarStateInEnvironment
+from envs.strategy.reward.a_reward_strategy import ARewardStrategy
+from utils.print_utils.printer import Printer
+
+
+# pylint: disable=too-few-public-methods
+# pylint: disable=R0801
+class FourthMinimap40PercentRewardStrategy(ARewardStrategy):
+    """
+    This class is an implementation of the ARewardStrategy abstract class.
+    This implementation gives:
+     - positive reward for:
+        Completing The Race (Partially - 40%)
+
+     - negative reward for:
+        Not Completing The Race in specified count_of_steps
+        
+    """
+
+    # noinspection DuplicatedCode
+    def evaluate_reward(self, par_env_inputs: CarStateInEnvironment,
+                        par_game_steps_per_episode: int,
+                        par_env_steps_counter: int,
+                        par_terminal: bool) -> tuple[float, bool]:
+        """
+        This method calculates the reward of the current step for the ShortRaceEnv environment.
+
+        Args:
+            par_env_inputs (CarStateInEnvironment): Object containing car state represented by 
+                the environment
+            par_game_steps_per_episode (int): Count of Configured Game Steps per Env Episode
+            par_env_steps_counter: (int) Count of passed game Steps in Env
+            par_terminal (bool): If the environment has reached a terminal state.
+
+        Returns:
+            Tuple[float, bool]: The reward value and if the episode is finished.
+        """
+        reward: float = 0
+        terminal: bool = par_terminal
+
+        # Ako daleko som od idealnej linie?
+
+        # Fiat Punto Top Speed - 179 # Zatial docasne prec
+
+        # 0 - 50 - Negative Reward ((-1) - 0)
+        # if -1 >= tmp_speed < 50:
+        # reward += (((50 - tmp_speed) / 50) / 255) * -1
+        # 50 - 100 - Positive Reward ( 0 - 1)
+        # elif 50 <= tmp_speed <= 100:
+        # reward += (((tmp_speed - 50) / 50) / 255)
+        # 100 - 179 - Reward 1 - (-1)
+        # else:
+        # reward += (((179 - tmp_speed) / 39.5) - 1) / 255
+
+        tmp_normalization_value: int = par_game_steps_per_episode
+
+        reward += self.__lap_progress_reward(par_env_inputs.lap_progress_difference,
+                                             tmp_normalization_value)
+
+        if par_env_steps_counter >= par_game_steps_per_episode or par_env_inputs.lap_progress >= 40:
+            terminal = True
+            if par_env_steps_counter >= par_game_steps_per_episode:
+                Printer.print_info("Exceeded Step Limit", "FOURTH_REWARD_STRATEGY", )
+                reward += ((par_env_inputs.lap_progress / 20) - 1)
+            if par_env_inputs.lap_progress >= 40:
+                reward += 1
+                Printer.print_success("Lap Complete", "FOURTH_REWARD_STRATEGY")
+            Printer.print_info("TERMINAL STATE ACHIEVED", "FOURTH_REWARD_STRATEGY")
+        return reward, terminal
+
+    def __lap_progress_reward(self, par_lap_progress_diff: float,
+                              par_normalization_value: int) -> float:
+        """
+        Calculates the lap progress reward based on the difference in lap progress
+        between the current and previous time step.
+
+        :param par_lap_progress_diff: A float representing the difference in lap
+            progress between the current and previous time step. The value should
+            be between -1 and 1, where negative values represent falling behind
+            and positive values represent making progress.
+         :param par_normalization_value: An integer representing the normalization
+            value to use in the reward calculation. This value should be greater
+            than zero to avoid division by zero errors.
+
+        :return: A float representing the lap progress reward. The value will be
+            positive if the agent is making progress and negative if the agent
+            is falling behind. The magnitude of the reward will be proportional
+            to the magnitude of the lap progress difference, divided by the
+            normalization value.
+        """
+        Printer.print_basic("Progress: " + str(par_lap_progress_diff), "FOURTH_REWARD_STRATEGY")
+        return par_lap_progress_diff / par_normalization_value

envs/strategy/reward/fouth_minimap/fourth_minimap_40_percent_strategy.py

@@ -6,9 +6,13 @@
 from enum import Enum
 
 from envs.strategy.reward.a_reward_strategy import ARewardStrategy
-from envs.strategy.reward.fifth_reward_strategy import FifthRewardStrategy
 from envs.strategy.reward.first_reward_strategy import FirstRewardStrategy
-from envs.strategy.reward.fourth_reward_strategy import FourthRewardStrategy
+from envs.strategy.reward.fouth_minimap.fourth_minimap_10_percent_strateg import \
+    FourthMinimap10PercentRewardStrategy
+from envs.strategy.reward.fouth_minimap.fourth_minimap_20_percent_strategy import \
+    FourthMinimap20PercentRewardStrategy
+from envs.strategy.reward.fouth_minimap.fourth_minimap_40_percent_strategy import \
+    FourthMinimap40PercentRewardStrategy
 from envs.strategy.reward.second_reward_strategy import SecondRewardStrategy
 from envs.strategy.reward.third_reward_strategy import ThirdRewardStrategy
 
@@ -20,8 +24,9 @@ class RewardStrategyEnum(Enum):
     FIRST_REWARD_STRATEGY = 0
     SECOND_REWARD_STRATEGY = 1
     THIRD_REWARD_STRATEGY = 2
-    FOURTH_REWARD_STRATEGY = 3
-    FIFTH_REWARD_STRATEGY = 4
+    FOURTH_MINIMAP_10_PERCENT = 3
+    FOURTH_MINIMAP_20_PERCENT = 4
+    FOURTH_MINIMAP_40_PERCENT = 5
 
     def return_strategy(self) -> ARewardStrategy:
         """
@@ -42,8 +47,9 @@ def return_strategy(self) -> ARewardStrategy:
             self.FIRST_REWARD_STRATEGY: FirstRewardStrategy(),
             self.SECOND_REWARD_STRATEGY: SecondRewardStrategy(),
             self.THIRD_REWARD_STRATEGY: ThirdRewardStrategy(),
-            self.FOURTH_REWARD_STRATEGY: FourthRewardStrategy(),
-            self.FIFTH_REWARD_STRATEGY: FifthRewardStrategy(),
+            self.FOURTH_MINIMAP_10_PERCENT: FourthMinimap10PercentRewardStrategy(),
+            self.FOURTH_MINIMAP_20_PERCENT: FourthMinimap20PercentRewardStrategy(),
+            self.FOURTH_MINIMAP_40_PERCENT: FourthMinimap40PercentRewardStrategy(),
         }
 
         try:

envs/strategy/reward/reward_strategy_enum.py

@@ -0,0 +1,103 @@
+"""
+Module for implementing a state calculation strategy that stores a 5x5 matrix of past states
+ and actions and returns it as a normalized flattened Torch Tensor.
+
+This module defines the `BiggerStateStrategy` class, which inherits from the abstract base class
+ `StateCalculationStrategy`.
+This class overrides the `calculate_state` method to store the last
+ five observations and actions in a matrix, which is then flattened and returned as a Torch Tensor.
+"""
+from typing import Union
+
+import numpy
+import torch
+from numpy import ndarray
+
+from car_states.car_state_in_environment import CarStateInEnvironment
+from envs.strategy.state_calc.a_state_calc_strategy import AStateCalculationStrategy
+
+
+# pylint: disable=R0801
+# noinspection DuplicatedCode
+class MinimapStateNormalizedStrategy(AStateCalculationStrategy):
+    """
+    A state calculation strategy that stores the last five observations and actions in a 5x5 matrix
+     and returns it as a normalized flattened Torch Tensor.
+
+    This class inherits from the abstract base class `StateCalculationStrategy`. It overrides the
+     `calculate_state` method to store the last five observations and actions in a 5x5 matrix. The
+     matrix is then flattened and returned as a Torch Tensor.
+
+    Attributes:
+        a_car_state_in_environment (np.ndarray): A object represent car state in environment
+    """
+
+    a_car_state_in_environment: CarStateInEnvironment
+
+    def __init__(self):
+        self.a_car_state_in_environment = CarStateInEnvironment()
+
+    def calculate_state(self, par_car_state: CarStateInEnvironment,
+                        par_action_taken: Union[int, None]) -> torch.Tensor:
+        """
+        Calculate the state tensor for a given car state and action taken.
+
+        Args:
+            par_car_state (CarStateInEnvironment): The car state in the environment.
+            par_action_taken (Union[int, None]): The action taken by the car, or None if no 
+             has been taken.
+
+        Returns:
+            torch.Tensor: A 3D tensor of shape (4, 48, 48) representing the current state of the
+             car.
+            The four layers of the tensor represent the mini-map, lap progress, car speed,
+               and wrong-way indicator, respectively.
+            The values in the mini-map layer are resized to 48x48 pixels and the lap progress and
+             car speed layers are rounded to a specified number of digits.
+            The wrong-way indicator layer contains binary values indicating whether the car is going
+             the wrong way on the track or not.
+        """
+        normalized_state_values: CarStateInEnvironment = self.normalize_state_values(par_car_state)
+
+        mini_map_resized_2d = normalized_state_values.mini_map
+        lap_progress_2d = numpy.full((48, 48), normalized_state_values.lap_progress)
+        car_speed_2d = numpy.full((48, 48), normalized_state_values.speed_mph)
+        wrong_way_2d = numpy.full((48, 48), normalized_state_values.wrong_way_indicator)
+
+        new_state = numpy.stack((mini_map_resized_2d, lap_progress_2d, car_speed_2d, wrong_way_2d))
+
+        # returns 3D matrix of size 4x48x48
+        return torch.from_numpy(new_state)
+
+    def normalize_state_values(self, par_car_state_not_normalized: CarStateInEnvironment) \
+            -> CarStateInEnvironment:
+        """
+        Normalizes the input state values and returns a tuple of normalized values.
+        Args:
+            par_car_state_not_normalized (CarState): A CarState object of un-normalized state
+                values, including the car speed, distance offset, lap progress, and direction
+                offset.
+        Returns:
+            CarState: A CarState object of normalized state values, including the
+                normalized car speed, normalized distance offset, normalized lap progress,
+                and normalized direction offset.
+        """
+
+        self.a_car_state_in_environment.reset_car_state()
+
+        tmp_normalized_minimap: ndarray = \
+            par_car_state_not_normalized.mini_map.astype('float32') / 255.0
+
+        tmp_car_top_speed: float = 111
+        tmp_normalized_speed = par_car_state_not_normalized.speed_mph / tmp_car_top_speed
+
+        tmp_normalized_lap_progress: float = par_car_state_not_normalized.lap_progress / 100
+
+        self.a_car_state_in_environment.reset_car_state()
+        self.a_car_state_in_environment.assign_values(
+            par_speed_mph=tmp_normalized_speed,
+            par_lap_progress=tmp_normalized_lap_progress,
+            par_wrong_way_indicator=par_car_state_not_normalized.wrong_way_indicator,
+            par_mini_map=tmp_normalized_minimap
+        )
+        return self.a_car_state_in_environment

envs/strategy/state_calc/minimap_state_normalized_strategy.py

@@ -9,6 +9,8 @@
 from envs.strategy.state_calc.basic_state_strategy import BasicStateStrategy
 from envs.strategy.state_calc.bigger_state_normalized_strategy import BiggerStateNormalizedStrategy
 from envs.strategy.state_calc.bigger_state_strategy import BiggerStateStrategy
+from envs.strategy.state_calc.minimap_state_normalized_strategy import \
+    MinimapStateNormalizedStrategy
 from envs.strategy.state_calc.minimap_state_strategy import MinimapStateStrategy
 
 
@@ -20,6 +22,7 @@ class StateStrategyEnum(Enum):
     BIGGER_STATE_STRATEGY = 1
     BIGGER_STATE_STRATEGY_NORMALIZED = 2
     MINIMAP_STATE_STRATEGY = 3
+    MINIMAP_STATE_NORMALIZED_STRATEGY = 4
 
     def return_strategy(self) -> AStateCalculationStrategy:
         """
@@ -41,7 +44,8 @@ def return_strategy(self) -> AStateCalculationStrategy:
             self.BASIC_STATE_STRATEGY: BasicStateStrategy(),
             self.BIGGER_STATE_STRATEGY: BiggerStateStrategy(),
             self.BIGGER_STATE_STRATEGY_NORMALIZED: BiggerStateNormalizedStrategy(),
-            self.MINIMAP_STATE_STRATEGY: MinimapStateStrategy()
+            self.MINIMAP_STATE_STRATEGY: MinimapStateStrategy(),
+            self.MINIMAP_STATE_NORMALIZED_STRATEGY: MinimapStateNormalizedStrategy()
         }
 
         try:

envs/strategy/state_calc/state_strategy_enum.py

@@ -78,51 +78,39 @@ class Game:
     a_gps: GPS
 
     a_speedometer: Speedometer
-
     a_lap_progress: LapProgress
-
     a_lap_time: LapTime
-
     a_revolutions_per_minute: RevolutionsPerMinute
-
     a_wrong_way: WrongWay
 
     a_is_recording: bool
 
     a_user23 = ctypes.windll.user32
-
     a_list_bitmap: []
-
     a_controls: Controls
-
     a_font_settings: FontSettings
-
     a_speed: int
-
     a_car_distance_offset: float
-
     a_car_direction_offset: int
-
     a_race_initialised: bool
-
     a_cycles_passed: int
-
     a_cuda_device: None
 
     a_gps_img_rcg_strategy: AGpsImageRecognitionStrategy
-
     a_gps_strategy_enum: GPSStrategyEnum
-
     a_cheat_engine: CheatEngine
-
     a_image_manipulation: ImageManipulation
 
     a_dictionary_menus: dict[str, str]
-
     a_enabled_game_api_values: EnabledGameApiValues
-
     a_car_state: CarState
 
+    a_speed_with_visualiser: int
+    a_speed_without_visualiser: int
+
+    a_hwnd: None
+    a_visualise: bool
+
     def __init__(self) -> None:
         self.a_image_manipulation = ImageManipulation()
         self.a_image_manipulation.load_comparable_images()
@@ -132,7 +120,7 @@ def __init__(self) -> None:
         self.a_cycles_passed = 0
         self.a_cheat_engine = CheatEngine()
         self.a_list_bitmap = []
-        self.a_speed = 3
+        self.a_speed = 6
         self.a_controls = Controls()
         self.a_font_settings = FontSettings(
             par_font=cv2.FONT_HERSHEY_SIMPLEX,
@@ -146,6 +134,7 @@ def __init__(self) -> None:
             'standing_menu': 'standings_menu',
             'attention_restart': 'attention_restart'
         }
+        self.a_visualise = True
 
         cuda.printCudaDeviceInfo(0)
 
@@ -177,7 +166,10 @@ def init_game_memory_objects(self) -> None:
         self.a_wrong_way.construct()
 
     def initialize_game(self, par_game_inputs: GameInputs,
-                        par_enabled_game_api_values: EnabledGameApiValues) -> None:
+                        par_enabled_game_api_values: EnabledGameApiValues,
+                        par_max_speed_with_visualiser: int,
+                        par_max_speed_without_visualiser: int
+                        ) -> None:
         """
         Initializes the game by starting the game, waiting for it to start, creating and 
             initializing required game objects,and setting the game speed and cheat engine.
@@ -187,12 +179,17 @@ def initialize_game(self, par_game_inputs: GameInputs,
                 game inputs.
             par_enabled_game_api_values (EnabledGameApiValues): an instance of EnabledGameApiValues
                 class containing the enabled game api values. 
+            par_max_speed_with_visualiser (int): max speed multiplier with visualiser enabled
+            par_max_speed_without_visualiser (int): max speed multiplier without visualiser enabled
+
 
         Returns:
             None
         """
         self.a_game_state = GameStateStarting()
         self.a_enabled_game_api_values = par_enabled_game_api_values
+        self.a_speed_with_visualiser = par_max_speed_with_visualiser
+        self.a_speed_without_visualiser = par_max_speed_without_visualiser
         self.start_game()
         # self.start_cheat_engine()
 
@@ -237,19 +234,43 @@ class containing the enabled game api values.
 
         self.a_cycles_passed = 0
 
+        par_game_inputs.game_initialization_inputs.put((
+            self.a_race_initialised
+        ))
+
+        agent_settings: tuple[bool, bool] = par_game_inputs.agent_settings_to_game.get()
+
+        self.a_visualise = agent_settings[0]
+
+        # If Not Realtime set speed based on enabled visualiser
+        if not agent_settings[1]:
+            if self.a_visualise:
+                self.a_speed = self.a_speed_with_visualiser
+            else:
+                self.a_speed = self.a_speed_without_visualiser
+        # If Realtime set speed to 1
+        else:
+            self.a_speed = 1
+
         par_game_inputs.game_initialization_inputs.put((
             self.a_race_initialised,
             self.a_speed
         ))
 
+        self.a_cheat_engine.reconfigure_speed(self.a_speed)
+
         self.a_car_state = self.create_empty_car_state()
 
     # pylint: disable=too-many-locals
     # pylint: disable=too-many-branches
     # pylint: disable=too-many-statements
+    # pylint: disable=too-many-arguments
     def main_loop(self, par_game_inputs: GameInputs,
                   par_results_path: str,
-                  par_enabled_game_api_values: EnabledGameApiValues):
+                  par_enabled_game_api_values: EnabledGameApiValues,
+                  par_max_speed_with_visualiser: int,
+                  par_max_speed_without_visualiser: int
+                  ):
         """
         Main Loop That Controls All The Game Logic
 
@@ -259,12 +280,17 @@ def main_loop(self, par_game_inputs: GameInputs,
             par_results_path (str): Path of the folder containing results including graph images
             par_enabled_game_api_values (EnabledGameApiValues): an instance of EnabledGameApiValues
                 class containing the enabled game api values. 
+            par_max_speed_with_visualiser (int): max speed multiplier with visualiser enabled
+            par_max_speed_without_visualiser (int): max speed multiplier without visualiser enabled
 
         Returns:
             None: This method doesn't return anything.
         """
 
-        self.initialize_game(par_game_inputs, par_enabled_game_api_values)
+        self.initialize_game(par_game_inputs,
+                             par_enabled_game_api_values,
+                             par_max_speed_with_visualiser,
+                             par_max_speed_without_visualiser)
 
         tmp_start_time = time.time()
         tmp_speed_constant = 1 / self.a_speed
@@ -276,7 +302,7 @@ class containing the enabled game api values.
             try:
                 tmp_wrong_way_value = self.a_wrong_way.return_is_wrong_way()
             # pylint: disable=broad-except
-            except Exception as exception:
+            except pymem.exception.MemoryReadError as exception:
                 Printer.print_info(f"Waiting for pointers to initialize {exception}", "GAME")
                 time.sleep(1)
 
@@ -287,10 +313,12 @@ class containing the enabled game api values.
 
             self.a_car_state.reset_car_state()
 
-            # Check for quit key -> !! WARNING - Without this all the windows will be BLANK GREY !!!
-            if cv2.waitKey(1) == ord('q'):
-                cv2.destroyAllWindows()
-                break
+            if self.a_visualise:
+                # Check for quit key -> !! WARNING - Without this all the windows will be
+                # BLANK GREY !!!
+                if cv2.waitKey(1) == ord('q'):
+                    cv2.destroyAllWindows()
+                    break
 
             # Check for record key
             if keyboard.is_pressed('r'):
@@ -340,11 +368,12 @@ class containing the enabled game api values.
             self.a_car_distance_offset = tmp_car_offset_distance
             self.a_car_direction_offset = tmp_car_offset_direction
 
-            if tmp_contour is not None:
+            if tmp_contour is not None and self.a_visualise:
                 cv2.drawContours(self.a_screenshot, [tmp_contour], -1, (255, 0, 255), -1)
 
             backup_screenshot: ndarray = self.a_screenshot
-            self.show_graph(par_image_path=par_results_path + 'scatter_plot.png')
+            if self.a_visualise:
+                self.show_graph(par_image_path=par_results_path + 'scatter_plot.png')
 
             tmp_frame_counter += tmp_speed_constant
 
@@ -359,7 +388,8 @@ class containing the enabled game api values.
                 tmp_needs_restart: bool = par_game_inputs.game_restart_inputs.get()
                 if tmp_needs_restart:
                     self.a_game_state = GameStateRestarting()
-                    self.update_state_on_screen(self.a_screenshot)
+                    if self.a_visualise:
+                        self.update_state_on_screen(self.a_screenshot)
 
                     par_game_inputs.game_restart_inputs.put(tmp_needs_restart)
                     self.reset_game_race(0.7 / float(self.a_speed), 0.01 / float(self.a_speed))
@@ -387,16 +417,16 @@ class containing the enabled game api values.
             )
 
             par_game_inputs.agent_inputs_state.put(self.a_car_state, )
+            if self.a_visualise:
+                self.show_texts_on_image(par_image=backup_screenshot,
+                                         par_font_color=(159, 43, 104),
+                                         par_car_state=self.a_car_state
+                                         )
 
-            self.show_texts_on_image(par_image=backup_screenshot,
-                                     par_font_color=(159, 43, 104),
-                                     par_car_state=self.a_car_state
-                                     )
-
-            self.show_state_on_image(par_image=backup_screenshot,
-                                     par_game_state=self.a_game_state)
+                self.show_state_on_image(par_image=backup_screenshot,
+                                         par_game_state=self.a_game_state)
 
-            cv2.imshow('Main Vision', backup_screenshot)
+                cv2.imshow('Main Vision', backup_screenshot)
 
     def is_race_initialised(self) -> bool:
         """
@@ -633,9 +663,11 @@ def window_capture(self) -> Tuple[np.ndarray, int, int]:
                - int: Width of the captured image
                - int: Height of the captured image
         """
-        # Find the game window
+        # Find the game window4
         hwnd = win32gui.FindWindow(None, self.api_settings['game_title_name'])
 
+        # win32gui.SendMessage(hwnd, win32con.WM_ACTIVATE, win32con.WA_CLICKACTIVE, hwnd)
+
         # Get the window device context
         w_dc = win32gui.GetWindowDC(hwnd)
 
@@ -831,7 +863,8 @@ def is_in_correct_restart_state(self, par_screen_image: ndarray) -> RestartState
 
         return RestartStateEnum.UNKNOWN_STATE
 
-    def reset_game_race(self, par_sleep_time_delay: float, par_sleep_time_key_press: float) -> None:
+    def reset_game_race(self, par_sleep_time_delay: float,
+                        par_sleep_time_key_press: float) -> None:
         """
         Reset the game race to the initial state.
 
@@ -872,7 +905,8 @@ def reset_game_race(self, par_sleep_time_delay: float, par_sleep_time_key_press:
             # Press Enter - Restarts The Race
             # Then Prompt Will Appear - We Move To The OK Button
             for key_to_press in keys_to_press:
-                self.a_controls.press_and_release_key(key_to_press, par_sleep_time_key_press, True)
+                self.a_controls.press_and_release_key(key_to_press,
+                                                      par_sleep_time_key_press, True)
                 time.sleep(par_sleep_time_delay)
 
             tmp_restart_state = self.is_in_correct_restart_state(self.window_capture()[0])
@@ -887,12 +921,14 @@ def reset_game_race(self, par_sleep_time_delay: float, par_sleep_time_key_press:
         # If the prompt with Attention (Do you Really Want to Restart) appears press Enter
         time.sleep(par_sleep_time_delay)
 
-        self.a_controls.press_and_release_key(self.a_controls.ENTER, par_sleep_time_key_press, True)
+        self.a_controls.press_and_release_key(self.a_controls.ENTER,
+                                              par_sleep_time_key_press, True)
         time.sleep(par_sleep_time_delay)
 
         time.sleep(1 * self.a_speed)
 
-    def init_game_race(self, par_sleep_time_delay: float, par_sleep_time_key_press: float):
+    def init_game_race(self,
+                       par_sleep_time_delay: float, par_sleep_time_key_press: float):
         """
         Initializes a race in the game by navigating through the game's menu system.
 

game_api/game.py

@@ -24,12 +24,15 @@ class GameInputs:
     __a_game_initialization_inputs: mp.Queue
     # Inputs Given If Restart is Needed
     __a_game_restart_inputs: mp.Queue
+    # Settings from Agent to Game
+    __a_agent_settings_to_game: mp.Queue
 
     def __init__(self, par_agent_inputs_state: mp.Queue, par_game_initialization_inputs: mp.Queue,
-                 par_game_restart_inputs: mp.Queue):
+                 par_game_restart_inputs: mp.Queue, par_agent_settings_to_game: mp.Queue):
         self.__a_agent_inputs_state = par_agent_inputs_state
         self.__a_game_initialization_inputs = par_game_initialization_inputs
         self.__a_game_restart_inputs = par_game_restart_inputs
+        self.__a_agent_settings_to_game = par_agent_settings_to_game
 
     @property
     def agent_inputs_state(self) -> mp.Queue:
@@ -60,3 +63,13 @@ def game_restart_inputs(self) -> mp.Queue:
             mp.Queue: The queue containing the inputs.
         """
         return self.__a_game_restart_inputs
+
+    @property
+    def agent_settings_to_game(self) -> mp.Queue:
+        """
+        Gets the settings given by agent to a game
+
+        Returns:
+            mp.Queue: The queue containing the settings.
+        """
+        return self.__a_agent_settings_to_game

game_inputs.py

@@ -51,7 +51,10 @@ def game_loop_thread(par_game_inputs: GameInputs) -> None:
         tmp_game.main_loop(
             par_game_inputs=par_game_inputs,
             par_results_path=results_path,
-            par_enabled_game_api_values=selected_configuration.return_enabled_game_api_values()
+            par_enabled_game_api_values=selected_configuration.return_enabled_game_api_values(),
+            par_max_speed_with_visualiser=selected_configuration.return_max_speed_visualised(),
+            par_max_speed_without_visualiser=
+            selected_configuration.return_max_speed_non_visualised(),
         )
     except Exception as exception:
         Printer.print_error("An error occurred in Game Api", "MAIN", exception)
@@ -74,7 +77,7 @@ def agent_loop(par_game_inputs: GameInputs) -> None:
     settings = {
         'create_scatter_plot': False,
         'load_previous_model': True,
-        'previous_model_iter_number': 4420
+        'previous_model_iter_number': 8000
     }
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     Printer.print_basic(torch.version.cuda, "MAIN")
@@ -86,9 +89,7 @@ def agent_loop(par_game_inputs: GameInputs) -> None:
 
     env_param = (
         par_game_inputs,
-        selected_configuration.return_reward_strategy(),
-        selected_configuration.return_state_calc_strategy(),
-        selected_configuration.return_enabled_game_api_values()
+        selected_configuration
     )
 
     count_of_iterations = 20000
@@ -140,20 +141,12 @@ def agent_loop(par_game_inputs: GameInputs) -> None:
         pass
     results_time = ''
 
-    tmp_game_variables: tuple = par_game_inputs.game_initialization_inputs.get()
-
-    tmp_is_game_started: bool = tmp_game_variables[0]
-
-    par_game_inputs.game_initialization_inputs.put(tmp_game_variables)
+    tmp_is_game_started: bool = par_game_inputs.game_initialization_inputs.get()
 
     while not tmp_is_game_started:
         Printer.print_info("Waiting for Race to Initialise", "MAIN")
 
-        tmp_game_variables: tuple = par_game_inputs.game_initialization_inputs.get()
-
-        tmp_is_game_started: bool = tmp_game_variables[0]
-
-        par_game_inputs.game_initialization_inputs.put(tmp_game_variables)
+        tmp_is_game_started: bool = par_game_inputs.game_initialization_inputs.get()
 
     time.sleep(1)
 
@@ -192,11 +185,13 @@ def agent_loop(par_game_inputs: GameInputs) -> None:
     tmp_queue_env_inputs: multiprocessing.Queue = multiprocessing.Queue()
     tmp_queue_game_started_inputs: multiprocessing.Queue = multiprocessing.Queue()
     tmp_queue_restart_game_input: multiprocessing.Queue = multiprocessing.Queue()
+    tmp_queue_agent_settings_to_game: multiprocessing.Queue = multiprocessing.Queue()
 
     game_inputs: GameInputs = GameInputs(
         tmp_queue_env_inputs,
         tmp_queue_game_started_inputs,
-        tmp_queue_restart_game_input
+        tmp_queue_restart_game_input,
+        tmp_queue_agent_settings_to_game,
     )
 
     tmp_game_thread = multiprocessing.Process(target=game_loop_thread, args=(game_inputs,))