From 2e5fd8c3a9f8c44bc783a02a51e397cd37d2c046 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 28 Jun 2023 17:54:45 -0400 Subject: [PATCH 1/7] Remove gym package and hiway-v0 --- CHANGELOG.md | 3 + Makefile | 3 +- docs/sim/env.rst | 2 +- docs/sim/visualization.rst | 2 +- .../inference/contrib_policy/filter_obs.py | 2 +- .../inference/contrib_policy/format_action.py | 2 +- .../inference/contrib_policy/frame_stack.py | 2 +- .../inference/contrib_policy/make_dict.py | 2 +- .../drive/inference/contrib_policy/network.py | 2 +- examples/rl/drive/train/preprocess.py | 2 +- examples/rl/drive/train/run.py | 2 +- .../inference/contrib_policy/filter_obs.py | 2 +- .../inference/contrib_policy/format_action.py | 2 +- .../inference/contrib_policy/frame_stack.py | 2 +- .../inference/contrib_policy/make_dict.py | 2 +- .../inference/contrib_policy/network.py | 2 +- examples/rl/platoon/train/preprocess.py | 2 +- examples/rl/platoon/train/run.py | 2 +- setup.cfg | 6 +- smarts/env/__init__.py | 7 - smarts/env/gymnasium/hiway_env_v1.py | 3 +- .../env/gymnasium/wrappers/api_reversion.py | 2 +- smarts/env/hiway_env.py | 328 ------------------ smarts/env/tests/test_hiway_env.py | 92 ----- smarts/env/wrappers/gif_recorder.py | 2 +- smarts/env/wrappers/record_video.py | 6 +- smarts/env/wrappers/recorder_wrapper.py | 3 +- smarts/zoo/agent_spec.py | 2 +- zoo/evaluation/npc_ego_example.py | 2 +- .../cross_rl_agent/train/run_test.py | 2 +- .../cross_rl_agent/train/run_train.py | 2 +- zoo/policies/rl-agent/rl_agent/agent.py | 2 +- zoo/policies/rl-agent/rl_agent/lane_space.py | 2 +- 33 files changed, 36 insertions(+), 463 deletions(-) delete mode 100644 smarts/env/hiway_env.py delete mode 100644 smarts/env/tests/test_hiway_env.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 49ec54ff1d..d20db15105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,11 +18,14 @@ Copy and pasting the git commit messages is __NOT__ enough. - Updated `RLlibHiWayEnv` to use the `gymnasium` interface. - Renamed `rllib/rllib.py` to `rllib/pg_pbt_example.py`. - Loosened constraint of `gymnasium` from `==0.27.0` to `>=0.26.3`. +- Changed all uses of `gym` to use `gymnasium`. ### Deprecated ### Fixed - Missing neighborhood vehicle state `'lane_id'` is now added to the `hiway-v1` formatted observations. - Fixed a regression where `pybullet` build time messages returned. ### Removed +- Removed `gym` as a core dependency. +- Removed `hiway-v0` env. ### Security ## [1.2.0] # 2023-06-14 diff --git a/Makefile b/Makefile index 7d06a01a9f..cfdb026e77 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ test: build-all-scenarios --dist=loadscope \ -n `expr \( \`nproc\` \/ 2 \& \`nproc\` \> 3 \) \| 2` \ --nb-exec-timeout 65536 \ - ./examples/tests ./smarts/env ./envision ./smarts/core ./smarts/sstudio ./tests \ + ./examples/tests ./smarts/env ./envision ./smarts/core ./smarts/sstudio \ --ignore=./smarts/core/waymo_map.py \ --ignore=./smarts/core/argoverse_map.py \ --ignore=./smarts/core/tests/test_argoverse.py \ @@ -18,6 +18,7 @@ test: build-all-scenarios --ignore=./smarts/core/tests/test_notebook.py \ --ignore=./smarts/env/tests/test_benchmark.py \ --ignore=./examples/tests/test_learning.py \ + --ignore=./examples/tests/test_rl.py \ -k 'not test_long_determinism' rm -f .coverage.* rm -f .coverage* diff --git a/docs/sim/env.rst b/docs/sim/env.rst index a3e4ac9cd1..4afe78f3e8 100644 --- a/docs/sim/env.rst +++ b/docs/sim/env.rst @@ -25,7 +25,7 @@ Refer to :class:`~smarts.env.hiway_env.HiWayEnv` for more details. import gymnasium as gym # Make env env = gym.make( - "smarts.env:hiway-v0", # Env entry name. + "smarts.env:hiway-v1", # Env entry name. scenarios=[scenario_path], # List of paths to scenario folders. agent_interfaces={AGENT_ID: agent_spec.interface}, # Dictionary mapping agents to agent interfaces. headless=False, # False to enable Envision visualization of the environment. diff --git a/docs/sim/visualization.rst b/docs/sim/visualization.rst index 403222abb7..1bc1cf28a7 100644 --- a/docs/sim/visualization.rst +++ b/docs/sim/visualization.rst @@ -35,7 +35,7 @@ For recording, simply feed ``envision_record_data_replay_path`` with desired rec .. code-block:: python env = gym.make( - "smarts.env:hiway-v0", + "smarts.env:hiway-v1", scenarios=args.scenarios, agents={AGENT_ID: agent}, headless=args.headless, diff --git a/examples/rl/drive/inference/contrib_policy/filter_obs.py b/examples/rl/drive/inference/contrib_policy/filter_obs.py index e366355c16..4be368171e 100644 --- a/examples/rl/drive/inference/contrib_policy/filter_obs.py +++ b/examples/rl/drive/inference/contrib_policy/filter_obs.py @@ -1,7 +1,7 @@ import math from typing import Any, Dict, Sequence, Tuple -import gym +import gymnasium as gym import numpy as np from smarts.core.agent_interface import RGB diff --git a/examples/rl/drive/inference/contrib_policy/format_action.py b/examples/rl/drive/inference/contrib_policy/format_action.py index 646f43f8f8..d1603d0862 100644 --- a/examples/rl/drive/inference/contrib_policy/format_action.py +++ b/examples/rl/drive/inference/contrib_policy/format_action.py @@ -1,6 +1,6 @@ from typing import Callable, Tuple -import gym +import gymnasium as gym import numpy as np from smarts.core.controllers import ActionSpaceType diff --git a/examples/rl/drive/inference/contrib_policy/frame_stack.py b/examples/rl/drive/inference/contrib_policy/frame_stack.py index 9da17cd193..8f7d8d7193 100644 --- a/examples/rl/drive/inference/contrib_policy/frame_stack.py +++ b/examples/rl/drive/inference/contrib_policy/frame_stack.py @@ -1,7 +1,7 @@ import copy from collections import deque -import gym +import gymnasium as gym import numpy as np diff --git a/examples/rl/drive/inference/contrib_policy/make_dict.py b/examples/rl/drive/inference/contrib_policy/make_dict.py index f995c93afc..599a79719f 100644 --- a/examples/rl/drive/inference/contrib_policy/make_dict.py +++ b/examples/rl/drive/inference/contrib_policy/make_dict.py @@ -1,6 +1,6 @@ from typing import Dict -import gym +import gymnasium as gym import numpy as np diff --git a/examples/rl/drive/inference/contrib_policy/network.py b/examples/rl/drive/inference/contrib_policy/network.py index 4ed514d1df..b1ae6bd98d 100644 --- a/examples/rl/drive/inference/contrib_policy/network.py +++ b/examples/rl/drive/inference/contrib_policy/network.py @@ -1,4 +1,4 @@ -import gym +import gymnasium as gym import torch as th import torch.nn as nn from stable_baselines3.common.preprocessing import get_flattened_obs_dim diff --git a/examples/rl/drive/train/preprocess.py b/examples/rl/drive/train/preprocess.py index ec2f890809..5cc0368d74 100644 --- a/examples/rl/drive/train/preprocess.py +++ b/examples/rl/drive/train/preprocess.py @@ -1,4 +1,4 @@ -import gym +import gymnasium as gym from contrib_policy.filter_obs import FilterObs from contrib_policy.format_action import FormatAction from contrib_policy.frame_stack import FrameStack diff --git a/examples/rl/drive/train/run.py b/examples/rl/drive/train/run.py index c42b9cab7c..8f52e949b7 100644 --- a/examples/rl/drive/train/run.py +++ b/examples/rl/drive/train/run.py @@ -12,7 +12,7 @@ from itertools import cycle, islice from typing import Any, Dict -import gym +import gymnasium as gym # Load inference module to register agent import inference diff --git a/examples/rl/platoon/inference/contrib_policy/filter_obs.py b/examples/rl/platoon/inference/contrib_policy/filter_obs.py index e366355c16..4be368171e 100644 --- a/examples/rl/platoon/inference/contrib_policy/filter_obs.py +++ b/examples/rl/platoon/inference/contrib_policy/filter_obs.py @@ -1,7 +1,7 @@ import math from typing import Any, Dict, Sequence, Tuple -import gym +import gymnasium as gym import numpy as np from smarts.core.agent_interface import RGB diff --git a/examples/rl/platoon/inference/contrib_policy/format_action.py b/examples/rl/platoon/inference/contrib_policy/format_action.py index 532770b3a1..305cb4bc05 100644 --- a/examples/rl/platoon/inference/contrib_policy/format_action.py +++ b/examples/rl/platoon/inference/contrib_policy/format_action.py @@ -1,6 +1,6 @@ from typing import Callable, Tuple -import gym +import gymnasium as gym import numpy as np from smarts.core.controllers import ActionSpaceType diff --git a/examples/rl/platoon/inference/contrib_policy/frame_stack.py b/examples/rl/platoon/inference/contrib_policy/frame_stack.py index 9da17cd193..8f7d8d7193 100644 --- a/examples/rl/platoon/inference/contrib_policy/frame_stack.py +++ b/examples/rl/platoon/inference/contrib_policy/frame_stack.py @@ -1,7 +1,7 @@ import copy from collections import deque -import gym +import gymnasium as gym import numpy as np diff --git a/examples/rl/platoon/inference/contrib_policy/make_dict.py b/examples/rl/platoon/inference/contrib_policy/make_dict.py index f995c93afc..599a79719f 100644 --- a/examples/rl/platoon/inference/contrib_policy/make_dict.py +++ b/examples/rl/platoon/inference/contrib_policy/make_dict.py @@ -1,6 +1,6 @@ from typing import Dict -import gym +import gymnasium as gym import numpy as np diff --git a/examples/rl/platoon/inference/contrib_policy/network.py b/examples/rl/platoon/inference/contrib_policy/network.py index 4ed514d1df..b1ae6bd98d 100644 --- a/examples/rl/platoon/inference/contrib_policy/network.py +++ b/examples/rl/platoon/inference/contrib_policy/network.py @@ -1,4 +1,4 @@ -import gym +import gymnasium as gym import torch as th import torch.nn as nn from stable_baselines3.common.preprocessing import get_flattened_obs_dim diff --git a/examples/rl/platoon/train/preprocess.py b/examples/rl/platoon/train/preprocess.py index 5a79d78960..1b4521e88d 100644 --- a/examples/rl/platoon/train/preprocess.py +++ b/examples/rl/platoon/train/preprocess.py @@ -1,4 +1,4 @@ -import gym +import gymnasium as gym from contrib_policy.filter_obs import FilterObs from contrib_policy.format_action import FormatAction from contrib_policy.frame_stack import FrameStack diff --git a/examples/rl/platoon/train/run.py b/examples/rl/platoon/train/run.py index 6ffcc140c8..fe4ac7a473 100644 --- a/examples/rl/platoon/train/run.py +++ b/examples/rl/platoon/train/run.py @@ -13,7 +13,7 @@ from itertools import cycle, islice from typing import Any, Dict -import gym +import gymnasium as gym # Load inference module to register agent import inference diff --git a/setup.cfg b/setup.cfg index 93e463ba8e..400e4f3507 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,6 @@ install_requires = pybullet>=3,<4.0 # The following are planned for removal - gym>=0.17.3,<=0.19.0 cloudpickle>=1.3.0,<=2.1.0 scipy @@ -71,7 +70,6 @@ dev = diagnostic = py-cpuinfo==9.0.0 mdutils==1.4.0 - gym>=0.17.3,<=0.19.0 matplotlib>=3.2.2 doc = myst-parser>=0.18.1 @@ -90,13 +88,12 @@ gym = opendrive = opendrive2lanelet>=1.2.1 Rtree>=0.9.7 -remote_agent = - grpcio==1.32.0 rllib = opencv-python==4.1.2.30 opencv-python-headless==4.1.2.30 ray[rllib]~=2.5.0 tensorflow-probability + tensorflow>=2.12.0 ray = ray~=2.5.0 ros = @@ -136,7 +133,6 @@ all = %(extras)s %(gym)s %(opendrive)s - %(remote_agent)s %(rllib)s # %(ray)s # incompatible with [rllib] for now %(ros)s diff --git a/smarts/env/__init__.py b/smarts/env/__init__.py index 707acddb4a..1eb3d85fe3 100644 --- a/smarts/env/__init__.py +++ b/smarts/env/__init__.py @@ -18,12 +18,5 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from gym.envs.registration import register - -register( - id="hiway-v0", - entry_point="smarts.env.hiway_env:HiWayEnv", -) - # Do NOT remove. import smarts.env.gymnasium diff --git a/smarts/env/gymnasium/hiway_env_v1.py b/smarts/env/gymnasium/hiway_env_v1.py index 1b749533de..8d99707ed5 100644 --- a/smarts/env/gymnasium/hiway_env_v1.py +++ b/smarts/env/gymnasium/hiway_env_v1.py @@ -39,7 +39,8 @@ import gymnasium as gym import numpy as np -from gym import spaces + +from gymnasium import spaces from gymnasium.core import ActType, ObsType from gymnasium.envs.registration import EnvSpec diff --git a/smarts/env/gymnasium/wrappers/api_reversion.py b/smarts/env/gymnasium/wrappers/api_reversion.py index 110ccfbc0a..74a2bb22f7 100644 --- a/smarts/env/gymnasium/wrappers/api_reversion.py +++ b/smarts/env/gymnasium/wrappers/api_reversion.py @@ -21,7 +21,7 @@ # THE SOFTWARE. from typing import Any, Dict, SupportsFloat, Tuple -import gym +import gymnasium as gym class Api021Reversion(gym.Wrapper): diff --git a/smarts/env/hiway_env.py b/smarts/env/hiway_env.py deleted file mode 100644 index d8c74222f4..0000000000 --- a/smarts/env/hiway_env.py +++ /dev/null @@ -1,328 +0,0 @@ -# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import logging -import os -import warnings -from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Set, Tuple, Union - -import gym - -from envision import types as envision_types -from envision.client import Client as Envision -from envision.data_formatter import EnvisionDataFormatterArgs -from smarts.core import seed as smarts_seed -from smarts.core.agent_interface import AgentInterface -from smarts.core.local_traffic_provider import LocalTrafficProvider -from smarts.core.observations import Observation -from smarts.core.scenario import Scenario -from smarts.core.smarts import SMARTS -from smarts.env.utils.action_conversion import ActionOptions, ActionSpacesFormatter -from smarts.zoo.agent_spec import AgentSpec - - -class HiWayEnv(gym.Env): - """A generic environment for various driving tasks simulated by SMARTS. - - Args: - scenarios (Sequence[str]): A list of scenario directories that - will be simulated. - agent_specs (Dict[str, AgentSpec]): Specification of the agents - that will run in the environment. (Deprecated in favor of - agent_interfaces) - sim_name (Optional[str], optional): Simulation name. Defaults to - None. - shuffle_scenarios (bool, optional): If true, order of scenarios - will be randomized, else it will be maintained. Defaults to - True. - headless (bool, optional): If True, disables visualization in - Envision. Defaults to False. - visdom (bool): Deprecated. Use SMARTS_VISDOM_ENABLED. - fixed_timestep_sec (Optional[float], optional): Step duration for - all components of the simulation. May be None if time deltas - are externally-driven. Defaults to None. - seed (int, optional): Random number generator seed. Defaults to 42. - num_external_sumo_clients (int, optional): Number of SUMO clients - beyond SMARTS. Defaults to 0. - sumo_headless (bool, optional): If True, disables visualization in - SUMO GUI. Defaults to True. - sumo_port (Optional[str], optional): SUMO port. Defaults to None. - sumo_auto_start (bool, optional): Automatic starting of SUMO. - Defaults to True. - envision_endpoint (Optional[str], optional): Envision's uri. - Defaults to None. - envision_record_data_replay_path (Optional[str], optional): - Envision's data replay output directory. Defaults to None. - timestep_sec (Optional[float], optional): [description]. Defaults - to None. - agent_interfaces (Dict[str, AgentInterface]): Specification of the agents - needs that will be used to configure the environment. - action_options (ActionOptions, str): Specifies format of the action space - of the environment. - """ - - metadata = {"render.modes": ["human"]} - """Metadata for gym's use""" - - def __init__( - self, - scenarios: Sequence[str], - agent_specs: Optional[Dict[str, AgentSpec]] = None, # (deprecated) - sim_name: Optional[str] = None, - shuffle_scenarios: bool = True, - headless: bool = True, - visdom: bool = False, - fixed_timestep_sec: Optional[float] = None, - seed: int = 42, - num_external_sumo_clients: int = 0, - sumo_headless: bool = True, - sumo_port: Optional[str] = None, - sumo_auto_start: bool = True, - envision_endpoint: Optional[str] = None, - envision_record_data_replay_path: Optional[str] = None, - agent_interfaces: Optional[Dict[str, AgentInterface]] = None, - timestep_sec: Optional[ - float - ] = None, # for backwards compatibility (deprecated) - action_options: Union[ActionOptions, str] = ActionOptions.unformatted, - ): - self._log = logging.getLogger(self.__class__.__name__) - self.seed(seed) - - if timestep_sec and not fixed_timestep_sec: - warnings.warn( - "timestep_sec has been deprecated in favor of fixed_timestep_sec. Please update your code.", - category=DeprecationWarning, - ) - if agent_specs is not None: - warnings.warn( - "agent_specs has been deprecated in favor of agent_interfaces. Please update your code.", - category=DeprecationWarning, - ) - if not fixed_timestep_sec: - fixed_timestep_sec = timestep_sec or 0.1 - - self._agent_interfaces = agent_interfaces or {} - if isinstance(agent_interfaces, dict): - self._agent_specs = { - a_id: AgentSpec(a_inter) for a_id, a_inter in agent_interfaces.items() - } - elif isinstance(agent_specs, dict): - self._agent_specs = agent_specs - self._agent_interfaces = { - agent_id: agent.interface for agent_id, agent in agent_specs.items() - } - else: - raise TypeError( - f"agent_interface must be supplied as `{Dict[str, AgentInterface]}`." - ) - self._dones_registered = 0 - - scenarios = [str(Path(scenario).resolve()) for scenario in scenarios] - self._scenarios_iterator = Scenario.scenario_variations( - scenarios, - list(self._agent_interfaces.keys()), - shuffle_scenarios, - ) - self._action_space_formatter = ActionSpacesFormatter( - agent_interfaces=self.agent_interfaces, - action_options=action_options, - ) - - envision_client = None - if not headless or envision_record_data_replay_path: - envision_client = Envision( - endpoint=envision_endpoint, - sim_name=sim_name, - output_dir=envision_record_data_replay_path, - headless=headless, - data_formatter_args=EnvisionDataFormatterArgs( - "base", enable_reduction=False - ), - ) - preamble = envision_types.Preamble(scenarios=scenarios) - envision_client.send(preamble) - - self._env_renderer = None - - traffic_sims = [] - if Scenario.any_support_sumo_traffic(scenarios): - from smarts.core.sumo_traffic_simulation import SumoTrafficSimulation - - sumo_traffic = SumoTrafficSimulation( - headless=sumo_headless, - time_resolution=fixed_timestep_sec, - num_external_sumo_clients=num_external_sumo_clients, - sumo_port=sumo_port, - auto_start=sumo_auto_start, - ) - traffic_sims += [sumo_traffic] - smarts_traffic = LocalTrafficProvider() - traffic_sims += [smarts_traffic] - - self._smarts = SMARTS( - agent_interfaces=self._agent_interfaces, - traffic_sims=traffic_sims, - envision=envision_client, - visdom=visdom, - fixed_timestep_sec=fixed_timestep_sec, - ) - - @property - def agent_ids(self) -> Set[str]: - """Agent ids of all agents that potentially will be in the environment. - Returns: - (Set[str]): Agent ids. - """ - return set(self._agent_interfaces) - - @property - def agent_interfaces(self) -> Dict[str, AgentInterface]: - """Agents' interfaces used in this simulation. - - Returns: - (Dict[str, AgentInterface]): Agents' interfaces. - """ - return self._agent_interfaces - - @property - def agent_specs(self) -> Dict[str, AgentSpec]: - """Agents' specifications used in this simulation. - - Returns: - (Dict[str, AgentSpec]): Agents' specifications. - """ - return self._agent_specs - - @property - def scenario_log(self) -> Dict[str, Union[float, str]]: - """Simulation steps log. - - Returns: - Dict[str, Union[float,str]]: A dictionary with the following keys. - - + fixed_timestep_sec: Simulation time-step. - + scenario_map: Name of the current scenario. - + scenario_traffic: Traffic spec(s) used. - + mission_hash: Hash identifier for the current scenario. - """ - - scenario = self._smarts.scenario - return { - "fixed_timestep_sec": self._smarts.fixed_timestep_sec, - "scenario_map": scenario.name, - "scenario_traffic": ",".join(map(os.path.basename, scenario.traffic_specs)), - "mission_hash": str(hash(frozenset(scenario.missions.items()))), - } - - @property - def scenario(self) -> Scenario: - """Returns underlying scenario. - - Returns: - Current simulated scenario. - """ - return self._smarts.scenario - - def seed(self, seed: int) -> int: - """Sets random number generator seed number. - - Args: - seed (int): Seed number. - - Returns: - int: Seed number. - """ - smarts_seed(seed) - return seed - - def step( - self, agent_actions - ) -> Tuple[ - Dict[str, Observation], Dict[str, float], Dict[str, bool], Dict[str, Any] - ]: - """Steps the environment. - - Args: - agent_actions (Dict[str, Any]): Action taken for each agent. - - Returns: - Tuple[ Dict[str, Observation], Dict[str, float], Dict[str, bool], Dict[str, Any] ]: - Observations, rewards, dones, and :spelling:ignore:`infos` for active agents. - """ - assert isinstance(agent_actions, dict) and all( - isinstance(key, str) for key in agent_actions.keys() - ), "Expected Dict[str, any]" - - observations, rewards, dones, extras = self._smarts.step( - self._action_space_formatter.format(agent_actions) - ) - - infos = { - agent_id: { - "score": value, - "env_obs": observations[agent_id], - "done": dones[agent_id], - } - for agent_id, value in extras["scores"].items() - } - - if self._env_renderer is not None: - self._env_renderer.step(observations, rewards, dones, infos) - - for done in dones.values(): - self._dones_registered += 1 if done else 0 - - dones["__all__"] = self._dones_registered >= len(self._agent_specs) - - return observations, rewards, dones, infos - - def reset(self) -> Dict[str, Observation]: - """Reset the environment and initialize to the next scenario. - - Returns: - Dict[str, Observation]: Agents' observation. - """ - scenario = next(self._scenarios_iterator) - - self._dones_registered = 0 - observations = self._smarts.reset(scenario) - - if self._env_renderer is not None: - self._env_renderer.reset(observations) - - return observations - - def render(self, mode="human"): - """Renders according to metadata requirements.""" - - if "rgb_array" in self.metadata["render.modes"]: - if self._env_renderer is None: - from smarts.env.utils.record import AgentCameraRGBRender - - self._env_renderer = AgentCameraRGBRender(self) - - return self._env_renderer.render(env=self) - - def close(self): - """Closes the environment and releases all resources.""" - if self._smarts is not None: - self._smarts.destroy() diff --git a/smarts/env/tests/test_hiway_env.py b/smarts/env/tests/test_hiway_env.py deleted file mode 100644 index eae430909d..0000000000 --- a/smarts/env/tests/test_hiway_env.py +++ /dev/null @@ -1,92 +0,0 @@ -# MIT License -# -# Copyright (C) 2021. Huawei Technologies Co., Ltd. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -import gym -import pytest - -from smarts.core.agent import Agent -from smarts.core.agent_interface import AgentInterface, AgentType -from smarts.core.utils.episodes import episodes -from smarts.env.hiway_env import HiWayEnv -from smarts.zoo.agent_spec import AgentSpec - -AGENT_ID = "Agent-007" -MAX_EPISODES = 3 - - -@pytest.fixture -def agent_spec(): - return AgentSpec( - interface=AgentInterface.from_type(AgentType.Laner, max_episode_steps=100), - agent_builder=lambda: Agent.from_function(lambda _: "keep_lane"), - ) - - -@pytest.fixture -def env(agent_spec: AgentSpec): - agent_interfaces = {AGENT_ID: agent_spec.interface} - agent_ids = set(agent_interfaces) - env: HiWayEnv = gym.make( - "smarts.env:hiway-v0", - scenarios=["scenarios/sumo/loop"], - agent_interfaces=agent_interfaces, - headless=True, - fixed_timestep_sec=0.01, - ) - assert isinstance(env, HiWayEnv) - assert not (agent_ids - set(env.agent_interfaces)) - matching_items = [ - env.agent_interfaces[k] == agent_interfaces[k] - for k in env.agent_interfaces - if k in agent_interfaces - ] - assert all(matching_items) - assert len(env.agent_specs) == len(agent_interfaces) - assert not (agent_ids - env.agent_ids) - yield env - env.close() - - -def test_hiway_env(env: HiWayEnv, agent_spec: AgentSpec): - episode = None - for episode in episodes(n=MAX_EPISODES): - agent = agent_spec.build_agent() - observations = env.reset() - episode.record_scenario(env.scenario_log) - - dones = {"__all__": False} - while not dones["__all__"]: - obs = observations[AGENT_ID] - observations, rewards, dones, infos = env.step({AGENT_ID: agent.act(obs)}) - - # Reward is currently the delta in distance travelled by the agent. - # Ensure that it is infact a delta and not total distance travelled - # since this bug has appeared a few times. Verify by ensuring the - # reward does not grow unbounded. - assert all( - [-3 < reward < 3 for reward in rewards.values()] - ), f"Expected bounded reward per timestep, but got {rewards}." - - episode.record_step(observations, rewards, dones, dones, infos) - - assert episode is not None and episode.index == ( - MAX_EPISODES - 1 - ), "Simulation must cycle through to the final episode." diff --git a/smarts/env/wrappers/gif_recorder.py b/smarts/env/wrappers/gif_recorder.py index aac5ced746..c4b43ccfd3 100644 --- a/smarts/env/wrappers/gif_recorder.py +++ b/smarts/env/wrappers/gif_recorder.py @@ -22,7 +22,7 @@ import logging import sys -import gym +import gymnasium as gym import numpy as np try: diff --git a/smarts/env/wrappers/record_video.py b/smarts/env/wrappers/record_video.py index 74fdf234a1..08fd0e311b 100644 --- a/smarts/env/wrappers/record_video.py +++ b/smarts/env/wrappers/record_video.py @@ -34,9 +34,9 @@ import os from typing import Callable -import gym -from gym import logger -from gym.wrappers.monitoring import video_recorder +import gymnasium as gym +from gymnasium import logger +from gymnasium.wrappers.monitoring import video_recorder def capped_cubic_video_schedule(episode_id, cap=1000): diff --git a/smarts/env/wrappers/recorder_wrapper.py b/smarts/env/wrappers/recorder_wrapper.py index 1f95c5f409..fa07ef1e1a 100644 --- a/smarts/env/wrappers/recorder_wrapper.py +++ b/smarts/env/wrappers/recorder_wrapper.py @@ -22,8 +22,7 @@ import os from pathlib import Path -import gym -import gym.envs +import gymnasium as gym from smarts.env.wrappers.gif_recorder import GifRecorder diff --git a/smarts/zoo/agent_spec.py b/smarts/zoo/agent_spec.py index 72058e95b0..b44f23e4f6 100644 --- a/smarts/zoo/agent_spec.py +++ b/smarts/zoo/agent_spec.py @@ -43,7 +43,7 @@ class AgentSpec(object): ) env = gym.make( - "smarts.env:hiway-v0", + "smarts.env:hiway-v1", scenarios=["scenarios/sumo/loop"], agent_specs={agent_id: agent_spec}, ) diff --git a/zoo/evaluation/npc_ego_example.py b/zoo/evaluation/npc_ego_example.py index 893d025344..233e6d42f9 100644 --- a/zoo/evaluation/npc_ego_example.py +++ b/zoo/evaluation/npc_ego_example.py @@ -2,7 +2,7 @@ import logging from importlib import import_module -import gym +import gymnasium as gym from smarts.core.utils.episodes import episodes diff --git a/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_test.py b/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_test.py index d194912ef2..e94c40c54f 100644 --- a/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_test.py +++ b/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_test.py @@ -25,7 +25,7 @@ import logging import warnings -import gym +import gymnasium as gym import numpy as np import tensorflow as tf from ac_network import ActorNetwork, CriticNetwork diff --git a/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_train.py b/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_train.py index 1bd85a8f3c..41894f34cb 100644 --- a/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_train.py +++ b/zoo/policies/cross-rl-agent/cross_rl_agent/train/run_train.py @@ -26,7 +26,7 @@ import pickle import warnings -import gym +import gymnasium as gym import numpy as np import tensorflow as tf from ac_network import ActorNetwork, CriticNetwork diff --git a/zoo/policies/rl-agent/rl_agent/agent.py b/zoo/policies/rl-agent/rl_agent/agent.py index 4ea6c0eaaa..a94e74abe4 100644 --- a/zoo/policies/rl-agent/rl_agent/agent.py +++ b/zoo/policies/rl-agent/rl_agent/agent.py @@ -3,7 +3,7 @@ """ import pickle -import gym +import gymnasium as gym import tensorflow.compat.v1 as tf tf.disable_v2_behavior() diff --git a/zoo/policies/rl-agent/rl_agent/lane_space.py b/zoo/policies/rl-agent/rl_agent/lane_space.py index 7550133d62..b1f0dc9e18 100644 --- a/zoo/policies/rl-agent/rl_agent/lane_space.py +++ b/zoo/policies/rl-agent/rl_agent/lane_space.py @@ -3,7 +3,7 @@ """ import math -import gym +import gymnasium as gym import numpy as np from smarts.core.agent_interface import OGM, AgentInterface, NeighborhoodVehicles From f12ba647c2135777d1b00d27ae0decb8aa12736e Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 28 Jun 2023 18:18:29 -0400 Subject: [PATCH 2/7] Fix tests and docs --- docs/sim/env.rst | 32 +------------------------------- examples/rl/platoon/train/run.py | 2 ++ 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/docs/sim/env.rst b/docs/sim/env.rst index 4afe78f3e8..dd929a0992 100644 --- a/docs/sim/env.rst +++ b/docs/sim/env.rst @@ -9,41 +9,11 @@ Base environments SMARTS environment module is defined in :mod:`~smarts.env` package. Currently SMARTS provides two kinds of training environments, namely: -+ ``HiWayEnv`` utilizing a ``gymnasium.Env`` interface ++ ``HiWayEnvV1`` utilizing a ``gymnasium.Env`` interface + ``RLlibHiwayEnv`` customized for `RLlib `_ training .. image:: ../_static/env.png -HiWayEnv -^^^^^^^^ - -``HiWayEnv`` inherits class ``gymnasium.Env`` and supports gym APIs like ``reset``, ``step``, ``close``. An usage example is shown below. -Refer to :class:`~smarts.env.hiway_env.HiWayEnv` for more details. - -.. code-block:: python - - import gymnasium as gym - # Make env - env = gym.make( - "smarts.env:hiway-v1", # Env entry name. - scenarios=[scenario_path], # List of paths to scenario folders. - agent_interfaces={AGENT_ID: agent_spec.interface}, # Dictionary mapping agents to agent interfaces. - headless=False, # False to enable Envision visualization of the environment. - seed=42, # RNG seed. Seeds are set at the start of simulation, and never automatically re-seeded. - ) - - # Reset env and build agent. - observations = env.reset() - agent = agent_spec.build_agent() - - # Step env. - agent_obs = observations[AGENT_ID] - agent_action = agent.act(agent_obs) - observations, rewards, dones, infos = env.step({AGENT_ID: agent_action}) - - # Close env. - env.close() - HiWayEnvV1 ^^^^^^^^^^ diff --git a/examples/rl/platoon/train/run.py b/examples/rl/platoon/train/run.py index fe4ac7a473..9102573416 100644 --- a/examples/rl/platoon/train/run.py +++ b/examples/rl/platoon/train/run.py @@ -15,6 +15,8 @@ import gymnasium as gym +sys.modules["gym"] = gym + # Load inference module to register agent import inference import stable_baselines3 as sb3lib From 0f3d1cd68784f18da36859e75f172b80c394acfe Mon Sep 17 00:00:00 2001 From: Saul Field Date: Fri, 30 Jun 2023 12:13:17 -0400 Subject: [PATCH 3/7] Make gymnasium optional, remove api_reversion wrapper --- CHANGELOG.md | 9 ++-- docs/sim/env.rst | 13 ------ examples/rl/drive/train/env.py | 1 - examples/rl/platoon/train/env.py | 1 - setup.cfg | 15 ++++--- .../env/gymnasium/wrappers/api_reversion.py | 42 ------------------- 6 files changed, 13 insertions(+), 68 deletions(-) delete mode 100644 smarts/env/gymnasium/wrappers/api_reversion.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 13339197b3..09dc917889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,17 +19,20 @@ Copy and pasting the git commit messages is __NOT__ enough. - Renamed `rllib/rllib.py` to `rllib/pg_pbt_example.py`. - Loosened constraint of `gymnasium` from `==0.27.0` to `>=0.26.3`. - `LaneFollowingController` now uses a different pole placement method to compute lateral/heading gains. Numerical behaviour is unchanged. Performance is slightly faster. -- Changed all uses of `gym` to use `gymnasium`. - Upgraded Stable Baselines3 from v1.7.0 to v2.0.0, and switched to Gymnasium backend, in Drive and VehicleFollowing RL examples. +- Changed all uses of `gym` to use `gymnasium`. +- Changed `gymnasium` to be an optional dependency. Use `pip install -e .[gymnasium]` to install it. +- Renamed the `[gym]` optional install to `[gif_recorder]`. ### Deprecated ### Fixed - Missing neighborhood vehicle state `'lane_id'` is now added to the `hiway-v1` formatted observations. - Fixed a regression where `pybullet` build time messages returned. ### Removed -- Removed `hiway-v0` env. - Removed `TruncatedDistribution` from scenario studio. - Removed `scipy` as a core package dependency. -- Removed `gym` as a core dependency. +- Removed `gym` as a core package dependency. +- Removed `hiway-v0` env. +- Removed `Api021Reversion` gym wrapper class. ### Security ## [1.2.0] # 2023-06-14 diff --git a/docs/sim/env.rst b/docs/sim/env.rst index dd929a0992..6b61e59d92 100644 --- a/docs/sim/env.rst +++ b/docs/sim/env.rst @@ -48,19 +48,6 @@ exactly matches the `env.observation_space`, and `ObservationOptions.multi_agent # Close env. env.close() -To use this environment with certain frameworks you may want to convert the environment back into a 0.21 api version of gym. -This can be done with :class:`~smarts.env.gymnasium.wrappers.api_reversion.Api021Reversion`. - -.. code-block:: python - - import gymnasium as gym - # Make env - env = gym.make( - "smarts.env:hiway-v1", # Env entry name. - scenarios=[scenario_path], # List of paths to scenario folders. - ) - env = Api021Reversion(env) # Turns the environment into roughly a 0.21 gym environment - RLlibHiwayEnv ^^^^^^^^^^^^^ diff --git a/examples/rl/drive/train/env.py b/examples/rl/drive/train/env.py index 7f5469d259..8829f4a7b0 100644 --- a/examples/rl/drive/train/env.py +++ b/examples/rl/drive/train/env.py @@ -14,7 +14,6 @@ def make_env(env_id, scenario, agent_spec: AgentSpec, config, seed): from reward import Reward from stable_baselines3.common.monitor import Monitor - from smarts.env.gymnasium.wrappers.api_reversion import Api021Reversion from smarts.env.gymnasium.wrappers.single_agent import SingleAgent env = gym.make( diff --git a/examples/rl/platoon/train/env.py b/examples/rl/platoon/train/env.py index 7f5469d259..8829f4a7b0 100644 --- a/examples/rl/platoon/train/env.py +++ b/examples/rl/platoon/train/env.py @@ -14,7 +14,6 @@ def make_env(env_id, scenario, agent_spec: AgentSpec, config, seed): from reward import Reward from stable_baselines3.common.monitor import Monitor - from smarts.env.gymnasium.wrappers.api_reversion import Api021Reversion from smarts.env.gymnasium.wrappers.single_agent import SingleAgent env = gym.make( diff --git a/setup.cfg b/setup.cfg index ed75300eb9..a6ee95109f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,12 +35,8 @@ install_requires = PyYAML>=3.13 twisted>=21.7.0 # for scenario requirements.txt files - # The following are planned to be made optional - gymnasium>=0.26.3 - pybullet>=3,<4.0 - - # The following are planned for removal - cloudpickle>=1.3.0,<=2.1.0 + pybullet>=3,<4.0 # planned to be made optional + cloudpickle>=1.3.0,<=2.1.0 # planned for removal [options.packages.find] exclude = @@ -82,8 +78,10 @@ envision = websocket-client>=1.2.1 ijson>=3.1.4 extras = pynput>=1.7.4 # Used by HumanKeyboardAgent -gym = +gif_recorder = moviepy == 1.0.3 +gymnasium = + gymnasium>=0.26.3 opendrive = opendrive2lanelet>=1.2.1 Rtree>=0.9.7 @@ -130,7 +128,8 @@ all = %(doc)s %(envision)s %(extras)s - %(gym)s + %(gif_recorder)s + %(gymnasium)s %(opendrive)s %(rllib)s # %(ray)s # incompatible with [rllib] for now diff --git a/smarts/env/gymnasium/wrappers/api_reversion.py b/smarts/env/gymnasium/wrappers/api_reversion.py deleted file mode 100644 index 74a2bb22f7..0000000000 --- a/smarts/env/gymnasium/wrappers/api_reversion.py +++ /dev/null @@ -1,42 +0,0 @@ -# MIT License -# -# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -from typing import Any, Dict, SupportsFloat, Tuple - -import gymnasium as gym - - -class Api021Reversion(gym.Wrapper): - """This wrapper reverts the API of a gym|gymnasium environment to v0.21 of gym.""" - - def step(self, action) -> Tuple[Dict, SupportsFloat, bool, Dict[str, Any]]: - """Converts :meth:`step` return from ``obs, reward, term, trunc, info`` to ``obs, reward, done, info``.""" - obs, reward, terminated, _, info = super().step(action) - return obs, reward, terminated, info - - def reset(self, **kwargs) -> Dict[str, Tuple[Any]]: - """Converts :meth:`reset` return from ``obs, info`` to ``obs``.""" - obs, _ = super().reset() - return obs - - def render(self, mode="human", **kwargs) -> Any: - """Uses the :meth:`render` of the env that can be overwritten to change the returned data.""" - return self.env.render() From cd371adff0f91201069916574dc2bd40c2561acd Mon Sep 17 00:00:00 2001 From: Saul Field Date: Fri, 30 Jun 2023 12:21:29 -0400 Subject: [PATCH 4/7] Fix CI --- .github/workflows/ci-base-tests-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-base-tests-linux.yml b/.github/workflows/ci-base-tests-linux.yml index 416ec60a04..6a08adb398 100644 --- a/.github/workflows/ci-base-tests-linux.yml +++ b/.github/workflows/ci-base-tests-linux.yml @@ -30,7 +30,7 @@ jobs: . ${{env.venv_dir}}/bin/activate pip install --upgrade pip pip install wheel==0.38.4 - pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gym,argoverse,envision,sumo] + pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,argoverse,envision,sumo] if echo ${{matrix.tests}} | grep -q -e "test_rllib_hiway_env.py" -e "test_examples.py"; then pip install -e .[rllib]; fi if echo ${{matrix.tests}} | grep -q -e "/smarts/ray"; then pip install -e .[ray]; fi - name: Build scenarios @@ -84,7 +84,7 @@ jobs: pip install --upgrade pip pip install wheel==0.38.4 pip install -e ./../../../.[camera_obs,argoverse,sumo,test] - pip install -e ./inference/ + pip install -e ./inference/ - name: Run smoke tests run: | cd ${GITHUB_WORKSPACE}/examples/rl/${{matrix.tests}} From a3dd6919907503716562619a0fe268f4255a1701 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Fri, 30 Jun 2023 12:37:18 -0400 Subject: [PATCH 5/7] Fix CI --- .github/workflows/ci-base-tests-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-base-tests-linux.yml b/.github/workflows/ci-base-tests-linux.yml index 6a08adb398..685e3229f1 100644 --- a/.github/workflows/ci-base-tests-linux.yml +++ b/.github/workflows/ci-base-tests-linux.yml @@ -30,7 +30,7 @@ jobs: . ${{env.venv_dir}}/bin/activate pip install --upgrade pip pip install wheel==0.38.4 - pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,argoverse,envision,sumo] + pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gymnasium,argoverse,envision,sumo] if echo ${{matrix.tests}} | grep -q -e "test_rllib_hiway_env.py" -e "test_examples.py"; then pip install -e .[rllib]; fi if echo ${{matrix.tests}} | grep -q -e "/smarts/ray"; then pip install -e .[ray]; fi - name: Build scenarios From f5830052d33dbb03517a6db10e3382cbead9f586 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Fri, 30 Jun 2023 12:50:35 -0400 Subject: [PATCH 6/7] Fix CI --- .github/workflows/ci-base-tests-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-base-tests-linux.yml b/.github/workflows/ci-base-tests-linux.yml index 685e3229f1..12925b8846 100644 --- a/.github/workflows/ci-base-tests-linux.yml +++ b/.github/workflows/ci-base-tests-linux.yml @@ -30,7 +30,7 @@ jobs: . ${{env.venv_dir}}/bin/activate pip install --upgrade pip pip install wheel==0.38.4 - pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gymnasium,argoverse,envision,sumo] + pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gif_recorder,gymnasium,argoverse,envision,sumo] if echo ${{matrix.tests}} | grep -q -e "test_rllib_hiway_env.py" -e "test_examples.py"; then pip install -e .[rllib]; fi if echo ${{matrix.tests}} | grep -q -e "/smarts/ray"; then pip install -e .[ray]; fi - name: Build scenarios From add65bbc1d61b0b2cfcbef468f695c29b400b21a Mon Sep 17 00:00:00 2001 From: Saul Field Date: Fri, 30 Jun 2023 16:25:03 -0400 Subject: [PATCH 7/7] Try removing tf dependency --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index a6ee95109f..fe501c49c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -90,7 +90,6 @@ rllib = opencv-python-headless==4.1.2.30 ray[rllib]~=2.5.0 tensorflow-probability - tensorflow>=2.12.0 ray = ray~=2.5.0 ros =