Move optimizers.py and lr_schedulers.py to d3rlpy.optimizers directory

d3rlpy/__init__.py

@@ -15,6 +15,7 @@
     models,
     notebook_utils,
     ope,
+    optimizers,
     preprocessing,
     tokenizers,
     types,
@@ -34,6 +35,7 @@
     "logging",
     "metrics",
     "models",
+    "optimizers",
     "notebook_utils",
     "ope",
     "preprocessing",

d3rlpy/algos/qlearning/awac.py

@@ -9,9 +9,9 @@
     create_normal_policy,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
 from ...models.torch import Parameter
+from ...optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.awac_impl import AWACImpl
@@ -53,9 +53,9 @@ class AWACConfig(LearnableConfig):
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/bc.py

@@ -8,7 +8,7 @@
     create_normal_policy,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.bc_impl import (
@@ -39,7 +39,7 @@ class BCConfig(LearnableConfig):
 
     Args:
         learning_rate (float): Learing rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.
@@ -129,7 +129,7 @@ class DiscreteBCConfig(LearnableConfig):
 
     Args:
         learning_rate (float): Learing rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.

d3rlpy/algos/qlearning/bcq.py

@@ -11,9 +11,9 @@
     create_vae_encoder,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
 from ...models.torch import CategoricalPolicy, compute_output_size
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.bcq_impl import (
@@ -110,11 +110,11 @@ class BCQConfig(LearnableConfig):
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
         imitator_learning_rate (float): Learning rate for Conditional VAE.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        imitator_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        imitator_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the conditional VAE.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.
@@ -316,7 +316,7 @@ class DiscreteBCQConfig(LearnableConfig):
             Observation preprocessor.
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         learning_rate (float): Learning rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory or str):
             Encoder factory.

d3rlpy/algos/qlearning/bear.py

@@ -11,8 +11,8 @@
     create_vae_encoder,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.bear_impl import BEARImpl, BEARModules
@@ -74,15 +74,15 @@ class BEARConfig(LearnableConfig):
             function.
         temp_learning_rate (float): Learning rate for temperature parameter.
         alpha_learning_rate (float): Learning rate for :math:`\alpha`.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        imitator_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        imitator_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the behavior policy.
-        temp_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        temp_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the temperature.
-        alpha_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        alpha_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for :math:`\alpha`.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/cal_ql.py

@@ -43,13 +43,13 @@ class CalQLConfig(CQLConfig):
         temp_learning_rate (float):
             Learning rate for temperature parameter of SAC.
         alpha_learning_rate (float): Learning rate for :math:`\alpha`.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        temp_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        temp_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the temperature.
-        alpha_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        alpha_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for :math:`\alpha`.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/cql.py

@@ -10,8 +10,8 @@
     create_parameter,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.cql_impl import CQLImpl, CQLModules, DiscreteCQLImpl
@@ -74,13 +74,13 @@ class CQLConfig(LearnableConfig):
         temp_learning_rate (float):
             Learning rate for temperature parameter of SAC.
         alpha_learning_rate (float): Learning rate for :math:`\alpha`.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        temp_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        temp_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the temperature.
-        alpha_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        alpha_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for :math:`\alpha`.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.
@@ -260,7 +260,7 @@ class DiscreteCQLConfig(LearnableConfig):
             Observation preprocessor.
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         learning_rate (float): Learning rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.

d3rlpy/algos/qlearning/crr.py

@@ -7,8 +7,8 @@
     create_normal_policy,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.crr_impl import CRRImpl, CRRModules
@@ -72,9 +72,9 @@ class CRRConfig(LearnableConfig):
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/ddpg.py

@@ -7,8 +7,8 @@
     create_deterministic_policy,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.ddpg_impl import DDPGImpl, DDPGModules
@@ -55,9 +55,9 @@ class DDPGConfig(LearnableConfig):
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q function.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/dqn.py

@@ -4,8 +4,8 @@
 from ...constants import ActionSpace
 from ...models.builders import create_discrete_q_function
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.dqn_impl import DoubleDQNImpl, DQNImpl, DQNModules
@@ -34,7 +34,7 @@ class DQNConfig(LearnableConfig):
             Observation preprocessor.
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         learning_rate (float): Learning rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.
@@ -140,7 +140,7 @@ class DoubleDQNConfig(DQNConfig):
             Observation preprocessor.
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         learning_rate (float): Learning rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.

d3rlpy/algos/qlearning/iql.py

@@ -8,8 +8,8 @@
     create_value_function,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import MeanQFunctionFactory
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.iql_impl import IQLImpl, IQLModules
@@ -62,9 +62,9 @@ class IQLConfig(LearnableConfig):
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/nfq.py

@@ -4,8 +4,8 @@
 from ...constants import ActionSpace
 from ...models.builders import create_discrete_q_function
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.dqn_impl import DQNImpl, DQNModules
@@ -38,7 +38,7 @@ class NFQConfig(LearnableConfig):
             Observation preprocessor.
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         learning_rate (float): Learning rate.
-        optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory.
         encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory.

d3rlpy/algos/qlearning/plas.py

@@ -10,8 +10,8 @@
     create_vae_encoder,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.plas_impl import (
@@ -56,11 +56,11 @@ class PLASConfig(LearnableConfig):
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
         imitator_learning_rate (float): Learning rate for Conditional VAE.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        imitator_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        imitator_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the conditional VAE.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.
@@ -224,11 +224,11 @@ class PLASWithPerturbationConfig(PLASConfig):
         actor_learning_rate (float): Learning rate for policy function.
         critic_learning_rate (float): Learning rate for Q functions.
         imitator_learning_rate (float): Learning rate for Conditional VAE.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
-        imitator_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        imitator_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the conditional VAE.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.

d3rlpy/algos/qlearning/rebrac.py

@@ -7,8 +7,8 @@
     create_deterministic_policy,
 )
 from ...models.encoders import EncoderFactory, make_encoder_field
-from ...models.optimizers import OptimizerFactory, make_optimizer_field
 from ...models.q_functions import QFunctionFactory, make_q_func_field
+from ...optimizers.optimizers import OptimizerFactory, make_optimizer_field
 from ...types import Shape
 from .base import QLearningAlgoBase
 from .torch.ddpg_impl import DDPGModules
@@ -51,9 +51,9 @@ class ReBRACConfig(LearnableConfig):
         reward_scaler (d3rlpy.preprocessing.RewardScaler): Reward preprocessor.
         actor_learning_rate (float): Learning rate for a policy function.
         critic_learning_rate (float): Learning rate for Q functions.
-        actor_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        actor_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the actor.
-        critic_optim_factory (d3rlpy.models.optimizers.OptimizerFactory):
+        critic_optim_factory (d3rlpy.optimizers.OptimizerFactory):
             Optimizer factory for the critic.
         actor_encoder_factory (d3rlpy.models.encoders.EncoderFactory):
             Encoder factory for the actor.