From e4c5fe785d681232fe7f4e607b4ff42468916f47 Mon Sep 17 00:00:00 2001 From: Jan Stenner Date: Thu, 15 Jun 2023 14:30:49 +0200 Subject: [PATCH 1/5] example script for the controllers paper --- .../RL_Classical_Controllers_Merge_2.jl | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 examples/scripts/RL_Classical_Controllers_Merge_2.jl diff --git a/examples/scripts/RL_Classical_Controllers_Merge_2.jl b/examples/scripts/RL_Classical_Controllers_Merge_2.jl new file mode 100644 index 00000000..b8d8493a --- /dev/null +++ b/examples/scripts/RL_Classical_Controllers_Merge_2.jl @@ -0,0 +1,136 @@ +using ElectricGrid + +CM = [ 0. 0. 1. + 0. 0. 2. + -1. -2. 0.] + +R_load, L_load, _, _ = ParallelLoadImpedance(50e3, 0.95, 230) + +parameters = +Dict{Any, Any}( + "source" => Any[ + Dict{Any, Any}( + "pwr" => 200e3, + "control_type" => "RL", + "fltr" => "L", + #"L1" => 0.0008, + ), + Dict{Any, Any}( + "pwr" => 200e3, + "fltr" => "LC", + "control_type" => "classic", + "mode" => "Droop",), + ], + "grid" => Dict{Any, Any}( + "phase" => 3, + "ramp_end" => 0.04,) +) + + + + +function reference(t) + if t < 0.04 + return [0.0, 0.0, 0.0] + end + + θ = 2*pi*50*t + θph = [θ; θ - 120π/180; θ + 120π/180] + return +10 * cos.(θph) +end + +featurize_ddpg = function(state, env, name) + if name == "ElectricGrid_ddpg_1" + + #state = state[findall(x -> split(x, "_")[2] == "i" , env.agent_dict["ElectricGrid_ddpg_1"]["state_ids"])] + + norm_ref = env.nc.parameters["source"][1]["i_limit"] + state = vcat(state, reference(env.t)/norm_ref) + + # θ = 2*pi*50*env.t + + # state_to_control_1 = env.state[findfirst(x -> x == "source1_i_L1_a", env.state_ids)] + # state_to_control_2 = env.state[findfirst(x -> x == "source1_i_L1_b", env.state_ids)] + # state_to_control_3 = env.state[findfirst(x -> x == "source1_i_L1_c", env.state_ids)] + + # state_to_control = [state_to_control_1, state_to_control_2, state_to_control_3] + + # Il_dq0 = DQ0Transform(state_to_control, θ) + # state = vcat(state, Il_dq0) + end +end + +function reward_function(env, name = nothing) + if name == "classic" + return 0 + + else + state_to_control_1 = env.state[findfirst(x -> x == "source1_i_L1_a", env.state_ids)] + state_to_control_2 = env.state[findfirst(x -> x == "source1_i_L1_b", env.state_ids)] + state_to_control_3 = env.state[findfirst(x -> x == "source1_i_L1_c", env.state_ids)] + + state_to_control = [state_to_control_1, state_to_control_2, state_to_control_3] + + if any(abs.(state_to_control).>1) + return -1 + else + + refs = reference(env.t) + norm_ref = env.nc.parameters["source"][1]["i_limit"] + r = 1-1/3*(sum((abs.(refs/norm_ref - state_to_control)/2).^0.5)) + return r + end + end + +end + +env = ElectricGridEnv( + #CM = CM, + parameters = parameters, + t_end = 1, + reward_function = reward_function, + featurize = featurize_ddpg, + action_delay = 0, + verbosity = 0) + + +controllers = SetupAgents(env) + +learnhook = DataHook() + +function learn() + num_steps = 50_000 + + an_scheduler_loops = 20 + + while true + if length(controllers.hook.df[!,"reward"]) <= 1_500_000 + println("Steps so far: $(length(controllers.hook.df[!,"reward"]))") + Learn(controllers, env, steps = num_steps, hook = learnhook) + else + for j in 1:10 + an = 0.01 * exp10.(collect(LinRange(0.0, -10, an_scheduler_loops))) + for i in 1:an_scheduler_loops + controllers.agents["ElectricGrid_ddpg_1"]["policy"].policy.policy.act_noise = an[i] + println("Steps so far: $(length(controllers.hook.df[!,"reward"]))") + println("next action noise level: $(an[i])") + Learn(controllers, env, steps = num_steps, hook = learnhook) + end + end + end + end +end + + +learn() + +hook = DataHook(collect_state_ids = env.state_ids, + collect_action_ids = env.action_ids) + +Simulate(controllers, env, hook=hook) + + +RenderHookResults(hook = hook, + states_to_plot = env.state_ids, + actions_to_plot = env.action_ids, + plot_reward=true) From 77dcc4bcff7ad8c91e9bb11a04d7f79f46b7cd85 Mon Sep 17 00:00:00 2001 From: Jan Stenner Date: Tue, 27 Jun 2023 13:42:01 +0200 Subject: [PATCH 2/5] example fix --- examples/scripts/RL_Classical_Controllers_Merge_2.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/scripts/RL_Classical_Controllers_Merge_2.jl b/examples/scripts/RL_Classical_Controllers_Merge_2.jl index b8d8493a..7fd596c5 100644 --- a/examples/scripts/RL_Classical_Controllers_Merge_2.jl +++ b/examples/scripts/RL_Classical_Controllers_Merge_2.jl @@ -103,6 +103,7 @@ function learn() an_scheduler_loops = 20 + Learn(controllers, env, steps = num_steps, hook = learnhook) while true if length(controllers.hook.df[!,"reward"]) <= 1_500_000 println("Steps so far: $(length(controllers.hook.df[!,"reward"]))") From 7aa2a2177f1ecd5dac2dabed6a08fc5cc116f3f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98ystein=20S=C3=B8rensen?= Date: Sun, 2 Jul 2023 07:22:20 +0200 Subject: [PATCH 3/5] Add space between "[" and "to" (#107) --- docs/JOSS/paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/JOSS/paper.md b/docs/JOSS/paper.md index 20fe966e..58aa90d4 100644 --- a/docs/JOSS/paper.md +++ b/docs/JOSS/paper.md @@ -158,7 +158,7 @@ The package should be installed using the julia package manager. In a julia term add ElectricGrid ``` -Alternatively it can also be installed from the Github source code. To do that, clone the repository, start Julia, activate the project by pressing `]`to access Pkg mode and then `activate path/to/ElectricGrid` or `activate .` If you started Julia in your ElectricGrid directory and afterwards run `instantiate`. +Alternatively it can also be installed from the Github source code. To do that, clone the repository, start Julia, activate the project by pressing `]` to access Pkg mode and then `activate path/to/ElectricGrid` or `activate .` If you started Julia in your ElectricGrid directory and afterwards run `instantiate`. The source code, guide and examples are available on the GitHub repository (https://github.com/upb-lea/JuliaElectricGrid.jl). From cf9c71be72a2e95603c3fe0899ac0e1d2bb51ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98ystein=20S=C3=B8rensen?= Date: Sun, 2 Jul 2023 07:24:37 +0200 Subject: [PATCH 4/5] Suggested change to JOSS submission (#104) * Citation should be in-text, not parenthesized. * Capitalize Jupyter Notebook --- docs/JOSS/paper.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/JOSS/paper.md b/docs/JOSS/paper.md index 58aa90d4..c08e06ad 100644 --- a/docs/JOSS/paper.md +++ b/docs/JOSS/paper.md @@ -106,7 +106,7 @@ Already provided are classic controllers (i.e., industry standard contollers) li Many basic auxiliary functionalities for the essential operation of electric power grids are provided too such as coordinate transformations for basic controller classes, data logging, measurement of real and imaginary powers, and phase-locked loops for frequency and phase angle extraction. -The interface provided by [@Tian2020Reinforcement] is also available for training +The interface provided by @Tian2020Reinforcement is also available for training data-driven control approaches like RL. This enables users who want to integrate contemporary open-source Julia-based RL toolboxes such as ``ReinforcementLearning.jl`` [@Tian2020Reinforcement]. @@ -141,7 +141,7 @@ The ``ElectricGrid.jl`` toolbox provides the following key features: * Interesting use cases applying data-driven learning. # Examples -For illustration and interactive introduction, jupyter notebooks are available for each topic. +For illustration and interactive introduction, Jupyter Notebooks are available for each topic. These provide clear and easy-to-expand examples of: - [Utilising ElectricGrid.jl to build an energy grid](https://github.com/upb-lea/JuliaElectricGrid.jl/blob/main/examples/notebooks/Env_Create_DEMO.ipynb), - [Theoretical principles behind the calculations](https://github.com/upb-lea/JuliaElectricGrid.jl/blob/main/examples/notebooks/NodeConstructor_Theory_DEMO.ipynb), From 08bd9e55d64227b7c51dd4028394231b1def406c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98ystein=20S=C3=B8rensen?= Date: Sun, 2 Jul 2023 07:25:17 +0200 Subject: [PATCH 5/5] Update paper.bib (#108) Protect uppercase names in BibTeX source --- docs/JOSS/paper.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/JOSS/paper.bib b/docs/JOSS/paper.bib index fc435060..78e46d84 100644 --- a/docs/JOSS/paper.bib +++ b/docs/JOSS/paper.bib @@ -21,7 +21,7 @@ @inproceedings{Coffrin2018 @misc{Tian2020Reinforcement, author = {Jun Tian and other contributors}, - title = {ReinforcementLearning.jl: A Reinforcement Learning Package for the Julia Programming Language}, + title = {ReinforcementLearning.jl: A Reinforcement Learning Package for the {Julia} Programming Language}, year = 2020, url = {https://github.com/JuliaReinforcementLearning/ReinforcementLearning.jl} }