Merge branch 'refs/heads/dev' into df/#930-refactor-handleInfeed

# Conflicts: # CHANGELOG.md
ie3-institute · Sep 26, 2024 · 47e4052 · 47e4052
2 parents c1e37ee + a8be401
commit 47e4052
Show file tree

Hide file tree

Showing 23 changed files with 608 additions and 76 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,6 +33,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add gradle application plugin for command line execution with gradle run [#890](https://github.com/ie3-institute/simona/issues/890)
 - Additional tests to check flexibility options of thermal house and storage [#729](https://github.com/ie3-institute/simona/issues/729)
 - EmAgents should be able to handle initialization [#945](https://github.com/ie3-institute/simona/issues/945)
+- Added option to directly zip the output files [#793](https://github.com/ie3-institute/simona/issues/793)
+- Added weatherData HowTo for Copernicus ERA5 data [#967](https://github.com/ie3-institute/simona/issues/967)
 
 ### Changed
 - Adapted to changed data source in PSDM [#435](https://github.com/ie3-institute/simona/issues/435)
@@ -113,6 +115,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix determineState of ThermalHouse [#926](https://github.com/ie3-institute/simona/issues/926)
 - Fix activation of Hp when not under control of an EM [#922](https://github.com/ie3-institute/simona/issues/922)
 - Fix expected secondaryData in baseStateData [#955](https://github.com/ie3-institute/simona/issues/955)
+- Improve code quality in fixedloadmodelspec and other tests [#919](https://github.com/ie3-institute/simona/issues/919)
+- Fix power flow calculation with em agents [#962](https://github.com/ie3-institute/simona/issues/962)
+- Fix scheduling at Evcs with more than one Ev at a time without Em [#787](https://github.com/ie3-institute/simona/issues/787)
 - Refactoring of `ThermalGrid.handleInfeed` to fix thermal storage recharge correctly when empty [#930](https://github.com/ie3-institute/simona/issues/930)
 
 ## [3.0.0] - 2023-08-07

diff --git a/build.gradle b/build.gradle
@@ -149,7 +149,7 @@ dependencies {
   implementation 'javax.measure:unit-api:2.2'
   implementation 'tech.units:indriya:2.2' // quantities
   implementation "org.typelevel:squants_${scalaVersion}:1.8.3"
-  implementation 'org.apache.commons:commons-csv:1.11.0'
+  implementation 'org.apache.commons:commons-csv:1.12.0'
   implementation 'org.scalanlp:breeze_2.13:2.1.0' // scientific calculations (http://www.scalanlp.org/)
   implementation 'de.lmu.ifi.dbs.elki:elki:0.7.5' // Statistics (for random load model)
   implementation 'org.jgrapht:jgrapht-core:1.5.2'

diff --git a/docs/readthedocs/_static/bibliography/bibtexAll.bib b/docs/readthedocs/_static/bibliography/bibtexAll.bib
@@ -189,4 +189,10 @@ @Book{Kittl_2022
   address   = {Düren},
   year      = {2022},
   doi       = {10.17877/DE290R-22548}
+}
+
+@MISC{Radiation_ECMWF,
+  author = {Robin Hogan},
+  title = {Radiation Quantities in the ECMWF model and MARS},
+  howpublished = {\url{https://www.ecmwf.int/sites/default/files/elibrary/2015/18490-radiation-quantities-ecmwf-model-and-mars.pdf}}
 }
diff --git a/docs/readthedocs/config.md b/docs/readthedocs/config.md
@@ -94,9 +94,13 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 ```
 
+While using a csv sink, the raw data output files can be zipped directly when `zipFiles = true` is used.
+
+
 #### Output configuration of the grid
 
 The grid output configuration defines for which grid components simulation values are to be output.

diff --git a/docs/readthedocs/howto/weatherDataHowToCopernicusERA5.md b/docs/readthedocs/howto/weatherDataHowToCopernicusERA5.md
@@ -0,0 +1,34 @@
+(weatherDataHowToCopernicusERA5)=
+
+# How To use Copernicus ERA5 weather data in SIMONA
+
+To use weather data from the past within SIMONA we recommend to use the dataset [ERA5 hourly data on single levels from 1940 to present](https://cds-beta.climate.copernicus.eu/datasets/reanalysis-era5-single-levels?tab=download) of [Copernicus Climate Data Store](https://cds-beta.climate.copernicus.eu/).
+
+The following data parameter should be used
+
+- Wind
+    - 100m u-component of wind
+    - 100m v-component of wind
+- Radiation
+  - Total sky direct solar radiation at surface (FDIR)
+  - Surface solar radiation downwards (SSRD)
+- Temperature
+  - 2m temperature
+
+Since SIMONAs [PV Model](pv_model) requires direct and diffuse solar radiation, the diffuse solar radiation need to be determined from the ERA5 data.
+
+## Pre-Processing solar radiation weather data
+
+To obtain diffuse solar radiation data from ERA5 weather data, the necessary diffuse solar radiation (FDIFF) at surface can be calculated by
+
+$$
+ FDIFF = SSRD - FDIR
+$$
+
+*with*\
+**SSRD** = Surface solar radiation downwards\
+**FDIR** = Total sky direct solar radiation at surface
+
+
+**References:**
+* {cite:cts}`Radiation_ECMWF`
diff --git a/docs/readthedocs/usersguide.md b/docs/readthedocs/usersguide.md
@@ -119,6 +119,13 @@ Besides a configuration and the actual grid and grid participants, SIMONA also e
 There is an option to use sample weather data, but if you want sensible results, definitely consider supplying suitable data.
 Information on the expected data format and different supported sources are given in the input parameters section of the {doc}`config` file.
 
+The following How-To's are available:
+```{toctree}
+---
+maxdepth: 1
+---
+howto/weatherDataHowToCopernicusERA5
+```
 
 ## Simulation Outputs
 

diff --git a/input/samples/vn_simona/vn_simona.conf b/input/samples/vn_simona/vn_simona.conf
@@ -51,6 +51,7 @@ simona.output.sink.csv {
   fileFormat = ".csv"
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = false
 }
 
 simona.output.grid = {

diff --git a/src/main/resources/config/config-template.conf b/src/main/resources/config/config-template.conf
@@ -269,6 +269,7 @@ simona.output.sink.csv {
   isHierarchic = Boolean | false
   filePrefix = ""
   fileSuffix = ""
+  zipFiles = "Boolean" | false
 }
 #@optional
 simona.output.sink.influxDb1x {

diff --git a/src/main/scala/edu/ie3/simona/agent/participant/ParticipantAgentFundamentals.scala b/src/main/scala/edu/ie3/simona/agent/participant/ParticipantAgentFundamentals.scala
@@ -808,6 +808,7 @@ protected trait ParticipantAgentFundamentals[
       nextActivation,
     )
 
+    unstashAll()
     stay() using stateDataFinal
   }
 

diff --git a/src/main/scala/edu/ie3/simona/agent/participant/evcs/EvcsAgentFundamentals.scala b/src/main/scala/edu/ie3/simona/agent/participant/evcs/EvcsAgentFundamentals.scala
@@ -65,7 +65,7 @@ import squants.{Dimensionless, Each, Power}
 
 import java.time.ZonedDateTime
 import java.util.UUID
-import scala.collection.SortedSet
+import scala.collection.immutable.SortedSet
 import scala.reflect.{ClassTag, classTag}
 
 protected trait EvcsAgentFundamentals
@@ -494,9 +494,10 @@ protected trait EvcsAgentFundamentals
     val relevantData =
       createCalcRelevantData(modelBaseStateData, tick)
 
+    val lastState = getLastOrInitialStateData(modelBaseStateData, tick)
+
     val updatedBaseStateData = {
       if (relevantData.arrivals.nonEmpty) {
-        val lastState = getLastOrInitialStateData(modelBaseStateData, tick)
 
         val currentEvs = modelBaseStateData.model.determineCurrentEvs(
           relevantData,
@@ -528,6 +529,15 @@ protected trait EvcsAgentFundamentals
         modelBaseStateData
     }
 
+    // if the lastState's tick is the same as the actual tick the results have already been determined and announced when we handled the departedEvs
+    if (lastState.tick != tick) {
+      determineResultsAnnounceUpdateValueStore(
+        lastState,
+        currentTick,
+        modelBaseStateData,
+      )
+    }
+
     // We're only here if we're not flex-controlled, thus sending a Completion is always right
     goToIdleReplyCompletionAndScheduleTriggerForNextAction(
       updatedBaseStateData,

diff --git a/src/main/scala/edu/ie3/simona/config/SimonaConfig.scala b/src/main/scala/edu/ie3/simona/config/SimonaConfig.scala
@@ -2054,6 +2054,7 @@ object SimonaConfig {
             filePrefix: java.lang.String,
             fileSuffix: java.lang.String,
             isHierarchic: scala.Boolean,
+            zipFiles: scala.Boolean,
         )
         object Csv {
           def apply(
@@ -2073,6 +2074,7 @@ object SimonaConfig {
                 else "",
               isHierarchic =
                 c.hasPathOrNull("isHierarchic") && c.getBoolean("isHierarchic"),
+              zipFiles = c.hasPathOrNull("zipFiles") && c.getBoolean("zipFiles"),
             )
           }
         }

diff --git a/src/main/scala/edu/ie3/simona/io/result/ResultSinkType.scala b/src/main/scala/edu/ie3/simona/io/result/ResultSinkType.scala
@@ -21,6 +21,7 @@ object ResultSinkType {
       fileFormat: String = ".csv",
       filePrefix: String = "",
       fileSuffix: String = "",
+      zipFiles: Boolean = false,
   ) extends ResultSinkType
 
   final case class InfluxDb1x(url: String, database: String, scenario: String)
@@ -48,7 +49,12 @@ object ResultSinkType {
 
     sink.headOption match {
       case Some(params: SimonaConfig.Simona.Output.Sink.Csv) =>
-        Csv(params.fileFormat, params.filePrefix, params.fileSuffix)
+        Csv(
+          params.fileFormat,
+          params.filePrefix,
+          params.fileSuffix,
+          params.zipFiles,
+        )
       case Some(params: SimonaConfig.Simona.Output.Sink.InfluxDb1x) =>
         InfluxDb1x(buildInfluxDb1xUrl(params), params.database, runName)
       case Some(params: SimonaConfig.ResultKafkaParams) =>

diff --git a/src/main/scala/edu/ie3/simona/main/RunSimonaStandalone.scala b/src/main/scala/edu/ie3/simona/main/RunSimonaStandalone.scala
@@ -10,12 +10,17 @@ import edu.ie3.simona.config.{ArgsParser, ConfigFailFast, SimonaConfig}
 import edu.ie3.simona.main.RunSimona._
 import edu.ie3.simona.sim.SimonaSim
 import edu.ie3.simona.sim.setup.SimonaStandaloneSetup
+import edu.ie3.util.io.FileIOUtils
 import org.apache.pekko.actor.typed.scaladsl.AskPattern._
 import org.apache.pekko.actor.typed.{ActorSystem, Scheduler}
 import org.apache.pekko.util.Timeout
 
+import java.nio.file.Path
 import scala.concurrent.Await
-import scala.concurrent.duration.DurationInt
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration.{Duration, DurationInt}
+import scala.jdk.FutureConverters.CompletionStageOps
+import scala.util.{Failure, Success}
 
 /** Run a standalone simulation of simona
   *
@@ -24,6 +29,7 @@ import scala.concurrent.duration.DurationInt
 object RunSimonaStandalone extends RunSimona[SimonaStandaloneSetup] {
 
   override implicit val timeout: Timeout = Timeout(12.hours)
+  implicit val compressTimeoutDuration: Duration = 15.minutes
 
   override def setup(args: Array[String]): SimonaStandaloneSetup = {
     // get the config and prepare it with the provided args
@@ -56,9 +62,38 @@ object RunSimonaStandalone extends RunSimona[SimonaStandaloneSetup] {
       case SimonaEnded(successful) =>
         simonaSim.terminate()
 
+        val config = SimonaConfig(simonaSetup.typeSafeConfig).simona.output
+
+        config.sink.csv.map(_.zipFiles).foreach { zipFiles =>
+          if (zipFiles) {
+            val rawOutputPath =
+              Path.of(simonaSetup.resultFileHierarchy.rawOutputDataDir)
+
+            rawOutputPath.toFile.listFiles().foreach { file =>
+              val fileName = file.getName
+              val archiveName = fileName.replace(".csv", "")
+              val filePath = rawOutputPath.resolve(fileName)
+
+              val compressFuture =
+                FileIOUtils
+                  .compressFile(filePath, rawOutputPath.resolve(archiveName))
+                  .asScala
+              compressFuture.onComplete {
+                case Success(_) =>
+                  FileIOUtils.deleteRecursively(filePath)
+                case Failure(exception) =>
+                  logger.error(
+                    s"Compression of output file to '$archiveName' has failed. Keep raw data.",
+                    exception,
+                  )
+              }
+              Await.ready(compressFuture, compressTimeoutDuration)
+            }
+          }
+        }
+
         successful
     }
-
   }
 
 }
diff --git a/src/main/scala/edu/ie3/simona/model/participant/evcs/EvcsModel.scala b/src/main/scala/edu/ie3/simona/model/participant/evcs/EvcsModel.scala
@@ -19,7 +19,6 @@ import edu.ie3.simona.model.participant.evcs.uncontrolled.{
   ConstantPowerCharging,
   MaximumPowerCharging,
 }
-import edu.ie3.util.scala.quantities.DefaultQuantities._
 import edu.ie3.simona.model.participant.{
   CalcRelevantData,
   FlexChangeIndicator,
@@ -32,7 +31,8 @@ import edu.ie3.simona.util.TickUtil.TickLong
 import edu.ie3.util.quantities.PowerSystemUnits._
 import edu.ie3.util.quantities.QuantityUtils.RichQuantityDouble
 import edu.ie3.util.scala.OperationInterval
-import squants.energy.{KilowattHours, Kilowatts}
+import edu.ie3.util.scala.quantities.DefaultQuantities._
+import squants.energy.Kilowatts
 import squants.time.Seconds
 import squants.{Dimensionless, Energy, Power}
 import tech.units.indriya.unit.Units.PERCENT
@@ -514,13 +514,17 @@ final case class EvcsModel(
       modelState: EvcsState,
       data: EvcsRelevantData,
   ): ApparentPower =
-    throw new NotImplementedError("Use calculatePowerAndEvSoc() instead.")
+    throw new NotImplementedError(
+      "Use calculateNewScheduling() or chargeEv() instead."
+    )
 
   override protected def calculateActivePower(
       modelState: EvcsState,
       data: EvcsRelevantData,
   ): Power =
-    throw new NotImplementedError("Use calculatePowerAndEvSoc() instead.")
+    throw new NotImplementedError(
+      "Use calculateNewScheduling() or chargeEv() instead."
+    )
 
   override def determineFlexOptions(
       data: EvcsRelevantData,

diff --git a/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala b/src/main/scala/edu/ie3/simona/sim/setup/SimonaStandaloneSetup.scala
@@ -63,7 +63,7 @@ import scala.jdk.CollectionConverters._
 class SimonaStandaloneSetup(
     val typeSafeConfig: Config,
     simonaConfig: SimonaConfig,
-    resultFileHierarchy: ResultFileHierarchy,
+    val resultFileHierarchy: ResultFileHierarchy,
     runtimeEventQueue: Option[LinkedBlockingQueue[RuntimeEvent]] = None,
     override val args: Array[String],
 ) extends SimonaSetup {