Skip to content

Commit

Permalink
Merge pull request msmasnadi#27 from msmasnadi/stream-validation
Browse files Browse the repository at this point in the history 
Stream validation
  • Loading branch information
@mbarlow12
mbarlow12 authored Nov 1, 2024
2 parents 78e31eb + fe6aeec commit 57155e3
Show file tree
Hide file tree
Showing 66 changed files with 733 additions and 131 deletions.
1 change: 1 addition & 0 deletions opgee/analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ def __init__(self, name, parent=None, attr_dict=None, field_names=None, groups=N

fields = [model.get_field(name) for name in self._field_names]

# Add to 'fields' any Field that matches Groups declared by the Analysis
for group in self.groups:
text = group.text
if group.is_regex:
Expand Down
3 changes: 2 additions & 1 deletion opgee/built_ins/csv2xml_plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ def import_fields(csv_path, xml_path, analysis_name, count=0, skip_fields=None,
from ..xml_utils import attr_to_xml

fields, dtypes = read_fields(csv_path,
from_package=from_package, skip_fields=skip_fields)
from_package=from_package,
skip_fields=skip_fields)

if count:
fields = fields[fields.columns[:count]]
Expand Down
5 changes: 5 additions & 0 deletions opgee/container.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ def __init__(self, name, attr_dict=None, parent=None):
self.energy = Energy()
self.import_export = ImportExport()

# N.B. These are the Process and Container instances directly inside
# the present Container, not including those held by sub-Containers.
self.procs = None
self.aggs = None

def add_children(self, aggs=None, procs=None, **kwargs):
self.aggs = self.adopt(aggs)
self.procs = self.adopt(procs)
Expand Down
28 changes: 25 additions & 3 deletions opgee/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import datetime

from . import ureg
from .error import OpgeeException, AbstractMethodError
from .error import OpgeeException, AbstractMethodError, ModelValidationError
from .log import getLogger
from .utils import coercible, getBooleanXML

Expand Down Expand Up @@ -157,6 +157,22 @@ def __str__(self):
name_str = f' name="{self.name}"' if self.name else ''
return f'<{type_str}{name_str} enabled={self.enabled}>'

def print_in_context(self):
"""
Print the object along with its parents, to provide context.
:return: none
"""
obj = self
seq = [obj]
while ((obj := obj.parent) is not None):
seq.insert(0, obj)

indent = 0
for obj in seq:
print(" " * indent, obj)
indent += 1

def is_enabled(self):
return self.enabled

Expand All @@ -166,7 +182,7 @@ def set_enabled(self, value):
def adopt(self, objs, asDict=False):
"""
Set the `parent` of each object to self. This is used to create back pointers
up the hieararchy so Processes and Streams can find their Field and Analysis
up the hierarchy so Processes and Streams can find their Field and Analysis
containers. Return the objects either as a list or dict.
:param objs: (None or list of XmlInstantiable)
Expand All @@ -176,11 +192,17 @@ def adopt(self, objs, asDict=False):
otherwise return the objs either in a list or dict.
"""
objs = [] if objs is None else objs
dct = {}

for obj in objs:
if (existing := dct.get(obj.name)):
obj.print_in_context()
existing.print_in_context()
raise ModelValidationError(f"Tried to adopt {obj} which is a duplicate of {existing}.")
dct[obj.name] = obj
obj.set_parent(self)

return {obj.name: obj for obj in objs} if asDict else objs
return dct if asDict else objs

def find_container(self, cls):
"""
Expand Down
5 changes: 4 additions & 1 deletion opgee/etc/attributes.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
<Model schema_version="4.0.a">
<AttrDefs xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="opgee.xsd">
<ClassAttrs name="Model">
<!-- Allow test XML to skip Process and Stream validation -->
<AttrDef name="skip_validation" type="binary" desc="Skip validation step">0</AttrDef>

<!-- Maximum number of iterations for process loops -->
<AttrDef name="maximum_iterations" type="int">100</AttrDef>
<AttrDef name="iteration_tolerance" type="float">0.000001</AttrDef>
Expand Down Expand Up @@ -346,8 +349,8 @@
<AttrDef name="is_REC" type="str" desc="Fracturing flowback captured with reduced emissons completions?">Yes</AttrDef>
<AttrDef name="frac_well_fractured" type="float" unit="frac" desc="Fraction of wells fractured">0.0</AttrDef>


</ClassAttrs>

<ClassAttrs name="SurveyTruck">
<AttrDef name="distance" type="float" desc="Distance of travel for survey" unit="mi">10000</AttrDef>
<AttrDef name="weight" type="float" desc="Weight of land survey vehicle" unit="tons">25</AttrDef>
Expand Down
134 changes: 72 additions & 62 deletions opgee/etc/opgee.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,71 +20,81 @@

<Field name="gas_lifting_field" modifies="template">
<A name="downhole_pump">1</A>
<A name="water_reinjection">0</A>
<A name="natural_gas_reinjection">0</A>
<A name="water_flooding">0</A>
<A name="gas_lifting">0</A>
<A name="gas_flooding">0</A>
<A name="steam_flooding">0</A>
<A name="country">Venezuela</A>
<A name="name">Hamaca</A>
<A name="age">13.0</A>
<A name="depth">2200.0</A>
<A name="oil_prod">158441.0</A>
<A name="num_prod_wells">190</A>
<A name="num_water_inj_wells">0</A>
<A name="well_diam">5.5</A>
<A name="prod_index">10.0</A>
<A name="res_press">950.0</A>
<A name="res_temp">109.6</A>
<A name="offshore">0.0</A>
<A name="API">8.6</A>
<A name="gas_comp_N2">0.1</A>
<A name="gas_comp_CO2">5.9</A>
<A name="gas_comp_C1">91.9</A>
<A name="gas_comp_C2">0.86</A>
<A name="gas_comp_C3">0.39</A>
<A name="gas_comp_C4">0.85</A>
<A name="gas_comp_H2S">0.0</A>
<A name="GOR">111.0</A>
<A name="WOR">0.1</A>
<A name="WIR">1.1</A>
<A name="GLIR">363.8</A>
<A name="GFIR">166.5</A>
<A name="flood_gas_type">NG</A>
<A name="SOR">0.0</A>
<A name="fraction_elec_onsite">1.0</A>
<A name="fraction_remaining_gas_inj">0.0</A>
<A name="fraction_water_reinjected">0.0</A>
<A name="fraction_steam_cogen">0.0</A>
<A name="fraction_steam_solar">0.0</A>
<Process class="CrudeOilDewatering">
<A name="heater_treater">1</A>
</Process>
<A name="stabilizer_column">0</A>
<A name="upgrader_type">Delayed coking</A>
<A name="gas_processing_path">Acid Wet Gas</A>
<A name="FOR">99.9</A>
<A name="frac_venting">0.002</A>
<A name="water_reinjection">0</A>
<A name="natural_gas_reinjection">0</A>
<A name="water_flooding">0</A>
<A name="gas_lifting">0</A>
<A name="gas_flooding">0</A>
<A name="steam_flooding">0</A>
<A name="country">Venezuela</A>
<A name="name">Hamaca</A>
<A name="age">13.0</A>
<A name="depth">2200.0</A>
<A name="oil_prod">158441.0</A>
<A name="num_prod_wells">190</A>
<A name="num_water_inj_wells">0</A>
<A name="well_diam">5.5</A>
<A name="prod_index">10.0</A>
<A name="res_press">950.0</A>
<A name="res_temp">109.6</A>
<A name="offshore">0.0</A>
<A name="API">8.6</A>
<A name="gas_comp_N2">0.1</A>
<A name="gas_comp_CO2">5.9</A>
<A name="gas_comp_C1">91.9</A>
<A name="gas_comp_C2">0.86</A>
<A name="gas_comp_C3">0.39</A>
<A name="gas_comp_C4">0.85</A>
<A name="gas_comp_H2S">0.0</A>
<A name="GOR">111.0</A>
<A name="WOR">0.1</A>
<A name="WIR">1.1</A>
<A name="GLIR">363.8</A>
<A name="GFIR">166.5</A>
<A name="flood_gas_type">NG</A>
<A name="SOR">0.0</A>
<A name="fraction_elec_onsite">1.0</A>
<A name="fraction_remaining_gas_inj">0.0</A>
<A name="fraction_water_reinjected">0.0</A>
<A name="fraction_steam_cogen">0.0</A>
<A name="fraction_steam_solar">0.0</A>
<A name="stabilizer_column">0</A>
<A name="upgrader_type">Delayed coking</A>
<A name="gas_processing_path">Acid Wet Gas</A>
<A name="FOR">99.9</A>
<A name="frac_venting">0.002</A>
<A name="frac_transport_tanker">1.0</A>
<A name="frac_transport_barge">0.0</A>
<A name="frac_transport_pipeline">1.0</A>
<A name="frac_transport_rail">0.0</A>
<A name="frac_transport_truck">0.0</A>
<A name="transport_dist_tanker">2443.0</A>
<A name="transport_dist_barge">500.0</A>
<A name="transport_dist_pipeline">140.0</A>
<A name="transport_dist_rail">800.0</A>
<A name="transport_dist_truck">100.0</A>
<A name="oil_sands_mine">None</A>
<A name="oil_processing_path">Upgrading</A>
<A name="frac_CO2_breakthrough">59</A>
<A name="ecosystem_richness">High carbon</A>
<A name="field_development_intensity">High</A>
<A name="prod_water_inlet_temp">140.0</A>

<!--
Set attributes with correct nesting to Process declarations,
so model / template merger works properly.
-->
<Aggregator name="ProductionExtraction">
<Process class="CrudeOilDewatering">
<A name="heater_treater">1</A>
</Process>
</Aggregator>

<Aggregator name="SurfaceProcessing">
<Process class="HeavyOilDilution">
<A name="fraction_diluent">0.0</A>
</Process>
<A name="frac_transport_tanker">1.0</A>
<A name="frac_transport_barge">0.0</A>
<A name="frac_transport_pipeline">1.0</A>
<A name="frac_transport_rail">0.0</A>
<A name="frac_transport_truck">0.0</A>
<A name="transport_dist_tanker">2443.0</A>
<A name="transport_dist_barge">500.0</A>
<A name="transport_dist_pipeline">140.0</A>
<A name="transport_dist_rail">800.0</A>
<A name="transport_dist_truck">100.0</A>
<A name="oil_sands_mine">None</A>
<A name="oil_processing_path">Upgrading</A>
<A name="frac_CO2_breakthrough">59</A>
<A name="ecosystem_richness">High carbon</A>
<A name="field_development_intensity">High</A>
<A name="prod_water_inlet_temp">140.0</A>
</Aggregator>
</Field>

<Field name="template">
Expand Down
61 changes: 37 additions & 24 deletions opgee/field.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,9 @@

from . import ureg
from .config import getParamAsList
from .constants import (
SIMPLE_RESULT,
DETAILED_RESULT,
ERROR_RESULT,
DEFAULT_RESULT_TYPE,
USER_RESULT_TYPES,
ALL_RESULT_TYPES,
)
from .constants import DETAILED_RESULT
from .container import Container
from .core import elt_name, instantiate_subelts, dict_from_list, STP, magnitude
from .core import elt_name, instantiate_subelts, dict_from_list, STP
from .energy import Energy
from .error import (
OpgeeException,
Expand Down Expand Up @@ -324,9 +317,7 @@ def _children(self):
super()._children()
) # + self.streams() # Adding this caused several errors...

def add_children(
self, aggs=None, procs=None, streams=None, process_choice_dict=None
):
def add_children(self, aggs=None, procs=None, streams=None, process_choice_dict=None):
# Note that `procs` include only Processes defined at the top-level of the field.
# Other Processes maybe defined within the Aggregators in `aggs`.
super().add_children(aggs=aggs, procs=procs)
Expand All @@ -342,7 +333,7 @@ def add_children(
known_boundaries = self.known_boundaries

# Remember streams that declare themselves as system boundaries. Keys must be one of the
# values in the tuples in the _known_boundaries dictionary above. s
# values in the tuples in the _known_boundaries dictionary above.
boundary_dict = self.boundary_dict

# Save references to boundary processes by name; fail if duplicate definitions are found.
Expand Down Expand Up @@ -372,10 +363,8 @@ def add_children(

self.check_attr_constraints(self.attr_dict)

(
self.component_fugitive_table,
self.loss_mat_gas_ave_df,
) = self.get_component_fugitive()
self.component_fugitive_table, self.loss_mat_gas_ave_df = \
self.get_component_fugitive()

self.finalize_process_graph()

Expand Down Expand Up @@ -1050,7 +1039,13 @@ def validate(self):
:return: none
:raises ModelValidationError: raised if any validation condition is violated.
"""
super().validate()

# Allow test models to skip validation to avoid overly complicating all tests
if self.model.attr("skip_validation"):
_logger.warning(f"{self} skipping Process and Stream validation")
else:
for child in self.children():
child.validate()

# Accumulate error msgs so user can correct them all at once.
msgs = []
Expand Down Expand Up @@ -1465,10 +1460,10 @@ def _collect(process_list, obj):
else:
_collect(process_list, child)

processes = (
self.builtin_procs.copy()
) # copy since we're appending to this list recursively
# use a copy since we append to this list recursively
processes = self.builtin_procs.copy()
_collect(processes, self)

return processes

def save_process_data(self, **kwargs):
Expand Down Expand Up @@ -1530,9 +1525,8 @@ def resolve_process_choices(self, process_choice_dict=None):
for group_name, group in choice.groups_dict.items():
procs, streams = group.processes_and_streams(self)

if (
group_name == selected_group_name
): # remember the ones to turn back on
# remember the ones to enable
if (group_name == selected_group_name):
to_enable.extend(procs)
to_enable.extend(streams)

Expand Down Expand Up @@ -1606,6 +1600,24 @@ def instances_by_class(self, cls):

return None

def print_process_list(self):
"""
Debugging tool
"""
p_dict = self.process_dict

for name in sorted(p_dict.keys()):
proc = p_dict[name]
print(f"\n{proc}")

print(" Inputs:")
for s in proc.inputs:
print(f" {s} contains '{s.contents}'")

print(" Outputs:")
for s in proc.outputs:
print(f" {s} contains '{s.contents}'")

#
# Smart Defaults and Distributions
#
Expand Down Expand Up @@ -1806,3 +1818,4 @@ def num_gas_inj_wells_default(self, num_prod_wells):
return num_prod_wells * 0.25

# TODO: decide how to handle "associated gas defaults", which is just global vs CA-LCFS values currently

4 changes: 4 additions & 0 deletions opgee/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,10 @@ def const(self, name):
except KeyError:
raise OpgeeException(f"No known constant with name '{name}'")

def validate(self):
for child in self.children():
child.validate()

def _children(self, include_disabled=False):
"""
Return a list of all children objects. External callers should use children()
Expand Down
Loading

0 comments on commit 57155e3

Please sign in to comment.