Skip to content

Commit

Permalink
add missing variables and models
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-zippenfenig
patrick-zippenfenig committed Oct 23, 2023
1 parent 8c6741c commit 8fde2d9
Show file tree
Hide file tree
Showing 7 changed files with 301 additions and 291 deletions.
7 changes: 6 additions & 1 deletion flatbuffers/openmeteo_sdk.fbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ enum Model: byte {
MPI_ESM1_2_XR,
NICAM16_8S,
cams_europe,
cams_global
cams_global,
cfsv2
}

enum Variable: byte {
Expand All @@ -114,6 +115,7 @@ enum Variable: byte {
cloudcover_high,
cloudcover_low,
cloudcover_mid,
daylight_duration,
dewpoint,
diffuse_radiation,
diffuse_radiation_instant,
Expand All @@ -131,6 +133,7 @@ enum Variable: byte {
lifted_index,
lightning_potential,
precipitation,
precipitation_hours,
precipitation_probability,
pressure_msl,
rain,
Expand All @@ -145,6 +148,8 @@ enum Variable: byte {
snowfall,
snowfall_height,
snowfall_water_equivalent,
sunrise,
sunset,
soil_moisture,
soil_moisture_index,
soil_temperature,
Expand Down
14 changes: 2 additions & 12 deletions python/openmeteo_sdk/ApiResponse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,19 +120,9 @@ def Hourly(self) -> Optional[SeriesAndTime]:
return obj
return None

# ApiResponse
def ThreeHourly(self) -> Optional[SeriesAndTime]:
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(28))
if o != 0:
x = self._tab.Indirect(o + self._tab.Pos)
obj = SeriesAndTime()
obj.Init(self._tab.Bytes, x)
return obj
return None

# ApiResponse
def SixHourly(self) -> Optional[SeriesAndTime]:
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30))
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(28))
if o != 0:
x = self._tab.Indirect(o + self._tab.Pos)
obj = SeriesAndTime()
Expand All @@ -142,7 +132,7 @@ def SixHourly(self) -> Optional[SeriesAndTime]:

# ApiResponse
def Minutely15(self) -> Optional[SeriesAndTime]:
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(32))
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(30))
if o != 0:
x = self._tab.Indirect(o + self._tab.Pos)
obj = SeriesAndTime()
Expand Down
1 change: 1 addition & 0 deletions python/openmeteo_sdk/Model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,3 +56,4 @@ class Model(Enum):
NICAM16_8S = 48
cams_europe = 49
cams_global = 50
cfsv2 = 51
188 changes: 96 additions & 92 deletions python/openmeteo_sdk/Variable.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,95 +12,99 @@ class Variable(Enum):
cloudcover_high = 4
cloudcover_low = 5
cloudcover_mid = 6
dewpoint = 7
diffuse_radiation = 8
diffuse_radiation_instant = 9
direct_normal_irradiance = 10
direct_normal_irradiance_instant = 11
direct_radiation = 12
direct_radiation_instant = 13
et0_fao_evapotranspiration = 14
evapotranspiration = 15
freezinglevel_height = 16
growing_degree_days = 17
is_day = 18
latent_heatflux = 19
leaf_wetness_probability = 20
lifted_index = 21
lightning_potential = 22
precipitation = 23
precipitation_probability = 24
pressure_msl = 25
rain = 26
relativehumidity = 27
runoff = 28
sensible_heatflux = 29
shortwave_radiation = 30
shortwave_radiation_instant = 31
showers = 32
snow_depth = 33
snow_height = 34
snowfall = 35
snowfall_height = 36
snowfall_water_equivalent = 37
soil_moisture = 38
soil_moisture_index = 39
soil_temperature = 40
surface_pressure = 41
surface_temperature = 42
temperature = 43
terrestrial_radiation = 44
terrestrial_radiation_instant = 45
total_column_integrated_water_vapour = 46
updraft = 47
uv_index = 48
uv_index_clear_sky = 49
vapor_pressure_deficit = 50
visibility = 51
weathercode = 52
winddirection = 53
windgusts = 54
windspeed = 55
vertical_velocity = 56
geopotential_height = 57
wet_bulb_temperature = 58
river_discharge = 59
wave_height = 60
wave_period = 61
wave_direction = 62
wind_wave_height = 63
wind_wave_period = 64
wind_wave_peak_period = 65
wind_wave_direction = 66
swell_wave_height = 67
swell_wave_period = 68
swell_wave_peak_period = 69
swell_wave_direction = 70
pm10 = 71
pm2p5 = 72
dust = 73
aerosol_optical_depth = 74
carbon_monoxide = 75
nitrogen_dioxide = 76
ammonia = 77
ozone = 78
sulphur_dioxide = 79
alder_pollen = 80
birch_pollen = 81
grass_pollen = 82
mugwort_pollen = 83
olive_pollen = 84
ragweed_pollen = 85
european_aqi = 86
european_aqi_pm2p5 = 87
european_aqi_pm10 = 88
european_aqi_no2 = 89
european_aqi_o3 = 90
european_aqi_so2 = 91
us_aqi = 92
us_aqi_pm2p5 = 93
us_aqi_pm10 = 94
us_aqi_no2 = 95
us_aqi_o3 = 96
us_aqi_so2 = 97
us_aqi_co = 98
daylight_duration = 7
dewpoint = 8
diffuse_radiation = 9
diffuse_radiation_instant = 10
direct_normal_irradiance = 11
direct_normal_irradiance_instant = 12
direct_radiation = 13
direct_radiation_instant = 14
et0_fao_evapotranspiration = 15
evapotranspiration = 16
freezinglevel_height = 17
growing_degree_days = 18
is_day = 19
latent_heatflux = 20
leaf_wetness_probability = 21
lifted_index = 22
lightning_potential = 23
precipitation = 24
precipitation_hours = 25
precipitation_probability = 26
pressure_msl = 27
rain = 28
relativehumidity = 29
runoff = 30
sensible_heatflux = 31
shortwave_radiation = 32
shortwave_radiation_instant = 33
showers = 34
snow_depth = 35
snow_height = 36
snowfall = 37
snowfall_height = 38
snowfall_water_equivalent = 39
sunrise = 40
sunset = 41
soil_moisture = 42
soil_moisture_index = 43
soil_temperature = 44
surface_pressure = 45
surface_temperature = 46
temperature = 47
terrestrial_radiation = 48
terrestrial_radiation_instant = 49
total_column_integrated_water_vapour = 50
updraft = 51
uv_index = 52
uv_index_clear_sky = 53
vapor_pressure_deficit = 54
visibility = 55
weathercode = 56
winddirection = 57
windgusts = 58
windspeed = 59
vertical_velocity = 60
geopotential_height = 61
wet_bulb_temperature = 62
river_discharge = 63
wave_height = 64
wave_period = 65
wave_direction = 66
wind_wave_height = 67
wind_wave_period = 68
wind_wave_peak_period = 69
wind_wave_direction = 70
swell_wave_height = 71
swell_wave_period = 72
swell_wave_peak_period = 73
swell_wave_direction = 74
pm10 = 75
pm2p5 = 76
dust = 77
aerosol_optical_depth = 78
carbon_monoxide = 79
nitrogen_dioxide = 80
ammonia = 81
ozone = 82
sulphur_dioxide = 83
alder_pollen = 84
birch_pollen = 85
grass_pollen = 86
mugwort_pollen = 87
olive_pollen = 88
ragweed_pollen = 89
european_aqi = 90
european_aqi_pm2p5 = 91
european_aqi_pm10 = 92
european_aqi_no2 = 93
european_aqi_o3 = 94
european_aqi_so2 = 95
us_aqi = 96
us_aqi_pm2p5 = 97
us_aqi_pm10 = 98
us_aqi_no2 = 99
us_aqi_o3 = 100
us_aqi_so2 = 101
us_aqi_co = 102
Loading

3 comments on commit 8fde2d9

@DadiBit
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keep in mind that for backward compatibility, you should always add new variables at the end, otherwise using an old client will decode the enum byte as a different variable!

@DadiBit
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 great data structure, I love it. Do you think we could suppress the value: float field in favor of the values: [float]?
Is there any scenario where we would find both value and values set @patrick-zippenfenig ? If we remove it we can use the same amount of bytes and reduce a little bit the generated code 😄
Here are some tests:

Both value and values in schema

test.fbs schema

table Test {
  value: float;
  values: [float];
}

root_type Test;

Python script

import flatbuffers
import Test

builder = flatbuffers.Builder(1024)

# comment accordingly 
Test.StartValuesVector(builder, 1)
builder.PrependFloat32(3.14)
values = builder.EndVector()

Test.Start(builder)
Test.AddValues(builder, values)
# Test.AddValue(builder, 3.14)
test = Test.End(builder)

builder.Finish(test)
buf = builder.Output()

print(len(buf))

Value set to 3.14, values empty: 28 bytes
Value unset, values set to [3.14]: 28 bytes
Value set to 3.14, values set to [3.14]: 32 bytes

Just values

test.fbs schema

table Test {
  // value: float;
  values: [float];
}

root_type Test;

Same python script, just remove the value stuff.
values set to [3.14]: 28 bytes

@patrick-zippenfenig
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keep in mind that for backward compatibility, you should always add new variables at the end

Sure. I will also add unit tests that confirm binary compatibility and will test for the last case in each enumeration.

Do you think we could suppress the value: float field in favor of the values: [float]?

Technically, it should reduce the indirection level. The value should be inlined into flatbuffers vtable while an array cannot. Your test was a bit surprising to me. Adding more attributes to the beginning of a table, but not using them, does increase the message size. The Flatbuffer docs, are a bit misleading.

table Test {
    a: int16;
    b: int32;
    c: int16;
    d: int32;
    e: int16;
    f: int32;
    value: float;
    values: [float];
}

Only setting value => 32 bytes. I already suspected this behaviour, because the old format with attributes for each variable, had a lot of zeros inside.

table Test {
    value: float;
}

=> 20 bytes

table Test {
    values: [float];
}

Setting values to [3.14] => 28 bytes

This shows that 4 bytes are used for the array vtable offset and 4 bytes for the array size. Hence, value: float is (probably) inlined in the vtable while arrays use indirection. I am not sure yet what is the most efficient implementation. I will check first, how I can optimise the order of attributes to reduce the message size.

Please sign in to comment.