Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Jun 17, 2024
1 parent 306e0f0 commit e085b31
Show file tree
Hide file tree
Showing 8 changed files with 57 additions and 44 deletions.
6 changes: 3 additions & 3 deletions pipelines/rj_cor/meteorologia/precipitacao_inea/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,9 @@ def treat_data(
].copy()

# Replace all values bigger than 10000 on "altura_agua" to nan
dfr_fluviometric.loc[
dfr_fluviometric["altura_agua"] > 10000, "altura_agua"
] = np.nan
dfr_fluviometric.loc[dfr_fluviometric["altura_agua"] > 10000, "altura_agua"] = (
np.nan
)

fluviometric_cols_order = [
"id_estacao",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -316,18 +316,18 @@ def process_ppi(
: hdf[primary_key]["where"].attrs["nbins"],
el_index,
] = rhohv_data_array
full_startazA_matrix[
indices_dict[primary_key]["indices"], :, el_index
] = startazA.reshape(startazA.shape[0], 1)
full_stopazA_matrix[
indices_dict[primary_key]["indices"], :, el_index
] = stopazA.reshape(stopazA.shape[0], 1)
full_startazT_matrix[
indices_dict[primary_key]["indices"], :, el_index
] = startazT.reshape(startazT.shape[0], 1)
full_stopazT_matrix[
indices_dict[primary_key]["indices"], :, el_index
] = stopazT.reshape(stopazT.shape[0], 1)
full_startazA_matrix[indices_dict[primary_key]["indices"], :, el_index] = (
startazA.reshape(startazA.shape[0], 1)
)
full_stopazA_matrix[indices_dict[primary_key]["indices"], :, el_index] = (
stopazA.reshape(stopazA.shape[0], 1)
)
full_startazT_matrix[indices_dict[primary_key]["indices"], :, el_index] = (
startazT.reshape(startazT.shape[0], 1)
)
full_stopazT_matrix[indices_dict[primary_key]["indices"], :, el_index] = (
stopazT.reshape(stopazT.shape[0], 1)
)

full_elevation_matrix[:, :, el_index] = hdf[primary_key]["where"].attrs[
"elangle"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,9 @@ def run_model_prediction(
)

assert new_radar_data.nrays == NRAYS, f"nrays should be {NRAYS}."
radar_data_dict[
f"{specs_dict['process_type']}-{specs_dict['feature']}"
] = new_radar_data
radar_data_dict[f"{specs_dict['process_type']}-{specs_dict['feature']}"] = (
new_radar_data
)

radar_data_hdfs.append(radar_data_dict)

Expand Down
8 changes: 5 additions & 3 deletions pipelines/rj_smtr/br_rj_riodejaneiro_diretorios/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,11 @@
LABELS = get_current_flow_labels()

table_params = task(
lambda tables, exclude: [t for t in tables if t["table_id"] not in exclude]
if exclude is not None
else tables,
lambda tables, exclude: (
[t for t in tables if t["table_id"] not in exclude]
if exclude is not None
else tables
),
checkpoint=False,
name="get_tables_to_run",
)(tables=constants.DIRETORIO_MATERIALIZACAO_TABLE_PARAMS.value, exclude=exclude)
Expand Down
6 changes: 3 additions & 3 deletions pipelines/rj_smtr/br_rj_riodejaneiro_rdo/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ def download_and_save_local_from_ftp(file_info: dict):
)

# Set general local path to save file (bucket_modes: raw or staging)
file_info[
"local_path"
] = f"""{base_path}/{table_id}/{file_info["partitions"]}/{file_info['filename']}.{{file_ext}}"""
file_info["local_path"] = (
f"""{base_path}/{table_id}/{file_info["partitions"]}/{file_info['filename']}.{{file_ext}}"""
)
# Get raw data
file_info["raw_path"] = file_info["local_path"].format(
bucket_mode="raw", file_ext="txt"
Expand Down
8 changes: 5 additions & 3 deletions pipelines/rj_smtr/br_rj_riodejaneiro_recursos/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,11 @@
]

table_params = task(
lambda tables, exclude: [t for t in tables if t["table_id"] not in exclude]
if exclude is not None
else tables,
lambda tables, exclude: (
[t for t in tables if t["table_id"] not in exclude]
if exclude is not None
else tables
),
checkpoint=False,
name="get_tables_to_run",
)(tables=constants.SUBSIDIO_SPPO_RECURSOS_TABLE_IDS.value, exclude=exclude)
Expand Down
29 changes: 18 additions & 11 deletions pipelines/rj_smtr/projeto_subsidio_sppo/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ def subsidio_data_quality_check(

if mode == "pos":
request_params["end_timestamp"] = f"""{params["end_date"]} 00:00:00"""
request_params[
"dataset_id"
] = smtr_constants.SUBSIDIO_SPPO_DASHBOARD_DATASET_ID.value
request_params["dataset_id"] = (
smtr_constants.SUBSIDIO_SPPO_DASHBOARD_DATASET_ID.value
)

checks_list = (
smtr_constants.SUBSIDIO_SPPO_DATA_CHECKS_PRE_LIST.value
Expand Down Expand Up @@ -139,14 +139,21 @@ def subsidio_data_quality_check(

if not test_check:
at_code_owners = [
f' - <@{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"] == "user"
else f' - <@!{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"]
== "user_nickname"
else f' - <#{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"] == "channel"
else f' - <@&{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
(
f' - <@{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"] == "user"
else (
f' - <@!{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"]
== "user_nickname"
else (
f' - <#{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
if constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["type"]
== "channel"
else f' - <@&{constants.OWNERS_DISCORD_MENTIONS.value[code_owner]["user_id"]}>\n'
)
)
)
for code_owner in code_owners
]

Expand Down
14 changes: 8 additions & 6 deletions pipelines/utils/georeference/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,14 @@ def georeference_dataframe(
locations.append(location)

geolocated_addresses = [
{
"latitude": location.latitude,
"longitude": location.longitude,
}
if location is not None
else {"latitude": None, "longitude": None}
(
{
"latitude": location.latitude,
"longitude": location.longitude,
}
if location is not None
else {"latitude": None, "longitude": None}
)
for location in locations
]

Expand Down

0 comments on commit e085b31

Please sign in to comment.