From 5676231b1c59ce185100d5f93c78d6c4e2884ed0 Mon Sep 17 00:00:00 2001 From: Michele Simionato Date: Thu, 14 Sep 2023 09:37:48 +0200 Subject: [PATCH 1/2] Directly removing hdf5 files with oq engine --dc --- openquake/commands/engine.py | 3 ++- openquake/commands/purge.py | 2 +- openquake/server/db/actions.py | 12 +++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/openquake/commands/engine.py b/openquake/commands/engine.py index dfea049c5671..892d56933806 100644 --- a/openquake/commands/engine.py +++ b/openquake/commands/engine.py @@ -59,11 +59,12 @@ def del_calculation(job_id, confirmed=False): 'all associated outputs?\nThis action cannot be undone. (y/n): '): try: abort([job.id]) - resp = logs.dbcmd('del_calc', job.id, getpass.getuser()) + resp = logs.dbcmd('del_calc', job.id, getpass.getuser(), False) except RuntimeError as err: safeprint(err) else: if 'success' in resp: + os.remove(resp['hdf5path']) print('Removed %d' % job.id) else: print(resp['error']) diff --git a/openquake/commands/purge.py b/openquake/commands/purge.py index cac74353c2b7..bce48e4bbd34 100644 --- a/openquake/commands/purge.py +++ b/openquake/commands/purge.py @@ -28,7 +28,7 @@ def purge_one(calc_id, user, force): """ Remove one calculation ID from the database and remove its datastore """ - logs.dbcmd('del_calc', calc_id, user, force) + logs.dbcmd('del_calc', calc_id, user, False, force) f1 = os.path.join(datadir, 'calc_%s.hdf5' % calc_id) f2 = os.path.join(datadir, 'calc_%s_tmp.hdf5' % calc_id) for f in [f1, f2]: diff --git a/openquake/server/db/actions.py b/openquake/server/db/actions.py index da69882cd856..19107d165ae6 100644 --- a/openquake/server/db/actions.py +++ b/openquake/server/db/actions.py @@ -318,13 +318,14 @@ def finish(db, job_id, status): job_id) -def del_calc(db, job_id, user, force=False): +def del_calc(db, job_id, user, delete_file=True, force=False): """ Delete a calculation and all associated outputs, if possible. :param db: a :class:`openquake.commonlib.dbapi.Db` instance :param job_id: job ID, can be an integer or a string :param user: username + :param delete_file: also delete the HDF5 file :param force: delete even if there are dependent calculations :returns: a dict with key "success" and value indicating the job id of the calculation or of its ancestor, or key "error" @@ -338,7 +339,7 @@ def del_calc(db, job_id, user, force=False): if not force and job_id in job_ids: # jobarray err = [] for jid in job_ids: - res = del_calc(db, jid, user, force=True) + res = del_calc(db, jid, user, delete_file, force=True) if "error" in res: err.append(res["error"]) if err: @@ -363,13 +364,14 @@ def del_calc(db, job_id, user, force=False): '%s and you are %s' % (job_id, owner, user)} fname = path + ".hdf5" - # A calculation could fail before it produces a hdf5 - if os.path.isfile(fname): + # A calculation could fail before it produces a hdf5, or somebody + # may have canceled the file, so it could not exist + if delete_file and os.path.isfile(fname): try: os.remove(fname) except OSError as exc: # permission error return {"error": 'Could not remove %s: %s' % (fname, exc)} - return {"success": str(job_id)} + return {"success": str(job_id), "hdf5path": fname} def log(db, job_id, timestamp, level, process, message): From af6ab537a6a86d9e81a954c6abd636d39d6a5ffe Mon Sep 17 00:00:00 2001 From: Michele Simionato Date: Thu, 14 Sep 2023 10:00:03 +0200 Subject: [PATCH 2/2] Updated changelog [ci skip] --- debian/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debian/changelog b/debian/changelog index 0f05f017f30c..1bfead10c379 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,6 @@ [Michele Simionato] + * Enhanced `oq engine --delete-calculation` to remove calculation files + even if the DbServer is on a remote machine * Fixed another site collection filtering bug in conditioned GMFs * Fixed a regression in `oq reaggregate` * Better error message for logic trees with branchsets exceeding the limit