From ba205f161842333608e0f7c268654d0b3be41586 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 19 Oct 2023 17:57:37 +0100 Subject: [PATCH] HPCC-29215 Add DFU Copy Ensure functionality Add an 'ensure' option to dfu copy request, which copies the file if necessary, but published the meta information only if the physical files already exist. Signed-off-by: Jake Smith --- dali/dfu/dfurun.cpp | 43 +++++++++++++++++++++++++---- dali/dfu/dfuwu.cpp | 12 +++++++- dali/dfu/dfuwu.hpp | 2 ++ esp/scm/ws_fs.ecm | 1 + esp/services/ws_fs/ws_fsService.cpp | 1 + 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 72989b5ed34..36de1b7a9eb 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -1442,6 +1442,8 @@ class CDFUengine: public CInterface, implements IDFUengine } break; } + + bool ensureLfnAlreadyPublished = false; // fill dstfile for commands that need it switch (cmd) { case DFUcmd_copymerge: @@ -1542,6 +1544,14 @@ class CDFUengine: public CInterface, implements IDFUengine Owned oldfile = wsdfs::lookup(tmp.str(),userdesc,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser,INFINITE); if (oldfile) { + if (options->getEnsure()) + { + // logical file already exists. + ensureLfnAlreadyPublished = true; + dstFile.setown(oldfile.getClear()); + dstName.set(tmp); + break; + } StringBuffer reason; bool canRemove = oldfile->canRemove(reason); oldfile.clear(); @@ -1700,12 +1710,35 @@ class CDFUengine: public CInterface, implements IDFUengine } } else { - fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid); - if (!abortnotify.abortRequested()) { - if (needrep) - replicating = true; + bool performCopy = true; + if (options->getEnsure()) + { + if (ensureLfnAlreadyPublished) + performCopy = false; else - dstFile->attach(dstName.get(),userdesc); + { + if (dstFile->existsPhysicalPartFiles(0)) + { + dstFile->attach(dstName.get(), userdesc); + performCopy = false; + } + } + if (!performCopy) + { + feedback.repmode=cProgressReporter::REPnone; + feedback.displaySummary(nullptr, 0); + Audit("COPYENSURE", userdesc, srcFile?srcName.str():nullptr, dstName.get()); + } + } + if (performCopy) + { + fsys.copy(srcFile,dstFile,recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid); + if (!abortnotify.abortRequested()) { + if (needrep) + replicating = true; + else + dstFile->attach(dstName.get(),userdesc); + } Audit("COPY",userdesc,srcFile?srcName.str():NULL,dstName.get()); } } diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp index 2444d37fd41..26306e8db90 100644 --- a/dali/dfu/dfuwu.cpp +++ b/dali/dfu/dfuwu.cpp @@ -2002,6 +2002,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions return queryRoot()->getPropInt("@overwrite")!=0; } + bool getEnsure() const + { + return queryRoot()->getPropInt("@ensure")!=0; + } + DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const { repeatlast = false; @@ -2146,7 +2151,7 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions queryRoot()->setPropInt("@throttle",val); } - void setTransferBufferSize(unsigned val) + void setTransferBufferSize(size32_t val) { queryRoot()->setPropInt("@transferBufferSize",val); } @@ -2161,6 +2166,11 @@ class CDFUoptions: public CLinkedDFUWUchild, implements IDFUoptions queryRoot()->setPropInt("@overwrite",val?1:0); } + void setEnsure(bool val=true) + { + queryRoot()->setPropInt("@ensure",val?1:0); + } + void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false) { queryRoot()->setPropInt("@replicatemode",(int)val); diff --git a/dali/dfu/dfuwu.hpp b/dali/dfu/dfuwu.hpp index 59c15a49e9d..923630a719c 100644 --- a/dali/dfu/dfuwu.hpp +++ b/dali/dfu/dfuwu.hpp @@ -153,6 +153,7 @@ interface IConstDFUoptions : extends IInterface virtual size32_t getTransferBufferSize() const = 0; virtual bool getVerify() const = 0; virtual bool getOverwrite() const = 0; + virtual bool getEnsure() const = 0; virtual DFUreplicateMode getReplicateMode(StringBuffer &cluster, bool &repeatlast,bool &onlyrepeated) const = 0; virtual const char *queryPartFilter() const = 0; virtual bool getKeepHeader() const = 0; @@ -195,6 +196,7 @@ interface IDFUoptions : extends IConstDFUoptions virtual void setTransferBufferSize(size32_t val) = 0; virtual void setVerify(bool val=true) = 0; virtual void setOverwrite(bool val=true) = 0; + virtual void setEnsure(bool val=true) = 0; virtual void setReplicateMode(DFUreplicateMode val,const char *cluster=NULL,bool repeatlast=false,bool onlyrepeated=false) = 0; virtual void setPartFilter(const char *filter) = 0; // format n,n-n,n etc virtual void setKeepHeader(bool val=true) = 0; diff --git a/esp/scm/ws_fs.ecm b/esp/scm/ws_fs.ecm index 6aab351fb1c..064039e1e39 100644 --- a/esp/scm/ws_fs.ecm +++ b/esp/scm/ws_fs.ecm @@ -481,6 +481,7 @@ ESPrequest [nil_remove] Copy string srcusername; string srcpassword; bool overwrite; + bool ensure; bool replicate; int ReplicateOffset(1); diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index dd41506ce37..10bab1b8edf 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -2790,6 +2790,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse wuFSpecDest->setLogicalName(dstname); wuFSpecDest->setFileMask(fileMask.str()); wuOptions->setOverwrite(req.getOverwrite()); + wuOptions->setEnsure(req.getEnsure()); wuOptions->setPreserveCompression(req.getPreserveCompression()); if (!req.getExpireDays_isNull()) wuOptions->setExpireDays(req.getExpireDays());