diff --git a/requirements.txt b/requirements.txt index 8746f3b29e..631fd73e3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,15 +13,15 @@ # https://www.python.org/dev/peps/pep-0440/#version-specifiers Cheetah3~=3.2.6.post1 # wmcore,wmagent,reqmgr2,reqmon -CherryPy~=18.8.0 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms +CherryPy~=18.8.0 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner CMSCouchapp~=1.3.4 # wmcore,wmagent -CMSMonitoring~=0.6.9 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms +CMSMonitoring~=0.6.9 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner coverage~=5.4 # wmcore,wmagent,wmagent-devtools cx-Oracle~=8.3.0 # wmcore,wmagent dbs3-client~=4.0.10 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue -future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms -gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms -httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms +future~=0.18.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +gfal2-python~=1.11.0.post3 # wmcore,reqmgr2ms-unmerged +httplib2~=0.20.4 # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner htcondor~=8.9.7 # wmcore,wmagent Jinja2~=3.1.2 # wmcore,wmagent memory-profiler~=0.60.0 # wmcore,wmagent-devtools @@ -32,13 +32,13 @@ nose~=1.3.7 # wmcore,wmagent-devtools nose2~=0.12.0 # wmcore,wmagent-devtools pycodestyle~=2.8.0 # wmcore,wmagent-devtools psutil~=5.9.1 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue -pycurl~=7.45.1 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms +pycurl~=7.45.1 # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner pylint~=2.14.5 # wmcore,wmagent-devtools -pymongo~=4.2.0 # wmcore,wmagent-devtools,reqmgr2ms +pymongo~=4.2.0 # wmcore,wmagent-devtools,reqmgr2ms-unmerged,reqmgr2ms-output pyOpenSSL~=18.0.0 # wmcore,wmagent pyzmq~=23.2.1 # wmcore,wmagent -retry~=0.9.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms -rucio-clients~=1.25.5 # wmcore,wmagent,global-workqueue,reqmgr2ms -Sphinx~=5.1.1 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms +retry~=0.9.2 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +rucio-clients~=1.25.5 # wmcore,wmagent,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner +Sphinx~=5.1.1 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue SQLAlchemy~=1.4.40 # wmcore,wmagent -PyJWT~=2.4.0 # wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms,wmcore +PyJWT~=2.4.0 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner diff --git a/setup_dependencies.py b/setup_dependencies.py index 56204779ef..0f84691b2e 100644 --- a/setup_dependencies.py +++ b/setup_dependencies.py @@ -110,9 +110,44 @@ 'WMCore.ReqMgr.DataStructs.RequestStatus', 'WMCore.ReqMgr.DataStructs.RequestType' ], - 'systems': ['wmc-rest', 'wmc-runtime', 'wmc-database'], + 'systems': ['wmc-rest', 'wmc-database'], 'statics': [], }, + 'reqmgr2ms-core': { + 'packages': ['WMCore.MicroService.MSCore', 'WMCore.MicroService.DataStructs', + 'WMCore.MicroService.Tools', 'WMCore.MicroService.CherryPyThreads', + 'WMCore.MicroService.Service', 'WMCore.MicroService.WebGui', + 'Utils', 'WMCore.Services+'], + 'modules': ['WMCore.Wrappers.__init__', + 'WMCore.Wrappers.JsonWrapper.__init__', + 'WMCore.Wrappers.JsonWrapper.JSONThunker', + 'WMCore.ReqMgr.__init__', 'WMCore.ReqMgr.DataStructs.__init__', + 'WMCore.ReqMgr.DataStructs.RequestStatus', + 'WMCore.ReqMgr.DataStructs.RequestType' + ], + 'systems': ['wmc-rest', 'wmc-database'], + 'statics': [], + }, + 'reqmgr2ms-unmerged': { + 'packages': ['WMCore.MicroService.MSUnmerged'], + 'systems': ['reqmgr2ms-core'], + }, + 'reqmgr2ms-output': { + 'packages': ['WMCore.MicroService.MSOutput'], + 'systems': ['reqmgr2ms-core'], + }, + 'reqmgr2ms-transferor': { + 'packages': ['WMCore.MicroService.MSTransferor'], + 'systems': ['reqmgr2ms-core'], + }, + 'reqmgr2ms-monitor': { + 'packages': ['WMCore.MicroService.MSMonitor'], + 'systems': ['reqmgr2ms-core'], + }, + 'reqmgr2ms-rulecleaner': { + 'packages': ['WMCore.MicroService.MSRuleCleaner'], + 'systems': ['reqmgr2ms-core'], + }, 'global-workqueue': { 'packages': ['WMCore.GlobalWorkQueue+', 'WMCore.WorkQueue+', 'WMCore.Wrappers+', 'WMCore.Services+', diff --git a/src/python/WMCore/MicroService/MSCore.py b/src/python/WMCore/MicroService/MSCore/MSCore.py similarity index 100% rename from src/python/WMCore/MicroService/MSCore.py rename to src/python/WMCore/MicroService/MSCore/MSCore.py diff --git a/src/python/WMCore/MicroService/MSManager.py b/src/python/WMCore/MicroService/MSCore/MSManager.py similarity index 99% rename from src/python/WMCore/MicroService/MSManager.py rename to src/python/WMCore/MicroService/MSCore/MSManager.py index 6e93b961b0..5a2d0d8046 100644 --- a/src/python/WMCore/MicroService/MSManager.py +++ b/src/python/WMCore/MicroService/MSCore/MSManager.py @@ -31,7 +31,7 @@ # WMCore modules from WMCore.MicroService.Tools.Common import getMSLogger -from WMCore.MicroService.TaskManager import start_new_thread +from WMCore.MicroService.MSCore.TaskManager import start_new_thread from Utils.Utilities import strToBool def daemon(func, reqStatus, interval, logger): diff --git a/src/python/WMCore/MicroService/TaskManager.py b/src/python/WMCore/MicroService/MSCore/TaskManager.py similarity index 100% rename from src/python/WMCore/MicroService/TaskManager.py rename to src/python/WMCore/MicroService/MSCore/TaskManager.py diff --git a/src/python/WMCore/MicroService/MSCore/__init__.py b/src/python/WMCore/MicroService/MSCore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py b/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py index 8dab01e1fe..e3268ad953 100644 --- a/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py +++ b/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py @@ -13,7 +13,7 @@ # WMCore modules from WMCore.MicroService.DataStructs.DefaultStructs import MONITOR_REPORT -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore from WMCore.Services.Rucio.Rucio import Rucio diff --git a/src/python/WMCore/MicroService/MSOutput/MSOutput.py b/src/python/WMCore/MicroService/MSOutput/MSOutput.py index 6a9d7731a0..ae6a1deb1a 100644 --- a/src/python/WMCore/MicroService/MSOutput/MSOutput.py +++ b/src/python/WMCore/MicroService/MSOutput/MSOutput.py @@ -18,7 +18,7 @@ # WMCore modules from WMCore.MicroService.DataStructs.DefaultStructs import OUTPUT_REPORT -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore from WMCore.MicroService.Tools.Common import gigaBytes from WMCore.Services.CRIC.CRIC import CRIC from WMCore.Services.DBS.DBS3Reader import getDataTiers diff --git a/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py b/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py index ed9598068d..50cf3147f5 100644 --- a/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py +++ b/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py @@ -27,7 +27,7 @@ # WMCore modules from WMCore.MicroService.DataStructs.DefaultStructs import RULECLEANER_REPORT from WMCore.MicroService.MSRuleCleaner.MSRuleCleanerWflow import MSRuleCleanerWflow -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore from WMCore.Services.pycurl_manager import RequestHandler from WMCore.Services.Rucio.Rucio import WMRucioDIDNotFoundException from WMCore.ReqMgr.DataStructs import RequestStatus diff --git a/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py b/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py index 967dce479b..6cecbdee55 100644 --- a/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py +++ b/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py @@ -25,7 +25,7 @@ from WMCore.MicroService.DataStructs.DefaultStructs import TRANSFEROR_REPORT,\ TRANSFER_RECORD, TRANSFER_COUCH_DOC from WMCore.MicroService.Tools.Common import gigaBytes, teraBytes, isRelVal -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore from WMCore.MicroService.MSTransferor.RequestInfo import RequestInfo from WMCore.MicroService.MSTransferor.DataStructs.RSEQuotas import RSEQuotas from WMCore.Services.CRIC.CRIC import CRIC diff --git a/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py b/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py index bfa6855f30..657381da6a 100644 --- a/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py +++ b/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py @@ -26,7 +26,7 @@ from WMCore.MicroService.Tools.Common import (elapsedTime, findBlockParents, findParent, getBlocksByDsetAndRun, getFileLumisInBlock, getRunsInBlock) -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore def isNanoWorkflow(reqDict): diff --git a/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py b/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py index 25b5d09785..2523d0c690 100644 --- a/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py +++ b/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py @@ -32,7 +32,7 @@ # WMCore modules from WMCore.MicroService.DataStructs.DefaultStructs import UNMERGED_REPORT -from WMCore.MicroService.MSCore import MSCore +from WMCore.MicroService.MSCore.MSCore import MSCore from WMCore.MicroService.MSUnmerged.MSUnmergedRSE import MSUnmergedRSE from WMCore.Services.RucioConMon.RucioConMon import RucioConMon from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer diff --git a/test/python/WMCore_t/MicroService_t/MSManager_t.py b/test/python/WMCore_t/MicroService_t/MSManager_t.py index d65387540d..e10c79ecd8 100644 --- a/test/python/WMCore_t/MicroService_t/MSManager_t.py +++ b/test/python/WMCore_t/MicroService_t/MSManager_t.py @@ -11,7 +11,7 @@ from Utils.PythonVersion import PY3 from WMCore.Agent.Configuration import Configuration -from WMCore.MicroService.MSManager import MSManager +from WMCore.MicroService.MSCore.MSManager import MSManager class MSManagerTest(unittest.TestCase): diff --git a/test/python/WMCore_t/MicroService_t/TaskManager_t.py b/test/python/WMCore_t/MicroService_t/TaskManager_t.py index 1f23cfdfa9..0dda9967b7 100644 --- a/test/python/WMCore_t/MicroService_t/TaskManager_t.py +++ b/test/python/WMCore_t/MicroService_t/TaskManager_t.py @@ -11,10 +11,9 @@ from builtins import str as newstr from future.utils import listvalues -from WMCore.MicroService.TaskManager import \ +from WMCore.MicroService.MSCore.TaskManager import \ TaskManager, genkey, UidSet - def myFunc(interval, results): "Test function" time.sleep(interval) diff --git a/tools/build_pypi_packages.sh b/tools/build_pypi_packages.sh index c9a6b542eb..9b642e4055 100644 --- a/tools/build_pypi_packages.sh +++ b/tools/build_pypi_packages.sh @@ -14,7 +14,9 @@ set -x # package passed as parameter, can be one of PACKAGES or "all" TOBUILD=$1 # list of packages that can be built and uploaded to pypi -PACKAGES="wmagent wmagent-devtools wmcore reqmon reqmgr2 reqmgr2ms global-workqueue acdcserver" +PACKAGES="wmagent wmagent-devtools wmcore reqmon reqmgr2 global-workqueue acdcserver + reqmgr2ms-unmerged reqmgr2ms-output reqmgr2ms-rulecleaner reqmgr2ms-transferor reqmgr2ms-monitor" + PACKAGE_REGEX="^($(echo $PACKAGES | sed 's/\ /|/g')|all)$" if [[ -z $TOBUILD ]]; then