Split microservices into separate packages

dmwm · Oct 6, 2022 · b5388b5 · b5388b5
1 parent 2d3f8d8
commit b5388b5
Show file tree

Hide file tree

Showing 15 changed files with 59 additions and 23 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -13,15 +13,15 @@
 # https://www.python.org/dev/peps/pep-0440/#version-specifiers
 
 Cheetah3~=3.2.6.post1         # wmcore,wmagent,reqmgr2,reqmon
-CherryPy~=18.8.0              # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+CherryPy~=18.8.0              # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
 CMSCouchapp~=1.3.4            # wmcore,wmagent
-CMSMonitoring~=0.6.9          # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+CMSMonitoring~=0.6.9          # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
 coverage~=5.4                 # wmcore,wmagent,wmagent-devtools
 cx-Oracle~=8.3.0              # wmcore,wmagent
 dbs3-client~=4.0.10           # wmcore,wmagent,reqmgr2,reqmon,global-workqueue
-future~=0.18.2                # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
-gfal2-python~=1.11.0.post3    # wmcore,reqmgr2ms
-httplib2~=0.20.4              # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
+future~=0.18.2                # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
+gfal2-python~=1.11.0.post3    # wmcore,reqmgr2ms-unmerged
+httplib2~=0.20.4              # wmcore,wmagent,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
 htcondor~=8.9.7               # wmcore,wmagent
 Jinja2~=3.1.2                 # wmcore,wmagent
 memory-profiler~=0.60.0       # wmcore,wmagent-devtools
@@ -32,13 +32,13 @@ nose~=1.3.7                   # wmcore,wmagent-devtools
 nose2~=0.12.0                 # wmcore,wmagent-devtools
 pycodestyle~=2.8.0            # wmcore,wmagent-devtools
 psutil~=5.9.1                 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue
-pycurl~=7.45.1                # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms
+pycurl~=7.45.1                # wmcore,wmagent,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
 pylint~=2.14.5                # wmcore,wmagent-devtools
-pymongo~=4.2.0                # wmcore,wmagent-devtools,reqmgr2ms
+pymongo~=4.2.0                # wmcore,wmagent-devtools,reqmgr2ms-unmerged,reqmgr2ms-output
 pyOpenSSL~=18.0.0             # wmcore,wmagent
 pyzmq~=23.2.1                 # wmcore,wmagent
-retry~=0.9.2                  # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms
-rucio-clients~=1.25.5         # wmcore,wmagent,global-workqueue,reqmgr2ms
-Sphinx~=5.1.1                 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms
+retry~=0.9.2                  # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
+rucio-clients~=1.25.5         # wmcore,wmagent,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
+Sphinx~=5.1.1                 # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue
 SQLAlchemy~=1.4.40            # wmcore,wmagent
-PyJWT~=2.4.0                  # wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms,wmcore
+PyJWT~=2.4.0                  # wmcore,wmagent,wmagent-devtools,reqmgr2,reqmon,acdcserver,global-workqueue,reqmgr2ms-unmerged,reqmgr2ms-output,reqmgr2ms-monitor,reqmgr2ms-transferor,reqmgr2ms-rulecleaner
diff --git a/setup_dependencies.py b/setup_dependencies.py
@@ -110,9 +110,44 @@
                     'WMCore.ReqMgr.DataStructs.RequestStatus',
                     'WMCore.ReqMgr.DataStructs.RequestType'
                     ],
-        'systems': ['wmc-rest', 'wmc-runtime', 'wmc-database'],
+        'systems': ['wmc-rest', 'wmc-database'],
         'statics': [],
     },
+    'reqmgr2ms-core': {
+        'packages': ['WMCore.MicroService.MSCore', 'WMCore.MicroService.DataStructs',
+                     'WMCore.MicroService.Tools', 'WMCore.MicroService.CherryPyThreads',
+                     'WMCore.MicroService.Service', 'WMCore.MicroService.WebGui',
+                     'Utils', 'WMCore.Services+'],
+        'modules': ['WMCore.Wrappers.__init__',
+                    'WMCore.Wrappers.JsonWrapper.__init__',
+                    'WMCore.Wrappers.JsonWrapper.JSONThunker',
+                    'WMCore.ReqMgr.__init__', 'WMCore.ReqMgr.DataStructs.__init__',
+                    'WMCore.ReqMgr.DataStructs.RequestStatus',
+                    'WMCore.ReqMgr.DataStructs.RequestType'
+                    ],
+        'systems': ['wmc-rest', 'wmc-database'],
+        'statics': [],
+    },
+    'reqmgr2ms-unmerged': {
+        'packages': ['WMCore.MicroService.MSUnmerged'],
+        'systems': ['reqmgr2ms-core'],
+    },
+    'reqmgr2ms-output': {
+        'packages': ['WMCore.MicroService.MSOutput'],
+        'systems': ['reqmgr2ms-core'],
+    },
+    'reqmgr2ms-transferor': {
+        'packages': ['WMCore.MicroService.MSTransferor'],
+        'systems': ['reqmgr2ms-core'],
+    },
+    'reqmgr2ms-monitor': {
+        'packages': ['WMCore.MicroService.MSMonitor'],
+        'systems': ['reqmgr2ms-core'],
+    },
+    'reqmgr2ms-rulecleaner': {
+        'packages': ['WMCore.MicroService.MSRuleCleaner'],
+        'systems': ['reqmgr2ms-core'],
+    },
     'global-workqueue': {
         'packages': ['WMCore.GlobalWorkQueue+', 'WMCore.WorkQueue+',
                      'WMCore.Wrappers+', 'WMCore.Services+',

diff --git a/src/python/WMCore/MicroService/MSCore.py → ...thon/WMCore/MicroService/MSCore/MSCore.py b/src/python/WMCore/MicroService/MSCore.py → ...thon/WMCore/MicroService/MSCore/MSCore.py
diff --git a/src/python/WMCore/MicroService/MSManager.py → ...n/WMCore/MicroService/MSCore/MSManager.py b/src/python/WMCore/MicroService/MSManager.py → ...n/WMCore/MicroService/MSCore/MSManager.py
@@ -31,7 +31,7 @@
 
 # WMCore modules
 from WMCore.MicroService.Tools.Common import getMSLogger
-from WMCore.MicroService.TaskManager import start_new_thread
+from WMCore.MicroService.MSCore.TaskManager import start_new_thread
 from Utils.Utilities import strToBool
 
 def daemon(func, reqStatus, interval, logger):

diff --git a/...python/WMCore/MicroService/TaskManager.py → ...WMCore/MicroService/MSCore/TaskManager.py b/...python/WMCore/MicroService/TaskManager.py → ...WMCore/MicroService/MSCore/TaskManager.py
diff --git a/src/python/WMCore/MicroService/MSCore/__init__.py b/src/python/WMCore/MicroService/MSCore/__init__.py
diff --git a/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py b/src/python/WMCore/MicroService/MSMonitor/MSMonitor.py
@@ -13,7 +13,7 @@
 
 # WMCore modules
 from WMCore.MicroService.DataStructs.DefaultStructs import MONITOR_REPORT
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 from WMCore.Services.Rucio.Rucio import Rucio
 
 

diff --git a/src/python/WMCore/MicroService/MSOutput/MSOutput.py b/src/python/WMCore/MicroService/MSOutput/MSOutput.py
@@ -18,7 +18,7 @@
 
 # WMCore modules
 from WMCore.MicroService.DataStructs.DefaultStructs import OUTPUT_REPORT
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 from WMCore.MicroService.Tools.Common import gigaBytes
 from WMCore.Services.CRIC.CRIC import CRIC
 from WMCore.Services.DBS.DBS3Reader import getDataTiers

diff --git a/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py b/src/python/WMCore/MicroService/MSRuleCleaner/MSRuleCleaner.py
@@ -27,7 +27,7 @@
 # WMCore modules
 from WMCore.MicroService.DataStructs.DefaultStructs import RULECLEANER_REPORT
 from WMCore.MicroService.MSRuleCleaner.MSRuleCleanerWflow import MSRuleCleanerWflow
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 from WMCore.Services.pycurl_manager import RequestHandler
 from WMCore.Services.Rucio.Rucio import WMRucioDIDNotFoundException
 from WMCore.ReqMgr.DataStructs import RequestStatus

diff --git a/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py b/src/python/WMCore/MicroService/MSTransferor/MSTransferor.py
@@ -25,7 +25,7 @@
 from WMCore.MicroService.DataStructs.DefaultStructs import TRANSFEROR_REPORT,\
     TRANSFER_RECORD, TRANSFER_COUCH_DOC
 from WMCore.MicroService.Tools.Common import gigaBytes, teraBytes, isRelVal
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 from WMCore.MicroService.MSTransferor.RequestInfo import RequestInfo
 from WMCore.MicroService.MSTransferor.DataStructs.RSEQuotas import RSEQuotas
 from WMCore.Services.CRIC.CRIC import CRIC

diff --git a/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py b/src/python/WMCore/MicroService/MSTransferor/RequestInfo.py
@@ -26,7 +26,7 @@
 from WMCore.MicroService.Tools.Common import (elapsedTime, findBlockParents,
                                               findParent, getBlocksByDsetAndRun,
                                               getFileLumisInBlock, getRunsInBlock)
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 
 
 def isNanoWorkflow(reqDict):

diff --git a/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py b/src/python/WMCore/MicroService/MSUnmerged/MSUnmerged.py
@@ -32,7 +32,7 @@
 
 # WMCore modules
 from WMCore.MicroService.DataStructs.DefaultStructs import UNMERGED_REPORT
-from WMCore.MicroService.MSCore import MSCore
+from WMCore.MicroService.MSCore.MSCore import MSCore
 from WMCore.MicroService.MSUnmerged.MSUnmergedRSE import MSUnmergedRSE
 from WMCore.Services.RucioConMon.RucioConMon import RucioConMon
 from WMCore.Services.WMStatsServer.WMStatsServer import WMStatsServer

diff --git a/test/python/WMCore_t/MicroService_t/MSManager_t.py b/test/python/WMCore_t/MicroService_t/MSManager_t.py
@@ -11,7 +11,7 @@
 from Utils.PythonVersion import PY3
 
 from WMCore.Agent.Configuration import Configuration
-from WMCore.MicroService.MSManager import MSManager
+from WMCore.MicroService.MSCore.MSManager import MSManager
 
 
 class MSManagerTest(unittest.TestCase):

diff --git a/test/python/WMCore_t/MicroService_t/TaskManager_t.py b/test/python/WMCore_t/MicroService_t/TaskManager_t.py
@@ -11,10 +11,9 @@
 from builtins import str as newstr
 from future.utils import listvalues
 
-from WMCore.MicroService.TaskManager import \
+from WMCore.MicroService.MSCore.TaskManager import \
     TaskManager, genkey, UidSet
 
-
 def myFunc(interval, results):
     "Test function"
     time.sleep(interval)

diff --git a/tools/build_pypi_packages.sh b/tools/build_pypi_packages.sh
@@ -14,7 +14,9 @@ set -x
 # package passed as parameter, can be one of PACKAGES or "all"
 TOBUILD=$1
 # list of packages that can be built and uploaded to pypi
-PACKAGES="wmagent wmagent-devtools wmcore reqmon reqmgr2 reqmgr2ms global-workqueue acdcserver"
+PACKAGES="wmagent wmagent-devtools wmcore reqmon reqmgr2 global-workqueue acdcserver
+          reqmgr2ms-unmerged reqmgr2ms-output reqmgr2ms-rulecleaner reqmgr2ms-transferor reqmgr2ms-monitor"
+
 PACKAGE_REGEX="^($(echo $PACKAGES | sed 's/\ /|/g')|all)$"
 
 if [[ -z $TOBUILD ]]; then