[WIP] Add support for he start/stop

ovirtlago/cmd.py

@@ -261,12 +261,19 @@ def do_ovirt_status(prefix, out_format, **kwargs):
 def do_ovirt_start(prefix, with_vms, vms_timeout, **kwargs):
     with LogTask('Starting oVirt environment'):
         prefix.start()
+        he = prefix.virt_env.engine_vm().is_hosted_engine()
+        if he:
+            with LogTask('Start HE VM'):
+                prefix.virt_env.engine_vm().start_he_vm()
         with LogTask('Waiting for ovirt-engine status'):
             prefix.virt_env.assert_engine_alive(timeout=3 * 60)
         with LogTask('Waiting for vdsmd status'):
             prefix.virt_env.assert_vdsm_alive(timeout=3 * 60)
         with LogTask('Updating Clusters CPU'):
             prefix.virt_env.update_clusters_cpu()
+        if he:
+            with LogTask('Moving out from global maintenance mode'):
+                prefix.virt_env.set_global_maintenance(False)
         with LogTask('Activating Engine Hosts'):
             prefix.virt_env.engine_vm().start_all_hosts(timeout=5 * 60)
         if with_vms:
@@ -302,11 +309,19 @@ def do_ovirt_start(prefix, with_vms, vms_timeout, **kwargs):
 @in_ovirt_prefix
 @with_logging
 def do_ovirt_stop(prefix, **kwargs):
+    he = prefix.virt_env.engine_vm().is_hosted_engine()
+
     with LogTask('Stopping oVirt environment'):
         with LogTask('Stopping Engine VMs'):
             prefix.virt_env.engine_vm().stop_all_vms()
+        if he:
+            with LogTask('Moving to global maintenance mode'):
+                prefix.virt_env.set_global_maintenance(True)
         with LogTask('Putting hosts in maintenance mode'):
             prefix.virt_env.engine_vm().stop_all_hosts()
+        if he:
+            with LogTask('Shutting down HE VM'):
+                prefix.virt_env.engine_vm().shutdown_he_vm()
         with LogTask('Shutdown Lago VMs'):
             prefix.shutdown()
 

ovirtlago/testlib.py

@@ -224,6 +224,10 @@ def _instance_of_any(obj, cls_list):
     return any(True for cls in cls_list if isinstance(obj, cls))
 
 
+def instance_of_any(obj, cls_list):
+    return _instance_of_any(obj, cls_list)
+
+
 def assert_equals_within(
     func, value, timeout, allowed_exceptions=None, initial_wait=10
 ):

ovirtlago/virt.py

@@ -21,9 +21,11 @@
 import time
 import warnings
 import lago
+from lago.utils import LagoException
 import lago.vm
 import logging
 import yaml
+import json
 from collections import OrderedDict
 from lago.config import config as lago_config
 from ovirtlago import utils
@@ -225,8 +227,8 @@ def _vdsm_up(host):
             testlib.assert_true_within(
                 partial(_vdsm_up, host),
                 timeout=timeout,
-                allowed_exceptions=self._get_check_running_allowed_exceptions(
-                ),
+                allowed_exceptions=
+                self._get_check_running_allowed_exceptions(),
             )
 
     def assert_engine_alive(self, timeout=2 * 60):
@@ -255,6 +257,51 @@ def _ovirt_engine_up(host):
             allowed_exceptions=self._get_check_running_allowed_exceptions(),
         )
 
+    def set_global_maintenance(self, enable):
+        self.first_result_from_host(
+            self._set_global_maintenance, [RuntimeError], enable
+        )
+
+    def _set_global_maintenance(self, host, enable):
+        ret = host().ssh(
+            [
+                'hosted-engine', '--set-maintenance',
+                '--mode={}'.format('global' if enable else 'none')
+            ]
+        )
+        if ret:
+            raise RuntimeError('Failed to change HE maintenance mode')
+
+        testlib.assert_true_within_long(
+            lambda: self.engine_vm()
+            .he_status()['global_maintenance'] == enable
+        )
+
+    def is_up(self):
+        return all([vm.alive() for vm in self._vms.itervalues()]) \
+            and all([net.alive() for net in self._nets.itervalues()])
+
+    def is_down(self):
+        return not self.is_up()
+
+    def host_map(self, func, *args, **kwargs):
+        return (func(h, *args, **kwargs) for h in self._host_vms)
+
+    def first_result_from_host(self, func, allowed_exc, *args, **kwargs):
+        for host in self._host_vms:
+            try:
+                return func(host, *args, **kwargs)
+            except Exception as exc:
+                LOGGER.debug(
+                    '{} did not satisfy {}. Got {}'.
+                    format(host.name(), func.__name__, str(exc))
+                )
+
+                if not testlib.instance_of_any(exc, allowed_exc):
+                    raise
+
+        raise RuntimeError('No host satisfy {}'.format(func.__name__))
+
 
 # TODO : solve the problem of ssh to the Node
 class NodeVM(lago.vm.DefaultVM):
@@ -436,6 +483,79 @@ def _vm_is_down(srv, id):
                 partial(_vm_is_down, srv=vm_srv, id=id), timeout=timeout
             )
 
+    def shutdown(self):
+        if not self.is_hosted_engine():
+            return super(EngineVM, self).shutdown()
+
+    def _is_hosted_engine(self, host):
+        ret = host.ssh(['hosted-engine', '--check-deployed'], tries=5)
+
+        return not ret
+
+    def is_hosted_engine(self):
+        return self.virt_env.first_result_from_host(
+            self._is_hosted_engine,
+            [RuntimeError],
+        )
+
+    def _he_status(self, host):
+        ret = host.ssh(['hosted-engine', '--vm-status', '--json'])
+        if not ret:
+            return json.loads(ret.out)
+
+        raise RuntimeError('Failed to get HE status')
+
+    def he_status(self):
+        self.virt_env.first_result_from_host(
+            self._he_status,
+            [RuntimeError, ValueError],
+        )
+
+    def he_host(self):
+        status = self.he_status()
+        running_on = ''
+        for k, v in status.iteritems():
+            if not k.isdigit():
+                continue
+
+            if v['engine-status']['vm'] == 'up':
+                running_on = v['hostname'].split('.', 1)[0]
+                break
+        try:
+            return next(
+                h for h in self.virt_env.host_vms() if h.name() == running_on
+            )
+        except StopIteration:
+            raise HEVMIsNotRunningError()
+
+    def change_he_vm_state(self, state, verify_func):
+        LOGGER.debug('{} HE VM'.format(state))
+        ret = self.he_host().ssh(['hosted-engine', '--vm-{}'.format(state)])
+        if ret:
+            raise RuntimeError('Failed to {} HE VM'.format(state))
+
+        testlib.assert_true_within_short(verify_func)
+        LOGGER.debug('Succeed to {} HE VM'.format(state))
+
+    def shutdown_he_vm(self):
+        try:
+            self.change_he_vm_state(
+                'shutdown', lambda: all(
+                    v['engine-status']['vm'] != 'up'
+                    for k, v in self.he_status().iteritems() if k.isdigit()
+                )
+            )
+        except HEVMIsNotRunningError:
+            LOGGER.debug('HE VM is not running. Nothing to do')
+
+    def start_he_vm(self):
+        self.change_he_vm_state(
+            'start', lambda: any(
+                v['engine-status']['vm'] == 'up'
+                for k, v in self.he_status().iteritems() if k.isdigit()
+            )
+        )
+
     @require_sdk(version='4')
     def stop_all_hosts(self, timeout=5 * 60):
         api = self.get_api_v4(check=True)
@@ -556,3 +676,8 @@ class HEHostVM(HostVM):
     def _artifact_paths(self):
         inherited_artifacts = super(HEHostVM, self)._artifact_paths()
         return set(inherited_artifacts)
+
+
+class HEVMIsNotRunningError(LagoException):
+    def __init__(self):
+        super(HEVMIsNotRunningError, self).__init__('HE VM is not running')