-
Notifications
You must be signed in to change notification settings - Fork 91
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
CA-379329: check for missing iSCSI sessions and reconnect
If, when an iSCSI SR is attached, some of the paths are not reachable they will not be subsequently connected should the connectivity issues be resolved, without detaching the SR or rebooting the host. Neither of these options give a good experience for users of the software. Add a systemd timer which will, periodically (default 10 minutes), ask each SR to do SR specific "health" checks. Currently only a checker for the iSCSI sessions is present but more can be added to this pattern in the future as needs arise. Signed-off-by: Mark Syms <[email protected]>
- Loading branch information
1 parent
fdca39c
commit d28dcc1
Showing
9 changed files
with
520 additions
and
232 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
#!/usr/bin/python3 | ||
|
||
# Copyright (C) Cloud Software Group, Inc. | ||
# | ||
# This program is free software; you can redistribute it and/or modify | ||
# it under the terms of the GNU Lesser General Public License as published | ||
# by the Free Software Foundation; version 2.1 only. | ||
# | ||
# This program is distributed in the hope that it will be useful, | ||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
# GNU Lesser General Public License for more details. | ||
# | ||
# You should have received a copy of the GNU Lesser General Public License | ||
# along with this program; if not, write to the Free Software Foundation, Inc., | ||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
|
||
""" | ||
Health check for SR, to be triggered periodically by a systemd timer. What is checked is | ||
SR implementation type dependent. | ||
""" | ||
|
||
import SR | ||
import util | ||
|
||
|
||
def main(): | ||
""" | ||
For all locally plugged SRs check that they are healthy | ||
""" | ||
try: | ||
session = util.get_localAPI_session() | ||
except SR.SROSError: | ||
util.SMlog("Unable to open local XAPI session", priority=util.LOG_ERR) | ||
return | ||
|
||
localhost = util.get_localhost_ref(session) | ||
|
||
sm_types = [x['type'] for x in session.xenapi.SM.get_all_records_where( | ||
'field "required_api_version" = "1.0"').values()] | ||
for sm_type in sm_types: | ||
srs = session.xenapi.SR.get_all_records_where( | ||
f'field "type" = "{sm_type}"') | ||
for sr in srs: | ||
pbds = session.xenapi.PBD.get_all_records_where( | ||
f'field "SR" = "{sr}" and field "host" = "{localhost}"') | ||
if not pbds: | ||
continue | ||
|
||
pbd_ref, pbd = pbds.popitem() | ||
if not pbd['currently_attached']: | ||
continue | ||
|
||
sr_uuid = srs[sr]['uuid'] | ||
sr_obj = SR.SR.from_uuid(session, sr_uuid) | ||
sr_obj.check_sr(sr_uuid) | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
[Unit] | ||
Description=Healthy Check service for Storage Repositories | ||
Wants=xapi-init-complete.target | ||
|
||
[Service] | ||
Type=oneshot | ||
ExecStart=/opt/xensource/sm/sr_health_check.py | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
[Unit] | ||
Description=Period SR health check | ||
|
||
[Timer] | ||
# Jitter it a bit | ||
RandomizedDelaySec=30 | ||
# Run 10 minutes after first activated... | ||
OnActiveSec=600 | ||
# ...and at 10-minute intervals thereafter | ||
OnUnitInactiveSec=600 | ||
|
||
[Install] | ||
WantedBy=timers.target |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
import unittest | ||
import unittest.mock as mock | ||
|
||
import sr_health_check | ||
from SR import SR | ||
|
||
TEST_HOST = 'test_host' | ||
|
||
SR_UUID = 'sr uuid' | ||
|
||
|
||
class TestSrHealthCheck(unittest.TestCase): | ||
|
||
def setUp(self): | ||
util_patcher = mock.patch('sr_health_check.util') | ||
self.mock_util = util_patcher.start() | ||
self.mock_session = mock.MagicMock() | ||
self.mock_util.get_localAPI_session.return_value = self.mock_session | ||
sr_patcher = mock.patch('sr_health_check.SR.SR', autospec=True) | ||
self.mock_sr = sr_patcher.start() | ||
|
||
self.addCleanup(mock.patch.stopall) | ||
|
||
def expect_good_sr_record(self): | ||
self.mock_session.xenapi.SR.get_all_records_where.return_value = { | ||
"iscsi_ref": {'uuid': SR_UUID, 'host': TEST_HOST} | ||
} | ||
|
||
def expect_good_localhost(self): | ||
self.mock_util.get_localhost_ref.return_value = TEST_HOST | ||
|
||
def expect_good_sm_types(self): | ||
self.mock_session.xenapi.SM.get_all_records_where.return_value = { | ||
'lvmoiscsi_type_ref': {'type': 'lvmoiscsi'} | ||
} | ||
|
||
def test_health_check_no_srs(self): | ||
# Arrange | ||
self.expect_good_sm_types() | ||
self.mock_session.xenapi.SR.get_all_records_where.return_value = {} | ||
|
||
# Act | ||
sr_health_check.main() | ||
|
||
# Assert | ||
self.mock_session.xenapi.SR.get_all_records_where.assert_called() | ||
|
||
def test_health_check_no_local_pbd(self): | ||
# Arrange | ||
self.expect_good_localhost() | ||
self.expect_good_sm_types() | ||
self.expect_good_sr_record() | ||
self.mock_session.xenapi.PBD.get_all_records_where.return_value = {} | ||
|
||
# Act | ||
sr_health_check.main() | ||
|
||
# Assert | ||
self.mock_session.xenapi.PBD.get_all_records_where.assert_called_with( | ||
f'field "SR" = "iscsi_ref" and field "host" = "{TEST_HOST}"') | ||
|
||
def test_health_check_sr_not_plugged(self): | ||
# Arrange | ||
self.expect_good_localhost() | ||
self.expect_good_sm_types() | ||
self.expect_good_sr_record() | ||
self.mock_session.xenapi.PBD.get_all_records_where.return_value = { | ||
'pbd_ref': {'currently_attached': False} | ||
} | ||
|
||
# Act | ||
sr_health_check.main() | ||
|
||
# Assert | ||
self.mock_session.xenapi.PBD.get_all_records_where.assert_called_with( | ||
f'field "SR" = "iscsi_ref" and field "host" = "{TEST_HOST}"') | ||
|
||
def test_health_check_run_sr_check(self): | ||
# Arrange | ||
self.expect_good_localhost() | ||
self.expect_good_sm_types() | ||
self.expect_good_sr_record() | ||
self.mock_session.xenapi.PBD.get_all_records_where.return_value = { | ||
'pbd_ref': {'currently_attached': True} | ||
} | ||
mock_sr = mock.create_autospec(SR) | ||
self.mock_sr.from_uuid.return_value = mock_sr | ||
|
||
# Act | ||
sr_health_check.main() | ||
|
||
# Assert | ||
mock_sr.check_sr.assert_called_with(SR_UUID) |