From 941d6afcd49e15dd2be897783d243fce8f7c889e Mon Sep 17 00:00:00 2001 From: Rui Marinho <@uphold.com> Date: Thu, 22 Aug 2024 12:42:07 +0100 Subject: [PATCH] Add support for AWS Snapshot Lock --- barman/clients/cloud_backup.py | 31 ++- barman/cloud_providers/__init__.py | 8 + barman/cloud_providers/aws_s3.py | 84 +++++++- barman/config.py | 22 ++ barman/utils.py | 48 +++++ tests/test_cli.py | 1 + tests/test_cloud_snapshot_interface.py | 275 ++++++++++++++++++++++++- tests/test_config.py | 8 + tests/test_infofile.py | 3 + tests/test_utils.py | 34 +++ tests/testing_helpers.py | 4 + 11 files changed, 509 insertions(+), 9 deletions(-) diff --git a/barman/clients/cloud_backup.py b/barman/clients/cloud_backup.py index 7ec654b54..c0161bc78 100755 --- a/barman/clients/cloud_backup.py +++ b/barman/clients/cloud_backup.py @@ -45,7 +45,7 @@ UnrecoverableHookScriptError, ) from barman.postgres import PostgreSQLConnection -from barman.utils import check_backup_name, check_positive, check_size, force_str +from barman.utils import check_aws_snapshot_lock_duration_range, check_aws_snapshot_lock_cool_off_period_range, check_backup_name, check_positive, check_size, check_timestamp, force_str _find_space = re.compile(r"[\s]").search @@ -419,6 +419,26 @@ def parse_arguments(args=None): "timing out (default: 3600 seconds)", type=check_positive, ) + s3_arguments.add_argument( + "--aws-snapshot-lock-mode", + help="The lock mode to apply to the snapshot. Allowed values: 'governance'|'compliance'.", + choices=["governance", "compliance"], + ) + s3_arguments.add_argument( + "--aws-snapshot-lock-duration", + help="The duration (in days) for which the snapshot should be locked. Must be between 1 and 36500. To lock a snapshopt, you must specify either this argument or --aws-snapshot-lock-expiration-date, but not both.", + type=check_aws_snapshot_lock_duration_range, + ) + s3_arguments.add_argument( + "--aws-snapshot-lock-cool-off-period", + help="Specifies the cool-off period (in hours) for a snapshot locked in 'compliance' mode, allowing you to unlock or modify lock settings after it is locked. Must be between 1 and 72 hours. To lock the snapshot immediately without a cool-off period, leave this option unset.", + type=check_aws_snapshot_lock_cool_off_period_range, + ) + s3_arguments.add_argument( + "--aws-snapshot-lock-expiration-date", + help="The expiration date for a locked snapshot in the format YYYY-MM-DDThh:mm:ss.sssZ. To lock a snapshot, you must specify either this argument or --aws-snapshot-lock-duration, but not both.", + type=check_timestamp + ) azure_arguments.add_argument( "--encryption-scope", help="The name of an encryption scope defined in the Azure Blob Storage " @@ -434,7 +454,14 @@ def parse_arguments(args=None): help="The name of the Azure resource group to which the compute instance and " "disks defined by the --snapshot-instance and --snapshot-disk arguments belong.", ) - return parser.parse_args(args=args) + + parsed_args = parser.parse_args(args=args) + + # Perform mutual exclusivity check + if parsed_args.aws_snapshot_lock_duration is not None and parsed_args.aws_snapshot_lock_expiration_date is not None: + parser.error("You must specify either --aws-snapshot-lock-duration or --aws-snapshot-lock-expiration-date, but not both.") + + return parsed_args if __name__ == "__main__": diff --git a/barman/cloud_providers/__init__.py b/barman/cloud_providers/__init__.py index c65317bfe..1c5b31e55 100644 --- a/barman/cloud_providers/__init__.py +++ b/barman/cloud_providers/__init__.py @@ -201,6 +201,10 @@ def get_snapshot_interface(config): config.aws_profile, config.aws_region, config.aws_await_snapshots_timeout, + config.aws_snapshot_lock_mode, + config.aws_snapshot_lock_duration, + config.aws_snapshot_lock_cool_off_period, + config.aws_snapshot_lock_expiration_date, ] return AwsCloudSnapshotInterface(*args) else: @@ -253,6 +257,10 @@ def get_snapshot_interface_from_server_config(server_config): server_config.aws_profile, server_config.aws_region, server_config.aws_await_snapshots_timeout, + server_config.aws_snapshot_lock_mode, + server_config.aws_snapshot_lock_duration, + server_config.aws_snapshot_lock_cool_off_period, + server_config.aws_snapshot_lock_expiration_date, ) else: raise CloudProviderUnsupported( diff --git a/barman/cloud_providers/aws_s3.py b/barman/cloud_providers/aws_s3.py index 7cf4cb330..340318439 100644 --- a/barman/cloud_providers/aws_s3.py +++ b/barman/cloud_providers/aws_s3.py @@ -463,7 +463,16 @@ class AwsCloudSnapshotInterface(CloudSnapshotInterface): https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-creating-snapshot.html """ - def __init__(self, profile_name=None, region=None, await_snapshots_timeout=3600): + def __init__( + self, + profile_name=None, + region=None, + await_snapshots_timeout=3600, + lock_mode=None, + lock_duration=None, + lock_cool_off_period=None, + lock_expiration_date=None + ): """ Creates the client necessary for creating and managing snapshots. @@ -471,13 +480,24 @@ def __init__(self, profile_name=None, region=None, await_snapshots_timeout=3600) :param str region: The AWS region in which snapshot resources are located. :param int await_snapshots_timeout: The maximum time in seconds to wait for snapshots to complete. + :param str lock_mode: The lock mode to apply to the snapshot. + :param int lock_duration: The duration (in days) for which the snapshot + should be locked. + :param int lock_cool_off_period: The cool-off period (in hours) for the snapshot. + :param str lock_expiration_date: The expiration date for the snapshot in the format + YYYY-MM-DDThh:mm:ss.sssZ. """ + self.session = boto3.Session(profile_name=profile_name) # If a specific region was provided then this overrides any region which may be # defined in the profile self.region = region or self.session.region_name self.ec2_client = self.session.client("ec2", region_name=self.region) self.await_snapshots_timeout = await_snapshots_timeout + self.lock_mode = lock_mode + self.lock_duration = lock_duration + self.lock_cool_off_period = lock_cool_off_period + self.lock_expiration_date = lock_expiration_date def _get_waiter_config(self): delay = 15 @@ -761,10 +781,12 @@ def take_snapshot_backup(self, backup_info, instance_identifier, volumes): snapshot_name, snapshot_resp = self._create_snapshot( backup_info, volume_identifier, volume_metadata.id ) + snapshots.append( AwsSnapshotMetadata( snapshot_id=snapshot_resp["SnapshotId"], snapshot_name=snapshot_name, + snapshot_lock_mode=self.lock_mode, device_name=attached_volumes[0]["DeviceName"], mount_options=volume_metadata.mount_options, mount_point=volume_metadata.mount_point, @@ -783,14 +805,57 @@ def take_snapshot_backup(self, backup_info, instance_identifier, volumes): WaiterConfig=self._get_waiter_config(), ) + # Apply lock on snapshots if lock mode is specified + if self.lock_mode: + self._lock_snapshots( + snapshots, + self.lock_mode, + self.lock_duration, + self.lock_cool_off_period, + self.lock_expiration_date + ) + backup_info.snapshots_info = AwsSnapshotsInfo( snapshots=snapshots, region=self.region, # All snapshots will have the same OwnerId so we get it from the last # snapshot response. account_id=snapshot_resp["OwnerId"], + ) + def _lock_snapshots(self, snapshots, lock_mode, lock_duration, lock_cool_off_period, lock_expiration_date): + lock_snapshot_default_args = { + "LockMode": lock_mode + } + + if lock_duration: + lock_snapshot_default_args["LockDuration"] = lock_duration + + if lock_cool_off_period: + lock_snapshot_default_args["CoolOffPeriod"] = lock_cool_off_period + + if lock_expiration_date: + lock_snapshot_default_args["ExpirationDate"] = lock_expiration_date + + for snapshot in snapshots: + lock_snapshot_args = lock_snapshot_default_args.copy() + lock_snapshot_args["SnapshotId"] = snapshot.identifier + + resp = self.ec2_client.lock_snapshot(**lock_snapshot_args) + + logging.info( + "Snapshot %s locked in state '%s' (lock duration: %s days, cool-off period: %s hours, " + "cool-off period expires on: %s, lock expires on: %s, lock duration time: %s)", + snapshot.identifier, + resp["LockState"], + resp["LockDuration"], + resp["CoolOffPeriod"], + resp["CoolOffPeriodExpiresOn"], + resp["LockExpiresOn"], + resp["LockDurationStartTime"] + ) + def _delete_snapshot(self, snapshot_id): """ Delete the specified snapshot. @@ -824,6 +889,16 @@ def delete_snapshot_backup(self, backup_info): snapshot.identifier, backup_info.backup_id, ) + + if snapshot.snapshot_lock_mode is not None: + resp = self.ec2_client.describe_locked_snapshots( + SnapshotIds=[snapshot.identifier], + ) + + if resp["Snapshots"] and resp["Snapshots"][0]["LockState"] != "expired": + logging.warning("Skipping deletion of snapshot %s as it not expired yet", snapshot.identifier) + continue + self._delete_snapshot(snapshot.identifier) def get_attached_volumes( @@ -1003,11 +1078,11 @@ class AwsSnapshotMetadata(SnapshotMetadata): """ Specialization of SnapshotMetadata for AWS EBS snapshots. - Stores the device_name, snapshot_id and snapshot_name in the provider-specific + Stores the device_name, snapshot_id, snapshot_name and snapshot_lock_mode in the provider-specific field. """ - _provider_fields = ("device_name", "snapshot_id", "snapshot_name") + _provider_fields = ("device_name", "snapshot_id", "snapshot_name", "snapshot_lock_mode") def __init__( self, @@ -1016,6 +1091,7 @@ def __init__( device_name=None, snapshot_id=None, snapshot_name=None, + snapshot_lock_mode=None, ): """ Constructor saves additional metadata for AWS snapshots. @@ -1027,12 +1103,14 @@ def __init__( :param str device_name: The device name used in the AWS API. :param str snapshot_id: The snapshot ID used in the AWS API. :param str snapshot_name: The snapshot name stored in the `Name` tag. + :param str snapshot_lock_mode: The mode with which the snapshot has been locked, if set. :param str project: The AWS project name. """ super(AwsSnapshotMetadata, self).__init__(mount_options, mount_point) self.device_name = device_name self.snapshot_id = snapshot_id self.snapshot_name = snapshot_name + self.snapshot_lock_mode = snapshot_lock_mode @property def identifier(self): diff --git a/barman/config.py b/barman/config.py index b569e0241..09a43b464 100644 --- a/barman/config.py +++ b/barman/config.py @@ -232,6 +232,17 @@ def parse_time_interval(value): return time_delta +def parse_datetime(value): + """ + Parse a string, transforming it in a datetime object. + Accepted format: YYYY-MM-DDThh:mm:ss.sssZ + + :param str value: the string to evaluate + """ + + return datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ") + + def parse_si_suffix(value): """ Parse a string, transforming it into integer and multiplying by @@ -489,6 +500,10 @@ class ServerConfig(BaseConfig): "archiver_batch_size", "autogenerate_manifest", "aws_await_snapshots_timeout", + "aws_snapshot_lock_mode", + "aws_snapshot_lock_duration", + "aws_snapshot_lock_cool_off_period", + "aws_snapshot_lock_expiration_date", "aws_profile", "aws_region", "azure_credential", @@ -587,6 +602,10 @@ class ServerConfig(BaseConfig): "archiver_batch_size", "autogenerate_manifest", "aws_await_snapshots_timeout", + "aws_snapshot_lock_mode", + "aws_snapshot_lock_duration", + "aws_snapshot_lock_cool_off_period", + "aws_snapshot_lock_expiration_date", "aws_profile", "aws_region", "azure_credential", @@ -710,6 +729,9 @@ class ServerConfig(BaseConfig): "archiver_batch_size": int, "autogenerate_manifest": parse_boolean, "aws_await_snapshots_timeout": int, + "aws_snapshot_lock_duration": int, + "aws_snapshot_lock_cool_off_period": int, + "aws_snapshot_lock_expiration_date": parse_datetime, "backup_compression": parse_backup_compression, "backup_compression_format": parse_backup_compression_format, "backup_compression_level": int, diff --git a/barman/utils.py b/barman/utils.py index 08752e67d..695a6896c 100644 --- a/barman/utils.py +++ b/barman/utils.py @@ -726,6 +726,44 @@ def check_positive(value): return int_value +def check_aws_snapshot_lock_duration_range(value): + """ + Check for AWS Snapshot Lock duration range option + + :param value: str containing the value to check + """ + if value is None: + return None + try: + int_value = int(value) + except Exception: + raise ArgumentTypeError("'%s' is not a valid input" % value) + + if int_value < 1 or int_value > 36500: + raise ArgumentTypeError("'%s' is outside supported range of 1-36500 days" % value) + + return int_value + + +def check_aws_snapshot_lock_cool_off_period_range(value): + """ + Check for AWS Snapshot Lock cool-off period range option + + :param value: str containing the value to check + """ + if value is None: + return None + try: + int_value = int(value) + except Exception: + raise ArgumentTypeError("'%s' is not a valid input" % value) + + if int_value < 1 or int_value > 72: + raise ArgumentTypeError("'%s' is outside supported range of 1-72 hours" % value) + + return int_value + + def check_tli(value): """ Check for a positive integer option, and also make "current" and "latest" acceptable values @@ -783,6 +821,16 @@ def check_size(value): return int_value +def check_timestamp(value): + try: + # Attempt to parse the input date string into a datetime object + return datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%fZ") + except ValueError: + raise ArgumentTypeError( + "Invalid expiration date: '%s'. Expected format is 'YYYY-MM-DDThh:mm:ss.sssZ'." % value + ) + + def check_backup_name(backup_name): """ Verify that a backup name is not a backup ID or reserved identifier. diff --git a/tests/test_cli.py b/tests/test_cli.py index a11d6ef95..51afa26ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1761,6 +1761,7 @@ def test_help_output(self, minimal_parser, capsys): if sys.version_info < (3, 10): options_label = "optional arguments" expected_output = self._expected_help_output.format(options_label=options_label) + assert expected_output == out diff --git a/tests/test_cloud_snapshot_interface.py b/tests/test_cloud_snapshot_interface.py index 17993bf06..70997e85c 100644 --- a/tests/test_cloud_snapshot_interface.py +++ b/tests/test_cloud_snapshot_interface.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Barman. If not, see . +import datetime import logging import mock import pytest @@ -143,13 +144,22 @@ def test_from_config_aws(self, mock_boto3): aws_region="us-east-2", aws_profile="default", aws_await_snapshots_timeout=7200, + aws_snapshot_lock_mode="compliance", + aws_snapshot_lock_duration=1, + aws_snapshot_lock_cool_off_period=2, + aws_snapshot_lock_expiration_date=datetime.datetime(2024, 1, 1), ) + # WHEN get_snapshot_interface_from_server_config is called snapshot_interface = get_snapshot_interface_from_server_config(mock_config) # THEN the config values are passed to the snapshot interface assert isinstance(snapshot_interface, AwsCloudSnapshotInterface) assert snapshot_interface.region == "us-east-2" assert snapshot_interface.await_snapshots_timeout == 7200 + assert snapshot_interface.lock_mode == "compliance" + assert snapshot_interface.lock_duration == 1 + assert snapshot_interface.lock_cool_off_period == 2 + assert snapshot_interface.lock_expiration_date == datetime.datetime(2024, 1, 1) mock_boto3.Session.assert_called_once_with(profile_name="default") @pytest.mark.parametrize( @@ -374,6 +384,10 @@ def test_from_args_aws(self, mock_boto3): aws_region="us-east-2", aws_profile="default", aws_await_snapshots_timeout=7200, + aws_snapshot_lock_mode="compliance", + aws_snapshot_lock_duration=1, + aws_snapshot_lock_cool_off_period=2, + aws_snapshot_lock_expiration_date=datetime.datetime(2024, 1, 1), ) # WHEN get_snapshot_interface is called snapshot_interface = get_snapshot_interface(mock_config) @@ -381,6 +395,10 @@ def test_from_args_aws(self, mock_boto3): assert isinstance(snapshot_interface, AwsCloudSnapshotInterface) assert snapshot_interface.region == "us-east-2" assert snapshot_interface.await_snapshots_timeout == 7200 + assert snapshot_interface.lock_mode == "compliance" + assert snapshot_interface.lock_duration == 1 + assert snapshot_interface.lock_cool_off_period == 2 + assert snapshot_interface.lock_expiration_date == datetime.datetime(2024, 1, 1) mock_boto3.Session.assert_called_once_with(profile_name="default") @@ -2624,6 +2642,44 @@ def _get_mock_describe_volumes_resp(self, disks): ] } + def _get_mock_lock_snapshot_resp(self, snapshot_id, lock_mode, lock_cool_off_period, lock_duration): + """Helper which returns a mock lock_snapshot response.""" + lock_created_on = datetime.datetime(2024, 1, 1) + + return { + "SnapshotId": snapshot_id, + "LockState": lock_mode, + "LockDuration": lock_duration, + "CoolOffPeriod": lock_cool_off_period, + "CoolOffPeriodExpiresOn": (lock_created_on + datetime.timedelta(hours=lock_cool_off_period)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockCreatedOn": lock_created_on.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockExpiresOn": (lock_created_on + datetime.timedelta(days=lock_duration)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockDurationStartTime": (lock_created_on + datetime.timedelta(hours=lock_cool_off_period)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z' + } + + def _get_mock_describe_locked_snapshots_resp(self, snapshot_id, lock_mode, lock_cool_off_period, lock_duration, owner_id="123456789012", next_token=None): + """Helper which returns a mock describe_locked_snapshots response.""" + lock_created_on = datetime.datetime(2024, 1, 1) + + snapshot = { + "OwnerId": owner_id, + "SnapshotId": snapshot_id, + "LockState": lock_mode, + "LockDuration": lock_duration, + "CoolOffPeriod": lock_cool_off_period, + "CoolOffPeriodExpiresOn": (lock_created_on + datetime.timedelta(hours=lock_cool_off_period)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockCreatedOn": lock_created_on.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockDurationStartTime": (lock_created_on + datetime.timedelta(hours=lock_cool_off_period)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z', + "LockExpiresOn": (lock_created_on + datetime.timedelta(days=lock_duration)).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + 'Z' + } + + response = { + "Snapshots": [snapshot], + "NextToken": next_token if next_token else "" + } + + return response + def _get_snapshot_id(self, disk): """Helper which forges the expected snapshot id for the given disk id.""" return disk["id"].replace("vol", "snap") @@ -2786,7 +2842,7 @@ def test_take_snapshot_backup(self, number_of_disks, mock_ec2_client): mock_ec2_client.describe_instances.return_value = ( self._get_mock_describe_instances_resp(disks) ) - # AND the mock EC2 client returns successful create_snapashot responses + # AND the mock EC2 client returns successful create_snapshot responses mock_ec2_client.create_snapshot.side_effect = self._get_mock_create_snapshot( disks ) @@ -2824,6 +2880,68 @@ def test_take_snapshot_backup(self, number_of_disks, mock_ec2_client): assert snapshot.mount_options == disk["mount_options"] assert snapshot.mount_point == disk["mount_point"] + @pytest.mark.parametrize("number_of_disks", (1, 2, 3)) + def test_take_snapshot_backup_with_lock(self, number_of_disks, mock_ec2_client): + """ + Verify that take_snapshot_backup waits for completion of all snapshots and + updates the backup_info when complete. + """ + # GIVEN a set of disks, represented as VolumeMetadata + disks = self.aws_disks[:number_of_disks] + assert len(disks) == number_of_disks + volumes = self._get_mock_volumes(disks) + # AND a backup_info for a given server name and backup ID + backup_info = mock.Mock(backup_id=self.backup_id, server_name=self.server_name) + # AND a mock EC2 client which returns an instance with the required disks + # attached + mock_ec2_client.describe_instances.return_value = ( + self._get_mock_describe_instances_resp(disks) + ) + # AND the mock EC2 client returns successful create_snapshot responses + mock_ec2_client.create_snapshot.side_effect = self._get_mock_create_snapshot( + disks + ) + # AND a new AwsCloudSnapshotInterface + snapshot_interface = AwsCloudSnapshotInterface( + region=self.aws_region, + lock_mode="compliance", + lock_duration=1, + lock_cool_off_period=2, + lock_expiration_date=datetime.datetime(2024, 1, 1), + ) + + # WHEN take_snapshot_backup is called + snapshot_interface.take_snapshot_backup( + backup_info, self.aws_instance_id, volumes + ) + + # THEN we waited for completion of all snapshots + expected_snapshot_ids = [self._get_snapshot_id(disk) for disk in disks] + mock_ec2_client.get_waiter.return_value.wait.assert_called_once_with( + Filters=[{"Name": "snapshot-id", "Values": expected_snapshot_ids}], + WaiterConfig={"Delay": 15, "MaxAttempts": 240}, + ) + + # AND the backup_info is updated with the expected snapshot metadata + snapshots_info = backup_info.snapshots_info + assert snapshots_info.account_id == self.aws_account_id + assert snapshots_info.region == self.aws_region + assert snapshots_info.provider == "aws" + assert len(snapshots_info.snapshots) == len(disks) + for disk in disks: + snapshot_id = self._get_snapshot_id(disk) + snapshot = next( + snapshot + for snapshot in snapshots_info.snapshots + if snapshot.identifier == snapshot_id + ) + assert snapshot.identifier == snapshot_id + assert snapshot.snapshot_name == self._get_snapshot_name(disk) + assert snapshot.snapshot_lock_mode == "compliance" + assert snapshot.device_name == disk["device"] + assert snapshot.mount_options == disk["mount_options"] + assert snapshot.mount_point == disk["mount_point"] + def test_take_snapshot_backup_instance_not_found(self, mock_ec2_client): """ Verify that a SnapshotBackupException is raised if the instance cannot be @@ -2908,7 +3026,7 @@ def test_take_snapshot_backup_wait( mock_ec2_client.describe_instances.return_value = ( self._get_mock_describe_instances_resp(disks) ) - # AND the mock EC2 client returns successful create_snapashot responses + # AND the mock EC2 client returns successful create_snapshot responses mock_ec2_client.create_snapshot.side_effect = self._get_mock_create_snapshot( disks ) @@ -2930,6 +3048,73 @@ def test_take_snapshot_backup_wait( WaiterConfig=expected_wait_config, ) + def test_take_snapshot_with_lock( + self, mock_ec2_client + ): + """ + Verify that take_snapshot_backup locks each snapshot once snapshots are complete. + """ + # GIVEN a set of disks, represented as VolumeMetadata + number_of_disks = 2 + disks = self.aws_disks[:number_of_disks] + assert len(disks) == number_of_disks + volumes = self._get_mock_volumes(disks) + # AND a backup_info for a given server name and backup ID + backup_info = mock.Mock(backup_id=self.backup_id, server_name=self.server_name) + # AND a mock EC2 client which returns an instance with the required disks + # attached + mock_ec2_client.describe_instances.return_value = ( + self._get_mock_describe_instances_resp(disks) + ) + # AND the mock EC2 client returns successful create_snapshot responses + mock_ec2_client.create_snapshot.side_effect = self._get_mock_create_snapshot( + disks + ) + # AND the mock EC2 client returns lock_snapshot_responses for these disks + mock_ec2_client.lock_snapshot.side_effect = [ + self._get_mock_lock_snapshot_resp( + self._get_snapshot_id(disk), + lock_mode="compliance", + lock_duration=1, + lock_cool_off_period=1, + ) for disk in disks + ] + + # AND a new AwsCloudSnapshotInterface + kwargs = { + "region": self.aws_region, + "lock_mode": "compliance", + "lock_duration": 1, + "lock_cool_off_period": 1, + "lock_expiration_date": datetime.datetime(2024, 1, 1), + } + snapshot_interface = AwsCloudSnapshotInterface(**kwargs) + + # WHEN take_snapshot_backup is called + snapshot_interface.take_snapshot_backup( + backup_info, self.aws_instance_id, volumes + ) + + expected_snapshot_ids = [self._get_snapshot_id(disk) for disk in disks] + + # THEN _lock_mock_ec2_client.create_snapshot.side_effect is called + mock_ec2_client.lock_snapshot.assert_has_calls([ + mock.call( + SnapshotId=expected_snapshot_ids[0], + LockMode="compliance", + LockDuration=1, + CoolOffPeriod=1, + ExpirationDate=datetime.datetime(2024, 1, 1) + ), + mock.call( + SnapshotId=expected_snapshot_ids[1], + LockMode="compliance", + LockDuration=1, + CoolOffPeriod=1, + ExpirationDate=datetime.datetime(2024, 1, 1) + )] + ) + aws_live_states = ["pending", "running", "shutting-down", "stopping", "stopped"] def test_get_instance_metadata_by_id(self, mock_ec2_client): @@ -3439,8 +3624,11 @@ def test_delete_snapshot_failed(self, mock_ec2_client, caplog): "snapshots_list", ( [], - [mock.Mock(identifier="snap-0123")], - [mock.Mock(identifier="snap-0123"), mock.Mock(identifier="snap0124")], + [mock.Mock(identifier="snap-0123", snapshot_lock_mode=None)], + [ + mock.Mock(identifier="snap-0123", snapshot_lock_mode=None), + mock.Mock(identifier="snap0124", snapshot_lock_mode=None) + ], ), ) def test_delete_snapshot_backup(self, snapshots_list, mock_ec2_client, caplog): @@ -3466,6 +3654,85 @@ def test_delete_snapshot_backup(self, snapshots_list, mock_ec2_client, caplog): ] mock_ec2_client.delete_snapshot.assert_has_calls(expected_calls) + @pytest.mark.parametrize( + "snapshots_list", + ( + [ + mock.Mock(identifier="snap-0123", snapshot_lock_mode="compliance"), + mock.Mock(identifier="snap-0124", snapshot_lock_mode="compliance") + ], + ), + ) + def test_delete_snapshot_backup_with_locked_snapshots(self, snapshots_list, mock_ec2_client, caplog): + """Verify that snapshots for a backup are not deleted if lock has not expired yet.""" + # GIVEN a backup_info specifying zero or more snapshots + backup_info = mock.Mock( + backup_id=self.backup_id, + snapshots_info=mock.Mock(snapshots=snapshots_list), + ) + # AND log level is info + caplog.set_level(logging.INFO) + # AND the describe_locked_snapshots request is successful + mock_ec2_client.describe_locked_snapshots.side_effect = [ + self._get_mock_describe_locked_snapshots_resp( + snapshot_id=snapshot.identifier, + lock_mode="compliance", + lock_cool_off_period=1, + lock_duration=1 + ) for snapshot in snapshots_list + ] + # AND the snapshot delete requests are successful + mock_ec2_client.delete_snapshot.return_value = {} + # AND a new AwsCloudSnapshotInterface + snapshot_interface = AwsCloudSnapshotInterface(region=self.aws_region) + + # WHEN delete_snapshot_backup is called + snapshot_interface.delete_snapshot_backup(backup_info) + + # THEN delete_snapshot should not have been called for any of the snapshots + mock_ec2_client.delete_snapshot.assert_not_called() + + @pytest.mark.parametrize( + "snapshots_list", + ( + [ + mock.Mock(identifier="snap-0123", snapshot_lock_mode="compliance"), + mock.Mock(identifier="snap-0124", snapshot_lock_mode="compliance") + ], + ), + ) + def test_delete_snapshot_backup_with_expired_snapshots(self, snapshots_list, mock_ec2_client, caplog): + """Verify that snapshots for a backup are deleted if lock exists but has expired.""" + # GIVEN a backup_info specifying zero or more snapshots + backup_info = mock.Mock( + backup_id=self.backup_id, + snapshots_info=mock.Mock(snapshots=snapshots_list), + ) + # AND log level is info + caplog.set_level(logging.INFO) + # AND the describe_locked_snapshots request is successful + mock_ec2_client.describe_locked_snapshots.side_effect = [ + self._get_mock_describe_locked_snapshots_resp( + snapshot_id=snapshot.identifier, + lock_mode="expired", + lock_cool_off_period=1, + lock_duration=1 + ) for snapshot in snapshots_list + ] + # AND the snapshot delete requests are successful + mock_ec2_client.delete_snapshot.return_value = {} + # AND a new AwsCloudSnapshotInterface + snapshot_interface = AwsCloudSnapshotInterface(region=self.aws_region) + + # WHEN delete_snapshot_backup is called + snapshot_interface.delete_snapshot_backup(backup_info) + + # THEN delete_snapshot was called for each snapshot + expected_calls = [ + mock.call(SnapshotId=snapshot.identifier) for snapshot in snapshots_list + ] + mock_ec2_client.delete_snapshot.assert_has_calls(expected_calls) + class TestAwsVolumeMetadata(object): """Verify behaviour of AwsVolumeMetadata.""" diff --git a/tests/test_config.py b/tests/test_config.py index 23c065fb6..9702cacc6 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1336,6 +1336,10 @@ def test_to_json(self, model_config): "archiver_batch_size": None, "autogenerate_manifest": None, "aws_await_snapshots_timeout": None, + "aws_snapshot_lock_mode": None, + "aws_snapshot_lock_duration": None, + "aws_snapshot_lock_cool_off_period": None, + "aws_snapshot_lock_expiration_date": None, "aws_profile": None, "aws_region": None, "azure_credential": None, @@ -1425,6 +1429,10 @@ def test_to_json_with_config_source(self, model_config): "archiver_batch_size": {"source": "SOME_SOURCE", "value": None}, "autogenerate_manifest": {"source": "SOME_SOURCE", "value": None}, "aws_await_snapshots_timeout": {"source": "SOME_SOURCE", "value": None}, + "aws_snapshot_lock_mode": {"source": "SOME_SOURCE", "value": None}, + "aws_snapshot_lock_duration": {"source": "SOME_SOURCE", "value": None}, + "aws_snapshot_lock_cool_off_period": {"source": "SOME_SOURCE", "value": None}, + "aws_snapshot_lock_expiration_date": {"source": "SOME_SOURCE", "value": None}, "aws_profile": {"source": "SOME_SOURCE", "value": None}, "aws_region": {"source": "SOME_SOURCE", "value": None}, "azure_credential": {"source": "SOME_SOURCE", "value": None}, diff --git a/tests/test_infofile.py b/tests/test_infofile.py index 7b5954afa..c738d4297 100644 --- a/tests/test_infofile.py +++ b/tests/test_infofile.py @@ -868,6 +868,7 @@ def test_with_snapshots_info_aws(self, tmpdir): device_name="/dev/sdf", snapshot_name="user-assigned name", snapshot_id="snap-0123", + snapshot_lock_mode="compliance", ) ], ) @@ -884,6 +885,7 @@ def test_with_snapshots_info_aws(self, tmpdir): assert snapshot0.device_name == "/dev/sdf" assert snapshot0.snapshot_name == "user-assigned name" assert snapshot0.snapshot_id == "snap-0123" + assert snapshot0.snapshot_lock_mode == "compliance" # AND the snapshots_info is included in the JSON output snapshots_json = b_info.to_json()["snapshots_info"] @@ -895,6 +897,7 @@ def test_with_snapshots_info_aws(self, tmpdir): assert snapshot0_json["provider"]["device_name"] == "/dev/sdf" assert snapshot0_json["provider"]["snapshot_name"] == "user-assigned name" assert snapshot0_json["provider"]["snapshot_id"] == "snap-0123" + assert snapshot0_json["provider"]["snapshot_lock_mode"] == "compliance" def test_with_no_snapshots_info(self, tmpdir): """ diff --git a/tests/test_utils.py b/tests/test_utils.py index 24a94f7a4..3f5019819 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -19,6 +19,7 @@ import decimal import json import logging +import random import signal import sys import re @@ -1022,6 +1023,39 @@ def test_get_backup_info_from_name_no_match(self, mock_backup_info_list): assert backup_info is None +class TestDateTimeTimestampFormat(object): + @pytest.mark.parametrize("timestamp", ["2024-01-01T00:00:00.000Z"]) + def test_parse(self, timestamp): + assert isinstance(barman.utils.check_timestamp(timestamp), datetime) + + @pytest.mark.parametrize("timestamp", ["2024-01-01", "2024-01-01T00:00:00", "2024-01-01T00:00:00.000"]) + def test_parse_error(self, timestamp): + with pytest.raises(ArgumentTypeError): + barman.utils.check_timestamp(timestamp) + + +class TestAWSSnapshotLockDurationRange(object): + @pytest.mark.parametrize("duration", ["1", random.choice(range(1, 36500)), "36500"]) + def test_parse(self, duration): + assert barman.utils.check_aws_snapshot_lock_duration_range(duration) == int(duration) + + @pytest.mark.parametrize("duration", ["-1", "0", "36501"]) + def test_parse_error(self, duration): + with pytest.raises(ArgumentTypeError): + barman.utils.check_aws_snapshot_lock_duration_range(duration) + + +class TestAWSSnapshotCoolOffPeriodRange(object): + @pytest.mark.parametrize("duration", ["1", random.choice(range(2, 71)), "72"]) + def test_parse(self, duration): + assert barman.utils.check_aws_snapshot_lock_cool_off_period_range(duration) == int(duration) + + @pytest.mark.parametrize("duration", ["-1", "0", "36501"]) + def test_parse_error(self, duration): + with pytest.raises(ArgumentTypeError): + barman.utils.check_aws_snapshot_lock_cool_off_period_range(duration) + + class TestEditConfig: def test_edit_config_existing_section(self, tmpdir): # Create a temporary file diff --git a/tests/testing_helpers.py b/tests/testing_helpers.py index a97bf0214..b427a8512 100644 --- a/tests/testing_helpers.py +++ b/tests/testing_helpers.py @@ -283,6 +283,10 @@ def build_config_dictionary(config_keys=None): "archiver_batch_size": 0, "autogenerate_manifest": False, "aws_await_snapshots_timeout": 3600, + "aws_snapshot_lock_mode": None, + "aws_snapshot_lock_duration": None, + "aws_snapshot_lock_cool_off_period": None, + "aws_snapshot_lock_expiration_date": None, "aws_profile": None, "aws_region": None, "azure_credential": None,