Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Proposed fix for issue #5 #6

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Proposed fix for issue #5 #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8c35a87
Proposed fix for issue #5
nonspecialist Aug 22, 2017
ad27d82
Fixed issues and added new tests to cover changes.
stevemac007 Aug 23, 2017
a6ea17d
Added more logic to determine RDS cluster for listing and deleting sn…
stevemac007 Aug 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions development.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@ s3transfer==0.1.10
six==1.10.0
Werkzeug==0.11.15
xmltodict==0.10.2
mock==2.0.0
52 changes: 42 additions & 10 deletions lambda/backuplambda.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ def process_backup(self):

for i in range(delta):
self.message += ' Deleting snapshot ' + self.resolve_snapshot_name(deletelist[i]) + '\n'
self.delete_snapshot(deletelist[i])
self.delete_snapshot(resource=backup_item,
snapshot=deletelist[i])
total_deletes += 1
# time.sleep(3)
except Exception as ex:
Expand Down Expand Up @@ -156,7 +157,7 @@ def process_backup(self):
"total_deletes": total_deletes,
}

def delete_snapshot(self, snapshot):
def delete_snapshot(self, resource, snapshot):
pass


Expand Down Expand Up @@ -242,7 +243,7 @@ def resolve_snapshot_name(self, resource):
def resolve_snapshot_time(self, resource):
return resource['StartTime']

def delete_snapshot(self, snapshot):
def delete_snapshot(self, resource, snapshot):
self.conn.delete_snapshot(SnapshotId=snapshot["SnapshotId"])


Expand Down Expand Up @@ -324,27 +325,58 @@ def snapshot_resource(self, resource, description, tags):
date = datetime.today().strftime('%d-%m-%Y-%H-%M-%S')
snapshot_id = self.period+'-'+self.resolve_backupable_id(resource)+"-"+date+"-"+self.date_suffix

current_snap = self.conn.create_db_snapshot(DBInstanceIdentifier=self.resolve_backupable_id(resource),
DBSnapshotIdentifier=snapshot_id,
Tags=aws_tagset)
if self.is_cluster(resource):
current_snap = self.conn.create_db_cluster_snapshot(
DBClusterIdentifier=self.resolve_backupable_id(resource),
DBClusterSnapshotIdentifier=snapshot_id,
Tags=aws_tagset)
else:
current_snap = self.conn.create_db_snapshot(
DBInstanceIdentifier=self.resolve_backupable_id(resource),
DBSnapshotIdentifier=snapshot_id,
Tags=aws_tagset)

def is_cluster(self, resource):
return 'DBClusterIdentifier' in resource or 'DBClusterSnapshotIdentifier' in resource

def list_snapshots_for_resource(self, resource):
snapshots = self.conn.describe_db_snapshots(DBInstanceIdentifier=self.resolve_backupable_id(resource),
SnapshotType='manual')

if self.is_cluster(resource):
snapshots = self.conn.describe_db_cluster_snapshots(DBClusterIdentifier=self.resolve_backupable_id(resource),
SnapshotType='manual')
else:
snapshots = self.conn.describe_db_snapshots(DBInstanceIdentifier=self.resolve_backupable_id(resource),
SnapshotType='manual')
return snapshots['DBSnapshots']

def resolve_backupable_id(self, resource):

if self.is_cluster(resource):
return resource["DBClusterIdentifier"]

return resource["DBInstanceIdentifier"]

def resolve_snapshot_name(self, resource):

if self.is_cluster(resource):
return resource['DBClusterSnapshotIdentifier']

return resource['DBSnapshotIdentifier']

def resolve_snapshot_time(self, resource):
now = datetime.utcnow()
return resource.get('SnapshotCreateTime', now)

def delete_snapshot(self, snapshot):
self.conn.delete_db_snapshot(DBSnapshotIdentifier=snapshot["DBSnapshotIdentifier"])
def delete_snapshot(self, resource, snapshot):

if self.is_cluster(resource):
self.conn.delete_db_cluster_snapshot(
DBClusterSnapshotIdentifier=snapshot['DBClusterSnapshotIdentifier']
)
else:
self.conn.delete_db_snapshot(
DBSnapshotIdentifier=snapshot["DBSnapshotIdentifier"]
)

def db_has_tag(self, db_instance):
arn = self.build_arn(db_instance)
Expand Down
165 changes: 164 additions & 1 deletion tests/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import boto3
import json

from mock import MagicMock, call, ANY

from moto import mock_ec2, mock_sns
from backuplambda import *

Expand Down Expand Up @@ -41,13 +43,174 @@ def test_resolve_resource_bytag(self):
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count="2")
keep_count=2)

volumes = mgr.get_backable_resources()

assert len(volumes) == 1


class RDSBackupManagerTest(unittest.TestCase):

def test_process_backup_no_instances(self):

mgr = RDSBackupManager(rds_region_name="ap-southeast-1",
period="day",
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count=2)

mgr.conn = MagicMock()
mgr.conn.describe_db_instances = MagicMock(return_value={"DBInstances": []})

result = mgr.process_backup()

self.assertEqual(mgr.conn.mock_calls, [
call.describe_db_instances()
])

self.assertEqual({'total_errors': 0,
'total_creates': 0,
'total_deletes': 0,
'total_resources': 0},
result)

def test_process_backup_single_instance(self):

mgr = RDSBackupManager(rds_region_name="ap-southeast-1",
period="day",
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count=2)

instance = {'DBInstanceIdentifier': "db-id1234"}
list_tags = {"TagList": [{"Key": "Snapshot", "Value": "True"}]}

mock_sg = MagicMock()
mock_db_snapshots = {'DBSnapshots': []}

mgr.conn = MagicMock()
mgr.conn.describe_db_instances = MagicMock(return_value={"DBInstances": [instance]})
mgr.conn.describe_db_security_groups = MagicMock(return_value=mock_sg)
mgr.conn.meta.region_name = "blart-bling-1"
mgr.conn.list_tags_for_resource = MagicMock(return_value=list_tags)
mgr.conn.describe_db_snapshots = MagicMock(return_value=mock_db_snapshots)

result = mgr.process_backup()

self.assertEqual(mgr.conn.mock_calls, [
call.describe_db_instances(),
call.describe_db_security_groups(),
call.list_tags_for_resource(
ResourceName="arn:aws:rds:blart-bling-1:0:db:db-id1234"),
call.list_tags_for_resource(
ResourceName="arn:aws:rds:blart-bling-1:0:db:db-id1234"),
call.create_db_snapshot(DBInstanceIdentifier='db-id1234',
DBSnapshotIdentifier=ANY,
Tags=[{'Value': 'True', 'Key': 'Snapshot'}]),
call.describe_db_snapshots(DBInstanceIdentifier='db-id1234', SnapshotType='manual'),
])

self.assertEqual({'total_errors': 0,
'total_creates': 1,
'total_deletes': 0,
'total_resources': 1},
result)

def test_process_backup_single_cluster_instance(self):

mgr = RDSBackupManager(rds_region_name="ap-southeast-1",
period="day",
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count=2)

instance = {'DBInstanceIdentifier': "db-id1234", "DBClusterIdentifier": "cluster-5678"}
list_tags = {"TagList": [{"Key": "Snapshot", "Value": "True"}]}

mock_sg = MagicMock()
mock_db_snapshots = {'DBSnapshots': []}

mgr.conn = MagicMock()
mgr.conn.describe_db_instances = MagicMock(return_value={"DBInstances": [instance]})
mgr.conn.describe_db_security_groups = MagicMock(return_value=mock_sg)
mgr.conn.meta.region_name = "blart-bling-1"
mgr.conn.list_tags_for_resource = MagicMock(return_value=list_tags)
mgr.conn.describe_db_cluster_snapshots = MagicMock(return_value=mock_db_snapshots)

result = mgr.process_backup()

self.assertEqual(mgr.conn.mock_calls, [
call.describe_db_instances(),
call.describe_db_security_groups(),
call.list_tags_for_resource(
ResourceName="arn:aws:rds:blart-bling-1:0:db:db-id1234"),
call.list_tags_for_resource(
ResourceName="arn:aws:rds:blart-bling-1:0:db:db-id1234"),
call.create_db_cluster_snapshot(DBClusterIdentifier='cluster-5678',
DBClusterSnapshotIdentifier=ANY,
Tags=[{'Value': 'True', 'Key': 'Snapshot'}]),
call.describe_db_cluster_snapshots(DBClusterIdentifier='cluster-5678', SnapshotType='manual'),
])

self.assertEqual({'total_errors': 0,
'total_creates': 1,
'total_deletes': 0,
'total_resources': 1},
result)

def test_snapshot_resource_for_standard(self):

mgr = RDSBackupManager(rds_region_name="ap-southeast-1",
period="day",
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count="2")

mgr.conn = MagicMock()

resource = {"DBInstanceIdentifier": "db-1234"}

description = ""
tags = {"value1": "value2"}

mgr.snapshot_resource(resource, description, tags)

self.assertEqual(mgr.conn.mock_calls, [
call.create_db_snapshot(DBInstanceIdentifier='db-1234',
DBSnapshotIdentifier=ANY,
Tags=[{'Value': 'value2', 'Key': 'value1'}])
])

def test_snapshot_resource_for_cluster(self):

mgr = RDSBackupManager(rds_region_name="ap-southeast-1",
period="day",
tag_name="Snapshot",
tag_value="True",
date_suffix="dd",
keep_count="2")

mgr.conn = MagicMock()

resource = {"DBInstanceIdentifier": "db-1234", "DBClusterIdentifier": "cluster-5678"}

description = ""
tags = {"value1": "value2"}

mgr.snapshot_resource(resource, description, tags)

self.assertEqual(mgr.conn.mock_calls, [
call.create_db_cluster_snapshot(DBClusterIdentifier='cluster-5678',
DBClusterSnapshotIdentifier=ANY,
Tags=[{'Value': 'value2', 'Key': 'value1'}])
])


class LambdaHandlerTest(unittest.TestCase):

@mock_ec2
Expand Down