diff --git a/README.rst b/README.rst index c0ad43a..ea357ff 100644 --- a/README.rst +++ b/README.rst @@ -55,17 +55,21 @@ dcgc Remove old docker containers and docker images. ``dcgc`` will remove stopped containers and unused images that are older than -"max age". Running containers, and images which are used by a container are -never removed. +"max age" or images that have count of tags greater than a given threshold. +Running containers, and images which are used by a container are never removed. Maximum age can be specificied with any format supported by `pytimeparse `_. +If max count of tags is specified - newer tags are kept and older are removed. +If both max age and max count of tags is specified images are cleared if any of +the limits is exceeded. + Example: .. code:: sh - dcgc --max-container-age 3days --max-image-age 30days + dcgc --max-container-age 3days --max-image-age 30days --max-tags-count 5 Prevent images from being removed diff --git a/docker_custodian/docker_gc.py b/docker_custodian/docker_gc.py index 9049c92..dccf128 100644 --- a/docker_custodian/docker_gc.py +++ b/docker_custodian/docker_gc.py @@ -4,6 +4,7 @@ """ import argparse +import collections import fnmatch import logging import sys @@ -81,7 +82,8 @@ def get_dangling_volumes(client): return volumes -def cleanup_images(client, max_image_age, dry_run, exclude_set): +def cleanup_images(client, max_image_age, max_tags_count, + dry_run, exclude_set): # re-fetch container list so that we don't include removed containers image_tags_in_use = set( container['Image'] for container in get_all_containers(client)) @@ -89,8 +91,36 @@ def cleanup_images(client, max_image_age, dry_run, exclude_set): images = filter_images_in_use(get_all_images(client), image_tags_in_use) images = filter_excluded_images(images, exclude_set) - for image_summary in reversed(list(images)): - remove_image(client, image_summary, max_image_age, dry_run) + tags_counter = collections.Counter() + for image_summary in images: + exceeds_max_tags_count = check_exceeds_max_tags_count( + image_summary, + tags_counter, + max_tags_count) + remove_image( + client, + image_summary, + max_image_age, + exceeds_max_tags_count, + dry_run) + + +def check_exceeds_max_tags_count(image_summary, tags_counter, max_tags_count): + """Check if tags count is exceeded for all repositories. + + Updates tags_counter that keeps state for the current session. + + """ + image_tags = image_summary.get('RepoTags') + if max_tags_count is None or no_image_tags(image_tags): + return False + exceeds_max_tags_count = True + for tag in image_tags: + repo, _, _ = tag.rpartition(':') + tags_counter[repo] += 1 + if tags_counter[repo] <= max_tags_count: + exceeds_max_tags_count = False + return exceeds_max_tags_count def filter_excluded_images(images, exclude_set): @@ -111,7 +141,7 @@ def get_tag_set(image_summary): image_tags = image_summary.get('RepoTags') if no_image_tags(image_tags): # The repr of the image Id used by client.containers() - return set(['%s:latest' % image_summary['Id'][:12]]) + return {'%s:latest' % image_summary['Id'][:12]} return set(image_tags) def image_not_in_use(image_summary): @@ -121,16 +151,18 @@ def image_not_in_use(image_summary): def is_image_old(image, min_date): - return dateutil.parser.parse(image['Created']) < min_date + return min_date and dateutil.parser.parse(image['Created']) < min_date def no_image_tags(image_tags): return not image_tags or image_tags == [':'] -def remove_image(client, image_summary, min_date, dry_run): +def remove_image(client, image_summary, min_date, + exceeds_max_tags_count, dry_run): image = api_call(client.inspect_image, image=image_summary['Id']) - if not image or not is_image_old(image, min_date): + if not image or not ( + exceeds_max_tags_count or is_image_old(image, min_date)): return log.info("Removing image %s" % format_image(image, image_summary)) @@ -212,11 +244,16 @@ def main(): if args.max_container_age: cleanup_containers(client, args.max_container_age, args.dry_run) - if args.max_image_age: + if args.max_image_age or args.max_tags_count: exclude_set = build_exclude_set( args.exclude_image, args.exclude_image_file) - cleanup_images(client, args.max_image_age, args.dry_run, exclude_set) + cleanup_images( + client, + args.max_image_age, + args.max_tags_count, + args.dry_run, + exclude_set) if args.dangling_volumes: cleanup_volumes(client, args.dry_run) @@ -233,9 +270,14 @@ def get_args(args=None): parser.add_argument( '--max-image-age', type=timedelta_type, - help="Maxium age for an image. Images older than this age will be " + help="Maximum age for an image. Images older than this age will be " "removed. Age can be specified in any pytimeparse supported " "format.") + parser.add_argument( + '--max-tags-count', + type=int, + help="Maximum number of tags to keep for an image. " + "Only the most recently added tags are kept.") parser.add_argument( '--dangling-volumes', action="store_true", diff --git a/tests/docker_gc_test.py b/tests/docker_gc_test.py index a7dbfd9..20758cc 100644 --- a/tests/docker_gc_test.py +++ b/tests/docker_gc_test.py @@ -1,3 +1,6 @@ +import datetime + +from dateutil import tz from six import StringIO import textwrap @@ -7,6 +10,7 @@ except ImportError: import mock import requests.exceptions +import pytest from docker_custodian import docker_gc @@ -87,10 +91,60 @@ def test_cleanup_images(mock_client, now): ] mock_client.inspect_image.side_effect = iter(mock_images) - docker_gc.cleanup_images(mock_client, max_image_age, False, set()) + docker_gc.cleanup_images( + mock_client, max_image_age, None, False, set()) assert mock_client.remove_image.mock_calls == [ - mock.call(image=image['Id']) for image in reversed(images) + mock.call(image=image['Id']) for image in images + ] + + +@pytest.mark.parametrize('max_image_age,expected_remove_calls', [ + # First two cases image 'abce' cannot be removed because it has tag + # that points to repository 'user/two' for the first time. + ( + None, + [mock.call(image=i) for i in ['user/two:aaaa', 'user/one:bbbb']] + ), + ( + datetime.datetime(2014, 1, 1, 0, 0, tzinfo=tz.tzutc()), + [mock.call(image=i) for i in ['user/two:aaaa', 'user/one:bbbb']] + ), + # All images should be removed because max_image_age is greater than + # the age of all images. + ( + datetime.datetime(2014, 2, 1, 0, 0, tzinfo=tz.tzutc()), + [mock.call(image=i) for i in ['user/one:latest', 'user/one:abcd', + 'user/two:latest', 'user/one:efgh', + 'user/two:aaaa', 'user/one:bbbb']] + ), +]) +def test_cleanup_images_max_tags_count(mock_client, + max_image_age, expected_remove_calls): + mock_client.images.return_value = [ + {'Id': 'abcd', 'RepoTags': ['user/one:latest', 'user/one:abcd']}, + {'Id': 'abce', 'RepoTags': ['user/two:latest', 'user/one:efgh']}, + {'Id': 'abcf', 'RepoTags': ['user/two:aaaa', 'user/one:bbbb']}, + ] + mock_images = [ + { + 'Id': 'abcd', + 'Created': '2014-01-01T01:01:01Z' + }, + { + 'Id': 'abce', + 'Created': '2014-01-01T01:01:01Z' + }, + { + 'Id': 'abcf', + 'Created': '2014-01-01T01:01:01Z' + }, ] + mock_client.inspect_image.side_effect = iter(mock_images) + max_tags_count = 1 + docker_gc.cleanup_images( + mock_client, max_image_age, max_tags_count, False, set()) + # Keep at least max_tags_count tags for each repository. + assert mock_client.remove_image.mock_calls == expected_remove_calls def test_cleanup_volumes(mock_client): @@ -253,11 +307,15 @@ def test_is_image_old_false(image, later_time): assert not docker_gc.is_image_old(image, later_time) +def test_is_image_old_none(image, later_time): + assert not docker_gc.is_image_old(image, None) + + def test_remove_image_no_tags(mock_client, image, now): image_id = 'abcd' image_summary = {'Id': image_id} mock_client.inspect_image.return_value = image - docker_gc.remove_image(mock_client, image_summary, now, False) + docker_gc.remove_image(mock_client, image_summary, now, False, False) mock_client.remove_image.assert_called_once_with(image=image_id) @@ -266,7 +324,8 @@ def test_remove_image_new_image_not_removed(mock_client, image, later_time): image_id = 'abcd' image_summary = {'Id': image_id} mock_client.inspect_image.return_value = image - docker_gc.remove_image(mock_client, image_summary, later_time, False) + docker_gc.remove_image( + mock_client, image_summary, later_time, False, False) assert not mock_client.remove_image.mock_calls @@ -279,7 +338,22 @@ def test_remove_image_with_tags(mock_client, image, now): 'RepoTags': repo_tags } mock_client.inspect_image.return_value = image - docker_gc.remove_image(mock_client, image_summary, now, False) + docker_gc.remove_image(mock_client, image_summary, now, False, False) + + assert mock_client.remove_image.mock_calls == [ + mock.call(image=tag) for tag in repo_tags + ] + + +def test_remove_image_exceeds_max_tag_count(mock_client, image, now): + image_id = 'abcd' + repo_tags = ['user/one:latest', 'user/one:12345'] + image_summary = { + 'Id': image_id, + 'RepoTags': repo_tags + } + mock_client.inspect_image.return_value = image + docker_gc.remove_image(mock_client, image_summary, None, True, False) assert mock_client.remove_image.mock_calls == [ mock.call(image=tag) for tag in repo_tags