Add --max-tags-count parameter

README.rst

@@ -55,17 +55,21 @@ dcgc
 Remove old docker containers and docker images.
 
 ``dcgc`` will remove stopped containers and unused images that are older than
-"max age".  Running containers, and images which are used by a container are
-never removed.
+"max age" or images that have count of tags greater than a given threshold.
+Running containers, and images which are used by a container are never removed.
 
 Maximum age can be specificied with any format supported by
 `pytimeparse <https://github.com/wroberts/pytimeparse>`_.
 
+If max count of tags is specified - newer tags are kept and older are removed.
+If both max age and max count of tags is specified images are cleared if any of
+the limits is exceeded.
+
 Example:
 
 .. code:: sh
 
-    dcgc --max-container-age 3days --max-image-age 30days
+    dcgc --max-container-age 3days --max-image-age 30days --max-tags-count 5
 
 
 Prevent images from being removed

docker_custodian/docker_gc.py

@@ -4,6 +4,7 @@
 
 """
 import argparse
+import collections
 import fnmatch
 import logging
 import sys
@@ -81,16 +82,45 @@ def get_dangling_volumes(client):
     return volumes
 
 
-def cleanup_images(client, max_image_age, dry_run, exclude_set):
+def cleanup_images(client, max_image_age, max_tags_count,
+                   dry_run, exclude_set):
     # re-fetch container list so that we don't include removed containers
     image_tags_in_use = set(
         container['Image'] for container in get_all_containers(client))
 
     images = filter_images_in_use(get_all_images(client), image_tags_in_use)
     images = filter_excluded_images(images, exclude_set)
 
-    for image_summary in reversed(list(images)):
-        remove_image(client, image_summary, max_image_age, dry_run)
+    tags_counter = collections.Counter()
+    for image_summary in images:
+        exceeds_max_tags_count = check_exceeds_max_tags_count(
+            image_summary,
+            tags_counter,
+            max_tags_count)
+        remove_image(
+            client,
+            image_summary,
+            max_image_age,
+            exceeds_max_tags_count,
+            dry_run)
+
+
+def check_exceeds_max_tags_count(image_summary, tags_counter, max_tags_count):
+    """Check if tags count is exceeded for all repositories.
+
+    Updates tags_counter that keeps state for the current session.
+
+    """
+    image_tags = image_summary.get('RepoTags')
+    if max_tags_count is None or no_image_tags(image_tags):
+        return False
+    exceeds_max_tags_count = True
+    for tag in image_tags:
+        repo, _, _ = tag.rpartition(':')
+        tags_counter[repo] += 1
+        if tags_counter[repo] <= max_tags_count:
+            exceeds_max_tags_count = False
+    return exceeds_max_tags_count
 
 
 def filter_excluded_images(images, exclude_set):
@@ -111,7 +141,7 @@ def get_tag_set(image_summary):
         image_tags = image_summary.get('RepoTags')
         if no_image_tags(image_tags):
             # The repr of the image Id used by client.containers()
-            return set(['%s:latest' % image_summary['Id'][:12]])
+            return {'%s:latest' % image_summary['Id'][:12]}
         return set(image_tags)
 
     def image_not_in_use(image_summary):
@@ -121,16 +151,18 @@ def image_not_in_use(image_summary):
 
 
 def is_image_old(image, min_date):
-    return dateutil.parser.parse(image['Created']) < min_date
+    return min_date and dateutil.parser.parse(image['Created']) < min_date
 
 
 def no_image_tags(image_tags):
     return not image_tags or image_tags == ['<none>:<none>']
 
 
-def remove_image(client, image_summary, min_date, dry_run):
+def remove_image(client, image_summary, min_date,
+                 exceeds_max_tags_count, dry_run):
     image = api_call(client.inspect_image, image=image_summary['Id'])
-    if not image or not is_image_old(image, min_date):
+    if not image or not (
+            exceeds_max_tags_count or is_image_old(image, min_date)):
         return
 
     log.info("Removing image %s" % format_image(image, image_summary))
@@ -212,11 +244,16 @@ def main():
     if args.max_container_age:
         cleanup_containers(client, args.max_container_age, args.dry_run)
 
-    if args.max_image_age:
+    if args.max_image_age or args.max_tags_count:
         exclude_set = build_exclude_set(
             args.exclude_image,
             args.exclude_image_file)
-        cleanup_images(client, args.max_image_age, args.dry_run, exclude_set)
+        cleanup_images(
+            client,
+            args.max_image_age,
+            args.max_tags_count,
+            args.dry_run,
+            exclude_set)
 
     if args.dangling_volumes:
         cleanup_volumes(client, args.dry_run)
@@ -233,9 +270,14 @@ def get_args(args=None):
     parser.add_argument(
         '--max-image-age',
         type=timedelta_type,
-        help="Maxium age for an image. Images older than this age will be "
+        help="Maximum age for an image. Images older than this age will be "
              "removed. Age can be specified in any pytimeparse supported "
              "format.")
+    parser.add_argument(
+        '--max-tags-count',
+        type=int,
+        help="Maximum number of tags to keep for an image. "
+             "Only the most recently added tags are kept.")
     parser.add_argument(
         '--dangling-volumes',
         action="store_true",

tests/docker_gc_test.py

@@ -1,3 +1,6 @@
+import datetime
+
+from dateutil import tz
 from six import StringIO
 import textwrap
 
@@ -7,6 +10,7 @@
 except ImportError:
     import mock
 import requests.exceptions
+import pytest
 
 from docker_custodian import docker_gc
 
@@ -87,10 +91,60 @@ def test_cleanup_images(mock_client, now):
     ]
     mock_client.inspect_image.side_effect = iter(mock_images)
 
-    docker_gc.cleanup_images(mock_client, max_image_age, False, set())
+    docker_gc.cleanup_images(
+        mock_client, max_image_age, None, False, set())
     assert mock_client.remove_image.mock_calls == [
-        mock.call(image=image['Id']) for image in reversed(images)
+        mock.call(image=image['Id']) for image in images
+    ]
+
+
+@pytest.mark.parametrize('max_image_age,expected_remove_calls', [
+    # First two cases image 'abce' cannot be removed because it has tag
+    # that points to repository 'user/two' for the first time.
+    (
+        None,
+        [mock.call(image=i) for i in ['user/two:aaaa', 'user/one:bbbb']]
+    ),
+    (
+        datetime.datetime(2014, 1, 1, 0, 0, tzinfo=tz.tzutc()),
+        [mock.call(image=i) for i in ['user/two:aaaa', 'user/one:bbbb']]
+    ),
+    # All images should be removed because max_image_age is greater than
+    # the age of all images.
+    (
+        datetime.datetime(2014, 2, 1, 0, 0, tzinfo=tz.tzutc()),
+        [mock.call(image=i) for i in ['user/one:latest', 'user/one:abcd',
+                                      'user/two:latest', 'user/one:efgh',
+                                      'user/two:aaaa', 'user/one:bbbb']]
+    ),
+])
+def test_cleanup_images_max_tags_count(mock_client,
+                                       max_image_age, expected_remove_calls):
+    mock_client.images.return_value = [
+        {'Id': 'abcd', 'RepoTags': ['user/one:latest', 'user/one:abcd']},
+        {'Id': 'abce', 'RepoTags': ['user/two:latest', 'user/one:efgh']},
+        {'Id': 'abcf', 'RepoTags': ['user/two:aaaa', 'user/one:bbbb']},
+    ]
+    mock_images = [
+        {
+            'Id': 'abcd',
+            'Created': '2014-01-01T01:01:01Z'
+        },
+        {
+            'Id': 'abce',
+            'Created': '2014-01-01T01:01:01Z'
+        },
+        {
+            'Id': 'abcf',
+            'Created': '2014-01-01T01:01:01Z'
+        },
     ]
+    mock_client.inspect_image.side_effect = iter(mock_images)
+    max_tags_count = 1
+    docker_gc.cleanup_images(
+        mock_client, max_image_age, max_tags_count, False, set())
+    # Keep at least max_tags_count tags for each repository.
+    assert mock_client.remove_image.mock_calls == expected_remove_calls
 
 
 def test_cleanup_volumes(mock_client):
@@ -253,11 +307,15 @@ def test_is_image_old_false(image, later_time):
     assert not docker_gc.is_image_old(image, later_time)
 
 
+def test_is_image_old_none(image, later_time):
+    assert not docker_gc.is_image_old(image, None)
+
+
 def test_remove_image_no_tags(mock_client, image, now):
     image_id = 'abcd'
     image_summary = {'Id': image_id}
     mock_client.inspect_image.return_value = image
-    docker_gc.remove_image(mock_client, image_summary, now, False)
+    docker_gc.remove_image(mock_client, image_summary, now, False, False)
 
     mock_client.remove_image.assert_called_once_with(image=image_id)
 
@@ -266,7 +324,8 @@ def test_remove_image_new_image_not_removed(mock_client, image, later_time):
     image_id = 'abcd'
     image_summary = {'Id': image_id}
     mock_client.inspect_image.return_value = image
-    docker_gc.remove_image(mock_client, image_summary, later_time, False)
+    docker_gc.remove_image(
+        mock_client, image_summary, later_time, False, False)
 
     assert not mock_client.remove_image.mock_calls
 
@@ -279,7 +338,22 @@ def test_remove_image_with_tags(mock_client, image, now):
             'RepoTags': repo_tags
     }
     mock_client.inspect_image.return_value = image
-    docker_gc.remove_image(mock_client, image_summary, now, False)
+    docker_gc.remove_image(mock_client, image_summary, now, False, False)
+
+    assert mock_client.remove_image.mock_calls == [
+        mock.call(image=tag) for tag in repo_tags
+    ]
+
+
+def test_remove_image_exceeds_max_tag_count(mock_client, image, now):
+    image_id = 'abcd'
+    repo_tags = ['user/one:latest', 'user/one:12345']
+    image_summary = {
+        'Id': image_id,
+        'RepoTags': repo_tags
+    }
+    mock_client.inspect_image.return_value = image
+    docker_gc.remove_image(mock_client, image_summary, None, True, False)
 
     assert mock_client.remove_image.mock_calls == [
         mock.call(image=tag) for tag in repo_tags