imapd.c: Initial implementation of IMAP UIDBATCHES

cassandane/Cassandane/Cyrus/UIDbatches.pm

@@ -0,0 +1,174 @@
+#!/usr/bin/perl
+#
+#  Copyright (c) 2011-2017 FastMail Pty Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+#  3. The name "Fastmail Pty Ltd" must not be used to
+#     endorse or promote products derived from this software without
+#     prior written permission. For permission or any legal
+#     details, please contact
+#      FastMail Pty Ltd
+#      PO Box 234
+#      Collins St West 8007
+#      Victoria
+#      Australia
+#
+#  4. Redistributions of any form whatsoever must retain the following
+#     acknowledgment:
+#     "This product includes software developed by Fastmail Pty. Ltd."
+#
+#  FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO
+#  THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+#  AND FITNESS, IN NO EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE
+#  FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+#  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+#  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+#  OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+package Cassandane::Cyrus::UIDbatches;
+use strict;
+use warnings;
+use Cwd qw(abs_path);
+use DateTime;
+use Data::Dumper;
+
+use lib '.';
+use base qw(Cassandane::Cyrus::TestCase);
+use Cassandane::Util::Log;
+
+sub new
+{
+    my $class = shift;
+    my $config = Cassandane::Config->default()->clone();
+    $config->set(conversations => 'on');
+    return $class->SUPER::new({adminstore => 1, config => $config}, @_);
+}
+
+sub set_up
+{
+    my ($self) = @_;
+    $self->SUPER::set_up();
+}
+
+sub tear_down
+{
+    my ($self) = @_;
+    $self->SUPER::tear_down();
+}
+
+sub test_uidbatches
+{
+    my ($self) = @_;
+
+    my $imaptalk = $self->{store}->get_client();
+
+    my @results;
+    my %handlers =
+    (
+        uidbatches => sub
+        {
+            my (undef, $uidbatches) = @_;
+            push(@results, $uidbatches);
+        },
+    );
+
+    $imaptalk->examine('INBOX');
+
+    # batch size of 10 on empty mailbox
+    @results = ();
+    my $tag = $imaptalk->{CmdId};
+    my $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('TAG', $results[0][0][0]);
+    $self->assert_str_equals($tag, $results[0][0][1]);
+    $self->assert_str_equals('UID', $results[0][1]);
+    $self->assert_null($results[0][2]);
+
+    xlog $self, "append some messages";
+    my %exp;
+    my $N = 50;
+    for (1..$N)
+    {
+        my $msg = $self->make_message("Message $_");
+        $exp{$_} = $msg;
+    }
+    xlog $self, "check the messages got there";
+    $self->check_messages(\%exp);
+
+    # expunge some messages
+    $imaptalk->store('1,14:17,41,43,45', '+FLAGS', '(\\Deleted)');
+    $imaptalk->expunge();
+
+    # for manual debugging
+    #$imaptalk->fetch('1:*', 'UID');
+
+    # batch size of 50 with less than 50 messages
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '50');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_null($results[0][2]);
+
+    # batch size of 25
+    @results = ();
+    $tag = $imaptalk->{CmdId};
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '25');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('TAG', $results[0][0][0]);
+    $self->assert_str_equals($tag, $results[0][0][1]);
+    $self->assert_str_equals('UID', $results[0][1]);
+    $self->assert_str_equals('ALL', $results[0][2]);
+    $self->assert_str_equals('22', $results[0][3]);
+    $self->assert_null($results[0][4]);
+
+    # batch size of 10
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('37', $results[0][3]);
+    $self->assert_str_equals('27', $results[0][4]);
+    $self->assert_str_equals('13', $results[0][5]);
+    $self->assert_str_equals('3', $results[0][6]);
+    $self->assert_null($results[0][7]);
+
+    # batch size of 10, first batch only
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10', '1:1');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('37', $results[0][3]);
+    $self->assert_null($results[0][4]);
+
+    # batch size of 10, second & third batches
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10', '2:3');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('27', $results[0][3]);
+    $self->assert_str_equals('13', $results[0][4]);
+    $self->assert_null($results[0][5]);
+
+    # batch size of 10, fourth & fifth batches
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10', '4:5');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_str_equals('3', $results[0][3]);
+    $self->assert_null($results[0][4]);
+
+    # batch size of 10, fifth & sixth batches
+    @results = ();
+    $res = $imaptalk->_imap_cmd('UIDBATCHES', 0, \%handlers, '10', '5:6');
+    $self->assert_str_equals('ok', $imaptalk->get_last_completion_response());
+    $self->assert_null($results[0][2]);
+}
+
+1;

changes/next/imap-uidbatches

@@ -0,0 +1,19 @@
+
+Description:
+
+Adds support for IMAP UIDBATCHES (draft-eggert-uidbatches).
+
+
+Config changes:
+
+None.
+
+
+Upgrade instructions:
+
+None.
+
+
+GitHub issue:
+
+None.

docsrc/imap/rfc-support.rst

@@ -947,6 +947,10 @@ draft-ietf-jmap-sharing
 
     JMAP Sharing
 
+draft-eggert-uidbatches
+
+   IMAP UIDBATCHES Extension
+
 draft-murchison-lmtp-ignorequota
 
     LMTP Service Extension for Ignoring Recipient Quotas

imap/imapd.c

@@ -472,6 +472,7 @@ static struct capa_struct base_capabilities[] = {
     { "STATUS=SIZE",           CAPA_POSTAUTH,           { 0 } }, /* RFC 8438 */
     { "THREAD=ORDEREDSUBJECT", CAPA_POSTAUTH,           { 0 } }, /* RFC 5256 */
     { "THREAD=REFERENCES",     CAPA_POSTAUTH,           { 0 } }, /* RFC 5256 */
+    { "UIDBATCHES",            CAPA_POSTAUTH,           { 0 } }, /* draft-eggert-uidbatches */
     { "UIDONLY",               CAPA_POSTAUTH,           { 0 } }, /* draft-ietf-extra-imap-uidonly */
     { "UIDPLUS",               CAPA_POSTAUTH,           { 0 } }, /* RFC 4315 */
     { "UNAUTHENTICATE",        CAPA_POSTAUTH|CAPA_STATE,         /* RFC 8437 */
@@ -610,6 +611,8 @@ static void cmd_notify(char *tag, int set);
 static void push_updates(int idling);
 
 static void cmd_getjmapaccess(char* tag);
+static void cmd_uidbatches(char *tag, uint32_t size,
+                           uint32_t low, uint32_t high);
 
 static int parsecreateargs(struct dlist **extargs);
 
@@ -2545,6 +2548,26 @@ static void cmdloop(void)
                     eatline(imapd_in, c);
                 }
             }
+            else if (!strcmp(cmd.s, "Uidbatches")) {
+                uint32_t size, low = 1, high = UINT32_MAX;
+
+                if (!imapd_index && !backend_current) goto nomailbox;
+
+                if (c != ' ') goto missingargs;
+                c = getuint32(imapd_in, &size);
+                if (c <= EOF) goto missingargs;
+                if (c == ' ' ) {
+                    c = getuint32(imapd_in, &low);
+                    if (c != ':') goto badsequence;
+                    c = getuint32(imapd_in, &high);
+                    if (c <= EOF) goto badsequence;
+                }
+                if (!IS_EOL(c, imapd_in)) goto extraargs;
+
+                cmd_uidbatches(tag.s, size, low, high);
+
+                prometheus_increment(CYRUS_IMAP_UIDBATCHES_TOTAL);
+            }
             else if (!strcmp(cmd.s, "Unauthenticate")) {
                 if (!imapd_userisadmin) goto adminsonly;
 
@@ -15731,3 +15754,33 @@ static void cmd_getjmapaccess(char *tag)
     prot_printf(imapd_out, "%s OK %s\r\n", tag,
                 "This server is also accessible via JMAP, see RFC8620");
 }
+
+static void cmd_uidbatches(char *tag, uint32_t size, uint32_t low, uint32_t high)
+{
+    if (backend_current) {
+        /* remote mailbox */
+        prot_printf(backend_current->out, "%s UIDBATCHES %u", tag, size);
+        if (low > 1 || high < UINT32_MAX) {
+            prot_printf(backend_current->out, " %u:%u", low, high);
+        }
+        prot_puts(backend_current->out, "\r\n");
+        pipe_including_tag(backend_current, tag, 0);
+        return;
+    }
+
+    prot_printf(imapd_out, "* UIDBATCHES (TAG \"%s\") UID", tag);
+    if (low <= imapd_index->exists / size) {
+        int64_t msgno = imapd_index->exists - low * size;
+        uint32_t batch = low;
+
+        prot_puts(imapd_out, " ALL");
+
+        for (; msgno > 0 && batch <= high; msgno -= size, batch++) {
+
+            prot_printf(imapd_out, " %u", index_getuid(imapd_index, msgno));
+        }
+    }
+    prot_puts(imapd_out, "\r\n");
+    prot_printf(imapd_out, "%s OK %s\r\n",
+                tag, error_message(IMAP_OK_COMPLETED));
+}

imap/promdata.p

@@ -68,6 +68,7 @@ metric counter cyrus_imap_sort_total                    The total number of IMAP
 metric counter cyrus_imap_status_total                  The total number of IMAP STATUSs
 metric counter cyrus_imap_scan_total                    The total number of IMAP SCANs
 metric counter cyrus_imap_thread_total                  The total number of IMAP THREADs
+metric counter cyrus_imap_uidbatches_total              The total number of IMAP UIDBATCHES
 metric counter cyrus_imap_unauthenticate_total          The total number of IMAP UNAUTHENTICATEs
 metric counter cyrus_imap_unsubscribe_total             The total number of IMAP UNSUBSCRIBEs
 metric counter cyrus_imap_unselect_total                The total number of IMAP UNSELECTs