WIP: 64bit nanosecond jmapids

cassandane/Cassandane/Cyrus/Append.pm

@@ -0,0 +1,88 @@
+#!/usr/bin/perl
+#
+#  Copyright (c) 2011-2023 FastMail Pty Ltd. All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#  2. Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#
+#  3. The name "Fastmail Pty Ltd" must not be used to
+#     endorse or promote products derived from this software without
+#     prior written permission. For permission or any legal
+#     details, please contact
+#      FastMail Pty Ltd
+#      PO Box 234
+#      Collins St West 8007
+#      Victoria
+#      Australia
+#
+#  4. Redistributions of any form whatsoever must retain the following
+#     acknowledgment:
+#     "This product includes software developed by Fastmail Pty. Ltd."
+#
+#  FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+#  INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY  AND FITNESS, IN NO
+#  EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE FOR ANY SPECIAL, INDIRECT
+#  OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+#  USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+#  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+#  OF THIS SOFTWARE.
+#
+
+package Cassandane::Cyrus::Append;
+use strict;
+use warnings;
+use DateTime;
+use JSON;
+use JSON::XS;
+use Mail::JMAPTalk 0.13;
+use Data::Dumper;
+use Storable 'dclone';
+use File::Basename;
+use IO::File;
+
+use lib '.';
+use base qw(Cassandane::Cyrus::TestCase);
+use Cassandane::Util::Log;
+
+use charnames ':full';
+
+sub new
+{
+    my ($class, @args) = @_;
+
+    my $config = Cassandane::Config->default()->clone();
+
+    $config->set(conversations => 'yes');
+
+    my $self = $class->SUPER::new({
+        config => $config,
+        services => [ 'imap' ]
+    }, @args);
+
+    return $self;
+}
+
+sub set_up
+{
+    my ($self) = @_;
+    $self->SUPER::set_up();
+}
+
+sub tear_down
+{
+    my ($self) = @_;
+    $self->SUPER::tear_down();
+}
+
+use Cassandane::Tiny::Loader 'tiny-tests/Append';
+
+1;

cassandane/Cassandane/Cyrus/Reconstruct.pm

@@ -617,7 +617,7 @@ sub test_downgrade_upgrade
     $msg{A}->set_attribute(flags => ['\\Seen']);
     $self->check_messages(\%msg);
 
-    for my $version (12, 14, 16, 'max') {
+    for my $version (12, 14, 16, 19, 'max') {
         xlog $self, "Set to version $version";
         $self->{instance}->run_command({ cyrus => 1 }, 'reconstruct', '-V', $version);
 

cassandane/tiny-tests/Append/append_duplicate_diff_internaldate

@@ -0,0 +1,43 @@
+#!perl
+use Cassandane::Tiny;
+
+sub test_append_duplicate_diff_internaldate
+{
+    my ($self) = @_;
+
+    my $imaptalk = $self->{store}->get_client();
+
+    xlog $self, "Append duplicate messages with different internaldates";
+    my $mimeMessage = <<'EOF';
+From: <from@local>
+To: to@local
+Subject: test
+Date: Mon, 13 Apr 2020 15:34:03 +0200
+MIME-Version: 1.0
+Content-Type: text/plain
+
+test
+EOF
+    $mimeMessage =~ s/\r?\n/\r\n/gs;
+
+
+    $imaptalk->append('INBOX', $mimeMessage) || die $@;
+    $imaptalk->append('INBOX',
+                      '17-Aug-2023 15:13:54 +0200', $mimeMessage) || die $@;
+
+    $imaptalk->create('foo');
+    $imaptalk->append('foo',
+                      '26-Jan-2025 15:13:54 -0500', $mimeMessage) || die $@;
+
+    xlog $self, "Verify that all messages have the same internaldates";
+    $imaptalk->examine('INBOX');
+    my $res = $imaptalk->fetch('1:*', 'INTERNALDATE');
+    my $internaldate = $res->{1}->{internaldate};
+    $self->assert_str_equals($internaldate, $res->{2}->{internaldate});
+
+    $imaptalk->examine('foo');
+    $res = $imaptalk->fetch('1:*', 'INTERNALDATE');
+    $self->assert_str_equals($internaldate, $res->{1}->{internaldate});
+}
+
+1;

cassandane/tiny-tests/Append/append_same_internaldate

@@ -0,0 +1,36 @@
+#!perl
+use Cassandane::Tiny;
+
+# This test is used to verify that our JMAPID conflict detection/resolution works
+# but it takes a very long time to run and may not even generate a conflict
+sub bogus_test_append_same_internaldate
+{
+    my ($self) = @_;
+
+    my $imaptalk = $self->{store}->get_client();
+
+    xlog $self, "Append messages with same internaldates";
+    for (1..100000) {
+        my $mimeMessage = << "EOF";
+From: <from\@local>
+To: to\@local
+Subject: test $_
+
+test
+EOF
+        $mimeMessage =~ s/\r?\n/\r\n/gs;
+
+        $imaptalk->append('INBOX',
+                          '26-Jan-2025 15:13:54 -0500', $mimeMessage) || die $@;
+    }
+
+    xlog $self, "Verify that all messages have the same internaldates";
+    $imaptalk->examine('INBOX');
+    my $res = $imaptalk->fetch('1:*', 'INTERNALDATE');
+
+    # Did we log an conflicts?
+    $self->assert_syslog_matches($self->{instance},
+                                 qr/IOERROR: JMAPID conflict during append/);
+}
+
+1;

cassandane/tiny-tests/JMAPEmail/email_set_guidsearch_updated_internaldate

@@ -116,6 +116,9 @@ sub test_email_set_guidsearch_updated_internaldate
     xlog "Append blob of message A via IMAP";
     $imap->append('INBOX', $emailBlobA) || die $@;
 
+    $res = $imap->fetch('3', "(emailid)");
+    my $emailIdC = $res->{3}{emailid}[0];
+
     xlog $self, "run incremental squatter";
     $self->{instance}->run_command({cyrus => 1}, 'squatter', '-i');
 
@@ -132,5 +135,5 @@ sub test_email_set_guidsearch_updated_internaldate
         }, 'R1'],
     ], $using);
     $self->assert_equals(JSON::true, $res->[0][1]{performance}{details}{isGuidSearch});
-    $self->assert_deep_equals([$emailIdB, $emailIdA], $res->[0][1]{ids});
+    $self->assert_deep_equals([$emailIdB, $emailIdC], $res->[0][1]{ids});
 }

cunit/annotate.testc

@@ -1146,7 +1146,6 @@ static void test_msg_copy(void)
     buf_printf(&path2, "%s/data/uuid/%c/%c/%s/cyrus.annotations", DBDIR,
                u2[0], u2[1], u2);
     CU_ASSERT_EQUAL(fexists(buf_cstring(&path1)), 0);
-    CU_ASSERT_EQUAL(fexists(buf_cstring(&path2)), 0);
     mailbox_close(&mailbox1);
 
     /* copy MBOXNAME1,1 -> MBOXNAME2,1 */
@@ -2002,12 +2001,14 @@ static int create_messages(struct mailbox *mailbox, int count)
         struct appendstate as;
         quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_DONTCARE_INITIALIZER;
         FILE *fp;
-        time_t internaldate = time(NULL);
+        struct timespec internaldate;
         struct body *body = NULL;
         struct buf buf = BUF_INITIALIZER;
 
+        clock_gettime(CLOCK_REALTIME, &internaldate);
+
         /* Write the message to the filesystem */
-        if (!(fp = append_newstage(mailbox_name(mailbox), internaldate, 0, &stage))) {
+        if (!(fp = append_newstage(mailbox_name(mailbox), internaldate.tv_sec, 0, &stage))) {
             fprintf(stderr, "append_newstage(%s) failed", mailbox_name(mailbox));
             return IMAP_IOERROR;
         }
@@ -2028,7 +2029,7 @@ static int create_messages(struct mailbox *mailbox, int count)
                     strerror(errno));
             return r;
         }
-        r = append_fromstage(&as, &body, stage, internaldate, 0, NULL, 0, NULL);
+        r = append_fromstage(&as, &body, stage, &internaldate, 0, NULL, 0, NULL);
         if (r) {
             fprintf(stderr, "append_fromstage(%s) failed: %s", mailbox_name(mailbox),
                     error_message(r));

cunit/charset.testc

@@ -1332,6 +1332,14 @@ static void test_charset_decode(void)
     TESTCASE("vu_A3g", 6, "\xbe\xef\xc0\xde", 4, ENCODING_BASE64URL);
     TESTCASE("vu_A3g==", 6, "\xbe\xef\xc0\xde", 4, ENCODING_BASE64URL);
 
+    /* Base64 JMAPID encoding */
+    TESTCASE("-----------", 11,
+             "\x00\x00\x00\x00\x00\x00\x00\x00", 8, ENCODING_BASE64JMAPID);
+    TESTCASE("zzzzzzzzzzw", 11,
+             "\xff\xff\xff\xff\xff\xff\xff\xff", 8, ENCODING_BASE64JMAPID);
+    TESTCASE("012-abc_TUV", 11,
+             "\x04\x20\xc0\x9a\x7a\x25\x79\xf8", 8, ENCODING_BASE64JMAPID);
+
 #undef TESTCASE
 
     struct buf buf = BUF_INITIALIZER;
@@ -1354,6 +1362,11 @@ static void test_charset_decode(void)
     CU_ASSERT_EQUAL(-1, r);
     buf_reset(&buf);
 
+    /* Base64jmapid with invalid characters */
+    r = charset_decode(&buf, "Zm9v@@@YmFy", 11, ENCODING_BASE64JMAPID);
+    CU_ASSERT_EQUAL(-1, r);
+    buf_reset(&buf);
+
     buf_free(&buf);
 }
 

cunit/mailbox.testc

@@ -119,22 +119,23 @@ static void test_aligned_record_offsets(void)
      * not need to care about that.  Instead, keep this list sorted
      * alphabetically by the OFFSET_... name, for ease of maintenance.
      */
+    CU_ASSERT_EQUAL(0, OFFSET_BASECID       % alignof(r.basecid));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_CRC     % alignof(r.cache_crc));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_OFFSET  % alignof(XXX_CACHE32_TYPE));
     CU_ASSERT_EQUAL(0, OFFSET_CACHE_VERSION % alignof(XXX_CACHE32_TYPE));
+    CU_ASSERT_EQUAL(0, OFFSET_CID           % alignof(r.cid));
     CU_ASSERT_EQUAL(0, OFFSET_CREATEDMODSEQ % alignof(r.createdmodseq));
-    CU_ASSERT_EQUAL(0, OFFSET_GMTIME        % alignof(XXX_TIME32_TYPE));
+    CU_ASSERT_EQUAL(0, OFFSET_GMTIME        % alignof(r.gmtime));
     CU_ASSERT_EQUAL(0, OFFSET_HEADER_SIZE   % alignof(r.header_size));
-    CU_ASSERT_EQUAL(0, OFFSET_INTERNALDATE  % alignof(XXX_TIME32_TYPE));
-    CU_ASSERT_EQUAL(0, OFFSET_LAST_UPDATED  % alignof(XXX_TIME32_TYPE));
+    CU_ASSERT_EQUAL(0, OFFSET_INTERNALDATE  % alignof(r.internaldate));
+    CU_ASSERT_EQUAL(0, OFFSET_LAST_UPDATED  % alignof(r.last_updated));
     CU_ASSERT_EQUAL(0, OFFSET_MESSAGE_GUID  % alignof(char)); /* r/w uses memcpy */
     CU_ASSERT_EQUAL(0, OFFSET_MODSEQ        % alignof(r.modseq));
     CU_ASSERT_EQUAL(0, OFFSET_RECORD_CRC    % alignof(uint32_t)); /* not in struct */
-    CU_ASSERT_EQUAL(0, OFFSET_SAVEDATE      % alignof(XXX_TIME32_TYPE));
-    CU_ASSERT_EQUAL(0, OFFSET_SENTDATE      % alignof(XXX_TIME32_TYPE));
+    CU_ASSERT_EQUAL(0, OFFSET_SAVEDATE      % alignof(r.savedate));
+    CU_ASSERT_EQUAL(0, OFFSET_SENTDATE      % alignof(r.sentdate));
     CU_ASSERT_EQUAL(0, OFFSET_SIZE          % alignof(r.size));
     CU_ASSERT_EQUAL(0, OFFSET_SYSTEM_FLAGS  % alignof(r.system_flags));
-    CU_ASSERT_EQUAL(0, OFFSET_THRID         % alignof(r.cid));
     CU_ASSERT_EQUAL(0, OFFSET_UID           % alignof(r.uid));
     CU_ASSERT_EQUAL(0, OFFSET_USER_FLAGS    % alignof(r.user_flags));
     /* this list is sorted alphabetically, don't just append */
@@ -243,22 +244,23 @@ static void test_unique_record_offsets(void)
         /* Keep this sorted alphabetically by the OFFSET_... name, for ease of
          * maintenance.  We'll qsort it into the order the test needs shortly.
          */
+        OFFSET(OFFSET_BASECID,          sizeof(r.basecid)),
         OFFSET(OFFSET_CACHE_CRC,        sizeof(r.cache_crc)),
         OFFSET(OFFSET_CACHE_OFFSET,     sizeof(XXX_CACHE32_TYPE)),
         OFFSET(OFFSET_CACHE_VERSION,    sizeof(XXX_CACHE32_TYPE)),
+        OFFSET(OFFSET_CID,              sizeof(r.cid)),
         OFFSET(OFFSET_CREATEDMODSEQ,    sizeof(r.createdmodseq)),
-        OFFSET(OFFSET_GMTIME,           sizeof(XXX_TIME32_TYPE)),
+        OFFSET(OFFSET_GMTIME,           sizeof(r.gmtime)),
         OFFSET(OFFSET_HEADER_SIZE,      sizeof(r.header_size)),
-        OFFSET(OFFSET_INTERNALDATE,     sizeof(XXX_TIME32_TYPE)),
-        OFFSET(OFFSET_LAST_UPDATED,     sizeof(XXX_TIME32_TYPE)),
+        OFFSET(OFFSET_INTERNALDATE,     sizeof(r.internaldate.tv_sec)),
+        OFFSET(OFFSET_LAST_UPDATED,     sizeof(r.last_updated)),
         OFFSET(OFFSET_MESSAGE_GUID,     MESSAGE_GUID_SIZE),
         OFFSET(OFFSET_MODSEQ,           sizeof(r.modseq)),
         OFFSET(OFFSET_RECORD_CRC,       sizeof(uint32_t)), /* not in struct */
-        OFFSET(OFFSET_SAVEDATE,         sizeof(XXX_TIME32_TYPE)),
-        OFFSET(OFFSET_SENTDATE,         sizeof(XXX_TIME32_TYPE)),
+        OFFSET(OFFSET_SAVEDATE,         sizeof(r.savedate)),
+        OFFSET(OFFSET_SENTDATE,         sizeof(r.sentdate)),
         OFFSET(OFFSET_SIZE,             sizeof(r.size)),
         OFFSET(OFFSET_SYSTEM_FLAGS,     sizeof(r.system_flags)),
-        OFFSET(OFFSET_THRID,            sizeof(r.cid)),
         OFFSET(OFFSET_UID,              sizeof(r.uid)),
         OFFSET(OFFSET_USER_FLAGS,       sizeof(r.user_flags)),
         /* this list is sorted alphabetically, don't just append */

imap/annotate.c

@@ -1882,7 +1882,7 @@ static void annotation_get_foldermodseq(annotate_state_t *state,
     annotate_state_need_mbentry(state);
     assert(state->mbentry);
 
-    buf_printf(&value, "%llu", state->mbentry->foldermodseq);
+    buf_printf(&value, MODSEQ_FMT, state->mbentry->foldermodseq);
     output_entryatt(state, entry->name, "", &value);
 
     buf_free(&value);
@@ -1903,7 +1903,7 @@ static void annotation_get_usermodseq(annotate_state_t *state,
     mboxname = mboxname_user_mbox(state->userid, NULL);
     mboxname_read_counters(mboxname, &counters);
 
-    buf_printf(&value, "%llu", counters.highestmodseq);
+    buf_printf(&value, MODSEQ_FMT, counters.highestmodseq);
 
     output_entryatt(state, entry->name, state->userid, &value);
     free(mboxname);
@@ -1923,7 +1923,13 @@ static void annotation_get_usercounters(annotate_state_t *state,
     mboxname = mboxname_user_mbox(state->userid, NULL);
     int r = mboxname_read_counters(mboxname, &counters);
 
-    if (!r) buf_printf(&value, "%u %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %u",
+    if (!r) buf_printf(&value, "%u "  MODSEQ_FMT " "
+                       MODSEQ_FMT " " MODSEQ_FMT " "
+                       MODSEQ_FMT " " MODSEQ_FMT " "
+                       MODSEQ_FMT " " MODSEQ_FMT " "
+                       MODSEQ_FMT " " MODSEQ_FMT " "
+                       MODSEQ_FMT " " MODSEQ_FMT " "
+                       "%u",
                        counters.version, counters.highestmodseq,
                        counters.mailmodseq, counters.caldavmodseq,
                        counters.carddavmodseq, counters.notesmodseq,
@@ -2065,6 +2071,7 @@ static const annotate_entrydesc_t message_builtin_entries[] =
     },
     {
         /* we use 'basethrid' to support split threads */
+        /* prior to version 20, there was no storage for basethrid, so it became an annotation */
         IMAP_ANNOT_NS "basethrid",
         ATTRIB_TYPE_STRING,
         BACKEND_ONLY,