From 26776b8cded4b2ddce3031b21d787c5803b1d5f2 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Thu, 2 Jan 2025 09:34:34 -0500 Subject: [PATCH 01/16] charset.c: implement base64 jmapid encoding --- cunit/charset.testc | 13 +++++++++ lib/charset.c | 65 +++++++++++++++++++++++++++++++++++++++++---- lib/charset.h | 7 ++--- 3 files changed, 77 insertions(+), 8 deletions(-) diff --git a/cunit/charset.testc b/cunit/charset.testc index 39a7d3e662..cf01430971 100644 --- a/cunit/charset.testc +++ b/cunit/charset.testc @@ -1332,6 +1332,14 @@ static void test_charset_decode(void) TESTCASE("vu_A3g", 6, "\xbe\xef\xc0\xde", 4, ENCODING_BASE64URL); TESTCASE("vu_A3g==", 6, "\xbe\xef\xc0\xde", 4, ENCODING_BASE64URL); + /* Base64 JMAPID encoding */ + TESTCASE("-----------", 11, + "\x00\x00\x00\x00\x00\x00\x00\x00", 8, ENCODING_BASE64JMAPID); + TESTCASE("zzzzzzzzzzw", 11, + "\xff\xff\xff\xff\xff\xff\xff\xff", 8, ENCODING_BASE64JMAPID); + TESTCASE("012-abc_TUV", 11, + "\x04\x20\xc0\x9a\x7a\x25\x79\xf8", 8, ENCODING_BASE64JMAPID); + #undef TESTCASE struct buf buf = BUF_INITIALIZER; @@ -1354,6 +1362,11 @@ static void test_charset_decode(void) CU_ASSERT_EQUAL(-1, r); buf_reset(&buf); + /* Base64jmapid with invalid characters */ + r = charset_decode(&buf, "Zm9v@@@YmFy", 11, ENCODING_BASE64JMAPID); + CU_ASSERT_EQUAL(-1, r); + buf_reset(&buf); + buf_free(&buf); } diff --git a/lib/charset.c b/lib/charset.c index 46fe299cf6..0c2c61b3e7 100644 --- a/lib/charset.c +++ b/lib/charset.c @@ -347,6 +347,28 @@ static const char index_64url[256] = { XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, }; + +/* + * Table for decoding base64jmapid + */ +static const char index_64jmapid[256] = { + XX,XX,XX,XX, XX,XX,XX,XX, XX,XS,XS,XX, XX,XS,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XS,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX, 0,XX,XX, + 1, 2, 3, 4, 5, 6, 7, 9, 9,10,XX,XX, XX,64,XX,XX, + XX,11,12,13, 14,15,16,17, 18,19,20,21, 22,23,24,25, + 26,27,28,29, 30,31,32,33, 34,35,36,XX, XX,XX,XX,37, + XX,38,39,40, 41,42,43,44, 45,46,47,48, 49,50,51,52, + 53,54,55,56, 57,58,59,60, 61,62,63,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, + XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, +}; #define CHAR64(c, index) (index[(unsigned char)(c)]) EXPORTED int encoding_lookupname(const char *s) @@ -374,6 +396,8 @@ EXPORTED int encoding_lookupname(const char *s) return ENCODING_BASE64; if (!strcasecmp(s, "BASE64URL")) return ENCODING_BASE64URL; + if (!strcasecmp(s, "BASE64JMAPID")) + return ENCODING_BASE64JMAPID; if (!strcasecmp(s, "BINARY")) return ENCODING_NONE; break; @@ -405,6 +429,7 @@ EXPORTED const char *encoding_name(int encoding) case ENCODING_QP: return "QUOTED-PRINTABLE"; case ENCODING_BASE64: return "BASE64"; case ENCODING_BASE64URL: return "BASE64URL"; + case ENCODING_BASE64JMAPID: return "BASE64JMAPID"; case ENCODING_UNKNOWN: return "UNKNOWN"; default: return "WTF"; } @@ -574,7 +599,7 @@ static int b64_flush(struct convert_rock *rock) { struct b64_state *s = (struct b64_state *)rock->state; if (s->invalid) { - if (s->index == index_64url) + if (s->index != index_64) return -1; else xsyslog(LOG_WARNING, "ignoring invalid base64 characters", NULL); @@ -2348,7 +2373,12 @@ static struct convert_rock *b64_init(struct convert_rock *next, int enc) { struct convert_rock *rock = xzmalloc(sizeof(struct convert_rock)); struct b64_state *state = xzmalloc(sizeof(struct b64_state)); - state->index = enc == ENCODING_BASE64URL ? index_64url : index_64; + if (enc == ENCODING_BASE64JMAPID) + state->index = index_64jmapid; + else if (enc == ENCODING_BASE64URL) + state->index = index_64url; + else + state->index = index_64; rock->state = state; rock->f = b64_2byte; rock->flush = b64_flush; @@ -2819,6 +2849,7 @@ EXPORTED char *charset_to_imaputf7(const char *msg_base, size_t len, charset_t c case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -2978,6 +3009,7 @@ EXPORTED int charset_to_utf8(struct buf *dst, const char *src, size_t len, chars case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -3040,6 +3072,7 @@ EXPORTED int charset_decode(struct buf *dst, const char *src, size_t len, int en case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -3063,8 +3096,25 @@ static void encode_b64(struct buf *dst, const char *src, size_t len, int encodin "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; static const char b64url[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - const char *b64 = encoding == ENCODING_BASE64URL ? b64url : b64std; - char pad = encoding == ENCODING_BASE64URL ? '\0' : '='; + static const char b64jmapid[] = + "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; + const char *b64; + char pad; + + switch (encoding) { + case ENCODING_BASE64JMAPID: + b64 = b64jmapid; + pad = '\0'; + break; + case ENCODING_BASE64URL: + b64 = b64url; + pad = '\0'; + break; + default: + b64 = b64std; + pad = '='; + break; + } const uint8_t *s = (uint8_t*)src; size_t r = len; @@ -3098,7 +3148,8 @@ EXPORTED int charset_encode(struct buf *dst, const char *src, size_t len, int en buf_setmap(dst, src, len); return 0; } - else if (encoding == ENCODING_BASE64 || encoding == ENCODING_BASE64URL) { + else if (encoding == ENCODING_BASE64 || + encoding == ENCODING_BASE64URL || ENCODING_BASE64JMAPID) { encode_b64(dst, src, len, encoding); return 0; } @@ -3140,6 +3191,7 @@ EXPORTED int charset_decode_sha1(uint8_t dest[SHA1_DIGEST_LENGTH], size_t *decod case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -3606,6 +3658,7 @@ EXPORTED int charset_searchfile(const char *substr, comp_pat *pat, case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -3684,6 +3737,7 @@ EXPORTED int charset_extract(int (*cb)(const struct buf *, void *), case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: input = b64_init(input, encoding); /* XXX have to have nl-mapping base64 in order to * properly count \n as 2 raw characters @@ -3754,6 +3808,7 @@ EXPORTED const char *charset_decode_mimebody(const char *msg_base, size_t len, i case ENCODING_BASE64: case ENCODING_BASE64URL: + case ENCODING_BASE64JMAPID: tobuffer = buffer_init(len); input = b64_init(tobuffer, encoding); break; diff --git a/lib/charset.h b/lib/charset.h index 5fba6fd2d3..feb360f596 100644 --- a/lib/charset.h +++ b/lib/charset.h @@ -43,9 +43,10 @@ #define INCLUDED_CHARSET_H #define ENCODING_NONE 0 -#define ENCODING_QP 1 -#define ENCODING_BASE64 2 -#define ENCODING_BASE64URL 3 +#define ENCODING_QP 1 // RFC 2045, Section 6.7 +#define ENCODING_BASE64 2 // RFC 4648, Section 4 +#define ENCODING_BASE64URL 3 // RFC 4648, Section 5 +#define ENCODING_BASE64JMAPID 4 // Base 64 using URL alphabet in ASCII order #define ENCODING_UNKNOWN 255 #define CHARSET_SKIPDIACRIT (1<<0) From eaf69c57efe5db15bed87d6e100b9e606ed33e08 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Wed, 8 Jan 2025 15:41:57 -0500 Subject: [PATCH 02/16] util.h: use consistent data types and formatting for 64-bit values --- imap/annotate.c | 12 +++++++++--- imap/conversations.h | 2 +- imap/ctl_mboxlist.c | 6 ++++-- imap/dlist.c | 10 +++++----- imap/http_caldav.c | 2 +- imap/http_dav.c | 3 ++- imap/jmap_ical.c | 4 ++-- imap/jmap_mail.c | 2 +- imap/mailbox.c | 16 ++++++++-------- imap/mbexamine.c | 2 +- imap/mboxevent.c | 8 +++++++- imap/mboxlist.c | 3 ++- imap/message.c | 14 +++++++++----- imap/sync_support.c | 10 +++++++--- lib/util.h | 27 ++++++++++++++++----------- 15 files changed, 75 insertions(+), 46 deletions(-) diff --git a/imap/annotate.c b/imap/annotate.c index b961b75ede..1022cc2382 100644 --- a/imap/annotate.c +++ b/imap/annotate.c @@ -1882,7 +1882,7 @@ static void annotation_get_foldermodseq(annotate_state_t *state, annotate_state_need_mbentry(state); assert(state->mbentry); - buf_printf(&value, "%llu", state->mbentry->foldermodseq); + buf_printf(&value, MODSEQ_FMT, state->mbentry->foldermodseq); output_entryatt(state, entry->name, "", &value); buf_free(&value); @@ -1903,7 +1903,7 @@ static void annotation_get_usermodseq(annotate_state_t *state, mboxname = mboxname_user_mbox(state->userid, NULL); mboxname_read_counters(mboxname, &counters); - buf_printf(&value, "%llu", counters.highestmodseq); + buf_printf(&value, MODSEQ_FMT, counters.highestmodseq); output_entryatt(state, entry->name, state->userid, &value); free(mboxname); @@ -1923,7 +1923,13 @@ static void annotation_get_usercounters(annotate_state_t *state, mboxname = mboxname_user_mbox(state->userid, NULL); int r = mboxname_read_counters(mboxname, &counters); - if (!r) buf_printf(&value, "%u %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %u", + if (!r) buf_printf(&value, "%u " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + "%u", counters.version, counters.highestmodseq, counters.mailmodseq, counters.caldavmodseq, counters.carddavmodseq, counters.notesmodseq, diff --git a/imap/conversations.h b/imap/conversations.h index 8404bf17ef..b71bd94912 100644 --- a/imap/conversations.h +++ b/imap/conversations.h @@ -66,7 +66,7 @@ (!mbentry ? NULL : (state->folders_byname ? mbentry->name : mbentry->uniqueid)) typedef bit64 conversation_id_t; -#define CONV_FMT "%016llx" +#define CONV_FMT BIT64_FMT #define NULLCONVERSATION (0ULL) struct index_record; diff --git a/imap/ctl_mboxlist.c b/imap/ctl_mboxlist.c index 63226d2dda..27284da068 100644 --- a/imap/ctl_mboxlist.c +++ b/imap/ctl_mboxlist.c @@ -696,7 +696,8 @@ static void do_undump_legacy(void) line++; sscanf(buf, "%m[^\t]\t%d %ms %m[^>]>%ms " TIME_T_FMT " %" SCNu32 - " %llu %llu %m[^\n]\n", &newmbentry->name, &newmbentry->mbtype, + " " MODSEQ_FMT " " MODSEQ_FMT " %m[^\n]\n", + &newmbentry->name, &newmbentry->mbtype, &newmbentry->partition, &newmbentry->acl, &newmbentry->uniqueid, &newmbentry->mtime, &newmbentry->uidvalidity, &newmbentry->foldermodseq, &newmbentry->createdmodseq, &newmbentry->legacy_specialuse); @@ -713,7 +714,8 @@ static void do_undump_legacy(void) mboxlist_entry_free(&newmbentry); newmbentry = mboxlist_entry_create(); sscanf(buf, "%m[^\t]\t%d %ms >%ms " TIME_T_FMT " %" SCNu32 - " %llu %llu %m[^\n]\n", &newmbentry->name, &newmbentry->mbtype, + " " MODSEQ_FMT " " MODSEQ_FMT " %m[^\n]\n", + &newmbentry->name, &newmbentry->mbtype, &newmbentry->partition, &newmbentry->uniqueid, &newmbentry->mtime, &newmbentry->uidvalidity, &newmbentry->foldermodseq, &newmbentry->createdmodseq, &newmbentry->legacy_specialuse); diff --git a/imap/dlist.c b/imap/dlist.c index fbe15046ce..1101b447fd 100644 --- a/imap/dlist.c +++ b/imap/dlist.c @@ -109,7 +109,7 @@ static void printfile(struct protstream *out, const struct dlist *dl) size = sbuf.st_size; if (size != dl->nval) { xsyslog(LOG_ERR, "IOERROR: Size mismatch", - "sval=<%s> len=<%lu> expected=<" MODSEQ_FMT ">", + "sval=<%s> len=<%lu> expected=<" UINT64_FMT ">", dl->sval, size, dl->nval); prot_printf(out, "NIL"); fclose(f); @@ -673,7 +673,7 @@ EXPORTED void dlist_print(const struct dlist *dl, int printkeys, break; case DL_NUM: case DL_DATE: /* for now, we will format it later */ - prot_printf(out, "%llu", dl->nval); + prot_printf(out, UINT64_FMT, dl->nval); break; case DL_FILE: printfile(out, dl); @@ -690,7 +690,7 @@ EXPORTED void dlist_print(const struct dlist *dl, int printkeys, case DL_HEX: { char buf[17]; - snprintf(buf, 17, "%016llx", dl->nval); + snprintf(buf, 17, BIT64_FMT, dl->nval); prot_printf(out, "%s", buf); } break; @@ -1391,12 +1391,12 @@ HIDDEN int dlist_tomap(struct dlist *dl, const char **valp, size_t *lenp) switch (dl->type) { case DL_NUM: case DL_DATE: - snprintf(tmp, 30, "%llu", dl->nval); + snprintf(tmp, 30, UINT64_FMT, dl->nval); dlist_makeatom(dl, tmp); break; case DL_HEX: - snprintf(tmp, 30, "%016llx", dl->nval); + snprintf(tmp, 30, BIT64_FMT, dl->nval); dlist_makeatom(dl, tmp); break; diff --git a/imap/http_caldav.c b/imap/http_caldav.c index a25d1a6070..a5235c871c 100644 --- a/imap/http_caldav.c +++ b/imap/http_caldav.c @@ -1736,7 +1736,7 @@ static int export_calendar(struct transaction_t *txn) n = sscanf((char *) hdr[0], "\"%u-" MODSEQ_FMT "%1s", &uidvalidity, &syncmodseq, dquote /* trailing DQUOTE */); - syslog(LOG_DEBUG, "scanned token %s to %d %u %llu", + syslog(LOG_DEBUG, "scanned token %s to %d %u " MODSEQ_FMT, hdr[0], n, uidvalidity, syncmodseq); /* Sanity check the token components */ diff --git a/imap/http_dav.c b/imap/http_dav.c index 5d1ff5b850..3d8d3e62d9 100644 --- a/imap/http_dav.c +++ b/imap/http_dav.c @@ -7649,7 +7649,8 @@ int report_sync_col(struct transaction_t *txn, struct meth_params *rparams, &uidvalidity, &syncmodseq, &basemodseq, tokenuri /* test for trailing junk */); - syslog(LOG_DEBUG, "scanned token %s to %d %u %llu %llu", + syslog(LOG_DEBUG, + "scanned token %s to %d %u " MODSEQ_FMT " " MODSEQ_FMT, str, r, uidvalidity, syncmodseq, basemodseq); /* Sanity check the token components */ if (r < 2 || r > 3 || diff --git a/imap/jmap_ical.c b/imap/jmap_ical.c index 0cb16a60fc..40c197b780 100644 --- a/imap/jmap_ical.c +++ b/imap/jmap_ical.c @@ -1299,7 +1299,7 @@ static void format_datetime(const struct jmapical_datetime *dt, struct buf *dst) buf_printf(dst, "%04d-%02d-%02dT%02d:%02d:%02d", dt->year, dt->month, dt->day, dt->hour, dt->minute, dt->second); if (dt->nano) { - buf_printf(dst, ".%.9llu", dt->nano); + buf_printf(dst, UINT64_NANOSEC_FMT, dt->nano); int n = buf_len(dst); const char *b = buf_base(dst); while (b[n-1] == '0') n--; @@ -1516,7 +1516,7 @@ HIDDEN void jmapical_duration_as_string(const struct jmapical_duration *dur, str else { buf_putc(buf, '0'); } - buf_printf(buf, ".%.9llu", dur->nanos); + buf_printf(buf, UINT64_NANOSEC_FMT, dur->nanos); /* Truncate trailing zeros */ b = buf_base(buf); n = buf_len(buf); diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index ebeb350298..3fa0ab3e9d 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -3640,7 +3640,7 @@ static int guidsearch_expr_eval(struct conversations_state *cstate, conversation_t conv = CONVERSATION_INIT; int flags = e->v.convflag.include_trash ? CONV_WITHFOLDERS : 0; if (conversation_load_advanced(cstate, match->cid, &conv, flags)) { - syslog(LOG_ERR, "%s: can't load cid %llx", + syslog(LOG_ERR, "%s: can't load cid " CONV_FMT, __func__, match->cid); return 1; } diff --git a/imap/mailbox.c b/imap/mailbox.c index 4c63603426..1405a7f8e3 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -2179,7 +2179,7 @@ static int _store_change(struct mailbox *mailbox, struct index_record *record, i struct buf annotval = BUF_INITIALIZER; if (record->cid && record->basecid && record->basecid != record->cid) - buf_printf(&annotval, "%016llx", record->basecid); + buf_printf(&annotval, CONV_FMT, record->basecid); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval); buf_free(&annotval); @@ -2225,7 +2225,7 @@ static int _commit_one(struct mailbox *mailbox, struct index_change *change) if (change->flags & CHANGE_ISAPPEND) /* note: messageid doesn't have <> wrappers because it already includes them */ syslog(LOG_NOTICE, "auditlog: append sessionid=<%s> " - "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " + "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<" MODSEQ_FMT "> " "sysflags=<%s> guid=<%s> cid=<%s> messageid=%s size=<%u>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, @@ -2234,7 +2234,7 @@ static int _commit_one(struct mailbox *mailbox, struct index_change *change) if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(change->flags & CHANGE_WASEXPUNGED)) syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> " - "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " + "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<" MODSEQ_FMT "> " "sysflags=<%s> guid=<%s> cid=<%s> size=<%u>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, @@ -2243,7 +2243,7 @@ static int _commit_one(struct mailbox *mailbox, struct index_change *change) if ((record->internal_flags & FLAG_INTERNAL_UNLINKED) && !(change->flags & CHANGE_WASUNLINKED)) syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> " - "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " + "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<" MODSEQ_FMT "> " "sysflags=<%s> guid=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, @@ -3408,7 +3408,7 @@ static uint32_t crc_virtannot(struct mailbox *mailbox, struct buf buf = BUF_INITIALIZER; if (record->cid && mailbox->i.minor_version >= 13) { - buf_printf(&buf, "%llx", record->cid); + buf_printf(&buf, CONV_FMT, record->cid); crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "thrid", "", &buf); buf_reset(&buf); } @@ -3420,7 +3420,7 @@ static uint32_t crc_virtannot(struct mailbox *mailbox, } if (record->createdmodseq && mailbox->i.minor_version >= 16) { - buf_printf(&buf, "%llu", record->createdmodseq); + buf_printf(&buf, MODSEQ_FMT, record->createdmodseq); crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); buf_reset(&buf); } @@ -5323,7 +5323,7 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) if (mailbox->i.minor_version >= 13 && repack->newmailbox.i.minor_version < 13) { if (record->cid) { buf_reset(&buf); - buf_printf(&buf, "%llx", record->cid); + buf_printf(&buf, CONV_FMT, record->cid); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "thrid", "", &buf); if (r) goto done; } @@ -5369,7 +5369,7 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) if (mailbox->i.minor_version >= 16 && repack->newmailbox.i.minor_version < 16) { if (record->createdmodseq) { buf_reset(&buf); - buf_printf(&buf, "%llu", record->createdmodseq); + buf_printf(&buf, MODSEQ_FMT, record->createdmodseq); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (r) goto done; } diff --git a/imap/mbexamine.c b/imap/mbexamine.c index 7a1e1eeac7..cb78e8c37b 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -326,7 +326,7 @@ static int do_examine(struct findall_data *data, void *rock) } if (mailbox->i.minor_version >= 13) { - printf(" THRID: %llx", record->cid); + printf(" THRID: " CONV_FMT, record->cid); } printf("\n"); diff --git a/imap/mboxevent.c b/imap/mboxevent.c index eef87aadae..406a4e6ccd 100644 --- a/imap/mboxevent.c +++ b/imap/mboxevent.c @@ -1757,7 +1757,13 @@ EXPORTED void mboxevent_extract_mailbox(struct mboxevent *event, struct buf value = BUF_INITIALIZER; int r = mboxname_read_counters(mailbox_name(mailbox), &counters); - if (!r) buf_printf(&value, "%u %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %u", + if (!r) buf_printf(&value, "%u " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + MODSEQ_FMT " " MODSEQ_FMT " " + "%u", counters.version, counters.highestmodseq, counters.mailmodseq, counters.caldavmodseq, counters.carddavmodseq, counters.notesmodseq, diff --git a/imap/mboxlist.c b/imap/mboxlist.c index b738512b34..14688510e5 100644 --- a/imap/mboxlist.c +++ b/imap/mboxlist.c @@ -1304,7 +1304,8 @@ static int mboxlist_update_entry_full(const char *name, const mbentry_t *mbentry xsyslog(LOG_NOTICE, "auditlog: acl", "sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> mbtype=<%s> " - "oldacl=<%s> acl=<%s> foldermodseq=<%llu>", + "oldacl=<%s> acl=<%s> " + "foldermodseq=<" MODSEQ_FMT ">", session_id(), name, mbentry->uniqueid, mboxlist_mbtype_to_string(mbentry->mbtype), old ? old->acl : "NONE", mbentry->acl, mbentry->foldermodseq); diff --git a/imap/message.c b/imap/message.c index 57bef80c56..abd0691544 100644 --- a/imap/message.c +++ b/imap/message.c @@ -3963,7 +3963,8 @@ EXPORTED int message_update_conversations(struct conversations_state *state, /* try finding a CID in the match list, or if we came in with it */ struct buf annotkey = BUF_INITIALIZER; struct buf annotval = BUF_INITIALIZER; - buf_printf(&annotkey, "%snewcid/%016llx", IMAP_ANNOT_NS, record->cid); + buf_printf(&annotkey, "%snewcid/" CONV_FMT, + IMAP_ANNOT_NS, record->cid); r = annotatemore_lookup(state->annotmboxname, buf_cstring(&annotkey), "", &annotval); if (annotval.len == 16) { const char *p = buf_cstring(&annotval); @@ -3995,7 +3996,8 @@ EXPORTED int message_update_conversations(struct conversations_state *state, conversation_id_t was = record->cid; record->cid = generate_conversation_id(record); - syslog(LOG_NOTICE, "splitting conversation for %s %u base:%016llx was:%016llx now:%016llx", + syslog(LOG_NOTICE, "splitting conversation for %s %u " + "base:" CONV_FMT " was:" CONV_FMT " now:" CONV_FMT, mailbox_name(mailbox), record->uid, record->basecid, was, record->cid); if (!record->basecid) record->basecid = was; @@ -4018,8 +4020,9 @@ EXPORTED int message_update_conversations(struct conversations_state *state, struct buf annotkey = BUF_INITIALIZER; struct buf annotval = BUF_INITIALIZER; - buf_printf(&annotkey, "%snewcid/%016llx", IMAP_ANNOT_NS, record->basecid); - buf_printf(&annotval, "%016llx", record->cid); + buf_printf(&annotkey, "%snewcid/" CONV_FMT, + IMAP_ANNOT_NS, record->basecid); + buf_printf(&annotval, CONV_FMT, record->cid); r = annotate_state_write(astate, buf_cstring(&annotkey), "", &annotval); buf_free(&annotkey); buf_free(&annotval); @@ -5724,7 +5727,8 @@ EXPORTED int message_extract_cids(message_t *msg, conversation_id_t newcid = 0; buf_reset(&annotkey); buf_reset(&annotval); - buf_printf(&annotkey, "%snewcid/%016llx", IMAP_ANNOT_NS, (conversation_id_t) arrayu64_nth(cids, i)); + buf_printf(&annotkey, "%snewcid/" CONV_FMT, + IMAP_ANNOT_NS, (conversation_id_t) arrayu64_nth(cids, i)); annotatemore_lookup(cstate->annotmboxname, buf_cstring(&annotkey), "", &annotval); if (buf_len(&annotval) == 16) { const char *p = buf_cstring(&annotval); diff --git a/imap/sync_support.c b/imap/sync_support.c index 9d689cbd63..d9681e566e 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -3093,11 +3093,14 @@ int sync_apply_mailbox(struct dlist *kin, /* skip out now, it's going to mismatch for sure! */ if (xconvmodseq < ourxconvmodseq) { if (opt_force) { - syslog(LOG_NOTICE, "forcesync: higher xconvmodseq on replica %s - %llu < %llu", + syslog(LOG_NOTICE, + "forcesync: higher xconvmodseq on replica %s - " + MODSEQ_FMT " < " MODSEQ_FMT, mboxname, xconvmodseq, ourxconvmodseq); } else { - syslog(LOG_ERR, "higher xconvmodseq on replica %s - %llu < %llu", + syslog(LOG_ERR, "higher xconvmodseq on replica %s - " + MODSEQ_FMT " < " MODSEQ_FMT, mboxname, xconvmodseq, ourxconvmodseq); r = IMAP_SYNC_CHECKSUM; goto done; @@ -4863,7 +4866,8 @@ static int find_reserve_all(struct sync_name_list *mboxname_list, ispartial = calculate_intermediate_state(mailbox, frommodseq, fromuid, batchsize, &touid, &tomodseq); if (ispartial) { - syslog(LOG_DEBUG, "doing partial sync: %s (%u/%u/%u) (%llu/%llu/%llu)", + syslog(LOG_DEBUG, "doing partial sync: %s (%u/%u/%u) " + "(" MODSEQ_FMT "/" MODSEQ_FMT "/" MODSEQ_FMT ")", mailbox_name(mailbox), fromuid, touid, mailbox->i.last_uid, frommodseq, tomodseq, mailbox->i.highestmodseq); } diff --git a/lib/util.h b/lib/util.h index ea97582fe1..a958616d2f 100644 --- a/lib/util.h +++ b/lib/util.h @@ -94,20 +94,25 @@ extern const char CYRUS_VERSION[]; #endif #define BIT32_MAX 4294967295U - -#if UINT_MAX == BIT32_MAX -typedef unsigned int bit32; -#elif ULONG_MAX == BIT32_MAX -typedef unsigned long bit32; -#elif USHRT_MAX == BIT32_MAX -typedef unsigned short bit32; +#define BIT64_MAX 18446744073709551615UL + +#if ULONG_MAX == BIT64_MAX +#define BIT64_FMT "%016lx" +#define UINT64_FMT "%lu" +#define UINT64_NANOSEC_FMT ".%.9lu" +#elif ULLONG_MAX == BIT64_MAX +#define BIT64_FMT "%016lx" +#define UINT64_FMT "%llu" +#define UINT64_NANOSEC_FMT ".%.9llu" #else -#error dont know what to use for bit32 +#error dont know what to use for bit64 #endif -typedef unsigned long long int bit64; -typedef unsigned long long int modseq_t; -#define MODSEQ_FMT "%llu" +typedef uint32_t bit32; +typedef uint64_t bit64; +typedef uint64_t modseq_t; + +#define MODSEQ_FMT UINT64_FMT #define atomodseq_t(s) strtoull(s, NULL, 10) char *modseqtoa(modseq_t modseq); From ba4ee1220afb760d6a5e1cb6ca64318e789ef251 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Fri, 10 Jan 2025 09:53:01 -0500 Subject: [PATCH 03/16] mailbox.c: v20 records (grow size and time fields to 64-bits) --- cassandane/Cassandane/Cyrus/Reconstruct.pm | 2 +- cunit/mailbox.testc | 20 +- imap/backend.c | 8 +- imap/http_dav.c | 4 +- imap/index.c | 2 +- imap/ipurge.c | 2 +- imap/mailbox.c | 314 +++++++++++++-------- imap/mailbox.h | 61 ++-- imap/mbexamine.c | 9 +- imap/mboxevent.c | 7 +- imap/sync_support.c | 2 +- imap/unexpunge.c | 2 +- lib/util.h | 2 + perl/imap/Cyrus/IndexFile.pm | 121 +++++++- 14 files changed, 388 insertions(+), 168 deletions(-) diff --git a/cassandane/Cassandane/Cyrus/Reconstruct.pm b/cassandane/Cassandane/Cyrus/Reconstruct.pm index 155019d117..20095642a0 100644 --- a/cassandane/Cassandane/Cyrus/Reconstruct.pm +++ b/cassandane/Cassandane/Cyrus/Reconstruct.pm @@ -617,7 +617,7 @@ sub test_downgrade_upgrade $msg{A}->set_attribute(flags => ['\\Seen']); $self->check_messages(\%msg); - for my $version (12, 14, 16, 'max') { + for my $version (12, 14, 16, 19, 'max') { xlog $self, "Set to version $version"; $self->{instance}->run_command({ cyrus => 1 }, 'reconstruct', '-V', $version); diff --git a/cunit/mailbox.testc b/cunit/mailbox.testc index 670f43c4ce..3bef5faa62 100644 --- a/cunit/mailbox.testc +++ b/cunit/mailbox.testc @@ -123,15 +123,15 @@ static void test_aligned_record_offsets(void) CU_ASSERT_EQUAL(0, OFFSET_CACHE_OFFSET % alignof(XXX_CACHE32_TYPE)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_VERSION % alignof(XXX_CACHE32_TYPE)); CU_ASSERT_EQUAL(0, OFFSET_CREATEDMODSEQ % alignof(r.createdmodseq)); - CU_ASSERT_EQUAL(0, OFFSET_GMTIME % alignof(XXX_TIME32_TYPE)); + CU_ASSERT_EQUAL(0, OFFSET_GMTIME % alignof(r.gmtime)); CU_ASSERT_EQUAL(0, OFFSET_HEADER_SIZE % alignof(r.header_size)); - CU_ASSERT_EQUAL(0, OFFSET_INTERNALDATE % alignof(XXX_TIME32_TYPE)); - CU_ASSERT_EQUAL(0, OFFSET_LAST_UPDATED % alignof(XXX_TIME32_TYPE)); + CU_ASSERT_EQUAL(0, OFFSET_INTERNALDATE % alignof(r.internaldate)); + CU_ASSERT_EQUAL(0, OFFSET_LAST_UPDATED % alignof(r.last_updated)); CU_ASSERT_EQUAL(0, OFFSET_MESSAGE_GUID % alignof(char)); /* r/w uses memcpy */ CU_ASSERT_EQUAL(0, OFFSET_MODSEQ % alignof(r.modseq)); CU_ASSERT_EQUAL(0, OFFSET_RECORD_CRC % alignof(uint32_t)); /* not in struct */ - CU_ASSERT_EQUAL(0, OFFSET_SAVEDATE % alignof(XXX_TIME32_TYPE)); - CU_ASSERT_EQUAL(0, OFFSET_SENTDATE % alignof(XXX_TIME32_TYPE)); + CU_ASSERT_EQUAL(0, OFFSET_SAVEDATE % alignof(r.savedate)); + CU_ASSERT_EQUAL(0, OFFSET_SENTDATE % alignof(r.sentdate)); CU_ASSERT_EQUAL(0, OFFSET_SIZE % alignof(r.size)); CU_ASSERT_EQUAL(0, OFFSET_SYSTEM_FLAGS % alignof(r.system_flags)); CU_ASSERT_EQUAL(0, OFFSET_THRID % alignof(r.cid)); @@ -247,15 +247,15 @@ static void test_unique_record_offsets(void) OFFSET(OFFSET_CACHE_OFFSET, sizeof(XXX_CACHE32_TYPE)), OFFSET(OFFSET_CACHE_VERSION, sizeof(XXX_CACHE32_TYPE)), OFFSET(OFFSET_CREATEDMODSEQ, sizeof(r.createdmodseq)), - OFFSET(OFFSET_GMTIME, sizeof(XXX_TIME32_TYPE)), + OFFSET(OFFSET_GMTIME, sizeof(r.gmtime)), OFFSET(OFFSET_HEADER_SIZE, sizeof(r.header_size)), - OFFSET(OFFSET_INTERNALDATE, sizeof(XXX_TIME32_TYPE)), - OFFSET(OFFSET_LAST_UPDATED, sizeof(XXX_TIME32_TYPE)), + OFFSET(OFFSET_INTERNALDATE, sizeof(r.internaldate)), + OFFSET(OFFSET_LAST_UPDATED, sizeof(r.last_updated)), OFFSET(OFFSET_MESSAGE_GUID, MESSAGE_GUID_SIZE), OFFSET(OFFSET_MODSEQ, sizeof(r.modseq)), OFFSET(OFFSET_RECORD_CRC, sizeof(uint32_t)), /* not in struct */ - OFFSET(OFFSET_SAVEDATE, sizeof(XXX_TIME32_TYPE)), - OFFSET(OFFSET_SENTDATE, sizeof(XXX_TIME32_TYPE)), + OFFSET(OFFSET_SAVEDATE, sizeof(r.savedate)), + OFFSET(OFFSET_SENTDATE, sizeof(r.sentdate)), OFFSET(OFFSET_SIZE, sizeof(r.size)), OFFSET(OFFSET_SYSTEM_FLAGS, sizeof(r.system_flags)), OFFSET(OFFSET_THRID, sizeof(r.cid)), diff --git a/imap/backend.c b/imap/backend.c index fbbaab0447..362182f9d7 100644 --- a/imap/backend.c +++ b/imap/backend.c @@ -1339,8 +1339,12 @@ EXPORTED int backend_version(struct backend *be) return MAILBOX_MINOR_VERSION; } else if (major == 3) { - if (minor >= 10) { - /* all versions since 3.10 have been 19 so far */ + if (minor >= 13) { + /* all versions since 3.13 have been 20 so far */ + return 20; + } + else if (minor >= 10) { + /* version 3.10 - 3.12 were 19 */ return 19; } else if (minor >= 3) { diff --git a/imap/http_dav.c b/imap/http_dav.c index 3d8d3e62d9..15cef9b74c 100644 --- a/imap/http_dav.c +++ b/imap/http_dav.c @@ -1946,7 +1946,7 @@ int propfind_getlength(const xmlChar *name, xmlNsPtr ns, buf_reset(&fctx->buf); if (fctx->record) { - buf_printf(&fctx->buf, "%u", + buf_printf(&fctx->buf, UINT64_FMT, fctx->record->size - fctx->record->header_size); } @@ -2974,7 +2974,7 @@ EXPORTED int propfind_quota(const xmlChar *name, xmlNsPtr ns, } else if (fctx->record) { /* Bytes used by resource */ - buf_printf(&fctx->buf, "%u", fctx->record->size); + buf_printf(&fctx->buf, UINT64_FMT, fctx->record->size); } else if (fctx->mailbox) { /* Bytes used by calendar collection */ diff --git a/imap/index.c b/imap/index.c index a445473416..1b372cea82 100644 --- a/imap/index.c +++ b/imap/index.c @@ -4485,7 +4485,7 @@ static int index_fetchreply(struct index_state *state, uint32_t msgno, sepchar = ' '; } if (fetchitems & FETCH_SIZE) { - prot_printf(state->out, "%cRFC822.SIZE %u", + prot_printf(state->out, "%cRFC822.SIZE " UINT64_FMT, sepchar, record.size); sepchar = ' '; } diff --git a/imap/ipurge.c b/imap/ipurge.c index 437512f3f1..c9e477aeee 100644 --- a/imap/ipurge.c +++ b/imap/ipurge.c @@ -387,7 +387,7 @@ static void print_record(struct mailbox *mailbox, const struct index_record *record) { printf("UID: %u\n", record->uid); - printf("\tSize: %u\n", record->size); + printf("\tSize: " UINT64_FMT "\n", record->size); printf("\tSent: %s", ctime(&record->sentdate)); printf("\tRecv: %s", ctime(&record->internaldate)); diff --git a/imap/mailbox.c b/imap/mailbox.c index 1405a7f8e3..4fbc887dee 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -1836,6 +1836,7 @@ static int mailbox_buf_to_index_header(const char *buf, size_t len, case 17: case 18: case 19: + case 20: headerlen = 160; break; default: @@ -2032,91 +2033,122 @@ static int mailbox_buf_to_index_record(const char *buf, int version, static int mailbox_buf_to_index_record(const char *buf, int version, struct index_record *record, int dirty) { - uint32_t crc; uint32_t stored_system_flags = 0; - int n; + uint64_t cache_offset_field; + uint64_t cache_version_field; + uint32_t offset_cache_crc; + uint32_t offset_record_crc; + int n, r = 0; /* tracking fields - initialise */ memset(record, 0, sizeof(struct index_record)); /* parse the shared bits first */ record->uid = ntohl(*((bit32 *)(buf+OFFSET_UID))); - record->internaldate = ntohl(*((bit32 *)(buf+OFFSET_INTERNALDATE))); - record->sentdate = ntohl(*((bit32 *)(buf+OFFSET_SENTDATE))); - record->size = ntohl(*((bit32 *)(buf+OFFSET_SIZE))); - record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE))); - record->gmtime = ntohl(*((bit32 *)(buf+OFFSET_GMTIME))); - uint64_t cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET))); - record->last_updated = ntohl(*((bit32 *)(buf+OFFSET_LAST_UPDATED))); - stored_system_flags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS))); - /* de-serialise system flags and internal flags */ - record->system_flags = stored_system_flags & 0x0000ffff; - record->internal_flags = stored_system_flags & 0xffff0000; + /* v20 grew the of size and time fields to 64-bits + and rearranged fields so that these would fall on 8-byte boundaries */ + if (version < 20) { + record->internaldate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE))); + record->sentdate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SENTDATE))); + record->size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SIZE))); + record->header_size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_HEADER_SIZE))); + record->gmtime = ntohl(*((bit32 *)(buf+PRE20_OFFSET_GMTIME))); + cache_offset_field = ntohl(*((bit32 *)(buf+PRE20_OFFSET_CACHE_OFFSET))); + record->last_updated = ntohl(*((bit32 *)(buf+PRE20_OFFSET_LAST_UPDATED))); + stored_system_flags = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SYSTEM_FLAGS))); + + for (n = 0; n < MAX_USER_FLAGS/32; n++) { + record->user_flags[n] = + ntohl(*((bit32 *)(buf+PRE20_OFFSET_USER_FLAGS+4*n))); + } + cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION))); + + if (version < 8) + goto done; - for (n = 0; n < MAX_USER_FLAGS/32; n++) { - record->user_flags[n] = ntohl(*((bit32 *)(buf+OFFSET_USER_FLAGS+4*n))); - } - uint64_t cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION))); + if (version < 10) { + /* modseq was at 72 before the GUID move */ + record->modseq = ntohll(*((bit64 *)(buf+PRE10_OFFSET_MODSEQ))); + goto done; + } - /* keep the low bits of the version field for the version */ - record->cache_version = cache_version_field & 0xffff; - /* use the high bits of the version field to extend the cache offset */ - record->cache_offset = cache_offset_field | ((cache_version_field & 0xffff0000) << 16); + message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID); + record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ))); + if (version < 12) + goto done; - if (version < 8) - return 0; + /* THRID got inserted before cache_crc32 in version 12 */ + if (version < 13) { + offset_cache_crc = PRE13_OFFSET_CACHE_CRC; + offset_record_crc = PRE13_OFFSET_RECORD_CRC; + goto crc; + } - if (version < 10) { - /* modseq was at 72 before the GUID move */ - record->modseq = ntohll(*((bit64 *)(buf+72))); - return 0; - } + record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); + if (version > 14) { + record->savedate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SAVEDATE))); + } - message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID); - record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ))); - if (version < 12) - return 0; + /* createdmodseq was added in version 16, pushing the CRCs down */ + if (version < 16) { + offset_cache_crc = PRE16_OFFSET_CACHE_CRC; + offset_record_crc = PRE16_OFFSET_RECORD_CRC; + goto crc; + } - /* THRID got inserted before cache_crc32 in version 12 */ - if (version < 13) { - record->cache_crc = ntohl(*((bit32 *)(buf+88))); + record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ)); - if (dirty) return 0; - /* check CRC32 */ - crc = crc32_map(buf, 92); - if (crc != ntohl(*((bit32 *)(buf+92)))) - return IMAP_MAILBOX_CHECKSUM; - return 0; + offset_cache_crc = PRE20_OFFSET_CACHE_CRC; + offset_record_crc = PRE20_OFFSET_RECORD_CRC; + goto crc; } - record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); - if (version > 14) { - record->savedate = ntohl(*((bit32 *)(buf+OFFSET_SAVEDATE))); + cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET))); + record->internaldate = ntohll(*((bit64 *)(buf+OFFSET_INTERNALDATE))); + record->sentdate = ntohll(*((bit64 *)(buf+OFFSET_SENTDATE))); + record->size = ntohll(*((bit64 *)(buf+OFFSET_SIZE))); + record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE))); + stored_system_flags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS))); + + for (n = 0; n < MAX_USER_FLAGS/32; n++) { + record->user_flags[n] = ntohl(*((bit32 *)(buf+OFFSET_USER_FLAGS+4*n))); } - /* createdmodseq was added in version 16, pushing the CRCs down */ - if (version < 16) { - record->cache_crc = ntohl(*((bit32 *)(buf+96))); + cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION))); + message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID); + record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ))); + record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); + record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ)); + record->gmtime = ntohll(*((bit64 *)(buf+OFFSET_GMTIME))); + record->last_updated = ntohll(*((bit64 *)(buf+OFFSET_LAST_UPDATED))); + record->savedate = ntohll(*((bit64 *)(buf+OFFSET_SAVEDATE))); + + offset_cache_crc = OFFSET_CACHE_CRC; + offset_record_crc = OFFSET_RECORD_CRC; + + crc: + record->cache_crc = ntohl(*((bit32 *)(buf+offset_cache_crc))); - if (dirty) return 0; + if (!dirty) { /* check CRC32 */ - crc = crc32_map(buf, 100); - if (crc != ntohl(*((bit32 *)(buf+100)))) - return IMAP_MAILBOX_CHECKSUM; - return 0; + uint32_t crc = crc32_map(buf, offset_record_crc); + if (crc != ntohl(*((bit32 *)(buf+offset_record_crc)))) + r = IMAP_MAILBOX_CHECKSUM; } - record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ)); - record->cache_crc = ntohl(*((bit32 *)(buf+OFFSET_CACHE_CRC))); + done: + /* de-serialise system flags and internal flags */ + record->system_flags = stored_system_flags & 0x0000ffff; + record->internal_flags = stored_system_flags & 0xffff0000; - if (dirty) return 0; - /* check CRC32 */ - crc = crc32_map(buf, OFFSET_RECORD_CRC); - if (crc != ntohl(*((bit32 *)(buf+OFFSET_RECORD_CRC)))) - return IMAP_MAILBOX_CHECKSUM; + /* keep the low bits of the version field for the version */ + record->cache_version = cache_version_field & 0xffff; + /* use the high bits of the version field to extend the cache offset */ + record->cache_offset = + cache_offset_field | ((cache_version_field & 0xffff0000) << 16); - return 0; + return r; } static struct index_change *_find_change(struct mailbox *mailbox, uint32_t recno) @@ -2226,7 +2258,7 @@ static int _commit_one(struct mailbox *mailbox, struct index_change *change) /* note: messageid doesn't have <> wrappers because it already includes them */ syslog(LOG_NOTICE, "auditlog: append sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<" MODSEQ_FMT "> " - "sysflags=<%s> guid=<%s> cid=<%s> messageid=%s size=<%u>", + "sysflags=<%s> guid=<%s> cid=<%s> messageid=%s size=<" UINT64_FMT">", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, message_guid_encode(&record->guid), conversation_id_encode(record->cid), @@ -2235,7 +2267,7 @@ static int _commit_one(struct mailbox *mailbox, struct index_change *change) if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(change->flags & CHANGE_WASEXPUNGED)) syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<" MODSEQ_FMT "> " - "sysflags=<%s> guid=<%s> cid=<%s> size=<%u>", + "sysflags=<%s> guid=<%s> cid=<%s> size=<" UINT64_FMT ">", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, message_guid_encode(&record->guid), conversation_id_encode(record->cid), @@ -3161,91 +3193,126 @@ EXPORTED int mailbox_commit(struct mailbox *mailbox) /* * Put an index record into a buffer suitable for writing to a file. */ -static bit32 mailbox_index_record_to_buf(struct index_record *record, int version, - unsigned char *buf) +static bit32 mailbox_index_record_to_buf(struct index_record *record, + int version, + unsigned char *buf) { int n; bit32 crc; uint32_t system_flags = 0; + uint32_t offset_cache_crc; + uint32_t offset_record_crc; memset(buf, 0, INDEX_RECORD_SIZE); /* keep the low bits of the offset in the offset field */ uint32_t cache_offset_field = record->cache_offset & 0xffffffff; /* mix in the high bits of the offset to the top half of the version field */ - uint32_t cache_version_field = (uint32_t)record->cache_version | (record->cache_offset & 0xffff00000000) >> 16; + uint32_t cache_version_field = + (uint32_t)record->cache_version | (record->cache_offset & 0xffff00000000) >> 16; + + /* serialise system flags and internal flags */ + system_flags = record->system_flags | record->internal_flags; *((bit32 *)(buf+OFFSET_UID)) = htonl(record->uid); - *((bit32 *)(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate); - *((bit32 *)(buf+OFFSET_SENTDATE)) = htonl(record->sentdate); - *((bit32 *)(buf+OFFSET_SIZE)) = htonl(record->size); - *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); - if (version >= 12) { - *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->gmtime); - } - else { - /* content_offset was always the same */ - *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->header_size); + + /* v20 grew the of size and time fields to 64-bits + and rearranged fields so that these would fall on 8-byte boundaries */ + if (version < 20) { + *((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE)) = htonl(record->internaldate); + *((bit32 *)(buf+PRE20_OFFSET_SENTDATE)) = htonl(record->sentdate); + *((bit32 *)(buf+PRE20_OFFSET_SIZE)) = htonl(record->size); + *((bit32 *)(buf+PRE20_OFFSET_HEADER_SIZE)) = htonl(record->header_size); + if (version >= 12) { + *((bit32 *)(buf+PRE20_OFFSET_GMTIME)) = htonl(record->gmtime); + } + else { + /* content_offset was always the same */ + *((bit32 *)(buf+PRE20_OFFSET_GMTIME)) = htonl(record->header_size); + } + *((bit32 *)(buf+PRE20_OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); + *((bit32 *)(buf+PRE20_OFFSET_LAST_UPDATED)) = htonl(record->last_updated); + *((bit32 *)(buf+PRE20_OFFSET_SYSTEM_FLAGS)) = htonl(system_flags); + + for (n = 0; n < MAX_USER_FLAGS/32; n++) { + *((bit32 *)(buf+PRE20_OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]); + } + if (version > 14) { + *((bit32 *)(buf+PRE20_OFFSET_SAVEDATE)) = htonl(record->savedate); + } + else { + *((bit32 *)(buf+PRE20_OFFSET_SAVEDATE)) = 0; // blank out old content_lines + } + *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field); + + /* versions less than 8 had no modseq */ + if (version < 8) { + return 0; + } + + /* versions 8 and 9 only had a smaller UUID, which we will ignore, + * but the modseq existed and was at offset 72 and 76 */ + if (version < 10) { + *((bit64 *)(buf+PRE10_OFFSET_MODSEQ)) = htonll(record->modseq); + return 0; + } + + /* otherwise we have the GUID and MODSEQ in their current place */ + message_guid_export(&record->guid, (char *)buf+OFFSET_MESSAGE_GUID); + *((bit64 *)(buf+OFFSET_MODSEQ)) = htonll(record->modseq); + + /* version 12 added the CACHE_CRC and RECORD_CRC, but at a lower point */ + if (version < 13) { + offset_cache_crc = PRE13_OFFSET_CACHE_CRC; + offset_record_crc = PRE13_OFFSET_RECORD_CRC; + goto crc; + } + + *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid); + + /* version 16 added createdmodseq, pushing the CRCs down */ + if (version < 16) { + offset_cache_crc = PRE16_OFFSET_CACHE_CRC; + offset_record_crc = PRE16_OFFSET_RECORD_CRC; + goto crc; + } + + *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq); + + offset_cache_crc = PRE20_OFFSET_CACHE_CRC; + offset_record_crc = PRE20_OFFSET_RECORD_CRC; + goto crc; } - *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); - *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(record->last_updated); - /* serialise system flags and internal flags */ - system_flags = record->system_flags | record->internal_flags; + *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); + *((bit64 *)(buf+OFFSET_INTERNALDATE)) = htonll(record->internaldate); + *((bit64 *)(buf+OFFSET_SENTDATE)) = htonll(record->sentdate); + *((bit64 *)(buf+OFFSET_SIZE)) = htonll(record->size); + *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); *((bit32 *)(buf+OFFSET_SYSTEM_FLAGS)) = htonl(system_flags); for (n = 0; n < MAX_USER_FLAGS/32; n++) { *((bit32 *)(buf+OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]); } - if (version > 14) { - *((bit32 *)(buf+OFFSET_SAVEDATE)) = htonl(record->savedate); - } - else { - *((bit32 *)(buf+OFFSET_SAVEDATE)) = 0; // blank out old content_lines - } - *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field); - - /* versions less than 8 had no modseq */ - if (version < 8) { - return 0; - } - - /* versions 8 and 9 only had a smaller UUID, which we will ignore, - * but the modseq existed and was at offset 72 and 76 */ - if (version < 10) { - *((bit64 *)(buf+72)) = htonll(record->modseq); - return 0; - } - /* otherwise we have the GUID and MODSEQ in their current place */ + *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field); message_guid_export(&record->guid, (char *)buf+OFFSET_MESSAGE_GUID); *((bit64 *)(buf+OFFSET_MODSEQ)) = htonll(record->modseq); - - /* version 12 added the CACHE_CRC and RECORD_CRC, but at a lower point */ - if (version < 13) { - *((bit32 *)(buf+88)) = htonl(record->cache_crc); - /* calculate the checksum */ - crc = crc32_map((char *)buf, 92); - *((bit32 *)(buf+92)) = htonl(crc); - return crc; - } - *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid); + *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq); + *((bit64 *)(buf+OFFSET_GMTIME)) = htonll(record->gmtime); + *((bit64 *)(buf+OFFSET_LAST_UPDATED)) = htonll(record->last_updated); + *((bit64 *)(buf+OFFSET_SAVEDATE)) = htonll(record->savedate); - /* version 16 added createdmodseq, pushing the CRCs down */ - if (version < 16) { - *((bit32 *)(buf+96)) = htonl(record->cache_crc); - crc = crc32_map((char *)buf, 100); - *((bit32 *)(buf+100)) = htonl(crc); - return crc; - } + offset_cache_crc = OFFSET_CACHE_CRC; + offset_record_crc = OFFSET_RECORD_CRC; - *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq); - *((bit32 *)(buf+OFFSET_CACHE_CRC)) = htonl(record->cache_crc); + crc: + *((bit32 *)(buf+offset_cache_crc)) = htonl(record->cache_crc); /* calculate the checksum */ - crc = crc32_map((char *)buf, OFFSET_RECORD_CRC); - *((bit32 *)(buf+OFFSET_RECORD_CRC)) = htonl(crc); + crc = crc32_map((char *)buf, offset_record_crc); + *((bit32 *)(buf+offset_record_crc)) = htonl(crc); return crc; } @@ -3268,7 +3335,7 @@ static void mailbox_quota_dirty(struct mailbox *mailbox) #define UPDATE_QUOTA_USED(quota_used, size, is_add) do { \ if (is_add) quota_used += size; \ /* corruption prevention - check we don't go negative */ \ - else if (quota_used > size) quota_used -= size; \ + else if ((uint64_t) quota_used > size) quota_used -= size; \ else quota_used = 0; \ } while (0) @@ -4970,6 +5037,11 @@ static int mailbox_repack_setup(struct mailbox *mailbox, int version, repack->newmailbox.i.start_offset = 160; repack->newmailbox.i.record_size = 112; break; + case 20: + repack->newmailbox.i.start_offset = 160; + /* version 20 grew size and time fields to 64-bits */ + repack->newmailbox.i.record_size = 136; + break; default: fatal("index version not supported", EX_SOFTWARE); } diff --git a/imap/mailbox.h b/imap/mailbox.h index b44f715e6b..d514ef55fd 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -78,7 +78,7 @@ * If you change MAILBOX_MINOR_VERSION you MUST also make corresponding * changes to backend_version() in backend.c. */ -#define MAILBOX_MINOR_VERSION (19) /* read comment above! */ +#define MAILBOX_MINOR_VERSION (20) /* read comment above! */ #define MAILBOX_CACHE_MINOR_VERSION (13) #define FNAME_HEADER "/cyrus.header" @@ -152,7 +152,7 @@ struct index_record { uint32_t uid; time_t internaldate; time_t sentdate; - uint32_t size; + uint64_t size; uint32_t header_size; time_t gmtime; size_t cache_offset; @@ -395,24 +395,45 @@ struct mailbox_iter; * add new fields to the record, don't forget to add them to the tests * too! */ -#define OFFSET_UID 0 -#define OFFSET_INTERNALDATE 4 -#define OFFSET_SENTDATE 8 -#define OFFSET_SIZE 12 -#define OFFSET_HEADER_SIZE 16 -#define OFFSET_GMTIME 20 -#define OFFSET_CACHE_OFFSET 24 -#define OFFSET_LAST_UPDATED 28 -#define OFFSET_SYSTEM_FLAGS 32 -#define OFFSET_USER_FLAGS 36 -#define OFFSET_SAVEDATE 52 /* added in v15 */ -#define OFFSET_CACHE_VERSION 56 -#define OFFSET_MESSAGE_GUID 60 -#define OFFSET_MODSEQ 80 /* CONDSTORE (64-bit modseq) */ -#define OFFSET_THRID 88 /* conversation id, added in v13 */ -#define OFFSET_CREATEDMODSEQ 96 /* modseq of creation time, added in v16 */ -#define OFFSET_CACHE_CRC 104 /* CRC32 of cache record */ -#define OFFSET_RECORD_CRC 108 +#define OFFSET_UID 0 +#define OFFSET_CACHE_OFFSET 4 +#define OFFSET_INTERNALDATE 8 /* grew to 64-bit in v20 */ +#define OFFSET_SENTDATE 16 /* grew to 64-bit in v20 */ +#define OFFSET_SIZE 24 /* grew to 64-bit in v20 */ +#define OFFSET_HEADER_SIZE 32 +#define OFFSET_SYSTEM_FLAGS 36 +#define OFFSET_USER_FLAGS 40 +#define OFFSET_CACHE_VERSION 56 +#define OFFSET_MESSAGE_GUID 60 +#define OFFSET_MODSEQ 80 /* CONDSTORE (64-bit modseq) */ +#define OFFSET_THRID 88 /* conversation id, added in v13 */ +#define OFFSET_CREATEDMODSEQ 96 /* modseq of creation time, added in v16 */ +#define OFFSET_GMTIME 104 /* grew to 64-bit in v20 */ +#define OFFSET_LAST_UPDATED 112 /* grew to 64-bit in v20 */ +#define OFFSET_SAVEDATE 120 /* added in v15 */ +#define OFFSET_CACHE_CRC 128 /* CRC32 of cache record */ +#define OFFSET_RECORD_CRC 132 + +#define PRE20_OFFSET_INTERNALDATE 4 +#define PRE20_OFFSET_SENTDATE 8 +#define PRE20_OFFSET_SIZE 12 +#define PRE20_OFFSET_HEADER_SIZE 16 +#define PRE20_OFFSET_GMTIME 20 +#define PRE20_OFFSET_CACHE_OFFSET 24 +#define PRE20_OFFSET_LAST_UPDATED 28 +#define PRE20_OFFSET_SYSTEM_FLAGS 32 +#define PRE20_OFFSET_USER_FLAGS 36 +#define PRE20_OFFSET_SAVEDATE 52 +#define PRE20_OFFSET_CACHE_CRC 104 +#define PRE20_OFFSET_RECORD_CRC 108 + +#define PRE16_OFFSET_CACHE_CRC 96 +#define PRE16_OFFSET_RECORD_CRC 100 + +#define PRE13_OFFSET_CACHE_CRC 88 +#define PRE13_OFFSET_RECORD_CRC 92 + +#define PRE10_OFFSET_MODSEQ 72 #define INDEX_HEADER_SIZE (OFFSET_HEADER_CRC+4) #define INDEX_RECORD_SIZE (OFFSET_RECORD_CRC+4) diff --git a/imap/mbexamine.c b/imap/mbexamine.c index cb78e8c37b..7714c44332 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -307,9 +307,10 @@ static int do_examine(struct findall_data *data, void *rock) } printf("%06u> UID:%08u" - " INT_DATE:" TIME_T_FMT " SENTDATE:" TIME_T_FMT " SAVEDATE:" TIME_T_FMT " SIZE:%-6u\n", + " INT_DATE:" TIME_T_FMT " SENTDATE:" TIME_T_FMT + " SAVEDATE:" TIME_T_FMT " SIZE: " UINT64_LALIGN_FMT "\n", msgno, record->uid, record->internaldate, - record->sentdate, record->savedate, record->size); + record->sentdate, record->savedate, 6, record->size); printf(" > HDRSIZE:%-6u LASTUPD :" TIME_T_FMT " SYSFLAGS:%08X", record->header_size, record->last_updated, record->system_flags); @@ -582,11 +583,11 @@ static int do_compare(struct findall_data *data, void *rock) printf("\n"); printf(" Size: "); - if (record) printf("%-50u", record->size); + if (record) printf(UINT64_LALIGN_FMT, 50, record->size); else printf("%-50s", ""); if (fs_record.uid && !message_guid_isnull(&fs_record.guid)) - printf("\t%-50u", fs_record.size); + printf(UINT64_LALIGN_FMT, 50, fs_record.size); printf("\n"); if (record) time_to_rfc5322(record->sentdate, sent, sizeof(sent)); diff --git a/imap/mboxevent.c b/imap/mboxevent.c index 406a4e6ccd..158c3839fc 100644 --- a/imap/mboxevent.c +++ b/imap/mboxevent.c @@ -1491,7 +1491,8 @@ void mboxevent_extract_content(struct mboxevent *event, { const char *base = NULL; size_t offset, size, len = 0; - int64_t truncate; + int64_t config_truncate; + uint64_t truncate; if (!event) return; @@ -1499,8 +1500,8 @@ void mboxevent_extract_content(struct mboxevent *event, if (!mboxevent_expected_param(event->type, EVENT_MESSAGE_CONTENT)) return; - truncate = config_getbytesize(IMAPOPT_EVENT_CONTENT_SIZE, 'B'); - if (truncate < 0) truncate = 0; + config_truncate = config_getbytesize(IMAPOPT_EVENT_CONTENT_SIZE, 'B'); + truncate = (config_truncate < 0) ? 0 : config_truncate; switch (config_getenum(IMAPOPT_EVENT_CONTENT_INCLUSION_MODE)) { /* include message up to 'truncate' in size with the notification */ diff --git a/imap/sync_support.c b/imap/sync_support.c index d9681e566e..6461b3037f 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1774,7 +1774,7 @@ int parse_upload(struct dlist *kr, struct mailbox *mailbox, return IMAP_PROTOCOL_BAD_PARAMETERS; if (!dlist_getdate(kr, "INTERNALDATE", &record->internaldate)) return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getnum32(kr, "SIZE", &record->size)) + if (!dlist_getnum64(kr, "SIZE", &record->size)) return IMAP_PROTOCOL_BAD_PARAMETERS; if (!dlist_getguid(kr, "GUID", &tmpguid)) return IMAP_PROTOCOL_BAD_PARAMETERS; diff --git a/imap/unexpunge.c b/imap/unexpunge.c index 04c9bc584e..ef4ccf2f0b 100644 --- a/imap/unexpunge.c +++ b/imap/unexpunge.c @@ -153,7 +153,7 @@ static void list_expunged(const char *mboxname, for (i = 0; i < num; i++) { const struct index_record *record = &records[i]; printf("UID: %u\n", record->uid); - printf("\tSize: %u\n", record->size); + printf("\tSize: " UINT64_FMT "\n", record->size); printf("\tSent: %s", ctime(&record->sentdate)); printf("\tRecv: %s", ctime(&record->internaldate)); printf("\tExpg: %s", ctime(&record->last_updated)); diff --git a/lib/util.h b/lib/util.h index a958616d2f..22ef9beb28 100644 --- a/lib/util.h +++ b/lib/util.h @@ -99,10 +99,12 @@ extern const char CYRUS_VERSION[]; #if ULONG_MAX == BIT64_MAX #define BIT64_FMT "%016lx" #define UINT64_FMT "%lu" +#define UINT64_LALIGN_FMT "%-*lu" #define UINT64_NANOSEC_FMT ".%.9lu" #elif ULLONG_MAX == BIT64_MAX #define BIT64_FMT "%016lx" #define UINT64_FMT "%llu" +#define UINT64_LALIGN_FMT "%-*llu" #define UINT64_NANOSEC_FMT ".%.9llu" #else #error dont know what to use for bit64 diff --git a/perl/imap/Cyrus/IndexFile.pm b/perl/imap/Cyrus/IndexFile.pm index 9eba72fc83..20af9b4fbd 100755 --- a/perl/imap/Cyrus/IndexFile.pm +++ b/perl/imap/Cyrus/IndexFile.pm @@ -35,6 +35,7 @@ format as generated by Cyrus IMAPd. * int32 4 - 32 bit value taking 4 octets on disk. Visible in perl as an integer * int64 8 - 64 bit value taking 8 octets on disk. Visible in perl as an integer * time_t 4 - same as int32 + * time64 8 - same as int64 * bitmap N - a bitmap taking up N octets on disk. Visible in perl as a string of 1s and 0s. * hex N - a big value taking up N octets on disk. Visible in perl as a hexadecimal string (0-9a-f) @@ -595,6 +596,64 @@ SKIPPED VERSION 11 - Fastmail internal only 16: CacheCRC int32 4 17: RecordCRC int32 4 + Version 20: + =========== + Header: + 0: Generation int32 4 + 1: Format int32 4 + 2: MinorVersion int32 4 + 3: StartOffset int32 4 + 4: RecordSize int32 4 + 5: Exists int32 4 + 6: LastAppenddate time_t 4 + 7: LastUid int32 4 + 8: QuotaUsed int64 8 + 9: Pop3LastLogin time_t 4 + 10: UidValidity int32 4 + 11: Deleted int32 4 + 12: Answered int32 4 + 13: Flagged int32 4 + 14: Exists int32 4 + 15: Options bitmap 4 + 16: LeakedCache int32 4 + 17: HighestModseq int64 8 + 18: DeletedModseq int64 8 + 19: FirstExpunged time_t 4 + 20: LastCleanup time_t 4 + 21: HeaderFileCRC int32 4 + 22: SyncCRC int32 4 + 23: RecentUid int32 4 + 24: RecentTime time_t 4 + 25: Pop3ShowAfter int32 4 + 26: QuotaAnnotUsed int32 4 + 27: SyncCRCsAnnot int32 4 + 28: Unseen int32 4 + 29: CreatedModseq int64 8 + 30: QuotaDeletedUsed int64 8 + 31: QuotaExpungedUsed int64 8 + 32: ChangesEpoch int32 4 + 33: HeaderCRC int32 4 + + Record: + 0: Uid int32 4 + 1: CacheOffset int32 4 + 2: InternalDate time_t 8 + 3: SentDate time_t 8 + 4: Size int64 8 + 5: HeaderSize int32 4 + 6: SystemFlags bitmap 4 + 7: UserFlags bitmap 16 + 8: CacheVersion int32 4 + 9: MessageGuid hex 20 + 10: Modseq int64 8 + 11: CID hex 8 + 12: CreatedModseq int64 8 + 13: GmTime time_t 8 + 14: LastUpdated time_t 8 + 15: SaveDate time_t 8 + 16: CacheCRC int32 4 + 17: RecordCRC int32 4 + =cut # Set up header and record formatting information {{{ @@ -1208,6 +1267,66 @@ CID hex 8 CreatedModseq int64 8 CacheCrc int32 4 RecordCrc int32 4 +EOF + }, + 20 => { + HeaderSize => 160, + _make_fields('Header',< 136, # defined in file too, check it! + _make_fields('Record', < Date: Fri, 10 Jan 2025 14:05:44 -0500 Subject: [PATCH 04/16] mailbox.c: use struct timespec to store internaldate --- cunit/mailbox.testc | 2 +- imap/caldav_alarm.c | 11 ++++---- imap/caldav_util.c | 2 +- imap/conversations.c | 8 +++--- imap/dav_util.c | 2 +- imap/http_caldav.c | 6 ++-- imap/http_rss.c | 2 +- imap/index.c | 12 ++++---- imap/ipurge.c | 4 +-- imap/jmap_backup.c | 4 +-- imap/jmap_calendar.c | 2 +- imap/jmap_mail_submission.c | 8 +++--- imap/mailbox.c | 56 ++++++++++++++++++------------------- imap/mailbox.h | 2 +- imap/mbdump.c | 4 +-- imap/mbexamine.c | 2 +- imap/mbtool.c | 6 ++-- imap/message.c | 4 +-- imap/msgrecord.c | 6 ++-- imap/pop3d.c | 2 +- imap/sync_support.c | 10 +++---- imap/unexpunge.c | 2 +- 22 files changed, 79 insertions(+), 78 deletions(-) diff --git a/cunit/mailbox.testc b/cunit/mailbox.testc index 3bef5faa62..46c3edcb46 100644 --- a/cunit/mailbox.testc +++ b/cunit/mailbox.testc @@ -249,7 +249,7 @@ static void test_unique_record_offsets(void) OFFSET(OFFSET_CREATEDMODSEQ, sizeof(r.createdmodseq)), OFFSET(OFFSET_GMTIME, sizeof(r.gmtime)), OFFSET(OFFSET_HEADER_SIZE, sizeof(r.header_size)), - OFFSET(OFFSET_INTERNALDATE, sizeof(r.internaldate)), + OFFSET(OFFSET_INTERNALDATE, sizeof(r.internaldate.tv_sec)), OFFSET(OFFSET_LAST_UPDATED, sizeof(r.last_updated)), OFFSET(OFFSET_MESSAGE_GUID, MESSAGE_GUID_SIZE), OFFSET(OFFSET_MODSEQ, sizeof(r.modseq)), diff --git a/imap/caldav_alarm.c b/imap/caldav_alarm.c index 437015b234..54d00c1413 100644 --- a/imap/caldav_alarm.c +++ b/imap/caldav_alarm.c @@ -1109,7 +1109,8 @@ HIDDEN int caldav_alarm_add_record(struct mailbox *mailbox, if (has_alarms(data, mailbox, record->uid, &num_rcpts)) { enum alarm_type atype = mbtype_to_alarm_type(mailbox_mbtype(mailbox)); - update_alarmdb(mailbox_name(mailbox), record->uid, record->internaldate, + update_alarmdb(mailbox_name(mailbox), record->uid, + record->internaldate.tv_sec, atype, num_rcpts, 0, 0, NULL); } @@ -1345,7 +1346,7 @@ static int process_valarms(struct mailbox *mailbox, struct lastalarm_data data; if (read_lastalarm(mailbox, record, &data)) - data.lastrun = record->internaldate; + data.lastrun = record->internaldate.tv_sec; /* Process VALARMs in iCalendar resource */ char *userid = mboxname_to_userid(mboxname); @@ -1520,7 +1521,7 @@ static int move_to_mailboxid(struct mailbox *srcmbox, if (r) goto done; /* Append the message to the mailbox */ - r = append_fromstage_full(&as, &body, stage, record->internaldate, + r = append_fromstage_full(&as, &body, stage, record->internaldate.tv_sec, savedate, 0, flags, 0, &annots); if (r) { append_abort(&as); @@ -2062,8 +2063,8 @@ static void process_one_record(struct caldav_alarm_data *data, time_t runtime, i } #ifdef WITH_JMAP case ALARM_SEND: - if (record.internaldate > runtime || dryrun) { - caldav_alarm_bump_nextcheck(data, record.internaldate, 0, NULL); + if (record.internaldate.tv_sec > runtime || dryrun) { + caldav_alarm_bump_nextcheck(data, record.internaldate.tv_sec, 0, NULL); goto done; } r = process_futurerelease(data, mailbox, &record, runtime); diff --git a/imap/caldav_util.c b/imap/caldav_util.c index 11080643af..a89d1e1326 100644 --- a/imap/caldav_util.c +++ b/imap/caldav_util.c @@ -196,7 +196,7 @@ EXPORTED int caldav_get_validators(struct mailbox *mailbox, void *data, dlist_getdate(dl, "LASTMOD", &user_lastmod); /* Per-user Last-Modified is latest mod time */ - *lastmod = MAX(record->internaldate, user_lastmod); + *lastmod = MAX(record->internaldate.tv_sec, user_lastmod); } dlist_free(&dl); diff --git a/imap/conversations.c b/imap/conversations.c index b2a104f493..2fac7b19a4 100644 --- a/imap/conversations.c +++ b/imap/conversations.c @@ -2361,11 +2361,11 @@ static int conversations_set_guid(struct conversations_state *state, base, record->cid, record->basecid, record->system_flags, record->internal_flags, - record->internaldate, + record->internaldate.tv_sec, add); if (!r) r = _guid_addbody(state, record->cid, record->basecid, record->system_flags, record->internal_flags, - record->internaldate, body, base, add); + record->internaldate.tv_sec, body, base, add); message_free_body(body); free(body); @@ -2518,7 +2518,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate, if (new) { if (!old || old->system_flags != new->system_flags || old->internal_flags != new->internal_flags || - old->internaldate != new->internaldate) { + old->internaldate.tv_sec != new->internaldate.tv_sec) { r = conversations_set_guid(cstate, mailbox, new, /*add*/1); if (r) goto done; } @@ -2634,7 +2634,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate, conversation_update_thread(conv, &record->guid, - record->internaldate, + record->internaldate.tv_sec, record->createdmodseq, delta_exists); diff --git a/imap/dav_util.c b/imap/dav_util.c index 815a9f1f48..33150a71c5 100644 --- a/imap/dav_util.c +++ b/imap/dav_util.c @@ -92,7 +92,7 @@ EXPORTED int dav_get_validators(struct mailbox *mailbox, void *data, } if (etag) *etag = message_guid_encode(&record->guid); - if (lastmod) *lastmod = record->internaldate; + if (lastmod) *lastmod = record->internaldate.tv_sec; } else { /* Unmapped URL (empty resource) */ diff --git a/imap/http_caldav.c b/imap/http_caldav.c index a5235c871c..10b5f6d1e3 100644 --- a/imap/http_caldav.c +++ b/imap/http_caldav.c @@ -2569,7 +2569,7 @@ static int caldav_get(struct transaction_t *txn, struct mailbox *mailbox, check if conditional matches previous ETag */ if (check_precond(txn, cdata->sched_tag, - record->internaldate) == HTTP_NOT_MODIFIED) { + record->internaldate.tv_sec) == HTTP_NOT_MODIFIED) { /* Fill in previous ETag and don't return Last-Modified */ txn->resp_body.etag = cdata->sched_tag; txn->resp_body.lastmod = 0; @@ -2615,7 +2615,7 @@ static int caldav_get(struct transaction_t *txn, struct mailbox *mailbox, /* Fill in new ETag and Last-Modified */ txn->resp_body.etag = message_guid_encode(&record->guid); - txn->resp_body.lastmod = record->internaldate; + txn->resp_body.lastmod = record->internaldate.tv_sec; caldav_close(caldavdb); } @@ -2885,7 +2885,7 @@ static int caldav_post_attach(struct transaction_t *txn, int rights) } etag = message_guid_encode(&record.guid); - lastmod = record.internaldate; + lastmod = record.internaldate.tv_sec; /* Load and parse message containing the resource */ ical = record_to_ical(calendar, &record, &schedule_addresses); diff --git a/imap/http_rss.c b/imap/http_rss.c index 01428ee2d4..445775043b 100644 --- a/imap/http_rss.c +++ b/imap/http_rss.c @@ -242,7 +242,7 @@ static int meth_get(struct transaction_t *txn, } etag = message_guid_encode(&record.guid); - lastmod = record.internaldate; + lastmod = record.internaldate.tv_sec; precond = check_precond(txn, etag, lastmod); switch (precond) { diff --git a/imap/index.c b/imap/index.c index 1b372cea82..7c72d3dada 100644 --- a/imap/index.c +++ b/imap/index.c @@ -3324,7 +3324,7 @@ static int index_appendremote(struct index_state *state, uint32_t msgno, } /* add internal date */ - time_to_rfc3501(record.internaldate, datebuf, sizeof(datebuf)); + time_to_rfc3501(record.internaldate.tv_sec, datebuf, sizeof(datebuf)); prot_printf(pout, ") \"%s\" ", datebuf); /* message literal */ @@ -4458,7 +4458,7 @@ static int index_fetchreply(struct index_state *state, uint32_t msgno, } if (fetchitems & FETCH_INTERNALDATE) { - time_t msgdate = record.internaldate; + time_t msgdate = record.internaldate.tv_sec; char datebuf[RFC3501_DATETIME_MAX+1]; time_to_rfc3501(msgdate, datebuf, sizeof(datebuf)); @@ -4577,7 +4577,7 @@ static int index_fetchreply(struct index_state *state, uint32_t msgno, char datebuf[RFC3501_DATETIME_MAX+1]; // handle internaldate - if (!msgdate) msgdate = record.internaldate; + if (!msgdate) msgdate = record.internaldate.tv_sec; time_to_rfc3501(msgdate, datebuf, sizeof(datebuf)); @@ -6336,7 +6336,7 @@ MsgData **index_msgdata_load(struct index_state *state, cur->sentdate = record.gmtime; /* fall through */ case SORT_ARRIVAL: - cur->internaldate = record.internaldate; + cur->internaldate = record.internaldate.tv_sec; break; case SORT_FROM: cur->from = get_localpart_addr(cacheitem_base(&record, CACHE_FROM)); @@ -6379,7 +6379,7 @@ MsgData **index_msgdata_load(struct index_state *state, } else { /* If not in mailboxId, we use receivedAt */ - cur->internaldate = record.internaldate; + cur->internaldate = record.internaldate.tv_sec; } break; case SORT_SNOOZEDUNTIL: @@ -6405,7 +6405,7 @@ MsgData **index_msgdata_load(struct index_state *state, #endif if (!cur->savedate) { /* If not snoozed in mailboxId, we use receivedAt */ - cur->internaldate = record.internaldate; + cur->internaldate = record.internaldate.tv_sec; } break; case LOAD_IDS: diff --git a/imap/ipurge.c b/imap/ipurge.c index c9e477aeee..756a3b6253 100644 --- a/imap/ipurge.c +++ b/imap/ipurge.c @@ -319,7 +319,7 @@ static unsigned purge_check(struct mailbox *mailbox, mbox_stats_t *stats = (mbox_stats_t *) deciderock; my_time = time(0); - senttime = use_sentdate ? record->sentdate : record->internaldate; + senttime = use_sentdate ? record->sentdate : record->internaldate.tv_sec; stats->total++; stats->total_bytes += record->size; @@ -389,7 +389,7 @@ static void print_record(struct mailbox *mailbox, printf("UID: %u\n", record->uid); printf("\tSize: " UINT64_FMT "\n", record->size); printf("\tSent: %s", ctime(&record->sentdate)); - printf("\tRecv: %s", ctime(&record->internaldate)); + printf("\tRecv: %s", ctime(&record->internaldate.tv_sec)); if (mailbox_cacherecord(mailbox, record)) { printf("\tERROR: cache record missing or corrupt, " diff --git a/imap/jmap_backup.c b/imap/jmap_backup.c index 7b44f1b277..0fd02fc661 100644 --- a/imap/jmap_backup.c +++ b/imap/jmap_backup.c @@ -562,7 +562,7 @@ static int recreate_resource(message_t *msg, struct mailbox *tomailbox, strarray_add(flags, "$restored"); /* append the message to the mailbox. */ - r = append_fromstage(&as, &body, stage, record->internaldate, + r = append_fromstage(&as, &body, stage, record->internaldate.tv_sec, is_update ? record->createdmodseq : 0, flags, /*nolink*/0, &annots); @@ -1642,7 +1642,7 @@ static int restore_message_list_cb(const mbentry_t *mbentry, void *rock) " intdate: " TIME_T_FMT ", updated: " TIME_T_FMT, record->uid, (record->internal_flags & FLAG_INTERNAL_EXPUNGED), (record->system_flags & FLAG_DRAFT), - record->internaldate, record->last_updated); + record->internaldate.tv_sec, record->last_updated); /* Suppress fetching of Message-ID if not restoring drafts */ if (rrock->jrestore->mode & UNDO_DRAFTS) { diff --git a/imap/jmap_calendar.c b/imap/jmap_calendar.c index 499adcd994..39c6a36e0c 100644 --- a/imap/jmap_calendar.c +++ b/imap/jmap_calendar.c @@ -9918,7 +9918,7 @@ static void notifsearch_run(const char *userid, entry.is_tombstone = (!seenuids && (record->system_flags & FLAG_SEEN)) || (seenuids && seqset_ismember(seenuids, record->uid)); } - entry.created = record->internaldate; + entry.created = record->internaldate.tv_sec; message_guid_copy(&entry.guid, &record->guid); entry.modseq = record->modseq; diff --git a/imap/jmap_mail_submission.c b/imap/jmap_mail_submission.c index 18559f0475..41c6b50e60 100644 --- a/imap/jmap_mail_submission.c +++ b/imap/jmap_mail_submission.c @@ -1007,7 +1007,7 @@ static void _emailsubmission_update(struct mailbox *submbox, else if (!strcmp(arg, "sendAt")) { time_t t = 0; if (time_from_iso8601(strval, &t) == (int) strlen(strval) && - t == record->internaldate) { + t == record->internaldate.tv_sec) { continue; } } @@ -1927,10 +1927,10 @@ static int submission_filter_match(void *vf, void *rock) const struct index_record *record = msg_record(sfrock->msg); /* before */ - if (record->internaldate >= f->before) return 0; + if (record->internaldate.tv_sec >= f->before) return 0; /* after */ - if (record->internaldate < f->after) return 0; + if (record->internaldate.tv_sec < f->after) return 0; /* createdBefore */ if (record->savedate >= f->createdBefore) return 0; @@ -2183,7 +2183,7 @@ static int jmap_emailsubmission_query(jmap_req_t *req) sprintf(match->id, "S%u", record->uid); match->uid = record->uid; match->created = record->savedate; - match->sentAt = record->internaldate; + match->sentAt = record->internaldate.tv_sec; match->emailId = sfrock.emailId; match->threadId = sfrock.threadId; match->submission = sfrock.submission; diff --git a/imap/mailbox.c b/imap/mailbox.c index 4fbc887dee..7ffd9667d4 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -2049,7 +2049,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, /* v20 grew the of size and time fields to 64-bits and rearranged fields so that these would fall on 8-byte boundaries */ if (version < 20) { - record->internaldate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE))); + record->internaldate.tv_sec = ntohl(*((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE))); record->sentdate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SENTDATE))); record->size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SIZE))); record->header_size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_HEADER_SIZE))); @@ -2105,7 +2105,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, } cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET))); - record->internaldate = ntohll(*((bit64 *)(buf+OFFSET_INTERNALDATE))); + record->internaldate.tv_sec = ntohll(*((bit64 *)(buf+OFFSET_INTERNALDATE))); record->sentdate = ntohll(*((bit64 *)(buf+OFFSET_SENTDATE))); record->size = ntohll(*((bit64 *)(buf+OFFSET_SIZE))); record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE))); @@ -3219,7 +3219,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, /* v20 grew the of size and time fields to 64-bits and rearranged fields so that these would fall on 8-byte boundaries */ if (version < 20) { - *((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE)) = htonl(record->internaldate); + *((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE)) = htonl(record->internaldate.tv_sec); *((bit32 *)(buf+PRE20_OFFSET_SENTDATE)) = htonl(record->sentdate); *((bit32 *)(buf+PRE20_OFFSET_SIZE)) = htonl(record->size); *((bit32 *)(buf+PRE20_OFFSET_HEADER_SIZE)) = htonl(record->header_size); @@ -3285,7 +3285,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, } *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); - *((bit64 *)(buf+OFFSET_INTERNALDATE)) = htonll(record->internaldate); + *((bit64 *)(buf+OFFSET_INTERNALDATE)) = htonll(record->internaldate.tv_sec); *((bit64 *)(buf+OFFSET_SENTDATE)) = htonll(record->sentdate); *((bit64 *)(buf+OFFSET_SIZE)) = htonll(record->size); *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); @@ -3424,7 +3424,7 @@ static uint32_t crc_basic(const struct mailbox *mailbox, snprintf(buf, sizeof(buf), "%u " MODSEQ_FMT " " TIME_T_FMT " (%u) " TIME_T_FMT " %s", record->uid, record->modseq, record->last_updated, flagcrc, - record->internaldate, + record->internaldate.tv_sec, message_guid_encode(&record->guid)); return crc32_cstring(buf); @@ -3752,7 +3752,7 @@ static int mailbox_update_carddav(struct mailbox *mailbox, cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; if (!cdata->dav.creationdate) - cdata->dav.creationdate = new->internaldate; + cdata->dav.creationdate = new->internaldate.tv_sec; /* Load message containing the resource and parse vcard data */ #ifdef HAVE_LIBICALVCARD @@ -3895,7 +3895,7 @@ static int mailbox_update_caldav(struct mailbox *mailbox, if (r) goto alarmdone; } - cdata->dav.creationdate = new->internaldate; + cdata->dav.creationdate = new->internaldate.tv_sec; cdata->dav.imap_uid = new->uid; cdata->dav.modseq = new->modseq; cdata->dav.createdmodseq = new->createdmodseq; @@ -3996,7 +3996,7 @@ static int mailbox_update_webdav(struct mailbox *mailbox, buf_len(&msg_buf) - new->header_size); buf_free(&msg_buf); - wdata->dav.creationdate = new->internaldate; + wdata->dav.creationdate = new->internaldate.tv_sec; wdata->dav.imap_uid = new->uid; wdata->dav.modseq = new->modseq; wdata->dav.createdmodseq = new->createdmodseq; @@ -4333,7 +4333,7 @@ static int mailbox_update_sieve(struct mailbox *mailbox, } } - sdata->lastupdated = new->internaldate; + sdata->lastupdated = new->internaldate.tv_sec; sdata->mailbox = mailbox_uniqueid(mailbox); sdata->imap_uid = new->uid; sdata->modseq = new->modseq; @@ -4345,7 +4345,7 @@ static int mailbox_update_sieve(struct mailbox *mailbox, sdata->contentid = message_guid_encode(&body->content_guid); if (!sdata->creationdate) - sdata->creationdate = new->internaldate; + sdata->creationdate = new->internaldate.tv_sec; r = sievedb_write(sievedb, sdata); } @@ -4763,12 +4763,12 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, } } - if (!record->internaldate) - record->internaldate = time(NULL); + if (!record->internaldate.tv_sec) + record->internaldate.tv_sec = time(NULL); if (!record->gmtime) - record->gmtime = record->internaldate; + record->gmtime = record->internaldate.tv_sec; if (!record->sentdate) { - struct tm *tm = localtime(&record->internaldate); + struct tm *tm = localtime(&record->internaldate.tv_sec); /* truncate to the day */ tm->tm_sec = 0; tm->tm_min = 0; @@ -4799,7 +4799,7 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, if (!(record->internal_flags & FLAG_INTERNAL_UNLINKED)) { /* make the file timestamp correct */ - settime.actime = settime.modtime = record->internaldate; + settime.actime = settime.modtime = record->internaldate.tv_sec; if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) // mabe there is no file in directory. if (utime(mailbox_record_fname(mailbox, record), &settime) == -1) return IMAP_IOERROR; @@ -5532,7 +5532,7 @@ EXPORTED unsigned mailbox_should_archive(struct mailbox *mailbox, return 0; /* archive all other old messages */ - if (record->internaldate <= cutoff) + if (record->internaldate.tv_sec <= cutoff) return 1; /* and don't archive anything else! */ @@ -7244,7 +7244,7 @@ static int records_match(const char *mboxname, int match = 1; int userflags_dirty = 0; - if (old->internaldate != new->internaldate) { + if (old->internaldate.tv_sec != new->internaldate.tv_sec) { printf("%s uid %u mismatch: internaldate\n", mboxname, new->uid); match = 0; @@ -7398,21 +7398,21 @@ static int mailbox_reconstruct_compare_update(struct mailbox *mailbox, /* copy once the cache record is read in... */ copy = *record; - if (!record->internaldate) { + if (!record->internaldate.tv_sec) { re_parse = 1; } /* re-calculate all the "derived" fields by parsing the file on disk */ if (re_parse) { /* set NULL in case parse finds a new value */ - record->internaldate = 0; + record->internaldate.tv_sec = 0; r = message_parse(fname, record); if (r) goto out; /* unchanged, keep the old value */ - if (!record->internaldate) - record->internaldate = copy.internaldate; + if (!record->internaldate.tv_sec) + record->internaldate.tv_sec = copy.internaldate.tv_sec; /* it's not the same message! */ if (!message_guid_equal(&record->guid, ©.guid)) { @@ -7492,16 +7492,16 @@ static int mailbox_reconstruct_compare_update(struct mailbox *mailbox, } /* get internaldate from the file if not set */ - if (!record->internaldate) { + if (!record->internaldate.tv_sec) { if (did_stat || stat(fname, &sbuf) != -1) - record->internaldate = sbuf.st_mtime; + record->internaldate.tv_sec = sbuf.st_mtim.tv_sec; else - record->internaldate = time(NULL); + record->internaldate.tv_sec = time(NULL); } if (!record->gmtime) - record->gmtime = record->internaldate; + record->gmtime = record->internaldate.tv_sec; if (!record->sentdate) { - struct tm *tm = localtime(&record->internaldate); + struct tm *tm = localtime(&record->internaldate.tv_sec); /* truncate to the day */ tm->tm_sec = 0; tm->tm_min = 0; @@ -7665,8 +7665,8 @@ static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid, int record.internal_flags |= FLAG_INTERNAL_SNOOZED; /* copy the timestamp from the file if not calculated */ - if (!record.internaldate) - record.internaldate = sbuf.st_mtime; + if (!record.internaldate.tv_sec) + record.internaldate.tv_sec = sbuf.st_mtim.tv_sec; if (uid > mailbox->i.last_uid) { printf("%s uid %u found - adding\n", mailbox_name(mailbox), uid); diff --git a/imap/mailbox.h b/imap/mailbox.h index d514ef55fd..27e0d3e36e 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -150,7 +150,7 @@ struct statusdata { struct index_record { uint32_t uid; - time_t internaldate; + struct timespec internaldate; time_t sentdate; uint64_t size; uint32_t header_size; diff --git a/imap/mbdump.c b/imap/mbdump.c index 7de1fa8271..61858950c5 100644 --- a/imap/mbdump.c +++ b/imap/mbdump.c @@ -154,7 +154,7 @@ static void downgrade_record(const struct index_record *record, char *buf, UP_modseqbase = OFFSET_MESSAGE_GUID+12; *((bit32 *)(buf+OFFSET_UID)) = htonl(record->uid); - *((bit32 *)(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate); + *((bit32 *)(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate.tv_sec); *((bit32 *)(buf+OFFSET_SENTDATE)) = htonl(record->sentdate); *((bit32 *)(buf+OFFSET_SIZE)) = htonl(record->size); *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); @@ -1312,7 +1312,7 @@ EXPORTED int undump_mailbox(const char *mbname, while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); fname = mailbox_record_fname(mailbox, record); - settime.actime = settime.modtime = record->internaldate; + settime.actime = settime.modtime = record->internaldate.tv_sec; if (utime(fname, &settime) == -1) { r = IMAP_IOERROR; mailbox_iter_done(&iter); diff --git a/imap/mbexamine.c b/imap/mbexamine.c index 7714c44332..f8c33b94e6 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -309,7 +309,7 @@ static int do_examine(struct findall_data *data, void *rock) printf("%06u> UID:%08u" " INT_DATE:" TIME_T_FMT " SENTDATE:" TIME_T_FMT " SAVEDATE:" TIME_T_FMT " SIZE: " UINT64_LALIGN_FMT "\n", - msgno, record->uid, record->internaldate, + msgno, record->uid, record->internaldate.tv_sec, record->sentdate, record->savedate, 6, record->size); printf(" > HDRSIZE:%-6u LASTUPD :" TIME_T_FMT " SYSFLAGS:%08X", record->header_size, record->last_updated, diff --git a/imap/mbtool.c b/imap/mbtool.c index 0099073ebe..c23a8096e0 100644 --- a/imap/mbtool.c +++ b/imap/mbtool.c @@ -205,17 +205,17 @@ static int do_timestamp(const mbname_t *mbname) while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* 1 day is close enough */ - if (llabs(record->internaldate - record->gmtime) < 86400) + if (llabs(record->internaldate.tv_sec - record->gmtime) < 86400) continue; struct index_record copyrecord = *record; - time_to_rfc5322(copyrecord.internaldate, olddate, sizeof(olddate)); + time_to_rfc5322(copyrecord.internaldate.tv_sec, olddate, sizeof(olddate)); time_to_rfc5322(copyrecord.gmtime, newdate, sizeof(newdate)); printf(" %u: %s => %s\n", copyrecord.uid, olddate, newdate); /* switch internaldate */ - copyrecord.internaldate = copyrecord.gmtime; + copyrecord.internaldate.tv_sec = copyrecord.gmtime; r = mailbox_rewrite_index_record(mailbox, ©record); if (r) goto done; diff --git a/imap/message.c b/imap/message.c index abd0691544..0fffe12342 100644 --- a/imap/message.c +++ b/imap/message.c @@ -5298,7 +5298,7 @@ EXPORTED int message_get_savedate(message_t *m, time_t *datep) int r = message_need(m, M_RECORD); if (r) return r; *datep = m->record.savedate; - if (!*datep) *datep = m->record.internaldate; + if (!*datep) *datep = m->record.internaldate.tv_sec; return 0; } @@ -5330,7 +5330,7 @@ EXPORTED int message_get_internaldate(message_t *m, time_t *datep) { int r = message_need(m, M_RECORD); if (r) return r; - *datep = m->record.internaldate; + *datep = m->record.internaldate.tv_sec; return 0; } diff --git a/imap/msgrecord.c b/imap/msgrecord.c index 38aaaff7b7..5ce6198fcd 100644 --- a/imap/msgrecord.c +++ b/imap/msgrecord.c @@ -291,7 +291,7 @@ EXPORTED int msgrecord_get_internaldate(msgrecord_t *mr, time_t *t) int r = msgrecord_need(mr, M_RECORD); if (r) return r; } - *t = mr->record.internaldate; + *t = mr->record.internaldate.tv_sec; return 0; } @@ -304,7 +304,7 @@ EXPORTED int msgrecord_get_savedate(msgrecord_t *mr, time_t *t) if (mr->record.savedate) *t = mr->record.savedate; else - *t = mr->record.internaldate; + *t = mr->record.internaldate.tv_sec; return 0; } @@ -683,7 +683,7 @@ EXPORTED int msgrecord_set_internaldate(msgrecord_t *mr, time_t internaldate) int r = msgrecord_need(mr, M_RECORD); if (r) return r; } - mr->record.internaldate = internaldate; + mr->record.internaldate.tv_sec = internaldate; return 0; } diff --git a/imap/pop3d.c b/imap/pop3d.c index 0dde3569af..eeaf7f5846 100644 --- a/imap/pop3d.c +++ b/imap/pop3d.c @@ -1814,7 +1814,7 @@ int openinbox(void) while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (popd_mailbox->i.pop3_show_after && - record->internaldate <= popd_mailbox->i.pop3_show_after) { + record->internaldate.tv_sec <= popd_mailbox->i.pop3_show_after) { /* Ignore messages older than the "show after" date */ continue; } diff --git a/imap/sync_support.c b/imap/sync_support.c index 6461b3037f..02e264dff7 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1772,7 +1772,7 @@ int parse_upload(struct dlist *kr, struct mailbox *mailbox, return IMAP_PROTOCOL_BAD_PARAMETERS; if (!dlist_getlist(kr, "FLAGS", &fl)) return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!dlist_getdate(kr, "INTERNALDATE", &record->internaldate)) + if (!dlist_getdate(kr, "INTERNALDATE", &record->internaldate.tv_sec)) return IMAP_PROTOCOL_BAD_PARAMETERS; if (!dlist_getnum64(kr, "SIZE", &record->size)) return IMAP_PROTOCOL_BAD_PARAMETERS; @@ -2126,7 +2126,7 @@ static int sync_prepare_dlists(struct mailbox *mailbox, dlist_setnum64(il, "MODSEQ", mymodseq); dlist_setdate(il, "LAST_UPDATED", record->last_updated); sync_print_flags(il, mailbox, record); - dlist_setdate(il, "INTERNALDATE", record->internaldate); + dlist_setdate(il, "INTERNALDATE", record->internaldate.tv_sec); dlist_setnum32(il, "SIZE", record->size); dlist_setatom(il, "GUID", message_guid_encode(&record->guid)); @@ -2650,7 +2650,7 @@ static int sync_mailbox_compare_update(struct mailbox *mailbox, copy.basecid = mrecord.basecid; copy.modseq = mrecord.modseq; copy.last_updated = mrecord.last_updated; - copy.internaldate = mrecord.internaldate; + copy.internaldate.tv_sec = mrecord.internaldate.tv_sec; copy.savedate = mrecord.savedate; copy.createdmodseq = mrecord.createdmodseq; copy.system_flags = mrecord.system_flags; @@ -5810,7 +5810,7 @@ static void log_record(const char *name, struct mailbox *mailbox, syslog(LOG_NOTICE, "SYNCNOTICE: %s uid:%u modseq:" MODSEQ_FMT " " "last_updated:" TIME_T_FMT " internaldate:" TIME_T_FMT " flags:(%s) cid:" CONV_FMT, name, record->uid, record->modseq, - record->last_updated, record->internaldate, + record->last_updated, record->internaldate.tv_sec, make_flags(mailbox, record), record->cid); } @@ -5881,7 +5881,7 @@ static int compare_one_record(struct sync_client_state *sync_cs, goto diff; if (mp->last_updated != rp->last_updated) goto diff; - if (mp->internaldate != rp->internaldate) + if (mp->internaldate.tv_sec != rp->internaldate.tv_sec) goto diff; if (mp->system_flags != rp->system_flags) goto diff; diff --git a/imap/unexpunge.c b/imap/unexpunge.c index ef4ccf2f0b..2da660914b 100644 --- a/imap/unexpunge.c +++ b/imap/unexpunge.c @@ -155,7 +155,7 @@ static void list_expunged(const char *mboxname, printf("UID: %u\n", record->uid); printf("\tSize: " UINT64_FMT "\n", record->size); printf("\tSent: %s", ctime(&record->sentdate)); - printf("\tRecv: %s", ctime(&record->internaldate)); + printf("\tRecv: %s", ctime(&record->internaldate.tv_sec)); printf("\tExpg: %s", ctime(&record->last_updated)); if (mailbox_cacherecord(mailbox, record)) { From 9af04c1dbb335373b458b7ae3cd070258fe0d627 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Thu, 23 Jan 2025 13:49:52 -0500 Subject: [PATCH 05/16] msgrecord.c: use a struct timespec for msgrecord_get/set_internaldate() --- imap/append.c | 14 ++++++++------ imap/jmap_mail.c | 12 +++++++----- imap/msgrecord.c | 10 ++++++---- imap/msgrecord.h | 4 ++-- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/imap/append.c b/imap/append.c index e060142c6a..575100295c 100644 --- a/imap/append.c +++ b/imap/append.c @@ -1057,8 +1057,6 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, msgrec = msgrecord_new(mailbox); r = msgrecord_set_uid(msgrec, uid); if (r) goto out; - r = msgrecord_set_internaldate(msgrec, internaldate); - if (r) goto out; r = msgrecord_set_createdmodseq(msgrec, createdmodseq); if (r) goto out; r = msgrecord_set_bodystructure(msgrec, *body); @@ -1069,9 +1067,9 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, } /* And make sure it has a timestamp */ - r = msgrecord_get_internaldate(msgrec, &internaldate); - if (!r && !internaldate) - r = msgrecord_set_internaldate(msgrec, time(NULL)); + struct timespec t = { internaldate, UTIME_NOW }; + if (!internaldate) t.tv_sec = time(NULL); + r = msgrecord_set_internaldate(msgrec, &t); if (r) goto out; /* should we archive it straight away? */ @@ -1268,7 +1266,11 @@ EXPORTED int append_fromstream(struct appendstate *as, struct body **body, msgrec = msgrecord_new(mailbox); r = msgrecord_set_uid(msgrec, as->baseuid + as->nummsg); if (r) goto out; - r = msgrecord_set_internaldate(msgrec, internaldate); + + /* And make sure it has a timestamp */ + struct timespec t = { internaldate, UTIME_NOW }; + if (!internaldate) t.tv_sec = time(NULL); + r = msgrecord_set_internaldate(msgrec, &t); if (r) goto out; /* Create message file */ diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index 3fa0ab3e9d..ac092ba7ad 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -7337,10 +7337,10 @@ static int _email_get_meta(jmap_req_t *req, /* receivedAt */ if (jmap_wantprop(props, "receivedAt")) { char datestr[RFC3339_DATETIME_MAX]; - time_t t; + struct timespec t; r = msgrecord_get_internaldate(msg->mr, &t); if (r) goto done; - time_to_rfc3339(t, datestr, RFC3339_DATETIME_MAX); + time_to_rfc3339(t.tv_sec, datestr, RFC3339_DATETIME_MAX); json_object_set_new(email, "receivedAt", json_string(datestr)); } @@ -13220,9 +13220,11 @@ static void _email_bulkupdate_exec_setflags(struct email_bulkupdate *bulk) if (update->received_at) { /* Write internaldate (Email/copy only) */ - time_t internaldate; - time_from_iso8601(update->received_at, &internaldate); - r = msgrecord_set_internaldate(mrw, internaldate); + struct timespec now, internaldate; + clock_gettime(CLOCK_REALTIME, &now); + internaldate.tv_nsec = now.tv_nsec; + time_from_iso8601(update->received_at, &internaldate.tv_sec); + r = msgrecord_set_internaldate(mrw, &internaldate); } /* Determine if to write the aggregated or updated JMAP keywords */ diff --git a/imap/msgrecord.c b/imap/msgrecord.c index 5ce6198fcd..a44d65c98f 100644 --- a/imap/msgrecord.c +++ b/imap/msgrecord.c @@ -285,13 +285,14 @@ EXPORTED int msgrecord_hasflag(msgrecord_t *mr, const char *flag, int *has) return 0; } -EXPORTED int msgrecord_get_internaldate(msgrecord_t *mr, time_t *t) +EXPORTED int msgrecord_get_internaldate(msgrecord_t *mr, struct timespec *t) { if (!mr->isappend) { int r = msgrecord_need(mr, M_RECORD); if (r) return r; } - *t = mr->record.internaldate.tv_sec; + t->tv_sec = mr->record.internaldate.tv_sec; + t->tv_nsec = mr->record.internaldate.tv_nsec; return 0; } @@ -677,13 +678,14 @@ EXPORTED int msgrecord_set_cache_offset(msgrecord_t *mr, size_t offset) return 0; } -EXPORTED int msgrecord_set_internaldate(msgrecord_t *mr, time_t internaldate) +EXPORTED int msgrecord_set_internaldate(msgrecord_t *mr, struct timespec *internaldate) { if (!mr->isappend) { int r = msgrecord_need(mr, M_RECORD); if (r) return r; } - mr->record.internaldate.tv_sec = internaldate; + mr->record.internaldate.tv_sec = internaldate->tv_sec; + mr->record.internaldate.tv_nsec = internaldate->tv_nsec; return 0; } diff --git a/imap/msgrecord.h b/imap/msgrecord.h index aa8791d6cf..0752f9f5fa 100644 --- a/imap/msgrecord.h +++ b/imap/msgrecord.h @@ -71,7 +71,7 @@ extern int msgrecord_get_uid(msgrecord_t *mr, uint32_t *uid); extern int msgrecord_get_messageid(msgrecord_t *mr, struct buf *buf); extern int msgrecord_get_modseq(msgrecord_t *mr, modseq_t *modseq); extern int msgrecord_get_createdmodseq(msgrecord_t *mr, modseq_t *modseq); -extern int msgrecord_get_internaldate(msgrecord_t *mr, time_t *t); +extern int msgrecord_get_internaldate(msgrecord_t *mr, struct timespec *t); extern int msgrecord_get_savedate(msgrecord_t *mr, time_t *t); extern int msgrecord_get_lastupdated(msgrecord_t *mr, time_t *t); extern int msgrecord_get_message(msgrecord_t *mr, message_t **msg); @@ -107,7 +107,7 @@ extern int msgrecord_set_userflag(msgrecord_t *mr, uint32_t user_flag, int bit); extern int msgrecord_set_uid(msgrecord_t *mr, uint32_t uid); extern int msgrecord_set_cid(msgrecord_t *mr, bit64 cid); extern int msgrecord_set_bodystructure(msgrecord_t *mr, struct body *body); -extern int msgrecord_set_internaldate(msgrecord_t *mr, time_t internaldate); +extern int msgrecord_set_internaldate(msgrecord_t *mr, struct timespec *internaldate); extern int msgrecord_set_savedate(msgrecord_t *mr, time_t savedate); extern int msgrecord_set_cache_offset(msgrecord_t *mr, size_t offset); extern int msgrecord_set_createdmodseq(msgrecord_t *mr, modseq_t modseq); From 123675d090bc7610019946c1077e4eeb9bd2f757 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Wed, 22 Jan 2025 16:56:15 -0500 Subject: [PATCH 06/16] append.c: use a struct timespec for append_from*() --- cunit/annotate.testc | 8 +++++--- imap/append.c | 22 ++++++++++++++-------- imap/append.h | 4 ++-- imap/caldav_alarm.c | 2 +- imap/carddav_db.c | 10 ++++++---- imap/cyr_virusscan.c | 8 ++++---- imap/dav_util.c | 2 +- imap/http_jmap.c | 12 +++++++----- imap/imapd.c | 9 ++++++--- imap/jmap_backup.c | 3 ++- imap/jmap_mail.c | 37 ++++++++++++++++++++++--------------- imap/jmap_mail_submission.c | 21 +++++++++++++-------- imap/jmap_notes.c | 12 +++++++----- imap/jmap_notif.c | 24 +++++++++++++----------- imap/lmtp_sieve.c | 9 ++++++--- imap/lmtpd.c | 11 +++++++---- imap/sieve_db.c | 2 +- 17 files changed, 117 insertions(+), 79 deletions(-) diff --git a/cunit/annotate.testc b/cunit/annotate.testc index 39de2bce62..3d185567ee 100644 --- a/cunit/annotate.testc +++ b/cunit/annotate.testc @@ -2002,12 +2002,14 @@ static int create_messages(struct mailbox *mailbox, int count) struct appendstate as; quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_DONTCARE_INITIALIZER; FILE *fp; - time_t internaldate = time(NULL); + struct timespec internaldate; struct body *body = NULL; struct buf buf = BUF_INITIALIZER; + clock_gettime(CLOCK_REALTIME, &internaldate); + /* Write the message to the filesystem */ - if (!(fp = append_newstage(mailbox_name(mailbox), internaldate, 0, &stage))) { + if (!(fp = append_newstage(mailbox_name(mailbox), internaldate.tv_sec, 0, &stage))) { fprintf(stderr, "append_newstage(%s) failed", mailbox_name(mailbox)); return IMAP_IOERROR; } @@ -2028,7 +2030,7 @@ static int create_messages(struct mailbox *mailbox, int count) strerror(errno)); return r; } - r = append_fromstage(&as, &body, stage, internaldate, 0, NULL, 0, NULL); + r = append_fromstage(&as, &body, stage, &internaldate, 0, NULL, 0, NULL); if (r) { fprintf(stderr, "append_fromstage(%s) failed: %s", mailbox_name(mailbox), error_message(r)); diff --git a/imap/append.c b/imap/append.c index 575100295c..7dfe36baad 100644 --- a/imap/append.c +++ b/imap/append.c @@ -935,7 +935,7 @@ static int findstage_cb(const conv_guidrec_t *rec, void *vrock) */ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, struct stagemsg *stage, - time_t internaldate, time_t savedate, + struct timespec *internaldate, time_t savedate, modseq_t createdmodseq, const strarray_t *flags, int nolink, struct entryattlist **user_annotsp) @@ -1067,9 +1067,12 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, } /* And make sure it has a timestamp */ - struct timespec t = { internaldate, UTIME_NOW }; - if (!internaldate) t.tv_sec = time(NULL); - r = msgrecord_set_internaldate(msgrec, &t); + struct timespec now; + if (!internaldate) { + clock_gettime(CLOCK_REALTIME, &now); + internaldate = &now; + } + r = msgrecord_set_internaldate(msgrec, internaldate); if (r) goto out; /* should we archive it straight away? */ @@ -1250,7 +1253,7 @@ EXPORTED int append_removestage(struct stagemsg *stage) EXPORTED int append_fromstream(struct appendstate *as, struct body **body, struct protstream *messagefile, unsigned long size, - time_t internaldate, + struct timespec *internaldate, const strarray_t *flags) { struct mailbox *mailbox = as->mailbox; @@ -1268,9 +1271,12 @@ EXPORTED int append_fromstream(struct appendstate *as, struct body **body, if (r) goto out; /* And make sure it has a timestamp */ - struct timespec t = { internaldate, UTIME_NOW }; - if (!internaldate) t.tv_sec = time(NULL); - r = msgrecord_set_internaldate(msgrec, &t); + struct timespec now; + if (!internaldate) { + clock_gettime(CLOCK_REALTIME, &now); + internaldate = &now; + } + r = msgrecord_set_internaldate(msgrec, internaldate); if (r) goto out; /* Create message file */ diff --git a/imap/append.h b/imap/append.h index 27eaf6d89d..b7684dc202 100644 --- a/imap/append.h +++ b/imap/append.h @@ -121,7 +121,7 @@ extern FILE *append_newstage_full(const char *mailboxname, time_t internaldate, /* adds a new mailbox to the stage initially created by append_newstage() */ extern int append_fromstage_full(struct appendstate *mailbox, struct body **body, struct stagemsg *stage, - time_t internaldate, time_t savedate, + struct timespec *internaldate, time_t savedate, modseq_t createdmodseq, const strarray_t *flags, int nolink, struct entryattlist **annotations); @@ -133,7 +133,7 @@ extern int append_removestage(struct stagemsg *stage); extern int append_fromstream(struct appendstate *as, struct body **body, struct protstream *messagefile, - unsigned long size, time_t internaldate, + unsigned long size, struct timespec *internaldate, const strarray_t *flags); extern int append_copy(struct mailbox *mailbox, diff --git a/imap/caldav_alarm.c b/imap/caldav_alarm.c index 54d00c1413..b4ca83d6f8 100644 --- a/imap/caldav_alarm.c +++ b/imap/caldav_alarm.c @@ -1521,7 +1521,7 @@ static int move_to_mailboxid(struct mailbox *srcmbox, if (r) goto done; /* Append the message to the mailbox */ - r = append_fromstage_full(&as, &body, stage, record->internaldate.tv_sec, + r = append_fromstage_full(&as, &body, stage, &record->internaldate, savedate, 0, flags, 0, &annots); if (r) { append_abort(&as); diff --git a/imap/carddav_db.c b/imap/carddav_db.c index 78dd5d6d3f..567177ef98 100644 --- a/imap/carddav_db.c +++ b/imap/carddav_db.c @@ -1210,7 +1210,7 @@ static int _carddav_store(struct mailbox *mailbox, struct buf *vcard, char *header; quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_DONTCARE_INITIALIZER; struct appendstate as; - time_t now = time(0); + struct timespec now; char *freeme = NULL; char datestr[80]; static int64_t vcard_max_size = -1; @@ -1223,8 +1223,10 @@ static int _carddav_store(struct mailbox *mailbox, struct buf *vcard, init_internal(); + clock_gettime(CLOCK_REALTIME, &now); + /* Prepare to stage the message */ - if (!(f = append_newstage(mailbox_name(mailbox), now, 0, &stage))) { + if (!(f = append_newstage(mailbox_name(mailbox), now.tv_sec, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mailbox)); return -1; } @@ -1243,7 +1245,7 @@ static int _carddav_store(struct mailbox *mailbox, struct buf *vcard, if (!resource) resource = freeme = strconcat(uid, ".vcf", (char *)NULL); mbuserid = mboxname_to_userid(mailbox_name(mailbox)); - time_to_rfc5322(now, datestr, sizeof(datestr)); + time_to_rfc5322(now.tv_sec, datestr, sizeof(datestr)); /* XXX This needs to be done via an LDAP/DB lookup */ header = charset_encode_mimeheader(mbuserid, 0, 0); @@ -1299,7 +1301,7 @@ static int _carddav_store(struct mailbox *mailbox, struct buf *vcard, struct body *body = NULL; - r = append_fromstage(&as, &body, stage, now, createdmodseq, flags, 0, annots); + r = append_fromstage(&as, &body, stage, &now, createdmodseq, flags, 0, annots); if (body) { message_free_body(body); free(body); diff --git a/imap/cyr_virusscan.c b/imap/cyr_virusscan.c index 8c8996f045..df92ffb1d1 100644 --- a/imap/cyr_virusscan.c +++ b/imap/cyr_virusscan.c @@ -717,7 +717,7 @@ static void append_notifications(const struct buf *template) if (i_mbox->msgs) { FILE *f = NULL; struct infected_msg *msg; - time_t t; + struct timespec now; struct protstream *pout; struct appendstate as; struct body *body = NULL; @@ -739,8 +739,8 @@ static void append_notifications(const struct buf *template) goto user_done; } - t = time(NULL); - put_notification_headers(f, outgoing_count++, t, owner); + clock_gettime(CLOCK_REALTIME, &now); + put_notification_headers(f, outgoing_count++, now.tv_sec, owner); first = 1; while ((msg = i_mbox->msgs)) { @@ -804,7 +804,7 @@ static void append_notifications(const struct buf *template) if (!r) { pout = prot_new(fd, 0); prot_rewind(pout); - r = append_fromstream(&as, &body, pout, msgsize, t, NULL); + r = append_fromstream(&as, &body, pout, msgsize, &now, NULL); /* n.b. append_fromstream calls append_abort itself if it fails */ if (!r) r = append_commit(&as); diff --git a/imap/dav_util.c b/imap/dav_util.c index 33150a71c5..3d6f014078 100644 --- a/imap/dav_util.c +++ b/imap/dav_util.c @@ -257,7 +257,7 @@ EXPORTED int dav_store_resource(struct transaction_t *txn, } /* Append the message to the mailbox */ - if ((r = append_fromstage(&as, &body, stage, now, + if ((r = append_fromstage(&as, &body, stage, NULL, createdmodseq, flaglist, 0, &annots))) { syslog(LOG_ERR, "append_fromstage(%s) failed: %s", mailbox_name(mailbox), error_message(r)); diff --git a/imap/http_jmap.c b/imap/http_jmap.c index 3f4c3fb498..8b17d60c5b 100644 --- a/imap/http_jmap.c +++ b/imap/http_jmap.c @@ -1026,7 +1026,7 @@ static int jmap_upload(struct transaction_t *txn) struct stagemsg *stage = NULL; FILE *f = NULL; const char **hdr; - time_t now = time(NULL); + struct timespec now; struct appendstate as; char *accountid = NULL; char *normalisedtype = NULL; @@ -1080,8 +1080,10 @@ static int jmap_upload(struct transaction_t *txn) goto done; } + clock_gettime(CLOCK_REALTIME, &now); + /* Prepare to stage the message */ - if (!(f = append_newstage(mailbox_name(mailbox), now, 0, &stage))) { + if (!(f = append_newstage(mailbox_name(mailbox), now.tv_sec, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mailbox)); txn->error.desc = "append_newstage() failed"; ret = HTTP_SERVER_ERROR; @@ -1157,7 +1159,7 @@ static int jmap_upload(struct transaction_t *txn) } else { char datestr[80]; - time_to_rfc5322(now, datestr, sizeof(datestr)); + time_to_rfc5322(now.tv_sec, datestr, sizeof(datestr)); fprintf(f, "Date: %s\r\n", datestr); } @@ -1204,7 +1206,7 @@ static int jmap_upload(struct transaction_t *txn) /* Append the message to the mailbox */ strarray_append(&flags, "\\Deleted"); strarray_append(&flags, "\\Expunged"); // custom flag to insta-expunge! - r = append_fromstage(&as, &body, stage, now, 0, &flags, 0, /*annots*/NULL); + r = append_fromstage(&as, &body, stage, &now, 0, &flags, 0, /*annots*/NULL); if (r) { append_abort(&as); @@ -1232,7 +1234,7 @@ static int jmap_upload(struct transaction_t *txn) } char datestr[RFC3339_DATETIME_MAX]; - time_to_rfc3339(now + 86400, datestr, RFC3339_DATETIME_MAX); + time_to_rfc3339(now.tv_sec + 86400, datestr, RFC3339_DATETIME_MAX); char blob_id[JMAP_BLOBID_SIZE]; jmap_set_blobid(rawmessage ? &body->guid : &body->content_guid, blob_id); diff --git a/imap/imapd.c b/imap/imapd.c index 12e4210140..054bf2725c 100644 --- a/imap/imapd.c +++ b/imap/imapd.c @@ -254,7 +254,7 @@ struct appendstage { struct stagemsg *stage; FILE *f; strarray_t flags; - time_t internaldate; + struct timespec internaldate; int binary; struct entryattlist *annotations; }; @@ -4318,6 +4318,9 @@ static int cmd_append(char *tag, char *name, const char *cur_name, int isreplace curstage = xzmalloc(sizeof(*curstage)); ptrarray_push(&stages, curstage); + /* Initialize the internaldate to "now" */ + clock_gettime(CLOCK_REALTIME, &curstage->internaldate); + /* Set limit on the total number of bytes allowed for mailbox+append-opts */ maxargssize_mark = prot_bytes_in(imapd_in) + (maxargssize - strlen(name)); @@ -4357,7 +4360,7 @@ static int cmd_append(char *tag, char *name, const char *cur_name, int isreplace /* Parse internaldate */ if (c == '\"' && !arg.s[0]) { prot_ungetc(c, imapd_in); - c = getdatetime(&(curstage->internaldate)); + c = getdatetime(&(curstage->internaldate.tv_sec)); if (c != ' ') { parseerr = "Invalid date-time in Append command"; r = IMAP_PROTOCOL_ERROR; @@ -4525,7 +4528,7 @@ static int cmd_append(char *tag, char *name, const char *cur_name, int isreplace } if (!r) { r = append_fromstage(&appendstate, &body, curstage->stage, - curstage->internaldate, /*createdmodseq*/0, + &curstage->internaldate, /*createdmodseq*/0, &curstage->flags, 0, &curstage->annotations); } diff --git a/imap/jmap_backup.c b/imap/jmap_backup.c index 0fd02fc661..97c55c68c1 100644 --- a/imap/jmap_backup.c +++ b/imap/jmap_backup.c @@ -562,7 +562,8 @@ static int recreate_resource(message_t *msg, struct mailbox *tomailbox, strarray_add(flags, "$restored"); /* append the message to the mailbox. */ - r = append_fromstage(&as, &body, stage, record->internaldate.tv_sec, + r = append_fromstage(&as, &body, stage, + (struct timespec *) &record->internaldate, is_update ? record->createdmodseq : 0, flags, /*nolink*/0, &annots); diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index ac092ba7ad..dbae6e6195 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -8965,7 +8965,7 @@ struct email_append_detail { static void _email_append(jmap_req_t *req, json_t *mailboxids, strarray_t *keywords, - time_t internaldate, + struct timespec *internaldate, json_t *snoozed, int has_attachment, const char *sourcefile, @@ -8987,6 +8987,7 @@ static void _email_append(jmap_req_t *req, size_t len; int r = 0; time_t savedate = 0; + struct timespec now; if (json_object_size(mailboxids) > JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL) { *err = json_pack("{s:s}", "type", "tooManyMailboxes"); @@ -8997,7 +8998,8 @@ static void _email_append(jmap_req_t *req, goto done; } - if (!internaldate) internaldate = time(NULL); + clock_gettime(CLOCK_REALTIME, &now); + if (!internaldate) internaldate = &now; /* Pick the mailbox to create the message in, prefer \Snoozed then \Drafts */ mailboxes = json_object(); /* maps mailbox ids to mboxnames */ @@ -9072,7 +9074,7 @@ static void _email_append(jmap_req_t *req, if (r) goto done; if (sourcefile) { - if (!(f = append_newstage_full(mailbox_name(mbox), internaldate, 0, &stage, sourcefile))) { + if (!(f = append_newstage_full(mailbox_name(mbox), internaldate->tv_sec, 0, &stage, sourcefile))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mbox)); r = HTTP_SERVER_ERROR; goto done; @@ -9080,7 +9082,7 @@ static void _email_append(jmap_req_t *req, } else { /* Write the message to the filesystem */ - if (!(f = append_newstage(mailbox_name(mbox), internaldate, 0, &stage))) { + if (!(f = append_newstage(mailbox_name(mbox), internaldate->tv_sec, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mbox)); r = HTTP_SERVER_ERROR; goto done; @@ -9300,7 +9302,7 @@ struct email { struct headers headers; /* parsed headers */ json_t *jemail; /* original Email JSON object */ struct emailpart *body; /* top-level MIME part */ - time_t internaldate; /* RFC 3501 internaldate aka receivedAt */ + struct timespec internaldate; /* RFC 3501 internaldate aka receivedAt */ int has_attachment; /* set the HasAttachment flag */ json_t *snoozed; /* set snoozed annotation */ }; @@ -10539,9 +10541,10 @@ static void _parse_email(jmap_req_t *req, jmap_parser_invalid(parser, "sentAt"); } /* receivedAt */ + clock_gettime(CLOCK_REALTIME, &email->internaldate); prop = json_object_get(jemail, "receivedAt"); if (json_is_utcdate(prop)) { - time_from_iso8601(json_string_value(prop), &email->internaldate); + time_from_iso8601(json_string_value(prop), &email->internaldate.tv_sec); } else if (JNOTNULL(prop)) { jmap_parser_invalid(parser, "receivedAt"); @@ -11289,7 +11292,7 @@ static void _email_create(jmap_req_t *req, /* Parse Email object into internal representation */ struct jmap_parser parser = JMAP_PARSER_INITIALIZER; - struct email email = { HEADERS_INITIALIZER, NULL, NULL, time(NULL), 0, NULL }; + struct email email = { HEADERS_INITIALIZER, NULL, NULL, { 0 }, 0, NULL }; _parse_email(req, jemail, &parser, &email); /* Validate mailboxIds */ @@ -11325,7 +11328,7 @@ static void _email_create(jmap_req_t *req, /* Append MIME-encoded Email to mailboxes and write keywords */ - _email_append(req, jmailboxids, &keywords, email.internaldate, email.snoozed, + _email_append(req, jmailboxids, &keywords, &email.internaldate, email.snoozed, config_getswitch(IMAPOPT_JMAP_SET_HAS_ATTACHMENT) ? email.has_attachment : 0, NULL, _email_to_mime, &email, &detail, set_err); @@ -13895,24 +13898,28 @@ static void _email_import(jmap_req_t *req, } /* set receivedAt property */ - time_t internaldate = 0; + struct timespec now, internaldate; const char *received_at = json_string_value(json_object_get(jemail_import, "receivedAt")); + + clock_gettime(CLOCK_REALTIME, &now); + internaldate.tv_nsec = now.tv_nsec; + if (received_at) { - time_from_iso8601(received_at, &internaldate); + time_from_iso8601(received_at, &internaldate.tv_sec); } else { - internaldate = _email_import_parse_received_at(buf_base(&content), - buf_len(&content)); + internaldate.tv_sec = _email_import_parse_received_at(buf_base(&content), + buf_len(&content)); } - if (!internaldate) - internaldate = time(NULL); + if (!internaldate.tv_sec) + internaldate.tv_sec = now.tv_sec; // mailbox will be readonly, drop the lock so it can be make writable if (mbox) mailbox_unlock_index(mbox, NULL); /* Write the message to the file system */ - _email_append(req, jmailbox_ids, &keywords, internaldate, snoozed, + _email_append(req, jmailbox_ids, &keywords, &internaldate, snoozed, has_attachment, sourcefile, _email_import_cb, &content, &detail, err); msgrecord_unref(&mr); diff --git a/imap/jmap_mail_submission.c b/imap/jmap_mail_submission.c index 41c6b50e60..8414ab407c 100644 --- a/imap/jmap_mail_submission.c +++ b/imap/jmap_mail_submission.c @@ -372,13 +372,15 @@ static int store_submission(jmap_req_t *req, struct mailbox *mailbox, size_t msglen = buf_len(msg); FILE *f = NULL; int r; - time_t now = time(0); - time_t internaldate = holduntil; + struct timespec now, internaldate; + + clock_gettime(CLOCK_REALTIME, &now); + internaldate.tv_nsec = now.tv_nsec; if (!holduntil) { /* Already sent */ msglen = 0; - internaldate = now; + internaldate.tv_sec = now.tv_sec; strarray_append(&flags, "\\Answered"); if (config_getswitch(IMAPOPT_JMAPSUBMISSION_DELETEONSEND)) { /* delete the EmailSubmission object immediately */ @@ -387,16 +389,19 @@ static int store_submission(jmap_req_t *req, struct mailbox *mailbox, strarray_append(&flags, "\\Expunged"); } } + else { + internaldate.tv_sec = holduntil; + } /* Prepare to stage the message */ - if (!(f = append_newstage(mailbox_name(mailbox), internaldate, 0, &stage))) { + if (!(f = append_newstage(mailbox_name(mailbox), internaldate.tv_sec, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mailbox)); r = IMAP_IOERROR; goto done; } /* Stage the message to send as message/rfc822 */ - time_to_rfc5322(now, datestr, sizeof(datestr)); + time_to_rfc5322(now.tv_sec, datestr, sizeof(datestr)); if (strchr(httpd_userid, '@')) { /* XXX This needs to be done via an LDAP/DB lookup */ @@ -449,7 +454,7 @@ static int store_submission(jmap_req_t *req, struct mailbox *mailbox, } /* Append the message to the mailbox */ - r = append_fromstage_full(&as, &body, stage, internaldate, now, + r = append_fromstage_full(&as, &body, stage, &internaldate, now.tv_sec, /*cmodseq*/0, &flags, /*nolink*/0, /*annots*/NULL); if (r) { @@ -471,7 +476,7 @@ static int store_submission(jmap_req_t *req, struct mailbox *mailbox, sprintf(sub_id, "S%u", mailbox->i.last_uid); char sendat[RFC3339_DATETIME_MAX]; - time_to_rfc3339(internaldate, sendat, RFC3339_DATETIME_MAX); + time_to_rfc3339(internaldate.tv_sec, sendat, RFC3339_DATETIME_MAX); // XXX: we should include all the other fields from the spec *new_submission = json_pack("{s:s, s:s, s:s}", @@ -482,7 +487,7 @@ static int store_submission(jmap_req_t *req, struct mailbox *mailbox, if (jmap_is_using(req, JMAP_MAIL_EXTENSION)) { char created[RFC3339_DATETIME_MAX]; - time_to_rfc3339(now, created, RFC3339_DATETIME_MAX); + time_to_rfc3339(now.tv_sec, created, RFC3339_DATETIME_MAX); json_object_set_new(*new_submission, "created", json_string(created)); } diff --git a/imap/jmap_notes.c b/imap/jmap_notes.c index 2c91743c73..c0ffb12d42 100644 --- a/imap/jmap_notes.c +++ b/imap/jmap_notes.c @@ -566,12 +566,14 @@ static int _note_create(struct mailbox *mailbox, json_t *note, json_t **new_note char datestr[80], *from, *title, *body; FILE *f = NULL; int r = 0, isFlagged = 0, isHTML = 0, qpencode = 0; - time_t now = time(0); + struct timespec now; json_t *prop; const char *uid = NULL, *created = NULL; + clock_gettime(CLOCK_REALTIME, &now); + /* Prepare to stage the message */ - if (!(f = append_newstage(mailbox_name(mailbox), now, 0, &stage))) { + if (!(f = append_newstage(mailbox_name(mailbox), now.tv_sec, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mailbox)); r = IMAP_IOERROR; goto done; @@ -586,10 +588,10 @@ static int _note_create(struct mailbox *mailbox, json_t *note, json_t **new_note json_object_set_new(*new_note, "id", json_string(uid)); } - time_to_rfc3339(now, datestr, sizeof(datestr)); + time_to_rfc3339(now.tv_sec, datestr, sizeof(datestr)); json_object_set_new(*new_note, "lastSaved", json_string(datestr)); - time_to_rfc5322(now, datestr, sizeof(datestr)); + time_to_rfc5322(now.tv_sec, datestr, sizeof(datestr)); prop = json_object_get(note, "created"); if (prop) created = json_string_value(prop); @@ -679,7 +681,7 @@ static int _note_create(struct mailbox *mailbox, json_t *note, json_t **new_note /* Append the message to the mailbox */ if (isFlagged) strarray_append(&flags, "\\Flagged"); - r = append_fromstage(&as, &bodypart, stage, now, 0, + r = append_fromstage(&as, &bodypart, stage, &now, 0, &flags, 0, /*annots*/NULL); if (r) { diff --git a/imap/jmap_notif.c b/imap/jmap_notif.c index a767172982..d6cfe9e00e 100644 --- a/imap/jmap_notif.c +++ b/imap/jmap_notif.c @@ -140,7 +140,7 @@ static int append_eventnotif(const char *from, const struct auth_state *authstate, struct mailbox *notifmbox, const char *calmboxname, - time_t created, + struct timespec *created, json_t *jnotif) { struct stagemsg *stage = NULL; @@ -206,7 +206,7 @@ static int append_eventnotif(const char *from, buf_reset(&buf); // Append new notification. - FILE *fp = append_newstage(mailbox_name(notifmbox), created, + FILE *fp = append_newstage(mailbox_name(notifmbox), created->tv_sec, strhash(ical_uid), &stage); if (!fp) { xsyslog(LOG_ERR, "append_newstage failed", "name=%s", mailbox_name(notifmbox)); @@ -221,18 +221,18 @@ static int append_eventnotif(const char *from, fputs("Subject: " JMAP_NOTIF_CALENDAREVENT "\r\n", fp); char date5322[RFC5322_DATETIME_MAX+1]; - time_to_rfc5322(created, date5322, RFC5322_DATETIME_MAX); + time_to_rfc5322(created->tv_sec, date5322, RFC5322_DATETIME_MAX); fputs("Date: ", fp); fputs(date5322, fp); fputs("\r\n", fp); fprintf(fp, "Message-ID: <%s-" TIME_T_FMT "@%s>\r\n", - makeuuid(), created, config_servername); + makeuuid(), created->tv_sec, config_servername); fputs("Content-Type: application/json; charset=utf-8\r\n", fp); fputs("Content-Transfer-Encoding: 8bit\r\n", fp); struct dlist *dl = dlist_newkvlist(NULL, "N"); - dlist_setdate(dl, "S", created); + dlist_setdate(dl, "S", created->tv_sec); dlist_setatom(dl, "T", JMAP_NOTIF_CALENDAREVENT); dlist_setatom(dl, "ID", ical_uid); dlist_setatom(dl, "NT", type); @@ -354,7 +354,8 @@ HIDDEN int jmap_create_caleventnotif(struct mailbox *notifmbox, return 0; } - time_t now = time(NULL); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); const char *byemail = schedule_addresses ? strarray_nth(schedule_addresses, 0) : NULL; @@ -365,13 +366,13 @@ HIDDEN int jmap_create_caleventnotif(struct mailbox *notifmbox, annotatemore_lookupmask(calhomename, annotname, userid, &byname); free(calhomename); - json_t *jnotif = build_eventnotif(type, now, userid, + json_t *jnotif = build_eventnotif(type, now.tv_sec, userid, buf_cstring(&byname), byemail, ical_uid, comment, is_draft, jevent, jpatch); char *from = jmap_caleventnotif_format_fromheader(userid); int r = append_eventnotif(from, userid, authstate, notifmbox, - calmboxname, now, jnotif); + calmboxname, &now, jnotif); free(from); json_decref(jnotif); @@ -393,7 +394,7 @@ HIDDEN int jmap_create_caldaveventnotif(struct transaction_t *txn, const char *accountid = mbname_userid(mbname); struct mailbox *notifmbox = NULL; mbentry_t *notifmb = NULL; - time_t now = time(NULL); + struct timespec now; json_t *jevent = NULL; json_t *jpatch = NULL; int r = 0; @@ -470,12 +471,13 @@ HIDDEN int jmap_create_caldaveventnotif(struct transaction_t *txn, byemail = strarray_nth(schedule_addresses, 0); } - json_t *jnotif = build_eventnotif(type, now, + clock_gettime(CLOCK_REALTIME, &now); + json_t *jnotif = build_eventnotif(type, now.tv_sec, byprincipal, buf_cstring(&byname), byemail, ical_uid, NULL, is_draft, jevent, jpatch); r = append_eventnotif(from, userid, authstate, notifmbox, - calmboxname, now, jnotif); + calmboxname, &now, jnotif); json_decref(jnotif); buf_free(&byname); diff --git a/imap/lmtp_sieve.c b/imap/lmtp_sieve.c index 7636d3870b..cc3f3207c8 100644 --- a/imap/lmtp_sieve.c +++ b/imap/lmtp_sieve.c @@ -1986,10 +1986,13 @@ static void do_fcc(script_data_t *sdata, sieve_fileinto_context_t *fcc, } if (!r) { struct stagemsg *stage; + struct timespec internaldate; + + clock_gettime(CLOCK_REALTIME, &internaldate); /* post-date by 1 sec in an effort to have the FCC threaded AFTER the incoming message */ - time_t internaldate = time(NULL) + 1; - FILE *f = append_newstage(intname, internaldate, + internaldate.tv_sec += 1; + FILE *f = append_newstage(intname, internaldate.tv_sec, strhash(intname) /* unique msgnum for reply */, &stage); if (f) { @@ -1999,7 +2002,7 @@ static void do_fcc(script_data_t *sdata, sieve_fileinto_context_t *fcc, fclose(f); r = append_fromstage(&as, &body, stage, - internaldate, /* createdmodseq */ 0, + &internaldate, /* createdmodseq */ 0, fcc->imapflags, 0, /* annotations */ NULL); if (!r) r = append_commit(&as); diff --git a/imap/lmtpd.c b/imap/lmtpd.c index 697acf79a0..4bf5f6f503 100644 --- a/imap/lmtpd.c +++ b/imap/lmtpd.c @@ -528,7 +528,7 @@ int deliver_mailbox(FILE *f, char *uuid = NULL; duplicate_key_t dkey = DUPLICATE_INITIALIZER; quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_DONTCARE_INITIALIZER; - time_t internaldate = 0; + struct timespec internaldate = { 0 }; /* make sure we have an IMAP mailbox */ if (mboxname_isnondeliverymailbox(mailboxname, 0/*mbtype*/)) { @@ -585,15 +585,18 @@ int deliver_mailbox(FILE *f, r = message_parse_file_buf(f, &content->map, &content->body, NULL); } + /* initialize internaldate to "now" */ + clock_gettime(CLOCK_REALTIME, &internaldate); + /* if the body contains an x-deliveredinternaldate then that overrides all else */ if (content->body->x_deliveredinternaldate) { time_from_rfc5322(content->body->x_deliveredinternaldate, - &internaldate, DATETIME_FULL); + &internaldate.tv_sec, DATETIME_FULL); } /* Otherwise we'll use a received date if there's one */ else if (content->body->received_date) { time_from_rfc5322(content->body->received_date, - &internaldate, DATETIME_FULL); + &internaldate.tv_sec, DATETIME_FULL); } if (!r) { @@ -613,7 +616,7 @@ int deliver_mailbox(FILE *f, } r = append_fromstage_full(&as, &content->body, stage, - internaldate, savedate, /*createdmodseq*/0, + &internaldate, savedate, /*createdmodseq*/0, flags, !singleinstance, &annotations); if (r) { diff --git a/imap/sieve_db.c b/imap/sieve_db.c index 315c1610fc..9b0a43c1d7 100644 --- a/imap/sieve_db.c +++ b/imap/sieve_db.c @@ -640,7 +640,7 @@ static int store_script(struct mailbox *mailbox, struct sieve_data *sdata, else { struct body *body = NULL; - r = append_fromstage(&as, &body, stage, now, + r = append_fromstage(&as, &body, stage, NULL, sdata->createdmodseq, &flags, 0, NULL); if (body) { message_free_body(body); From 371bf19c462962b5c2dc9494aaec8cf1a1a3d94d Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Thu, 23 Jan 2025 08:33:53 -0500 Subject: [PATCH 07/16] reduce the number of strcmp and numcmp when handling virtual annots --- imap/mailbox.c | 81 +++++++++++++++------ imap/sync_support.c | 173 +++++++++++++++++++++++++++----------------- 2 files changed, 166 insertions(+), 88 deletions(-) diff --git a/imap/mailbox.c b/imap/mailbox.c index 7ffd9667d4..d35f26ae19 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -3449,17 +3449,33 @@ static uint32_t crc_annot(unsigned int uid, const char *entry, static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry) { - if (mailbox->i.minor_version < 13) return 0; - // thrid was introduced in v13 - if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) return 1; + switch (mailbox->i.minor_version) { + case 20: + case 19: + case 18: + case 17: + case 16: + // createdmodseq was introduced in v16 + if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) return 1; - if (mailbox->i.minor_version < 15) return 0; - // savedate was introduced in v15 - if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) return 1; + GCC_FALLTHROUGH - if (mailbox->i.minor_version < 16) return 0; - // createdmodseq was introduced in v16 - if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) return 1; + case 15: + // savedate was introduced in v15 + if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) return 1; + + GCC_FALLTHROUGH + + case 14: + case 13: + // thrid was introduced in v13 + if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) return 1; + + GCC_FALLTHROUGH + + default: + break; + } return 0; } @@ -3474,22 +3490,41 @@ static uint32_t crc_virtannot(struct mailbox *mailbox, uint32_t crc = 0; struct buf buf = BUF_INITIALIZER; - if (record->cid && mailbox->i.minor_version >= 13) { - buf_printf(&buf, CONV_FMT, record->cid); - crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "thrid", "", &buf); - buf_reset(&buf); - } + switch (mailbox->i.minor_version) { + case 20: + case 19: + case 18: + case 17: + case 16: + if (record->createdmodseq) { + buf_printf(&buf, MODSEQ_FMT, record->createdmodseq); + crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); + buf_reset(&buf); + } - if (record->savedate && mailbox->i.minor_version >= 15) { - buf_printf(&buf, TIME_T_FMT, record->savedate); - crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "savedate", "", &buf); - buf_reset(&buf); - } + GCC_FALLTHROUGH + + case 15: + if (record->savedate) { + buf_printf(&buf, TIME_T_FMT, record->savedate); + crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "savedate", "", &buf); + buf_reset(&buf); + } - if (record->createdmodseq && mailbox->i.minor_version >= 16) { - buf_printf(&buf, MODSEQ_FMT, record->createdmodseq); - crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); - buf_reset(&buf); + GCC_FALLTHROUGH + + case 14: + case 13: + if (record->cid) { + buf_printf(&buf, CONV_FMT, record->cid); + crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "thrid", "", &buf); + buf_reset(&buf); + } + + GCC_FALLTHROUGH + + default: + break; } buf_free(&buf); diff --git a/imap/sync_support.c b/imap/sync_support.c index 02e264dff7..dea352a6a6 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1448,34 +1448,55 @@ void encode_annotations(struct dlist *parent, } } - if (record && record->cid && mailbox->i.minor_version >= 13) { - if (!annots) - annots = dlist_newlist(parent, "ANNOTATIONS"); - aa = dlist_newkvlist(annots, NULL); - dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "thrid"); - dlist_setatom(aa, "USERID", ""); - dlist_setnum64(aa, "MODSEQ", 0); - dlist_sethex64(aa, "VALUE", record->cid); - } + if (!record) return; + + switch (mailbox->i.minor_version) { + case 20: + case 19: + case 18: + case 17: + case 16: + if (record->createdmodseq) { + if (!annots) + annots = dlist_newlist(parent, "ANNOTATIONS"); + aa = dlist_newkvlist(annots, NULL); + dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "createdmodseq"); + dlist_setatom(aa, "USERID", ""); + dlist_setnum64(aa, "MODSEQ", 0); + dlist_setnum64(aa, "VALUE", record->createdmodseq); + } - if (record && record->savedate && mailbox->i.minor_version >= 15) { - if (!annots) - annots = dlist_newlist(parent, "ANNOTATIONS"); - aa = dlist_newkvlist(annots, NULL); - dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "savedate"); - dlist_setatom(aa, "USERID", ""); - dlist_setnum64(aa, "MODSEQ", 0); - dlist_setnum32(aa, "VALUE", record->savedate); - } + GCC_FALLTHROUGH + + case 15: + if (record->savedate) { + if (!annots) + annots = dlist_newlist(parent, "ANNOTATIONS"); + aa = dlist_newkvlist(annots, NULL); + dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "savedate"); + dlist_setatom(aa, "USERID", ""); + dlist_setnum64(aa, "MODSEQ", 0); + dlist_setnum32(aa, "VALUE", record->savedate); + } + + GCC_FALLTHROUGH + + case 14: + case 13: + if (record->cid) { + if (!annots) + annots = dlist_newlist(parent, "ANNOTATIONS"); + aa = dlist_newkvlist(annots, NULL); + dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "thrid"); + dlist_setatom(aa, "USERID", ""); + dlist_setnum64(aa, "MODSEQ", 0); + dlist_sethex64(aa, "VALUE", record->cid); + } + + GCC_FALLTHROUGH - if (record && record->createdmodseq && mailbox->i.minor_version >= 16) { - if (!annots) - annots = dlist_newlist(parent, "ANNOTATIONS"); - aa = dlist_newkvlist(annots, NULL); - dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "createdmodseq"); - dlist_setatom(aa, "USERID", ""); - dlist_setnum64(aa, "MODSEQ", 0); - dlist_setnum64(aa, "VALUE", record->createdmodseq); + default: + break; } } @@ -1512,47 +1533,69 @@ int decode_annotations(/*const*/struct dlist *annots, return IMAP_PROTOCOL_BAD_PARAMETERS; if (!dlist_getbuf(aa, "VALUE", &value)) return IMAP_PROTOCOL_BAD_PARAMETERS; - if (!strcmp(entry, IMAP_ANNOT_NS "thrid") && - record && mailbox->i.minor_version >= 13) { - const char *p = buf_cstring(&value); - parsehex(p, &p, 16, &record->cid); - } - else if (!strcmp(entry, IMAP_ANNOT_NS "savedate") && - record && mailbox->i.minor_version >= 15) { - const char *p = buf_cstring(&value); - bit64 newval; - parsenum(p, &p, 0, &newval); - record->savedate = newval; - } - else if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq") && - record && mailbox->i.minor_version >= 16) { - const char *p = buf_cstring(&value); - bit64 newval; - parsenum(p, &p, 0, &newval); - record->createdmodseq = newval; - } - else if (!strcmp(entry, IMAP_ANNOT_NS "basethrid") && record) { - /* this might double-apply the annotation, but oh well. It does mean that - * basethrid is paired in here when we do a comparison against new values - * from the replica later! */ - const char *p = buf_cstring(&value); - parsehex(p, &p, 16, &record->basecid); - /* XXX - check on p? */ - - /* "basethrid" is special, since it is written during mailbox - * appends and rewrites, using whatever modseq the index_record - * has at this moment. This might differ from the modseq we - * just parsed here, causing master and replica annotations - * to get out of sync. - * The fix is to set the basecid field both on the index - * record *and* adding the annotation to the annotation list. - * That way the local modseq of basethrid always gets over- - * written by whoever wins to be master of this annotation */ - sync_annot_list_add(*salp, entry, userid, &value, modseq); - } - else { + + if (!record) goto realannot; + + const char *p = buf_cstring(&value); + + /* Look for and process "virtual" annotations */ + switch (mailbox->i.minor_version) { + case 20: + case 19: + case 18: + case 17: + case 16: + if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) { + bit64 newval; + parsenum(p, &p, 0, &newval); + record->createdmodseq = newval; + break; + } + + GCC_FALLTHROUGH + + case 15: + if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) { + bit64 newval; + parsenum(p, &p, 0, &newval); + record->savedate = newval; + break; + } + + GCC_FALLTHROUGH + + case 14: + case 13: + if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) { + parsehex(p, &p, 16, &record->cid); + break; + } + else if (!strcmp(entry, IMAP_ANNOT_NS "basethrid")) { + /* this might double-apply the annotation, but oh well. + * It does mean that basethrid is paired in here when we do + * a comparison against new values from the replica later! */ + parsehex(p, &p, 16, &record->basecid); + /* XXX - check on p? */ + + /* "basethrid" is special, since it is written during mailbox + * appends and rewrites, using whatever modseq the index_record + * has at this moment. This might differ from the modseq we + * just parsed here, causing master and replica annotations + * to get out of sync. + * The fix is to set the basecid field both on the index + * record *and* adding the annotation to the annotation list. + * That way the local modseq of basethrid always gets over- + * written by whoever wins to be master of this annotation */ + } + + GCC_FALLTHROUGH + + realannot: + default: sync_annot_list_add(*salp, entry, userid, &value, modseq); + break; } + buf_free(&value); } return 0; From 2163d2b9d2212756c86f6ba40d2b47d2cc4a14ac Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Tue, 4 Feb 2025 13:43:41 -0500 Subject: [PATCH 08/16] mailbox.c: store INTERNALDATE as nanoseconds since epoch --- imap/mailbox.c | 93 ++++++++++++++++++++++++++++------ imap/mailbox.h | 4 +- imap/mbexamine.c | 38 +++++++++----- imap/sync_support.c | 27 +++++++++- lib/util.h | 7 +++ perl/imap/Cyrus/AccountSync.pm | 4 +- 6 files changed, 140 insertions(+), 33 deletions(-) diff --git a/imap/mailbox.c b/imap/mailbox.c index d35f26ae19..683941b5e3 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -2050,6 +2050,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, and rearranged fields so that these would fall on 8-byte boundaries */ if (version < 20) { record->internaldate.tv_sec = ntohl(*((bit32 *)(buf+PRE20_OFFSET_INTERNALDATE))); + record->internaldate.tv_nsec = UTIME_OMIT; record->sentdate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SENTDATE))); record->size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SIZE))); record->header_size = ntohl(*((bit32 *)(buf+PRE20_OFFSET_HEADER_SIZE))); @@ -2105,7 +2106,8 @@ static int mailbox_buf_to_index_record(const char *buf, int version, } cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET))); - record->internaldate.tv_sec = ntohll(*((bit64 *)(buf+OFFSET_INTERNALDATE))); + TIMESPEC_FROM_NANOSEC(&record->internaldate, + ntohll(*((bit64 *)(buf+OFFSET_INTERNALDATE)))); record->sentdate = ntohll(*((bit64 *)(buf+OFFSET_SENTDATE))); record->size = ntohll(*((bit64 *)(buf+OFFSET_SIZE))); record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE))); @@ -3285,7 +3287,8 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, } *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); - *((bit64 *)(buf+OFFSET_INTERNALDATE)) = htonll(record->internaldate.tv_sec); + *((bit64 *)(buf+OFFSET_INTERNALDATE)) = + htonll(TIMESPEC_TO_NANOSEC(&record->internaldate)); *((bit64 *)(buf+OFFSET_SENTDATE)) = htonll(record->sentdate); *((bit64 *)(buf+OFFSET_SIZE)) = htonll(record->size); *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); @@ -3451,6 +3454,11 @@ static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry) { switch (mailbox->i.minor_version) { case 20: + // internaldate.nsec was introduced in v20 + if (!strcmp(entry, IMAP_ANNOT_NS "internaldate.nsec")) return 1; + + GCC_FALLTHROUGH + case 19: case 18: case 17: @@ -3492,6 +3500,14 @@ static uint32_t crc_virtannot(struct mailbox *mailbox, switch (mailbox->i.minor_version) { case 20: + if (record->internaldate.tv_nsec != UTIME_OMIT) { + buf_printf(&buf, UINT64_FMT, record->internaldate.tv_nsec); + crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); + buf_reset(&buf); + } + + GCC_FALLTHROUGH + case 19: case 18: case 17: @@ -4737,7 +4753,7 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, struct index_record *record) { int r; - struct utimbuf settime; + struct timespec now; uint32_t changeflags = CHANGE_ISAPPEND; assert(mailbox_index_islocked(mailbox, 1)); @@ -4798,8 +4814,12 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, } } - if (!record->internaldate.tv_sec) - record->internaldate.tv_sec = time(NULL); + clock_gettime(CLOCK_REALTIME, &now); + + if (!record->internaldate.tv_sec) { + record->internaldate.tv_sec = now.tv_sec; + record->internaldate.tv_nsec = now.tv_nsec; + } if (!record->gmtime) record->gmtime = record->internaldate.tv_sec; if (!record->sentdate) { @@ -4834,10 +4854,18 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, if (!(record->internal_flags & FLAG_INTERNAL_UNLINKED)) { /* make the file timestamp correct */ - settime.actime = settime.modtime = record->internaldate.tv_sec; - if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) // mabe there is no file in directory. - if (utime(mailbox_record_fname(mailbox, record), &settime) == -1) + if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) { // maybe there is no file in directory. + struct timespec settimes[] = { + { record->internaldate.tv_sec, record->internaldate.tv_nsec }, + { record->internaldate.tv_sec, record->internaldate.tv_nsec } + }; + const char *fname = mailbox_record_fname(mailbox, record); + r = utimensat(AT_FDCWD, fname, settimes, 0); + if (r == -1) { + syslog(LOG_ERR, "failed to set mtime on %s: %m", fname); return IMAP_IOERROR; + } + } /* write the cache record before buffering the message, it * will set the cache_offset field. */ @@ -5482,6 +5510,29 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) } } + if (mailbox->i.minor_version < 20 && repack->newmailbox.i.minor_version >= 20) { + /* extract internaldate.nsec */ + buf_reset(&buf); + mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); + if (buf.len) { + const char *p = buf_cstring(&buf); + bit64 newval; + parsenum(p, &p, 0, &newval); + copyrecord.internaldate.tv_nsec = newval; + } + buf_reset(&buf); + r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); + if (r) goto done; + } + if (mailbox->i.minor_version >= 20 && repack->newmailbox.i.minor_version < 20) { + if (record->internaldate.tv_nsec != UTIME_OMIT) { + buf_reset(&buf); + buf_printf(&buf, UINT64_FMT, record->internaldate.tv_nsec); + r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); + if (r) goto done; + } + } + /* read in the old cache record */ r = mailbox_cacherecord(mailbox, ©record); if (r) goto done; @@ -7279,7 +7330,8 @@ static int records_match(const char *mboxname, int match = 1; int userflags_dirty = 0; - if (old->internaldate.tv_sec != new->internaldate.tv_sec) { + if (old->internaldate.tv_sec != new->internaldate.tv_sec || + old->internaldate.tv_nsec != new->internaldate.tv_nsec) { printf("%s uid %u mismatch: internaldate\n", mboxname, new->uid); match = 0; @@ -7440,14 +7492,17 @@ static int mailbox_reconstruct_compare_update(struct mailbox *mailbox, /* re-calculate all the "derived" fields by parsing the file on disk */ if (re_parse) { /* set NULL in case parse finds a new value */ - record->internaldate.tv_sec = 0; + record->internaldate.tv_sec = 0; + record->internaldate.tv_nsec = UTIME_OMIT; r = message_parse(fname, record); if (r) goto out; /* unchanged, keep the old value */ - if (!record->internaldate.tv_sec) + if (!record->internaldate.tv_sec) { record->internaldate.tv_sec = copy.internaldate.tv_sec; + record->internaldate.tv_nsec = copy.internaldate.tv_nsec; + } /* it's not the same message! */ if (!message_guid_equal(&record->guid, ©.guid)) { @@ -7528,10 +7583,16 @@ static int mailbox_reconstruct_compare_update(struct mailbox *mailbox, /* get internaldate from the file if not set */ if (!record->internaldate.tv_sec) { - if (did_stat || stat(fname, &sbuf) != -1) + if (did_stat || stat(fname, &sbuf) != -1) { record->internaldate.tv_sec = sbuf.st_mtim.tv_sec; - else - record->internaldate.tv_sec = time(NULL); + record->internaldate.tv_nsec = sbuf.st_mtim.tv_nsec; + } + else { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + record->internaldate.tv_sec = now.tv_sec; + record->internaldate.tv_nsec = now.tv_nsec; + } } if (!record->gmtime) record->gmtime = record->internaldate.tv_sec; @@ -7700,8 +7761,10 @@ static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid, int record.internal_flags |= FLAG_INTERNAL_SNOOZED; /* copy the timestamp from the file if not calculated */ - if (!record.internaldate.tv_sec) + if (!record.internaldate.tv_sec) { record.internaldate.tv_sec = sbuf.st_mtim.tv_sec; + record.internaldate.tv_nsec = sbuf.st_mtim.tv_nsec; + } if (uid > mailbox->i.last_uid) { printf("%s uid %u found - adding\n", mailbox_name(mailbox), uid); diff --git a/imap/mailbox.h b/imap/mailbox.h index 27e0d3e36e..5f2d4cddb2 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -150,7 +150,7 @@ struct statusdata { struct index_record { uint32_t uid; - struct timespec internaldate; + struct timespec internaldate; // nanoseconds since epoch time_t sentdate; uint64_t size; uint32_t header_size; @@ -397,7 +397,7 @@ struct mailbox_iter; */ #define OFFSET_UID 0 #define OFFSET_CACHE_OFFSET 4 -#define OFFSET_INTERNALDATE 8 /* grew to 64-bit in v20 */ +#define OFFSET_INTERNALDATE 8 /* grew to 64-bit in v20 (nsec since epoch) */ #define OFFSET_SENTDATE 16 /* grew to 64-bit in v20 */ #define OFFSET_SIZE 24 /* grew to 64-bit in v20 */ #define OFFSET_HEADER_SIZE 32 diff --git a/imap/mbexamine.c b/imap/mbexamine.c index f8c33b94e6..fe874d6869 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -307,30 +307,38 @@ static int do_examine(struct findall_data *data, void *rock) } printf("%06u> UID:%08u" - " INT_DATE:" TIME_T_FMT " SENTDATE:" TIME_T_FMT + " INT_DATE:" TIME_T_FMT, + msgno, record->uid, record->internaldate.tv_sec); + if (mailbox->i.minor_version >= 20 && + record->internaldate.tv_nsec < UTIME_OMIT) { + printf(UINT64_NANOSEC_FMT, record->internaldate.tv_nsec); + } + printf(" SENTDATE:" TIME_T_FMT " SAVEDATE:" TIME_T_FMT " SIZE: " UINT64_LALIGN_FMT "\n", - msgno, record->uid, record->internaldate.tv_sec, record->sentdate, record->savedate, 6, record->size); printf(" > HDRSIZE:%-6u LASTUPD :" TIME_T_FMT " SYSFLAGS:%08X", record->header_size, record->last_updated, record->system_flags); - if (mailbox->i.minor_version >= 6) - printf(" > CACHEVER:%-2u", record->cache_version); + printf("\n"); - if (mailbox->i.minor_version >= 7) { - printf(" GUID:%s", message_guid_encode(&record->guid)); - } + if (mailbox->i.minor_version >= 6) { + printf(" > CACHEVER:%-5u", record->cache_version); - if (mailbox->i.minor_version >= 8) { - printf(" MODSEQ:" MODSEQ_FMT, record->modseq); - } + if (mailbox->i.minor_version >= 7) { + printf(" GUID:%s", message_guid_encode(&record->guid)); - if (mailbox->i.minor_version >= 13) { - printf(" THRID: " CONV_FMT, record->cid); - } + if (mailbox->i.minor_version >= 8) { + printf(" MODSEQ:" MODSEQ_FMT, record->modseq); - printf("\n"); + if (mailbox->i.minor_version >= 13) { + printf(" THRID: " CONV_FMT, record->cid); + } + } + } + + printf("\n"); + } printf(" > INTERNALFLAGS:"); if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) @@ -344,6 +352,8 @@ static int do_examine(struct findall_data *data, void *rock) if (record->internal_flags & FLAG_INTERNAL_SNOOZED) printf(" FLAG_INTERNAL_SNOOZED"); + printf("\n"); + printf(" > SYSTEMFLAGS:"); if (record->system_flags & FLAG_SEEN) printf(" FLAG_SEEN"); if (record->system_flags & FLAG_DRAFT) printf(" FLAG_DRAFT"); diff --git a/imap/sync_support.c b/imap/sync_support.c index dea352a6a6..c41b3b5e22 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1452,6 +1452,18 @@ void encode_annotations(struct dlist *parent, switch (mailbox->i.minor_version) { case 20: + if (record->internaldate.tv_sec != UTIME_OMIT) { + if (!annots) + annots = dlist_newlist(parent, "ANNOTATIONS"); + aa = dlist_newkvlist(annots, NULL); + dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "internaldate.nsec"); + dlist_setatom(aa, "USERID", ""); + dlist_setnum64(aa, "MODSEQ", 0); + dlist_setnum64(aa, "VALUE", record->internaldate.tv_nsec); + } + + GCC_FALLTHROUGH + case 19: case 18: case 17: @@ -1541,6 +1553,15 @@ int decode_annotations(/*const*/struct dlist *annots, /* Look for and process "virtual" annotations */ switch (mailbox->i.minor_version) { case 20: + if (!strcmp(entry, IMAP_ANNOT_NS "internaldate.nsec")) { + bit64 newval; + parsenum(p, &p, 0, &newval); + record->internaldate.tv_nsec = newval; + break; + } + + GCC_FALLTHROUGH + case 19: case 18: case 17: @@ -1822,6 +1843,7 @@ int parse_upload(struct dlist *kr, struct mailbox *mailbox, if (!dlist_getguid(kr, "GUID", &tmpguid)) return IMAP_PROTOCOL_BAD_PARAMETERS; + record->internaldate.tv_nsec = UTIME_OMIT; record->guid = *tmpguid; /* parse the flags */ @@ -2693,7 +2715,8 @@ static int sync_mailbox_compare_update(struct mailbox *mailbox, copy.basecid = mrecord.basecid; copy.modseq = mrecord.modseq; copy.last_updated = mrecord.last_updated; - copy.internaldate.tv_sec = mrecord.internaldate.tv_sec; + copy.internaldate.tv_sec = mrecord.internaldate.tv_sec; + copy.internaldate.tv_nsec = mrecord.internaldate.tv_nsec; copy.savedate = mrecord.savedate; copy.createdmodseq = mrecord.createdmodseq; copy.system_flags = mrecord.system_flags; @@ -5926,6 +5949,8 @@ static int compare_one_record(struct sync_client_state *sync_cs, goto diff; if (mp->internaldate.tv_sec != rp->internaldate.tv_sec) goto diff; + if (mp->internaldate.tv_nsec != rp->internaldate.tv_nsec) + goto diff; if (mp->system_flags != rp->system_flags) goto diff; if ((mp->internal_flags & FLAG_INTERNAL_EXPUNGED) != diff --git a/lib/util.h b/lib/util.h index 22ef9beb28..e4ab9a3257 100644 --- a/lib/util.h +++ b/lib/util.h @@ -160,6 +160,13 @@ extern const unsigned char convert_to_uppercase[256]; } #endif +#define TIMESPEC_TO_NANOSEC(ts) ((ts)->tv_sec * 1000000000 + (ts)->tv_nsec) + +#define TIMESPEC_FROM_NANOSEC(ts, nanosec) { \ + (ts)->tv_sec = nanosec / 1000000000; \ + (ts)->tv_nsec = nanosec % 1000000000; \ +} + typedef struct keyvalue { char *key, *value; } keyvalue; diff --git a/perl/imap/Cyrus/AccountSync.pm b/perl/imap/Cyrus/AccountSync.pm index fc3ba0fb71..4330141db2 100644 --- a/perl/imap/Cyrus/AccountSync.pm +++ b/perl/imap/Cyrus/AccountSync.pm @@ -110,6 +110,7 @@ sub dump_user { modseq => $record->{MODSEQ} + 0, internalDate => $record->{INTERNALDATE} + 0, rawMessage => $data, + annotations => $record->{ANNOTATIONS}, ); if ($opts{objectid}) { $email{emailId} = "M" . substr($record->{GUID}, 0, 24); @@ -176,8 +177,9 @@ sub undump_user { my $internaldate = $email->{internalDate} // $time; my $modseq = $email->{modseq} || 1; my $flags = $email->{flags} || []; + my $annotations = $email->{annotations} || []; my %record = ( - ANNOTATIONS => [], # skip savedate and such for now + ANNOTATIONS => $annotations, GUID => $guid, FLAGS => $flags, INTERNALDATE => $internaldate, From 8476f1ec51043cfc18801866b76fcf2e4d7b0564 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Mon, 27 Jan 2025 12:29:11 -0500 Subject: [PATCH 09/16] conversations.c: store internaldates as nanseconds since epoch store the encoded nanosecond value as a J(MAPID) key that maps to the GUID of an email --- imap/conversations.c | 60 ++++++++++++++++++++++++++++++++------------ imap/conversations.h | 10 ++++---- imap/jmap_mail.c | 14 +++++++---- lib/util.h | 11 +++++--- 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/imap/conversations.c b/imap/conversations.c index 2fac7b19a4..29787dd078 100644 --- a/imap/conversations.c +++ b/imap/conversations.c @@ -962,8 +962,8 @@ static void conv_to_buf(conversation_t *conv, struct buf *buf, int flagcount) nn = dlist_newlist(n, "THREAD"); dlist_setguid(nn, "GUID", &thread->guid); dlist_setnum32(nn, "EXISTS", thread->exists); - dlist_setnum32(nn, "INTERNALDATE", thread->internaldate); - dlist_setnum32(nn, "CREATEDMODSEQ", thread->createdmodseq); + dlist_setnum64(nn, "INTERNALDATE", thread->internaldate); + dlist_setnum64(nn, "CREATEDMODSEQ", thread->createdmodseq); } dlist_setnum64(dl, "CREATEDMODSEQ", conv->createdmodseq); @@ -1490,7 +1490,7 @@ int _saxconvparse(int type, struct dlistsax_data *d) return 0; case 2: - rock->thread->internaldate = atol(d->data); + rock->thread->internaldate = atoll(d->data); rock->substate = 3; return 0; @@ -1775,8 +1775,12 @@ static int _thread_datesort(const void **a, const void **b) { const conv_thread_t *ta = (const conv_thread_t *)*a; const conv_thread_t *tb = (const conv_thread_t *)*b; + struct timespec ta_internaldate, tb_internaldate; - int r = (ta->internaldate - tb->internaldate); + TIMESPEC_FROM_NANOSEC(&ta_internaldate, ta->internaldate); + TIMESPEC_FROM_NANOSEC(&tb_internaldate, tb->internaldate); + + int r = (ta_internaldate.tv_sec - tb_internaldate.tv_sec); if (r < 0) return -1; if (r > 0) return 1; @@ -1817,7 +1821,7 @@ static void conversations_thread_sort(conversation_t *conv) EXPORTED void conversation_update_thread(conversation_t *conv, const struct message_guid *guid, - time_t internaldate, + uint64_t internaldate, modseq_t createdmodseq, int delta_exists) { @@ -1963,7 +1967,7 @@ static int _guid_one(struct guid_foreach_rock *frock, conversation_id_t basecid, uint32_t system_flags, uint32_t internal_flags, - time_t internaldate, + uint64_t internaldate, char version) { const char *p, *err; @@ -2071,7 +2075,7 @@ static int _guid_cb(void *rock, conversation_id_t basecid = 0; uint32_t system_flags = 0; uint32_t internal_flags = 0; - time_t internaldate = 0; + uint64_t internaldate = 0; char version = 0; if (datalen >= 16) { const char *p = data; @@ -2114,7 +2118,7 @@ static int _guid_cb(void *rock, internal_flags = ntohl(*((bit32*)p)); p += 4; /* internaldate*/ - internaldate = (time_t) ntohll(*((bit64*)p)); + internaldate = ntohll(*((bit64*)p)); p += 8; /* basecid */ basecid = ntohll(*((bit64*)p)); @@ -2228,7 +2232,7 @@ static int conversations_guid_setitem(struct conversations_state *state, conversation_id_t basecid, uint32_t system_flags, uint32_t internal_flags, - time_t internaldate, + uint64_t internaldate, int add) { struct buf key = BUF_INITIALIZER; @@ -2271,7 +2275,7 @@ static int conversations_guid_setitem(struct conversations_state *state, buf_appendbit64(&val, cid); buf_appendbit32(&val, system_flags); buf_appendbit32(&val, internal_flags); - buf_appendbit64(&val, (bit64)internaldate); + buf_appendbit64(&val, internaldate); } /* When bumping the G value version, make sure to update _guid_cb */ else { @@ -2279,7 +2283,7 @@ static int conversations_guid_setitem(struct conversations_state *state, buf_appendbit64(&val, cid); buf_appendbit32(&val, system_flags); buf_appendbit32(&val, internal_flags); - buf_appendbit64(&val, (bit64)internaldate); + buf_appendbit64(&val, internaldate); buf_appendbit64(&val, basecid == cid ? 0 : basecid); } @@ -2302,7 +2306,7 @@ static int _guid_addbody(struct conversations_state *state, conversation_id_t cid, conversation_id_t basecid, uint32_t system_flags, uint32_t internal_flags, - time_t internaldate, + uint64_t internaldate, struct body *body, const char *base, int add) { @@ -2357,15 +2361,38 @@ static int conversations_set_guid(struct conversations_state *state, buf_printf(&item, "%d:%u", folder, record->uid); const char *base = buf_cstring(&item); - r = conversations_guid_setitem(state, message_guid_encode(&record->guid), + const char *guidrep = message_guid_encode(&record->guid); + uint64_t internaldate = TIMESPEC_TO_NANOSEC(&record->internaldate); + r = conversations_guid_setitem(state, guidrep, base, record->cid, record->basecid, record->system_flags, record->internal_flags, - record->internaldate.tv_sec, + internaldate, add); + if (!r) { + struct buf key = BUF_INITIALIZER; + + /* Build J key */ + buf_setcstr(&key, "J"); + NANOSEC_TO_JMAPID(&key, internaldate); + + /* Do we have an existing toplevel G record? */ + if (!conversations_guid_cid_lookup(state, guidrep, NULL)) { + /* Remove J record */ + r = cyrusdb_delete(state->db, buf_base(&key), buf_len(&key), + &state->txn, /*force*/1); + } + else { + /* Add J record */ + r = cyrusdb_store(state->db, buf_base(&key), buf_len(&key), + guidrep, strlen(guidrep), &state->txn); + } + + buf_free(&key); + } if (!r) r = _guid_addbody(state, record->cid, record->basecid, record->system_flags, record->internal_flags, - record->internaldate.tv_sec, body, base, add); + internaldate, body, base, add); message_free_body(body); free(body); @@ -2518,6 +2545,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate, if (new) { if (!old || old->system_flags != new->system_flags || old->internal_flags != new->internal_flags || + old->internaldate.tv_nsec != new->internaldate.tv_nsec || old->internaldate.tv_sec != new->internaldate.tv_sec) { r = conversations_set_guid(cstate, mailbox, new, /*add*/1); if (r) goto done; @@ -2634,7 +2662,7 @@ EXPORTED int conversations_update_record(struct conversations_state *cstate, conversation_update_thread(conv, &record->guid, - record->internaldate.tv_sec, + TIMESPEC_TO_NANOSEC(&record->internaldate), record->createdmodseq, delta_exists); diff --git a/imap/conversations.h b/imap/conversations.h index b71bd94912..8cee1125e0 100644 --- a/imap/conversations.h +++ b/imap/conversations.h @@ -124,7 +124,7 @@ struct conv_thread { conv_thread_t *next; struct message_guid guid; uint32_t exists; - time_t internaldate; + uint64_t internaldate; // nanoseconds since epoch modseq_t createdmodseq; }; @@ -138,7 +138,7 @@ struct conv_folder { uint32_t prev_exists; }; -#define CONV_GUIDREC_VERSION 3 // (must be <= 127) +#define CONV_GUIDREC_VERSION 4 // (must be <= 127) #define CONV_GUIDREC_BYNAME_VERSION 1 // last folders byname version struct conv_guidrec { @@ -148,11 +148,11 @@ struct conv_guidrec { uint32_t uid; const char *part; conversation_id_t cid; - conversation_id_t basecid; + conversation_id_t basecid; // if version >= 3 char version; uint32_t system_flags; // if version >= 1 uint32_t internal_flags; // if version >= 1 - time_t internaldate; // if version >= 1 + uint64_t internaldate; // if version >= 4 (nanoseconds since epoch) }; struct conv_sender { @@ -354,7 +354,7 @@ extern void conversation_update_sender(conversation_t *conv, ssize_t delta_exists); extern void conversation_update_thread(conversation_t *conv, const struct message_guid *guid, - time_t internaldate, + uint64_t internaldate, modseq_t createdmodseq, int delta_exists); diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index dbae6e6195..676750a042 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -3235,9 +3235,9 @@ static int emailsearch_is_mutable(struct emailsearch *search) struct guidsearch_match { char guidrep[MESSAGE_GUID_SIZE*2+1]; uint32_t system_flags; - uint32_t internaldate; + uint64_t internaldate; // nanoseconds since epoch conversation_id_t cid; - bitvector_t folders; // only set if numfolders > 0 + bitvector_t folders; // only set if numfolders > 0 }; static void guidsearch_match_init(struct guidsearch_match *match, @@ -3261,10 +3261,14 @@ static int guidsearch_match_cmp QSORT_R_COMPAR_ARGS(const void *va, while (sort->key != SORT_SEQUENCE) { int ret; switch (sort->key) { - case SORT_ARRIVAL: - ret = a->internaldate < b->internaldate ? -1 : - a->internaldate > b->internaldate ? 1 : 0; + case SORT_ARRIVAL: { + struct timespec a_internaldate, b_internaldate; + TIMESPEC_FROM_NANOSEC(&a_internaldate, a->internaldate); + TIMESPEC_FROM_NANOSEC(&b_internaldate, b->internaldate); + ret = a_internaldate.tv_sec < b_internaldate.tv_sec ? -1 : + a_internaldate.tv_sec > b_internaldate.tv_sec ? 1 : 0; break; + } case SORT_GUID: ret = memcmp(a->guidrep, b->guidrep, MESSAGE_GUID_SIZE*2); break; diff --git a/lib/util.h b/lib/util.h index e4ab9a3257..a7adaae0c7 100644 --- a/lib/util.h +++ b/lib/util.h @@ -162,9 +162,14 @@ extern const unsigned char convert_to_uppercase[256]; #define TIMESPEC_TO_NANOSEC(ts) ((ts)->tv_sec * 1000000000 + (ts)->tv_nsec) -#define TIMESPEC_FROM_NANOSEC(ts, nanosec) { \ - (ts)->tv_sec = nanosec / 1000000000; \ - (ts)->tv_nsec = nanosec % 1000000000; \ +#define TIMESPEC_FROM_NANOSEC(ts, nanosec) { \ + (ts)->tv_sec = (nanosec) / 1000000000; \ + (ts)->tv_nsec = (nanosec) % 1000000000; \ +} + +#define NANOSEC_TO_JMAPID(buf, nanosec) { \ + uint64_t u64 = htonll(UINT64_MAX - (nanosec)); \ + charset_encode(buf, (const char *) &u64, 8, ENCODING_BASE64JMAPID); \ } typedef struct keyvalue { From 4b9ff880bdaacfc9ee454e21055c2afc30cdbaab Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Thu, 16 Jan 2025 15:50:00 -0500 Subject: [PATCH 10/16] index.c: use struct timespec to store internaldate in MsgData --- imap/index.c | 29 ++++++++++++++++------------- imap/index.h | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/imap/index.c b/imap/index.c index 7c72d3dada..41c14cb12f 100644 --- a/imap/index.c +++ b/imap/index.c @@ -6336,7 +6336,8 @@ MsgData **index_msgdata_load(struct index_state *state, cur->sentdate = record.gmtime; /* fall through */ case SORT_ARRIVAL: - cur->internaldate = record.internaldate.tv_sec; + cur->internaldate.tv_sec = record.internaldate.tv_sec; + cur->internaldate.tv_nsec = record.internaldate.tv_nsec; break; case SORT_FROM: cur->from = get_localpart_addr(cacheitem_base(&record, CACHE_FROM)); @@ -6379,7 +6380,8 @@ MsgData **index_msgdata_load(struct index_state *state, } else { /* If not in mailboxId, we use receivedAt */ - cur->internaldate = record.internaldate.tv_sec; + cur->internaldate.tv_sec = record.internaldate.tv_sec; + cur->internaldate.tv_nsec = record.internaldate.tv_nsec; } break; case SORT_SNOOZEDUNTIL: @@ -6405,7 +6407,8 @@ MsgData **index_msgdata_load(struct index_state *state, #endif if (!cur->savedate) { /* If not snoozed in mailboxId, we use receivedAt */ - cur->internaldate = record.internaldate.tv_sec; + cur->internaldate.tv_sec = record.internaldate.tv_sec; + cur->internaldate.tv_nsec = record.internaldate.tv_nsec; } break; case LOAD_IDS: @@ -6771,21 +6774,21 @@ static int index_sort_compare(MsgData *md1, MsgData *md2, ret = numcmp(md1->msgno, md2->msgno); break; case SORT_ARRIVAL: - ret = numcmp(md1->internaldate, md2->internaldate); + ret = numcmp(md1->internaldate.tv_sec, md2->internaldate.tv_sec); break; case SORT_CC: ret = strcmpsafe(md1->cc, md2->cc); break; case SORT_DATE: { - time_t d1 = md1->sentdate ? md1->sentdate : md1->internaldate; - time_t d2 = md2->sentdate ? md2->sentdate : md2->internaldate; + time_t d1 = md1->sentdate ? md1->sentdate : md1->internaldate.tv_sec; + time_t d2 = md2->sentdate ? md2->sentdate : md2->internaldate.tv_sec; ret = numcmp(d1, d2); break; } case SORT_SNOOZEDUNTIL: case SORT_SAVEDATE: { - time_t d1 = md1->savedate ? md1->savedate : md1->internaldate; - time_t d2 = md2->savedate ? md2->savedate : md2->internaldate; + time_t d1 = md1->savedate ? md1->savedate : md1->internaldate.tv_sec; + time_t d2 = md2->savedate ? md2->savedate : md2->internaldate.tv_sec; ret = numcmp(d1, d2); break; } @@ -6985,7 +6988,7 @@ static int index_sort_compare_arrival(const void *v1, const void *v2) MsgData *md2 = *(MsgData **)v2; int ret; - ret = md1->internaldate - md2->internaldate; + ret = md1->internaldate.tv_sec - md2->internaldate.tv_sec; if (ret) return ret; ret = md1->createdmodseq - md2->createdmodseq; @@ -7007,7 +7010,7 @@ static int index_sort_compare_reverse_arrival(const void *v1, const void *v2) MsgData *md2 = *(MsgData **)v2; int ret; - ret = md2->internaldate - md1->internaldate; + ret = md2->internaldate.tv_sec - md1->internaldate.tv_sec; if (ret) return ret; ret = md2->createdmodseq - md1->createdmodseq; @@ -7032,7 +7035,7 @@ static int index_sort_compare_reverse_flagged(const void *v1, const void *v2) ret = md2->hasflag - md1->hasflag; if (ret) return ret; - ret = md2->internaldate - md1->internaldate; + ret = md2->internaldate.tv_sec - md1->internaldate.tv_sec; if (ret) return ret; ret = md2->createdmodseq - md1->createdmodseq; @@ -7940,8 +7943,8 @@ static void find_most_recent(Thread *thread, MsgData *recent) Thread *child; /* test the head node */ - if (thread->msgdata->internaldate > recent->internaldate) - recent->internaldate = thread->msgdata->internaldate; + if (thread->msgdata->internaldate.tv_sec > recent->internaldate.tv_sec) + recent->internaldate.tv_sec = thread->msgdata->internaldate.tv_sec; /* test the children recursively */ child = thread->child; diff --git a/imap/index.h b/imap/index.h index dc9283eadc..e39a3deed7 100644 --- a/imap/index.h +++ b/imap/index.h @@ -163,7 +163,7 @@ typedef struct msgdata { strarray_t ref; /* array of references */ time_t sentdate; /* sent date & time of message from Date: header (adjusted by time zone) */ - time_t internaldate; /* internaldate */ + struct timespec internaldate;/* internaldate */ time_t savedate; /* savedate */ size_t size; /* message size */ modseq_t modseq; /* modseq of record*/ From 7ede7384e5ea69dab933d212487af9ba7d782f76 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Mon, 27 Jan 2025 12:33:33 -0500 Subject: [PATCH 11/16] jmap_mail.c: use nanosecond-based JMAPIDs as EMAILIDs --- .../email_set_guidsearch_updated_internaldate | 5 +- imap/caldav_alarm.c | 8 +- imap/conversations.c | 38 ++++ imap/conversations.h | 6 + imap/index.c | 17 +- imap/jmap_blob.c | 4 +- imap/jmap_mail.c | 215 ++++++++++++------ imap/jmap_util.c | 11 +- imap/jmap_util.h | 9 +- imap/mboxevent.c | 22 +- 10 files changed, 237 insertions(+), 98 deletions(-) diff --git a/cassandane/tiny-tests/JMAPEmail/email_set_guidsearch_updated_internaldate b/cassandane/tiny-tests/JMAPEmail/email_set_guidsearch_updated_internaldate index 215469832d..40c65a1c0c 100644 --- a/cassandane/tiny-tests/JMAPEmail/email_set_guidsearch_updated_internaldate +++ b/cassandane/tiny-tests/JMAPEmail/email_set_guidsearch_updated_internaldate @@ -116,6 +116,9 @@ sub test_email_set_guidsearch_updated_internaldate xlog "Append blob of message A via IMAP"; $imap->append('INBOX', $emailBlobA) || die $@; + $res = $imap->fetch('3', "(emailid)"); + my $emailIdC = $res->{3}{emailid}[0]; + xlog $self, "run incremental squatter"; $self->{instance}->run_command({cyrus => 1}, 'squatter', '-i'); @@ -132,5 +135,5 @@ sub test_email_set_guidsearch_updated_internaldate }, 'R1'], ], $using); $self->assert_equals(JSON::true, $res->[0][1]{performance}{details}{isGuidSearch}); - $self->assert_deep_equals([$emailIdB, $emailIdA], $res->[0][1]{ids}); + $self->assert_deep_equals([$emailIdB, $emailIdC], $res->[0][1]{ids}); } diff --git a/imap/caldav_alarm.c b/imap/caldav_alarm.c index b4ca83d6f8..30778eac35 100644 --- a/imap/caldav_alarm.c +++ b/imap/caldav_alarm.c @@ -1601,7 +1601,8 @@ static int find_scheduled_email(const char *emailid, struct conversations_state *cstate = NULL; int r; - if (emailid[0] != 'M' || strlen(emailid) != 25) { + if (emailid[0] != JMAP_EMAILID_PREFIX || + strlen(emailid) != JMAP_EMAILID_SIZE - 1) { return IMAP_NOTFOUND; } @@ -1612,8 +1613,9 @@ static int find_scheduled_email(const char *emailid, return r; } - const char *guid = emailid + 1; - r = conversations_guid_foreach(cstate, guid, find_sched_cb, frock); + char guid[2*MESSAGE_GUID_SIZE+1]; + r = conversations_jmapid_guidrep_lookup(cstate, emailid + 1, guid); + if (!r) r = conversations_guid_foreach(cstate, guid, find_sched_cb, frock); conversations_commit(&cstate); if (r == IMAP_OK_COMPLETED) r = 0; diff --git a/imap/conversations.c b/imap/conversations.c index 29787dd078..d4d5d56759 100644 --- a/imap/conversations.c +++ b/imap/conversations.c @@ -3333,4 +3333,42 @@ EXPORTED int conversations_zero_modseq(struct conversations_state *state) return r; } +EXPORTED int conversations_jmapid_guidrep_lookup(struct conversations_state *state, + const char *jidrep, char guidrep[]) +{ + static const char jmapid_alphabet[] = + "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; + char key[CONV_JMAPID_SIZE+2] = "J"; + size_t idlen, datalen = 0; + const char *data; + int r; + + /* sanity check the ID: + - to prevent strcat() from overrunning a fixed-length buffer + - to short-circuit an ill-fated DB fetch + */ + idlen = strspn(jidrep, jmapid_alphabet); + if (idlen != CONV_JMAPID_SIZE || jidrep[idlen] != '\0') { + /* we know this J key can't exist */ + return CYRUSDB_NOTFOUND; + } + + strcat(key, jidrep); + + r = cyrusdb_fetch(state->db, key, CONV_JMAPID_SIZE+1, + &data, &datalen, &state->txn); + if (r) return r; + + if (datalen < 2*MESSAGE_GUID_SIZE) { + xsyslog(LOG_NOTICE, "IOERROR: malformed data value in J record", + "key=<%s>", key); + return CYRUSDB_NOTFOUND; + } + + strncpy(guidrep, data, datalen); + guidrep[datalen] = '\0'; + + return 0; +} + #undef DB diff --git a/imap/conversations.h b/imap/conversations.h index 8cee1125e0..2378112dc8 100644 --- a/imap/conversations.h +++ b/imap/conversations.h @@ -289,6 +289,12 @@ extern int conversations_guid_cid_lookup(struct conversations_state *state, strcmpsafe(conv_guidrec_uniqueid(rec), mailbox_uniqueid(mbox)) : \ strcmpsafe(conv_guidrec_mboxname(rec), mailbox_name(mbox)) \ ) + +/* J record items */ +#define CONV_JMAPID_SIZE 11 // 64-bits base64-encoded w/o padding +extern int conversations_jmapid_guidrep_lookup(struct conversations_state *state, + const char *jidrep, char guidrep[]); + /* F record items */ extern int conversation_getstatus(struct conversations_state *state, const char *mailbox, diff --git a/imap/index.c b/imap/index.c index 41c14cb12f..c084f58564 100644 --- a/imap/index.c +++ b/imap/index.c @@ -4537,11 +4537,11 @@ static int index_fetchreply(struct index_state *state, uint32_t msgno, sepchar = ' '; } if (fetchitems & FETCH_EMAILID) { - char emailid[26]; - emailid[0] = 'M'; - memcpy(emailid+1, message_guid_encode(&record.guid), 24); - emailid[25] = '\0'; - prot_printf(state->out, "%cEMAILID (%s)", sepchar, emailid); + struct buf emailid = BUF_INITIALIZER; + buf_putc(&emailid, 'S'); + NANOSEC_TO_JMAPID(&emailid, TIMESPEC_TO_NANOSEC(&record.internaldate)); + prot_printf(state->out, "%cEMAILID (%s)", sepchar, buf_cstring(&emailid)); + buf_free(&emailid); sepchar = ' '; } if ((fetchitems & FETCH_CID) && @@ -6851,6 +6851,13 @@ static int index_sort_compare(MsgData *md1, MsgData *md2, case SORT_GUID: ret = message_guid_cmp(&md1->guid, &md2->guid); break; + case SORT_EMAILID: + // EMAILIDs are an ASCII-order encoding of + // (INT_MAX - nano_internaldate) => + // reverse the order of nano_internaldate comparison + ret = numcmp(TIMESPEC_TO_NANOSEC(&md2->internaldate), + TIMESPEC_TO_NANOSEC(&md1->internaldate)); + break; } } while (!ret && sortcrit[i++].key != SORT_SEQUENCE); diff --git a/imap/jmap_blob.c b/imap/jmap_blob.c index e4812e9dc6..462a2ff600 100644 --- a/imap/jmap_blob.c +++ b/imap/jmap_blob.c @@ -804,10 +804,12 @@ static int jmap_blob_lookup(jmap_req_t *req) /* Read message record */ struct message_guid guid; + struct timespec internaldate; bit64 cid = 0; msgrecord_t *mr = NULL; r = msgrecord_find(mbox, getblob->uid, &mr); if (!r) r = msgrecord_get_guid(mr, &guid); + if (!r) r = msgrecord_get_internaldate(mr, &internaldate); if (!r) r = msgrecord_get_cid(mr, &cid); msgrecord_unref(&mr); if (r) { @@ -848,7 +850,7 @@ static int jmap_blob_lookup(jmap_req_t *req) case DATATYPE_EMAIL: { char emailid[JMAP_EMAILID_SIZE]; - jmap_set_emailid(&guid, emailid); + jmap_set_emailid_from_timespec(&internaldate, emailid); strarray_add(ids, emailid); break; } diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c index 676750a042..5782d41354 100644 --- a/imap/jmap_mail.c +++ b/imap/jmap_mail.c @@ -1077,9 +1077,14 @@ static char *_emailbodies_to_html(struct emailbodies *bodies, const struct buf * return buf_releasenull(&buf); } -static const char *_guid_from_id(const char *msgid) +static const char *_guid_from_id(struct conversations_state *cstate, + const char *emailid) { - return msgid + 1; + static char guidrep[2*MESSAGE_GUID_SIZE+1]; + + int r = conversations_jmapid_guidrep_lookup(cstate, emailid + 1, guidrep); + + return (r ? NULL : guidrep); } static conversation_id_t _cid_from_id(const char *thrid) @@ -1091,15 +1096,15 @@ static conversation_id_t _cid_from_id(const char *thrid) } /* - * Lookup all mailboxes where msgid is contained in. + * Lookup all mailboxes where guidrep is contained in. * * The return value is a JSON object keyed by the mailbox unique id, * and its mailbox name as value. */ -static json_t *_email_mailboxes(jmap_req_t *req, const char *msgid) +static json_t *_email_mailboxes(jmap_req_t *req, const char *guidrep) { struct _email_mailboxes_rock data = { req, json_object() }; - conversations_guid_foreach(req->cstate, _guid_from_id(msgid), _email_mailboxes_cb, &data); + conversations_guid_foreach(req->cstate, guidrep, _email_mailboxes_cb, &data); return data.mboxs; } @@ -1208,20 +1213,68 @@ static int _email_find_cb(const conv_guidrec_t *rec, void *rock) return d->mboxname ? IMAP_OK_COMPLETED : 0; } +static int _email_find_by_guid(jmap_req_t *req, + struct conversations_state *cstate, + const char *guid, + char **mboxnameptr, + uint32_t *uidptr) + +{ + struct _email_find_rock rock = { req, NULL, 0 }; + + if (!guid || !*guid) return IMAP_NOTFOUND; + + int r = conversations_guid_foreach(cstate, guid, _email_find_cb, &rock); + + /* Set return values */ + if (r == IMAP_OK_COMPLETED) + r = 0; + else if (!rock.mboxname) + r = IMAP_NOTFOUND; + *mboxnameptr = rock.mboxname; + *uidptr = rock.uid; + return r; +} + +HIDDEN int jmap_email_find_by_guid(jmap_req_t *req, + const char *from_accountid, + const char *guid, + char **mboxnameptr, + uint32_t *uidptr) +{ + const char *accountid = from_accountid ? from_accountid : req->accountid; + int r; + + /* Open conversation state, if not already open */ + struct conversations_state *mycstate = NULL; + if (strcmp(req->accountid, accountid)) { + r = conversations_open_user(accountid, 1/*shared*/, &mycstate); + if (r) return r; + } + else { + mycstate = req->cstate; + } + + r = _email_find_by_guid(req, mycstate, guid, mboxnameptr, uidptr); + if (mycstate != req->cstate) { + conversations_commit(&mycstate); + } + return r; +} + static int _email_find_in_account(jmap_req_t *req, const char *account_id, const char *email_id, char **mboxnameptr, uint32_t *uidptr) { - struct _email_find_rock rock = { req, NULL, 0 }; int r; - /* must be prefixed with 'M' */ - if (email_id[0] != 'M') + /* must be prefixed with 'S' */ + if (email_id[0] != JMAP_EMAILID_PREFIX) return IMAP_NOTFOUND; - /* this is on a 24 character prefix only */ - if (strlen(email_id) != 25) + /* this is on a 11 character prefix only */ + if (strlen(email_id) != JMAP_EMAILID_SIZE - 1) return IMAP_NOTFOUND; /* Open conversation state, if not already open */ struct conversations_state *mycstate = NULL; @@ -1232,18 +1285,12 @@ static int _email_find_in_account(jmap_req_t *req, else { mycstate = req->cstate; } - r = conversations_guid_foreach(mycstate, _guid_from_id(email_id), - _email_find_cb, &rock); + + r = _email_find_by_guid(req, mycstate, _guid_from_id(mycstate, email_id), + mboxnameptr, uidptr); if (mycstate != req->cstate) { conversations_commit(&mycstate); } - /* Set return values */ - if (r == IMAP_OK_COMPLETED) - r = 0; - else if (!rock.mboxname) - r = IMAP_NOTFOUND; - *mboxnameptr = rock.mboxname; - *uidptr = rock.uid; return r; } @@ -1290,16 +1337,18 @@ static int _email_get_cid(jmap_req_t *req, const char *msgid, { int r; - /* must be prefixed with 'M' */ - if (msgid[0] != 'M') + /* must be prefixed with 'S' */ + if (msgid[0] != JMAP_EMAILID_PREFIX) return IMAP_NOTFOUND; - /* this is on a 24 character prefix only */ - if (strlen(msgid) != 25) + /* this is on a 11 character prefix only */ + if (strlen(msgid) != JMAP_EMAILID_SIZE - 1) return IMAP_NOTFOUND; int checkacl = strcmp(req->userid, req->accountid); struct email_getcid_rock rock = { req, checkacl, 0 }; - r = conversations_guid_foreach(req->cstate, _guid_from_id(msgid), _email_get_cid_cb, &rock); + r = conversations_guid_foreach(req->cstate, + _guid_from_id(req->cstate, msgid), + _email_get_cid_cb, &rock); if (r == IMAP_OK_COMPLETED) { *cidp = rock.cid; r = 0; @@ -2924,7 +2973,7 @@ static struct sortcrit *_email_buildsort(json_t *sort, int *sort_savedate) sortcrit[j].key = SORT_DISPLAYFROM; } else if (!strcmp(prop, "id")) { - sortcrit[j].key = SORT_GUID; + sortcrit[j].key = SORT_EMAILID; } else if (!strcmp(prop, "emailState")) { sortcrit[j].key = SORT_MODSEQ; @@ -2975,7 +3024,7 @@ static struct sortcrit *_email_buildsort(json_t *sort, int *sort_savedate) sortcrit[j+0].key = SORT_ARRIVAL; sortcrit[j+0].flags |= SORT_REVERSE; - sortcrit[j+1].key = SORT_GUID; + sortcrit[j+1].key = SORT_EMAILID; sortcrit[j+1].flags |= SORT_REVERSE; sortcrit[j+2].key = SORT_SEQUENCE; sortcrit[j+2].flags |= SORT_REVERSE; @@ -3046,7 +3095,7 @@ static void emailsearch_init(struct emailsearch *search, struct sortcrit *sort = xzmalloc(3 * sizeof(struct sortcrit)); sort[0].key = SORT_ARRIVAL; sort[0].flags |= SORT_REVERSE; - sort[1].key = SORT_GUID; + sort[1].key = SORT_EMAILID; sort[1].flags |= SORT_REVERSE; sort[2].key = SORT_SEQUENCE; sort[2].flags |= SORT_REVERSE; @@ -3132,6 +3181,7 @@ static int _email_parse_comparator(jmap_req_t *req, struct emailquery_match { struct message_guid guid; conversation_id_t cid; + uint64_t internaldate; // nanoseconds since epoch smallarrayu64_t partnums; }; @@ -3272,6 +3322,13 @@ static int guidsearch_match_cmp QSORT_R_COMPAR_ARGS(const void *va, case SORT_GUID: ret = memcmp(a->guidrep, b->guidrep, MESSAGE_GUID_SIZE*2); break; + case SORT_EMAILID: + // EMAILIDs are an ASCII-order encoding of + // (INT_MAX - internaldate) => + // reverse the order of nano_internaldate comparison + ret = b->internaldate < a->internaldate ? -1 : + b->internaldate > a->internaldate ? 1 : 0; + break; default: syslog(LOG_ERR, "%s: ignoring unexpected sort %d", __func__, sort->key); ret = 0; @@ -3867,7 +3924,8 @@ static int is_guidsearch_sort(struct sortcrit *sort) { if (sort) { for ( ; sort->key != SORT_SEQUENCE; sort++) { - if (sort->key != SORT_GUID && sort->key != SORT_ARRIVAL) + if (sort->key != SORT_GUID && + sort->key != SORT_ARRIVAL && sort->key != SORT_EMAILID) return 0; } } @@ -4207,6 +4265,7 @@ static void emailquery_guidsearch_result_ensure(struct emailquery *q, struct ema struct emailquery_match *match = &qc->uncollapsed_matches[qc->uncollapsed_len++]; message_guid_decode(&match->guid, gsqmatch->guidrep); match->cid = gsqmatch->cid; + match->internaldate = gsqmatch->internaldate; smallarrayu64_init(&match->partnums); if (q->collapse_threads && hashset_add(qc->seen_threads, &match->cid)) qc->collapsed_matches[qc->collapsed_len++] = *match; @@ -4314,6 +4373,7 @@ static void emailquery_uidsearch_result_ensure(struct emailquery *q, struct emai struct emailquery_match *match = &qc->uncollapsed_matches[qc->uncollapsed_len++]; match->guid = md->guid; match->cid = md->cid; + match->internaldate = TIMESPEC_TO_NANOSEC(&md->internaldate); smallarrayu64_init(&match->partnums); /* Set partIds */ @@ -4649,7 +4709,7 @@ static void emailquery_cache_slice(struct emailquery *q, } struct emailquery_match *match = matches + i; char emailid[JMAP_EMAILID_SIZE]; - jmap_set_emailid(&match->guid, emailid); + jmap_set_emailid_from_nanosec(match->internaldate, emailid); if (!strcmp(emailid, q->super.anchor)) { /* Found anchor */ found_anchor = 1; @@ -4724,7 +4784,7 @@ static void emailquery_buildresult(struct emailquery *q, struct emailquery_match *match = &matches[i]; /* Set email id */ char emailid[JMAP_EMAILID_SIZE]; - jmap_set_emailid(&match->guid, emailid); + jmap_set_emailid_from_nanosec(match->internaldate, emailid); json_array_append_new(q->super.ids, json_string(emailid)); /* Set email-part ids */ if (q->want_partids && qc->qr.partid) { @@ -4753,7 +4813,7 @@ static void emailquery_buildresult(struct emailquery *q, size_t tpos = (size_t) hashu64_lookup(match->cid, &qc->firstinthread); do { char emailid[JMAP_EMAILID_SIZE]; - jmap_set_emailid(&qc->uncollapsed_matches[tpos].guid, emailid); + jmap_set_emailid_from_nanosec(qc->uncollapsed_matches[tpos].internaldate, emailid); json_array_append_new(emailids, json_string(emailid)); tpos = qc->nextinthread[tpos]; } while (tpos < qc->uncollapsed_len); @@ -5127,7 +5187,7 @@ static void _email_querychanges_collapsed(jmap_req_t *req, continue; } - jmap_set_emailid(&md->guid, email_id); + jmap_set_emailid_from_timespec(&md->internaldate, email_id); hash_insert(email_id, (void*)1, &touched_ids); hashu64_insert(md->cid, (void*)1, &touched_cids); @@ -5137,7 +5197,7 @@ static void _email_querychanges_collapsed(jmap_req_t *req, for (i = 0 ; i < mdcount; i++) { MsgData *md = ptrarray_nth(msgdata, i); - jmap_set_emailid(&md->guid, email_id); + jmap_set_emailid_from_timespec(&md->internaldate, email_id); int is_expunged = (md->system_flags & FLAG_DELETED) || (md->internal_flags & FLAG_INTERNAL_EXPUNGED); @@ -5323,7 +5383,7 @@ static void _email_querychanges_uncollapsed(jmap_req_t *req, // for this phase, we only care that it has a change if (md->modseq <= since_modseq) continue; - jmap_set_emailid(&md->guid, email_id); + jmap_set_emailid_from_timespec(&md->internaldate, email_id); hash_insert(email_id, (void*)1, &touched_ids); } @@ -5332,7 +5392,7 @@ static void _email_querychanges_uncollapsed(jmap_req_t *req, for (i = 0 ; i < mdcount; i++) { MsgData *md = ptrarray_nth(msgdata, i); - jmap_set_emailid(&md->guid, email_id); + jmap_set_emailid_from_timespec(&md->internaldate, email_id); int is_expunged = (md->system_flags & FLAG_DELETED) || (md->internal_flags & FLAG_INTERNAL_EXPUNGED); @@ -5505,8 +5565,9 @@ static void _email_changes(jmap_req_t *req, struct jmap_changes *changes, json_t for (i = 0 ; i < msgdata->count; i++) { MsgData *md = ptrarray_nth(msgdata, i); + const char *guidrep = message_guid_encode(&md->guid); - jmap_set_emailid(&md->guid, email_id); + jmap_set_emailid_from_timespec(&md->internaldate, email_id); /* Skip already seen messages */ if (hash_lookup(email_id, &seen_ids)) continue; @@ -5523,7 +5584,7 @@ static void _email_changes(jmap_req_t *req, struct jmap_changes *changes, json_t highest_modseq = md->modseq; struct email_expunge_check rock = { req, changes->since_modseq, 0 }; - int r = conversations_guid_foreach(req->cstate, _guid_from_id(email_id), + int r = conversations_guid_foreach(req->cstate, guidrep, _email_is_expunged_cb, &rock); if (r) { *err = jmap_server_error(r); @@ -6341,7 +6402,7 @@ static int _thread_get(jmap_req_t *req, json_t *ids, if ((r && r != IMAP_OK_COMPLETED) || !rock.is_visible) { continue; } - jmap_set_emailid(&thread->guid, email_id); + jmap_set_emailid_from_nanosec(thread->internaldate, email_id); json_array_append_new(ids, json_string(email_id)); } @@ -6663,7 +6724,7 @@ static int _email_get_keywords_cb(const conv_guidrec_t *rec, void *vrock) static int _email_get_keywords(jmap_req_t *req, struct email_getcontext *ctx, - const char *msgid, + const char *guidrep, json_t **jkeywords) { /* Initialize seen.db and sequence set cache */ @@ -6677,7 +6738,7 @@ static int _email_get_keywords(jmap_req_t *req, /* Gather keywords for all message records */ struct email_get_keywords_rock rock = { req, _EMAIL_KEYWORDS_INITIALIZER }; _email_keywords_init(&rock.keywords, req->userid, ctx->seendb, &ctx->seenseq_by_mbox_id); - int r = conversations_guid_foreach(req->cstate, _guid_from_id(msgid), + int r = conversations_guid_foreach(req->cstate, guidrep, _email_get_keywords_cb, &rock); *jkeywords = _email_keywords_to_jmap(&rock.keywords); _email_keywords_fini(&rock.keywords); @@ -7168,7 +7229,7 @@ static const struct blob_header_t { { NULL, NULL } }; -static const char *_encode_emailheader_blobid(const char *emailid, +static const char *_encode_emailheader_blobid(const char *blobid, const char *hdr, struct buf *dst) { @@ -7178,7 +7239,7 @@ static const char *_encode_emailheader_blobid(const char *emailid, /* Smart blob prefix, emailid, hdrname index */ buf_reset(dst); - if (blob_headers[n].name) buf_printf(dst, "H%s-%u", emailid, n); + if (blob_headers[n].name) buf_printf(dst, "H%s-%u", blobid, n); return buf_cstring(dst); } @@ -7265,10 +7326,14 @@ static int _email_get_meta(jmap_req_t *req, /* Determine message id */ struct message_guid guid; + struct timespec internaldate; r = msgrecord_get_guid(msg->mr, &guid); + if (!r) r = msgrecord_get_internaldate(msg->mr, &internaldate); if (r) goto done; - jmap_set_emailid(&guid, email_id); + const char *guidrep = message_guid_encode(&guid); + + jmap_set_emailid_from_timespec(&internaldate, email_id); /* id */ if (jmap_wantprop(props, "id")) { @@ -7277,9 +7342,7 @@ static int _email_get_meta(jmap_req_t *req, /* blobId */ if (jmap_wantprop(props, "blobId")) { - char blob_id[JMAP_BLOBID_SIZE]; - jmap_set_blobid(&guid, blob_id); - json_object_set_new(email, "blobId", json_string(blob_id)); + json_object_set_new(email, "blobId", json_pack("s+", "G", guidrep)); } /* threadid */ @@ -7301,7 +7364,7 @@ static int _email_get_meta(jmap_req_t *req, jmap_wantprop(props, "addedDates") ? json_object() : NULL; json_t *removed = jmap_wantprop(props, "removedDates") ? json_object() : NULL; - json_t *mailboxes = _email_mailboxes(req, email_id); + json_t *mailboxes = _email_mailboxes(req, guidrep); json_t *val; const char *mboxid; @@ -7325,7 +7388,7 @@ static int _email_get_meta(jmap_req_t *req, /* keywords */ if (jmap_wantprop(props, "keywords")) { json_t *keywords = NULL; - r = _email_get_keywords(req, &args->ctx, email_id, &keywords); + r = _email_get_keywords(req, &args->ctx, guidrep, &keywords); if (r) goto done; json_object_set_new(email, "keywords", keywords); } @@ -7381,7 +7444,7 @@ static int _email_get_meta(jmap_req_t *req, struct email_get_snoozed_rock rock = { req, NULL }; /* Look for the first snoozed copy of this email_id */ - conversations_guid_foreach(req->cstate, _guid_from_id(email_id), + conversations_guid_foreach(req->cstate, guidrep, _email_get_snoozed_cb, &rock); json_object_set_new(email, "snoozed", @@ -7398,7 +7461,7 @@ static int _email_get_meta(jmap_req_t *req, if (!r) r = message_get_field(msg->_m, hdrname, MESSAGE_RAW, &buf); if (!r && buf_len(&buf)) { const char *blobid = - _encode_emailheader_blobid(email_id, hdrname, &buf); + _encode_emailheader_blobid(guidrep, hdrname, &buf); if (*blobid) jval = json_string(blobid); } json_object_set_new(email, "bimiBlobId", jval); @@ -7409,7 +7472,7 @@ static int _email_get_meta(jmap_req_t *req, struct email_get_createdmodseq_rock rock = { req, 0 }; /* Look for the smalled createdmodseq for all index records */ - conversations_guid_foreach(req->cstate, _guid_from_id(email_id), + conversations_guid_foreach(req->cstate, guidrep, _email_get_createdmodseq_cb, &rock); json_object_set_new(email, "createdModseq", json_integer(rock.modseq)); @@ -8356,10 +8419,12 @@ static void jmap_email_get_full(jmap_req_t *req, struct jmap_get *get, struct em struct _warmup_mboxcache_cb_rock rock = { req, PTRARRAY_INITIALIZER }; json_array_foreach(get->ids, i, val) { const char *email_id = json_string_value(val); - if (email_id[0] != 'M' || strlen(email_id) != 25) { + if (email_id[0] != JMAP_EMAILID_PREFIX || + strlen(email_id) != JMAP_EMAILID_SIZE - 1) { continue; } - int r = conversations_guid_foreach(req->cstate, _guid_from_id(email_id), + int r = conversations_guid_foreach(req->cstate, + _guid_from_id(req->cstate, email_id), _warmup_mboxcache_cb, &rock); if (r) { /* Ignore errors, they'll be handled in email_find */ @@ -9106,7 +9171,6 @@ static void _email_append(jmap_req_t *req, if ((addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0))) { struct message_guid guid; message_guid_generate(&guid, addr, len); - jmap_set_emailid(&guid, detail->email_id); jmap_set_blobid(&guid, detail->blob_id); detail->size = len; munmap(addr, len); @@ -9117,11 +9181,14 @@ static void _email_append(jmap_req_t *req, fclose(f); f = NULL; + jmap_set_emailid_from_timespec(internaldate, detail->email_id); + /* Check if a message with this GUID already exists and is * visible for the authenticated user. */ char *exist_mboxname = NULL; uint32_t exist_uid; - r = jmap_email_find(req, NULL, detail->email_id, &exist_mboxname, &exist_uid); + r = jmap_email_find_by_guid(req, req->accountid, detail->blob_id+1, + &exist_mboxname, &exist_uid); free(exist_mboxname); if (r != IMAP_NOTFOUND) { if (!r) r = IMAP_MAILBOX_EXISTS; @@ -11475,13 +11542,15 @@ static void _email_mboxrecs_read(jmap_req_t *req, int i; for (i = 0; i < strarray_size(email_ids); i++) { const char *email_id = strarray_nth(email_ids, i); - if (email_id[0] != 'M' || strlen(email_id) != 25) { + if (email_id[0] != JMAP_EMAILID_PREFIX || + strlen(email_id) != JMAP_EMAILID_SIZE - 1) { // not a valid emailId continue; } struct email_mboxrecs_make_rock rock = { req, email_id, mboxrecs }; - int r = conversations_guid_foreach(cstate, _guid_from_id(email_id), + int r = conversations_guid_foreach(cstate, + _guid_from_id(cstate, email_id), _email_mboxrecs_read_cb, &rock); if (r) { json_t *err = (r == IMAP_NOTFOUND || r == IMAP_PERMISSION_DENIED) ? @@ -13488,7 +13557,8 @@ static void _email_update_bulk(jmap_req_t *req, /* Validate patched mailbox ids */ if (update->patch_mailboxids && !json_array_size(parser.invalid)) { - json_t *cur = _email_mailboxes(req, email_id); + json_t *cur = _email_mailboxes(req, + _guid_from_id(req->cstate, email_id)); if (!json_object_size(cur)) { json_object_set_new(not_updated, email_id, json_pack("{s:s}", "type", "notFound")); @@ -14270,7 +14340,8 @@ static void _email_copy_bulk(jmap_req_t *req, /* Check if email already exists in to_account */ struct _email_exists_rock data = { req, 0, 0 }; - conversations_guid_foreach(req->cstate, _guid_from_id(update->email_id), + conversations_guid_foreach(req->cstate, + _guid_from_id(req->cstate, update->email_id), _email_exists_cb, &data); if (data.exists) { json_object_set_new(copy->not_created, creation_id, @@ -14319,7 +14390,8 @@ static void _email_copy_bulk(jmap_req_t *req, struct _email_exists_rock data = { req, 0, 0 }; conversations_guid_foreach(req->cstate, - _guid_from_id(update->email_id), + _guid_from_id(req->cstate, + update->email_id), _email_exists_cb, &data); jmap_set_threadid(data.cid, thread_id); @@ -14476,18 +14548,18 @@ static int jmap_email_matchmime_method(jmap_req_t *req) } static int _decode_emailheader_blobid(const char *blobid, - char **emailidptr, + char **blobidptr, const char **hdrnameptr, const char **mimetypeptr) { - char *emailid = NULL; + char *email_blobid = NULL; int is_valid = 0; - /* Decode emailid */ + /* Decode blobid */ const char *base = blobid+1; const char *p = strchr(base, '-'); - if (!p || p-base != JMAP_EMAILID_SIZE-1) goto done; - emailid = xstrndup(base, p-base); + if (!p || p-base != JMAP_BLOBID_SIZE-2) goto done; + email_blobid = xstrndup(base, p-base); base = p + 1; /* Decode hdrname */ @@ -14500,13 +14572,13 @@ static int _decode_emailheader_blobid(const char *blobid, base = endptr; /* All done */ - *emailidptr = emailid; + *blobidptr = email_blobid; *hdrnameptr = blob_headers[index].name; *mimetypeptr = blob_headers[index].type; is_valid = 1; done: - if (!is_valid) free(emailid); + if (!is_valid) free(email_blobid); return is_valid; } @@ -14514,7 +14586,7 @@ static int _decode_emailheader_blobid(const char *blobid, static int jmap_emailheader_getblob(jmap_req_t *req, jmap_getblob_context_t *ctx) { struct mailbox *mailbox = NULL; - char *emailid = NULL; + char *blobid = NULL; const char *hdrname = NULL; const char *mimetype = NULL; char *mboxname = NULL; @@ -14524,13 +14596,14 @@ static int jmap_emailheader_getblob(jmap_req_t *req, jmap_getblob_context_t *ctx if (ctx->blobid[0] != 'H') return 0; - if (!_decode_emailheader_blobid(ctx->blobid, &emailid, &hdrname, &mimetype)) { + if (!_decode_emailheader_blobid(ctx->blobid, &blobid, &hdrname, &mimetype)) { res = HTTP_BAD_REQUEST; goto done; } /* Lookup emailid */ - r = jmap_email_find(req, ctx->from_accountid, emailid, &mboxname, &uid); + r = jmap_email_find_by_guid(req, ctx->from_accountid, blobid, + &mboxname, &uid); if (r) { if (r == IMAP_NOTFOUND) res = HTTP_NOT_FOUND; else { @@ -14626,7 +14699,7 @@ static int jmap_emailheader_getblob(jmap_req_t *req, jmap_getblob_context_t *ctx ctx->errstr = desc; } mailbox_close(&mailbox); - free(emailid); + free(blobid); free(mboxname); return res; } diff --git a/imap/jmap_util.c b/imap/jmap_util.c index 9515bdf5fa..9b3dade1dd 100644 --- a/imap/jmap_util.c +++ b/imap/jmap_util.c @@ -1306,11 +1306,14 @@ EXPORTED void jmap_set_blobid(const struct message_guid *guid, char *buf) buf[JMAP_BLOBID_SIZE-1] = '\0'; } -EXPORTED void jmap_set_emailid(const struct message_guid *guid, char *buf) +EXPORTED void jmap_set_emailid_from_nanosec(uint64_t nanosec, char *emailid) { - buf[0] = 'M'; - memcpy(buf+1, message_guid_encode(guid), JMAP_EMAILID_SIZE-2); - buf[JMAP_EMAILID_SIZE-1] = '\0'; + // initialize a struct buf with char emailid[JMAP_EMAILID_SIZE] + struct buf buf = { emailid, 0, JMAP_EMAILID_SIZE, 0 }; + + buf_putc(&buf, JMAP_EMAILID_PREFIX); + NANOSEC_TO_JMAPID(&buf, nanosec); + buf_cstring(&buf); } EXPORTED void jmap_set_threadid(conversation_id_t cid, char *buf) diff --git a/imap/jmap_util.h b/imap/jmap_util.h index a288777409..1d914464b8 100644 --- a/imap/jmap_util.h +++ b/imap/jmap_util.h @@ -193,8 +193,13 @@ extern int jmap_is_valid_id(const char *id); #define JMAP_BLOBID_SIZE 42 extern void jmap_set_blobid(const struct message_guid *guid, char *buf); -#define JMAP_EMAILID_SIZE 26 -extern void jmap_set_emailid(const struct message_guid *guid, char *buf); +#define JMAP_EMAILID_PREFIX 'S' +#define JMAP_EMAILID_SIZE (CONV_JMAPID_SIZE + 2) + +#define jmap_set_emailid_from_timespec(ts, buf) \ + jmap_set_emailid_from_nanosec(TIMESPEC_TO_NANOSEC(ts), buf) + +extern void jmap_set_emailid_from_nanosec(uint64_t nanosec, char *buf); #define JMAP_THREADID_SIZE 18 extern void jmap_set_threadid(conversation_id_t cid, char *buf); diff --git a/imap/mboxevent.c b/imap/mboxevent.c index 158c3839fc..5e02258921 100644 --- a/imap/mboxevent.c +++ b/imap/mboxevent.c @@ -973,11 +973,11 @@ static const char *threadid(bit64 cid) return id; } -static json_t *jmap_email(struct message_guid *guid, bit64 cid, struct body *body) +static json_t *jmap_email(struct timespec *internaldate, bit64 cid, struct body *body) { char emailid[JMAP_EMAILID_SIZE]; - jmap_set_emailid(guid, emailid); + jmap_set_emailid_from_timespec(internaldate, emailid); return json_pack("{ s:s s:s s:o s:o s:o s:o s:o s:o s:o s:o s:o s:o }", "id", emailid, @@ -1066,7 +1066,7 @@ EXPORTED void mboxevent_extract_record(struct mboxevent *event, struct mailbox * /* add message EMAILID */ if (mboxevent_expected_param(event->type, EVENT_MESSAGE_EMAILID)) { char *emailid = xmalloc(JMAP_EMAILID_SIZE); - jmap_set_emailid(&record->guid, emailid); + jmap_set_emailid_from_timespec(&record->internaldate, emailid); FILL_STRING_PARAM(event, EVENT_MESSAGE_EMAILID, emailid); } @@ -1082,7 +1082,7 @@ EXPORTED void mboxevent_extract_record(struct mboxevent *event, struct mailbox * return; message_read_bodystructure(record, &body); - json_t *email = jmap_email(&record->guid, record->cid, body); + json_t *email = jmap_email(&record->internaldate, record->cid, body); FILL_JSON_PARAM(event, EVENT_JMAP_EMAIL, email); } @@ -1241,13 +1241,13 @@ EXPORTED void mboxevent_extract_msgrecord(struct mboxevent *event, msgrecord_t * /* add message EMAILID */ if (mboxevent_expected_param(event->type, EVENT_MESSAGE_EMAILID)) { - struct message_guid guid; - if ((r = msgrecord_get_guid(msgrec, &guid))) { + struct timespec internaldate; + if ((r = msgrecord_get_internaldate(msgrec, &internaldate))) { syslog(LOG_ERR, "mboxevent: can't extract guid: %s", error_message(r)); return; } char *emailid = xmalloc(JMAP_EMAILID_SIZE); - jmap_set_emailid(&guid, emailid); + jmap_set_emailid_from_timespec(&internaldate, emailid); FILL_STRING_PARAM(event, EVENT_MESSAGE_EMAILID, emailid); } @@ -1263,10 +1263,10 @@ EXPORTED void mboxevent_extract_msgrecord(struct mboxevent *event, msgrecord_t * /* add vnd.fastmail.jmapEmail */ if (mboxevent_expected_param(event->type, EVENT_JMAP_EMAIL)) { - struct message_guid guid; + struct timespec internaldate; bit64 cid; - if ((r = msgrecord_get_guid(msgrec, &guid))) { - syslog(LOG_ERR, "mboxevent: can't extract guid: %s", error_message(r)); + if ((r = msgrecord_get_internaldate(msgrec, &internaldate))) { + syslog(LOG_ERR, "mboxevent: can't extract internaldate: %s", error_message(r)); return; } if ((r = msgrecord_get_cid(msgrec, &cid))) { @@ -1277,7 +1277,7 @@ EXPORTED void mboxevent_extract_msgrecord(struct mboxevent *event, msgrecord_t * syslog(LOG_ERR, "mboxevent: can't extract body: %s", error_message(r)); return; } - json_t *email = jmap_email(&guid, cid, body); + json_t *email = jmap_email(&internaldate, cid, body); FILL_JSON_PARAM(event, EVENT_JMAP_EMAIL, email); } From 12fbe0025ff61eb1d5ec77f5915578864836f670 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Tue, 21 Jan 2025 08:25:38 -0500 Subject: [PATCH 12/16] mailbox.c: rename OFFSET_THRID to OFFSET_CID --- cunit/mailbox.testc | 4 ++-- imap/mailbox.c | 8 ++++---- imap/mailbox.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cunit/mailbox.testc b/cunit/mailbox.testc index 46c3edcb46..94180ea8ac 100644 --- a/cunit/mailbox.testc +++ b/cunit/mailbox.testc @@ -122,6 +122,7 @@ static void test_aligned_record_offsets(void) CU_ASSERT_EQUAL(0, OFFSET_CACHE_CRC % alignof(r.cache_crc)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_OFFSET % alignof(XXX_CACHE32_TYPE)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_VERSION % alignof(XXX_CACHE32_TYPE)); + CU_ASSERT_EQUAL(0, OFFSET_CID % alignof(r.cid)); CU_ASSERT_EQUAL(0, OFFSET_CREATEDMODSEQ % alignof(r.createdmodseq)); CU_ASSERT_EQUAL(0, OFFSET_GMTIME % alignof(r.gmtime)); CU_ASSERT_EQUAL(0, OFFSET_HEADER_SIZE % alignof(r.header_size)); @@ -134,7 +135,6 @@ static void test_aligned_record_offsets(void) CU_ASSERT_EQUAL(0, OFFSET_SENTDATE % alignof(r.sentdate)); CU_ASSERT_EQUAL(0, OFFSET_SIZE % alignof(r.size)); CU_ASSERT_EQUAL(0, OFFSET_SYSTEM_FLAGS % alignof(r.system_flags)); - CU_ASSERT_EQUAL(0, OFFSET_THRID % alignof(r.cid)); CU_ASSERT_EQUAL(0, OFFSET_UID % alignof(r.uid)); CU_ASSERT_EQUAL(0, OFFSET_USER_FLAGS % alignof(r.user_flags)); /* this list is sorted alphabetically, don't just append */ @@ -246,6 +246,7 @@ static void test_unique_record_offsets(void) OFFSET(OFFSET_CACHE_CRC, sizeof(r.cache_crc)), OFFSET(OFFSET_CACHE_OFFSET, sizeof(XXX_CACHE32_TYPE)), OFFSET(OFFSET_CACHE_VERSION, sizeof(XXX_CACHE32_TYPE)), + OFFSET(OFFSET_CID, sizeof(r.cid)), OFFSET(OFFSET_CREATEDMODSEQ, sizeof(r.createdmodseq)), OFFSET(OFFSET_GMTIME, sizeof(r.gmtime)), OFFSET(OFFSET_HEADER_SIZE, sizeof(r.header_size)), @@ -258,7 +259,6 @@ static void test_unique_record_offsets(void) OFFSET(OFFSET_SENTDATE, sizeof(r.sentdate)), OFFSET(OFFSET_SIZE, sizeof(r.size)), OFFSET(OFFSET_SYSTEM_FLAGS, sizeof(r.system_flags)), - OFFSET(OFFSET_THRID, sizeof(r.cid)), OFFSET(OFFSET_UID, sizeof(r.uid)), OFFSET(OFFSET_USER_FLAGS, sizeof(r.user_flags)), /* this list is sorted alphabetically, don't just append */ diff --git a/imap/mailbox.c b/imap/mailbox.c index 683941b5e3..e700b50667 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -2086,7 +2086,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, goto crc; } - record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); + record->cid = ntohll(*(bit64 *)(buf+OFFSET_CID)); if (version > 14) { record->savedate = ntohl(*((bit32 *)(buf+PRE20_OFFSET_SAVEDATE))); } @@ -2120,7 +2120,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION))); message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID); record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ))); - record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); + record->cid = ntohll(*(bit64 *)(buf+OFFSET_CID)); record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ)); record->gmtime = ntohll(*((bit64 *)(buf+OFFSET_GMTIME))); record->last_updated = ntohll(*((bit64 *)(buf+OFFSET_LAST_UPDATED))); @@ -3270,7 +3270,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, goto crc; } - *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid); + *((bit64 *)(buf+OFFSET_CID)) = htonll(record->cid); /* version 16 added createdmodseq, pushing the CRCs down */ if (version < 16) { @@ -3301,7 +3301,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field); message_guid_export(&record->guid, (char *)buf+OFFSET_MESSAGE_GUID); *((bit64 *)(buf+OFFSET_MODSEQ)) = htonll(record->modseq); - *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid); + *((bit64 *)(buf+OFFSET_CID)) = htonll(record->cid); *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq); *((bit64 *)(buf+OFFSET_GMTIME)) = htonll(record->gmtime); *((bit64 *)(buf+OFFSET_LAST_UPDATED)) = htonll(record->last_updated); diff --git a/imap/mailbox.h b/imap/mailbox.h index 5f2d4cddb2..378450e8ca 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -406,7 +406,7 @@ struct mailbox_iter; #define OFFSET_CACHE_VERSION 56 #define OFFSET_MESSAGE_GUID 60 #define OFFSET_MODSEQ 80 /* CONDSTORE (64-bit modseq) */ -#define OFFSET_THRID 88 /* conversation id, added in v13 */ +#define OFFSET_CID 88 /* conversation id, added in v13 */ #define OFFSET_CREATEDMODSEQ 96 /* modseq of creation time, added in v16 */ #define OFFSET_GMTIME 104 /* grew to 64-bit in v20 */ #define OFFSET_LAST_UPDATED 112 /* grew to 64-bit in v20 */ From 688bc4b9b0feb21b07d2f9c8036b73322d96621a Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Wed, 22 Jan 2025 12:06:02 -0500 Subject: [PATCH 13/16] mailbox.c: add BASECID to v20 index record use a "virtual" annotation to pass this value via replication --- cunit/annotate.testc | 1 - cunit/mailbox.testc | 2 ++ imap/annotate.c | 1 + imap/mailbox.c | 66 ++++++++++++++++++++++++++++++------ imap/mailbox.h | 9 ++--- imap/mbexamine.c | 5 ++- imap/message.c | 4 +++ imap/sync_support.c | 12 +++++++ perl/imap/Cyrus/IndexFile.pm | 4 ++- 9 files changed, 86 insertions(+), 18 deletions(-) diff --git a/cunit/annotate.testc b/cunit/annotate.testc index 3d185567ee..5867b545d5 100644 --- a/cunit/annotate.testc +++ b/cunit/annotate.testc @@ -1146,7 +1146,6 @@ static void test_msg_copy(void) buf_printf(&path2, "%s/data/uuid/%c/%c/%s/cyrus.annotations", DBDIR, u2[0], u2[1], u2); CU_ASSERT_EQUAL(fexists(buf_cstring(&path1)), 0); - CU_ASSERT_EQUAL(fexists(buf_cstring(&path2)), 0); mailbox_close(&mailbox1); /* copy MBOXNAME1,1 -> MBOXNAME2,1 */ diff --git a/cunit/mailbox.testc b/cunit/mailbox.testc index 94180ea8ac..64a5ce6488 100644 --- a/cunit/mailbox.testc +++ b/cunit/mailbox.testc @@ -119,6 +119,7 @@ static void test_aligned_record_offsets(void) * not need to care about that. Instead, keep this list sorted * alphabetically by the OFFSET_... name, for ease of maintenance. */ + CU_ASSERT_EQUAL(0, OFFSET_BASECID % alignof(r.basecid)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_CRC % alignof(r.cache_crc)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_OFFSET % alignof(XXX_CACHE32_TYPE)); CU_ASSERT_EQUAL(0, OFFSET_CACHE_VERSION % alignof(XXX_CACHE32_TYPE)); @@ -243,6 +244,7 @@ static void test_unique_record_offsets(void) /* Keep this sorted alphabetically by the OFFSET_... name, for ease of * maintenance. We'll qsort it into the order the test needs shortly. */ + OFFSET(OFFSET_BASECID, sizeof(r.basecid)), OFFSET(OFFSET_CACHE_CRC, sizeof(r.cache_crc)), OFFSET(OFFSET_CACHE_OFFSET, sizeof(XXX_CACHE32_TYPE)), OFFSET(OFFSET_CACHE_VERSION, sizeof(XXX_CACHE32_TYPE)), diff --git a/imap/annotate.c b/imap/annotate.c index 1022cc2382..efeb55996f 100644 --- a/imap/annotate.c +++ b/imap/annotate.c @@ -2071,6 +2071,7 @@ static const annotate_entrydesc_t message_builtin_entries[] = }, { /* we use 'basethrid' to support split threads */ + /* prior to version 20, there was no storage for basethrid, so it became an annotation */ IMAP_ANNOT_NS "basethrid", ATTRIB_TYPE_STRING, BACKEND_ONLY, diff --git a/imap/mailbox.c b/imap/mailbox.c index e700b50667..703fa2cb9d 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -2125,6 +2125,7 @@ static int mailbox_buf_to_index_record(const char *buf, int version, record->gmtime = ntohll(*((bit64 *)(buf+OFFSET_GMTIME))); record->last_updated = ntohll(*((bit64 *)(buf+OFFSET_LAST_UPDATED))); record->savedate = ntohll(*((bit64 *)(buf+OFFSET_SAVEDATE))); + record->basecid = ntohll(*(bit64 *)(buf+OFFSET_BASECID)); offset_cache_crc = OFFSET_CACHE_CRC; offset_record_crc = OFFSET_RECORD_CRC; @@ -2207,18 +2208,20 @@ static int _store_change(struct mailbox *mailbox, struct index_record *record, i free(c_env); } - annotate_state_t *astate = NULL; - int r = mailbox_get_annotate_state(mailbox, record->uid, &astate); - if (r) return r; + if (mailbox->i.minor_version < 20) { + annotate_state_t *astate = NULL; + int r = mailbox_get_annotate_state(mailbox, record->uid, &astate); + if (r) return r; - struct buf annotval = BUF_INITIALIZER; - if (record->cid && record->basecid && record->basecid != record->cid) - buf_printf(&annotval, CONV_FMT, record->basecid); + struct buf annotval = BUF_INITIALIZER; + if (record->cid && record->basecid && record->basecid != record->cid) + buf_printf(&annotval, CONV_FMT, record->basecid); - r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval); - buf_free(&annotval); + r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval); + buf_free(&annotval); - if (r) return r; + if (r) return r; + } return 0; } @@ -3306,6 +3309,7 @@ static bit32 mailbox_index_record_to_buf(struct index_record *record, *((bit64 *)(buf+OFFSET_GMTIME)) = htonll(record->gmtime); *((bit64 *)(buf+OFFSET_LAST_UPDATED)) = htonll(record->last_updated); *((bit64 *)(buf+OFFSET_SAVEDATE)) = htonll(record->savedate); + *((bit64 *)(buf+OFFSET_BASECID)) = htonll(record->basecid); offset_cache_crc = OFFSET_CACHE_CRC; offset_record_crc = OFFSET_RECORD_CRC; @@ -3457,6 +3461,9 @@ static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry) // internaldate.nsec was introduced in v20 if (!strcmp(entry, IMAP_ANNOT_NS "internaldate.nsec")) return 1; + // basethrid was introduced as virtual in v20 + else if (!strcmp(entry, IMAP_ANNOT_NS "basethrid")) return 1; + GCC_FALLTHROUGH case 19: @@ -3506,6 +3513,12 @@ static uint32_t crc_virtannot(struct mailbox *mailbox, buf_reset(&buf); } + if (record->basecid) { + buf_printf(&buf, CONV_FMT, record->basecid); + crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "basethrid", "", &buf); + buf_reset(&buf); + } + GCC_FALLTHROUGH case 19: @@ -5102,8 +5115,8 @@ static int mailbox_repack_setup(struct mailbox *mailbox, int version, break; case 20: repack->newmailbox.i.start_offset = 160; - /* version 20 grew size and time fields to 64-bits */ - repack->newmailbox.i.record_size = 136; + /* version 20 grew size and time fields to 64-bits, and added basecid */ + repack->newmailbox.i.record_size = 144; break; default: fatal("index version not supported", EX_SOFTWARE); @@ -5523,6 +5536,17 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) buf_reset(&buf); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); if (r) goto done; + + /* extract BaseCID */ + buf_reset(&buf); + mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "basethrid", "", &buf); + if (buf.len == 16) { + const char *p = buf_cstring(&buf); + parsehex(p, &p, 16, ©record.basecid); + } + buf_reset(&buf); + r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &buf); + if (r) goto done; } if (mailbox->i.minor_version >= 20 && repack->newmailbox.i.minor_version < 20) { if (record->internaldate.tv_nsec != UTIME_OMIT) { @@ -5531,6 +5555,13 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); if (r) goto done; } + + if (record->basecid) { + buf_reset(&buf); + buf_printf(&buf, CONV_FMT, record->basecid); + r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &buf); + if (r) goto done; + } } /* read in the old cache record */ @@ -8197,6 +8228,19 @@ EXPORTED int mailbox_reconstruct(const char *name, int flags, struct mailbox **m if (r) goto close; } } + + if (mailbox->i.minor_version >= 20) { + buf_reset(&buf); + mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", "", &buf); + if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", NULL, &buf); + if (buf.len) { + syslog(LOG_NOTICE, "removing stale basethrid for %u", record.uid); + printf("removing stale basethrid for %u\n", record.uid); + buf_reset(&buf); + r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "basethrid", "", &buf); + if (r) goto close; + } + } } /* make sure appends will be allowed, fixing last_uid to be at least diff --git a/imap/mailbox.h b/imap/mailbox.h index 378450e8ca..210a294704 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -164,15 +164,15 @@ struct index_record { uint16_t cache_version; struct message_guid guid; modseq_t modseq; - bit64 cid; modseq_t createdmodseq; + bit64 cid; + bit64 basecid; bit32 cache_crc; /* metadata */ uint32_t recno; unsigned silentupdate:1; unsigned ignorelimits:1; - bit64 basecid; struct cacherecord crec; }; @@ -411,8 +411,9 @@ struct mailbox_iter; #define OFFSET_GMTIME 104 /* grew to 64-bit in v20 */ #define OFFSET_LAST_UPDATED 112 /* grew to 64-bit in v20 */ #define OFFSET_SAVEDATE 120 /* added in v15 */ -#define OFFSET_CACHE_CRC 128 /* CRC32 of cache record */ -#define OFFSET_RECORD_CRC 132 +#define OFFSET_BASECID 128 /* base conversation id, added in v20 */ +#define OFFSET_CACHE_CRC 136 /* CRC32 of cache record */ +#define OFFSET_RECORD_CRC 140 #define PRE20_OFFSET_INTERNALDATE 4 #define PRE20_OFFSET_SENTDATE 8 diff --git a/imap/mbexamine.c b/imap/mbexamine.c index fe874d6869..800eba247a 100644 --- a/imap/mbexamine.c +++ b/imap/mbexamine.c @@ -332,7 +332,10 @@ static int do_examine(struct findall_data *data, void *rock) printf(" MODSEQ:" MODSEQ_FMT, record->modseq); if (mailbox->i.minor_version >= 13) { - printf(" THRID: " CONV_FMT, record->cid); + printf(" CID: " CONV_FMT, record->cid); + + if (mailbox->i.minor_version >= 20) + printf(" BASECID: " CONV_FMT, record->basecid); } } } diff --git a/imap/message.c b/imap/message.c index 0fffe12342..e658cd8861 100644 --- a/imap/message.c +++ b/imap/message.c @@ -4047,6 +4047,10 @@ EXPORTED int message_update_conversations(struct conversations_state *state, /* mark that it's split so basecid gets saved */ if (record->basecid != record->cid) record->internal_flags |= FLAG_INTERNAL_SPLITCONVERSATION; + else { + /* otherwise, we DO NOT want/expect basecid to be set */ + record->basecid = NULLCONVERSATION; + } out: message_unref(&msg); diff --git a/imap/sync_support.c b/imap/sync_support.c index c41b3b5e22..5fa7898b6b 100644 --- a/imap/sync_support.c +++ b/imap/sync_support.c @@ -1462,6 +1462,16 @@ void encode_annotations(struct dlist *parent, dlist_setnum64(aa, "VALUE", record->internaldate.tv_nsec); } + if (record->basecid) { + if (!annots) + annots = dlist_newlist(parent, "ANNOTATIONS"); + aa = dlist_newkvlist(annots, NULL); + dlist_setatom(aa, "ENTRY", IMAP_ANNOT_NS "basethrid"); + dlist_setatom(aa, "USERID", ""); + dlist_setnum64(aa, "MODSEQ", 0); + dlist_sethex64(aa, "VALUE", record->basecid); + } + GCC_FALLTHROUGH case 19: @@ -1598,6 +1608,8 @@ int decode_annotations(/*const*/struct dlist *annots, parsehex(p, &p, 16, &record->basecid); /* XXX - check on p? */ + if (mailbox->i.minor_version >= 20) break; + /* "basethrid" is special, since it is written during mailbox * appends and rewrites, using whatever modseq the index_record * has at this moment. This might differ from the modseq we diff --git a/perl/imap/Cyrus/IndexFile.pm b/perl/imap/Cyrus/IndexFile.pm index 20af9b4fbd..a5bd8d59ba 100755 --- a/perl/imap/Cyrus/IndexFile.pm +++ b/perl/imap/Cyrus/IndexFile.pm @@ -651,6 +651,7 @@ SKIPPED VERSION 11 - Fastmail internal only 13: GmTime time_t 8 14: LastUpdated time_t 8 15: SaveDate time_t 8 + 11: BaseCID hex 8 16: CacheCRC int32 4 17: RecordCRC int32 4 @@ -1307,7 +1308,7 @@ QuotaExpungedUsed int64 8 ChangesEpoch int32 4 HeaderCrc int32 4 EOF - RecordSize => 136, # defined in file too, check it! + RecordSize => 144, # defined in file too, check it! _make_fields('Record', < Date: Fri, 7 Feb 2025 08:22:57 -0500 Subject: [PATCH 14/16] append.c: use internaldate of existing message when appending a duplicate --- cassandane/Cassandane/Cyrus/Append.pm | 88 +++++++++++++++++++ .../Append/append_duplicate_diff_internaldate | 43 +++++++++ imap/append.c | 58 +++++++++--- 3 files changed, 175 insertions(+), 14 deletions(-) create mode 100644 cassandane/Cassandane/Cyrus/Append.pm create mode 100644 cassandane/tiny-tests/Append/append_duplicate_diff_internaldate diff --git a/cassandane/Cassandane/Cyrus/Append.pm b/cassandane/Cassandane/Cyrus/Append.pm new file mode 100644 index 0000000000..5c625fd9d2 --- /dev/null +++ b/cassandane/Cassandane/Cyrus/Append.pm @@ -0,0 +1,88 @@ +#!/usr/bin/perl +# +# Copyright (c) 2011-2023 FastMail Pty Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. The name "Fastmail Pty Ltd" must not be used to +# endorse or promote products derived from this software without +# prior written permission. For permission or any legal +# details, please contact +# FastMail Pty Ltd +# PO Box 234 +# Collins St West 8007 +# Victoria +# Australia +# +# 4. Redistributions of any form whatsoever must retain the following +# acknowledgment: +# "This product includes software developed by Fastmail Pty. Ltd." +# +# FASTMAIL PTY LTD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO +# EVENT SHALL OPERA SOFTWARE AUSTRALIA BE LIABLE FOR ANY SPECIAL, INDIRECT +# OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# + +package Cassandane::Cyrus::Append; +use strict; +use warnings; +use DateTime; +use JSON; +use JSON::XS; +use Mail::JMAPTalk 0.13; +use Data::Dumper; +use Storable 'dclone'; +use File::Basename; +use IO::File; + +use lib '.'; +use base qw(Cassandane::Cyrus::TestCase); +use Cassandane::Util::Log; + +use charnames ':full'; + +sub new +{ + my ($class, @args) = @_; + + my $config = Cassandane::Config->default()->clone(); + + $config->set(conversations => 'yes'); + + my $self = $class->SUPER::new({ + config => $config, + services => [ 'imap' ] + }, @args); + + return $self; +} + +sub set_up +{ + my ($self) = @_; + $self->SUPER::set_up(); +} + +sub tear_down +{ + my ($self) = @_; + $self->SUPER::tear_down(); +} + +use Cassandane::Tiny::Loader 'tiny-tests/Append'; + +1; diff --git a/cassandane/tiny-tests/Append/append_duplicate_diff_internaldate b/cassandane/tiny-tests/Append/append_duplicate_diff_internaldate new file mode 100644 index 0000000000..2ed69dedc0 --- /dev/null +++ b/cassandane/tiny-tests/Append/append_duplicate_diff_internaldate @@ -0,0 +1,43 @@ +#!perl +use Cassandane::Tiny; + +sub test_append_duplicate_diff_internaldate +{ + my ($self) = @_; + + my $imaptalk = $self->{store}->get_client(); + + xlog $self, "Append duplicate messages with different internaldates"; + my $mimeMessage = <<'EOF'; +From: +To: to@local +Subject: test +Date: Mon, 13 Apr 2020 15:34:03 +0200 +MIME-Version: 1.0 +Content-Type: text/plain + +test +EOF + $mimeMessage =~ s/\r?\n/\r\n/gs; + + + $imaptalk->append('INBOX', $mimeMessage) || die $@; + $imaptalk->append('INBOX', + '17-Aug-2023 15:13:54 +0200', $mimeMessage) || die $@; + + $imaptalk->create('foo'); + $imaptalk->append('foo', + '26-Jan-2025 15:13:54 -0500', $mimeMessage) || die $@; + + xlog $self, "Verify that all messages have the same internaldates"; + $imaptalk->examine('INBOX'); + my $res = $imaptalk->fetch('1:*', 'INTERNALDATE'); + my $internaldate = $res->{1}->{internaldate}; + $self->assert_str_equals($internaldate, $res->{2}->{internaldate}); + + $imaptalk->examine('foo'); + $res = $imaptalk->fetch('1:*', 'INTERNALDATE'); + $self->assert_str_equals($internaldate, $res->{1}->{internaldate}); +} + +1; diff --git a/imap/append.c b/imap/append.c index 7dfe36baad..e20242859e 100644 --- a/imap/append.c +++ b/imap/append.c @@ -886,6 +886,7 @@ static int append_apply_flags(struct appendstate *as, struct findstage_cb_rock { const char *partition; const char *guid; + struct timespec *internaldate; char *fname; }; @@ -893,13 +894,32 @@ static int findstage_cb(const conv_guidrec_t *rec, void *vrock) { struct findstage_cb_rock *rock = vrock; mbentry_t *mbentry = NULL; + int r, ret = 0; if (rec->part) return 0; + + if (rock->internaldate && + !(rec->internal_flags & FLAG_INTERNAL_EXPUNGED)) { + r = conv_guidrec_mbentry(rec, &mbentry); + if (r) return 0; + + if (mbtype_isa(mbentry->mbtype) == MBTYPE_EMAIL) { + // found a non-expunged duplicate email; use its internaldate + TIMESPEC_FROM_NANOSEC(rock->internaldate, rec->internaldate); + if (!rock->partition) { + ret = CYRUSDB_DONE; + goto done; + } + } + } + // no point copying from archive, spool is on data - if (rec->internal_flags & FLAG_INTERNAL_ARCHIVED) return 0; + if (rec->internal_flags & FLAG_INTERNAL_ARCHIVED) goto done; - int r = conv_guidrec_mbentry(rec, &mbentry); - if (r) return 0; + if (!mbentry) { + r = conv_guidrec_mbentry(rec, &mbentry); + if (r) return 0; + } if (!strcmp(rock->partition, mbentry->partition)) { struct stat sbuf; @@ -921,9 +941,10 @@ static int findstage_cb(const conv_guidrec_t *rec, void *vrock) } } + done: mboxlist_entry_free(&mbentry); - return rock->fname ? CYRUSDB_DONE : 0; + return rock->fname ? CYRUSDB_DONE : ret; } /* @@ -974,30 +995,39 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, mboxlist_findstage(mailbox_name(mailbox), stagefile, sizeof(stagefile)); strlcat(stagefile, stage->fname, sizeof(stagefile)); - if (!nolink) { + uint32_t mbtype = mbtype_isa(mailbox_mbtype(mailbox)); + struct timespec existing_internaldate = { 0, UTIME_OMIT }; + if (!nolink || mbtype == MBTYPE_EMAIL) { /* attempt to find an existing message with the same guid - and use it as the stagefile */ + and use it as the stagefile and/or use its internaldate */ struct conversations_state *cstate = mailbox_get_cstate(mailbox); if (cstate) { - char *guid = xstrdup(message_guid_encode(&(*body)->guid)); - struct findstage_cb_rock rock = { mailbox_partition(mailbox), guid, NULL }; + char guid[2*MESSAGE_GUID_SIZE+1]; + struct findstage_cb_rock rock = { + nolink ? NULL : mailbox_partition(mailbox), + strcpy(guid, message_guid_encode(&(*body)->guid)), + mbtype == MBTYPE_EMAIL ? &existing_internaldate : NULL, + NULL/*fname*/ + }; // ignore errors, it's OK for this to fail conversations_guid_foreach(cstate, guid, findstage_cb, &rock); - // if we found a file, remember it + // if we found a matching message, use its internaldate + if (existing_internaldate.tv_nsec != UTIME_OMIT) + internaldate = &existing_internaldate; + + // if we found a file, use it if (rock.fname) { - syslog(LOG_NOTICE, "found existing file %s for %s; linking", guid, rock.fname); + syslog(LOG_NOTICE, "found existing file %s for %s; linking", + guid, rock.fname); linkfile = rock.fname; + goto havefile; } - - free(guid); } } - if (linkfile) goto havefile; - for (i = 0 ; i < stage->parts.count ; i++) { /* ok, we've successfully created the file */ if (!strcmp(stagefile, stage->parts.data[i])) { From 517d65ebd0ec34ee01bbfe7e27a8429052294bd6 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Tue, 28 Jan 2025 12:18:41 -0500 Subject: [PATCH 15/16] append.c: check for and resolve JMAP ID conflicts during append --- .../Append/append_same_internaldate | 36 +++++++++++++++++++ imap/append.c | 5 +++ imap/conversations.c | 33 +++++++++++++++++ imap/conversations.h | 3 ++ 4 files changed, 77 insertions(+) create mode 100644 cassandane/tiny-tests/Append/append_same_internaldate diff --git a/cassandane/tiny-tests/Append/append_same_internaldate b/cassandane/tiny-tests/Append/append_same_internaldate new file mode 100644 index 0000000000..a5e447ca62 --- /dev/null +++ b/cassandane/tiny-tests/Append/append_same_internaldate @@ -0,0 +1,36 @@ +#!perl +use Cassandane::Tiny; + +# This test is used to verify that our JMAPID conflict detection/resolution works +# but it takes a very long time to run and may not even generate a conflict +sub bogus_test_append_same_internaldate +{ + my ($self) = @_; + + my $imaptalk = $self->{store}->get_client(); + + xlog $self, "Append messages with same internaldates"; + for (1..100000) { + my $mimeMessage = << "EOF"; +From: +To: to\@local +Subject: test $_ + +test +EOF + $mimeMessage =~ s/\r?\n/\r\n/gs; + + $imaptalk->append('INBOX', + '26-Jan-2025 15:13:54 -0500', $mimeMessage) || die $@; + } + + xlog $self, "Verify that all messages have the same internaldates"; + $imaptalk->examine('INBOX'); + my $res = $imaptalk->fetch('1:*', 'INTERNALDATE'); + + # Did we log an conflicts? + $self->assert_syslog_matches($self->{instance}, + qr/IOERROR: JMAPID conflict during append/); +} + +1; diff --git a/imap/append.c b/imap/append.c index e20242859e..992f3929aa 100644 --- a/imap/append.c +++ b/imap/append.c @@ -1018,6 +1018,11 @@ EXPORTED int append_fromstage_full(struct appendstate *as, struct body **body, if (existing_internaldate.tv_nsec != UTIME_OMIT) internaldate = &existing_internaldate; + else if (internaldate) { + // make sure we don't have a JMAP ID (internaldate) clash + conversations_adjust_internaldate(cstate, guid, internaldate); + } + // if we found a file, use it if (rock.fname) { syslog(LOG_NOTICE, "found existing file %s for %s; linking", diff --git a/imap/conversations.c b/imap/conversations.c index d4d5d56759..405635ddf1 100644 --- a/imap/conversations.c +++ b/imap/conversations.c @@ -3371,4 +3371,37 @@ EXPORTED int conversations_jmapid_guidrep_lookup(struct conversations_state *sta return 0; } +EXPORTED void conversations_adjust_internaldate(struct conversations_state *cstate, + const char *my_guid, + struct timespec *internaldate) +{ + struct buf jidrep = BUF_INITIALIZER; + + // check for a JMAPID (internaldate) clash, and adjust nanosec as needed + do { + char existing_guid[2*MESSAGE_GUID_SIZE+1]; + + buf_reset(&jidrep); + NANOSEC_TO_JMAPID(&jidrep, TIMESPEC_TO_NANOSEC(internaldate)); + int r = conversations_jmapid_guidrep_lookup(cstate, + buf_cstring(&jidrep), + existing_guid); + if (r || !strcmp(my_guid, existing_guid)) { + // JMAP ID doesn't exist or it references our GUID + break; + } + + xsyslog(LOG_INFO, "IOERROR: JMAPID conflict during append," + " incrementing internaldate.tv_nsec", + "JMAPID=<%s> GUID=<%s> existing_GUID=<%s>", + buf_cstring(&jidrep), my_guid, existing_guid); + + // try the next nanosecond */ + internaldate->tv_nsec = (internaldate->tv_nsec + 1) % 1000000000; + + } while (1); + + buf_free(&jidrep); +} + #undef DB diff --git a/imap/conversations.h b/imap/conversations.h index 2378112dc8..903d79b4ca 100644 --- a/imap/conversations.h +++ b/imap/conversations.h @@ -294,6 +294,9 @@ extern int conversations_guid_cid_lookup(struct conversations_state *state, #define CONV_JMAPID_SIZE 11 // 64-bits base64-encoded w/o padding extern int conversations_jmapid_guidrep_lookup(struct conversations_state *state, const char *jidrep, char guidrep[]); +extern void conversations_adjust_internaldate(struct conversations_state *cstate, + const char *guidrep, + struct timespec *internaldate); /* F record items */ extern int conversation_getstatus(struct conversations_state *state, From a91fca054d06be8008def63fe8e06313d7b4f078 Mon Sep 17 00:00:00 2001 From: Ken Murchison Date: Fri, 7 Feb 2025 12:55:12 -0500 Subject: [PATCH 16/16] mailbox.c: properly reconstruct v19 -> v20 - use the internaldate of an existing message with same UID - pick internaldate.nsec in a deterministic manner - make sure internaldate doesn't conflict with an existing JMAPID - set the timestamps on the message data file to internaldate - update G record and add J record in conv.db --- imap/mailbox.c | 130 ++++++++++++++++++++++++++++++++++++++++++------- imap/mailbox.h | 3 ++ 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/imap/mailbox.c b/imap/mailbox.c index 703fa2cb9d..d97c2877b3 100644 --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -4860,25 +4860,10 @@ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, } } - int object_storage_enabled = 0 ; -#if defined ENABLE_OBJECTSTORE - object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; -#endif - if (!(record->internal_flags & FLAG_INTERNAL_UNLINKED)) { /* make the file timestamp correct */ - if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) { // maybe there is no file in directory. - struct timespec settimes[] = { - { record->internaldate.tv_sec, record->internaldate.tv_nsec }, - { record->internaldate.tv_sec, record->internaldate.tv_nsec } - }; - const char *fname = mailbox_record_fname(mailbox, record); - r = utimensat(AT_FDCWD, fname, settimes, 0); - if (r == -1) { - syslog(LOG_ERR, "failed to set mtime on %s: %m", fname); - return IMAP_IOERROR; - } - } + r = mailbox_set_datafile_timestamps(mailbox, record); + if (r) return r; /* write the cache record before buffering the message, it * will set the cache_offset field. */ @@ -5378,10 +5363,38 @@ HIDDEN int mailbox_repack_commit(struct mailbox_repack **repackptr) return r; } +static int find_dup_cb(const conv_guidrec_t *rec, void *rock) +{ + struct timespec *internaldate = (struct timespec *) rock; + mbentry_t *mbentry = NULL; + int ret = 0; + + if (rec->part) return 0; + + if (!(rec->internal_flags & FLAG_INTERNAL_EXPUNGED)) { + int r = conv_guidrec_mbentry(rec, &mbentry); + if (r) return 0; + + if (mbtype_isa(mbentry->mbtype) == MBTYPE_EMAIL) { + // found a non-expunged duplicate email; use its internaldate + TIMESPEC_FROM_NANOSEC(internaldate, rec->internaldate); + ret = CYRUSDB_DONE; + goto done; + } + } + + done: + mboxlist_entry_free(&mbentry); + + return ret; +} + /* need a mailbox exclusive lock, we're rewriting files */ static int mailbox_index_repack(struct mailbox *mailbox, int version) { struct mailbox_repack *repack = NULL; + struct conversations_state *cstate = NULL; + uint32_t mbtype = mbtype_isa(mailbox_mbtype(mailbox)); const message_t *msg; struct mailbox_iter *iter = NULL; struct buf buf = BUF_INITIALIZER; @@ -5392,6 +5405,13 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) r = mailbox_repack_setup(mailbox, version, &repack); if (r) goto done; + if (mailbox->i.minor_version < 20 && + repack->newmailbox.i.minor_version >= 20 && + !(cstate = mailbox_get_cstate(mailbox))) { + r = IMAP_IOERROR; + goto done; + } + iter = mailbox_iter_init(mailbox, 0, 0); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); @@ -5533,6 +5553,55 @@ static int mailbox_index_repack(struct mailbox *mailbox, int version) parsenum(p, &p, 0, &newval); copyrecord.internaldate.tv_nsec = newval; } + else { + /* assign internaldate.nsec in a deterministic manner - + * use the first 29 bits of GUID + * (0x1FFFFFFF < 999999999 nanoseconds) + */ + copyrecord.internaldate.tv_nsec = + *((uint32_t *) record->guid.value) >> 3; + + if (mbtype == MBTYPE_EMAIL) { + /* attempt to find an existing message with the same guid + and use its internaldate instead */ + struct timespec existing_internaldate = { 0, UTIME_OMIT }; + char guid[2*MESSAGE_GUID_SIZE+1]; + + strcpy(guid, message_guid_encode(&record->guid)); + + // ignore errors, it's OK for this to fail + conversations_guid_foreach(cstate, guid, find_dup_cb, + &existing_internaldate); + + // if we found a matching message, use its internaldate + if (existing_internaldate.tv_nsec != UTIME_OMIT) { + copyrecord.internaldate.tv_sec = + existing_internaldate.tv_sec; + copyrecord.internaldate.tv_nsec = + existing_internaldate.tv_nsec; + } + else { + // make sure we don't have a JMAP ID (internaldate) clash + conversations_adjust_internaldate(cstate, guid, + ©record.internaldate); + } + } + + /* make the file timestamp correct */ + r = mailbox_set_datafile_timestamps(mailbox, ©record); + if (r) goto done; + + /* update G & J records */ + r = mailbox_update_conversations(mailbox, record, ©record); + if (r) goto done; + + /* add virtual annotation to old crc */ + buf_reset(&buf); + buf_printf(&buf, UINT64_FMT, copyrecord.internaldate.tv_nsec); + repack->crcs.annot ^= + crc_annot(record->uid, + IMAP_ANNOT_NS "internaldate.nsec", "", &buf); + } buf_reset(&buf); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "internaldate.nsec", "", &buf); if (r) goto done; @@ -8758,3 +8827,30 @@ EXPORTED struct mboxlist_entry *mailbox_mbentry_from_path(const char *header_pat return mbentry; } + +EXPORTED int mailbox_set_datafile_timestamps(struct mailbox *mailbox, + struct index_record *record) +{ + int object_storage_enabled = 0; +#if defined ENABLE_OBJECTSTORE + object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; +#endif + + if (object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED)) { + // there is no file in directory + return 0; + } + + const char *fname = mailbox_record_fname(mailbox, record); + struct timespec settimes[] = { + { record->internaldate.tv_sec, record->internaldate.tv_nsec }, + { record->internaldate.tv_sec, record->internaldate.tv_nsec } + }; + + if (utimensat(AT_FDCWD, fname, settimes, 0) == -1) { + syslog(LOG_ERR, "failed to set mtime on %s: %m", fname); + return IMAP_IOERROR; + } + + return 0; +} diff --git a/imap/mailbox.h b/imap/mailbox.h index 210a294704..8b0780dde6 100644 --- a/imap/mailbox.h +++ b/imap/mailbox.h @@ -801,4 +801,7 @@ extern int mailbox_parse_datafilename(const char *name, uint32_t *uidp); extern struct mboxlist_entry *mailbox_mbentry_from_path(const char *header_path); +extern int mailbox_set_datafile_timestamps(struct mailbox *mailbox, + struct index_record *record); + #endif /* INCLUDED_MAILBOX_H */