From 7a035e6a53449b77cceae1d8045c51c4063b0ab6 Mon Sep 17 00:00:00 2001
From: Dan Callaghan <djc@djc.id.au>
Date: Sun, 12 May 2024 14:07:11 +1000
Subject: [PATCH] use a fallback date if Date header is missing

If a message doesn't have a usable Date header, we can fall back to
using the modtime of the file for Maildir, or the delivery time which
should have been written to the From line in mbox.

If all else fails, use a fixed date in the past, rather than Time.now.
This will prevent the mails from popping back to the top of the thread
list every time Sup reloads them.

Fixes #596.
---
 lib/sup/maildir.rb               |  4 ++++
 lib/sup/mbox.rb                  | 21 +++++++++++++++++++++
 lib/sup/message.rb               | 16 +++++++---------
 test/dummy_source.rb             |  4 ++++
 test/integration/test_maildir.rb | 16 +++++++++++++++-
 test/integration/test_mbox.rb    | 10 ++++++++++
 6 files changed, 61 insertions(+), 10 deletions(-)

diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
index 28b433ed0..1883844a2 100644
--- a/lib/sup/maildir.rb
+++ b/lib/sup/maildir.rb
@@ -114,6 +114,10 @@ def raw_message id
     with_file_for(id) { |f| f.read }
   end
 
+  def fallback_date_for_message id
+    File.mtime File.join(@dir, id)
+  end
+
   ## XXX use less memory
   def poll
     added = []
diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb
index 61a220eca..ec931e349 100644
--- a/lib/sup/mbox.rb
+++ b/lib/sup/mbox.rb
@@ -1,5 +1,6 @@
 require 'uri'
 require 'set'
+require 'time'
 
 module Redwood
 
@@ -137,6 +138,26 @@ def each_raw_message_line offset
     end
   end
 
+  def fallback_date_for_message offset
+    ## This is a bit awkward... We treat the From line as a delimiter,
+    ## not part of the message. So the offset is pointing *after* the
+    ## From line for the desired message. With a bit of effort we can
+    ## scan backwards to find its From line and extract a date from it.
+    buf = @mutex.synchronize do
+      ensure_open
+      start = offset
+      loop do
+        start = (start - 200).clamp 0, 2**64
+        @f.seek start
+        buf = @f.read (offset - start)
+        break buf if buf.include? ?\n or start == 0
+      end
+    end
+    BREAK_RE.match buf.lines.last do |m|
+      Time.strptime m[1], "%a %b %d %H:%M:%S %Y"
+    end
+  end
+
   def default_labels
     [:inbox, :unread]
   end
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
index 6ee185370..0a0bcc6eb 100644
--- a/lib/sup/message.rb
+++ b/lib/sup/message.rb
@@ -103,16 +103,10 @@ def parse_header encoded_header
     when Time
       date
     when String
-      begin
-        Time.parse date
-      rescue ArgumentError
-        #debug "faking mangled date header for #{@id} (orig #{header['date'].inspect} gave error: #{e.message})"
-        Time.now
-      end
-    else
-      #debug "faking non-existent date header for #{@id}"
-      Time.now
+      Time.parse date rescue nil
     end
+    @date = location.fallback_date if @date.nil?
+    @date = Time.utc 1970, 1, 1 if @date.nil?
 
     subj = header["subject"]
     subj = subj ? subj.fix_encoding! : nil
@@ -808,6 +802,10 @@ def parsed_message
     source.load_message info
   end
 
+  def fallback_date
+    source.fallback_date_for_message info
+  end
+
   def valid?
     source.valid? info
   end
diff --git a/test/dummy_source.rb b/test/dummy_source.rb
index ac907a238..d85153136 100644
--- a/test/dummy_source.rb
+++ b/test/dummy_source.rb
@@ -59,6 +59,10 @@ def each_raw_message_line id
       end
     end
   end
+
+  def fallback_date_for_message id
+    Time.utc 2001, 2, 3, 4, 56, 57
+  end
 end
 
 end
diff --git a/test/integration/test_maildir.rb b/test/integration/test_maildir.rb
index 2211b4402..9249227a6 100644
--- a/test/integration/test_maildir.rb
+++ b/test/integration/test_maildir.rb
@@ -34,7 +34,9 @@ def create_a_maildir(extra='')
   end
 
   def create_a_maildir_email(folder, content)
-    File.write(File.join(folder, "#{Time.now.to_f}.hostname:2,S"), content)
+    filename = File.join folder, "#{Time.now.to_f}.hostname:2,S"
+    File.write filename, content
+    filename
   end
 
   def start_sup_and_add_source(source)
@@ -74,5 +76,17 @@ def test_can_index_a_maildir_directory_with_special_characters
 
   end
 
+  def test_missing_date_header
+    ## The message is missing a Date header so we should use its modtime
+    ## as a fallback.
+    fallback_date = Time.new 2004, 4, 19, 11, 12, 13
+    maildir = create_a_maildir
+    filename = create_a_maildir_email(File.join(maildir, 'cur'), @test_message_1)
+    File.utime fallback_date, fallback_date, filename
+    start_sup_and_add_source Maildir.new "maildir:#{maildir}"
+
+    messages_in_index = Index.instance.enum_for(:each_message).to_a
+    assert_equal fallback_date, messages_in_index.first.date
+  end
 end
 
diff --git a/test/integration/test_mbox.rb b/test/integration/test_mbox.rb
index 421b86a7b..ebd6076d8 100644
--- a/test/integration/test_mbox.rb
+++ b/test/integration/test_mbox.rb
@@ -69,4 +69,14 @@ def test_can_index_a_mbox_directory_with_special_characters
 
   end
 
+  def test_missing_date_header
+    ## The message is missing a Date header so we should use envelope date
+    ## stored in the From line as a fallback.
+    fallback_date = Time.new 2009, 4, 27, 12, 56, 18
+    mbox = create_a_mbox
+    start_sup_and_add_source MBox.new "mbox:#{mbox}"
+
+    messages_in_index = Index.instance.enum_for(:each_message).to_a
+    assert_equal fallback_date, messages_in_index.first.date
+  end
 end