Use advertized/server/live time for sorting instead of arrival time

fixes #310

7 files changed, 40 insertions, 84 deletions

diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala
index 20aa6c83..dab058af 100644
--- a/libdino/src/plugin/interfaces.vala
+++ b/libdino/src/plugin/interfaces.vala
@@ -93,10 +93,8 @@ public abstract interface NotificationPopulator : Object {
 public abstract class MetaConversationItem : Object {
     public virtual string populator_id { get; set; }
     public virtual Jid? jid { get; set; default=null; }
-    public virtual DateTime sort_time { get; set; default = new DateTime.now_utc(); }
-    public virtual long seccondary_sort_indicator { get; set; }
-    public virtual long tertiary_sort_indicator { get; set; }
-    public virtual DateTime? display_time { get; set; default = null; }
+    public virtual DateTime time { get; set; default = new DateTime.now_utc(); }
+    public virtual int secondary_sort_indicator { get; set; }
     public virtual Encryption encryption { get; set; default = Encryption.NONE; }
     public virtual Entities.Message.Marked mark { get; set; default = Entities.Message.Marked.NONE; }
 
diff --git a/libdino/src/service/chat_interaction.vala b/libdino/src/service/chat_interaction.vala
index d777db90..240d22af 100644
--- a/libdino/src/service/chat_interaction.vala
+++ b/libdino/src/service/chat_interaction.vala
@@ -37,11 +37,10 @@ public class ChatInteraction : StreamInteractionModule, Object {
         if (read_up_to_item == null) return 0;
 
         Database db = Dino.Application.get_default().db;
-        string local_time = read_up_to_item.sort_time.to_unix().to_string();
-        string time = read_up_to_item.display_time.to_unix().to_string();
+        string time = read_up_to_item.time.to_unix().to_string();
         string id = read_up_to_item.id.to_string();
         return (int)db.content_item.select()
-                .where(@"local_time > ? OR (local_time = ? AND time > ?) OR (local_time = ? AND time = ? AND id > ?)", { local_time, local_time, time, local_time, time, id })
+                .where(@"time > ? OR (time = ? AND id > ?)", { time, time, id })
                 .with(db.content_item.conversation_id, "=", conversation.id)
                 .with(db.content_item.hide, "=", false)
                 .count();
diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala
index 357d2300..a6d79267 100644
--- a/libdino/src/service/content_item_store.vala
+++ b/libdino/src/service/content_item_store.vala
@@ -46,14 +46,12 @@ public class ContentItemStore : StreamInteractionModule, Object {
             int provider = row[db.content_item.content_type];
             int foreign_id = row[db.content_item.foreign_id];
             DateTime time = new DateTime.from_unix_utc(row[db.content_item.time]);
-            DateTime local_time = new DateTime.from_unix_utc(row[db.content_item.local_time]);
             switch (provider) {
                 case 1:
                     Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_id(foreign_id, conversation);
                     if (message != null) {
                         var message_item = new MessageItem(message, conversation, row[db.content_item.id]);
-                        message_item.display_time = time;
-                        message_item.sort_time = local_time;
+                        message_item.time = time;
                         items.add(message_item);
                     }
                     break;
@@ -122,36 +120,34 @@ public class ContentItemStore : StreamInteractionModule, Object {
         QueryBuilder select = db.content_item.select()
             .with(db.content_item.conversation_id, "=", conversation.id)
             .with(db.content_item.hide, "=", false)
-            .order_by(db.content_item.local_time, "DESC")
             .order_by(db.content_item.time, "DESC")
+            .order_by(db.content_item.id, "DESC")
             .limit(count);
 
         return get_items_from_query(select, conversation);
     }
 
     public Gee.List<ContentItem> get_before(Conversation conversation, ContentItem item, int count) {
-        long local_time = (long) item.sort_time.to_unix();
-        long time = (long) item.display_time.to_unix();
+        long time = (long) item.time.to_unix();
         QueryBuilder select = db.content_item.select()
-            .where(@"local_time < ? OR (local_time = ? AND time < ?) OR (local_time = ? AND time = ? AND id < ?)", { local_time.to_string(), local_time.to_string(), time.to_string(), local_time.to_string(), time.to_string(), item.id.to_string() })
+            .where(@"time < ? OR (time = ? AND id < ?)", { time.to_string(), time.to_string(), item.id.to_string() })
             .with(db.content_item.conversation_id, "=", conversation.id)
             .with(db.content_item.hide, "=", false)
-            .order_by(db.content_item.local_time, "DESC")
             .order_by(db.content_item.time, "DESC")
+            .order_by(db.content_item.id, "DESC")
             .limit(count);
 
         return get_items_from_query(select, conversation);
     }
 
     public Gee.List<ContentItem> get_after(Conversation conversation, ContentItem item, int count) {
-        long local_time = (long) item.sort_time.to_unix();
-        long time = (long) item.display_time.to_unix();
+        long time = (long) item.time.to_unix();
         QueryBuilder select = db.content_item.select()
-            .where(@"local_time > ? OR (local_time = ? AND time > ?) OR (local_time = ? AND time = ? AND id > ?)", { local_time.to_string(), local_time.to_string(), time.to_string(), local_time.to_string(), time.to_string(), item.id.to_string() })
+            .where(@"time > ? OR (time = ? AND id > ?)", { time.to_string(), time.to_string(), item.id.to_string() })
             .with(db.content_item.conversation_id, "=", conversation.id)
             .with(db.content_item.hide, "=", false)
-            .order_by(db.content_item.local_time, "ASC")
             .order_by(db.content_item.time, "ASC")
+            .order_by(db.content_item.id, "ASC")
             .limit(count);
 
         return get_items_from_query(select, conversation);
@@ -228,17 +224,15 @@ public abstract class ContentItem : Object {
     public int id { get; set; }
     public string type_ { get; set; }
     public Jid jid { get; set; }
-    public DateTime sort_time { get; set; }
-    public DateTime display_time { get; set; }
+    public DateTime time { get; set; }
     public Encryption encryption { get; set; }
     public Entities.Message.Marked mark { get; set; }
 
-    ContentItem(int id, string ty, Jid jid, DateTime sort_time, DateTime display_time, Encryption encryption, Entities.Message.Marked mark) {
+    ContentItem(int id, string ty, Jid jid, DateTime time, Encryption encryption, Entities.Message.Marked mark) {
         this.id = id;
         this.type_ = ty;
         this.jid = jid;
-        this.sort_time = sort_time;
-        this.display_time = display_time;
+        this.time = time;
         this.encryption = encryption;
         this.mark = mark;
     }
@@ -248,10 +242,7 @@ public abstract class ContentItem : Object {
     }
 
     public static int compare_func(ContentItem a, ContentItem b) {
-        int res = a.sort_time.compare(b.sort_time);
-        if (res == 0) {
-            res = a.display_time.compare(b.display_time);
-        }
+        int res = a.time.compare(b.time);
         if (res == 0) {
             res = a.id - b.id > 0 ? 1 : -1;
         }
@@ -266,7 +257,7 @@ public class MessageItem : ContentItem {
     public Conversation conversation;
 
     public MessageItem(Message message, Conversation conversation, int id) {
-        base(id, TYPE, message.from, message.local_time, message.time, message.encryption, message.marked);
+        base(id, TYPE, message.from, message.time, message.encryption, message.marked);
 
         this.message = message;
         this.conversation = conversation;
@@ -287,7 +278,7 @@ public class FileItem : ContentItem {
         } else if (file_transfer.direction == FileTransfer.DIRECTION_SENT) {
             mark = file_to_message_state(file_transfer.state);
         }
-        base(id, TYPE, file_transfer.from, file_transfer.local_time, file_transfer.time, file_transfer.encryption, mark);
+        base(id, TYPE, file_transfer.from, file_transfer.time, file_transfer.encryption, mark);
 
         this.file_transfer = file_transfer;
         this.conversation = conversation;
diff --git a/libdino/src/service/conversation_manager.vala b/libdino/src/service/conversation_manager.vala
index 8b33f52f..99cc9039 100644
--- a/libdino/src/service/conversation_manager.vala
+++ b/libdino/src/service/conversation_manager.vala
@@ -177,7 +177,7 @@ public class ConversationManager : StreamInteractionModule, Object {
 
             if (stanza != null) {
                 bool is_mam_message = Xep.MessageArchiveManagement.MessageFlag.get_flag(stanza) != null;
-                bool is_recent = message.local_time.compare(new DateTime.now_utc().add_days(-3)) > 0;
+                bool is_recent = message.time.compare(new DateTime.now_utc().add_days(-3)) > 0;
                 if (is_mam_message && !is_recent) return false;
             }
             stream_interactor.get_module(ConversationManager.IDENTITY).start_conversation(conversation);
@@ -188,7 +188,7 @@ public class ConversationManager : StreamInteractionModule, Object {
     private void handle_sent_message(Entities.Message message, Conversation conversation) {
         conversation.last_active = message.time;
 
-        bool is_recent = message.local_time.compare(new DateTime.now_utc().add_hours(-24)) > 0;
+        bool is_recent = message.time.compare(new DateTime.now_utc().add_hours(-24)) > 0;
         if (is_recent) {
             start_conversation(conversation);
         }
diff --git a/libdino/src/service/database.vala b/libdino/src/service/database.vala
index 40967025..8b3e101d 100644
--- a/libdino/src/service/database.vala
+++ b/libdino/src/service/database.vala
@@ -7,7 +7,7 @@ using Dino.Entities;
 namespace Dino {
 
 public class Database : Qlite.Database {
-    private const int VERSION = 17;
+    private const int VERSION = 18;
 
     public class AccountTable : Table {
         public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true };
@@ -62,7 +62,7 @@ public class Database : Qlite.Database {
         internal ContentItemTable(Database db) {
             base(db, "content_item");
             init({id, conversation_id, time, local_time, content_type, foreign_id, hide});
-            index("contentitem_conversation_hide_localtime_time_idx", {conversation_id, hide, local_time, time});
+            index("contentitem_conversation_hide_time_idx", {conversation_id, hide, time});
             unique({content_type, foreign_id}, "IGNORE");
         }
     }
@@ -89,7 +89,7 @@ public class Database : Qlite.Database {
                 type_, time, local_time, body, encryption, marked});
 
             // get latest messages
-            index("message_account_counterpart_localtime_idx", {account_id, counterpart_id, local_time});
+            index("message_account_counterpart_time_idx", {account_id, counterpart_id, time});
 
             // deduplication
             index("message_account_counterpart_stanzaid_idx", {account_id, counterpart_id, stanza_id});
@@ -152,7 +152,6 @@ public class Database : Qlite.Database {
             base(db, "file_transfer");
             init({id, account_id, counterpart_id, counterpart_resource, our_resource, direction, time, local_time,
                     encryption, file_name, path, mime_type, size, state, provider, info});
-            index("filetransfer_localtime_counterpart_idx", {local_time, counterpart_id});
         }
     }
 
@@ -398,6 +397,19 @@ public class Database : Qlite.Database {
                 error("Failed to upgrade to database version 17: %s", e.message);
             }
         }
+        if (oldVersion < 18) {
+            try {
+                exec("DROP INDEX contentitem_conversation_hide_localtime_time_idx");
+                exec("CREATE INDEX contentitem_conversation_hide_time_idx ON content_item (conversation_id, hide, time)");
+
+                exec("DROP INDEX message_account_counterpart_localtime_idx");
+                exec("CREATE INDEX message_account_counterpart_time_idx ON message (account_id, counterpart_id, time)");
+
+                exec("DROP INDEX filetransfer_localtime_counterpart_idx");
+            } catch (Error e) {
+                error("Failed to upgrade to database version 18: %s", e.message);
+            }
+        }
     }
 
     public ArrayList<Account> get_accounts() {
@@ -448,22 +460,22 @@ public class Database : Qlite.Database {
 
         if (before != null) {
             if (id > 0) {
-                select.where(@"local_time < ? OR (local_time = ? AND message.id < ?)", { before.to_unix().to_string(), before.to_unix().to_string(), id.to_string() });
+                select.where(@"time < ? OR (time = ? AND message.id < ?)", { before.to_unix().to_string(), before.to_unix().to_string(), id.to_string() });
             } else {
                 select.with(message.id, "<", id);
             }
         }
         if (after != null) {
             if (id > 0) {
-                select.where(@"local_time > ? OR (local_time = ? AND message.id > ?)", { after.to_unix().to_string(), after.to_unix().to_string(), id.to_string() });
+                select.where(@"time > ? OR (time = ? AND message.id > ?)", { after.to_unix().to_string(), after.to_unix().to_string(), id.to_string() });
             } else {
-                select.with(message.local_time, ">", (long) after.to_unix());
+                select.with(message.time, ">", (long) after.to_unix());
             }
             if (id > 0) {
                 select.with(message.id, ">", id);
             }
         } else {
-            select.order_by(message.local_time, "DESC");
+            select.order_by(message.time, "DESC");
         }
 
         select.with(message.counterpart_id, "=", get_jid_id(jid))
diff --git a/libdino/src/service/file_manager.vala b/libdino/src/service/file_manager.vala
index 192dc8a8..f3256601 100644
--- a/libdino/src/service/file_manager.vala
+++ b/libdino/src/service/file_manager.vala
@@ -149,47 +149,6 @@ public class FileManager : StreamInteractionModule, Object {
         return false;
     }
 
-    public Gee.List<FileTransfer> get_latest_transfers(Account account, Jid counterpart, int n) {
-        Qlite.QueryBuilder select = db.file_transfer.select()
-                .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
-                .with(db.file_transfer.account_id, "=", account.id)
-                .order_by(db.file_transfer.local_time, "DESC")
-                .limit(n);
-        return get_transfers_from_qry(select);
-    }
-
-    public Gee.List<FileTransfer> get_transfers_before(Account account, Jid counterpart, DateTime before, int n) {
-        Qlite.QueryBuilder select = db.file_transfer.select()
-                .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
-                .with(db.file_transfer.account_id, "=", account.id)
-                .with(db.file_transfer.local_time, "<", (long)before.to_unix())
-                .order_by(db.file_transfer.local_time, "DESC")
-                .limit(n);
-        return get_transfers_from_qry(select);
-    }
-
-    public Gee.List<FileTransfer> get_transfers_after(Account account, Jid counterpart, DateTime after, int n) {
-        Qlite.QueryBuilder select = db.file_transfer.select()
-                .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
-                .with(db.file_transfer.account_id, "=", account.id)
-                .with(db.file_transfer.local_time, ">", (long)after.to_unix())
-                .limit(n);
-        return get_transfers_from_qry(select);
-    }
-
-    private Gee.List<FileTransfer> get_transfers_from_qry(Qlite.QueryBuilder select) {
-        Gee.List<FileTransfer> ret = new ArrayList<FileTransfer>();
-        foreach (Qlite.Row row in select) {
-            try {
-                FileTransfer file_transfer = new FileTransfer.from_row(db, row, get_storage_dir());
-                ret.insert(0, file_transfer);
-            } catch (InvalidJidError e) {
-                warning("Ignoring file transfer with invalid Jid: %s", e.message);
-            }
-        }
-        return ret;
-    }
-
     public void add_provider(FileProvider file_provider) {
         file_providers.add(file_provider);
         file_provider.file_incoming.connect((info, from, time, local_time, conversation, receive_data, file_meta) => {
diff --git a/libdino/src/service/message_storage.vala b/libdino/src/service/message_storage.vala
index c010a876..a44c0b02 100644
--- a/libdino/src/service/message_storage.vala
+++ b/libdino/src/service/message_storage.vala
@@ -40,7 +40,6 @@ public class MessageStorage : StreamInteractionModule, Object {
                 .with(db.message.account_id, "=", conversation.account.id)
                 .with(db.message.counterpart_id, "=", db.get_jid_id(conversation.counterpart))
                 .with(db.message.type_, "=", (int) Util.get_message_type_for_conversation(conversation))
-                .order_by(db.message.local_time, "DESC")
                 .order_by(db.message.time, "DESC")
                 .outer_join_with(db.message_correction, db.message_correction.message_id, db.message.id)
                 .limit(count);
@@ -111,7 +110,6 @@ public class MessageStorage : StreamInteractionModule, Object {
                 .with(db.message.counterpart_id, "=", db.get_jid_id(conversation.counterpart))
                 .with(db.message.type_, "=", (int) Util.get_message_type_for_conversation(conversation))
                 .with(db.message.stanza_id, "=", stanza_id)
-                .order_by(db.message.local_time, "DESC")
                 .order_by(db.message.time, "DESC")
                 .outer_join_with(db.message_correction, db.message_correction.message_id, db.message.id);
 
@@ -139,7 +137,6 @@ public class MessageStorage : StreamInteractionModule, Object {
                 .with(db.message.counterpart_id, "=", db.get_jid_id(conversation.counterpart))
                 .with(db.message.type_, "=", (int) Util.get_message_type_for_conversation(conversation))
                 .with(db.message.server_id, "=", server_id)
-                .order_by(db.message.local_time, "DESC")
                 .order_by(db.message.time, "DESC")
                 .outer_join_with(db.message_correction, db.message_correction.message_id, db.message.id);