Merge pull request #415 from bobufa/message-search

Message search

12 files changed, 694 insertions, 84 deletions

diff --git a/libdino/CMakeLists.txt b/libdino/CMakeLists.txt
index 9b3e991a..f45d08f5 100644
--- a/libdino/CMakeLists.txt
+++ b/libdino/CMakeLists.txt
@@ -29,6 +29,7 @@ SOURCES
     src/service/blocking_manager.vala
     src/service/chat_interaction.vala
     src/service/connection_manager.vala
+    src/service/content_item_store.vala
     src/service/conversation_manager.vala
     src/service/counterpart_interaction_manager.vala
     src/service/database.vala
@@ -42,6 +43,7 @@ SOURCES
     src/service/presence_manager.vala
     src/service/registration.vala
     src/service/roster_manager.vala
+    src/service/search_processor.vala
     src/service/stream_interactor.vala
     src/service/util.vala
 
diff --git a/libdino/src/application.vala b/libdino/src/application.vala
index 370618b2..7f278fa0 100644
--- a/libdino/src/application.vala
+++ b/libdino/src/application.vala
@@ -38,6 +38,8 @@ public interface Dino.Application : GLib.Application {
         ChatInteraction.start(stream_interactor);
         FileManager.start(stream_interactor, db);
         NotificationEvents.start(stream_interactor);
+        ContentItemStore.start(stream_interactor, db);
+        SearchProcessor.start(stream_interactor, db);
 
         create_actions();
 
diff --git a/libdino/src/entity/file_transfer.vala b/libdino/src/entity/file_transfer.vala
index e2542e74..be472796 100644
--- a/libdino/src/entity/file_transfer.vala
+++ b/libdino/src/entity/file_transfer.vala
@@ -23,7 +23,21 @@ public class FileTransfer : Object {
     public DateTime? local_time { get; set; }
     public Encryption encryption { get; set; }
 
-    public InputStream input_stream { get; set; }
+    private InputStream? input_stream_ = null;
+    public InputStream input_stream {
+        get {
+            if (input_stream_ == null) {
+                File file = File.new_for_path(Path.build_filename(storage_dir, path ?? file_name));
+                try {
+                    input_stream_ = file.read();
+                } catch (Error e) { }
+            }
+            return input_stream_;
+        }
+        set {
+            input_stream_ = value;
+        }
+    }
     public OutputStream output_stream { get; set; }
 
     public string file_name { get; set; }
@@ -41,9 +55,11 @@ public class FileTransfer : Object {
     public string info { get; set; }
 
     private Database? db;
+    private string storage_dir;
 
-    public FileTransfer.from_row(Database db, Qlite.Row row) {
+    public FileTransfer.from_row(Database db, Qlite.Row row, string storage_dir) {
         this.db = db;
+        this.storage_dir = storage_dir;
 
         id = row[db.file_transfer.id];
         account = db.get_account_by_id(row[db.file_transfer.account_id]); // TODO don’t have to generate acc new
@@ -61,7 +77,7 @@ public class FileTransfer : Object {
         }
         direction = row[db.file_transfer.direction];
         time = new DateTime.from_unix_utc(row[db.file_transfer.time]);
-        local_time = new DateTime.from_unix_utc(row[db.file_transfer.time]);
+        local_time = new DateTime.from_unix_utc(row[db.file_transfer.local_time]);
         encryption = (Encryption) row[db.file_transfer.encryption];
         file_name = row[db.file_transfer.file_name];
         path = row[db.file_transfer.path];
diff --git a/libdino/src/entity/message.vala b/libdino/src/entity/message.vala
index 6e34e458..ac54a7c2 100644
--- a/libdino/src/entity/message.vala
+++ b/libdino/src/entity/message.vala
@@ -82,7 +82,7 @@ public class Message : Object {
         }
         direction = row[db.message.direction];
         time = new DateTime.from_unix_utc(row[db.message.time]);
-        local_time = new DateTime.from_unix_utc(row[db.message.time]);
+        local_time = new DateTime.from_unix_utc(row[db.message.local_time]);
         body = row[db.message.body];
         marked = (Message.Marked) row[db.message.marked];
         encryption = (Encryption) row[db.message.encryption];
diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala
index 62260076..2378feb7 100644
--- a/libdino/src/plugin/interfaces.vala
+++ b/libdino/src/plugin/interfaces.vala
@@ -75,15 +75,16 @@ public interface ConversationTitlebarWidget : Object {
 public abstract interface ConversationItemPopulator : Object {
     public abstract string id { get; }
     public abstract void init(Conversation conversation, ConversationItemCollection summary, WidgetType type);
-    public virtual void populate_timespan(Conversation conversation, DateTime from, DateTime to) { }
-    public virtual void populate_between_widgets(Conversation conversation, DateTime from, DateTime to) { }
     public abstract void close(Conversation conversation);
 }
 
+public abstract interface ConversationAdditionPopulator : ConversationItemPopulator {
+    public virtual void populate_timespan(Conversation conversation, DateTime from, DateTime to) { }
+}
+
 public abstract class MetaConversationItem : Object {
+    public virtual string populator_id { get; set; }
     public virtual Jid? jid { get; set; default=null; }
-    public virtual string color { get; set; default=null; }
-    public virtual string display_name { get; set; default=null; }
     public virtual bool dim { get; set; default=false; }
     public virtual DateTime? sort_time { get; set; default=null; }
     public virtual double seccondary_sort_indicator { get; set; }
@@ -103,21 +104,4 @@ public interface ConversationItemCollection : Object {
     public signal void remove_item(MetaConversationItem item);
 }
 
-public interface MessageDisplayProvider : Object {
-    public abstract string id { get; set; }
-    public abstract double priority { get; set; }
-    public abstract bool can_display(Entities.Message? message);
-    public abstract MetaConversationItem? get_item(Entities.Message message, Entities.Conversation conversation);
-}
-
-public interface FileWidget : Object {
-    public abstract Object? get_widget(WidgetType type);
-}
-
-public interface FileDisplayProvider : Object {
-    public abstract double priority { get; }
-    public abstract bool can_display(Entities.Message? message);
-    public abstract FileWidget? get_item(Entities.Message? message);
-}
-
 }
diff --git a/libdino/src/plugin/registry.vala b/libdino/src/plugin/registry.vala
index 7b4410aa..2b496288 100644
--- a/libdino/src/plugin/registry.vala
+++ b/libdino/src/plugin/registry.vala
@@ -7,8 +7,7 @@ public class Registry {
     internal ArrayList<AccountSettingsEntry> account_settings_entries = new ArrayList<AccountSettingsEntry>();
     internal ArrayList<ContactDetailsProvider> contact_details_entries = new ArrayList<ContactDetailsProvider>();
     internal Map<string, TextCommand> text_commands = new HashMap<string, TextCommand>();
-    internal Gee.List<MessageDisplayProvider> message_displays = new ArrayList<MessageDisplayProvider>();
-    internal Gee.List<ConversationItemPopulator> conversation_item_populators = new ArrayList<ConversationItemPopulator>();
+    internal Gee.List<ConversationAdditionPopulator> conversation_addition_populators = new ArrayList<ConversationAdditionPopulator>();
     internal Gee.Collection<ConversationTitlebarEntry> conversation_titlebar_entries = new Gee.TreeSet<ConversationTitlebarEntry>((a, b) => {
         if (a.order < b.order) {
             return -1;
@@ -70,22 +69,12 @@ public class Registry {
         }
     }
 
-    public bool register_message_display(MessageDisplayProvider provider) {
-        lock (message_displays) {
-            foreach(MessageDisplayProvider p in message_displays) {
-                if (p.id == provider.id) return false;
-            }
-            message_displays.add(provider);
-            return true;
-        }
-    }
-
-    public bool register_conversation_item_populator(ConversationItemPopulator populator) {
-        lock (conversation_item_populators) {
-            foreach(ConversationItemPopulator p in conversation_item_populators) {
+    public bool register_conversation_addition_populator(ConversationAdditionPopulator populator) {
+        lock (conversation_addition_populators) {
+            foreach(ConversationItemPopulator p in conversation_addition_populators) {
                 if (p.id == populator.id) return false;
             }
-            conversation_item_populators.add(populator);
+            conversation_addition_populators.add(populator);
             return true;
         }
     }
diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala
new file mode 100644
index 00000000..83ba1da0
--- /dev/null
+++ b/libdino/src/service/content_item_store.vala
@@ -0,0 +1,245 @@
+using Gee;
+
+using Dino.Entities;
+using Qlite;
+using Xmpp;
+
+namespace Dino {
+
+public class ContentItemStore : StreamInteractionModule, Object {
+    public static ModuleIdentity<ContentItemStore> IDENTITY = new ModuleIdentity<ContentItemStore>("content_item_store");
+    public string id { get { return IDENTITY.id; } }
+
+    public signal void new_item(ContentItem item, Conversation conversation);
+
+    private StreamInteractor stream_interactor;
+    private Database db;
+    private Gee.List<ContentFilter> filters = new ArrayList<ContentFilter>();
+    private HashMap<Conversation, ContentItemCollection> collection_conversations = new HashMap<Conversation, ContentItemCollection>(Conversation.hash_func, Conversation.equals_func);
+
+    public static void start(StreamInteractor stream_interactor, Database db) {
+        ContentItemStore m = new ContentItemStore(stream_interactor, db);
+        stream_interactor.add_module(m);
+    }
+
+    public ContentItemStore(StreamInteractor stream_interactor, Database db) {
+        this.stream_interactor = stream_interactor;
+        this.db = db;
+
+        stream_interactor.get_module(MessageProcessor.IDENTITY).message_received.connect(on_new_message);
+        stream_interactor.get_module(MessageProcessor.IDENTITY).message_sent.connect(on_new_message);
+        stream_interactor.get_module(FileManager.IDENTITY).received_file.connect(insert_file_transfer);
+    }
+
+    public void init(Conversation conversation, ContentItemCollection item_collection) {
+        collection_conversations[conversation] = item_collection;
+    }
+
+    public void uninit(Conversation conversation, ContentItemCollection item_collection) {
+        collection_conversations.unset(conversation);
+    }
+
+    public Gee.List<ContentItem> get_items_from_query(QueryBuilder select, Conversation conversation) {
+        Gee.TreeSet<ContentItem> items = new Gee.TreeSet<ContentItem>(ContentItem.compare);
+
+        foreach (var row in select) {
+            int provider = row[db.content.content_type];
+            int foreign_id = row[db.content.foreign_id];
+            switch (provider) {
+                case 1:
+                    RowOption row_option = db.message.select().with(db.message.id, "=", foreign_id).row();
+                    if (row_option.is_present()) {
+                        Message message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_id(foreign_id, conversation);
+                        if (message == null) {
+                            message = new Message.from_row(db, row_option.inner);
+                        }
+                        items.add(new MessageItem(message, conversation, row[db.content.id]));
+                    }
+                    break;
+                case 2:
+                    RowOption row_option = db.file_transfer.select().with(db.file_transfer.id, "=", foreign_id).row();
+                    if (row_option.is_present()) {
+                        string storage_dir = stream_interactor.get_module(FileManager.IDENTITY).get_storage_dir();
+                        FileTransfer file_transfer = new FileTransfer.from_row(db, row_option.inner, storage_dir);
+                        items.add(new FileItem(file_transfer, row[db.content.id]));
+                    }
+                    break;
+            }
+        }
+
+        Gee.List<ContentItem> ret = new ArrayList<ContentItem>();
+        foreach (ContentItem item in items) {
+            ret.add(item);
+        }
+        return ret;
+    }
+
+    public Gee.List<ContentItem> get_latest(Conversation conversation, int count) {
+        QueryBuilder select = db.content.select()
+            .with(db.content.conversation_id, "=", conversation.id)
+            .order_by(db.content.local_time, "DESC")
+            .order_by(db.content.time, "DESC")
+            .limit(count);
+
+        return get_items_from_query(select, conversation);
+    }
+
+    public Gee.List<ContentItem> get_before(Conversation conversation, ContentItem item, int count) {
+        long local_time = (long) item.sort_time.to_unix();
+        long time = (long) item.display_time.to_unix();
+        QueryBuilder select = db.content.select()
+            .where(@"local_time < ? OR (local_time = ? AND time < ?) OR (local_time = ? AND time = ? AND id < ?)", { local_time.to_string(), local_time.to_string(), time.to_string(), local_time.to_string(), time.to_string(), item.id.to_string() })
+            .with(db.content.conversation_id, "=", conversation.id)
+            .order_by(db.content.local_time, "DESC")
+            .order_by(db.content.time, "DESC")
+            .limit(count);
+
+        return get_items_from_query(select, conversation);
+    }
+
+    public Gee.List<ContentItem> get_after(Conversation conversation, ContentItem item, int count) {
+        long local_time = (long) item.sort_time.to_unix();
+        long time = (long) item.display_time.to_unix();
+        QueryBuilder select = db.content.select()
+            .where(@"local_time > ? OR (local_time = ? AND time > ?) OR (local_time = ? AND time = ? AND id > ?)", { local_time.to_string(), local_time.to_string(), time.to_string(), local_time.to_string(), time.to_string(), item.id.to_string() })
+            .with(db.content.conversation_id, "=", conversation.id)
+            .order_by(db.content.local_time, "ASC")
+            .order_by(db.content.time, "ASC")
+            .limit(count);
+
+        return get_items_from_query(select, conversation);
+    }
+
+    public void add_filter(ContentFilter content_filter) {
+        filters.add(content_filter);
+    }
+
+    private void on_new_message(Message message, Conversation conversation) {
+        MessageItem item = new MessageItem(message, conversation, -1);
+        if (!discard(item)) {
+            item.id = db.add_content_item(conversation, message.time, message.local_time, 1, message.id);
+
+            if (collection_conversations.has_key(conversation)) {
+                collection_conversations.get(conversation).insert_item(item);
+            }
+            new_item(item, conversation);
+        }
+    }
+
+    private void insert_file_transfer(FileTransfer file_transfer, Conversation conversation) {
+        FileItem item = new FileItem(file_transfer, -1);
+        if (!discard(item)) {
+            item.id = db.add_content_item(conversation, file_transfer.time, file_transfer.local_time, 2, file_transfer.id);
+
+            if (collection_conversations.has_key(conversation)) {
+                collection_conversations.get(conversation).insert_item(item);
+            }
+            new_item(item, conversation);
+        }
+    }
+
+    private bool discard(ContentItem content_item) {
+        foreach (ContentFilter filter in filters) {
+            if (filter.discard(content_item)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+
+public interface ContentItemCollection : Object {
+    public abstract void insert_item(ContentItem item);
+    public abstract void remove_item(ContentItem item);
+}
+
+public interface ContentFilter : Object {
+    public abstract bool discard(ContentItem content_item);
+}
+
+public abstract class ContentItem : Object {
+    public int id { get; set; }
+    public string type_ { get; set; }
+    public Jid? jid { get; set; default=null; }
+    public DateTime? sort_time { get; set; default=null; }
+    public double seccondary_sort_indicator { get; set; }
+    public DateTime? display_time { get; set; default=null; }
+    public Encryption? encryption { get; set; default=null; }
+    public Entities.Message.Marked? mark { get; set; default=null; }
+
+    public ContentItem(int id, string ty, Jid jid, DateTime sort_time, double seccondary_sort_indicator, DateTime display_time, Encryption encryption, Entities.Message.Marked mark) {
+        this.id = id;
+        this.type_ = ty;
+        this.jid = jid;
+        this.sort_time = sort_time;
+        this.seccondary_sort_indicator = seccondary_sort_indicator;
+        this.display_time = display_time;
+        this.encryption = encryption;
+        this.mark = mark;
+    }
+
+    public static int compare(ContentItem a, ContentItem b) {
+        int res = a.sort_time.compare(b.sort_time);
+        if (res == 0) {
+            res = a.display_time.compare(b.display_time);
+        }
+        if (res == 0) {
+            res = a.seccondary_sort_indicator - b.seccondary_sort_indicator > 0 ? 1 : -1;
+        }
+        return res;
+    }
+}
+
+public class MessageItem : ContentItem {
+    public const string TYPE = "message";
+
+    public Message message;
+    public Conversation conversation;
+
+    public MessageItem(Message message, Conversation conversation, int id) {
+        base(id, TYPE, message.from, message.local_time, message.id + 0.0845, message.time, message.encryption, message.marked);
+        this.message = message;
+        this.conversation = conversation;
+
+        WeakRef weak_message = WeakRef(message);
+        message.notify["marked"].connect(() => {
+            Message? m = weak_message.get() as Message;
+            if (m == null) return;
+            mark = m.marked;
+        });
+    }
+}
+
+public class FileItem : ContentItem {
+    public const string TYPE = "file";
+
+    public FileTransfer file_transfer;
+    public Conversation conversation;
+
+    public FileItem(FileTransfer file_transfer, int id) {
+        Jid jid = file_transfer.direction == FileTransfer.DIRECTION_SENT ? file_transfer.account.bare_jid.with_resource(file_transfer.account.resourcepart) : file_transfer.counterpart;
+        base(id, TYPE, jid, file_transfer.local_time, file_transfer.id + 0.0845, file_transfer.time, file_transfer.encryption, file_to_message_state(file_transfer.state));
+
+        this.file_transfer = file_transfer;
+
+        file_transfer.notify["state"].connect_after(() => {
+            this.mark = file_to_message_state(file_transfer.state);
+        });
+    }
+
+    private static Entities.Message.Marked file_to_message_state(FileTransfer.State state) {
+        switch (state) {
+            case FileTransfer.State.IN_PROCESS:
+                return Entities.Message.Marked.UNSENT;
+            case FileTransfer.State.COMPLETE:
+                return Entities.Message.Marked.NONE;
+            case FileTransfer.State.NOT_STARTED:
+                return Entities.Message.Marked.UNSENT;
+            case FileTransfer.State.FAILED:
+                return Entities.Message.Marked.WONTSEND;
+        }
+        assert_not_reached();
+    }
+}
+
+}
diff --git a/libdino/src/service/counterpart_interaction_manager.vala b/libdino/src/service/counterpart_interaction_manager.vala
index fb10d20c..b4df9b8d 100644
--- a/libdino/src/service/counterpart_interaction_manager.vala
+++ b/libdino/src/service/counterpart_interaction_manager.vala
@@ -9,7 +9,7 @@ public class CounterpartInteractionManager : StreamInteractionModule, Object {
     public string id { get { return IDENTITY.id; } }
 
     public signal void received_state(Account account, Jid jid, string state);
-    public signal void received_marker(Account account, Jid jid, Entities.Message message, string marker);
+    public signal void received_marker(Account account, Jid jid, Entities.Message message, Entities.Message.Marked marker);
     public signal void received_message_received(Account account, Jid jid, Entities.Message message);
     public signal void received_message_displayed(Account account, Jid jid, Entities.Message message);
 
@@ -69,12 +69,12 @@ public class CounterpartInteractionManager : StreamInteractionModule, Object {
             if (marker != Xep.ChatMarkers.MARKER_DISPLAYED && marker != Xep.ChatMarkers.MARKER_ACKNOWLEDGED) return;
             Conversation? conversation = stream_interactor.get_module(MessageStorage.IDENTITY).get_conversation_for_stanza_id(account, stanza_id);
             if (conversation == null) return;
-            Entities.Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_id(stanza_id, conversation);
+            Entities.Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_stanza_id(stanza_id, conversation);
             if (message == null) return;
             conversation.read_up_to = message;
         } else {
             foreach (Conversation conversation in stream_interactor.get_module(ConversationManager.IDENTITY).get_conversations(jid, account)) {
-                Entities.Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_id(stanza_id, conversation);
+                Entities.Message? message = stream_interactor.get_module(MessageStorage.IDENTITY).get_message_by_stanza_id(stanza_id, conversation);
                 if (message != null) {
                     switch (marker) {
                         case Xep.ChatMarkers.MARKER_RECEIVED:
diff --git a/libdino/src/service/database.vala b/libdino/src/service/database.vala
index 25db82f8..8a470d12 100644
--- a/libdino/src/service/database.vala
+++ b/libdino/src/service/database.vala
@@ -6,7 +6,7 @@ using Dino.Entities;
 namespace Dino {
 
 public class Database : Qlite.Database {
-    private const int VERSION = 6;
+    private const int VERSION = 8;
 
     public class AccountTable : Table {
         public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true };
@@ -34,6 +34,21 @@ public class Database : Qlite.Database {
         }
     }
 
+    public class ContentTable : Table {
+        public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true };
+        public Column<int> conversation_id = new Column.Integer("conversation_id") { not_null = true };
+        public Column<long> time = new Column.Long("time") { not_null = true };
+        public Column<long> local_time = new Column.Long("local_time") { not_null = true };
+        public Column<int> content_type = new Column.Integer("content_type") { not_null = true };
+        public Column<int> foreign_id = new Column.Integer("foreign_id") { not_null = true };
+
+        internal ContentTable(Database db) {
+            base(db, "content_item");
+            init({id, conversation_id, time, local_time, content_type, foreign_id});
+            unique({content_type, foreign_id}, "IGNORE");
+        }
+    }
+
     public class MessageTable : Table {
         public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true };
         public Column<string> stanza_id = new Column.Text("stanza_id");
@@ -54,6 +69,7 @@ public class Database : Qlite.Database {
             init({id, stanza_id, account_id, counterpart_id, our_resource, counterpart_resource, direction,
                 type_, time, local_time, body, encryption, marked});
             index("message_localtime_counterpart_idx", {local_time, counterpart_id});
+            fts({body});
         }
     }
 
@@ -173,6 +189,7 @@ public class Database : Qlite.Database {
 
     public AccountTable account { get; private set; }
     public JidTable jid { get; private set; }
+    public ContentTable content { get; private set; }
     public MessageTable message { get; private set; }
     public RealJidTable real_jid { get; private set; }
     public FileTransferTable file_transfer { get; private set; }
@@ -190,6 +207,7 @@ public class Database : Qlite.Database {
         base(fileName, VERSION);
         account = new AccountTable(this);
         jid = new JidTable(this);
+        content = new ContentTable(this);
         message = new MessageTable(this);
         real_jid = new RealJidTable(this);
         file_transfer = new FileTransferTable(this);
@@ -198,7 +216,7 @@ public class Database : Qlite.Database {
         entity_feature = new EntityFeatureTable(this);
         roster = new RosterTable(this);
         settings = new SettingsTable(this);
-        init({ account, jid, message, real_jid, file_transfer, conversation, avatar, entity_feature, roster, settings });
+        init({ account, jid, content, message, real_jid, file_transfer, conversation, avatar, entity_feature, roster, settings });
         try {
             exec("PRAGMA synchronous=0");
         } catch (Error e) { }
@@ -206,6 +224,31 @@ public class Database : Qlite.Database {
 
     public override void migrate(long oldVersion) {
         // new table columns are added, outdated columns are still present
+        if (oldVersion < 7) {
+            message.fts_rebuild();
+        } else if (oldVersion < 8) {
+            exec("""
+            insert into content_item (conversation_id, time, local_time, content_type, foreign_id)
+            select conversation.id, message.time, message.local_time, 1, message.id
+            from message join conversation on
+                message.account_id=conversation.account_id and
+                message.counterpart_id=conversation.jid_id and
+                message.type=conversation.type+1 and
+                (message.counterpart_resource=conversation.resource or message.type != 3)
+            where
+                message.body not in (select info from file_transfer where info not null) and
+                message.id not in (select info from file_transfer where info not null)
+            union
+            select conversation.id, message.time, message.local_time, 2, file_transfer.id
+            from file_transfer
+            join message on
+                file_transfer.info=message.id
+            join conversation on
+                file_transfer.account_id=conversation.account_id and
+                file_transfer.counterpart_id=conversation.jid_id and
+                message.type=conversation.type+1 and
+                (message.counterpart_resource=conversation.resource or message.type != 3)""");
+        }
     }
 
     public ArrayList<Account> get_accounts() {
@@ -232,11 +275,41 @@ public class Database : Qlite.Database {
         }
     }
 
-    public Gee.List<Message> get_messages(Xmpp.Jid jid, Account account, Message.Type? type, int count, DateTime? before) {
-        QueryBuilder select = message.select()
-                .with(message.counterpart_id, "=", get_jid_id(jid))
+    public int add_content_item(Conversation conversation, DateTime time, DateTime local_time, int content_type, int foreign_id) {
+        return (int) content.insert()
+            .value(content.conversation_id, conversation.id)
+            .value(content.local_time, (long) local_time.to_unix())
+            .value(content.time, (long) time.to_unix())
+            .value(content.content_type, content_type)
+            .value(content.foreign_id, foreign_id)
+            .perform();
+    }
+
+    public Gee.List<Message> get_messages(Xmpp.Jid jid, Account account, Message.Type? type, int count, DateTime? before, DateTime? after, int id) {
+        QueryBuilder select = message.select();
+
+        if (before != null) {
+            if (id > 0) {
+                select.where(@"local_time < ? OR (local_time = ? AND id < ?)", { before.to_unix().to_string(), before.to_unix().to_string(), id.to_string() });
+            } else {
+                select.with(message.id, "<", id);
+            }
+        }
+        if (after != null) {
+            if (id > 0) {
+                select.where(@"local_time > ? OR (local_time = ? AND id > ?)", { after.to_unix().to_string(), after.to_unix().to_string(), id.to_string() });
+            } else {
+                select.with(message.local_time, ">", (long) after.to_unix());
+            }
+            if (id > 0) {
+                select.with(message.id, ">", id);
+            }
+        } else {
+            select.order_by(message.id, "DESC");
+        }
+
+        select.with(message.counterpart_id, "=", get_jid_id(jid))
                 .with(message.account_id, "=", account.id)
-                .order_by(message.id, "DESC")
                 .limit(count);
         if (jid.resourcepart != null) {
             select.with(message.counterpart_resource, "=", jid.resourcepart);
@@ -244,9 +317,6 @@ public class Database : Qlite.Database {
         if (type != null) {
             select.with(message.type_, "=", (int) type);
         }
-        if (before != null) {
-            select.with(message.local_time, "<", (long) before.to_unix());
-        }
 
         LinkedList<Message> ret = new LinkedList<Message>();
         foreach (Row row in select) {
diff --git a/libdino/src/service/file_manager.vala b/libdino/src/service/file_manager.vala
index 3def24af..340205af 100644
--- a/libdino/src/service/file_manager.vala
+++ b/libdino/src/service/file_manager.vala
@@ -11,7 +11,7 @@ public class FileManager : StreamInteractionModule, Object {
     public string id { get { return IDENTITY.id; } }
 
     public signal void upload_available(Account account);
-    public signal void received_file(FileTransfer file_transfer);
+    public signal void received_file(FileTransfer file_transfer, Conversation conversation);
 
     private StreamInteractor stream_interactor;
     private Database db;
@@ -68,7 +68,7 @@ public class FileManager : StreamInteractionModule, Object {
                 file_sender.send_file(conversation, file_transfer);
             }
         }
-        received_file(file_transfer);
+        received_file(file_transfer, conversation);
     }
 
     public bool is_upload_available(Conversation conversation) {
@@ -78,21 +78,38 @@ public class FileManager : StreamInteractionModule, Object {
         return false;
     }
 
-    public Gee.List<FileTransfer> get_file_transfers(Account account, Jid counterpart, DateTime after, DateTime before) {
+    public Gee.List<FileTransfer> get_latest_transfers(Account account, Jid counterpart, int n) {
+        Qlite.QueryBuilder select = db.file_transfer.select()
+                .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
+                .with(db.file_transfer.account_id, "=", account.id)
+                .order_by(db.file_transfer.local_time, "DESC")
+                .limit(n);
+        return get_transfers_from_qry(select);
+    }
+
+    public Gee.List<FileTransfer> get_transfers_before(Account account, Jid counterpart, DateTime before, int n) {
         Qlite.QueryBuilder select = db.file_transfer.select()
                 .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
                 .with(db.file_transfer.account_id, "=", account.id)
-                .with(db.file_transfer.local_time, ">", (long)after.to_unix())
                 .with(db.file_transfer.local_time, "<", (long)before.to_unix())
-                .order_by(db.file_transfer.id, "DESC");
+                .order_by(db.file_transfer.local_time, "DESC")
+                .limit(n);
+        return get_transfers_from_qry(select);
+    }
+
+    public Gee.List<FileTransfer> get_transfers_after(Account account, Jid counterpart, DateTime after, int n) {
+        Qlite.QueryBuilder select = db.file_transfer.select()
+                .with(db.file_transfer.counterpart_id, "=", db.get_jid_id(counterpart))
+                .with(db.file_transfer.account_id, "=", account.id)
+                .with(db.file_transfer.local_time, ">", (long)after.to_unix())
+                .limit(n);
+        return get_transfers_from_qry(select);
+    }
 
+    private Gee.List<FileTransfer> get_transfers_from_qry(Qlite.QueryBuilder select) {
         Gee.List<FileTransfer> ret = new ArrayList<FileTransfer>();
         foreach (Qlite.Row row in select) {
-            FileTransfer file_transfer = new FileTransfer.from_row(db, row);
-            File file = File.new_for_path(Path.build_filename(get_storage_dir(), file_transfer.path ?? file_transfer.file_name));
-            try {
-                file_transfer.input_stream = file.read();
-            } catch (Error e) { }
+            FileTransfer file_transfer = new FileTransfer.from_row(db, row, get_storage_dir());
             ret.insert(0, file_transfer);
         }
         return ret;
@@ -117,7 +134,7 @@ public class FileManager : StreamInteractionModule, Object {
         outgoing_processors.add(processor);
     }
 
-    private void handle_incomming_file(FileTransfer file_transfer) {
+    private void handle_incomming_file(FileTransfer file_transfer, Conversation conversation) {
         foreach (IncommingFileProcessor processor in incomming_processors) {
             if (processor.can_process(file_transfer)) {
                 processor.process(file_transfer);
@@ -131,7 +148,7 @@ public class FileManager : StreamInteractionModule, Object {
         } catch (Error e) { }
 
         file_transfer.persist(db);
-        received_file(file_transfer);
+        received_file(file_transfer, conversation);
     }
 
     private void save_file(FileTransfer file_transfer) {
@@ -152,7 +169,7 @@ public class FileManager : StreamInteractionModule, Object {
 }
 
 public interface FileProvider : Object {
-    public signal void file_incoming(FileTransfer file_transfer);
+    public signal void file_incoming(FileTransfer file_transfer, Conversation conversation);
 }
 
 public interface FileSender : Object {
diff --git a/libdino/src/service/message_storage.vala b/libdino/src/service/message_storage.vala
index 35e05074..50fc94b3 100644
--- a/libdino/src/service/message_storage.vala
+++ b/libdino/src/service/message_storage.vala
@@ -1,4 +1,5 @@
 using Gee;
+using Qlite;
 
 using Dino.Entities;
 
@@ -51,26 +52,47 @@ public class MessageStorage : StreamInteractionModule, Object {
         return null;
     }
 
-    public Gee.List<Message>? get_messages_before_message(Conversation? conversation, Message message, int count = 20) {
-        SortedSet<Message>? before = messages[conversation].head_set(message);
-        if (before != null && before.size >= count) {
-            Gee.List<Message> ret = new ArrayList<Message>(Message.equals_func);
-            Iterator<Message> iter = before.iterator();
-            iter.next();
-            for (int from_index = before.size - count; iter.has_next() && from_index > 0; from_index--) iter.next();
-            while(iter.has_next()) {
-                Message m = iter.get();
-                ret.add(m);
-                iter.next();
-            }
-            return ret;
-        } else {
-            Gee.List<Message> db_messages = db.get_messages(conversation.counterpart, conversation.account, Util.get_message_type_for_conversation(conversation), count, message.local_time);
-            return db_messages;
+    public Gee.List<MessageItem> get_messages_before_message(Conversation? conversation, DateTime before, int id, int count = 20) {
+//        SortedSet<Message>? before = messages[conversation].head_set(message);
+//        if (before != null && before.size >= count) {
+//            Gee.List<Message> ret = new ArrayList<Message>(Message.equals_func);
+//            Iterator<Message> iter = before.iterator();
+//            iter.next();
+//            for (int from_index = before.size - count; iter.has_next() && from_index > 0; from_index--) iter.next();
+//            while(iter.has_next()) {
+//                Message m = iter.get();
+//                ret.add(m);
+//                iter.next();
+//            }
+//            return ret;
+//        } else {
+        Gee.List<Message> db_messages = db.get_messages(conversation.counterpart, conversation.account, Util.get_message_type_for_conversation(conversation), count, before, null, id);
+        Gee.List<MessageItem> ret = new ArrayList<MessageItem>();
+        foreach (Message message in db_messages) {
+            ret.add(new MessageItem(message, conversation, -1));
+        }
+        return ret;
+//        }
+    }
+
+    public Gee.List<MessageItem> get_messages_after_message(Conversation? conversation, DateTime after, int id, int count = 20) {
+        Gee.List<Message> db_messages = db.get_messages(conversation.counterpart, conversation.account, Util.get_message_type_for_conversation(conversation), count, null, after, id);
+        Gee.List<MessageItem> ret = new ArrayList<MessageItem>();
+        foreach (Message message in db_messages) {
+            ret.add(new MessageItem(message, conversation, -1));
+        }
+        return ret;
+    }
+
+    public Message? get_message_by_id(int id, Conversation conversation) {
+        init_conversation(conversation);
+        foreach (Message message in messages[conversation]) {
+            if (message.id == id) return message;
         }
+        return null;
     }
 
-    public Message? get_message_by_id(string stanza_id, Conversation conversation) {
+    public Message? get_message_by_stanza_id(string stanza_id, Conversation conversation) {
         init_conversation(conversation);
         foreach (Message message in messages[conversation]) {
             if (message.stanza_id == stanza_id) return message;
@@ -100,7 +122,7 @@ public class MessageStorage : StreamInteractionModule, Object {
                 }
                 return res;
             });
-            Gee.List<Message> db_messages = db.get_messages(conversation.counterpart, conversation.account, Util.get_message_type_for_conversation(conversation), 50, null);
+            Gee.List<Message> db_messages = db.get_messages(conversation.counterpart, conversation.account, Util.get_message_type_for_conversation(conversation), 50, null, null, -1);
             messages[conversation].add_all(db_messages);
         }
     }
diff --git a/libdino/src/service/search_processor.vala b/libdino/src/service/search_processor.vala
new file mode 100644
index 00000000..5a14dfe7
--- /dev/null
+++ b/libdino/src/service/search_processor.vala
@@ -0,0 +1,263 @@
+using Gee;
+
+using Xmpp;
+using Qlite;
+using Dino.Entities;
+
+namespace Dino {
+
+public class SearchProcessor : StreamInteractionModule, Object {
+    public static ModuleIdentity<SearchProcessor> IDENTITY = new ModuleIdentity<SearchProcessor>("search_processor");
+    public string id { get { return IDENTITY.id; } }
+
+    private StreamInteractor stream_interactor;
+    private Database db;
+
+    public static void start(StreamInteractor stream_interactor, Database db) {
+        SearchProcessor m = new SearchProcessor(stream_interactor, db);
+        stream_interactor.add_module(m);
+    }
+
+    public SearchProcessor(StreamInteractor stream_interactor, Database db) {
+        this.stream_interactor = stream_interactor;
+        this.db = db;
+    }
+
+    private QueryBuilder prepare_search(string query, bool join_content) {
+        string words = "";
+        string? with = null;
+        string? in_ = null;
+        string? from = null;
+        foreach(string word in query.split(" ")) {
+            if (word.has_prefix("with:")) {
+                if (with == null) {
+                    with = word.substring(5);
+                } else {
+                    return db.message.select().where("0");
+                }
+            } else if (word.has_prefix("in:")) {
+                if (in_ == null) {
+                    in_ = word.substring(3);
+                } else {
+                    return db.message.select().where("0");
+                }
+            } else if (word.has_prefix("from:")) {
+                if (from == null) {
+                    from = word.substring(5);
+                } else {
+                    return db.message.select().where("0");
+                }
+            } else {
+                words += word + "* ";
+            }
+        }
+        if (in_ != null && with != null) {
+            return db.message.select().where("0");
+        }
+
+        QueryBuilder rows = db.message
+            .match(db.message.body, words)
+            .order_by(db.message.id, "DESC")
+            .join_with(db.jid, db.jid.id, db.message.counterpart_id)
+            .join_with(db.account, db.account.id, db.message.account_id)
+            .outer_join_with(db.real_jid, db.real_jid.message_id, db.message.id)
+            .with(db.account.enabled, "=", true);
+        if (join_content) {
+            rows.join_on(db.content, "message.id=content_item.foreign_id AND content_item.content_type=1")
+                .with(db.content.content_type, "=", 1);
+        }
+        if (with != null) {
+            if (with.index_of("/") > 0) {
+                rows.with(db.message.type_, "=", Message.Type.GROUPCHAT_PM)
+                    .with(db.jid.bare_jid, "LIKE", with.substring(0, with.index_of("/")))
+                    .with(db.message.counterpart_resource, "LIKE", with.substring(with.index_of("/") + 1));
+            } else {
+                rows.where(@"($(db.message.type_) = $((int)Message.Type.CHAT) AND $(db.jid.bare_jid) LIKE ?)"
+                    + @" OR ($(db.message.type_) = $((int)Message.Type.GROUPCHAT_PM) AND $(db.real_jid.real_jid) LIKE ?)"
+                    + @" OR ($(db.message.type_) = $((int)Message.Type.GROUPCHAT_PM) AND $(db.message.counterpart_resource) LIKE ?)", {with, with, with});
+            }
+        } else if (in_ != null) {
+            rows.with(db.jid.bare_jid, "LIKE", in_)
+                .with(db.message.type_, "=", Message.Type.GROUPCHAT);
+        }
+        if (from != null) {
+            rows.where(@"($(db.message.direction) = 1 AND $(db.account.bare_jid) LIKE ?)"
+                + @" OR ($(db.message.direction) = 1 AND $(db.message.type_) IN ($((int)Message.Type.GROUPCHAT), $((int)Message.Type.GROUPCHAT_PM)) AND $(db.message.our_resource) LIKE ?)"
+                + @" OR ($(db.message.direction) = 0 AND $(db.message.type_) = $((int)Message.Type.CHAT) AND $(db.jid.bare_jid) LIKE ?)"
+                + @" OR ($(db.message.direction) = 0 AND $(db.message.type_) IN ($((int)Message.Type.GROUPCHAT), $((int)Message.Type.GROUPCHAT_PM)) AND $(db.real_jid.real_jid) LIKE ?)"
+                + @" OR ($(db.message.direction) = 0 AND $(db.message.type_) IN ($((int)Message.Type.GROUPCHAT), $((int)Message.Type.GROUPCHAT_PM)) AND $(db.message.counterpart_resource) LIKE ?)", {from, from, from, from, from});
+        }
+        return rows;
+    }
+
+    public Gee.List<SearchSuggestion> suggest_auto_complete(string query, int cursor_position, int limit = 5) {
+        int after_prev_space = query.substring(0, cursor_position).last_index_of(" ") + 1;
+        int next_space = query.index_of(" ", after_prev_space);
+        if (next_space < 0) next_space = query.length;
+        string current_query = query.substring(after_prev_space, next_space - after_prev_space);
+        Gee.List<SearchSuggestion> suggestions = new ArrayList<SearchSuggestion>();
+
+        if (current_query.has_prefix("from:")) {
+            if (cursor_position < after_prev_space + 5) return suggestions;
+            string current_from = current_query.substring(5);
+            string[] splitted = query.split(" ");
+            foreach(string s in splitted) {
+                if (s.has_prefix("from:") && s != "from:" + current_from) {
+                    // Already have an from: filter -> no useful autocompletion possible
+                    return suggestions;
+                }
+            }
+            string? current_in = null;
+            string? current_with = null;
+            foreach(string s in splitted) {
+                if (s.has_prefix("in:")) {
+                    current_in = s.substring(3);
+                } else if (s.has_prefix("with:")) {
+                    current_with = s.substring(5);
+                }
+            }
+            if (current_in != null && current_with != null) {
+                // in: and with: -> no useful autocompletion possible
+                return suggestions;
+            }
+            if (current_with != null) {
+                // Can only be the other one or us
+
+                // Normal chat
+                QueryBuilder chats = db.conversation.select()
+                    .join_with(db.jid, db.jid.id, db.conversation.jid_id)
+                    .join_with(db.account, db.account.id, db.conversation.account_id)
+                    .with(db.jid.bare_jid, "=", current_with)
+                    .with(db.account.enabled, "=", true)
+                    .with(db.conversation.type_, "=", Conversation.Type.CHAT)
+                    .order_by(db.conversation.last_active, "DESC");
+                foreach(Row chat in chats) {
+                    if (suggestions.size == 0) {
+                        suggestions.add(new SearchSuggestion(new Account.from_row(db, chat), new Jid(chat[db.jid.bare_jid]), "from:"+chat[db.jid.bare_jid], after_prev_space, next_space));
+                    }
+                    suggestions.add(new SearchSuggestion(new Account.from_row(db, chat), new Jid(chat[db.account.bare_jid]), "from:"+chat[db.account.bare_jid], after_prev_space, next_space));
+                }
+                return suggestions;
+            }
+            if (current_in != null) {
+                // All members of the MUC with history
+                QueryBuilder msgs = db.message.select()
+                    .select_string(@"account.*, $(db.message.counterpart_resource)")
+                    .join_with(db.jid, db.jid.id, db.message.counterpart_id)
+                    .join_with(db.account, db.account.id, db.message.account_id)
+                    .with(db.jid.bare_jid, "=", current_in)
+                    .with(db.account.enabled, "=", true)
+                    .with(db.message.type_, "=", Message.Type.GROUPCHAT)
+                    .with(db.message.counterpart_resource, "LIKE", @"%$current_from%")
+                    .group_by({db.message.counterpart_resource})
+                    .order_by_name(@"MAX($(db.message.time))", "DESC")
+                    .limit(5);
+                foreach(Row msg in msgs) {
+                    suggestions.add(new SearchSuggestion(new Account.from_row(db, msg), new Jid(current_in).with_resource(msg[db.message.counterpart_resource]), "from:"+msg[db.message.counterpart_resource], after_prev_space, next_space));
+                }
+            }
+            // TODO: auto complete from
+        } else if (current_query.has_prefix("with:")) {
+            if (cursor_position < after_prev_space + 5) return suggestions;
+            string current_with = current_query.substring(5);
+            string[] splitted = query.split(" ");
+            foreach(string s in splitted) {
+                if ((s.has_prefix("with:") && s != "with:" + current_with) || s.has_prefix("in:")) {
+                    // Already have an in: or with: filter -> no useful autocompletion possible
+                    return suggestions;
+                }
+            }
+
+            // Normal chat
+            QueryBuilder chats = db.conversation.select()
+                .join_with(db.jid, db.jid.id, db.conversation.jid_id)
+                .join_with(db.account, db.account.id, db.conversation.account_id)
+                .outer_join_on(db.roster, @"$(db.jid.bare_jid) = $(db.roster.jid) AND $(db.account.id) = $(db.roster.account_id)")
+                .where(@"$(db.jid.bare_jid) LIKE ? OR $(db.roster.handle) LIKE ?", {@"%$current_with%", @"%$current_with%"})
+                .with(db.account.enabled, "=", true)
+                .with(db.conversation.type_, "=", Conversation.Type.CHAT)
+                .order_by(db.conversation.last_active, "DESC")
+                .limit(limit);
+            foreach(Row chat in chats) {
+                suggestions.add(new SearchSuggestion(new Account.from_row(db, chat), new Jid(chat[db.jid.bare_jid]), "with:"+chat[db.jid.bare_jid], after_prev_space, next_space) { order = chat[db.conversation.last_active]});
+            }
+
+            // Groupchat PM
+            if (suggestions.size < 5) {
+                chats = db.conversation.select()
+                    .join_with(db.jid, db.jid.id, db.conversation.jid_id)
+                    .join_with(db.account, db.account.id, db.conversation.account_id)
+                    .where(@"$(db.jid.bare_jid) LIKE ? OR $(db.conversation.resource) LIKE ?", {@"%$current_with%", @"%$current_with%"})
+                    .with(db.account.enabled, "=", true)
+                    .with(db.conversation.type_, "=", Conversation.Type.GROUPCHAT_PM)
+                    .order_by(db.conversation.last_active, "DESC")
+                    .limit(limit - suggestions.size);
+                foreach(Row chat in chats) {
+                    suggestions.add(new SearchSuggestion(new Account.from_row(db, chat), new Jid(chat[db.jid.bare_jid]).with_resource(chat[db.conversation.resource]), "with:"+chat[db.jid.bare_jid]+"/"+chat[db.conversation.resource], after_prev_space, next_space) { order = chat[db.conversation.last_active]});
+                }
+                suggestions.sort((a, b) => (int)(b.order - a.order));
+            }
+        } else if (current_query.has_prefix("in:")) {
+            if (cursor_position < after_prev_space + 3) return suggestions;
+            string current_in = current_query.substring(3);
+            string[] splitted = query.split(" ");
+            foreach(string s in splitted) {
+                if ((s.has_prefix("in:") && s != "in:" + current_in) || s.has_prefix("with:")) {
+                    // Already have an in: or with: filter -> no useful autocompletion possible
+                    return suggestions;
+                }
+            }
+            QueryBuilder groupchats = db.conversation.select()
+                .join_with(db.jid, db.jid.id, db.conversation.jid_id)
+                .join_with(db.account, db.account.id, db.conversation.account_id)
+                .with(db.jid.bare_jid, "LIKE", @"%$current_in%")
+                .with(db.account.enabled, "=", true)
+                .with(db.conversation.type_, "=", Conversation.Type.GROUPCHAT)
+                .order_by(db.conversation.last_active, "DESC")
+                .limit(limit);
+            foreach(Row chat in groupchats) {
+                suggestions.add(new SearchSuggestion(new Account.from_row(db, chat), new Jid(chat[db.jid.bare_jid]), "in:"+chat[db.jid.bare_jid], after_prev_space, next_space));
+            }
+        } else {
+            // Other auto complete?
+        }
+        return suggestions;
+    }
+
+    public Gee.List<MessageItem> match_messages(string query, int offset = -1) {
+        Gee.List<MessageItem> ret = new ArrayList<MessageItem>();
+        QueryBuilder rows = prepare_search(query, true).limit(10);
+        if (offset > 0) {
+            rows.offset(offset);
+        }
+        foreach (Row row in rows) {
+            Message message = new Message.from_row(db, row);
+            Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation_for_message(message);
+            ret.add(new MessageItem(message, conversation, row[db.content.id]));
+        }
+        return ret;
+    }
+
+    public int count_match_messages(string query) {
+        return (int)prepare_search(query, false).select({db.message.id}).count();
+    }
+}
+
+public class SearchSuggestion : Object {
+    public Account account { get; private set; }
+    public Jid? jid { get; private set; }
+    public string completion { get; private set; }
+    public int start_index { get; private set; }
+    public int end_index { get; private set; }
+    public long order { get; set; }
+
+    public SearchSuggestion(Account account, Jid? jid, string completion, int start_index, int end_index) {
+        this.account = account;
+        this.jid = jid;
+        this.completion = completion;
+        this.start_index = start_index;
+        this.end_index = end_index;
+    }
+}
+
+}