Restructure (http) file provider flow: Separate download&get_info, download in provider, move XmppStreamModule into xmpp-vala

10 files changed, 149 insertions, 122 deletions

diff --git a/libdino/src/entity/file_transfer.vala b/libdino/src/entity/file_transfer.vala
index be472796..2c45aa08 100644
--- a/libdino/src/entity/file_transfer.vala
+++ b/libdino/src/entity/file_transfer.vala
@@ -18,6 +18,12 @@ public class FileTransfer : Object {
     public Account account { get; set; }
     public Jid counterpart { get; set; }
     public Jid ourpart { get; set; }
+    public Jid? from {
+        get { return direction == DIRECTION_SENT ? ourpart : counterpart; }
+    }
+    public Jid? to {
+        get { return direction == DIRECTION_SENT ? counterpart : ourpart; }
+    }
     public bool direction { get; set; }
     public DateTime time { get; set; }
     public DateTime? local_time { get; set; }
@@ -47,7 +53,7 @@ public class FileTransfer : Object {
         set { server_file_name_ = value; }
     }
     public string path { get; set; }
-    public string mime_type { get; set; }
+    public string? mime_type { get; set; }
     public int size { get; set; }
 
     public State state { get; set; }
diff --git a/libdino/src/service/file_manager.vala b/libdino/src/service/file_manager.vala
index 340205af..b35a4e09 100644
--- a/libdino/src/service/file_manager.vala
+++ b/libdino/src/service/file_manager.vala
@@ -16,7 +16,7 @@ public class FileManager : StreamInteractionModule, Object {
     private StreamInteractor stream_interactor;
     private Database db;
     private Gee.List<FileSender> file_senders = new ArrayList<FileSender>();
-    private Gee.List<IncommingFileProcessor> incomming_processors = new ArrayList<IncommingFileProcessor>();
+    public Gee.List<IncommingFileProcessor> incomming_processors = new ArrayList<IncommingFileProcessor>();
     private Gee.List<OutgoingFileProcessor> outgoing_processors = new ArrayList<OutgoingFileProcessor>();
 
     public static void start(StreamInteractor stream_interactor, Database db) {
@@ -116,7 +116,7 @@ public class FileManager : StreamInteractionModule, Object {
     }
 
     public void add_provider(FileProvider file_provider) {
-        file_provider.file_incoming.connect(handle_incomming_file);
+        file_provider.file_incoming.connect((file_transfer, conversation) => { handle_incomming_file.begin(file_provider, file_transfer, conversation); });
     }
 
     public void add_sender(FileSender file_sender) {
@@ -134,13 +134,18 @@ public class FileManager : StreamInteractionModule, Object {
         outgoing_processors.add(processor);
     }
 
-    private void handle_incomming_file(FileTransfer file_transfer, Conversation conversation) {
-        foreach (IncommingFileProcessor processor in incomming_processors) {
-            if (processor.can_process(file_transfer)) {
-                processor.process(file_transfer);
-            }
-        }
-        save_file(file_transfer);
+    public bool is_sender_trustworthy(FileTransfer file_transfer, Conversation conversation) {
+        Jid relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(file_transfer.from, conversation.account) ?? conversation.counterpart;
+        bool in_roster = stream_interactor.get_module(RosterManager.IDENTITY).get_roster_item(conversation.account, relevant_jid) != null;
+        return file_transfer.direction == FileTransfer.DIRECTION_SENT || in_roster;
+    }
+
+    private async void handle_incomming_file(FileProvider file_provider, FileTransfer file_transfer, Conversation conversation) {
+        if (!is_sender_trustworthy(file_transfer, conversation)) return;
+
+        string filename = Random.next_int().to_string("%x") + "_" + file_transfer.file_name;
+        File file = File.new_for_path(Path.build_filename(get_storage_dir(), filename));
+        yield file_provider.download(file_transfer, file);
 
         try {
             FileInfo file_info = file_transfer.get_file().query_info("*", FileQueryInfoFlags.NONE);
@@ -170,6 +175,7 @@ public class FileManager : StreamInteractionModule, Object {
 
 public interface FileProvider : Object {
     public signal void file_incoming(FileTransfer file_transfer, Conversation conversation);
+    public abstract async void download(FileTransfer file_transfer, File file);
 }
 
 public interface FileSender : Object {
diff --git a/libdino/src/service/module_manager.vala b/libdino/src/service/module_manager.vala
index b9197f90..41a2c6a0 100644
--- a/libdino/src/service/module_manager.vala
+++ b/libdino/src/service/module_manager.vala
@@ -77,6 +77,7 @@ public class ModuleManager {
             module_map[account].add(new Xep.DelayedDelivery.Module());
             module_map[account].add(new StreamError.Module());
             module_map[account].add(new Xep.InBandRegistration.Module());
+            module_map[account].add(new Xep.HttpFileUpload.Module());
             initialize_account_modules(account, module_map[account]);
         }
     }
diff --git a/plugins/http-files/CMakeLists.txt b/plugins/http-files/CMakeLists.txt
index 340ff5b2..93ccbf67 100644
--- a/plugins/http-files/CMakeLists.txt
+++ b/plugins/http-files/CMakeLists.txt
@@ -13,7 +13,6 @@ SOURCES
     src/manager.vala
     src/plugin.vala
     src/register_plugin.vala
-    src/upload_stream_module.vala
 CUSTOM_VAPIS
     ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi
     ${CMAKE_BINARY_DIR}/exports/dino.vapi
diff --git a/plugins/http-files/src/file_provider.vala b/plugins/http-files/src/file_provider.vala
index 6f4b19f4..7b52a7fb 100644
--- a/plugins/http-files/src/file_provider.vala
+++ b/plugins/http-files/src/file_provider.vala
@@ -10,12 +10,14 @@ public class FileProvider : Dino.FileProvider, Object {
     public string id { get { return "http"; } }
 
     private StreamInteractor stream_interactor;
+    private Dino.Database dino_db;
     private Regex url_regex;
 
     private Gee.List<string> ignore_once = new ArrayList<string>();
 
     public FileProvider(StreamInteractor stream_interactor, Dino.Database dino_db) {
         this.stream_interactor = stream_interactor;
+        this.dino_db = dino_db;
         this.url_regex = new Regex("""^(?i)\b((?:[a-z][\w-]+:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}\/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))$""");
 
         stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(new ReceivedMessageListener(this));
@@ -39,77 +41,91 @@ public class FileProvider : Dino.FileProvider, Object {
         }
 
         public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) {
-            if (!outer.url_regex.match(message.body)) return false;
-            Jid relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(message.from, conversation.account) ?? conversation.counterpart;
-            bool in_roster = stream_interactor.get_module(RosterManager.IDENTITY).get_roster_item(conversation.account, relevant_jid) != null;
-            if (message.direction == Message.DIRECTION_RECEIVED && !in_roster) return false;
-
-            string? oob_url = Xmpp.Xep.OutOfBandData.get_url_from_message(message.stanza);
-            if (oob_url != null && oob_url == message.body) {
-                yield outer.download_url(message, conversation);
+            if (outer.url_regex.match(message.body)) {
+                string? oob_url = Xmpp.Xep.OutOfBandData.get_url_from_message(message.stanza);
+                if (oob_url != null && oob_url == message.body) {
+                    outer.on_file_message(message, conversation);
+                }
             }
             return false;
         }
     }
 
-    private async bool download_url(Message message, Conversation conversation) {
-        bool success = false;
+    private async void on_file_message(Entities.Message message, Conversation conversation) {
+        FileTransfer file_transfer = new FileTransfer();
+        file_transfer.account = conversation.account;
+        file_transfer.counterpart = message.counterpart;
+        file_transfer.ourpart = message.ourpart;
+        file_transfer.encryption = Encryption.NONE;
+        file_transfer.time = message.time;
+        file_transfer.local_time = message.local_time;
+        file_transfer.direction = message.direction;
+        file_transfer.file_name = message.body.substring(message.body.last_index_of("/") + 1);
+        file_transfer.mime_type = null;
+        file_transfer.size = -1;
+        file_transfer.state = FileTransfer.State.NOT_STARTED;
+        file_transfer.provider = 0;
+        file_transfer.info = message.id.to_string();
+
+        if (stream_interactor.get_module(FileManager.IDENTITY).is_sender_trustworthy(file_transfer, conversation)) {
+            ContentItem? content_item = stream_interactor.get_module(ContentItemStore.IDENTITY).get_item(conversation, 1, message.id);
+            if (content_item != null) {
+                stream_interactor.get_module(ContentItemStore.IDENTITY).set_item_hide(content_item, true);
+            }
+            yield get_meta_info(file_transfer);
+            file_incoming(file_transfer, conversation);
+        }
+    }
+
+    private async void get_meta_info(FileTransfer file_transfer) {
+        string url_body = dino_db.message.select({dino_db.message.body}).with(dino_db.message.id, "=", int.parse(file_transfer.info))[dino_db.message.body];
         var session = new Soup.Session();
-        var head_message = new Soup.Message("HEAD", message.body);
+        var head_message = new Soup.Message("HEAD", url_body);
         if (head_message != null) {
-            SourceFunc callback = download_url.callback;
-            session.send_async.begin(head_message, null, (obj, res) => {
-                string? content_type = null, content_length = null;
-                print(message.body + ":\n");
-                head_message.response_headers.foreach((name, val) => {
-                    print(name + " " + val + "\n");
-                    if (name == "Content-Type") content_type = val;
-                    if (name == "Content-Length") content_length = val;
-                });
-                if (content_length != null && int.parse(content_length) < 5000000) {
-                    FileTransfer file_transfer = new FileTransfer();
-                    try {
-                        Soup.Request request = session.request(message.body);
-                        request.send_async.begin(null, (obj, res) => {
-                            try {
-                                file_transfer.input_stream = request.send_async.end(res);
-                            } catch (Error e) {
-                                Idle.add((owned)callback);
-                                return;
-                            }
-                            file_transfer.account = conversation.account;
-                            file_transfer.counterpart = message.counterpart;
-                            file_transfer.ourpart = message.ourpart;
-                            file_transfer.encryption = Encryption.NONE;
-                            file_transfer.time = message.time;
-                            file_transfer.local_time = message.local_time;
-                            file_transfer.direction = message.direction;
-                            file_transfer.file_name = message.body.substring(message.body.last_index_of("/") + 1);
-                            file_transfer.mime_type = content_type;
-                            file_transfer.size = int.parse(content_length);
-                            file_transfer.state = FileTransfer.State.NOT_STARTED;
-                            file_transfer.provider = 0;
-                            file_transfer.info = message.id.to_string();
-                            file_incoming(file_transfer, conversation);
-
-                            ContentItem? content_item = stream_interactor.get_module(ContentItemStore.IDENTITY).get_item(conversation, 1, message.id);
-                            if (content_item != null) {
-                                stream_interactor.get_module(ContentItemStore.IDENTITY).set_item_hide(content_item, true);
-                            }
-
-                            success = true;
-                            Idle.add((owned)callback);
-                        });
-                    } catch (Error e) {
-                        Idle.add((owned)callback);
-                    }
-                } else {
-                    Idle.add((owned)callback);
-                }
+            SourceFunc callback = get_meta_info.callback;
+            yield session.send_async(head_message, null);
+
+            string? content_type = null, content_length = null;
+            print(url_body + ":\n");
+            head_message.response_headers.foreach((name, val) => {
+                print(name + " " + val + "\n");
+                if (name == "Content-Type") content_type = val;
+                if (name == "Content-Length") content_length = val;
             });
-            yield;
+            file_transfer.mime_type = content_type;
+            file_transfer.size = int.parse(content_length);
+        }
+    }
+
+    public async void download(FileTransfer file_transfer, File file_) {
+        try {
+            File file = file_;
+            string url_body = dino_db.message.select({dino_db.message.body}).with(dino_db.message.id, "=", int.parse(file_transfer.info))[dino_db.message.body];
+            var session = new Soup.Session();
+            Soup.Request request = session.request(url_body);
+
+            file_transfer.input_stream = yield request.send_async(null);
+
+            foreach (IncommingFileProcessor processor in stream_interactor.get_module(FileManager.IDENTITY).incomming_processors) {
+                if (processor.can_process(file_transfer)) {
+                    processor.process(file_transfer);
+                }
+            }
+
+            if (file_transfer.encryption == Encryption.PGP || file.get_path().has_suffix(".pgp")) {
+                file = File.new_for_path(file.get_path().substring(0, file.get_path().length - 4));
+            }
+
+            OutputStream os = file.create(FileCreateFlags.REPLACE_DESTINATION);
+            os.splice(file_transfer.input_stream, 0);
+            os.close();
+            file_transfer.path = file.get_basename();
+            file_transfer.input_stream = file.read();
+
+            file_transfer.state = FileTransfer.State.COMPLETE;
+        } catch (Error e) {
+            file_transfer.state = FileTransfer.State.FAILED;
         }
-        return success;
     }
 }
 
diff --git a/plugins/http-files/src/manager.vala b/plugins/http-files/src/manager.vala
index 78697f9c..b5aa0d26 100644
--- a/plugins/http-files/src/manager.vala
+++ b/plugins/http-files/src/manager.vala
@@ -29,10 +29,46 @@ public class Manager : StreamInteractionModule, FileSender, Object {
         stream_interactor.get_module(MessageProcessor.IDENTITY).build_message_stanza.connect(check_add_oob);
     }
 
+    public delegate void OnUploadOk(XmppStream stream, string url_down);
+    public delegate void OnError(XmppStream stream, string error);
+    public void upload(XmppStream stream, FileTransfer file_transfer, owned OnUploadOk listener, owned OnError error_listener) {
+        uint8[] buf = new uint8[256];
+        Array<uint8> data = new Array<uint8>(false, true, 0);
+        size_t len = -1;
+        do {
+            try {
+                len = file_transfer.input_stream.read(buf);
+            } catch (IOError error) {
+                error_listener(stream, @"HTTP upload: IOError reading stream: $(error.message)");
+            }
+            data.append_vals(buf, (uint) len);
+        } while(len > 0);
+
+        stream_interactor.module_manager.get_module(file_transfer.account, Xmpp.Xep.HttpFileUpload.Module.IDENTITY).request_slot(stream, file_transfer.server_file_name, (int) data.length, file_transfer.mime_type,
+            (stream, url_down, url_up) => {
+                Soup.Message message = new Soup.Message("PUT", url_up);
+                message.set_request(file_transfer.mime_type, Soup.MemoryUse.COPY, data.data);
+                Soup.Session session = new Soup.Session();
+                session.send_async.begin(message, null, (obj, res) => {
+                    try {
+                        session.send_async.end(res);
+                        if (message.status_code >= 200 && message.status_code < 300) {
+                            listener(stream, url_down);
+                        } else {
+                            error_listener(stream, "HTTP status code " + message.status_code.to_string());
+                        }
+                    } catch (Error e) {
+                        error_listener(stream, e.message);
+                    }
+                });
+            },
+            (stream, error) => error_listener(stream, error));
+    }
+
     public void send_file(Conversation conversation, FileTransfer file_transfer) {
         Xmpp.XmppStream? stream = stream_interactor.get_stream(file_transfer.account);
         if (stream != null) {
-            stream_interactor.module_manager.get_module(file_transfer.account, UploadStreamModule.IDENTITY).upload(stream, file_transfer.input_stream, file_transfer.server_file_name, file_transfer.mime_type,
+            upload(stream, file_transfer,
                 (stream, url_down) => {
                     uploaded(file_transfer, url_down);
                     file_transfer.info = url_down;
@@ -71,7 +107,7 @@ public class Manager : StreamInteractionModule, FileSender, Object {
     }
 
     private void on_stream_negotiated(Account account, XmppStream stream) {
-        stream_interactor.module_manager.get_module(account, UploadStreamModule.IDENTITY).feature_available.connect((stream, max_file_size) => {
+        stream_interactor.module_manager.get_module(account, Xmpp.Xep.HttpFileUpload.Module.IDENTITY).feature_available.connect((stream, max_file_size) => {
             lock (max_file_sizes) {
                 max_file_sizes[account] = max_file_size;
             }
diff --git a/plugins/http-files/src/plugin.vala b/plugins/http-files/src/plugin.vala
index bd136f31..e2ce5ad9 100644
--- a/plugins/http-files/src/plugin.vala
+++ b/plugins/http-files/src/plugin.vala
@@ -14,10 +14,6 @@ public class Plugin : RootInterface, Object {
 
         file_provider = new FileProvider(app.stream_interactor, app.db);
 
-        app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => {
-            list.add(new UploadStreamModule());
-        });
-
         app.stream_interactor.get_module(FileManager.IDENTITY).add_provider(file_provider);
         app.stream_interactor.get_module(ContentItemStore.IDENTITY).add_filter(new FileMessageFilter(app.db));
     }
diff --git a/plugins/openpgp/src/out_file_processor.vala b/plugins/openpgp/src/out_file_processor.vala
index 522e518a..40eef7f5 100644
--- a/plugins/openpgp/src/out_file_processor.vala
+++ b/plugins/openpgp/src/out_file_processor.vala
@@ -23,6 +23,7 @@ public class OutFileProcessor : OutgoingFileProcessor, Object {
             file_transfer.encryption = Encryption.PGP;
             file_transfer.server_file_name = Xmpp.random_uuid() + ".pgp";
         } catch (Error e) {
+            warning(@"PGP file encryption error: $(e.message)\n");
             file_transfer.state = FileTransfer.State.FAILED;
         }
     }
diff --git a/xmpp-vala/CMakeLists.txt b/xmpp-vala/CMakeLists.txt
index 1649411e..fba9966d 100644
--- a/xmpp-vala/CMakeLists.txt
+++ b/xmpp-vala/CMakeLists.txt
@@ -68,6 +68,7 @@ SOURCES
     "src/module/xep/0280_message_carbons.vala"
     "src/module/xep/0313_message_archive_management.vala"
     "src/module/xep/0333_chat_markers.vala"
+    "src/module/xep/0363_http_file_upload.vala"
     "src/module/xep/0368_srv_records_tls.vala"
     "src/module/xep/0380_explicit_encryption.vala"
     "src/module/xep/pixbuf_storage.vala"
diff --git a/plugins/http-files/src/upload_stream_module.vala b/xmpp-vala/src/module/xep/0363_http_file_upload.vala
index 934d4df5..d2e2af2c 100644
--- a/plugins/http-files/src/upload_stream_module.vala
+++ b/xmpp-vala/src/module/xep/0363_http_file_upload.vala
@@ -2,55 +2,20 @@ using Xmpp;
 using Xmpp;
 using Xmpp.Xep;
 
-namespace Dino.Plugins.HttpFiles {
+namespace Xmpp.Xep.HttpFileUpload {
 
 private const string NS_URI = "urn:xmpp:http:upload";
 private const string NS_URI_0 = "urn:xmpp:http:upload:0";
 
-public class UploadStreamModule : XmppStreamModule {
-    public static Xmpp.ModuleIdentity<UploadStreamModule> IDENTITY = new Xmpp.ModuleIdentity<UploadStreamModule>(NS_URI, "0363_http_file_upload");
+public class Module : XmppStreamModule {
+    public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0363_http_file_upload");
 
     public signal void feature_available(XmppStream stream, long max_file_size);
     public signal void received_url(XmppStream stream, MessageStanza message);
 
-    public delegate void OnUploadOk(XmppStream stream, string url_down);
+    public delegate void OnSlotOk(XmppStream stream, string url_get, string url_put);
     public delegate void OnError(XmppStream stream, string error);
-    public void upload(XmppStream stream, InputStream input_stream, string file_name, string file_content_type, owned OnUploadOk listener, owned OnError error_listener) {
-        uint8[] buf = new uint8[256];
-        Array<uint8> data = new Array<uint8>(false, true, 0);
-        size_t len = -1;
-        do {
-            try {
-                len = input_stream.read(buf);
-            } catch (IOError error) {
-                error_listener(stream, @"HTTP upload: IOError reading stream: $(error.message)");
-            }
-            data.append_vals(buf, (uint) len);
-        } while(len > 0);
-
-        request_slot(stream, file_name, (int) data.length, file_content_type,
-            (stream, url_down, url_up) => {
-                Soup.Message message = new Soup.Message("PUT", url_up);
-                message.set_request(file_content_type, Soup.MemoryUse.COPY, data.data);
-                Soup.Session session = new Soup.Session();
-                session.send_async.begin(message, null, (obj, res) => {
-                    try {
-                        session.send_async.end(res);
-                        if (message.status_code >= 200 && message.status_code < 300) {
-                            listener(stream, url_down);
-                        } else {
-                            error_listener(stream, "HTTP status code " + message.status_code.to_string());
-                        }
-                    } catch (Error e) {
-                        error_listener(stream, e.message);
-                    }
-                });
-            },
-            (stream, error) => error_listener(stream, error));
-    }
-
-    private delegate void OnSlotOk(XmppStream stream, string url_get, string url_put);
-    private void request_slot(XmppStream stream, string filename, int file_size, string? content_type, owned OnSlotOk listener, owned OnError error_listener) {
+    public void request_slot(XmppStream stream, string filename, int file_size, string? content_type, owned OnSlotOk listener, owned OnError error_listener) {
         Flag? flag = stream.get_flag(Flag.IDENTITY);
         if (flag == null) return;
 
@@ -73,7 +38,7 @@ public class UploadStreamModule : XmppStreamModule {
         Iq.Stanza iq = new Iq.Stanza.get(request_node) { to=flag.file_store_jid };
         stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq, (stream, iq) => {
             if (iq.is_error()) {
-                error_listener(stream, "Error getting upload/download url");
+                error_listener(stream, "Error getting upload/download url (Error Iq)");
                 return;
             }
             string? url_get = null, url_put = null;
@@ -170,14 +135,14 @@ public class ReceivedPipelineListener : StanzaListener<MessageStanza> {
     public override async bool run(XmppStream stream, MessageStanza message) {
         string? oob_url = OutOfBandData.get_url_from_message(message);
         if (oob_url != null && oob_url == message.body) {
-            stream.get_module(UploadStreamModule.IDENTITY).received_url(stream, message);
+            stream.get_module(Module.IDENTITY).received_url(stream, message);
         }
         return true;
     }
 }
 
 public class Flag : XmppStreamFlag {
-    public static FlagIdentity<Flag> IDENTITY = new FlagIdentity<Flag>(NS_URI, "service_discovery");
+    public static FlagIdentity<Flag> IDENTITY = new FlagIdentity<Flag>(NS_URI, "http_file_upload");
 
     public Jid file_store_jid;
     public string ns_ver;