Fix and improve stateless file-sharing

1 files changed, 77 insertions, 224 deletions

diff --git a/libdino/src/service/file_manager.vala b/libdino/src/service/file_manager.vala
index 32cf23c4..2a665e1e 100644
--- a/libdino/src/service/file_manager.vala
+++ b/libdino/src/service/file_manager.vala
@@ -22,6 +22,11 @@ public class FileManager : StreamInteractionModule, Object {
     private Gee.List<FileProvider> file_providers = new ArrayList<FileProvider>();
     private Gee.List<FileMetadataProvider> file_metadata_providers = new ArrayList<FileMetadataProvider>();
 
+    public StatelessFileSharing sfs {
+        owned get { return stream_interactor.get_module(StatelessFileSharing.IDENTITY); }
+        private set { }
+    }
+
     public static void start(StreamInteractor stream_interactor, Database db) {
         FileManager m = new FileManager(stream_interactor, db);
         stream_interactor.add_module(m);
@@ -40,15 +45,12 @@ public class FileManager : StreamInteractionModule, Object {
         this.add_sender(new JingleFileSender(stream_interactor));
         this.add_metadata_provider(new GenericFileMetadataProvider());
         this.add_metadata_provider(new ImageFileMetadataProvider());
-        this.stream_interactor.account_added.connect((account) => {
-            on_account_added(account);
-        });
     }
 
     public const int HTTP_PROVIDER_ID = 0;
     public const int SFS_PROVIDER_ID = 2;
 
-    private FileProvider? select_file_provider(FileTransfer file_transfer) {
+    public FileProvider? select_file_provider(FileTransfer file_transfer) {
         bool http_usable = file_transfer.provider == SFS_PROVIDER_ID;
         foreach (FileProvider file_provider in this.file_providers) {
             if (file_transfer.provider == file_provider.get_id()) {
@@ -61,98 +63,7 @@ public class FileManager : StreamInteractionModule, Object {
         return null;
     }
 
-    // For receiving out of band data as sfs
-    private async void on_backwards_compatible_sfs(FileProvider file_provider, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) {
-        Xep.StatelessFileSharing.SfsElement sfs_element = new Xep.StatelessFileSharing.SfsElement();
-
-        Xep.StatelessFileSharing.HttpSource source = new Xep.StatelessFileSharing.HttpSource();
-        HttpFileReceiveData http_receive_data = receive_data as HttpFileReceiveData;
-        source.url = http_receive_data.url;
-        sfs_element.sources.add(source);
-
-        FileTransfer file_transfer = new FileTransfer();
-
-        if (is_jid_trustworthy(from, conversation)) {
-            try {
-                file_meta = yield file_provider.get_meta_info(file_transfer, http_receive_data, file_meta);
-            } catch (Error e) {
-                warning("Can't accept oob data as stateless file sharing due to failed http request\n");
-            }
-        }
-
-        sfs_element.metadata.size = file_meta.size;
-        sfs_element.metadata.name = file_meta.file_name;
-        sfs_element.metadata.mime_type = file_meta.mime_type;
-        // Encryption unused in http file transfers
-
-        yield on_receive_sfs(from, conversation, sfs_element, null);
-    }
-
-    private async void on_receive_sfs(Jid from, Conversation conversation, Xep.StatelessFileSharing.SfsElement sfs_element, string? id) {
-        FileTransfer file_transfer = new FileTransfer();
-        file_transfer.account = conversation.account;
-        file_transfer.counterpart = file_transfer.direction == FileTransfer.DIRECTION_RECEIVED ? from : conversation.counterpart;
-        if (conversation.type_.is_muc_semantic()) {
-            file_transfer.ourpart = stream_interactor.get_module(MucManager.IDENTITY).get_own_jid(conversation.counterpart, conversation.account) ?? conversation.account.bare_jid;
-            file_transfer.direction = from.equals(file_transfer.ourpart) ? FileTransfer.DIRECTION_SENT : FileTransfer.DIRECTION_RECEIVED;
-        } else {
-            file_transfer.ourpart = conversation.account.full_jid;
-            file_transfer.direction = from.equals_bare(file_transfer.ourpart) ? FileTransfer.DIRECTION_SENT : FileTransfer.DIRECTION_RECEIVED;
-        }
-        file_transfer.time = new DateTime.now_utc();
-        // TODO: get time from message
-        file_transfer.local_time = new DateTime.now_utc();
-        file_transfer.provider = SFS_PROVIDER_ID;
-        file_transfer.with_metadata_element(sfs_element.metadata);
-        foreach (Xep.StatelessFileSharing.SfsSource source in sfs_element.sources) {
-            file_transfer.sfs_sources.append(new FileTransfer.SerializedSfsSource.from_sfs_source(source) as Object);
-        }
-        // FileTransfer.info stores the id of the MessageStanza for future SfsSourceAttachments
-        // Prior to sfs, info stored the id of the Message entity for oob
-        file_transfer.info = id;
-
-        stream_interactor.get_module(FileTransferStorage.IDENTITY).add_file(file_transfer);
-
-        if (is_sender_trustworthy(file_transfer, conversation)) {
-            if (file_transfer.size >= 0 && file_transfer.size < 500) {
-                FileProvider? file_provider = this.select_file_provider(file_transfer);
-                download_file_internal.begin(file_provider, file_transfer, conversation, (_, res) => {
-                    download_file_internal.end(res);
-                });
-            }
-        }
-
-        conversation.last_active = file_transfer.time;
-        received_file(file_transfer, conversation);
-    }
-
-    private void on_receive_sfs_attachment(Jid from, Conversation conversation, Xep.StatelessFileSharing.SfsSourceAttachment attachment) {
-        foreach (Qlite.Row file_transfer_row in this.db.file_transfer.select()
-                .with(db.file_transfer.info, "=", attachment.sfs_id)) {
-            FileTransfer file_transfer = new FileTransfer.from_row(this.db, file_transfer_row, FileManager.get_storage_dir());
-            if (file_transfer.hashes.supported_hashes().is_empty) {
-                return;
-            }
-            foreach (StatelessFileSharing.SfsSource source in attachment.sources) {
-                file_transfer.sfs_sources.append(new FileTransfer.SerializedSfsSource.from_sfs_source(source) as Object);
-            }
-        }
-
-    }
-
-    private void on_account_added(Account account) {
-        Xep.StatelessFileSharing.Module fsf_module = stream_interactor.module_manager.get_module(account, Xep.StatelessFileSharing.Module.IDENTITY);
-        fsf_module.received_sfs.connect((from, to, sfs_element, message) => {
-            Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).approx_conversation_for_stanza(from, to, account, message.type_);
-            on_receive_sfs(from, conversation, sfs_element, message.id);
-        });
-        fsf_module.received_sfs_attachment.connect((from, to, sfs_attachment, message) => {
-            Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).approx_conversation_for_stanza(from, to, account, message.type_);
-            on_receive_sfs_attachment(from, conversation, sfs_attachment);
-        });
-    }
-
-        public async HashMap<int, long> get_file_size_limits(Conversation conversation) {
+    public async HashMap<int, long> get_file_size_limits(Conversation conversation) {
         HashMap<int, long> ret = new HashMap<int, long>();
         foreach (FileSender sender in file_senders) {
             ret[sender.get_id()] = yield sender.get_file_size_limit(conversation);
@@ -180,7 +91,7 @@ public class FileManager : StreamInteractionModule, Object {
                 yield file_metadata_provider.fill_metadata(file, metadata);
             }
         }
-        file_transfer.with_metadata_element(metadata);
+        file_transfer.file_metadata = metadata;
 
         try {
             file_transfer.input_stream = yield file.read_async();
@@ -237,7 +148,16 @@ public class FileManager : StreamInteractionModule, Object {
                 file_send_data = file_encryptor.preprocess_send_file(conversation, file_transfer, file_send_data, file_meta);
             }
 
+            file_transfer.state = FileTransfer.State.IN_PROGRESS;
+
+            // Update current download progress in the FileTransfer
+            LimitInputStream? limit_stream = file_transfer.input_stream as LimitInputStream;
+            if (limit_stream != null) {
+                limit_stream.bind_property("retrieved-bytes", file_transfer, "transferred-bytes", BindingFlags.SYNC_CREATE);
+            }
+
             yield file_sender.send_file(conversation, file_transfer, file_send_data, file_meta);
+            file_transfer.state = FileTransfer.State.COMPLETE;
 
         } catch (Error e) {
             warning("Send file error: %s", e.message);
@@ -265,12 +185,7 @@ public class FileManager : StreamInteractionModule, Object {
     public void add_provider(FileProvider file_provider) {
         file_providers.add(file_provider);
         file_provider.file_incoming.connect((info, from, time, local_time, conversation, receive_data, file_meta) => {
-            if (receive_data is HttpFileReceiveData) {
-                printerr("Handling oob data as stateless file sharing");
-                this.on_backwards_compatible_sfs.begin(file_provider, from, time, local_time, conversation, receive_data, file_meta);
-            } else {
-                handle_incoming_file.begin(file_provider, info, from, time, local_time, conversation, receive_data, file_meta);
-            }
+            handle_incoming_file.begin(file_provider, info, from, time, local_time, conversation, receive_data, file_meta);
         });
     }
 
@@ -296,10 +211,12 @@ public class FileManager : StreamInteractionModule, Object {
         file_metadata_providers.add(file_metadata_provider);
     }
 
-    private bool is_jid_trustworthy(Jid from, Conversation conversation) {
+    public bool is_sender_trustworthy(FileTransfer file_transfer, Conversation conversation) {
+        if (file_transfer.direction == FileTransfer.DIRECTION_SENT) return true;
+
         Jid relevant_jid = conversation.counterpart;
         if (conversation.type_ == Conversation.Type.GROUPCHAT) {
-            relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(from, conversation.account);
+            relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(file_transfer.from, conversation.account);
         }
         if (relevant_jid == null) return false;
 
@@ -307,12 +224,6 @@ public class FileManager : StreamInteractionModule, Object {
         return in_roster;
     }
 
-    public bool is_sender_trustworthy(FileTransfer file_transfer, Conversation conversation) {
-        if (file_transfer.direction == FileTransfer.DIRECTION_SENT) return true;
-
-        return is_jid_trustworthy(file_transfer.from, conversation);
-    }
-
     private async FileMeta get_file_meta(FileProvider file_provider, FileTransfer file_transfer, Conversation conversation, FileReceiveData receive_data_) throws FileReceiveError {
         FileReceiveData receive_data = receive_data_;
         FileMeta file_meta = file_provider.get_file_meta(file_transfer);
@@ -337,7 +248,11 @@ public class FileManager : StreamInteractionModule, Object {
     private async void download_file_internal(FileProvider file_provider, FileTransfer file_transfer, Conversation conversation) {
         try {
             // Get meta info
-            FileReceiveData receive_data = yield file_provider.get_file_receive_data(file_transfer);
+            FileReceiveData? receive_data = file_provider.get_file_receive_data(file_transfer);
+            if (receive_data == null) {
+                warning("Don't have download data (yet)");
+                return;
+            }
             FileDecryptor? file_decryptor = null;
             foreach (FileDecryptor decryptor in file_decryptors) {
                 if (decryptor.can_decrypt_file(conversation, file_transfer, receive_data)) {
@@ -367,10 +282,17 @@ public class FileManager : StreamInteractionModule, Object {
                 input_stream = yield file_decryptor.decrypt_file(input_stream, conversation, file_transfer, receive_data);
             }
 
+            // Update current download progress in the FileTransfer
+            LimitInputStream? limit_stream = input_stream as LimitInputStream;
+            if (limit_stream != null) {
+                limit_stream.bind_property("retrieved-bytes", file_transfer, "transferred-bytes", BindingFlags.SYNC_CREATE);
+            }
+
             // Save file
             string filename = Random.next_int().to_string("%x") + "_" + file_transfer.file_name;
             File file = File.new_for_path(Path.build_filename(get_storage_dir(), filename));
 
+            // libsoup doesn't properly support splicing
             OutputStream os = file.create(FileCreateFlags.REPLACE_DESTINATION);
             uint8[] buffer = new uint8[1024];
             ssize_t read;
@@ -381,20 +303,49 @@ public class FileManager : StreamInteractionModule, Object {
             }
             yield input_stream.close_async(Priority.LOW, file_transfer.cancellable);
             yield os.close_async(Priority.LOW, file_transfer.cancellable);
+
+            // Verify the hash of the downloaded file, if it is known
+            var supported_hashes = Xep.CryptographicHashes.get_supported_hashes(file_transfer.hashes);
+            if (!supported_hashes.is_empty) {
+                var checksum_types = new ArrayList<ChecksumType>();
+                var hashes = new HashMap<ChecksumType, string>();
+                foreach (var hash in supported_hashes) {
+                    var checksum_type = Xep.CryptographicHashes.hash_string_to_type(hash.algo);
+                    checksum_types.add(checksum_type);
+                    hashes[checksum_type] = hash.val;
+                }
+
+                var computed_hashes = yield compute_file_hashes(file, checksum_types);
+                foreach (var checksum_type in hashes.keys) {
+                    if (hashes[checksum_type] != computed_hashes[checksum_type]) {
+                        warning("Hash of downloaded file does not equal advertised hash, discarding: %s. %s should be %s, was %s",
+                                file_transfer.file_name, checksum_type.to_string(), hashes[checksum_type], computed_hashes[checksum_type]);
+                        FileUtils.remove(file.get_path());
+                        file_transfer.state = FileTransfer.State.FAILED;
+                        return;
+                    }
+                }
+            }
+
             file_transfer.path = file.get_basename();
-            file_transfer.input_stream = yield file.read_async();
 
             FileInfo file_info = file_transfer.get_file().query_info("*", FileQueryInfoFlags.NONE);
             file_transfer.mime_type = file_info.get_content_type();
 
             file_transfer.state = FileTransfer.State.COMPLETE;
+        } catch (IOError.CANCELLED e) {
+            print("cancelled\n");
         } catch (Error e) {
             warning("Error downloading file: %s", e.message);
-            file_transfer.state = FileTransfer.State.FAILED;
+            if (file_transfer.provider == 0 || file_transfer.provider == FileManager.SFS_PROVIDER_ID) {
+                file_transfer.state = FileTransfer.State.NOT_STARTED;
+            } else {
+                file_transfer.state = FileTransfer.State.FAILED;
+            }
         }
     }
 
-    private async void handle_incoming_file(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) {
+    public FileTransfer create_file_transfer_from_provider_incoming(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) {
         FileTransfer file_transfer = new FileTransfer();
         file_transfer.account = conversation.account;
         file_transfer.counterpart = file_transfer.direction == FileTransfer.DIRECTION_RECEIVED ? from : conversation.counterpart;
@@ -426,6 +377,11 @@ public class FileManager : StreamInteractionModule, Object {
             }
         }
 
+        return file_transfer;
+    }
+
+    private async void handle_incoming_file(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) {
+        FileTransfer file_transfer = create_file_transfer_from_provider_incoming(file_provider, info, from, time, local_time, conversation, receive_data, file_meta);
         stream_interactor.get_module(FileTransferStorage.IDENTITY).add_file(file_transfer);
 
         if (is_sender_trustworthy(file_transfer, conversation)) {
@@ -451,10 +407,10 @@ public class FileManager : StreamInteractionModule, Object {
             string filename = Random.next_int().to_string("%x") + "_" + file_transfer.file_name;
             File file = File.new_for_path(Path.build_filename(get_storage_dir(), filename));
             OutputStream os = file.create(FileCreateFlags.REPLACE_DESTINATION);
-            yield os.splice_async(file_transfer.input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE|OutputStreamSpliceFlags.CLOSE_TARGET);
+            yield os.splice_async(file_transfer.input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE | OutputStreamSpliceFlags.CLOSE_TARGET);
             file_transfer.state = FileTransfer.State.COMPLETE;
             file_transfer.path = filename;
-            file_transfer.input_stream = yield file.read_async();
+            file_transfer.input_stream = new LimitInputStream(yield file.read_async(), file_transfer.size);
         } catch (Error e) {
             throw new FileSendError.SAVE_FAILED("Saving file error: %s".printf(e.message));
         }
@@ -467,10 +423,10 @@ public errordomain FileSendError {
     SAVE_FAILED
 }
 
+// Get rid of this Error and pass IoErrors instead - DOWNLOAD_FAILED already removed
 public errordomain FileReceiveError {
     GET_METADATA_FAILED,
-    DECRYPTION_FAILED,
-    DOWNLOAD_FAILED
+    DECRYPTION_FAILED
 }
 
 public class FileMeta {
@@ -505,10 +461,10 @@ public interface FileProvider : Object {
 
     public abstract Encryption get_encryption(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta);
     public abstract FileMeta get_file_meta(FileTransfer file_transfer) throws FileReceiveError;
-    public abstract async FileReceiveData? get_file_receive_data(FileTransfer file_transfer);
+    public abstract FileReceiveData? get_file_receive_data(FileTransfer file_transfer);
 
     public abstract async FileMeta get_meta_info(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws FileReceiveError;
-    public abstract async InputStream download(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws FileReceiveError;
+    public abstract async InputStream download(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws IOError;
 
     public abstract int get_id();
 }
@@ -541,107 +497,4 @@ public interface FileDecryptor : Object {
     public abstract async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError;
 }
 
-public interface FileMetadataProvider : Object {
-    public abstract bool supports_file(File file);
-
-    public abstract async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata);
-}
-
-class GenericFileMetadataProvider: Dino.FileMetadataProvider, Object {
-    public bool supports_file(File file) {
-        return true;
-    }
-
-    public async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata) {
-        FileInfo info = file.query_info("*", FileQueryInfoFlags.NONE);
-
-        metadata.name = info.get_display_name();
-        metadata.mime_type = info.get_content_type();
-        metadata.size = info.get_size();
-        metadata.date = info.get_modification_date_time();
-
-        Bytes file_data = file.load_bytes();
-        metadata.hashes.hashes.add(new CryptographicHashes.Hash.from_data(GLib.ChecksumType.SHA256, file_data.get_data()));
-        metadata.hashes.hashes.add(new CryptographicHashes.Hash.from_data(GLib.ChecksumType.SHA512, file_data.get_data()));
-    }
-}
-
-public class ImageFileMetadataProvider: Dino.FileMetadataProvider, Object {
-    public bool supports_file(File file) {
-        return file.query_info("*", FileQueryInfoFlags.NONE).get_content_type().has_prefix("image");
-    }
-
-    private const int[] THUMBNAIL_DIMS = { 1, 2, 3, 4, 8 };
-    private const string IMAGE_TYPE = "png";
-    private const string MIME_TYPE = "image/png";
-
-    public async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata) {
-        Pixbuf pixbuf = new Pixbuf.from_stream(yield file.read_async());
-        metadata.width = pixbuf.get_width();
-        metadata.height = pixbuf.get_height();
-        float ratio = (float)metadata.width / (float) metadata.height;
-
-        int thumbnail_width = -1;
-        int thumbnail_height = -1;
-        float diff = float.INFINITY;
-        for (int i = 0; i < THUMBNAIL_DIMS.length; i++) {
-            int test_width = THUMBNAIL_DIMS[i];
-            int test_height = THUMBNAIL_DIMS[THUMBNAIL_DIMS.length - 1 - i];
-            float test_ratio = (float)test_width / (float)test_height;
-            float test_diff = (test_ratio - ratio).abs();
-            if (test_diff < diff) {
-                thumbnail_width = test_width;
-                thumbnail_height = test_height;
-                diff = test_diff;
-            }
-        }
-
-        Pixbuf thumbnail_pixbuf = pixbuf.scale_simple(thumbnail_width, thumbnail_height, InterpType.BILINEAR);
-        uint8[] buffer;
-        thumbnail_pixbuf.save_to_buffer(out buffer, IMAGE_TYPE);
-        string base_64 = GLib.Base64.encode(buffer);
-        string uri = @"data:$MIME_TYPE;base64,$base_64";
-        Xep.JingleContentThumbnails.Thumbnail thumbnail = new Xep.JingleContentThumbnails.Thumbnail();
-        thumbnail.uri = uri;
-        thumbnail.media_type = MIME_TYPE;
-        thumbnail.width = thumbnail_width;
-        thumbnail.height = thumbnail_height;
-        metadata.thumbnails.add(thumbnail);
-    }
-
-    public static Pixbuf? parse_thumbnail(Xep.JingleContentThumbnails.Thumbnail thumbnail) {
-        string[] splits = thumbnail.uri.split(":", 2);
-        if (splits.length != 2) {
-            printerr("Thumbnail parsing error: ':' not found");
-            return null;
-        }
-        if (splits[0] != "data") {
-            printerr("Unsupported thumbnail: unimplemented uri type\n");
-            return null;
-        }
-        splits = splits[1].split(";", 2);
-        if (splits.length != 2) {
-            printerr("Thumbnail parsing error: ';' not found");
-            return null;
-        }
-        if (splits[0] != MIME_TYPE) {
-            printerr("Unsupported thumbnail: unsupported mime-type\n");
-            return null;
-        }
-        splits = splits[1].split(",", 2);
-        if (splits.length != 2) {
-            printerr("Thumbnail parsing error: ',' not found");
-            return null;
-        }
-        if (splits[0] != "base64") {
-            printerr("Unsupported thumbnail: data is not base64 encoded\n");
-            return null;
-        }
-        uint8[] data = Base64.decode(splits[1]);
-        MemoryInputStream input_stream = new MemoryInputStream.from_data(data);
-        Pixbuf pixbuf = new Pixbuf.from_stream(input_stream);
-        return pixbuf;
-    }
-}
-
 }