From 79f792e090330a05753f9edb27332a946eb0840d Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 2 Nov 2024 22:24:59 +0100 Subject: Fix and improve stateless file-sharing --- libdino/src/service/file_manager.vala | 301 +++++++++------------------------- 1 file changed, 77 insertions(+), 224 deletions(-) (limited to 'libdino/src/service/file_manager.vala') diff --git a/libdino/src/service/file_manager.vala b/libdino/src/service/file_manager.vala index 32cf23c4..2a665e1e 100644 --- a/libdino/src/service/file_manager.vala +++ b/libdino/src/service/file_manager.vala @@ -22,6 +22,11 @@ public class FileManager : StreamInteractionModule, Object { private Gee.List file_providers = new ArrayList(); private Gee.List file_metadata_providers = new ArrayList(); + public StatelessFileSharing sfs { + owned get { return stream_interactor.get_module(StatelessFileSharing.IDENTITY); } + private set { } + } + public static void start(StreamInteractor stream_interactor, Database db) { FileManager m = new FileManager(stream_interactor, db); stream_interactor.add_module(m); @@ -40,15 +45,12 @@ public class FileManager : StreamInteractionModule, Object { this.add_sender(new JingleFileSender(stream_interactor)); this.add_metadata_provider(new GenericFileMetadataProvider()); this.add_metadata_provider(new ImageFileMetadataProvider()); - this.stream_interactor.account_added.connect((account) => { - on_account_added(account); - }); } public const int HTTP_PROVIDER_ID = 0; public const int SFS_PROVIDER_ID = 2; - private FileProvider? select_file_provider(FileTransfer file_transfer) { + public FileProvider? select_file_provider(FileTransfer file_transfer) { bool http_usable = file_transfer.provider == SFS_PROVIDER_ID; foreach (FileProvider file_provider in this.file_providers) { if (file_transfer.provider == file_provider.get_id()) { @@ -61,98 +63,7 @@ public class FileManager : StreamInteractionModule, Object { return null; } - // For receiving out of band data as sfs - private async void on_backwards_compatible_sfs(FileProvider file_provider, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) { - Xep.StatelessFileSharing.SfsElement sfs_element = new Xep.StatelessFileSharing.SfsElement(); - - Xep.StatelessFileSharing.HttpSource source = new Xep.StatelessFileSharing.HttpSource(); - HttpFileReceiveData http_receive_data = receive_data as HttpFileReceiveData; - source.url = http_receive_data.url; - sfs_element.sources.add(source); - - FileTransfer file_transfer = new FileTransfer(); - - if (is_jid_trustworthy(from, conversation)) { - try { - file_meta = yield file_provider.get_meta_info(file_transfer, http_receive_data, file_meta); - } catch (Error e) { - warning("Can't accept oob data as stateless file sharing due to failed http request\n"); - } - } - - sfs_element.metadata.size = file_meta.size; - sfs_element.metadata.name = file_meta.file_name; - sfs_element.metadata.mime_type = file_meta.mime_type; - // Encryption unused in http file transfers - - yield on_receive_sfs(from, conversation, sfs_element, null); - } - - private async void on_receive_sfs(Jid from, Conversation conversation, Xep.StatelessFileSharing.SfsElement sfs_element, string? id) { - FileTransfer file_transfer = new FileTransfer(); - file_transfer.account = conversation.account; - file_transfer.counterpart = file_transfer.direction == FileTransfer.DIRECTION_RECEIVED ? from : conversation.counterpart; - if (conversation.type_.is_muc_semantic()) { - file_transfer.ourpart = stream_interactor.get_module(MucManager.IDENTITY).get_own_jid(conversation.counterpart, conversation.account) ?? conversation.account.bare_jid; - file_transfer.direction = from.equals(file_transfer.ourpart) ? FileTransfer.DIRECTION_SENT : FileTransfer.DIRECTION_RECEIVED; - } else { - file_transfer.ourpart = conversation.account.full_jid; - file_transfer.direction = from.equals_bare(file_transfer.ourpart) ? FileTransfer.DIRECTION_SENT : FileTransfer.DIRECTION_RECEIVED; - } - file_transfer.time = new DateTime.now_utc(); - // TODO: get time from message - file_transfer.local_time = new DateTime.now_utc(); - file_transfer.provider = SFS_PROVIDER_ID; - file_transfer.with_metadata_element(sfs_element.metadata); - foreach (Xep.StatelessFileSharing.SfsSource source in sfs_element.sources) { - file_transfer.sfs_sources.append(new FileTransfer.SerializedSfsSource.from_sfs_source(source) as Object); - } - // FileTransfer.info stores the id of the MessageStanza for future SfsSourceAttachments - // Prior to sfs, info stored the id of the Message entity for oob - file_transfer.info = id; - - stream_interactor.get_module(FileTransferStorage.IDENTITY).add_file(file_transfer); - - if (is_sender_trustworthy(file_transfer, conversation)) { - if (file_transfer.size >= 0 && file_transfer.size < 500) { - FileProvider? file_provider = this.select_file_provider(file_transfer); - download_file_internal.begin(file_provider, file_transfer, conversation, (_, res) => { - download_file_internal.end(res); - }); - } - } - - conversation.last_active = file_transfer.time; - received_file(file_transfer, conversation); - } - - private void on_receive_sfs_attachment(Jid from, Conversation conversation, Xep.StatelessFileSharing.SfsSourceAttachment attachment) { - foreach (Qlite.Row file_transfer_row in this.db.file_transfer.select() - .with(db.file_transfer.info, "=", attachment.sfs_id)) { - FileTransfer file_transfer = new FileTransfer.from_row(this.db, file_transfer_row, FileManager.get_storage_dir()); - if (file_transfer.hashes.supported_hashes().is_empty) { - return; - } - foreach (StatelessFileSharing.SfsSource source in attachment.sources) { - file_transfer.sfs_sources.append(new FileTransfer.SerializedSfsSource.from_sfs_source(source) as Object); - } - } - - } - - private void on_account_added(Account account) { - Xep.StatelessFileSharing.Module fsf_module = stream_interactor.module_manager.get_module(account, Xep.StatelessFileSharing.Module.IDENTITY); - fsf_module.received_sfs.connect((from, to, sfs_element, message) => { - Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).approx_conversation_for_stanza(from, to, account, message.type_); - on_receive_sfs(from, conversation, sfs_element, message.id); - }); - fsf_module.received_sfs_attachment.connect((from, to, sfs_attachment, message) => { - Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).approx_conversation_for_stanza(from, to, account, message.type_); - on_receive_sfs_attachment(from, conversation, sfs_attachment); - }); - } - - public async HashMap get_file_size_limits(Conversation conversation) { + public async HashMap get_file_size_limits(Conversation conversation) { HashMap ret = new HashMap(); foreach (FileSender sender in file_senders) { ret[sender.get_id()] = yield sender.get_file_size_limit(conversation); @@ -180,7 +91,7 @@ public class FileManager : StreamInteractionModule, Object { yield file_metadata_provider.fill_metadata(file, metadata); } } - file_transfer.with_metadata_element(metadata); + file_transfer.file_metadata = metadata; try { file_transfer.input_stream = yield file.read_async(); @@ -237,7 +148,16 @@ public class FileManager : StreamInteractionModule, Object { file_send_data = file_encryptor.preprocess_send_file(conversation, file_transfer, file_send_data, file_meta); } + file_transfer.state = FileTransfer.State.IN_PROGRESS; + + // Update current download progress in the FileTransfer + LimitInputStream? limit_stream = file_transfer.input_stream as LimitInputStream; + if (limit_stream != null) { + limit_stream.bind_property("retrieved-bytes", file_transfer, "transferred-bytes", BindingFlags.SYNC_CREATE); + } + yield file_sender.send_file(conversation, file_transfer, file_send_data, file_meta); + file_transfer.state = FileTransfer.State.COMPLETE; } catch (Error e) { warning("Send file error: %s", e.message); @@ -265,12 +185,7 @@ public class FileManager : StreamInteractionModule, Object { public void add_provider(FileProvider file_provider) { file_providers.add(file_provider); file_provider.file_incoming.connect((info, from, time, local_time, conversation, receive_data, file_meta) => { - if (receive_data is HttpFileReceiveData) { - printerr("Handling oob data as stateless file sharing"); - this.on_backwards_compatible_sfs.begin(file_provider, from, time, local_time, conversation, receive_data, file_meta); - } else { - handle_incoming_file.begin(file_provider, info, from, time, local_time, conversation, receive_data, file_meta); - } + handle_incoming_file.begin(file_provider, info, from, time, local_time, conversation, receive_data, file_meta); }); } @@ -296,10 +211,12 @@ public class FileManager : StreamInteractionModule, Object { file_metadata_providers.add(file_metadata_provider); } - private bool is_jid_trustworthy(Jid from, Conversation conversation) { + public bool is_sender_trustworthy(FileTransfer file_transfer, Conversation conversation) { + if (file_transfer.direction == FileTransfer.DIRECTION_SENT) return true; + Jid relevant_jid = conversation.counterpart; if (conversation.type_ == Conversation.Type.GROUPCHAT) { - relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(from, conversation.account); + relevant_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(file_transfer.from, conversation.account); } if (relevant_jid == null) return false; @@ -307,12 +224,6 @@ public class FileManager : StreamInteractionModule, Object { return in_roster; } - public bool is_sender_trustworthy(FileTransfer file_transfer, Conversation conversation) { - if (file_transfer.direction == FileTransfer.DIRECTION_SENT) return true; - - return is_jid_trustworthy(file_transfer.from, conversation); - } - private async FileMeta get_file_meta(FileProvider file_provider, FileTransfer file_transfer, Conversation conversation, FileReceiveData receive_data_) throws FileReceiveError { FileReceiveData receive_data = receive_data_; FileMeta file_meta = file_provider.get_file_meta(file_transfer); @@ -337,7 +248,11 @@ public class FileManager : StreamInteractionModule, Object { private async void download_file_internal(FileProvider file_provider, FileTransfer file_transfer, Conversation conversation) { try { // Get meta info - FileReceiveData receive_data = yield file_provider.get_file_receive_data(file_transfer); + FileReceiveData? receive_data = file_provider.get_file_receive_data(file_transfer); + if (receive_data == null) { + warning("Don't have download data (yet)"); + return; + } FileDecryptor? file_decryptor = null; foreach (FileDecryptor decryptor in file_decryptors) { if (decryptor.can_decrypt_file(conversation, file_transfer, receive_data)) { @@ -367,10 +282,17 @@ public class FileManager : StreamInteractionModule, Object { input_stream = yield file_decryptor.decrypt_file(input_stream, conversation, file_transfer, receive_data); } + // Update current download progress in the FileTransfer + LimitInputStream? limit_stream = input_stream as LimitInputStream; + if (limit_stream != null) { + limit_stream.bind_property("retrieved-bytes", file_transfer, "transferred-bytes", BindingFlags.SYNC_CREATE); + } + // Save file string filename = Random.next_int().to_string("%x") + "_" + file_transfer.file_name; File file = File.new_for_path(Path.build_filename(get_storage_dir(), filename)); + // libsoup doesn't properly support splicing OutputStream os = file.create(FileCreateFlags.REPLACE_DESTINATION); uint8[] buffer = new uint8[1024]; ssize_t read; @@ -381,20 +303,49 @@ public class FileManager : StreamInteractionModule, Object { } yield input_stream.close_async(Priority.LOW, file_transfer.cancellable); yield os.close_async(Priority.LOW, file_transfer.cancellable); + + // Verify the hash of the downloaded file, if it is known + var supported_hashes = Xep.CryptographicHashes.get_supported_hashes(file_transfer.hashes); + if (!supported_hashes.is_empty) { + var checksum_types = new ArrayList(); + var hashes = new HashMap(); + foreach (var hash in supported_hashes) { + var checksum_type = Xep.CryptographicHashes.hash_string_to_type(hash.algo); + checksum_types.add(checksum_type); + hashes[checksum_type] = hash.val; + } + + var computed_hashes = yield compute_file_hashes(file, checksum_types); + foreach (var checksum_type in hashes.keys) { + if (hashes[checksum_type] != computed_hashes[checksum_type]) { + warning("Hash of downloaded file does not equal advertised hash, discarding: %s. %s should be %s, was %s", + file_transfer.file_name, checksum_type.to_string(), hashes[checksum_type], computed_hashes[checksum_type]); + FileUtils.remove(file.get_path()); + file_transfer.state = FileTransfer.State.FAILED; + return; + } + } + } + file_transfer.path = file.get_basename(); - file_transfer.input_stream = yield file.read_async(); FileInfo file_info = file_transfer.get_file().query_info("*", FileQueryInfoFlags.NONE); file_transfer.mime_type = file_info.get_content_type(); file_transfer.state = FileTransfer.State.COMPLETE; + } catch (IOError.CANCELLED e) { + print("cancelled\n"); } catch (Error e) { warning("Error downloading file: %s", e.message); - file_transfer.state = FileTransfer.State.FAILED; + if (file_transfer.provider == 0 || file_transfer.provider == FileManager.SFS_PROVIDER_ID) { + file_transfer.state = FileTransfer.State.NOT_STARTED; + } else { + file_transfer.state = FileTransfer.State.FAILED; + } } } - private async void handle_incoming_file(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) { + public FileTransfer create_file_transfer_from_provider_incoming(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) { FileTransfer file_transfer = new FileTransfer(); file_transfer.account = conversation.account; file_transfer.counterpart = file_transfer.direction == FileTransfer.DIRECTION_RECEIVED ? from : conversation.counterpart; @@ -426,6 +377,11 @@ public class FileManager : StreamInteractionModule, Object { } } + return file_transfer; + } + + private async void handle_incoming_file(FileProvider file_provider, string info, Jid from, DateTime time, DateTime local_time, Conversation conversation, FileReceiveData receive_data, FileMeta file_meta) { + FileTransfer file_transfer = create_file_transfer_from_provider_incoming(file_provider, info, from, time, local_time, conversation, receive_data, file_meta); stream_interactor.get_module(FileTransferStorage.IDENTITY).add_file(file_transfer); if (is_sender_trustworthy(file_transfer, conversation)) { @@ -451,10 +407,10 @@ public class FileManager : StreamInteractionModule, Object { string filename = Random.next_int().to_string("%x") + "_" + file_transfer.file_name; File file = File.new_for_path(Path.build_filename(get_storage_dir(), filename)); OutputStream os = file.create(FileCreateFlags.REPLACE_DESTINATION); - yield os.splice_async(file_transfer.input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE|OutputStreamSpliceFlags.CLOSE_TARGET); + yield os.splice_async(file_transfer.input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE | OutputStreamSpliceFlags.CLOSE_TARGET); file_transfer.state = FileTransfer.State.COMPLETE; file_transfer.path = filename; - file_transfer.input_stream = yield file.read_async(); + file_transfer.input_stream = new LimitInputStream(yield file.read_async(), file_transfer.size); } catch (Error e) { throw new FileSendError.SAVE_FAILED("Saving file error: %s".printf(e.message)); } @@ -467,10 +423,10 @@ public errordomain FileSendError { SAVE_FAILED } +// Get rid of this Error and pass IoErrors instead - DOWNLOAD_FAILED already removed public errordomain FileReceiveError { GET_METADATA_FAILED, - DECRYPTION_FAILED, - DOWNLOAD_FAILED + DECRYPTION_FAILED } public class FileMeta { @@ -505,10 +461,10 @@ public interface FileProvider : Object { public abstract Encryption get_encryption(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta); public abstract FileMeta get_file_meta(FileTransfer file_transfer) throws FileReceiveError; - public abstract async FileReceiveData? get_file_receive_data(FileTransfer file_transfer); + public abstract FileReceiveData? get_file_receive_data(FileTransfer file_transfer); public abstract async FileMeta get_meta_info(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws FileReceiveError; - public abstract async InputStream download(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws FileReceiveError; + public abstract async InputStream download(FileTransfer file_transfer, FileReceiveData receive_data, FileMeta file_meta) throws IOError; public abstract int get_id(); } @@ -541,107 +497,4 @@ public interface FileDecryptor : Object { public abstract async InputStream decrypt_file(InputStream encrypted_stream, Conversation conversation, FileTransfer file_transfer, FileReceiveData receive_data) throws FileReceiveError; } -public interface FileMetadataProvider : Object { - public abstract bool supports_file(File file); - - public abstract async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata); -} - -class GenericFileMetadataProvider: Dino.FileMetadataProvider, Object { - public bool supports_file(File file) { - return true; - } - - public async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata) { - FileInfo info = file.query_info("*", FileQueryInfoFlags.NONE); - - metadata.name = info.get_display_name(); - metadata.mime_type = info.get_content_type(); - metadata.size = info.get_size(); - metadata.date = info.get_modification_date_time(); - - Bytes file_data = file.load_bytes(); - metadata.hashes.hashes.add(new CryptographicHashes.Hash.from_data(GLib.ChecksumType.SHA256, file_data.get_data())); - metadata.hashes.hashes.add(new CryptographicHashes.Hash.from_data(GLib.ChecksumType.SHA512, file_data.get_data())); - } -} - -public class ImageFileMetadataProvider: Dino.FileMetadataProvider, Object { - public bool supports_file(File file) { - return file.query_info("*", FileQueryInfoFlags.NONE).get_content_type().has_prefix("image"); - } - - private const int[] THUMBNAIL_DIMS = { 1, 2, 3, 4, 8 }; - private const string IMAGE_TYPE = "png"; - private const string MIME_TYPE = "image/png"; - - public async void fill_metadata(File file, Xep.FileMetadataElement.FileMetadata metadata) { - Pixbuf pixbuf = new Pixbuf.from_stream(yield file.read_async()); - metadata.width = pixbuf.get_width(); - metadata.height = pixbuf.get_height(); - float ratio = (float)metadata.width / (float) metadata.height; - - int thumbnail_width = -1; - int thumbnail_height = -1; - float diff = float.INFINITY; - for (int i = 0; i < THUMBNAIL_DIMS.length; i++) { - int test_width = THUMBNAIL_DIMS[i]; - int test_height = THUMBNAIL_DIMS[THUMBNAIL_DIMS.length - 1 - i]; - float test_ratio = (float)test_width / (float)test_height; - float test_diff = (test_ratio - ratio).abs(); - if (test_diff < diff) { - thumbnail_width = test_width; - thumbnail_height = test_height; - diff = test_diff; - } - } - - Pixbuf thumbnail_pixbuf = pixbuf.scale_simple(thumbnail_width, thumbnail_height, InterpType.BILINEAR); - uint8[] buffer; - thumbnail_pixbuf.save_to_buffer(out buffer, IMAGE_TYPE); - string base_64 = GLib.Base64.encode(buffer); - string uri = @"data:$MIME_TYPE;base64,$base_64"; - Xep.JingleContentThumbnails.Thumbnail thumbnail = new Xep.JingleContentThumbnails.Thumbnail(); - thumbnail.uri = uri; - thumbnail.media_type = MIME_TYPE; - thumbnail.width = thumbnail_width; - thumbnail.height = thumbnail_height; - metadata.thumbnails.add(thumbnail); - } - - public static Pixbuf? parse_thumbnail(Xep.JingleContentThumbnails.Thumbnail thumbnail) { - string[] splits = thumbnail.uri.split(":", 2); - if (splits.length != 2) { - printerr("Thumbnail parsing error: ':' not found"); - return null; - } - if (splits[0] != "data") { - printerr("Unsupported thumbnail: unimplemented uri type\n"); - return null; - } - splits = splits[1].split(";", 2); - if (splits.length != 2) { - printerr("Thumbnail parsing error: ';' not found"); - return null; - } - if (splits[0] != MIME_TYPE) { - printerr("Unsupported thumbnail: unsupported mime-type\n"); - return null; - } - splits = splits[1].split(",", 2); - if (splits.length != 2) { - printerr("Thumbnail parsing error: ',' not found"); - return null; - } - if (splits[0] != "base64") { - printerr("Unsupported thumbnail: data is not base64 encoded\n"); - return null; - } - uint8[] data = Base64.decode(splits[1]); - MemoryInputStream input_stream = new MemoryInputStream.from_data(data); - Pixbuf pixbuf = new Pixbuf.from_stream(input_stream); - return pixbuf; - } -} - } -- cgit v1.2.3-70-g09d2