diff options
62 files changed, 3628 insertions, 1254 deletions
diff --git a/crypto-vala/src/srtp.vala b/crypto-vala/src/srtp.vala index c7f45da3..22210e37 100644 --- a/crypto-vala/src/srtp.vala +++ b/crypto-vala/src/srtp.vala @@ -34,9 +34,8 @@ public class Session { if (res != ErrorStatus.ok) { throw new Error.UNKNOWN(@"SRTP encrypt failed: $res"); } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; + buf.length = buf_use; + return buf; } public uint8[] decrypt_rtp(uint8[] data) throws Error { @@ -65,9 +64,8 @@ public class Session { if (res != ErrorStatus.ok) { throw new Error.UNKNOWN(@"SRTCP encrypt failed: $res"); } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; + buf.length = buf_use; + return buf; } public uint8[] decrypt_rtcp(uint8[] data) throws Error { @@ -81,7 +79,7 @@ public class Session { case ErrorStatus.ok: break; default: - throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + throw new Error.UNKNOWN(@"SRTCP decrypt failed: $res"); } uint8[] ret = new uint8[buf_use]; GLib.Memory.copy(ret, buf, buf_use); @@ -105,6 +103,7 @@ public class Session { policy.key = new uint8[key.length + salt.length]; Memory.copy(policy.key, key, key.length); Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + policy.next = null; encrypt_context.add_stream(ref policy); has_encrypt = true; } @@ -115,6 +114,7 @@ public class Session { policy.key = new uint8[key.length + salt.length]; Memory.copy(policy.key, key, key.length); Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + policy.next = null; decrypt_context.add_stream(ref policy); has_decrypt = true; } diff --git a/crypto-vala/vapi/libsrtp2.vapi b/crypto-vala/vapi/libsrtp2.vapi index c0c50c1c..f8f2d297 100644 --- a/crypto-vala/vapi/libsrtp2.vapi +++ b/crypto-vala/vapi/libsrtp2.vapi @@ -42,9 +42,11 @@ public struct Policy { public uint8[] key; public ulong num_master_keys; public ulong window_size; - public int allow_repeat_tx; + [CCode (ctype = "int")] + public bool allow_repeat_tx; [CCode (array_length_cname = "enc_xtn_hdr_count")] public int[] enc_xtn_hdr; + public Policy* next; } [CCode (cname = "srtp_crypto_policy_t")] diff --git a/libdino/CMakeLists.txt b/libdino/CMakeLists.txt index d7f7583c..ce836f62 100644 --- a/libdino/CMakeLists.txt +++ b/libdino/CMakeLists.txt @@ -29,6 +29,8 @@ SOURCES src/service/avatar_manager.vala src/service/blocking_manager.vala src/service/call_store.vala + src/service/call_state.vala + src/service/call_peer_state.vala src/service/calls.vala src/service/chat_interaction.vala src/service/connection_manager.vala diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index 577b3ab8..8e5bc246 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -11,7 +11,7 @@ namespace Dino.Entities { RINGING, ESTABLISHING, IN_PROGRESS, - OTHER_DEVICE_ACCEPTED, + OTHER_DEVICE, ENDED, DECLINED, MISSED, @@ -21,13 +21,11 @@ namespace Dino.Entities { public int id { get; set; default=-1; } public Account account { get; set; } public Jid counterpart { get; set; } + public Gee.List<Jid> counterparts = new Gee.ArrayList<Jid>(Jid.equals_bare_func); public Jid ourpart { get; set; } - public Jid? from { + public Jid proposer { get { return direction == DIRECTION_OUTGOING ? ourpart : counterpart; } } - public Jid? to { - get { return direction == DIRECTION_OUTGOING ? counterpart : ourpart; } - } public bool direction { get; set; } public DateTime time { get; set; } public DateTime local_time { get; set; } @@ -44,10 +42,6 @@ namespace Dino.Entities { id = row[db.call.id]; account = db.get_account_by_id(row[db.call.account_id]); - counterpart = db.get_jid_by_id(row[db.call.counterpart_id]); - string counterpart_resource = row[db.call.counterpart_resource]; - if (counterpart_resource != null) counterpart = counterpart.with_resource(counterpart_resource); - string our_resource = row[db.call.our_resource]; if (our_resource != null) { ourpart = account.bare_jid.with_resource(our_resource); @@ -61,6 +55,21 @@ namespace Dino.Entities { encryption = (Encryption) row[db.call.encryption]; state = (State) row[db.call.state]; + Qlite.QueryBuilder counterparts_select = db.call_counterpart.select().with(db.call_counterpart.call_id, "=", id); + foreach (Qlite.Row counterparts_row in counterparts_select) { + Jid peer = db.get_jid_by_id(counterparts_row[db.call_counterpart.jid_id]); + if (!counterparts.contains(peer)) { // Legacy: The first peer is also in the `call` table. Don't add twice. + counterparts.add(peer); + } + } + + counterpart = db.get_jid_by_id(row[db.call.counterpart_id]); + string counterpart_resource = row[db.call.counterpart_resource]; + if (counterpart_resource != null) counterpart = counterpart.with_resource(counterpart_resource); + if (counterparts.is_empty) { + counterparts.add(counterpart); + } + notify.connect(on_update); } @@ -70,8 +79,6 @@ namespace Dino.Entities { this.db = db; Qlite.InsertBuilder builder = db.call.insert() .value(db.call.account_id, account.id) - .value(db.call.counterpart_id, db.get_jid_id(counterpart)) - .value(db.call.counterpart_resource, counterpart.resourcepart) .value(db.call.our_resource, ourpart.resourcepart) .value(db.call.direction, direction) .value(db.call.time, (long) time.to_unix()) @@ -83,11 +90,36 @@ namespace Dino.Entities { } else { builder.value(db.call.end_time, (long) local_time.to_unix()); } + if (counterpart != null) { + builder.value(db.call.counterpart_id, db.get_jid_id(counterpart)) + .value(db.call.counterpart_resource, counterpart.resourcepart); + } id = (int) builder.perform(); + foreach (Jid peer in counterparts) { + db.call_counterpart.insert() + .value(db.call_counterpart.call_id, id) + .value(db.call_counterpart.jid_id, db.get_jid_id(peer)) + .value(db.call_counterpart.resource, peer.resourcepart) + .perform(); + } + notify.connect(on_update); } + public void add_peer(Jid peer) { + if (counterparts.contains(peer)) return; + + counterparts.add(peer); + if (db != null) { + db.call_counterpart.insert() + .value(db.call_counterpart.call_id, id) + .value(db.call_counterpart.jid_id, db.get_jid_id(peer)) + .value(db.call_counterpart.resource, peer.resourcepart) + .perform(); + } + } + public bool equals(Call c) { return equals_func(this, c); } diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala index eadbb085..fb80fef6 100644 --- a/libdino/src/plugin/interfaces.vala +++ b/libdino/src/plugin/interfaces.vala @@ -106,11 +106,13 @@ public abstract interface VideoCallPlugin : Object { public abstract MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream stream, bool incoming); public abstract void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause); public abstract void set_device(Xmpp.Xep.JingleRtp.Stream stream, MediaDevice? device); + + public abstract void dump_dot(); } public abstract interface VideoCallWidget : Object { public signal void resolution_changed(uint width, uint height); - public abstract void display_stream(Xmpp.Xep.JingleRtp.Stream stream); // TODO: Multi participant + public abstract void display_stream(Xmpp.Xep.JingleRtp.Stream stream, Jid jid); public abstract void display_device(MediaDevice device); public abstract void detach(); } diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala new file mode 100644 index 00000000..09440371 --- /dev/null +++ b/libdino/src/service/call_peer_state.vala @@ -0,0 +1,457 @@ +using Dino.Entities; +using Gee; +using Xmpp; + +public class Dino.PeerState : Object { + public signal void counterpart_sends_video_updated(bool mute); + public signal void info_received(Xep.JingleRtp.CallSessionInfo session_info); + + public signal void connection_ready(); + public signal void session_terminated(bool we_terminated, string? reason_name, string? reason_text); + public signal void encryption_updated(Xep.Jingle.ContentEncryption? audio_encryption, Xep.Jingle.ContentEncryption? video_encryption, bool same); + + public StreamInteractor stream_interactor; + public Calls calls; + public Call call; + public Jid jid; + public Xep.Jingle.Session session; + public string sid; + public string internal_id = Xmpp.random_uuid(); + + public Xep.JingleRtp.Parameters? audio_content_parameter = null; + public Xep.JingleRtp.Parameters? video_content_parameter = null; + public Xep.Jingle.Content? audio_content = null; + public Xep.Jingle.Content? video_content = null; + public Xep.Jingle.ContentEncryption? video_encryption = null; + public Xep.Jingle.ContentEncryption? audio_encryption = null; + public bool encryption_keys_same = false; + public HashMap<string, Xep.Jingle.ContentEncryption>? video_encryptions = null; + public HashMap<string, Xep.Jingle.ContentEncryption>? audio_encryptions = null; + + public bool first_peer = false; + public bool accepted_jmi = false; + public bool waiting_for_inbound_muji_connection = false; + public Xep.Muji.GroupCall? group_call { get; set; } + + public bool counterpart_sends_video = false; + public bool we_should_send_audio { get; set; default=false; } + public bool we_should_send_video { get; set; default=false; } + + public PeerState(Jid jid, Call call, StreamInteractor stream_interactor) { + this.jid = jid; + this.call = call; + this.stream_interactor = stream_interactor; + this.calls = stream_interactor.get_module(Calls.IDENTITY); + + var session_info_type = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type; + session_info_type.mute_update_received.connect((session,mute, name) => { + if (this.sid != session.sid) return; + + foreach (Xep.Jingle.Content content in session.contents) { + if (name == null || content.content_name == name) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + on_counterpart_mute_update(mute, rtp_content_parameter.media); + } + } + } + }); + session_info_type.info_received.connect((session, session_info) => { + if (this.sid != session.sid) return; + + info_received(session_info); + }); + } + + public async void initiate_call(Jid counterpart) { + Gee.List<Jid> call_resources = yield calls.get_call_resources(call.account, counterpart); + + bool do_jmi = false; + Jid? jid_for_direct = null; + if (yield calls.contains_jmi_resources(call.account, call_resources)) { + do_jmi = true; + } else if (!call_resources.is_empty) { + jid_for_direct = call_resources[0]; + } else if (calls.has_jmi_resources(jid)) { + do_jmi = true; + } + + sid = Xmpp.random_uuid(); + + if (do_jmi) { + XmppStream? stream = stream_interactor.get_stream(call.account); + + calls.current_jmi_request_call[call.account] = calls.call_states[call]; + calls.current_jmi_request_peer[call.account] = this; + + var descriptions = new ArrayList<StanzaNode>(); + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "audio")); + if (we_should_send_video) { + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "video")); + } + + stream.get_module(Xmpp.Xep.JingleMessageInitiation.Module.IDENTITY).send_session_propose_to_peer(stream, jid, sid, descriptions); + } else if (jid_for_direct != null) { + yield call_resource(jid_for_direct); + } + } + + public async void call_resource(Jid full_jid) { + XmppStream? stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + if (sid == null) sid = Xmpp.random_uuid(); + + Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, we_should_send_video, sid, group_call != null ? group_call.muc_jid : null); + set_session(session); + } + + public void accept() { + if (session != null) { + foreach (Xep.Jingle.Content content in session.contents) { + content.accept(); + } + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + accepted_jmi = true; + + calls.current_jmi_request_call[call.account] = calls.call_states[call]; + calls.current_jmi_request_peer[call.account] = this; + + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, jid, sid); + } + } + + public void reject() { + if (session != null) { + foreach (Xep.Jingle.Content content in session.contents) { + content.reject(); + } + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_peer(stream, jid, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); + } + } + + public void end(string terminate_reason, string? reason_text = null) { + switch (terminate_reason) { + case Xep.Jingle.ReasonElement.SUCCESS: + if (session != null) { + session.terminate(terminate_reason, reason_text, "success"); + } + break; + case Xep.Jingle.ReasonElement.CANCEL: + if (session != null) { + session.terminate(terminate_reason, reason_text, "cancel"); + } else if (group_call != null) { + // We don't have to do anything (?) + } else { + // Only a JMI so far + XmppStream? stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_retract_to_peer(stream, jid, sid); + } + break; + } + } + + internal void mute_own_audio(bool mute) { + // Call isn't fully established yet. Audio will be muted once the stream is created. + if (session == null || audio_content_parameter == null || audio_content_parameter.stream == null) return; + + Xep.JingleRtp.Stream stream = audio_content_parameter.stream; + + // Inform our counterpart that we (un)muted our audio + stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_mute(session, mute, "audio"); + + // Start/Stop sending audio data + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + internal void mute_own_video(bool mute) { + + if (session == null) { + // Call hasn't been established yet + return; + } + + Xep.JingleRtp.Module rtp_module = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY); + + if (video_content_parameter != null && + video_content_parameter.stream != null && + session.senders_include_us(video_content.senders)) { + // A video content already exists + + // Start/Stop sending video data + Xep.JingleRtp.Stream stream = video_content_parameter.stream; + if (stream != null) { + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + // Inform our counterpart that we started/stopped our video + rtp_module.session_info_type.send_mute(session, mute, "video"); + } else if (!mute) { + // Add a new video content + XmppStream stream = stream_interactor.get_stream(call.account); + rtp_module.add_outgoing_video_content.begin(stream, session, group_call != null ? group_call.muc_jid : null, (_, res) => { + if (video_content_parameter == null) { + Xep.Jingle.Content content = rtp_module.add_outgoing_video_content.end(res); + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + connect_content_signals(content, rtp_content_parameter); + } + } + }); + } + // If video_content_parameter == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. + } + + public Xep.JingleRtp.Stream? get_video_stream(Call call) { + if (video_content_parameter != null) { + return video_content_parameter.stream; + } + return null; + } + + public Xep.JingleRtp.Stream? get_audio_stream(Call call) { + if (audio_content_parameter != null) { + return audio_content_parameter.stream; + } + return null; + } + + internal void set_session(Xep.Jingle.Session session) { + this.session = session; + this.sid = session.sid; + + session.terminated.connect((stream, we_terminated, reason_name, reason_text) => + session_terminated(we_terminated, reason_name, reason_text) + ); + session.additional_content_add_incoming.connect((session,stream, content) => + on_incoming_content_add(stream, session, content) + ); + + foreach (Xep.Jingle.Content content in session.contents) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter == null) continue; + + connect_content_signals(content, rtp_content_parameter); + } + } + + public PeerInfo get_info() { + var ret = new PeerInfo(); + if (audio_content != null || audio_content_parameter != null) { + ret.audio = get_content_info(audio_content, audio_content_parameter); + } + if (video_content != null || video_content_parameter != null) { + ret.video = get_content_info(video_content, video_content_parameter); + } + return ret; + } + + private PeerContentInfo get_content_info(Xep.Jingle.Content? content, Xep.JingleRtp.Parameters? parameter) { + PeerContentInfo ret = new PeerContentInfo(); + if (parameter != null) { + ret.rtcp_ready = parameter.rtcp_ready; + ret.rtp_ready = parameter.rtp_ready; + + if (parameter.agreed_payload_type != null) { + ret.codec = parameter.agreed_payload_type.name; + ret.clockrate = parameter.agreed_payload_type.clockrate; + } + if (parameter.stream != null && parameter.stream.remb_enabled) { + ret.target_receive_bytes = parameter.stream.target_receive_bitrate; + ret.target_send_bytes = parameter.stream.target_send_bitrate; + } + } + + if (content != null) { + Xmpp.Xep.Jingle.ComponentConnection? component0 = content.get_transport_connection(1); + if (component0 != null) { + ret.bytes_received = component0.bytes_received; + ret.bytes_sent = component0.bytes_sent; + } + } + return ret; + } + + + + private void connect_content_signals(Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { + if (rtp_content_parameter.media == "audio") { + audio_content = content; + audio_content_parameter = rtp_content_parameter; + } else if (rtp_content_parameter.media == "video") { + video_content = content; + video_content_parameter = rtp_content_parameter; + } + + debug(@"[%s] %s connecting content signals %s", call.account.bare_jid.to_string(), jid.to_string(), rtp_content_parameter.media); + rtp_content_parameter.stream_created.connect((stream) => on_stream_created(rtp_content_parameter.media, stream)); + rtp_content_parameter.connection_ready.connect((status) => { + Idle.add(() => { + on_connection_ready(content, rtp_content_parameter.media); + return false; + }); + }); + + content.senders_modify_incoming.connect((content, proposed_senders) => { + if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { + warning("counterpart set us to (not)sending %s. ignoring", content.content_name); + return; + } + + if (!content.session.senders_include_counterpart(content.senders) && content.session.senders_include_counterpart(proposed_senders)) { + // Counterpart wants to start sending. Ok. + content.accept_content_modify(proposed_senders); + on_counterpart_mute_update(false, "video"); + } + }); + } + + private void on_incoming_content_add(XmppStream stream, Xep.Jingle.Session session, Xep.Jingle.Content content) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + + if (rtp_content_parameter == null) { + content.reject(); + return; + } + + // Our peer shouldn't tell us to start sending, that's for us to initiate + if (session.senders_include_us(content.senders)) { + if (session.senders_include_counterpart(content.senders)) { + // If our peer wants to send, let them + content.modify(session.we_initiated ? Xep.Jingle.Senders.RESPONDER : Xep.Jingle.Senders.INITIATOR); + } else { + // If only we're supposed to send, reject + content.reject(); + } + } + + connect_content_signals(content, rtp_content_parameter); + content.accept(); + } + + private void on_stream_created(string media, Xep.JingleRtp.Stream stream) { + if (media == "video" && stream.receiving) { + counterpart_sends_video = true; + video_content_parameter.connection_ready.connect((status) => { + Idle.add(() => { + counterpart_sends_video_updated(false); + return false; + }); + }); + } + + // Outgoing audio/video might have been muted in the meanwhile. + if (media == "video" && !we_should_send_video) { + mute_own_video(true); + } else if (media == "audio" && !we_should_send_audio) { + mute_own_audio(true); + } + } + + private void on_counterpart_mute_update(bool mute, string? media) { + if (!call.equals(call)) return; + + if (media == "video") { + counterpart_sends_video = !mute; + debug(@"[%s] %s video muted %s", call.account.bare_jid.to_string(), jid.to_string(), mute.to_string()); + counterpart_sends_video_updated(mute); + } + } + + private void on_connection_ready(Xep.Jingle.Content content, string media) { + debug("[%s] %s on_connection_ready", call.account.bare_jid.to_string(), jid.to_string()); + connection_ready(); + + if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + call.state = Call.State.IN_PROGRESS; + } + + if (media == "audio") { + audio_encryptions = content.encryptions; + } else if (media == "video") { + video_encryptions = content.encryptions; + } + + if ((audio_encryptions != null && audio_encryptions.is_empty) || (video_encryptions != null && video_encryptions.is_empty)) { + call.encryption = Encryption.NONE; + encryption_updated(null, null, true); + return; + } + + HashMap<string, Xep.Jingle.ContentEncryption> encryptions = audio_encryptions ?? video_encryptions; + + Xep.Jingle.ContentEncryption? omemo_encryption = null, dtls_encryption = null, srtp_encryption = null; + foreach (string encr_name in encryptions.keys) { + if (video_encryptions != null && !video_encryptions.has_key(encr_name)) continue; + + var encryption = encryptions[encr_name]; + if (encryption.encryption_ns == "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification") { + omemo_encryption = encryption; + } else if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { + dtls_encryption = encryption; + } else if (encryption.encryption_name == "SRTP") { + srtp_encryption = encryption; + } + } + + if (omemo_encryption != null && dtls_encryption != null) { + call.encryption = Encryption.OMEMO; + omemo_encryption.peer_key = dtls_encryption.peer_key; + omemo_encryption.our_key = dtls_encryption.our_key; + audio_encryption = omemo_encryption; + encryption_keys_same = true; + video_encryption = video_encryptions != null ? video_encryptions["http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"] : null; + } else if (dtls_encryption != null) { + call.encryption = Encryption.DTLS_SRTP; + audio_encryption = dtls_encryption; + video_encryption = video_encryptions != null ? video_encryptions[Xep.JingleIceUdp.DTLS_NS_URI] : null; + encryption_keys_same = true; + if (video_encryption != null && dtls_encryption.peer_key.length == video_encryption.peer_key.length) { + for (int i = 0; i < dtls_encryption.peer_key.length; i++) { + if (dtls_encryption.peer_key[i] != video_encryption.peer_key[i]) { + encryption_keys_same = false; + break; + } + } + } + } else if (srtp_encryption != null) { + call.encryption = Encryption.SRTP; + audio_encryption = srtp_encryption; + video_encryption = video_encryptions != null ? video_encryptions["SRTP"] : null; + encryption_keys_same = false; + } else { + call.encryption = Encryption.NONE; + encryption_keys_same = true; + } + + encryption_updated(audio_encryption, video_encryption, encryption_keys_same); + } +} + +public class Dino.PeerContentInfo { + public bool rtp_ready { get; set; } + public bool rtcp_ready { get; set; } + public ulong? bytes_sent { get; set; default=0; } + public ulong? bytes_received { get; set; default=0; } + public string? codec { get; set; } + public uint32 clockrate { get; set; } + public uint target_receive_bytes { get; set; default=-1; } + public uint target_send_bytes { get; set; default=-1; } +} + +public class Dino.PeerInfo { + public PeerContentInfo? audio = null; + public PeerContentInfo? video = null; +}
\ No newline at end of file diff --git a/libdino/src/service/call_state.vala b/libdino/src/service/call_state.vala new file mode 100644 index 00000000..188a8321 --- /dev/null +++ b/libdino/src/service/call_state.vala @@ -0,0 +1,342 @@ +using Dino.Entities; +using Gee; +using Xmpp; + +public class Dino.CallState : Object { + + public signal void terminated(Jid who_terminated, string? reason_name, string? reason_text); + public signal void peer_joined(Jid jid, PeerState peer_state); + public signal void peer_left(Jid jid, PeerState peer_state, string? reason_name, string? reason_text); + + public StreamInteractor stream_interactor; + public Call call; + public Xep.Muji.GroupCall? group_call { get; set; } + public Jid? parent_muc { get; set; } + public Jid? invited_to_group_call = null; + public Jid? group_call_inviter = null; + public bool accepted { get; private set; default=false; } + + public bool we_should_send_audio { get; set; default=false; } + public bool we_should_send_video { get; set; default=false; } + public HashMap<Jid, PeerState> peers = new HashMap<Jid, PeerState>(Jid.hash_func, Jid.equals_func); + + private string message_type = Xmpp.MessageStanza.TYPE_CHAT; + + public CallState(Call call, StreamInteractor stream_interactor) { + this.call = call; + this.stream_interactor = stream_interactor; + + if (call.direction == Call.DIRECTION_OUTGOING) { + accepted = true; + + Timeout.add_seconds(30, () => { + if (this == null) return false; // TODO enough? + if (call.state == Call.State.ESTABLISHING) { + call.state = Call.State.MISSED; + terminated(call.account.bare_jid, null, null); + } + return false; + }); + } + } + + internal async void initiate_groupchat_call(Jid muc) { + parent_muc = muc; + message_type = Xmpp.MessageStanza.TYPE_GROUPCHAT; + + if (this.group_call == null) yield convert_into_group_call(); + if (this.group_call == null) return; + // The user might have retracted the call in the meanwhile + if (this.call.state != Call.State.RINGING) return; + + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + Gee.List<Jid> occupants = stream_interactor.get_module(MucManager.IDENTITY).get_other_occupants(muc, call.account); + foreach (Jid occupant in occupants) { + Jid? real_jid = stream_interactor.get_module(MucManager.IDENTITY).get_real_jid(occupant, call.account); + if (real_jid == null) continue; + debug(@"Adding MUC member as MUJI MUC owner %s", real_jid.bare_jid.to_string()); + yield stream.get_module(Xep.Muc.Module.IDENTITY).change_affiliation(stream, group_call.muc_jid, real_jid.bare_jid, null, "owner"); + } + + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite(stream, muc, group_call.muc_jid, we_should_send_video, message_type); + } + + internal PeerState set_first_peer(Jid peer) { + var peer_state = new PeerState(peer, call, stream_interactor); + peer_state.first_peer = true; + add_peer(peer_state); + return peer_state; + } + + internal void add_peer(PeerState peer) { + call.add_peer(peer.jid.bare_jid); + connect_peer_signals(peer); + peer_joined(peer.jid, peer); + } + + public void accept() { + accepted = true; + call.state = Call.State.ESTABLISHING; + + if (invited_to_group_call != null) { + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_accept_to_peer(stream, group_call_inviter, invited_to_group_call, message_type); + join_group_call.begin(invited_to_group_call); + } else { + foreach (PeerState peer in peers.values) { + peer.accept(); + } + } + } + + public void reject() { + call.state = Call.State.DECLINED; + + if (invited_to_group_call != null) { + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_reject_to_self(stream, invited_to_group_call); + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_reject_to_peer(stream, group_call_inviter, invited_to_group_call, message_type); + } + var peers_cpy = new ArrayList<PeerState>(); + peers_cpy.add_all(peers.values); + foreach (PeerState peer in peers_cpy) { + peer.reject(); + } + terminated(call.account.bare_jid, null, null); + } + + public void end() { + var peers_cpy = new ArrayList<PeerState>(); + peers_cpy.add_all(peers.values); + + if (group_call != null) { + stream_interactor.get_module(MucManager.IDENTITY).part(call.account, group_call.muc_jid); + } + + if (call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { + foreach (PeerState peer in peers_cpy) { + peer.end(Xep.Jingle.ReasonElement.SUCCESS); + } + call.state = Call.State.ENDED; + } else if (call.state == Call.State.RINGING) { + foreach (PeerState peer in peers_cpy) { + peer.end(Xep.Jingle.ReasonElement.CANCEL); + } + if (parent_muc != null && group_call != null) { + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_retract_to_peer(stream, parent_muc, group_call.muc_jid, message_type); + } + call.state = Call.State.MISSED; + } else { + return; + } + + call.end_time = new DateTime.now_utc(); + + terminated(call.account.bare_jid, null, null); + } + + public void mute_own_audio(bool mute) { + we_should_send_audio = !mute; + foreach (PeerState peer in peers.values) { + peer.mute_own_audio(mute); + } + } + + public void mute_own_video(bool mute) { + we_should_send_video = !mute; + foreach (PeerState peer in peers.values) { + peer.mute_own_video(mute); + } + } + + public bool should_we_send_video() { + return we_should_send_video; + } + + public async void invite_to_call(Jid invitee) { + if (this.group_call == null) yield convert_into_group_call(); + if (this.group_call == null) return; + + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + debug("[%s] Inviting to muji call %s", call.account.bare_jid.to_string(), invitee.to_string()); + yield stream.get_module(Xep.Muc.Module.IDENTITY).change_affiliation(stream, group_call.muc_jid, invitee, null, "owner"); + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite(stream, invitee, group_call.muc_jid, we_should_send_video); + + // If the peer hasn't accepted within a minute, retract the invite + Timeout.add_seconds(60, () => { + if (this == null) return false; + + bool contains_peer = false; + foreach (Jid peer in peers.keys) { + if (peer.equals_bare(invitee)) { + contains_peer = true; + } + } + + if (!contains_peer) { + debug("[%s] Retracting invite to %s from %s", call.account.bare_jid.to_string(), group_call.muc_jid.to_string(), invitee.to_string()); + stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_retract_to_peer(stream, invitee, group_call.muc_jid); + stream.get_module(Xep.Muc.Module.IDENTITY).change_affiliation.begin(stream, group_call.muc_jid, invitee, null, "none"); + } + return false; + }); + } + + internal void rename_peer(Jid from_jid, Jid to_jid) { + debug("[%s] Renaming %s to %s exists %b", call.account.bare_jid.to_string(), from_jid.to_string(), to_jid.to_string(), peers.has_key(from_jid)); + PeerState? peer_state = peers[from_jid]; + if (peer_state == null) return; + + // Adjust the internal mapping of this `PeerState` object + peers.unset(from_jid); + peers[to_jid] = peer_state; + peer_state.jid = to_jid; + } + + private void on_call_terminated(Jid who_terminated, bool we_terminated, string? reason_name, string? reason_text) { + if (call.state == Call.State.RINGING || call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { + call.end_time = new DateTime.now_utc(); + } + if (call.state == Call.State.IN_PROGRESS) { + call.state = Call.State.ENDED; + } else if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + if (reason_name == Xep.Jingle.ReasonElement.DECLINE) { + call.state = Call.State.DECLINED; + } else { + call.state = Call.State.FAILED; + } + } + + terminated(who_terminated, reason_name, reason_text); + } + + private void connect_peer_signals(PeerState peer_state) { + peers[peer_state.jid] = peer_state; + + this.bind_property("we-should-send-audio", peer_state, "we-should-send-audio", BindingFlags.SYNC_CREATE | BindingFlags.BIDIRECTIONAL); + this.bind_property("we-should-send-video", peer_state, "we-should-send-video", BindingFlags.SYNC_CREATE | BindingFlags.BIDIRECTIONAL); + this.bind_property("group-call", peer_state, "group-call", BindingFlags.SYNC_CREATE | BindingFlags.BIDIRECTIONAL); + + peer_state.session_terminated.connect((we_terminated, reason_name, reason_text) => { + peers.unset(peer_state.jid); + debug("[%s] Peer left %s left %i", call.account.bare_jid.to_string(), peer_state.jid.to_string(), peers.size); + + if (peers.is_empty) { + if (group_call != null) group_call.leave(stream_interactor.get_stream(call.account)); + on_call_terminated(peer_state.jid, we_terminated, reason_name, reason_text); + } else { + peer_left(peer_state.jid, peer_state, reason_name, reason_text); + } + }); + } + + public async void convert_into_group_call() { + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + Jid? muc_jid = null; + if (muc_jid == null) { + warning("Failed to initiate group call: MUC server not known."); + return; + } + + muc_jid = new Jid("%08x@".printf(Random.next_int()) + muc_jid.to_string()); // TODO longer? + + debug("[%s] Converting call to groupcall %s", call.account.bare_jid.to_string(), muc_jid.to_string()); + yield join_group_call(muc_jid); + + Xep.DataForms.DataForm? data_form = yield stream_interactor.get_module(MucManager.IDENTITY).get_config_form(call.account, muc_jid); + if (data_form == null) return; + + foreach (Xep.DataForms.DataForm.Field field in data_form.fields) { + switch (field.var) { + case "muc#roomconfig_allowinvites": + if (field.type_ == Xep.DataForms.DataForm.Type.BOOLEAN) { + ((Xep.DataForms.DataForm.BooleanField) field).value = true; + } + break; + case "muc#roomconfig_persistentroom": + if (field.type_ == Xep.DataForms.DataForm.Type.BOOLEAN) { + ((Xep.DataForms.DataForm.BooleanField) field).value = false; + } + break; + case "muc#roomconfig_membersonly": + if (field.type_ == Xep.DataForms.DataForm.Type.BOOLEAN) { + ((Xep.DataForms.DataForm.BooleanField) field).value = true; + } + break; + case "muc#roomconfig_whois": + if (field.type_ == Xep.DataForms.DataForm.Type.LIST_SINGLE) { + ((Xep.DataForms.DataForm.ListSingleField) field).value = "anyone"; + } + break; + } + } + yield stream_interactor.get_module(MucManager.IDENTITY).set_config_form(call.account, muc_jid, data_form); + + foreach (Jid peer_jid in peers.keys) { + debug("[%s] Group call inviting %s", call.account.bare_jid.to_string(), peer_jid.to_string()); + yield invite_to_call(peer_jid); + } + } + + public async void join_group_call(Jid muc_jid) { + debug("[%s] Joining group call %s", call.account.bare_jid.to_string(), muc_jid.to_string()); + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + this.group_call = yield stream.get_module(Xep.Muji.Module.IDENTITY).join_call(stream, muc_jid, we_should_send_video); + if (this.group_call == null) { + warning("[%s] Couldn't join MUJI MUC", call.account.bare_jid.to_string()); + return; + } + + this.group_call.peer_joined.connect((jid) => { + debug("[%s] Group call peer joined: %s", call.account.bare_jid.to_string(), jid.to_string()); + + // Newly joined peers have to call us, not the other way round + // Maybe they called us already. Accept the call. + // (Except for the first peer, we already have a connection to that one.) + if (peers.has_key(jid)) { + if (!peers[jid].first_peer) { + peers[jid].accept(); + } + // else: Connection to first peer already active + } else { + var peer_state = new PeerState(jid, call, stream_interactor); + peer_state.waiting_for_inbound_muji_connection = true; + debug("[%s] Waiting for call from %s", call.account.bare_jid.to_string(), jid.to_string()); + add_peer(peer_state); + } + }); + + this.group_call.peer_left.connect((jid) => { + debug("[%s] Group call peer left: %s", call.account.bare_jid.to_string(), jid.to_string()); + if (!peers.has_key(jid)) return; + // end() will in the end cause a `peer_left` signal and removal from `peers` + peers[jid].end(Xep.Jingle.ReasonElement.CANCEL, "Peer left the MUJI MUC"); + }); + + // Call all peers that are in the room already + foreach (Jid peer_jid in group_call.peers_to_connect_to) { + // Don't establish connection if we have one already (the person that invited us to the call) + if (peers.has_key(peer_jid)) continue; + + debug("[%s] Calling %s because they were in the MUC already", call.account.bare_jid.to_string(), peer_jid.to_string()); + + PeerState peer_state = new PeerState(peer_jid, call, stream_interactor); + add_peer(peer_state); + peer_state.call_resource.begin(peer_jid); + } + + debug("[%s] Finished joining MUJI muc %s", call.account.bare_jid.to_string(), muc_jid.to_string()); + } +}
\ No newline at end of file diff --git a/libdino/src/service/call_store.vala b/libdino/src/service/call_store.vala index fa6e63ee..bfc8255f 100644 --- a/libdino/src/service/call_store.vala +++ b/libdino/src/service/call_store.vala @@ -30,7 +30,7 @@ namespace Dino { cache_call(call); } - public Call? get_call_by_id(int id) { + public Call? get_call_by_id(int id, Conversation conversation) { Call? call = calls_by_db_id[id]; if (call != null) { return call; @@ -38,14 +38,17 @@ namespace Dino { RowOption row_option = db.call.select().with(db.call.id, "=", id).row(); - return create_call_from_row_opt(row_option); + return create_call_from_row_opt(row_option, conversation); } - private Call? create_call_from_row_opt(RowOption row_opt) { + private Call? create_call_from_row_opt(RowOption row_opt, Conversation conversation) { if (!row_opt.is_present()) return null; try { Call call = new Call.from_row(db, row_opt.inner); + if (conversation.type_.is_muc_semantic()) { + call.ourpart = conversation.counterpart.with_resource(call.ourpart.resourcepart); + } cache_call(call); return call; } catch (InvalidJidError e) { diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 6b69049e..76cd0c68 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -7,16 +7,11 @@ namespace Dino { public class Calls : StreamInteractionModule, Object { - public signal void call_incoming(Call call, Conversation conversation, bool video); - public signal void call_outgoing(Call call, Conversation conversation); + public signal void call_incoming(Call call, CallState state, Conversation conversation, bool video); + public signal void call_outgoing(Call call, CallState state, Conversation conversation); public signal void call_terminated(Call call, string? reason_name, string? reason_text); - public signal void counterpart_ringing(Call call); - public signal void counterpart_sends_video_updated(Call call, bool mute); - public signal void info_received(Call call, Xep.JingleRtp.CallSessionInfo session_info); - public signal void encryption_updated(Call call, Xep.Jingle.ContentEncryption? audio_encryption, Xep.Jingle.ContentEncryption? video_encryption, bool same); - - public signal void stream_created(Call call, string media); + public signal void conference_info_received(Call call, Xep.Coin.ConferenceInfo conference_info); public static ModuleIdentity<Calls> IDENTITY = new ModuleIdentity<Calls>("calls"); public string id { get { return IDENTITY.id; } } @@ -24,24 +19,9 @@ namespace Dino { private StreamInteractor stream_interactor; private Database db; - private HashMap<Account, HashMap<Call, string>> sid_by_call = new HashMap<Account, HashMap<Call, string>>(Account.hash_func, Account.equals_func); - private HashMap<Account, HashMap<string, Call>> call_by_sid = new HashMap<Account, HashMap<string, Call>>(Account.hash_func, Account.equals_func); - public HashMap<Call, Xep.Jingle.Session> sessions = new HashMap<Call, Xep.Jingle.Session>(Call.hash_func, Call.equals_func); - - public HashMap<Account, Call> jmi_call = new HashMap<Account, Call>(Account.hash_func, Account.equals_func); - public HashMap<Account, string> jmi_sid = new HashMap<Account, string>(Account.hash_func, Account.equals_func); - public HashMap<Account, bool> jmi_video = new HashMap<Account, bool>(Account.hash_func, Account.equals_func); - - private HashMap<Call, bool> counterpart_sends_video = new HashMap<Call, bool>(Call.hash_func, Call.equals_func); - private HashMap<Call, bool> we_should_send_video = new HashMap<Call, bool>(Call.hash_func, Call.equals_func); - private HashMap<Call, bool> we_should_send_audio = new HashMap<Call, bool>(Call.hash_func, Call.equals_func); - - private HashMap<Call, Xep.JingleRtp.Parameters> audio_content_parameter = new HashMap<Call, Xep.JingleRtp.Parameters>(Call.hash_func, Call.equals_func); - private HashMap<Call, Xep.JingleRtp.Parameters> video_content_parameter = new HashMap<Call, Xep.JingleRtp.Parameters>(Call.hash_func, Call.equals_func); - private HashMap<Call, Xep.Jingle.Content> audio_content = new HashMap<Call, Xep.Jingle.Content>(Call.hash_func, Call.equals_func); - private HashMap<Call, Xep.Jingle.Content> video_content = new HashMap<Call, Xep.Jingle.Content>(Call.hash_func, Call.equals_func); - private HashMap<Call, HashMap<string, Xep.Jingle.ContentEncryption>> video_encryptions = new HashMap<Call, HashMap<string, Xep.Jingle.ContentEncryption>>(Call.hash_func, Call.equals_func); - private HashMap<Call, HashMap<string, Xep.Jingle.ContentEncryption>> audio_encryptions = new HashMap<Call, HashMap<string, Xep.Jingle.ContentEncryption>>(Call.hash_func, Call.equals_func); + public HashMap<Account, CallState> current_jmi_request_call = new HashMap<Account, CallState>(Account.hash_func, Account.equals_func); + public HashMap<Account, PeerState> current_jmi_request_peer = new HashMap<Account, PeerState>(Account.hash_func, Account.equals_func); + public HashMap<Call, CallState> call_states = new HashMap<Call, CallState>(Call.hash_func, Call.equals_func); public static void start(StreamInteractor stream_interactor, Database db) { Calls m = new Calls(stream_interactor, db); @@ -55,212 +35,47 @@ namespace Dino { stream_interactor.account_added.connect(on_account_added); } - public Xep.JingleRtp.Stream? get_video_stream(Call call) { - if (video_content_parameter.has_key(call)) { - return video_content_parameter[call].stream; - } - return null; - } - - public Xep.JingleRtp.Stream? get_audio_stream(Call call) { - if (audio_content_parameter.has_key(call)) { - return audio_content_parameter[call].stream; - } - return null; - } - - public async Call? initiate_call(Conversation conversation, bool video) { + public async CallState? initiate_call(Conversation conversation, bool video) { Call call = new Call(); call.direction = Call.DIRECTION_OUTGOING; call.account = conversation.account; call.counterpart = conversation.counterpart; - call.ourpart = conversation.account.full_jid; + call.ourpart = stream_interactor.get_module(MucManager.IDENTITY).get_own_jid(conversation.counterpart, conversation.account) ?? conversation.account.full_jid; call.time = call.local_time = call.end_time = new DateTime.now_utc(); call.state = Call.State.RINGING; stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); - we_should_send_video[call] = video; - we_should_send_audio[call] = true; - - Gee.List<Jid> call_resources = yield get_call_resources(conversation); - - bool do_jmi = false; - Jid? jid_for_direct = null; - if (yield contains_jmi_resources(conversation.account, call_resources)) { - do_jmi = true; - } else if (!call_resources.is_empty) { - jid_for_direct = call_resources[0]; - } else if (has_jmi_resources(conversation)) { - do_jmi = true; - } - - if (do_jmi) { - XmppStream? stream = stream_interactor.get_stream(conversation.account); - jmi_call[conversation.account] = call; - jmi_video[conversation.account] = video; - jmi_sid[conversation.account] = Xmpp.random_uuid(); - - call_by_sid[call.account][jmi_sid[conversation.account]] = call; + var call_state = new CallState(call, stream_interactor); + call_state.we_should_send_video = video; + call_state.we_should_send_audio = true; + connect_call_state_signals(call_state); - var descriptions = new ArrayList<StanzaNode>(); - descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "audio")); - if (video) { - descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "video")); - } - - stream.get_module(Xmpp.Xep.JingleMessageInitiation.Module.IDENTITY).send_session_propose_to_peer(stream, conversation.counterpart, jmi_sid[call.account], descriptions); - } else if (jid_for_direct != null) { - yield call_resource(conversation.account, jid_for_direct, call, video); + if (conversation.type_ == Conversation.Type.CHAT) { + call.add_peer(conversation.counterpart); + PeerState peer_state = call_state.set_first_peer(conversation.counterpart); + yield peer_state.initiate_call(conversation.counterpart); + } else { + call_state.initiate_groupchat_call.begin(conversation.counterpart); } conversation.last_active = call.time; - call_outgoing(call, conversation); - return call; - } + call_outgoing(call, call_state, conversation); - private async void call_resource(Account account, Jid full_jid, Call call, bool video, string? sid = null) { - XmppStream? stream = stream_interactor.get_stream(account); - if (stream == null) return; - - try { - Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, video, sid); - sessions[call] = session; - sid_by_call[call.account][call] = session.sid; - - connect_session_signals(call, session); - } catch (Error e) { - warning("Failed to start call: %s", e.message); - } + return call_state; } - public void end_call(Conversation conversation, Call call) { - XmppStream? stream = stream_interactor.get_stream(call.account); - if (stream == null) return; - - if (call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { - sessions[call].terminate(Xep.Jingle.ReasonElement.SUCCESS, null, "success"); - call.state = Call.State.ENDED; - } else if (call.state == Call.State.RINGING) { - if (sessions.has_key(call)) { - sessions[call].terminate(Xep.Jingle.ReasonElement.CANCEL, null, "cancel"); - } else { - // Only a JMI so far - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_retract_to_peer(stream, call.counterpart, jmi_sid[call.account]); - } - call.state = Call.State.MISSED; - } else { - return; - } - - call.end_time = new DateTime.now_utc(); - - remove_call_from_datastructures(call); - } - - public void accept_call(Call call) { - call.state = Call.State.ESTABLISHING; + public async bool can_do_audio_calls_async(Conversation conversation) { + if (!can_do_audio_calls()) return false; - if (sessions.has_key(call)) { - foreach (Xep.Jingle.Content content in sessions[call].contents) { - content.accept(); - } + if (conversation.type_ == Conversation.Type.CHAT) { + return (yield get_call_resources(conversation.account, conversation.counterpart)).size > 0 || has_jmi_resources(conversation.counterpart); } else { - // Only a JMI so far - Account account = call.account; - string sid = sid_by_call[call.account][call]; - XmppStream stream = stream_interactor.get_stream(account); - if (stream == null) return; - - jmi_call[account] = call; - jmi_sid[account] = sid; - jmi_video[account] = we_should_send_video[call]; - - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, sid); - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, call.counterpart, sid); + return stream_interactor.get_module(MucManager.IDENTITY).is_private_room(conversation.account, conversation.counterpart); } } - public void reject_call(Call call) { - call.state = Call.State.DECLINED; - - if (sessions.has_key(call)) { - foreach (Xep.Jingle.Content content in sessions[call].contents) { - content.reject(); - } - remove_call_from_datastructures(call); - } else { - // Only a JMI so far - XmppStream stream = stream_interactor.get_stream(call.account); - if (stream == null) return; - - string sid = sid_by_call[call.account][call]; - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_peer(stream, call.counterpart, sid); - stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); - remove_call_from_datastructures(call); - } - } - - public void mute_own_audio(Call call, bool mute) { - we_should_send_audio[call] = !mute; - - Xep.JingleRtp.Stream stream = audio_content_parameter[call].stream; - // The user might mute audio before a feed was created. The feed will be muted as soon as it has been created. - if (stream == null) return; - - // Inform our counterpart that we (un)muted our audio - stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_mute(sessions[call], mute, "audio"); - - // Start/Stop sending audio data - Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); - } - - public void mute_own_video(Call call, bool mute) { - we_should_send_video[call] = !mute; - - if (!sessions.has_key(call)) { - // Call hasn't been established yet - return; - } - - Xep.JingleRtp.Module rtp_module = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY); - - if (video_content_parameter.has_key(call) && - video_content_parameter[call].stream != null && - sessions[call].senders_include_us(video_content[call].senders)) { - // A video feed has already been established - - // Start/Stop sending video data - Xep.JingleRtp.Stream stream = video_content_parameter[call].stream; - if (stream != null) { - // TODO maybe the user muted video before the feed was created... - Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); - } - - // Inform our counterpart that we started/stopped our video - rtp_module.session_info_type.send_mute(sessions[call], mute, "video"); - } else if (!mute) { - // Need to start a new video feed - XmppStream stream = stream_interactor.get_stream(call.account); - rtp_module.add_outgoing_video_content.begin(stream, sessions[call], (_, res) => { - if (video_content_parameter[call] == null) { - Xep.Jingle.Content content = rtp_module.add_outgoing_video_content.end(res); - Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; - if (rtp_content_parameter != null) { - connect_content_signals(call, content, rtp_content_parameter); - } - } - }); - } - // If video_feed == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. - } - - public async bool can_do_audio_calls_async(Conversation conversation) { - if (!can_do_audio_calls()) return false; - return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); - } - private bool can_do_audio_calls() { Plugins.VideoCallPlugin? plugin = Application.get_default().plugin_registry.video_call_plugin; if (plugin == null) return false; @@ -270,7 +85,12 @@ namespace Dino { public async bool can_do_video_calls_async(Conversation conversation) { if (!can_do_video_calls()) return false; - return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); + + if (conversation.type_ == Conversation.Type.CHAT) { + return (yield get_call_resources(conversation.account, conversation.counterpart)).size > 0 || has_jmi_resources(conversation.counterpart); + } else { + return stream_interactor.get_module(MucManager.IDENTITY).is_private_room(conversation.account, conversation.counterpart); + } } private bool can_do_video_calls() { @@ -280,13 +100,13 @@ namespace Dino { return plugin.supports("video"); } - private async Gee.List<Jid> get_call_resources(Conversation conversation) { + public async Gee.List<Jid> get_call_resources(Account account, Jid counterpart) { ArrayList<Jid> ret = new ArrayList<Jid>(); - XmppStream? stream = stream_interactor.get_stream(conversation.account); + XmppStream? stream = stream_interactor.get_stream(account); if (stream == null) return ret; - Gee.List<Jid>? full_jids = stream.get_flag(Presence.Flag.IDENTITY).get_resources(conversation.counterpart); + Gee.List<Jid>? full_jids = stream.get_flag(Presence.Flag.IDENTITY).get_resources(counterpart); if (full_jids == null) return ret; foreach (Jid full_jid in full_jids) { @@ -297,7 +117,7 @@ namespace Dino { return ret; } - private async bool contains_jmi_resources(Account account, Gee.List<Jid> full_jids) { + public async bool contains_jmi_resources(Account account, Gee.List<Jid> full_jids) { XmppStream? stream = stream_interactor.get_stream(account); if (stream == null) return false; @@ -308,26 +128,22 @@ namespace Dino { return false; } - private bool has_jmi_resources(Conversation conversation) { + public bool has_jmi_resources(Jid counterpart) { int64 jmi_resources = db.entity.select() - .with(db.entity.jid_id, "=", db.get_jid_id(conversation.counterpart)) + .with(db.entity.jid_id, "=", db.get_jid_id(counterpart)) .join_with(db.entity_feature, db.entity.caps_hash, db.entity_feature.entity) .with(db.entity_feature.feature, "=", Xep.JingleMessageInitiation.NS_URI) .count(); return jmi_resources > 0; } - public bool should_we_send_video(Call call) { - return we_should_send_video[call]; - } - - public Jid? is_call_in_progress() { - foreach (Call call in sessions.keys) { + public bool is_call_in_progress() { + foreach (Call call in call_states.keys) { if (call.state == Call.State.IN_PROGRESS || call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { - return call.counterpart; + return true; } } - return null; + return false; } private void on_incoming_call(Account account, Xep.Jingle.Session session) { @@ -336,56 +152,81 @@ namespace Dino { return; } + Jid? muji_muc = null; bool counterpart_wants_video = false; foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; if (rtp_content_parameter == null) continue; + muji_muc = rtp_content_parameter.muji_muc; if (rtp_content_parameter.media == "video" && session.senders_include_us(content.senders)) { counterpart_wants_video = true; } } - // Session might have already been accepted via Jingle Message Initiation - bool already_accepted = jmi_sid.has_key(account) && - jmi_sid[account] == session.sid && jmi_call[account].account.equals(account) && - jmi_call[account].counterpart.equals_bare(session.peer_full_jid) && - jmi_video[account] == counterpart_wants_video; - - Call? call = null; - if (already_accepted) { - call = jmi_call[account]; - } else { - call = create_received_call(account, session.peer_full_jid, account.full_jid, counterpart_wants_video); + // Check if this comes from a MUJI MUC => accept + if (muji_muc != null) { + debug("[%s] Incoming call from %s from MUJI muc %s", account.bare_jid.to_string(), session.peer_full_jid.to_string(), muji_muc.to_string()); + + foreach (CallState call_state in call_states.values) { + if (call_state.group_call != null && call_state.group_call.muc_jid.equals(muji_muc)) { + if (call_state.peers.keys.contains(session.peer_full_jid)) { + PeerState peer_state = call_state.peers[session.peer_full_jid]; + debug("[%s] Incoming call, we know the peer. Expected %b", account.bare_jid.to_string(), peer_state.waiting_for_inbound_muji_connection); + if (!peer_state.waiting_for_inbound_muji_connection) return; + + peer_state.set_session(session); + debug(@"[%s] Accepting incoming MUJI call from %s", account.bare_jid.to_string(), session.peer_full_jid.to_string()); + peer_state.accept(); + } else { + debug(@"[%s] Incoming call, but didn't see peer in MUC yet", account.bare_jid.to_string()); + PeerState peer_state = new PeerState(session.peer_full_jid, call_state.call, stream_interactor); + peer_state.set_session(session); + call_state.add_peer(peer_state); + } + return; + } + } + return; } - sessions[call] = session; - call_by_sid[account][session.sid] = call; - sid_by_call[account][call] = session.sid; + debug(@"[%s] Incoming call from %s", account.bare_jid.to_string(), session.peer_full_jid.to_string()); - connect_session_signals(call, session); + // Check if we already accepted this call via Jingle Message Initiation => accept + if (current_jmi_request_call.has_key(account) && + current_jmi_request_peer[account].sid == session.sid && + current_jmi_request_peer[account].we_should_send_video == counterpart_wants_video && + current_jmi_request_peer[account].accepted_jmi) { + current_jmi_request_peer[account].set_session(session); + current_jmi_request_call[account].accept(); - if (already_accepted) { - accept_call(call); - } else { - stream_interactor.module_manager.get_module(account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_ringing(session); + current_jmi_request_peer.unset(account); + current_jmi_request_call.unset(account); + return; } + + // This is a direct call without prior JMI. Ask user. + PeerState peer_state = create_received_call(account, session.peer_full_jid, account.full_jid, counterpart_wants_video); + peer_state.set_session(session); + stream_interactor.module_manager.get_module(account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_ringing(session); } - private Call create_received_call(Account account, Jid from, Jid to, bool video_requested) { + private PeerState create_received_call(Account account, Jid from, Jid to, bool video_requested) { Call call = new Call(); if (from.equals_bare(account.bare_jid)) { // Call requested by another of our devices call.direction = Call.DIRECTION_OUTGOING; call.ourpart = from; + call.state = Call.State.OTHER_DEVICE; call.counterpart = to; } else { call.direction = Call.DIRECTION_INCOMING; call.ourpart = account.full_jid; + call.state = Call.State.RINGING; call.counterpart = from; } + call.add_peer(call.counterpart); call.account = account; call.time = call.local_time = call.end_time = new DateTime.now_utc(); - call.state = Call.State.RINGING; Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(call.counterpart.bare_jid, account, Conversation.Type.CHAT); @@ -393,208 +234,93 @@ namespace Dino { conversation.last_active = call.time; - we_should_send_video[call] = video_requested; - we_should_send_audio[call] = true; + var call_state = new CallState(call, stream_interactor); + connect_call_state_signals(call_state); + PeerState peer_state = call_state.set_first_peer(call.counterpart); + call_state.we_should_send_video = video_requested; + call_state.we_should_send_audio = true; if (call.direction == Call.DIRECTION_INCOMING) { - call_incoming(call, conversation, video_requested); + call_incoming(call, call_state, conversation, video_requested); } else { - call_outgoing(call, conversation); - } - - return call; - } - - private void on_incoming_content_add(XmppStream stream, Call call, Xep.Jingle.Session session, Xep.Jingle.Content content) { - Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; - - if (rtp_content_parameter == null) { - content.reject(); - return; - } - - // Our peer shouldn't tell us to start sending, that's for us to initiate - if (session.senders_include_us(content.senders)) { - if (session.senders_include_counterpart(content.senders)) { - // If our peer wants to send, let them - content.modify(session.we_initiated ? Xep.Jingle.Senders.RESPONDER : Xep.Jingle.Senders.INITIATOR); - } else { - // If only we're supposed to send, reject - content.reject(); - } - } - - connect_content_signals(call, content, rtp_content_parameter); - content.accept(); - } - - private void on_call_terminated(Call call, bool we_terminated, string? reason_name, string? reason_text) { - if (call.state == Call.State.RINGING || call.state == Call.State.IN_PROGRESS || call.state == Call.State.ESTABLISHING) { - call.end_time = new DateTime.now_utc(); - } - if (call.state == Call.State.IN_PROGRESS) { - call.state = Call.State.ENDED; - } else if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { - if (reason_name == Xep.Jingle.ReasonElement.DECLINE) { - call.state = Call.State.DECLINED; - } else { - call.state = Call.State.FAILED; - } + call_outgoing(call, call_state, conversation); } - call_terminated(call, reason_name, reason_text); - remove_call_from_datastructures(call); + return peer_state; } - private void on_stream_created(Call call, string media, Xep.JingleRtp.Stream stream) { - if (media == "video" && stream.receiving) { - counterpart_sends_video[call] = true; - video_content_parameter[call].connection_ready.connect((status) => { - counterpart_sends_video_updated(call, false); - }); - } - stream_created(call, media); + private CallState? get_call_state_for_groupcall(Account account, Jid muc_jid) { + foreach (CallState call_state in call_states.values) { + if (!call_state.call.account.equals(account)) continue; - // Outgoing audio/video might have been muted in the meanwhile. - if (media == "video" && !we_should_send_video[call]) { - mute_own_video(call, true); - } else if (media == "audio" && !we_should_send_audio[call]) { - mute_own_audio(call, true); + if (call_state.group_call != null && call_state.group_call.muc_jid.equals(muc_jid)) return call_state; + if (call_state.invited_to_group_call != null && call_state.invited_to_group_call.equals(muc_jid)) return call_state; } + return null; } - private void on_counterpart_mute_update(Call call, bool mute, string? media) { - if (!call.equals(call)) return; + private async void on_muji_call_received(Account account, Jid inviter_jid, Jid muc_jid, Gee.List<StanzaNode> descriptions, string message_type) { + debug("[%s] Muji call received from %s for MUC %s, type %s", account.bare_jid.to_string(), inviter_jid.to_string(), muc_jid.to_string(), message_type); - if (media == "video") { - counterpart_sends_video[call] = !mute; - counterpart_sends_video_updated(call, mute); - } - } + foreach (Call call in call_states.keys) { + if (!call.account.equals(account)) return; - private void connect_session_signals(Call call, Xep.Jingle.Session session) { - session.terminated.connect((stream, we_terminated, reason_name, reason_text) => - on_call_terminated(call, we_terminated, reason_name, reason_text) - ); - session.additional_content_add_incoming.connect((session,stream, content) => - on_incoming_content_add(stream, call, session, content) - ); + // We already know the call; this is a reflection of our own invite + if (call_states[call].parent_muc.equals_bare(inviter_jid)) return; - foreach (Xep.Jingle.Content content in session.contents) { - Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; - if (rtp_content_parameter == null) continue; - - connect_content_signals(call, content, rtp_content_parameter); - } - } - - private void connect_content_signals(Call call, Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { - if (rtp_content_parameter.media == "audio") { - audio_content[call] = content; - audio_content_parameter[call] = rtp_content_parameter; - } else if (rtp_content_parameter.media == "video") { - video_content[call] = content; - video_content_parameter[call] = rtp_content_parameter; - } - - rtp_content_parameter.stream_created.connect((stream) => on_stream_created(call, rtp_content_parameter.media, stream)); - rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(call, content, rtp_content_parameter.media)); - - content.senders_modify_incoming.connect((content, proposed_senders) => { - if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { - warning("counterpart set us to (not)sending %s. ignoring", content.content_name); + if (call.counterparts.contains(inviter_jid) && call_states[call].accepted) { + // A call is converted into a group call. + yield call_states[call].join_group_call(muc_jid); return; } - - if (!content.session.senders_include_counterpart(content.senders) && content.session.senders_include_counterpart(proposed_senders)) { - // Counterpart wants to start sending. Ok. - content.accept_content_modify(proposed_senders); - on_counterpart_mute_update(call, false, "video"); - } - }); - } - - private void on_connection_ready(Call call, Xep.Jingle.Content content, string media) { - if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { - call.state = Call.State.IN_PROGRESS; } - if (media == "audio") { - audio_encryptions[call] = content.encryptions; - } else if (media == "video") { - video_encryptions[call] = content.encryptions; - } + bool audio_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "audio"); + bool video_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "video"); - if ((audio_encryptions.has_key(call) && audio_encryptions[call].is_empty) || (video_encryptions.has_key(call) && video_encryptions[call].is_empty)) { - call.encryption = Encryption.NONE; - encryption_updated(call, null, null, true); - return; - } + Call call = new Call(); + call.direction = Call.DIRECTION_INCOMING; + call.ourpart = account.full_jid; + call.counterpart = inviter_jid; + call.account = account; + call.time = call.local_time = call.end_time = new DateTime.now_utc(); + call.state = Call.State.RINGING; - HashMap<string, Xep.Jingle.ContentEncryption> encryptions = audio_encryptions[call] ?? video_encryptions[call]; + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(inviter_jid.bare_jid, account); + stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); + conversation.last_active = call.time; - Xep.Jingle.ContentEncryption? omemo_encryption = null, dtls_encryption = null, srtp_encryption = null; - foreach (string encr_name in encryptions.keys) { - if (video_encryptions.has_key(call) && !video_encryptions[call].has_key(encr_name)) continue; + CallState call_state = new CallState(call, stream_interactor); + connect_call_state_signals(call_state); + call_state.we_should_send_audio = true; + call_state.we_should_send_video = video_requested; + call_state.invited_to_group_call = muc_jid; + call_state.group_call_inviter = inviter_jid; - var encryption = encryptions[encr_name]; - if (encryption.encryption_ns == "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification") { - omemo_encryption = encryption; - } else if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { - dtls_encryption = encryption; - } else if (encryption.encryption_name == "SRTP") { - srtp_encryption = encryption; - } - } + debug("[%s] on_muji_call_received accepting", account.bare_jid.to_string()); + call_incoming(call_state.call, call_state, conversation, video_requested); + } - if (omemo_encryption != null && dtls_encryption != null) { - call.encryption = Encryption.OMEMO; - Xep.Jingle.ContentEncryption? video_encryption = video_encryptions.has_key(call) ? video_encryptions[call]["http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"] : null; - omemo_encryption.peer_key = dtls_encryption.peer_key; - omemo_encryption.our_key = dtls_encryption.our_key; - encryption_updated(call, omemo_encryption, video_encryption, true); - } else if (dtls_encryption != null) { - call.encryption = Encryption.DTLS_SRTP; - Xep.Jingle.ContentEncryption? video_encryption = video_encryptions.has_key(call) ? video_encryptions[call][Xep.JingleIceUdp.DTLS_NS_URI] : null; - bool same = true; - if (video_encryption != null && dtls_encryption.peer_key.length == video_encryption.peer_key.length) { - for (int i = 0; i < dtls_encryption.peer_key.length; i++) { - if (dtls_encryption.peer_key[i] != video_encryption.peer_key[i]) { same = false; break; } - } - } - encryption_updated(call, dtls_encryption, video_encryption, same); - } else if (srtp_encryption != null) { - call.encryption = Encryption.SRTP; - encryption_updated(call, srtp_encryption, video_encryptions[call]["SRTP"], false); - } else { - call.encryption = Encryption.NONE; - encryption_updated(call, null, null, true); + private void remove_call_from_datastructures(Call call) { + if (current_jmi_request_call.has_key(call.account) && current_jmi_request_call[call.account].call.equals(call)) { + current_jmi_request_call.unset(call.account); + current_jmi_request_peer.unset(call.account); } + call_states.unset(call); } - private void remove_call_from_datastructures(Call call) { - string? sid = sid_by_call[call.account][call]; - sid_by_call[call.account].unset(call); - if (sid != null) call_by_sid[call.account].unset(sid); - - sessions.unset(call); - - counterpart_sends_video.unset(call); - we_should_send_video.unset(call); - we_should_send_audio.unset(call); - - audio_content_parameter.unset(call); - video_content_parameter.unset(call); - audio_content.unset(call); - video_content.unset(call); - audio_encryptions.unset(call); - video_encryptions.unset(call); + private void connect_call_state_signals(CallState call_state) { + call_states[call_state.call] = call_state; + + ulong terminated_handler_id = -1; + terminated_handler_id = call_state.terminated.connect((who_terminated, reason_name, reason_text) => { + remove_call_from_datastructures(call_state.call); + call_terminated(call_state.call, reason_name, reason_text); + call_state.disconnect(terminated_handler_id); + }); } private void on_account_added(Account account) { - call_by_sid[account] = new HashMap<string, Call>(); - sid_by_call[account] = new HashMap<Call, string>(); - Xep.Jingle.Module jingle_module = stream_interactor.module_manager.get_module(account, Xep.Jingle.Module.IDENTITY); jingle_module.session_initiate_received.connect((stream, session) => { foreach (Xep.Jingle.Content content in session.contents) { @@ -606,27 +332,6 @@ namespace Dino { } }); - var session_info_type = stream_interactor.module_manager.get_module(account, Xep.JingleRtp.Module.IDENTITY).session_info_type; - session_info_type.mute_update_received.connect((session,mute, name) => { - if (!call_by_sid[account].has_key(session.sid)) return; - Call call = call_by_sid[account][session.sid]; - - foreach (Xep.Jingle.Content content in session.contents) { - if (name == null || content.content_name == name) { - Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; - if (rtp_content_parameter != null) { - on_counterpart_mute_update(call, mute, rtp_content_parameter.media); - } - } - } - }); - session_info_type.info_received.connect((session, session_info) => { - if (!call_by_sid[account].has_key(session.sid)) return; - Call call = call_by_sid[account][session.sid]; - - info_received(call, session_info); - }); - Xep.JingleMessageInitiation.Module mi_module = stream_interactor.module_manager.get_module(account, Xep.JingleMessageInitiation.Module.IDENTITY); mi_module.session_proposed.connect((from, to, sid, descriptions) => { if (!can_do_audio_calls()) { @@ -637,53 +342,105 @@ namespace Dino { bool audio_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "audio"); bool video_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "video"); if (!audio_requested && !video_requested) return; - Call call = create_received_call(account, from, to, video_requested); - call_by_sid[account][sid] = call; - sid_by_call[account][call] = sid; + + PeerState peer_state = create_received_call(account, from, to, video_requested); + peer_state.sid = sid; + peer_state.we_should_send_audio = true; + peer_state.we_should_send_video = video_requested; + + current_jmi_request_peer[account] = peer_state; + current_jmi_request_call[account] = call_states[peer_state.call]; }); - mi_module.session_accepted.connect((from, sid) => { - if (!call_by_sid[account].has_key(sid)) return; + mi_module.session_accepted.connect((from, to, sid) => { + if (!current_jmi_request_peer.has_key(account) || current_jmi_request_peer[account].sid != sid) return; if (from.equals_bare(account.bare_jid)) { // Carboned message from our account // Ignore carbon from ourselves if (from.equals(account.full_jid)) return; - Call call = call_by_sid[account][sid]; - call.state = Call.State.OTHER_DEVICE_ACCEPTED; + Call call = current_jmi_request_peer[account].call; + call.ourpart = from; + call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call); - } else if (from.equals_bare(call_by_sid[account][sid].counterpart)) { // Message from our peer + } else if (from.equals_bare(current_jmi_request_peer[account].jid) && to.equals(account.full_jid)) { // Message from our peer // We proposed the call - if (jmi_sid.has_key(account) && jmi_sid[account] == sid) { - call_resource.begin(account, from, jmi_call[account], jmi_video[account], jmi_sid[account]); - jmi_call.unset(account); - jmi_sid.unset(account); - jmi_video.unset(account); - } + // We know the full jid of our peer now + current_jmi_request_call[account].rename_peer(current_jmi_request_peer[account].jid, from); + current_jmi_request_peer[account].call_resource.begin(from); } }); mi_module.session_rejected.connect((from, to, sid) => { - if (!call_by_sid[account].has_key(sid)) return; - Call call = call_by_sid[account][sid]; + if (!current_jmi_request_peer.has_key(account) || current_jmi_request_peer[account].sid != sid) return; + Call call = current_jmi_request_peer[account].call; - bool outgoing_reject = call.direction == Call.DIRECTION_OUTGOING && from.equals_bare(call.counterpart); + bool outgoing_reject = call.direction == Call.DIRECTION_OUTGOING && from.equals_bare(call.counterparts[0]); bool incoming_reject = call.direction == Call.DIRECTION_INCOMING && from.equals_bare(account.bare_jid); - if (!(outgoing_reject || incoming_reject)) return; + if (!outgoing_reject && !incoming_reject) return; + + // We don't care if a single person in a group call rejected the call + if (incoming_reject && call_states[call].group_call != null) return; call.state = Call.State.DECLINED; + call_states[call].terminated(from, Xep.Jingle.ReasonElement.DECLINE, "JMI reject"); remove_call_from_datastructures(call); - call_terminated(call, null, null); }); mi_module.session_retracted.connect((from, to, sid) => { - if (!call_by_sid[account].has_key(sid)) return; - Call call = call_by_sid[account][sid]; + if (!current_jmi_request_peer.has_key(account) || current_jmi_request_peer[account].sid != sid) return; + Call call = current_jmi_request_peer[account].call; bool outgoing_retract = call.direction == Call.DIRECTION_OUTGOING && from.equals_bare(account.bare_jid); bool incoming_retract = call.direction == Call.DIRECTION_INCOMING && from.equals_bare(call.counterpart); if (!(outgoing_retract || incoming_retract)) return; call.state = Call.State.MISSED; + call_states[call].terminated(from, Xep.Jingle.ReasonElement.CANCEL, "JMI retract"); remove_call_from_datastructures(call); - call_terminated(call, null, null); + }); + + Xep.MujiMeta.Module muji_meta_module = stream_interactor.module_manager.get_module(account, Xep.MujiMeta.Module.IDENTITY); + muji_meta_module.call_proposed.connect((inviter_jid, to, muc_jid, descriptions, message_type) => { + if (inviter_jid.equals_bare(account.bare_jid)) return; + on_muji_call_received.begin(account, inviter_jid, muc_jid, descriptions, message_type); + }); + muji_meta_module.call_accepted.connect((from_jid, muc_jid, message_type) => { + if (!from_jid.equals_bare(account.bare_jid)) return; + + // We accepted the call from another device + CallState? call_state = get_call_state_for_groupcall(account, muc_jid); + if (call_state == null) return; + + call_state.call.state = Call.State.OTHER_DEVICE; + remove_call_from_datastructures(call_state.call); + }); + muji_meta_module.call_retracted.connect((from_jid, to_jid, muc_jid, message_type) => { + if (from_jid.equals_bare(account.bare_jid)) return; + + // The call was retracted by the counterpart + CallState? call_state = get_call_state_for_groupcall(account, muc_jid); + if (call_state == null) return; + + if (call_state.call.state != Call.State.RINGING) { + debug("%s tried to retract a call that's in state %s. Ignoring.", from_jid.to_string(), call_state.call.state.to_string()); + return; + } + + // TODO prevent other MUC occupants from retracting a call + + call_state.call.state = Call.State.MISSED; + remove_call_from_datastructures(call_state.call); + }); + muji_meta_module.call_rejected.connect((from_jid, to_jid, muc_jid, message_type) => { + if (from_jid.equals_bare(account.bare_jid)) return; + debug(@"[%s] rejected our MUJI invite to %s", account.bare_jid.to_string(), from_jid.to_string(), muc_jid.to_string()); + }); + + stream_interactor.module_manager.get_module(account, Xep.Coin.Module.IDENTITY).coin_info_received.connect((jid, info) => { + foreach (Call call in call_states.keys) { + if (call.counterparts[0].equals_bare(jid)) { + conference_info_received(call, info); + return; + } + } }); } } diff --git a/libdino/src/service/connection_manager.vala b/libdino/src/service/connection_manager.vala index 0eb6a6f5..d114b9ae 100644 --- a/libdino/src/service/connection_manager.vala +++ b/libdino/src/service/connection_manager.vala @@ -228,6 +228,8 @@ public class ConnectionManager : Object { stream.attached_modules.connect((stream) => { stream_attached_modules(account, stream); change_connection_state(account, ConnectionState.CONNECTED); + +// stream.get_module(Xep.Muji.Module.IDENTITY).join_call(stream, new Jid("test@muc.poez.io"), true); }); stream.get_module(Sasl.Module.IDENTITY).received_auth_failure.connect((stream, node) => { set_connection_error(account, new ConnectionError(ConnectionError.Source.SASL, null)); diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala index 87244a23..c6c47af4 100644 --- a/libdino/src/service/content_item_store.vala +++ b/libdino/src/service/content_item_store.vala @@ -68,7 +68,7 @@ public class ContentItemStore : StreamInteractionModule, Object { } break; case 3: - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(foreign_id); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(foreign_id, conversation); if (call != null) { var call_item = new CallItem(call, conversation, row[db.content_item.id]); items.add(call_item); @@ -177,7 +177,7 @@ public class ContentItemStore : StreamInteractionModule, Object { new_item(item, conversation); } - private void insert_call(Call call, Conversation conversation) { + private void insert_call(Call call, CallState call_state, Conversation conversation) { CallItem item = new CallItem(call, conversation, -1); item.id = db.add_content_item(conversation, call.time, call.local_time, 3, call.id, false); if (collection_conversations.has_key(conversation)) { @@ -299,7 +299,7 @@ public class CallItem : ContentItem { public Conversation conversation; public CallItem(Call call, Conversation conversation, int id) { - base(id, TYPE, call.from, call.time, call.encryption, Message.Marked.NONE); + base(id, TYPE, call.proposer, call.time, call.encryption, Message.Marked.NONE); this.call = call; this.conversation = conversation; diff --git a/libdino/src/service/database.vala b/libdino/src/service/database.vala index dab32749..0300112a 100644 --- a/libdino/src/service/database.vala +++ b/libdino/src/service/database.vala @@ -7,7 +7,7 @@ using Dino.Entities; namespace Dino { public class Database : Qlite.Database { - private const int VERSION = 21; + private const int VERSION = 22; public class AccountTable : Table { public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true }; @@ -174,6 +174,19 @@ public class Database : Qlite.Database { } } + public class CallCounterpartTable : Table { + public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true }; + public Column<int> call_id = new Column.Integer("call_id") { not_null = true }; + public Column<int> jid_id = new Column.Integer("jid_id") { not_null = true }; + public Column<string> resource = new Column.Text("resource"); + + internal CallCounterpartTable(Database db) { + base(db, "call_counterpart"); + init({call_id, jid_id, resource}); + index("call_counterpart_call_jid_idx", {call_id}); + } + } + public class ConversationTable : Table { public Column<int> id = new Column.Integer("id") { primary_key = true, auto_increment = true }; public Column<int> account_id = new Column.Integer("account_id") { not_null = true }; @@ -295,6 +308,7 @@ public class Database : Qlite.Database { public RealJidTable real_jid { get; private set; } public FileTransferTable file_transfer { get; private set; } public CallTable call { get; private set; } + public CallCounterpartTable call_counterpart { get; private set; } public ConversationTable conversation { get; private set; } public AvatarTable avatar { get; private set; } public EntityIdentityTable entity_identity { get; private set; } @@ -319,6 +333,7 @@ public class Database : Qlite.Database { real_jid = new RealJidTable(this); file_transfer = new FileTransferTable(this); call = new CallTable(this); + call_counterpart = new CallCounterpartTable(this); conversation = new ConversationTable(this); avatar = new AvatarTable(this); entity_identity = new EntityIdentityTable(this); @@ -327,7 +342,7 @@ public class Database : Qlite.Database { mam_catchup = new MamCatchupTable(this); settings = new SettingsTable(this); conversation_settings = new ConversationSettingsTable(this); - init({ account, jid, entity, content_item, message, message_correction, real_jid, file_transfer, call, conversation, avatar, entity_identity, entity_feature, roster, mam_catchup, settings, conversation_settings }); + init({ account, jid, entity, content_item, message, message_correction, real_jid, file_transfer, call, call_counterpart, conversation, avatar, entity_identity, entity_feature, roster, mam_catchup, settings, conversation_settings }); try { exec("PRAGMA journal_mode = WAL"); @@ -446,6 +461,19 @@ public class Database : Qlite.Database { error("Failed to upgrade to database version 18: %s", e.message); } } + if (oldVersion < 22) { + try { + exec("INSERT INTO call_counterpart (call_id, jid_id, resource) SELECT id, counterpart_id, counterpart_resource FROM call"); + } catch (Error e) { + error("Failed to upgrade to database version 22: %s", e.message); + } +// exec("ALTER TABLE call RENAME TO call2"); +// call.create_table_at_version(VERSION); +// exec("INSERT INTO call (id, account_id, our_resource, direction, time, local_time, end_time, encryption, state) +// SELECT id, account_id, our_resource, direction, time, local_time, end_time, encryption, state +// FROM call2"); +// exec("DROP TABLE call2"); + } } public ArrayList<Account> get_accounts() { diff --git a/libdino/src/service/module_manager.vala b/libdino/src/service/module_manager.vala index a6165392..39ed8a7c 100644 --- a/libdino/src/service/module_manager.vala +++ b/libdino/src/service/module_manager.vala @@ -80,6 +80,10 @@ public class ModuleManager { module_map[account].add(new Xep.LastMessageCorrection.Module()); module_map[account].add(new Xep.DirectMucInvitations.Module()); module_map[account].add(new Xep.JingleMessageInitiation.Module()); + module_map[account].add(new Xep.JingleRawUdp.Module()); + module_map[account].add(new Xep.Muji.Module()); + module_map[account].add(new Xep.MujiMeta.Module()); + module_map[account].add(new Xep.Coin.Module()); initialize_account_modules(account, module_map[account]); } } diff --git a/libdino/src/service/muc_manager.vala b/libdino/src/service/muc_manager.vala index 5a224a18..05473647 100644 --- a/libdino/src/service/muc_manager.vala +++ b/libdino/src/service/muc_manager.vala @@ -27,6 +27,7 @@ public class MucManager : StreamInteractionModule, Object { private ReceivedMessageListener received_message_listener; private HashMap<Account, BookmarksProvider> bookmarks_provider = new HashMap<Account, BookmarksProvider>(Account.hash_func, Account.equals_func); private HashMap<Account, Gee.List<Jid>> invites = new HashMap<Account, Gee.List<Jid>>(Account.hash_func, Account.equals_func); + public HashMap<Account, Jid> default_muc_server = new HashMap<Account, Jid>(Account.hash_func, Account.equals_func); public static void start(StreamInteractor stream_interactor) { MucManager m = new MucManager(stream_interactor); @@ -76,7 +77,7 @@ public class MucManager : StreamInteractionModule, Object { } mucs_todo[account].add(jid.with_resource(nick_)); - Muc.JoinResult? res = yield stream.get_module(Xep.Muc.Module.IDENTITY).enter(stream, jid.bare_jid, nick_, password, history_since); + Muc.JoinResult? res = yield stream.get_module(Xep.Muc.Module.IDENTITY).enter(stream, jid.bare_jid, nick_, password, history_since, null); mucs_joining[account].remove(jid); @@ -117,10 +118,10 @@ public class MucManager : StreamInteractionModule, Object { return yield stream.get_module(Xep.Muc.Module.IDENTITY).get_config_form(stream, jid); } - public void set_config_form(Account account, Jid jid, DataForms.DataForm data_form) { + public async void set_config_form(Account account, Jid jid, DataForms.DataForm data_form) { XmppStream? stream = stream_interactor.get_stream(account); if (stream == null) return; - stream.get_module(Xep.Muc.Module.IDENTITY).set_config_form(stream, jid, data_form); + yield stream.get_module(Xep.Muc.Module.IDENTITY).set_config_form(stream, jid, data_form); } public void change_subject(Account account, Jid jid, string subject) { @@ -170,7 +171,7 @@ public class MucManager : StreamInteractionModule, Object { public void change_affiliation(Account account, Jid jid, string nick, string role) { XmppStream? stream = stream_interactor.get_stream(account); - if (stream != null) stream.get_module(Xep.Muc.Module.IDENTITY).change_affiliation(stream, jid.bare_jid, nick, role); + if (stream != null) stream.get_module(Xep.Muc.Module.IDENTITY).change_affiliation.begin(stream, jid.bare_jid, null, nick, role); } public void change_role(Account account, Jid jid, string nick, string role) { @@ -401,6 +402,36 @@ public class MucManager : StreamInteractionModule, Object { }); } + private async void search_default_muc_server(Account account) { + XmppStream? stream = stream_interactor.get_stream(account); + if (stream == null) return; + + ServiceDiscovery.ItemsResult? items_result = yield stream.get_module(ServiceDiscovery.Module.IDENTITY).request_items(stream, stream.remote_name); + if (items_result == null) return; + + for (int i = 0; i < 2; i++) { + foreach (Xep.ServiceDiscovery.Item item in items_result.items) { + + // First try the promising items and only afterwards all the others + bool promising_upload_item = item.jid.to_string().has_prefix("conference") || + item.jid.to_string().has_prefix("muc") || + item.jid.to_string().has_prefix("chat"); + if ((i == 0 && !promising_upload_item) || (i == 1) && promising_upload_item) continue; + + Gee.Set<Xep.ServiceDiscovery.Identity> identities = yield stream_interactor.get_module(EntityInfo.IDENTITY).get_identities(account, item.jid); + if (identities == null) return; + + foreach (Xep.ServiceDiscovery.Identity identity in identities) { + if (identity.category == Xep.ServiceDiscovery.Identity.CATEGORY_CONFERENCE) { + default_muc_server[account] = item.jid; + debug("[%s] Default MUC: %s", account.bare_jid.to_string(), item.jid.to_string()); + return; + } + } + } + } + } + private async void on_stream_negotiated(Account account, XmppStream stream) { if (bookmarks_provider[account] == null) return; @@ -411,6 +442,10 @@ public class MucManager : StreamInteractionModule, Object { } else { sync_autojoin_active(account, conferences); } + + if (!default_muc_server.has_key(account)) { + search_default_muc_server.begin(account); + } } private void on_invite_received(Account account, Jid room_jid, Jid from_jid, string? password, string? reason) { diff --git a/libdino/src/service/notification_events.vala b/libdino/src/service/notification_events.vala index f87ebe0d..6f1d0fd4 100644 --- a/libdino/src/service/notification_events.vala +++ b/libdino/src/service/notification_events.vala @@ -106,7 +106,7 @@ public class NotificationEvents : StreamInteractionModule, Object { notifier.notify_subscription_request.begin(conversation); } - private void on_call_incoming(Call call, Conversation conversation, bool video) { + private void on_call_incoming(Call call, CallState call_state, Conversation conversation, bool video) { string conversation_display_name = get_conversation_display_name(stream_interactor, conversation, null); notifier.notify_call.begin(call, conversation, video, conversation_display_name); diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 4891abb0..3abe970a 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -16,6 +16,7 @@ find_packages(MAIN_PACKAGES REQUIRED ) set(RESOURCE_LIST + icons/dino-account-plus-symbolic.svg icons/dino-changes-allowed-symbolic.svg icons/dino-changes-prevent-symbolic.svg icons/dino-conversation-list-placeholder-arrow.svg @@ -139,9 +140,11 @@ SOURCES src/ui/call_window/audio_settings_popover.vala src/ui/call_window/call_bottom_bar.vala + src/ui/call_window/call_connection_details_window.vala src/ui/call_window/call_encryption_button.vala src/ui/call_window/call_window.vala src/ui/call_window/call_window_controller.vala + src/ui/call_window/participant_widget.vala src/ui/call_window/video_settings_popover.vala src/ui/chat_input/chat_input_controller.vala diff --git a/main/data/call_widget.ui b/main/data/call_widget.ui index 47fb0046..0c5d8bfa 100644 --- a/main/data/call_widget.ui +++ b/main/data/call_widget.ui @@ -66,19 +66,15 @@ <property name="reveal-child">True</property> <property name="visible">True</property> <child> - <object class="GtkBox"> + <object class="GtkBox" id="outer_additional_box"> <property name="visible">True</property> - <style> - <class name="incoming-call-box"/> - </style> <child> - <object class="GtkBox"> + <object class="GtkBox" id="incoming_call_box"> <property name="halign">end</property> <property name="orientation">horizontal</property> <property name="spacing">5</property> <property name="margin">10</property> <property name="hexpand">True</property> - <property name="visible">True</property> <child> <object class="GtkButton" id="reject_call_button"> <property name="label" translatable="yes">Reject</property> @@ -99,6 +95,13 @@ </child> </object> </child> + <child> + <object class="GtkBox" id="multiparty_peer_box"> + <property name="margin">10</property> + <property name="spacing">7</property> + <property name="hexpand">True</property> + </object> + </child> </object> </child> </object> diff --git a/main/data/icons/dino-account-plus-symbolic.svg b/main/data/icons/dino-account-plus-symbolic.svg new file mode 100644 index 00000000..cf743afa --- /dev/null +++ b/main/data/icons/dino-account-plus-symbolic.svg @@ -0,0 +1 @@ +<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="24" height="24" viewBox="0 0 24 24"><path d="M15,14C12.33,14 7,15.33 7,18V20H23V18C23,15.33 17.67,14 15,14M6,10V7H4V10H1V12H4V15H6V12H9V10M15,12A4,4 0 0,0 19,8A4,4 0 0,0 15,4A4,4 0 0,0 11,8A4,4 0 0,0 15,12Z" /></svg>
\ No newline at end of file diff --git a/main/data/theme.css b/main/data/theme.css index cf57ae96..36b3cb5b 100644 --- a/main/data/theme.css +++ b/main/data/theme.css @@ -114,6 +114,11 @@ window.dino-main .incoming-call-box { background: alpha(@theme_selected_bg_color, 0.1); } +window.dino-main .multiparty-participants { + border-top: 1px solid alpha(@theme_fg_color, 0.05); + background: alpha(@theme_fg_color, 0.04) +} + window.dino-main .dino-sidebar > frame.collapsed { border-bottom: 1px solid @borders; } @@ -280,20 +285,27 @@ box.dino-input-error label.input-status-highlight-once { margin: 0; } -.dino-call-window .encryption-box { +.dino-call-window .black-element { color: rgba(255,255,255,0.7); border-radius: 5px; background: rgba(0,0,0,0.5); - padding: 0px; border: none; box-shadow: none; } -.dino-call-window .encryption-box.unencrypted { +.dino-call-window label.black-element { + padding: 5px; +} + +.dino-call-window button.black-element { + padding: 0; +} + +.dino-call-window button.black-element.unencrypted { color: @error_color; } -.dino-call-window .encryption-box:hover { +.dino-call-window .black-element:hover { background: rgba(20,20,20,0.5); } @@ -303,22 +315,46 @@ box.dino-input-error label.input-status-highlight-once { border-radius: 0; } -.dino-call-window .call-header-bar, -.dino-call-window .call-header-bar image { +.dino-call-window .call-header-bar { color: #ededec; } +.dino-call-window .call-header-bar button image { + color: alpha(white, 0.7); +} + +.dino-call-window .call-header-bar button:hover image { + color: white; +} + +.dino-call-window .participant-title-button { + background: none; + border: 0; + border-radius: 0; + box-shadow: none; +} + .dino-call-window .call-bottom-bar { background: linear-gradient(rgba(0,0,0,0), rgba(0,0,0,0.3)); border: 0; } -.dino-call-window .video-placeholder-box { +.dino-call-window { background-color: #212121; } +.dino-call-window .participant-name { + color: white; + text-shadow: black; +} + .dino-call-window .text-no-controls { + color: black; background: white; border-radius: 5px; padding: 5px 10px; +} + +.dino-call-window .own-video { + box-shadow: 0 0 2px 0 rgba(0,0,0,0.5); }
\ No newline at end of file diff --git a/main/src/ui/application.vala b/main/src/ui/application.vala index bed6d01b..ecbea85e 100644 --- a/main/src/ui/application.vala +++ b/main/src/ui/application.vala @@ -206,21 +206,37 @@ public class Dino.Ui.Application : Gtk.Application, Dino.Application { }); add_action(open_shortcuts_action); - SimpleAction accept_call_action = new SimpleAction("accept-call", VariantType.INT32); + SimpleAction accept_call_action = new SimpleAction("accept-call", new VariantType.tuple(new VariantType[]{VariantType.INT32, VariantType.INT32})); accept_call_action.activate.connect((variant) => { - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); - stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + int conversation_id = variant.get_child_value(0).get_int32(); + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation_by_id(conversation_id); + if (conversation == null) return; + + int call_id = variant.get_child_value(1).get_int32(); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(call_id, conversation); + CallState? call_state = stream_interactor.get_module(Calls.IDENTITY).call_states[call]; + if (call_state == null) return; + + call_state.accept(); var call_window = new CallWindow(); - call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.controller = new CallWindowController(call_window, call_state, stream_interactor); call_window.present(); }); add_action(accept_call_action); - SimpleAction deny_call_action = new SimpleAction("deny-call", VariantType.INT32); + SimpleAction deny_call_action = new SimpleAction("reject-call", new VariantType.tuple(new VariantType[]{VariantType.INT32, VariantType.INT32})); deny_call_action.activate.connect((variant) => { - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); - stream_interactor.get_module(Calls.IDENTITY).reject_call(call); + int conversation_id = variant.get_child_value(0).get_int32(); + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation_by_id(conversation_id); + if (conversation == null) return; + + int call_id = variant.get_child_value(1).get_int32(); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(call_id, conversation); + CallState? call_state = stream_interactor.get_module(Calls.IDENTITY).call_states[call]; + if (call_state == null) return; + + call_state.reject(); }); add_action(deny_call_action); } diff --git a/main/src/ui/avatar_image.vala b/main/src/ui/avatar_image.vala index 81aa3ce1..f7731373 100644 --- a/main/src/ui/avatar_image.vala +++ b/main/src/ui/avatar_image.vala @@ -9,6 +9,7 @@ public class AvatarImage : Misc { public int height { get; set; default = 35; } public int width { get; set; default = 35; } public bool allow_gray { get; set; default = true; } + public bool force_gray { get; set; default = false; } public StreamInteractor? stream_interactor { get; set; } public AvatarManager? avatar_manager { owned get { return stream_interactor == null ? null : stream_interactor.get_module(AvatarManager.IDENTITY); } } public MucManager? muc_manager { owned get { return stream_interactor == null ? null : stream_interactor.get_module(MucManager.IDENTITY); } } @@ -78,14 +79,14 @@ public class AvatarImage : Misc { string user_color = Util.get_avatar_hex_color(stream_interactor, account, conversation.counterpart, conversation); if (avatar_manager.has_avatar_cached(account, conversation.counterpart)) { drawer = new AvatarDrawer().tile(avatar_manager.get_cached_avatar(account, conversation.counterpart), "#", user_color); - if (allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); + if (force_gray || allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); } else { Gee.List<Jid>? occupants = muc_manager.get_other_offline_members(conversation.counterpart, account); if (muc_manager.is_private_room(account, conversation.counterpart) && occupants != null && occupants.size > 0) { jids = occupants.to_array(); } else { drawer = new AvatarDrawer().tile(null, "#", user_color); - if (allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); + if (force_gray || allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); } try_load_avatar_async(conversation.counterpart); } @@ -116,7 +117,7 @@ public class AvatarImage : Misc { if (jids.length > 4) { drawer.plus(); } - if (allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); + if (force_gray || allow_gray && (!is_self_online() || !is_counterpart_online())) drawer.grayscale(); } diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index 8a0604b3..b3fa2093 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -25,17 +25,12 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; public VideoSettingsPopover? video_settings_popover; - public CallEntryptionButton encryption_button = new CallEntryptionButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; - private Label label = new Label("") { margin=20, halign=Align.CENTER, valign=Align.CENTER, wrap=true, wrap_mode=Pango.WrapMode.WORD_CHAR, hexpand=true, visible=true }; private Stack stack = new Stack() { visible=true }; public CallBottomBar() { Object(orientation:Orientation.HORIZONTAL, spacing:0); - Overlay default_control = new Overlay() { visible=true }; - default_control.add_overlay(encryption_button); - Box main_buttons = new Box(Orientation.HORIZONTAL, 20) { margin_start=40, margin_end=40, margin=20, halign=Align.CENTER, hexpand=true, visible=true }; audio_button.add(audio_image); @@ -66,11 +61,9 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { button_hang.clicked.connect(() => hang_up()); main_buttons.add(button_hang); - default_control.add(main_buttons); - label.get_style_context().add_class("text-no-controls"); - stack.add_named(default_control, "control-buttons"); + stack.add_named(main_buttons, "control-buttons"); stack.add_named(label, "label"); this.add(stack); @@ -159,6 +152,6 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { } public bool is_menu_active() { - return video_settings_button.active || audio_settings_button.active || encryption_button.active; + return video_settings_button.active || audio_settings_button.active; } }
\ No newline at end of file diff --git a/main/src/ui/call_window/call_connection_details_window.vala b/main/src/ui/call_window/call_connection_details_window.vala new file mode 100644 index 00000000..1d5265c9 --- /dev/null +++ b/main/src/ui/call_window/call_connection_details_window.vala @@ -0,0 +1,111 @@ +using Gtk; + +namespace Dino.Ui { + + public class CallConnectionDetailsWindow : Gtk.Window { + + public Box box = new Box(Orientation.VERTICAL, 15) { margin=10, halign=Align.CENTER, valign=Align.CENTER, visible=true }; + + private bool video_added = false; + private CallContentDetails audio_details = new CallContentDetails("Audio") { visible=true }; + private CallContentDetails video_details = new CallContentDetails("Video"); + + public CallConnectionDetailsWindow() { + box.add(audio_details); + box.add(video_details); + add(box); + } + + public void update_content(PeerInfo peer_info) { + if (peer_info.audio != null) { + audio_details.update_content(peer_info.audio); + } + if (peer_info.video != null) { + add_video_widgets(); + video_details.update_content(peer_info.video); + } + } + + private void add_video_widgets() { + if (video_added) return; + + video_details.visible = true; + video_added = true; + } + } + + public class CallContentDetails : Gtk.Grid { + + public Label rtp_title = new Label("RTP") { xalign=0, visible=true }; + public Label rtcp_title = new Label("RTCP") { xalign=0, visible=true }; + public Label target_recv_title = new Label("Target receive bitrate") { xalign=0, visible=true }; + public Label target_send_title = new Label("Target send bitrate") { xalign=0, visible=true }; + + public Label rtp_ready = new Label("?") { xalign=0, visible=true }; + public Label rtcp_ready = new Label("?") { xalign=0, visible=true }; + public Label sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label codec = new Label("?") { xalign=0, visible=true }; + public Label target_receive_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + public Label target_send_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + + private PeerContentInfo? prev_info = null; + private int row_at = 0; + + public CallContentDetails(string headline) { + attach(new Label("<b>%s</b>".printf(headline)) { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); + attach(rtp_title, 0, row_at, 1, 1); + attach(rtp_ready, 1, row_at++, 1, 1); + attach(rtcp_title, 0, row_at, 1, 1); + attach(rtcp_ready, 1, row_at++, 1, 1); + put_row("Sent"); + attach(sent_bps, 1, row_at++, 1, 1); + put_row("Received"); + attach(recv_bps, 1, row_at++, 1, 1); + put_row("Codec"); + attach(codec, 1, row_at++, 1, 1); + attach(target_recv_title, 0, row_at, 1, 1); + attach(target_receive_bitrate, 1, row_at++, 1, 1); + attach(target_send_title, 0, row_at, 1, 1); + attach(target_send_bitrate, 1, row_at++, 1, 1); + + this.column_spacing = 5; + } + + public void update_content(PeerContentInfo info) { + if (!info.rtp_ready) { + rtp_ready.visible = rtcp_ready.visible = true; + rtp_title.visible = rtcp_title.visible = true; + rtp_ready.label = info.rtp_ready.to_string(); + rtcp_ready.label = info.rtcp_ready.to_string(); + } else { + rtp_ready.visible = rtcp_ready.visible = false; + rtp_title.visible = rtcp_title.visible = false; + } + if (info.target_send_bytes != -1) { + target_receive_bitrate.visible = target_send_bitrate.visible = true; + target_recv_title.visible = target_send_title.visible = true; + target_receive_bitrate.label = "<span font_family='monospace'>%u</span> kbps".printf(info.target_receive_bytes); + target_send_bitrate.label = "<span font_family='monospace'>%u</span> kbps".printf(info.target_send_bytes); + } else { + target_receive_bitrate.visible = target_send_bitrate.visible = false; + target_recv_title.visible = target_send_title.visible = false; + } + + codec.label = info.codec + " " + info.clockrate.to_string(); + + if (prev_info != null) { + ulong audio_sent_kbps = (info.bytes_sent - prev_info.bytes_sent) * 8 / 1000; + sent_bps.label = "<span font_family='monospace'>%lu</span> kbps".printf(audio_sent_kbps); + ulong audio_recv_kbps = (info.bytes_received - prev_info.bytes_received) * 8 / 1000; + recv_bps.label = "<span font_family='monospace'>%lu</span> kbps".printf(audio_recv_kbps); + } + prev_info = info; + } + + private void put_row(string label) { + attach(new Label(label) { xalign=0, visible=true }, 0, row_at, 1, 1); + } + } +} + diff --git a/main/src/ui/call_window/call_encryption_button.vala b/main/src/ui/call_window/call_encryption_button.vala index 1d785d51..a7081954 100644 --- a/main/src/ui/call_window/call_encryption_button.vala +++ b/main/src/ui/call_window/call_encryption_button.vala @@ -2,19 +2,21 @@ using Dino.Entities; using Gtk; using Pango; -public class Dino.Ui.CallEntryptionButton : MenuButton { +public class Dino.Ui.CallEncryptionButton : MenuButton { - private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; + private Image encryption_image = new Image.from_icon_name("", IconSize.BUTTON) { visible=true }; + private bool has_been_set = false; + public bool controls_active { get; set; default=false; } - construct { + public CallEncryptionButton() { + this.opacity = 0; add(encryption_image); - get_style_context().add_class("encryption-box"); this.set_popover(popover); + + this.notify["controls-active"].connect(update_opacity); } public void set_icon(bool encrypted, string? icon_name) { - this.visible = true; - if (encrypted) { encryption_image.set_from_icon_name(icon_name ?? "changes-prevent-symbolic", IconSize.BUTTON); get_style_context().remove_class("unencrypted"); @@ -22,6 +24,8 @@ public class Dino.Ui.CallEntryptionButton : MenuButton { encryption_image.set_from_icon_name(icon_name ?? "changes-allow-symbolic", IconSize.BUTTON); get_style_context().add_class("unencrypted"); } + has_been_set = true; + update_opacity(); } public void set_info(string? title, bool show_keys, Xmpp.Xep.Jingle.ContentEncryption? audio_encryption, Xmpp.Xep.Jingle.ContentEncryption? video_encryption) { @@ -51,6 +55,10 @@ public class Dino.Ui.CallEntryptionButton : MenuButton { popover.add(box); } + public void update_opacity() { + this.opacity = controls_active && has_been_set ? 1 : 0; + } + private Grid create_media_encryption_grid(Xmpp.Xep.Jingle.ContentEncryption? encryption) { Grid ret = new Grid() { row_spacing=3, column_spacing=5, visible=true }; if (encryption.peer_key.length > 0) { diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala index 3b3d4dc2..c610444f 100644 --- a/main/src/ui/call_window/call_window.vala +++ b/main/src/ui/call_window/call_window.vala @@ -1,87 +1,155 @@ +using Gee; +using Xmpp; using Dino.Entities; using Gtk; namespace Dino.Ui { public class CallWindow : Gtk.Window { - public string counterpart_display_name { get; set; } - // TODO should find another place for this + public signal void menu_dump_dot(); + public CallWindowController controller; public Overlay overlay = new Overlay() { visible=true }; + public Grid grid = new Grid() { visible=true }; public EventBox event_box = new EventBox() { visible=true }; public CallBottomBar bottom_bar = new CallBottomBar() { visible=true }; public Revealer bottom_bar_revealer = new Revealer() { valign=Align.END, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; - public HeaderBar header_bar = new HeaderBar() { show_close_button=true, visible=true }; - public Revealer header_bar_revealer = new Revealer() { valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; - public Stack stack = new Stack() { visible=true }; - public Box own_video_box = new Box(Orientation.HORIZONTAL, 0) { expand=true, visible=true }; + public HeaderBar header_bar = new HeaderBar() { valign=Align.START, halign=Align.END, show_close_button=true, visible=true }; + public Revealer header_bar_revealer = new Revealer() { halign=Align.END, valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; + public Box own_video_box = new Box(Orientation.HORIZONTAL, 0) { halign=Align.END, valign=Align.END, visible=true }; + public Revealer invite_button_revealer = new Revealer() { margin_top=50, margin_right=30, halign=Align.END, valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200 }; + public Button invite_button = new Button.from_icon_name("dino-account-plus") { relief=ReliefStyle.NONE, visible=false }; private Widget? own_video = null; - private Box? own_video_border = new Box(Orientation.HORIZONTAL, 0) { expand=true }; // hack to draw a border around our own video, since we apparently can't draw a border around the Gst widget + private HashMap<string, ParticipantWidget> participant_widgets = new HashMap<string, ParticipantWidget>(); + private ArrayList<string> participants = new ArrayList<string>(); private int own_video_width = 150; private int own_video_height = 100; - private bool hide_controll_elements = false; - private uint hide_controll_handler = 0; - private Widget? main_widget = null; + private bool hide_control_elements = false; + private uint hide_control_handler = 0; + public bool controls_active { get; set; default=true; } construct { + Util.force_css(header_bar, "* { background: none; border: 0; border-radius: 0; }"); header_bar.get_style_context().add_class("call-header-bar"); header_bar_revealer.add(header_bar); + bottom_bar_revealer.add(bottom_bar); + invite_button.get_style_context().add_class("black-element"); + invite_button_revealer.add(invite_button); + own_video_box.get_style_context().add_class("own-video"); this.get_style_context().add_class("dino-call-window"); - bottom_bar_revealer.add(bottom_bar); - + overlay.add(grid); overlay.add_overlay(own_video_box); - overlay.add_overlay(own_video_border); overlay.add_overlay(bottom_bar_revealer); overlay.add_overlay(header_bar_revealer); + overlay.add_overlay(invite_button_revealer); + overlay.get_child_position.connect(on_get_child_position); event_box.add(overlay); add(event_box); - - Util.force_css(own_video_border, "* { border: 1px solid #616161; background-color: transparent; }"); } public CallWindow() { - event_box.events |= Gdk.EventMask.POINTER_MOTION_MASK; - event_box.events |= Gdk.EventMask.ENTER_NOTIFY_MASK; - event_box.events |= Gdk.EventMask.LEAVE_NOTIFY_MASK; - - this.bind_property("counterpart-display-name", header_bar, "title", BindingFlags.SYNC_CREATE); - this.bind_property("counterpart-display-name", bottom_bar, "counterpart-display-name", BindingFlags.SYNC_CREATE); + this.bind_property("controls-active", bottom_bar_revealer, "reveal-child", BindingFlags.SYNC_CREATE); + this.bind_property("controls-active", header_bar_revealer, "reveal-child", BindingFlags.SYNC_CREATE); + this.bind_property("controls-active", invite_button_revealer, "reveal-child", BindingFlags.SYNC_CREATE); event_box.motion_notify_event.connect(reveal_control_elements); event_box.enter_notify_event.connect(reveal_control_elements); event_box.leave_notify_event.connect(reveal_control_elements); this.configure_event.connect(reveal_control_elements); // upon resizing - this.configure_event.connect(update_own_video_position); + + this.configure_event.connect(reposition_participant_widgets); this.set_titlebar(new OutsideHeaderBar(this.header_bar) { visible=true }); reveal_control_elements(); } - public void set_video_fallback(StreamInteractor stream_interactor, Conversation conversation) { - hide_controll_elements = false; + public void add_participant(string participant, ParticipantWidget participant_widget) { + participant_widget.visible = true; + this.bind_property("controls-active", participant_widget, "controls-active", BindingFlags.SYNC_CREATE); + this.bind_property("controls-active", participant_widget.encryption_button, "controls-active", BindingFlags.SYNC_CREATE); + + participants.add(participant); + participant_widgets[participant] = participant_widget; + grid.attach(participant_widget, 0, 0); + + reposition_participant_widgets(); + } + + public void remove_participant(string participant) { + participants.remove(participant); + grid.remove(participant_widgets[participant]); + participant_widgets.unset(participant); - Box box = new Box(Orientation.HORIZONTAL, 0) { visible=true }; - box.get_style_context().add_class("video-placeholder-box"); - AvatarImage avatar = new AvatarImage() { hexpand=true, vexpand=true, halign=Align.CENTER, valign=Align.CENTER, height=100, width=100, visible=true }; - avatar.set_conversation(stream_interactor, conversation); - box.add(avatar); + reposition_participant_widgets(); + } - set_new_main_widget(box); + public void set_video(string participant, Widget widget) { + participant_widgets[participant].set_video(widget); + hide_control_elements = true; + timeout_hide_control_elements(); } - public void set_video(Widget widget) { - hide_controll_elements = true; + public void set_placeholder(string participant, Conversation? conversation, StreamInteractor stream_interactor) { + participant_widgets[participant].set_placeholder(conversation, stream_interactor); + hide_control_elements = false; + foreach (ParticipantWidget participant_widget in participant_widgets.values) { + if (participant_widget.shows_video) { + hide_control_elements = true; + } + } - widget.visible = true; - set_new_main_widget(widget); + if (!hide_control_elements) { + reveal_control_elements(); + } + } + + private bool reposition_participant_widgets() { + int width, height; + this.get_size(out width,out height); + reposition_participant_widgets_rec(participants, width, height, 0, 0, 0, 0); + return false; + } + + private void reposition_participant_widgets_rec(ArrayList<string> participants, int width, int height, int margin_top, int margin_right, int margin_bottom, int margin_left) { + if (participants.size == 0) return; + + if (participants.size == 1) { + participant_widgets[participants[0]].margin_top = margin_top; + participant_widgets[participants[0]].margin_end = margin_right; + participant_widgets[participants[0]].margin_bottom = margin_bottom; + participant_widgets[participants[0]].margin_start = margin_left; + + participant_widgets[participants[0]].on_lowest_row_changed(margin_bottom == 0); + participant_widgets[participants[0]].on_highest_row_changed(margin_top == 0); + return; + } + + ArrayList<string> first_part = new ArrayList<string>(); + ArrayList<string> last_part = new ArrayList<string>(); + + for (int i = 0; i < participants.size; i++) { + if (i < Math.ceil((double)participants.size / (double)2)) { + first_part.add(participants[i]); + } else { + last_part.add(participants[i]); + } + } + + if (width > height) { + reposition_participant_widgets_rec(first_part, width / 2, height, margin_top, margin_right + width / 2, margin_bottom, margin_left); + reposition_participant_widgets_rec(last_part, width / 2, height, margin_top, margin_right, margin_bottom, margin_left + width / 2); + } else { + reposition_participant_widgets_rec(first_part, width, height / 2, margin_top, margin_right, margin_bottom + height / 2, margin_left); + reposition_participant_widgets_rec(last_part, width, height / 2, margin_top + height / 2, margin_right, margin_bottom, margin_left); + } } public void set_own_video(Widget? widget_) { @@ -92,13 +160,7 @@ namespace Dino.Ui { own_video = new Box(Orientation.HORIZONTAL, 0) { expand=true }; } own_video.visible = true; - own_video.width_request = 150; - own_video.height_request = 100; own_video_box.add(own_video); - - own_video_border.visible = true; - - update_own_video_position(); } public void set_own_video_ratio(int width, int height) { @@ -109,78 +171,25 @@ namespace Dino.Ui { this.own_video_width = width * 100 / height; this.own_video_height = 100; } - - own_video.width_request = own_video_width; - own_video.height_request = own_video_height; - - update_own_video_position(); } public void unset_own_video() { own_video_box.foreach((widget) => { own_video_box.remove(widget); }); - - own_video_border.visible = false; - } - - public void set_test_video() { - hide_controll_elements = true; - - var pipeline = new Gst.Pipeline(null); - var src = Gst.ElementFactory.make("videotestsrc", null); - pipeline.add(src); - Gst.Video.Sink sink = (Gst.Video.Sink) Gst.ElementFactory.make("gtksink", null); - Gtk.Widget widget; - sink.get("widget", out widget); - widget.unparent(); - pipeline.add(sink); - src.link(sink); - widget.visible = true; - - pipeline.set_state(Gst.State.PLAYING); - - sink.get_static_pad("sink").notify["caps"].connect(() => { - int width, height; - sink.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); - sink.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); - widget.width_request = width; - widget.height_request = height; - }); - - set_new_main_widget(widget); } - private void set_new_main_widget(Widget widget) { - if (main_widget != null) overlay.remove(main_widget); - overlay.add(widget); - main_widget = widget; + public void set_status(string participant_id, string state) { + participant_widgets[participant_id].set_status(state); } - public void set_status(string state) { - switch (state) { - case "requested": - header_bar.subtitle = _("Calling…"); - break; - case "ringing": - header_bar.subtitle = _("Ringing…"); - break; - case "establishing": - header_bar.subtitle = _("Connecting…"); - break; - default: - header_bar.subtitle = null; - break; - } - } - - public void show_counterpart_ended(string? reason_name, string? reason_text) { - hide_controll_elements = false; + public void show_counterpart_ended(string who_terminated, string? reason_name, string? reason_text) { + hide_control_elements = false; reveal_control_elements(); string text = ""; if (reason_name == Xmpp.Xep.Jingle.ReasonElement.SUCCESS) { - text = _("%s ended the call").printf(counterpart_display_name); + text = _("%s ended the call").printf(who_terminated); } else if (reason_name == Xmpp.Xep.Jingle.ReasonElement.DECLINE || reason_name == Xmpp.Xep.Jingle.ReasonElement.BUSY) { - text = _("%s declined the call").printf(counterpart_display_name); + text = _("%s declined the call").printf(who_terminated); } else { text = "The call has been terminated: " + (reason_name ?? "") + " " + (reason_text ?? ""); } @@ -188,48 +197,53 @@ namespace Dino.Ui { bottom_bar.show_counterpart_ended(text); } - public bool reveal_control_elements() { + private bool reveal_control_elements() { if (!bottom_bar_revealer.child_revealed) { - bottom_bar_revealer.set_reveal_child(true); - header_bar_revealer.set_reveal_child(true); + controls_active = true; } - if (hide_controll_handler != 0) { - Source.remove(hide_controll_handler); - hide_controll_handler = 0; + timeout_hide_control_elements(); + return false; + } + + private void timeout_hide_control_elements() { + if (hide_control_handler != 0) { + Source.remove(hide_control_handler); + hide_control_handler = 0; } - if (!hide_controll_elements) { - return false; + if (!hide_control_elements) { + return; } - hide_controll_handler = Timeout.add_seconds(3, () => { - if (!hide_controll_elements) { + hide_control_handler = Timeout.add_seconds(3, () => { + if (!hide_control_elements) { return false; } if (bottom_bar.is_menu_active()) { - return true; + return false; } - header_bar_revealer.set_reveal_child(false); - bottom_bar_revealer.set_reveal_child(false); - hide_controll_handler = 0; + controls_active = false; + + hide_control_handler = 0; return false; }); - return false; } - private bool update_own_video_position() { - if (own_video == null) return false; - - int width, height; - this.get_size(out width,out height); - - own_video.margin_end = own_video.margin_bottom = own_video_border.margin_end = own_video_border.margin_bottom = 20; - own_video.margin_start = own_video_border.margin_start = width - own_video_width - 20; - own_video.margin_top = own_video_border.margin_top = height - own_video_height - 20; - + private bool on_get_child_position(Widget widget, out Gdk.Rectangle allocation) { + if (widget == own_video_box) { + int width, height; + this.get_size(out width,out height); + + allocation = Gdk.Rectangle(); + allocation.width = own_video_width; + allocation.height = own_video_height; + allocation.x = width - own_video_width - 20; + allocation.y = height - own_video_height - 20; + return true; + } return false; } } diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index b07b41b1..dbf2106c 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -1,3 +1,5 @@ +using Xmpp; +using Gee; using Dino.Entities; using Gtk; @@ -5,103 +7,227 @@ public class Dino.Ui.CallWindowController : Object { private CallWindow call_window; private Call call; - private Conversation conversation; + private CallState call_state; private StreamInteractor stream_interactor; private Calls calls; private Plugins.VideoCallPlugin call_plugin = Dino.Application.get_default().plugin_registry.video_call_plugin; private Plugins.VideoCallWidget? own_video = null; - private Plugins.VideoCallWidget? counterpart_video = null; + private HashMap<string, Plugins.VideoCallWidget> participant_videos = new HashMap<string, Plugins.VideoCallWidget>(); + private HashMap<string, ParticipantWidget> participant_widgets = new HashMap<string, ParticipantWidget>(); + private HashMap<string, PeerState> peer_states = new HashMap<string, PeerState>(); private int window_height = -1; private int window_width = -1; private bool window_size_changed = false; + private ulong[] call_window_handler_ids = new ulong[0]; + private ulong[] bottom_bar_handler_ids = new ulong[0]; + private ulong[] invite_handler_ids = new ulong[0]; - public CallWindowController(CallWindow call_window, Call call, StreamInteractor stream_interactor) { + public CallWindowController(CallWindow call_window, CallState call_state, StreamInteractor stream_interactor) { this.call_window = call_window; - this.call = call; + this.call = call_state.call; + this.call_state = call_state; this.stream_interactor = stream_interactor; this.calls = stream_interactor.get_module(Calls.IDENTITY); - this.conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(call.counterpart.bare_jid, call.account, Conversation.Type.CHAT); this.own_video = call_plugin.create_widget(Plugins.WidgetType.GTK); - this.counterpart_video = call_plugin.create_widget(Plugins.WidgetType.GTK); - call_window.counterpart_display_name = Util.get_conversation_display_name(stream_interactor, conversation); call_window.set_default_size(704, 528); // 640x480 * 1.1 - call_window.set_video_fallback(stream_interactor, conversation); - this.call_window.bottom_bar.video_enabled = calls.should_we_send_video(call); + this.call_window.bottom_bar.video_enabled = call_state.should_we_send_video(); - if (call.direction == Call.DIRECTION_INCOMING) { - call_window.set_status("establishing"); - } else { - call_window.set_status("requested"); + call_state.terminated.connect((who_terminated, reason_name, reason_text) => { + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(who_terminated.bare_jid, call.account, Conversation.Type.CHAT); + string display_name = conversation != null ? Util.get_conversation_display_name(stream_interactor, conversation) : who_terminated.bare_jid.to_string(); + + call_window.show_counterpart_ended(display_name, reason_name, reason_text); + Timeout.add_seconds(3, () => { + call_window.close(); + call_window.destroy(); + + return false; + }); + }); + call_state.peer_joined.connect((jid, peer_state) => { + connect_peer_signals(peer_state); + add_new_participant(peer_state.internal_id, peer_state.jid); + }); + call_state.peer_left.connect((jid, peer_state, reason_name, reason_text) => { + remove_participant(peer_state.internal_id); + }); + + foreach (PeerState peer_state in call_state.peers.values) { + connect_peer_signals(peer_state); + add_new_participant(peer_state.internal_id, peer_state.jid); } - call_window.bottom_bar.hang_up.connect(() => { - calls.end_call(conversation, call); + // Call window signals + + bottom_bar_handler_ids += call_window.bottom_bar.hang_up.connect(() => { + call_state.end(); call_window.close(); call_window.destroy(); this.dispose(); }); - call_window.destroy.connect(() => { - calls.end_call(conversation, call); + call_window_handler_ids += call_window.destroy.connect(() => { + call_state.end(); this.dispose(); }); - - call_window.bottom_bar.notify["audio-enabled"].connect(() => { - calls.mute_own_audio(call, !call_window.bottom_bar.audio_enabled); + bottom_bar_handler_ids += call_window.bottom_bar.notify["audio-enabled"].connect(() => { + call_state.mute_own_audio(!call_window.bottom_bar.audio_enabled); }); - call_window.bottom_bar.notify["video-enabled"].connect(() => { - calls.mute_own_video(call, !call_window.bottom_bar.video_enabled); + bottom_bar_handler_ids += call_window.bottom_bar.notify["video-enabled"].connect(() => { + call_state.mute_own_video(!call_window.bottom_bar.video_enabled); update_own_video(); }); + call_window_handler_ids += call_window.configure_event.connect((event) => { + if (window_width == -1 || window_height == -1) return false; + int current_height = this.call_window.get_allocated_height(); + int current_width = this.call_window.get_allocated_width(); + if (window_width != current_width || window_height != current_height) { + debug("Call window size changed by user. Disabling auto window-to-video size adaptation. %i->%i x %i->%i", window_width, current_width, window_height, current_height); + window_size_changed = true; + } + return false; + }); + call_window_handler_ids += call_window.realize.connect(() => { + capture_window_size(); + }); + invite_handler_ids += call_window.invite_button.clicked.connect(() => { + Gee.List<Account> acc_list = new ArrayList<Account>(Account.equals_func); + acc_list.add(call.account); + SelectContactDialog add_chat_dialog = new SelectContactDialog(stream_interactor, acc_list); + add_chat_dialog.set_transient_for((Window) call_window.get_toplevel()); + add_chat_dialog.title = _("Invite to Call"); + add_chat_dialog.ok_button.label = _("Invite"); + add_chat_dialog.selected.connect((account, jid) => { + call_state.invite_to_call.begin(jid); + }); + add_chat_dialog.present(); + }); - calls.counterpart_sends_video_updated.connect((call, mute) => { + calls.conference_info_received.connect((call, conference_info) => { if (!this.call.equals(call)) return; + var participants = new ArrayList<string>(); + participants.add_all(participant_videos.keys); + foreach (string participant in participants) { + remove_participant(participant); + } + foreach (Jid participant in conference_info.users.keys) { + add_new_participant(participant.to_string(), participant); + } + }); + + own_video.resolution_changed.connect((width, height) => { + if (width == 0 || height == 0) return; + call_window.set_own_video_ratio((int)width, (int)height); + }); + + call_window.menu_dump_dot.connect(() => { call_plugin.dump_dot(); }); + + update_own_video(); + } + + private void connect_peer_signals(PeerState peer_state) { + string peer_id = peer_state.internal_id; + Jid peer_jid = peer_state.jid; + peer_states[peer_id] = peer_state; + + peer_state.connection_ready.connect(() => { + call_window.set_status(peer_state.internal_id, ""); + if (participant_widgets.size == 1) { + // This is the first peer. + // If it can do MUJI, show invite button. + call_window.invite_button_revealer.visible = true; +// stream_interactor.get_module(EntityInfo.IDENTITY).has_feature.begin(call.account, peer_state.jid, Xep.Muji.NS_URI, (_, res) => { +// bool has_feature = stream_interactor.get_module(EntityInfo.IDENTITY).has_feature.end(res); +// call_window.invite_button_revealer.visible = has_feature; +// }); + + call_plugin.devices_changed.connect((media, incoming) => { + if (media == "audio") update_audio_device_choices(); + if (media == "video") update_video_device_choices(); + }); + + update_audio_device_choices(); + update_video_device_choices(); + } else if (participant_widgets.size >= 1) { + call_window.invite_button_revealer.visible = true; + } + }); + peer_state.counterpart_sends_video_updated.connect((mute) => { if (mute) { - call_window.set_video_fallback(stream_interactor, conversation); - counterpart_video.detach(); + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(peer_jid.bare_jid, call.account, Conversation.Type.CHAT); + call_window.set_placeholder(peer_id, conversation, stream_interactor); + participant_videos[peer_id].detach(); } else { - if (!(counterpart_video is Widget)) return; - Widget widget = (Widget) counterpart_video; - call_window.set_video(widget); - counterpart_video.display_stream(calls.get_video_stream(call)); + if (!(participant_videos[peer_id] is Widget)) return; + Widget widget = (Widget) participant_videos[peer_id]; + call_window.set_video(peer_id, widget); + participant_videos[peer_id].display_stream(peer_state.get_video_stream(call), peer_jid); } }); - calls.info_received.connect((call, session_info) => { - if (!this.call.equals(call)) return; + peer_state.info_received.connect((session_info) => { if (session_info == Xmpp.Xep.JingleRtp.CallSessionInfo.RINGING) { - call_window.set_status("ringing"); + call_window.set_status(peer_state.internal_id, "ringing"); } }); - calls.encryption_updated.connect((call, audio_encryption, video_encryption, same) => { - if (!this.call.equals(call)) return; + peer_state.encryption_updated.connect((audio_encryption, video_encryption, same) => { + update_encryption_indicator(participant_widgets[peer_id].encryption_button, audio_encryption, video_encryption, same); + }); + } - string? title = null; - string? icon_name = null; - bool show_keys = true; - Plugins.Registry registry = Dino.Application.get_default().plugin_registry; - Plugins.CallEncryptionEntry? encryption_entry = audio_encryption != null ? registry.call_encryption_entries[audio_encryption.encryption_ns] : null; - if (encryption_entry != null) { - Plugins.CallEncryptionWidget? encryption_widgets = encryption_entry.get_widget(call.account, audio_encryption); - if (encryption_widgets != null) { - title = encryption_widgets.get_title(); - icon_name = encryption_widgets.get_icon_name(); - show_keys = encryption_widgets.show_keys(); - } + private void update_encryption_indicator(CallEncryptionButton encryption_button, Xep.Jingle.ContentEncryption? audio_encryption, Xep.Jingle.ContentEncryption? video_encryption, bool same) { + string? title = null; + string? icon_name = null; + bool show_keys = true; + Plugins.Registry registry = Dino.Application.get_default().plugin_registry; + Plugins.CallEncryptionEntry? encryption_entry = audio_encryption != null ? registry.call_encryption_entries[audio_encryption.encryption_ns] : null; + if (encryption_entry != null) { + Plugins.CallEncryptionWidget? encryption_widgets = encryption_entry.get_widget(call.account, audio_encryption); + if (encryption_widgets != null) { + title = encryption_widgets.get_title(); + icon_name = encryption_widgets.get_icon_name(); + show_keys = encryption_widgets.show_keys(); } - call_window.bottom_bar.encryption_button.set_info(title, show_keys, audio_encryption, same ? null :video_encryption); - call_window.bottom_bar.encryption_button.set_icon(audio_encryption != null, icon_name); - }); + } - own_video.resolution_changed.connect((width, height) => { - if (width == 0 || height == 0) return; - call_window.set_own_video_ratio((int)width, (int)height); + encryption_button.set_info(title, show_keys, audio_encryption, same ? null : video_encryption); + encryption_button.set_icon(audio_encryption != null, icon_name); + } + + private void add_new_participant(string participant_id, Jid jid) { + if (participant_widgets.has_key(participant_id)) { + warning("[%s] Attempted to add same participant twice: %s", call.account.bare_jid.to_string(), jid.to_string()); + return; + } + debug("[%s] Call window controller | Add participant: %s", call.account.bare_jid.to_string(), jid.to_string()); + + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(jid.bare_jid, call.account, Conversation.Type.CHAT); + string participant_name = conversation != null ? Util.get_conversation_display_name(stream_interactor, conversation) : jid.bare_jid.to_string(); + + ParticipantWidget participant_widget = new ParticipantWidget(participant_name); + participant_widget.menu_button.clicked.connect((event) => { + var conn_details_window = new CallConnectionDetailsWindow() { title=participant_name, visible=true }; + conn_details_window.update_content(peer_states[participant_id].get_info()); + uint timeout_handle_id = Timeout.add_seconds(1, () => { + conn_details_window.update_content(peer_states[participant_id].get_info()); + return true; + }); + conn_details_window.set_transient_for(call_window); + conn_details_window.destroy.connect(() => Source.remove(timeout_handle_id)); + conn_details_window.present(); + this.call_window.destroy.connect(() => conn_details_window.close() ); }); - counterpart_video.resolution_changed.connect((width, height) => { - if (window_size_changed) return; + participant_widgets[participant_id] = participant_widget; + + call_window.add_participant(participant_id, participant_widget); + + participant_videos[participant_id] = call_plugin.create_widget(Plugins.WidgetType.GTK); + + participant_videos[participant_id].resolution_changed.connect((width, height) => { + if (window_size_changed || participant_widgets.size > 1) return; if (width == 0 || height == 0) return; if (width > height) { call_window.resize(704, (int) (height * 704 / width)); @@ -110,24 +236,22 @@ public class Dino.Ui.CallWindowController : Object { } capture_window_size(); }); - call_window.configure_event.connect((event) => { - if (window_width == -1 || window_height == -1) return false; - int current_height = this.call_window.get_allocated_height(); - int current_width = this.call_window.get_allocated_width(); - if (window_width != current_width || window_height != current_height) { - debug("Call window size changed by user. Disabling auto window-to-video size adaptation. %i->%i x %i->%i", window_width, current_width, window_height, current_height); - window_size_changed = true; - } - return false; - }); - call_window.realize.connect(() => { - capture_window_size(); - }); - call.notify["state"].connect(on_call_state_changed); - calls.call_terminated.connect(on_call_terminated); + participant_widget.set_placeholder(conversation, stream_interactor); + if (call.direction == Call.DIRECTION_INCOMING) { + call_window.set_status(participant_id, "establishing"); + } else { + call_window.set_status(participant_id, "requested"); + } + } - update_own_video(); + private void remove_participant(string participant_id) { + if (peer_states.has_key(participant_id)) debug(@"[%s] Call window controller | Remove participant: %s", call.account.bare_jid.to_string(), peer_states[participant_id].jid.to_string()); + + participant_videos.unset(participant_id); + participant_widgets.unset(participant_id); + peer_states.unset(participant_id); + call_window.remove_participant(participant_id); } private void capture_window_size() { @@ -137,33 +261,6 @@ public class Dino.Ui.CallWindowController : Object { this.window_width = this.call_window.get_allocated_width(); } - private void on_call_state_changed() { - if (call.state == Call.State.IN_PROGRESS) { - call_window.set_status(""); - call_plugin.devices_changed.connect((media, incoming) => { - if (media == "audio") update_audio_device_choices(); - if (media == "video") update_video_device_choices(); - }); - - update_audio_device_choices(); - update_video_device_choices(); - } - } - - private void on_call_terminated(Call call, string? reason_name, string? reason_text) { - call_window.show_counterpart_ended(reason_name, reason_text); - Timeout.add_seconds(3, () => { - call.notify["state"].disconnect(on_call_state_changed); - calls.call_terminated.disconnect(on_call_terminated); - - - call_window.close(); - call_window.destroy(); - - return false; - }); - } - private void update_audio_device_choices() { if (call_plugin.get_devices("audio", true).size == 0 || call_plugin.get_devices("audio", false).size == 0) { call_window.bottom_bar.show_audio_device_error(); @@ -190,13 +287,13 @@ public class Dino.Ui.CallWindowController : Object { });*/ } - private void update_current_audio_device(AudioSettingsPopover audio_settings_popover) { + /*private void update_current_audio_device(AudioSettingsPopover audio_settings_popover) { Xmpp.Xep.JingleRtp.Stream stream = calls.get_audio_stream(call); if (stream != null) { audio_settings_popover.current_microphone_device = call_plugin.get_device(stream, false); audio_settings_popover.current_speaker_device = call_plugin.get_device(stream, true); } - } + }*/ private void update_video_device_choices() { int device_count = call_plugin.get_devices("video", false).size; @@ -223,12 +320,37 @@ public class Dino.Ui.CallWindowController : Object { });*/ } - private void update_current_video_device(VideoSettingsPopover video_settings_popover) { + public void add_test_video() { + var pipeline = new Gst.Pipeline(null); + var src = Gst.ElementFactory.make("videotestsrc", null); + pipeline.add(src); + Gst.Video.Sink sink = (Gst.Video.Sink) Gst.ElementFactory.make("gtksink", null); + Gtk.Widget widget; + sink.get("widget", out widget); + widget.unparent(); + pipeline.add(sink); + src.link(sink); + widget.visible = true; + + pipeline.set_state(Gst.State.PLAYING); + + sink.get_static_pad("sink").notify["caps"].connect(() => { + int width, height; + sink.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + sink.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + widget.width_request = width; + widget.height_request = height; + }); + +// call_window.set_participant_video(Xmpp.random_uuid(), widget); + } + + /*private void update_current_video_device(VideoSettingsPopover video_settings_popover) { Xmpp.Xep.JingleRtp.Stream stream = calls.get_video_stream(call); if (stream != null) { video_settings_popover.current_device = call_plugin.get_device(stream, false); } - } + }*/ private void update_own_video() { if (this.call_window.bottom_bar.video_enabled) { @@ -247,8 +369,12 @@ public class Dino.Ui.CallWindowController : Object { } public override void dispose() { + foreach (ulong handler_id in call_window_handler_ids) call_window.disconnect(handler_id); + foreach (ulong handler_id in bottom_bar_handler_ids) call_window.bottom_bar.disconnect(handler_id); + foreach (ulong handler_id in invite_handler_ids) call_window.invite_button.disconnect(handler_id); + + call_window_handler_ids = bottom_bar_handler_ids = invite_handler_ids = new ulong[0]; + base.dispose(); - call.notify["state"].disconnect(on_call_state_changed); - calls.call_terminated.disconnect(on_call_terminated); } }
\ No newline at end of file diff --git a/main/src/ui/call_window/participant_widget.vala b/main/src/ui/call_window/participant_widget.vala new file mode 100644 index 00000000..cbf8df2d --- /dev/null +++ b/main/src/ui/call_window/participant_widget.vala @@ -0,0 +1,129 @@ +using Pango; +using Gee; +using Xmpp; +using Dino.Entities; +using Gtk; + +namespace Dino.Ui { + + public class ParticipantWidget : Gtk.Overlay { + + public Widget main_widget; + public Box outer_box = new Box(Orientation.HORIZONTAL, 0) { valign=Align.START, visible=true }; + public Box inner_box = new Box(Orientation.HORIZONTAL, 0) { margin_start=5, margin_top=5, hexpand=true, visible=true }; + public Box title_box = new Box(Orientation.VERTICAL, 0) { valign=Align.CENTER, hexpand=true, visible=true }; + public CallEncryptionButton encryption_button = new CallEncryptionButton() { opacity=0, relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_end=5, visible=true }; + public Label status_label = new Label("") { ellipsize=EllipsizeMode.MIDDLE }; + public Label name_label = new Label("") { ellipsize=EllipsizeMode.MIDDLE, visible=true }; + public Button menu_button = new Button.from_icon_name("view-more-horizontal-symbolic") { relief=ReliefStyle.NONE, visible=true }; + public bool shows_video = false; + public string? participant_name; + + bool is_highest_row = false; + bool is_lowest_row = false; + public bool controls_active { get; set; } + + public ParticipantWidget(string participant_name) { + this.participant_name = participant_name; + name_label.label = participant_name; + + name_label.attributes = new AttrList(); + name_label.attributes.filter((attr) => attr.equal(attr_weight_new(Weight.BOLD))); + + name_label.attributes = new AttrList(); + name_label.attributes.filter((attr) => attr.equal(attr_scale_new(0.9))); + status_label.get_style_context().add_class("dim-label"); + + Util.force_css(outer_box, "* { color: white; text-shadow: 1px 1px black; }"); + menu_button.get_style_context().add_class("participant-title-button"); + encryption_button.get_style_context().add_class("participant-title-button"); + + title_box.add(name_label); + title_box.add(status_label); + + outer_box.add(inner_box); + + inner_box.add(menu_button); + inner_box.add(encryption_button); + inner_box.add(title_box); + inner_box.add(new Button.from_icon_name("go-up-symbolic") { opacity=0, visible=true }); + inner_box.add(new Button.from_icon_name("go-up-symbolic") { opacity=0, visible=true }); + + this.add_overlay(outer_box); + + this.notify["controls-active"].connect(reveal_or_hide_controls); + } + + public void on_show_names_changed(bool show) { + name_label.visible = show; + reveal_or_hide_controls(); + } + + public void on_highest_row_changed(bool is_highest) { + is_highest_row = is_highest; + reveal_or_hide_controls(); + } + + public void on_lowest_row_changed(bool is_lowest) { + is_lowest_row = is_lowest; + reveal_or_hide_controls(); + } + + public void set_video(Widget widget) { + shows_video = true; + widget.visible = true; + set_participant_widget(widget); + } + + public void set_placeholder(Conversation? conversation, StreamInteractor stream_interactor) { + shows_video = false; + Box box = new Box(Orientation.HORIZONTAL, 0) { visible=true }; + box.get_style_context().add_class("video-placeholder-box"); + AvatarImage avatar = new AvatarImage() { allow_gray=false, hexpand=true, vexpand=true, halign=Align.CENTER, valign=Align.CENTER, height=100, width=100, visible=true }; + if (conversation != null) { + avatar.set_conversation(stream_interactor, conversation); + } else { + avatar.set_text("?", false); + } + box.add(avatar); + + set_participant_widget(box); + } + + private void set_participant_widget(Widget widget) { + widget.expand = true; + if (main_widget != null) this.remove(main_widget); + main_widget = widget; + this.add(main_widget); + } + + public void set_status(string state) { + status_label.visible = true; + + if (state == "requested") { + status_label.label = _("Calling…"); + } else if (state == "ringing") { + status_label.label = _("Ringing…"); + } else if (state == "establishing") { + status_label.label = _("Connecting…"); + } else { + status_label.visible = false; + } + } + + private void reveal_or_hide_controls() { + if (controls_active && name_label.visible) { + title_box.opacity = 1; + menu_button.opacity = 1; + } else { + title_box.opacity = 0; + menu_button.opacity = 0; + } + if (is_highest_row && controls_active) { + outer_box.get_style_context().add_class("call-header-bar"); + } else { + outer_box.get_style_context().remove_class("call-header-bar"); + } + } + } +}
\ No newline at end of file diff --git a/main/src/ui/contact_details/muc_config_form_provider.vala b/main/src/ui/contact_details/muc_config_form_provider.vala index f9f8d7e9..5b4184c5 100644 --- a/main/src/ui/contact_details/muc_config_form_provider.vala +++ b/main/src/ui/contact_details/muc_config_form_provider.vala @@ -33,7 +33,7 @@ public class MucConfigFormProvider : Plugins.ContactDetailsProvider, Object { contact_details.save.connect(() => { // Only send the config form if something was changed if (config_backup != data_form.stanza_node.to_string()) { - stream_interactor.get_module(MucManager.IDENTITY).set_config_form(conversation.account, conversation.counterpart, data_form); + stream_interactor.get_module(MucManager.IDENTITY).set_config_form.begin(conversation.account, conversation.counterpart, data_form); } }); }); diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index 62156761..a2c8c0c2 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -2,6 +2,7 @@ using Gee; using Gdk; using Gtk; using Pango; +using Xmpp; using Dino.Entities; @@ -18,7 +19,8 @@ namespace Dino.Ui { public override Object? get_widget(Plugins.WidgetType type) { CallItem call_item = content_item as CallItem; - return new CallWidget(stream_interactor, call_item.call, call_item.conversation) { visible=true }; + CallState? call_state = stream_interactor.get_module(Calls.IDENTITY).call_states[call_item.call]; + return new CallWidget(stream_interactor, call_item.call, call_state, call_item.conversation) { visible=true }; } public override Gee.List<Plugins.MessageAction>? get_item_actions(Plugins.WidgetType type) { return null; } @@ -31,10 +33,14 @@ namespace Dino.Ui { [GtkChild] public unowned Label title_label; [GtkChild] public unowned Label subtitle_label; [GtkChild] public unowned Revealer incoming_call_revealer; + [GtkChild] public unowned Box outer_additional_box; + [GtkChild] public unowned Box incoming_call_box; + [GtkChild] public unowned Box multiparty_peer_box; [GtkChild] public unowned Button accept_call_button; [GtkChild] public unowned Button reject_call_button; private StreamInteractor stream_interactor; + private CallState call_manager; private Call call; private Conversation conversation; public Call.State call_state { get; set; } // needs to be public for binding @@ -45,8 +51,10 @@ namespace Dino.Ui { size_request_mode = SizeRequestMode.HEIGHT_FOR_WIDTH; } - public CallWidget(StreamInteractor stream_interactor, Call call, Conversation conversation) { + /** @param call_state Null if it's an old call and we can't interact with it anymore */ + public CallWidget(StreamInteractor stream_interactor, Call call, CallState? call_state, Conversation conversation) { this.stream_interactor = stream_interactor; + this.call_manager = call_state; this.call = call; this.conversation = conversation; @@ -57,36 +65,63 @@ namespace Dino.Ui { }); call.bind_property("state", this, "call-state"); - this.notify["call-state"].connect(update_widget); + this.notify["call-state"].connect(update_call_state); + + if (call_manager != null && (call.state == Call.State.ESTABLISHING || call.state == Call.State.IN_PROGRESS)) { + call_manager.peer_joined.connect(update_counterparts); + } accept_call_button.clicked.connect(() => { - stream_interactor.get_module(Calls.IDENTITY).accept_call(call); + call_manager.accept(); var call_window = new CallWindow(); - call_window.controller = new CallWindowController(call_window, call, stream_interactor); + call_window.controller = new CallWindowController(call_window, call_state, stream_interactor); call_window.present(); }); - reject_call_button.clicked.connect(() => { - stream_interactor.get_module(Calls.IDENTITY).reject_call(call); - }); + reject_call_button.clicked.connect(call_manager.reject); + + update_call_state(); + } + + private void update_counterparts() { + if (call.state != Call.State.IN_PROGRESS && call.state != Call.State.ENDED) return; + if (call.counterparts.size <= 1 && conversation.type_ == Conversation.Type.CHAT) return; - update_widget(); + multiparty_peer_box.foreach((widget) => { multiparty_peer_box.remove(widget); }); + + foreach (Jid counterpart in call.counterparts) { + AvatarImage image = new AvatarImage() { force_gray=true, margin_top=2, visible=true }; + image.set_conversation_participant(stream_interactor, conversation, counterpart.bare_jid); + multiparty_peer_box.add(image); + } + AvatarImage image2 = new AvatarImage() { force_gray=true, margin_top=2, visible=true }; + image2.set_conversation_participant(stream_interactor, conversation, call.account.bare_jid); + multiparty_peer_box.add(image2); + + outer_additional_box.get_style_context().add_class("multiparty-participants"); + + multiparty_peer_box.visible = true; + incoming_call_box.visible = false; + incoming_call_revealer.reveal_child = true; } - private void update_widget() { + private void update_call_state() { incoming_call_revealer.reveal_child = false; incoming_call_revealer.get_style_context().remove_class("incoming"); + outer_additional_box.get_style_context().remove_class("incoming-call-box"); switch (call.state) { case Call.State.RINGING: image.set_from_icon_name("dino-phone-ring-symbolic", IconSize.LARGE_TOOLBAR); if (call.direction == Call.DIRECTION_INCOMING) { - bool video = stream_interactor.get_module(Calls.IDENTITY).should_we_send_video(call); + bool video = call_manager.should_we_send_video(); title_label.label = video ? _("Incoming video call") : _("Incoming call"); subtitle_label.label = "Ring ring…!"; + incoming_call_box.visible = true; incoming_call_revealer.reveal_child = true; incoming_call_revealer.get_style_context().add_class("incoming"); + outer_additional_box.get_style_context().add_class("incoming-call-box"); } else { title_label.label = _("Calling…"); subtitle_label.label = "Ring ring…?"; @@ -100,14 +135,16 @@ namespace Dino.Ui { subtitle_label.label = _("Started %s ago").printf(duration); time_update_handler_id = Timeout.add_seconds(get_next_time_change() + 1, () => { - Source.remove(time_update_handler_id); - time_update_handler_id = 0; - update_widget(); + if (time_update_handler_id != 0) { + Source.remove(time_update_handler_id); + time_update_handler_id = 0; + update_call_state(); + } return true; }); break; - case Call.State.OTHER_DEVICE_ACCEPTED: + case Call.State.OTHER_DEVICE: image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); title_label.label = call.direction == Call.DIRECTION_INCOMING ? _("Incoming call") : _("Outgoing call"); subtitle_label.label = _("You handled this call on another device"); @@ -128,7 +165,7 @@ namespace Dino.Ui { if (call.direction == Call.DIRECTION_INCOMING) { subtitle_label.label = _("You missed this call"); } else { - string who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + string who = Util.get_conversation_display_name(stream_interactor, conversation); subtitle_label.label = _("%s missed this call").printf(who); } break; @@ -138,7 +175,7 @@ namespace Dino.Ui { if (call.direction == Call.DIRECTION_INCOMING) { subtitle_label.label = _("You declined this call"); } else { - string who = Util.get_participant_display_name(stream_interactor, conversation, call.to); + string who = Util.get_conversation_display_name(stream_interactor, conversation); subtitle_label.label = _("%s declined this call").printf(who); } break; @@ -148,6 +185,8 @@ namespace Dino.Ui { subtitle_label.label = "Call failed to establish"; break; } + + update_counterparts(); } private string get_duration_string(TimeSpan duration) { @@ -201,6 +240,9 @@ namespace Dino.Ui { Source.remove(time_update_handler_id); time_update_handler_id = 0; } + if (call_manager != null) { + call_manager.peer_joined.disconnect(update_counterparts); + } } } } diff --git a/main/src/ui/conversation_content_view/conversation_item_skeleton.vala b/main/src/ui/conversation_content_view/conversation_item_skeleton.vala index b356d8cc..73f7c671 100644 --- a/main/src/ui/conversation_content_view/conversation_item_skeleton.vala +++ b/main/src/ui/conversation_content_view/conversation_item_skeleton.vala @@ -114,7 +114,6 @@ public class ConversationItemSkeleton : EventBox { [GtkTemplate (ui = "/im/dino/Dino/conversation_content_view/item_metadata_header.ui")] public class ItemMetaDataHeader : Box { [GtkChild] public unowned Label name_label; - [GtkChild] public unowned Label dot_label; [GtkChild] public unowned Label time_label; public Image received_image = new Image() { opacity=0.4 }; public Widget? encryption_image = null; diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index 4bf946d2..3b3a5b39 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -47,16 +47,16 @@ namespace Dino.Ui { Box box = new Box(Orientation.VERTICAL, 0) { margin=10, visible=true }; audio_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, false, (_, res) => { - Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); - open_call_window(call); + CallState call_state = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call_state); }); }); box.add(audio_button); video_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, true, (_, res) => { - Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); - open_call_window(call); + CallState call_state = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); + open_call_window(call_state); }); }); box.add(video_button); @@ -68,7 +68,7 @@ namespace Dino.Ui { popover_menu.visible = true; }); - stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect((call, conversation) => { + stream_interactor.get_module(Calls.IDENTITY).call_incoming.connect((call, state,conversation) => { update_button_state(); }); @@ -76,6 +76,7 @@ namespace Dino.Ui { update_button_state(); }); stream_interactor.get_module(PresenceManager.IDENTITY).show_received.connect((jid, account) => { + if (this.conversation == null) return; if (this.conversation.counterpart.equals_bare(jid) && this.conversation.account.equals(account)) { update_visibility.begin(); } @@ -83,11 +84,14 @@ namespace Dino.Ui { stream_interactor.connection_manager.connection_state_changed.connect((account, state) => { update_visibility.begin(); }); + Dino.Application.get_default().plugin_registry.video_call_plugin.devices_changed.connect((media, incoming) => { + update_visibility.begin(); + }); } - private void open_call_window(Call call) { + private void open_call_window(CallState call_state) { var call_window = new CallWindow(); - var call_controller = new CallWindowController(call_window, call, stream_interactor); + var call_controller = new CallWindowController(call_window, call_state, stream_interactor); call_window.controller = call_controller; call_window.present(); @@ -102,17 +106,15 @@ namespace Dino.Ui { } private void update_button_state() { - Jid? call_counterpart = stream_interactor.get_module(Calls.IDENTITY).is_call_in_progress(); - this.sensitive = call_counterpart == null; - - if (call_counterpart != null && call_counterpart.equals_bare(conversation.counterpart)) { - this.set_image(new Gtk.Image.from_icon_name("dino-phone-in-talk-symbolic", Gtk.IconSize.MENU) { visible=true }); - } else { - this.set_image(new Gtk.Image.from_icon_name("dino-phone-symbolic", Gtk.IconSize.MENU) { visible=true }); - } + this.sensitive = !stream_interactor.get_module(Calls.IDENTITY).is_call_in_progress(); } private async void update_visibility() { + if (conversation == null) { + visible = false; + return; + } + if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); diff --git a/main/src/ui/notifier_freedesktop.vala b/main/src/ui/notifier_freedesktop.vala index 48313be3..e8e2ba1d 100644 --- a/main/src/ui/notifier_freedesktop.vala +++ b/main/src/ui/notifier_freedesktop.vala @@ -123,6 +123,7 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { } public async void notify_call(Call call, Conversation conversation, bool video, string conversation_display_name) { + debug("[%s] Call notification", call.account.bare_jid.to_string()); string summary = Markup.escape_text(conversation_display_name); string body = video ? _("Incoming video call") : _("Incoming call"); @@ -140,10 +141,12 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { GLib.Application.get_default().activate_action("open-conversation", new Variant.int32(conversation.id)); }); add_action_listener(notification_id, "reject", () => { - GLib.Application.get_default().activate_action("deny-call", new Variant.int32(call.id)); + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.int32(call.id)}); + GLib.Application.get_default().activate_action("reject-call", variant); }); add_action_listener(notification_id, "accept", () => { - GLib.Application.get_default().activate_action("accept-call", new Variant.int32(call.id)); + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.int32(call.id)}); + GLib.Application.get_default().activate_action("accept-call", variant); }); } catch (Error e) { warning("Failed showing subscription request notification: %s", e.message); diff --git a/main/src/ui/notifier_gnotifications.vala b/main/src/ui/notifier_gnotifications.vala index 5fd3be4b..665d47c4 100644 --- a/main/src/ui/notifier_gnotifications.vala +++ b/main/src/ui/notifier_gnotifications.vala @@ -74,7 +74,7 @@ namespace Dino.Ui { notification.set_icon(new ThemedIcon.from_names(new string[] {"call-start-symbolic"})); notification.set_default_action_and_target_value("app.open-conversation", new Variant.int32(conversation.id)); - notification.add_button_with_target_value(_("Deny"), "app.deny-call", new Variant.int32(call.id)); + notification.add_button_with_target_value(_("Reject"), "app.reject-call", new Variant.int32(call.id)); notification.add_button_with_target_value(_("Accept"), "app.accept-call", new Variant.int32(call.id)); GLib.Application.get_default().send_notification(call.id.to_string(), notification); diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index ca398e69..c5b2eb75 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -37,40 +37,30 @@ public class Handler { this.own_fingerprint = creds.own_fingerprint; } - public uint8[]? process_incoming_data(uint component_id, uint8[] data) { + public uint8[]? process_incoming_data(uint component_id, uint8[] data) throws Crypto.Error { if (srtp_session.has_decrypt) { - try { - if (component_id == 1) { - if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { - return srtp_session.decrypt_rtcp(data); - } - return srtp_session.decrypt_rtp(data); + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.decrypt_rtcp(data); } - if (component_id == 2) return srtp_session.decrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return null; + return srtp_session.decrypt_rtp(data); } + if (component_id == 2) return srtp_session.decrypt_rtcp(data); } else if (component_id == 1) { on_data_rec(data); } return null; } - public uint8[]? process_outgoing_data(uint component_id, uint8[] data) { + public uint8[]? process_outgoing_data(uint component_id, uint8[] data) throws Crypto.Error { if (srtp_session.has_encrypt) { - try { - if (component_id == 1) { - if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { - return srtp_session.encrypt_rtcp(data); - } - return srtp_session.encrypt_rtp(data); + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.encrypt_rtcp(data); } - if (component_id == 2) return srtp_session.encrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return null; + return srtp_session.encrypt_rtp(data); } + if (component_id == 2) return srtp_session.encrypt_rtcp(data); } return null; } @@ -122,6 +112,19 @@ public class Handler { } public async Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection() { + MainContext context = MainContext.current_source().get_context(); + var thread = new Thread<Xmpp.Xep.Jingle.ContentEncryption>("dtls-connection", () => { + var res = setup_dtls_connection_thread(); + Source source = new IdleSource(); + source.set_callback(setup_dtls_connection.callback); + source.attach(context); + return res; + }); + yield; + return thread.join(); + } + + private Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection_thread() { buffer_mutex.lock(); if (stop) { restart = true; @@ -156,28 +159,23 @@ public class Handler { session.set_push_function(push_function); session.set_verify_function(verify_function); - Thread<int> thread = new Thread<int> (null, () => { - DateTime maximum_time = new DateTime.now_utc().add_seconds(20); - do { - err = session.handshake(); + DateTime maximum_time = new DateTime.now_utc().add_seconds(20); + do { + err = session.handshake(); + + DateTime current_time = new DateTime.now_utc(); + if (maximum_time.compare(current_time) < 0) { + warning("DTLS handshake timeouted"); + err = ErrorCode.APPLICATION_ERROR_MIN + 1; + break; + } + if (stop) { + debug("DTLS handshake stopped"); + err = ErrorCode.APPLICATION_ERROR_MIN + 2; + break; + } + } while (err < 0 && !((ErrorCode)err).is_fatal()); - DateTime current_time = new DateTime.now_utc(); - if (maximum_time.compare(current_time) < 0) { - warning("DTLS handshake timeouted"); - err = ErrorCode.APPLICATION_ERROR_MIN + 1; - break; - } - if (stop) { - debug("DTLS handshake stopped"); - err = ErrorCode.APPLICATION_ERROR_MIN + 2; - break; - } - } while (err < 0 && !((ErrorCode)err).is_fatal()); - Idle.add(setup_dtls_connection.callback); - return err; - }); - yield; - err = thread.join(); buffer_mutex.lock(); if (stop) { stop = false; @@ -186,7 +184,7 @@ public class Handler { buffer_mutex.unlock(); if (restart) { debug("Restarting DTLS handshake"); - return yield setup_dtls_connection(); + return setup_dtls_connection_thread(); } return null; } diff --git a/plugins/ice/src/plugin.vala b/plugins/ice/src/plugin.vala index f145dd6d..4abf042c 100644 --- a/plugins/ice/src/plugin.vala +++ b/plugins/ice/src/plugin.vala @@ -15,7 +15,12 @@ public class Dino.Plugins.Ice.Plugin : RootInterface, Object { list.add(new Module()); }); app.stream_interactor.stream_attached_modules.connect((account, stream) => { - stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + if (stream.get_module(Socks5Bytestreams.Module.IDENTITY) != null) { + stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + } + if (stream.get_module(JingleRawUdp.Module.IDENTITY) != null) { + stream.get_module(JingleRawUdp.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + } }); app.stream_interactor.stream_negotiated.connect(on_stream_negotiated); } diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 62c04906..f684e411 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -10,16 +10,14 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private bool remote_credentials_set; private Map<uint8, DatagramConnection> connections = new HashMap<uint8, DatagramConnection>(); private DtlsSrtp.Handler? dtls_srtp_handler; + private MainContext thread_context; + private MainLoop thread_loop; private class DatagramConnection : Jingle.DatagramConnection { private Nice.Agent agent; private DtlsSrtp.Handler? dtls_srtp_handler; private uint stream_id; private string? error; - private ulong sent; - private ulong sent_reported; - private ulong recv; - private ulong recv_reported; private ulong datagram_received_id; public DatagramConnection(Nice.Agent agent, DtlsSrtp.Handler? dtls_srtp_handler, uint stream_id, uint8 component_id) { @@ -28,11 +26,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport this.stream_id = stream_id; this.component_id = component_id; this.datagram_received_id = this.datagram_received.connect((datagram) => { - recv += datagram.length; - if (recv > recv_reported + 100000) { - debug("Received %lu bytes via stream %u component %u", recv, stream_id, component_id); - recv_reported = recv; - } + bytes_received += datagram.length; }); } @@ -45,16 +39,22 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport public override void send_datagram(Bytes datagram) { if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { - uint8[] encrypted_data = null; - if (dtls_srtp_handler != null) { - encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); - if (encrypted_data == null) return; - } - agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); - sent += datagram.length; - if (sent > sent_reported + 100000) { - debug("Sent %lu bytes via stream %u component %u", sent, stream_id, component_id); - sent_reported = sent; + try { + if (dtls_srtp_handler != null) { + uint8[] encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); + if (encrypted_data == null) return; + // TODO: Nonblocking might require certain libnice versions? + GLib.OutputVector[] vectors = {{ encrypted_data, encrypted_data.length }}; + Nice.OutputMessage[] messages = {{ vectors }}; + agent.send_messages_nonblocking(stream_id, component_id, messages); + } else { + GLib.OutputVector[] vectors = {{ datagram.get_data(), datagram.get_size() }}; + Nice.OutputMessage[] messages = {{ vectors }}; + agent.send_messages_nonblocking(stream_id, component_id, messages); + } + bytes_sent += datagram.length; + } catch (GLib.Error e) { + warning("%s while send_datagram stream %u component %u", e.message, stream_id, component_id); } } } @@ -93,6 +93,14 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent.controlling_mode = !incoming; stream_id = agent.add_stream(components); + thread_context = new MainContext(); + new Thread<void*>(@"ice-thread-$stream_id", () => { + thread_context.push_thread_default(); + thread_loop = new MainLoop(thread_context, false); + thread_loop.run(); + thread_context.pop_thread_default(); + return null; + }); if (turn_ip != null) { for (uint8 component_id = 1; component_id <= components; component_id++) { @@ -107,7 +115,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport for (uint8 component_id = 1; component_id <= components; component_id++) { // We don't properly get local candidates before this call - agent.attach_recv(stream_id, component_id, MainContext.@default(), on_recv); + agent.attach_recv(stream_id, component_id, thread_context, on_recv); } agent.gather_candidates(stream_id); @@ -260,8 +268,13 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (stream_id != this.stream_id) return; uint8[] decrypt_data = null; if (dtls_srtp_handler != null) { - decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); - if (decrypt_data == null) return; + try { + decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); + if (decrypt_data == null) return; + } catch (Crypto.Error e) { + warning("%s while on_recv stream %u component %u", e.message, stream_id, component_id); + return; + } } may_consider_ready(stream_id, component_id); if (connections.has_key((uint8) component_id)) { @@ -341,5 +354,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent = null; dtls_srtp_handler = null; connections.clear(); + if (thread_loop != null) { + thread_loop.quit(); + } } } diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 3264e24a..fa4f367c 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -16,6 +16,10 @@ if(GstRtp_VERSION VERSION_GREATER "1.16") set(RTP_DEFINITIONS GST_1_16) endif() +if(Vala_VERSION VERSION_GREATER "0.50") + set(RTP_DEFINITIONS VALA_0_50) +endif() + if(WebRTCAudioProcessing_VERSION GREATER "0.4") message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") unset(WebRTCAudioProcessing_FOUND) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 417dc4be..6fb5e7aa 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -97,7 +97,7 @@ public class Dino.Plugins.Rtp.CodecUtil { case "h264": return new string[] {/*"msdkh264enc", */"vaapih264enc", "x264enc"}; case "vp9": - return new string[] {/*"msdkvp9enc", */"vaapivp9enc" /*, "vp9enc" */}; + return new string[] {/*"msdkvp9enc", */"vaapivp9enc", "vp9enc"}; case "vp8": return new string[] {/*"msdkvp8enc", */"vaapivp8enc", "vp8enc"}; } @@ -140,13 +140,16 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 if (encode == "msdkh264enc") return @" rate-control=vbr"; - if (encode == "vaapih264enc") return @" tune=low-power"; - if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; + if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency bframes=0 cabac=false dct8x8=false"; // VP8 - if (encode == "msdkvp8enc") return " rate-control=vbr"; - if (encode == "vaapivp8enc") return " rate-control=vbr"; - if (encode == "vp8enc") return " deadline=1 error-resilient=1"; + if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; + + // VP9 + if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; // OPUS if (encode == "opusenc") { @@ -159,7 +162,8 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 - if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse"; + if (media == "video" && codec == "h264") return " ! capsfilter caps=video/x-h264,profile=constrained-baseline ! h264parse"; + if (media == "video" && codec == "vp8" && encode == "vp8enc") return " ! capsfilter caps=video/x-vp8,profile=(string)1"; return null; } @@ -171,26 +175,43 @@ public class Dino.Plugins.Rtp.CodecUtil { if (encode_name == null) return 0; Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); - bitrate = uint.min(2048000, bitrate); - switch (encode_name) { case "msdkh264enc": case "vaapih264enc": case "x264enc": + case "msdkvp9enc": + case "vaapivp9enc": case "msdkvp8enc": case "vaapivp8enc": bitrate = uint.min(2048000, bitrate); encode.set("bitrate", bitrate); return bitrate; + case "vp9enc": case "vp8enc": bitrate = uint.min(2147483, bitrate); - encode.set("target-bitrate", bitrate * 1000); + encode.set("target-bitrate", bitrate * 1024); return bitrate; } return 0; } + public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + rescale_caps.set("caps", caps); + } + + public Gst.Caps? get_rescale_caps(Gst.Element encode_element) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return null; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + Gst.Caps caps; + rescale_caps.get("caps", out caps); + return caps; + } + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { return null; } @@ -198,6 +219,7 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + if (decode == "vp8dec" || decode == "vp9dec") return " threads=8"; return null; } @@ -242,6 +264,7 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_decode_element_name(string media, string? codec) { + if (codec == "vp9") return null; // Temporary unsupport VP9 if (get_depay_element_name(media, codec) == null) return null; foreach (string candidate in get_decode_candidates(media, codec)) { if (is_element_supported(candidate)) return candidate; @@ -270,7 +293,7 @@ public class Dino.Plugins.Rtp.CodecUtil { string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; + return @"queue ! $depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -285,16 +308,29 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + string? desc1 = get_encode_bin_without_payloader_description(media, codec, payload_type, element_name, name); + string? desc2 = get_payloader_bin_description(media, codec, payload_type, name); + return @"$desc1 ! $desc2"; + } + + public string? get_payloader_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? name = null) { if (codec == null) return null; string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string? pay = get_pay_element_name(media, codec); + if (pay == null) return null; + return @"$pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + } + + public string? get_encode_bin_without_payloader_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string? encode = element_name ?? get_encode_element_name(media, codec); - if (pay == null || encode == null) return null; + if (encode == null) return null; string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps"; + return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -308,4 +344,26 @@ public class Dino.Plugins.Rtp.CodecUtil { return bin; } -}
\ No newline at end of file + public Gst.Element? get_encode_bin_without_payloader(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_without_payloader_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s without payloader: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public Gst.Element? get_payloader_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_payloader_bin_description(media, codec, payload_type, base_name); + if (desc == null) return null; + debug("Pipeline to payload %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +} diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index e25271b1..97258d0c 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -1,44 +1,64 @@ +using Xmpp.Xep.JingleRtp; +using Gee; + public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + private const int[] common_widths = {320, 360, 400, 480, 640, 960, 1280, 1920, 2560, 3840}; + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { return plugin.codec_util; } } public Gst.Device device { get; private set; } - private string device_name; - public string id { get { - return device_name; - }} - private string device_display_name; - public string display_name { get { - return device_display_name; - }} + public string id { get { return device_name; }} + public string display_name { get { return device_display_name; }} public string detail_name { get { return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; }} - public Gst.Pipeline pipe { get { - return plugin.pipe; - }} + + public Gst.Pipeline pipe { get { return plugin.pipe; }} public string? media { get { - if (device.device_class.has_prefix("Audio/")) { + if (device.has_classes("Audio")) { return "audio"; - } else if (device.device_class.has_prefix("Video/")) { + } else if (device.has_classes("Video")) { return "video"; } else { return null; } }} - public bool is_source { get { - return device.device_class.has_suffix("/Source"); - }} - public bool is_sink { get { - return device.device_class.has_suffix("/Sink"); - }} + public bool is_source { get { return device.has_classes("Source"); }} + public bool is_sink { get { return device.has_classes("Sink"); }} + + private string device_name; + private string device_display_name; + private Gst.Caps device_caps; private Gst.Element element; private Gst.Element tee; private Gst.Element dsp; - private Gst.Element mixer; + private Gst.Base.Aggregator mixer; private Gst.Element filter; - private Gst.Element rate; - private int links = 0; + private int links; + + // Codecs + private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func); + + // Payloaders + private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func); + + // Bitrate + private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func); + + private class CodecBitrate { + public uint bitrate; + public int64 timestamp; + + public CodecBitrate(uint bitrate) { + this.bitrate = bitrate; + this.timestamp = get_monotonic_time(); + } + } public Device(Plugin plugin, Gst.Device device) { this.plugin = plugin; @@ -57,25 +77,241 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } public Gst.Element? link_sink() { + if (!is_sink) return null; if (element == null) create(); links++; - if (mixer != null) return mixer; - if (is_sink && media == "audio") return filter; + if (mixer != null) { + Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())"); + pipe.add(rate); + rate.link(mixer); + return rate; + } + if (media == "audio") return filter; return element; } - public Gst.Element? link_source() { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) { + if (!is_source) return null; if (element == null) create(); links++; + if (payload_type != null && ssrc != 0 && tee != null) { + bool new_codec = false; + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (!codecs.has_key(payload_type)) { + codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder"); + pipe.add(codecs[payload_type]); + new_codec = true; + } + if (!codec_tees.has_key(payload_type)) { + codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee"); + codec_tees[payload_type].@set("allow-not-linked", true); + pipe.add(codec_tees[payload_type]); + codecs[payload_type].link(codec_tees[payload_type]); + } + if (!payloaders.has_key(payload_type)) { + payloaders[payload_type] = new HashMap<uint, Gst.Element>(); + } + if (!payloaders[payload_type].has_key(ssrc)) { + payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)"); + var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); + payload.ssrc = ssrc; + payload.seqnum_offset = seqnum_offset; + if (timestamp_offset != 0) { + payload.timestamp_offset = timestamp_offset; + } + pipe.add(payloaders[payload_type][ssrc]); + codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset); + } + if (!payloader_tees.has_key(payload_type)) { + payloader_tees[payload_type] = new HashMap<uint, Gst.Element>(); + } + if (!payloader_tees[payload_type].has_key(ssrc)) { + payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee"); + payloader_tees[payload_type][ssrc].@set("allow-not-linked", true); + pipe.add(payloader_tees[payload_type][ssrc]); + payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]); + } + if (!payloader_links.has_key(payload_type)) { + payloader_links[payload_type] = new HashMap<uint, uint>(); + } + if (!payloader_links[payload_type].has_key(ssrc)) { + payloader_links[payload_type][ssrc] = 1; + } else { + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1; + } + if (new_codec) { + tee.link(codecs[payload_type]); + } + return payloader_tees[payload_type][ssrc]; + } if (tee != null) return tee; return element; } - public void unlink() { + private static double get_target_bitrate(Gst.Caps caps) { + if (caps == null || caps.get_size() == 0) return uint.MAX; + unowned Gst.Structure? that = caps.get_structure(0); + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX; + if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX; + if (!that.has_field("framerate")) return uint.MAX; + Value framerate = that.get_value("framerate"); + if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX; + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + double pxs = ((double)num/(double)den) * (double)width * (double)height; + double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0; + if (br < 128.0) return 128.0; + return br; + } + + private Gst.Caps get_active_caps(PayloadType payload_type) { + return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; + } + + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { + plugin.pause(); + debug("Set scaled caps to %s", caps.to_string()); + codec_util.update_rescale_caps(codecs[payload_type], caps); + plugin.unpause(); + } + + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { + int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; + device_caps.get_structure(0).get_int("width", out device_caps_width); + device_caps.get_structure(0).get_int("height", out device_caps_height); + device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den); + Gst.Caps active_caps = get_active_caps(payload_type); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (new_width == active_caps_width) return; + int new_height = device_caps_height * new_width / device_caps_width; + Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); + double required_bitrate = get_target_bitrate(new_caps); + debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); + if (bitrate < required_bitrate && new_width > active_caps_width) return; + apply_caps(payload_type, new_caps); + } + + public void update_bitrate(PayloadType payload_type, uint bitrate) { + if (codecs.has_key(payload_type)) { + lock(codec_bitrates); + if (!codec_bitrates.has_key(payload_type)) { + codec_bitrates[payload_type] = new ArrayList<CodecBitrate>(); + } + codec_bitrates[payload_type].add(new CodecBitrate(bitrate)); + var remove = new ArrayList<CodecBitrate>(); + foreach (CodecBitrate rate in codec_bitrates[payload_type]) { + if (rate.timestamp < get_monotonic_time() - 5000000L) { + remove.add(rate); + continue; + } + if (rate.bitrate < bitrate) { + bitrate = rate.bitrate; + } + } + codec_bitrates[payload_type].remove_all(remove); + if (media == "video") { + if (bitrate < 128) bitrate = 128; + Gst.Caps active_caps = get_active_caps(payload_type); + double max_bitrate = get_target_bitrate(device_caps) * 2; + double current_target_bitrate = get_target_bitrate(active_caps); + int device_caps_width, active_caps_width; + device_caps.get_structure(0).get_int("width", out device_caps_width); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { + // Lower video resolution + int i = 1; + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) { + debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } + apply_width(payload_type, common_widths[i-1], bitrate); + } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { + // Higher video resolution + int i = 0; + for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] != active_caps_width) { + debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } + if (common_widths[i] > device_caps_width) { + // We never scale up, so just stick with what the device gives + apply_width(payload_type, device_caps_width, bitrate); + } else if (common_widths[i] != active_caps_width) { + apply_width(payload_type, common_widths[i], bitrate); + } + } + if (bitrate > max_bitrate) bitrate = (uint) max_bitrate; + } + codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); + unlock(codec_bitrates); + } + } + + public void unlink(Gst.Element? link = null) { if (links <= 0) { critical("Link count below zero."); return; } + if (link != null && is_source && tee != null) { + PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key; + uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key; + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1; + if (payloader_links[payload_type][ssrc] == 0) { + plugin.pause(); + + codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]); + payloaders[payload_type][ssrc].set_locked_state(true); + payloaders[payload_type][ssrc].set_state(Gst.State.NULL); + payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]); + pipe.remove(payloaders[payload_type][ssrc]); + payloaders[payload_type].unset(ssrc); + payloader_tees[payload_type][ssrc].set_locked_state(true); + payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL); + pipe.remove(payloader_tees[payload_type][ssrc]); + payloader_tees[payload_type].unset(ssrc); + + payloader_links[payload_type].unset(ssrc); + plugin.unpause(); + } + if (payloader_links[payload_type].size == 0) { + plugin.pause(); + + tee.unlink(codecs[payload_type]); + codecs[payload_type].set_locked_state(true); + codecs[payload_type].set_state(Gst.State.NULL); + codecs[payload_type].unlink(codec_tees[payload_type]); + pipe.remove(codecs[payload_type]); + codecs.unset(payload_type); + codec_tees[payload_type].set_locked_state(true); + codec_tees[payload_type].set_state(Gst.State.NULL); + pipe.remove(codec_tees[payload_type]); + codec_tees.unset(payload_type); + + payloaders.unset(payload_type); + payloader_tees.unset(payload_type); + payloader_links.unset(payload_type); + plugin.unpause(); + } + } + if (link != null && is_sink && mixer != null) { + plugin.pause(); + link.set_locked_state(true); + Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer(); + link.get_static_pad("src").unlink(mixer_sink_pad); + mixer_sink_pad.set_active(false); + link.set_state(Gst.State.NULL); + pipe.remove(link); + mixer.release_request_pad(mixer_sink_pad); + plugin.unpause(); + } links--; if (links == 0) { destroy(); @@ -154,11 +390,16 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { debug("Creating device %s", id); plugin.pause(); element = device.create_element(id); + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } pipe.add(element); + device_caps = get_best_caps(); if (is_source) { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); + filter.@set("caps", device_caps); pipe.add(filter); element.link(filter); #if WITH_VOICE_PROCESSOR @@ -174,22 +415,18 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(tee); (dsp ?? filter).link(tee); } - if (is_sink) { - element.@set("async", false); - element.@set("sync", false); - } if (is_sink && media == "audio") { - filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); - pipe.add(filter); - if (plugin.echoprobe != null) { - rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); - rate.@set("tolerance", 100000000); - pipe.add(rate); - filter.link(rate); - rate.link(plugin.echoprobe); + mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id"); + pipe.add(mixer); + mixer.link(pipe); + if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) { + mixer.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", device_caps); + pipe.add(filter); + mixer.link(filter); filter.link(element); } } @@ -197,38 +434,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } private void destroy() { - if (mixer != null) { - if (is_sink && media == "audio" && plugin.echoprobe != null) { - plugin.echoprobe.unlink(mixer); - } - int linked_sink_pads = 0; - mixer.foreach_sink_pad((_, pad) => { - if (pad.is_linked()) linked_sink_pads++; - return true; - }); - if (linked_sink_pads > 0) { - warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + if (is_sink) { + if (mixer != null) { + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.unlink(plugin.echoprobe ?? element); } - mixer.set_locked_state(true); - mixer.set_state(Gst.State.NULL); - mixer.unlink(element); - pipe.remove(mixer); - mixer = null; - } else if (is_sink && media == "audio") { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); + filter.unlink(element); pipe.remove(filter); filter = null; } - if (rate != null) { - rate.set_locked_state(true); - rate.set_state(Gst.State.NULL); - rate.unlink(plugin.echoprobe); - pipe.remove(rate); - rate = null; - } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } @@ -239,34 +463,42 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { else if (is_source) element.unlink(tee); pipe.remove(element); element = null; - if (filter != null) { - filter.set_locked_state(true); - filter.set_state(Gst.State.NULL); - filter.unlink(dsp ?? tee); - pipe.remove(filter); - filter = null; - } - if (dsp != null) { - dsp.set_locked_state(true); - dsp.set_state(Gst.State.NULL); - dsp.unlink(tee); - pipe.remove(dsp); - dsp = null; + if (mixer != null) { + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + pipe.remove(mixer); + mixer = null; } - if (tee != null) { - int linked_src_pads = 0; - tee.foreach_src_pad((_, pad) => { - if (pad.is_linked()) linked_src_pads++; - return true; - }); - if (linked_src_pads != 0) { - warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + if (is_source) { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; } - tee.set_locked_state(true); - tee.set_state(Gst.State.NULL); - pipe.remove(tee); - tee = null; } debug("Destroyed device %s", id); } -}
\ No newline at end of file +} diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index dfe224aa..4f48b6c4 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -63,7 +63,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return supported; } - private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { + public override async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; @@ -131,7 +131,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) { Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func); if (media == "audio") { - var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 }; + var opus = new JingleRtp.PayloadType() { channels = 1, clockrate = 48000, name = "opus", id = 99 }; opus.parameters["useinbandfec"] = "1"; var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 }; var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 }; diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d79fc2aa..6d6da79a 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -5,10 +5,10 @@ using Xmpp.Xep; public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Dino.Application app { get; private set; } public CodecUtil codec_util { get; private set; } - public Gst.DeviceMonitor device_monitor { get; private set; } - public Gst.Pipeline pipe { get; private set; } - public Gst.Bin rtpbin { get; private set; } - public Gst.Element echoprobe { get; private set; } + public Gst.DeviceMonitor? device_monitor { get; private set; } + public Gst.Pipeline? pipe { get; private set; } + public Gst.Bin? rtpbin { get; private set; } + public Gst.Element? echoprobe { get; private set; } private Gee.List<Stream> streams = new ArrayList<Stream>(); private Gee.List<Device> devices = new ArrayList<Device>(); @@ -42,18 +42,22 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { if (pause_count < 0) warning("Pause count below zero!"); } - public void startup() { + private void init_device_monitor() { + if (device_monitor != null) return; device_monitor = new Gst.DeviceMonitor(); device_monitor.show_all = true; device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); device_monitor.start(); foreach (Gst.Device device in device_monitor.get_devices()) { - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) continue; + if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue; if (device.properties.get_string("device.class") == "monitor") continue; if (devices.any_match((it) => it.matches(device))) continue; devices.add(new Device(this, device)); } + } + private void init_call_pipe() { + if (pipe != null) return; pipe = new Gst.Pipeline(null); // RTP @@ -66,7 +70,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { rtpbin.pad_added.connect(on_rtp_pad_added); rtpbin.@set("latency", 100); rtpbin.@set("do-lost", true); - rtpbin.@set("do-sync-event", true); +// rtpbin.@set("do-sync-event", true); rtpbin.@set("drop-on-latency", true); rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); @@ -86,6 +90,20 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.set_state(Gst.State.PLAYING); } + private void destroy_call_pipe() { + if (pipe == null) return; + pipe.set_state(Gst.State.NULL); + rtpbin = null; +#if WITH_VOICE_PROCESSOR + echoprobe = null; +#endif + pipe = null; + } + + public void startup() { + init_device_monitor(); + } + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { debug("request-pt-map"); return null; @@ -98,7 +116,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { uint8 rtpid = (uint8)int.parse(split[3]); foreach (Stream stream in streams) { if (stream.rtpid == rtpid) { - stream.on_ssrc_pad_added(split[4], pad); + stream.on_ssrc_pad_added((uint32) split[4].to_uint64(), pad); } } } @@ -179,51 +197,39 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { - Gst.Device old_device = null; - Gst.Device device = null; - Device old = null; + Gst.Device? old_gst_device = null; + Gst.Device? gst_device = null; + Device? device = null; switch (message.type) { case Gst.MessageType.DEVICE_ADDED: - message.parse_device_added(out device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; - devices.add(new Device(this, device)); + message.parse_device_added(out gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE; + device = new Device(this, gst_device); + devices.add(device); break; #if GST_1_16 case Gst.MessageType.DEVICE_CHANGED: - message.parse_device_changed(out device, out old_device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - old = devices.first_match((it) => it.matches(old_device)); - if (old != null) old.update(device); + message.parse_device_changed(out gst_device, out old_gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + device = devices.first_match((it) => it.matches(old_gst_device)); + if (device != null) device.update(gst_device); break; #endif case Gst.MessageType.DEVICE_REMOVED: - message.parse_device_removed(out device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - old = devices.first_match((it) => it.matches(device)); - if (old != null) devices.remove(old); + message.parse_device_removed(out gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + device = devices.first_match((it) => it.matches(gst_device)); + if (device != null) devices.remove(device); break; default: break; } if (device != null) { - switch (device.device_class) { - case "Audio/Source": - devices_changed("audio", false); - break; - case "Audio/Sink": - devices_changed("audio", true); - break; - case "Video/Source": - devices_changed("video", false); - break; - case "Video/Sink": - devices_changed("video", true); - break; - } + devices_changed(device.media, device.is_sink); } return Source.CONTINUE; } @@ -253,6 +259,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { // } public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + init_call_pipe(); var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; if (content_params == null) return null; Stream stream; @@ -271,15 +278,15 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } public void shutdown() { - device_monitor.stop(); - pipe.set_state(Gst.State.NULL); - rtpbin = null; - pipe = null; + if (device_monitor != null) { + device_monitor.stop(); + } + destroy_call_pipe(); Gst.deinit(); } public bool supports(string media) { - if (rtpbin == null) return false; + if (!codec_util.is_element_supported("rtpbin")) return false; if (media == "audio") { if (get_devices("audio", false).is_empty) return false; @@ -287,7 +294,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } if (media == "video") { - if (Gst.ElementFactory.make("gtksink", null) == null) return false; + if (!codec_util.is_element_supported("gtksink")) return false; if (get_devices("video", false).is_empty) return false; } @@ -295,6 +302,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } public VideoCallWidget? create_widget(WidgetType type) { + init_call_pipe(); if (type == WidgetType.GTK) { return new VideoWidget(this); } @@ -422,10 +430,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } } - private void dump_dot() { + public void dump_dot() { + if (pipe == null) return; string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); - debug("Stored pipe details as %s", name); + print(@"Stored pipe details as $name\n"); } public void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause) { diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index bd8a279f..dc712b61 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -18,22 +18,19 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.App.Sink send_rtcp; private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtcp; - private Gst.Element encode; - private Gst.RTP.BasePayload encode_pay; private Gst.Element decode; private Gst.RTP.BaseDepayload decode_depay; private Gst.Element input; + private Gst.Pad input_pad; private Gst.Element output; private Gst.Element session; private Device _input_device; public Device input_device { get { return _input_device; } set { if (!paused) { - if (this._input_device != null) { - this._input_device.unlink(); - this._input_device = null; - } - set_input(value != null ? value.link_source() : null); + var input = this.input; + set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null); + if (this._input_device != null) this._input_device.unlink(input); } this._input_device = value; }} @@ -47,7 +44,16 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public bool created { get; private set; default = false; } public bool paused { get; private set; default = false; } private bool push_recv_data = false; - private string participant_ssrc = null; + private uint our_ssrc = Random.next_int(); + private int next_seqnum_offset = -1; + private uint32 next_timestamp_offset_base = 0; + private int64 next_timestamp_offset_stamp = 0; + private uint32 next_timestamp_offset { get { + if (next_timestamp_offset_base == 0) return 0; + int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp; + return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate); + } } + private uint32 participant_ssrc = 0; private Gst.Pad recv_rtcp_sink_pad; private Gst.Pad recv_rtp_sink_pad; @@ -92,16 +98,22 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtp.async = false; send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.emit_signals = true; - send_rtp.sync = false; + send_rtp.sync = true; + send_rtp.drop = true; + send_rtp.wait_on_eos = false; send_rtp.new_sample.connect(on_new_sample); + send_rtp.connect("signal::eos", on_eos_static, this); pipe.add(send_rtp); send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; send_rtcp.async = false; send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.emit_signals = true; - send_rtcp.sync = false; + send_rtcp.sync = true; + send_rtcp.drop = true; + send_rtcp.wait_on_eos = false; send_rtcp.new_sample.connect(on_new_sample); + send_rtcp.connect("signal::eos", on_eos_static, this); pipe.add(send_rtcp); recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; @@ -125,18 +137,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); // Connect input - encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid"); - encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay"); - pipe.add(encode); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); - encode.get_static_pad("src").link(send_rtp_sink_pad); if (input != null) { - input.link(encode); + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); } // Connect output decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); - decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay"); pipe.add(decode); if (output != null) { decode.link(output); @@ -151,7 +160,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { plugin.unpause(); GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); - if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + if (session != null && remb_enabled) { Object internal_session; session.@get("internal-session", out internal_session); if (internal_session != null) { @@ -159,15 +168,16 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } Timeout.add(1000, () => remb_adjust()); } - if (media == "video") { - codec_util.update_bitrate(media, payload_type, encode, 256); + if (input_device != null && media == "video") { + input_device.update_bitrate(payload_type, target_send_bitrate); } } - private uint remb = 256; private int last_packets_lost = -1; - private uint64 last_packets_received; - private uint64 last_octets_received; + private uint64 last_packets_received = 0; + private uint64 last_octets_received = 0; + private uint max_target_receive_bitrate = 0; + private int64 last_remb_time = 0; private bool remb_adjust() { unowned Gst.Structure? stats; if (session == null) { @@ -185,73 +195,95 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { warning("No source-stats for session %u", rtpid); return Source.REMOVE; } + + if (input_device == null) return Source.CONTINUE; + foreach (Value value in source_stats.values) { unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); - uint ssrc; + uint32 ssrc; if (!source_stat.get_uint("ssrc", out ssrc)) continue; - if (ssrc.to_string() == participant_ssrc) { + if (ssrc == participant_ssrc) { int packets_lost; uint64 packets_received, octets_received; source_stat.get_int("packets-lost", out packets_lost); source_stat.get_uint64("packets-received", out packets_received); source_stat.get_uint64("octets-received", out octets_received); int new_lost = packets_lost - last_packets_lost; + if (new_lost < 0) new_lost = 0; uint64 new_received = packets_received - last_packets_received; + if (packets_received < last_packets_received) new_received = 0; uint64 new_octets = octets_received - last_octets_received; + if (octets_received < last_octets_received) octets_received = 0; if (new_received == 0) continue; last_packets_lost = packets_lost; last_packets_received = packets_received; last_octets_received = octets_received; double loss_rate = (double)new_lost / (double)(new_lost + new_received); + uint new_target_receive_bitrate; if (new_lost <= 0 || loss_rate < 0.02) { - remb = (uint)(1.08 * (double)remb); + new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { - remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); + } else { + new_target_receive_bitrate = target_receive_bitrate; } - remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); - remb = uint.max(16, remb); // Never go below 16 - uint8[] data = new uint8[] { - 143, 206, 0, 5, - 0, 0, 0, 0, - 0, 0, 0, 0, - 'R', 'E', 'M', 'B', - 1, 0, 0, 0, - 0, 0, 0, 0 - }; - data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); - data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); - data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); - data[7] = (uint8)(encode_pay.ssrc & 0xff); - uint8 br_exp = 0; - uint32 br_mant = remb * 1000; - uint8 bits = (uint8)Math.log2(br_mant); - if (bits > 16) { - br_exp = (uint8)bits - 16; - br_mant = br_mant >> br_exp; + if (last_remb_time == 0) { + last_remb_time = get_monotonic_time(); + } else { + int64 time_now = get_monotonic_time(); + int64 time_diff = time_now - last_remb_time; + last_remb_time = time_now; + uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); + new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); + max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate); + new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); + } + new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 + if (new_target_receive_bitrate != target_receive_bitrate) { + target_receive_bitrate = new_target_receive_bitrate; + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((our_ssrc >> 24) & 0xff); + data[5] = (uint8)((our_ssrc >> 16) & 0xff); + data[6] = (uint8)((our_ssrc >> 8) & 0xff); + data[7] = (uint8)(our_ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = target_receive_bitrate * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); } - data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); - data[18] = (uint8)((br_mant >> 8) & 0xff); - data[19] = (uint8)(br_mant & 0xff); - data[20] = (uint8)((ssrc >> 24) & 0xff); - data[21] = (uint8)((ssrc >> 16) & 0xff); - data[22] = (uint8)((ssrc >> 8) & 0xff); - data[23] = (uint8)(ssrc & 0xff); - encrypt_and_send_rtcp(data); } } return Source.CONTINUE; } private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { - if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { + if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) { // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 uint8[] data; fci.extract_dup(0, fci.get_size(), out data); if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; uint8 br_exp = data[5] >> 2; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; - uint bitrate = (br_mant << br_exp) / 1000; - self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); + self.target_send_bitrate = (br_mant << br_exp) / 1000; + self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate); } } @@ -267,32 +299,63 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("Sink is null"); return Gst.FlowReturn.EOS; } + if (sink != send_rtp && sink != send_rtcp) { + warning("unknown sample"); + return Gst.FlowReturn.NOT_SUPPORTED; + } Gst.Sample sample = sink.pull_sample(); Gst.Buffer buffer = sample.get_buffer(); + if (sink == send_rtp) { + uint buffer_ssrc = 0, buffer_seq = 0; + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + buffer_ssrc = rtp_buffer.get_ssrc(); + buffer_seq = rtp_buffer.get_seq(); + next_seqnum_offset = rtp_buffer.get_seq() + 1; + next_timestamp_offset_base = rtp_buffer.get_timestamp(); + next_timestamp_offset_stamp = get_monotonic_time(); + rtp_buffer.unmap(); + } + if (our_ssrc != buffer_ssrc) { + warning("Sending RTP %s buffer seq %u with SSRC %u when our ssrc is %u", media, buffer_seq, buffer_ssrc, our_ssrc); + } else { + debug("Sending RTP %s buffer seq %u with SSRC %u", media, buffer_seq, buffer_ssrc); + } + } + + prepare_local_crypto(); + uint8[] data; buffer.extract_dup(0, buffer.get_size(), out data); - prepare_local_crypto(); if (sink == send_rtp) { - if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtp(data); - } - on_send_rtp_data(new Bytes.take((owned) data)); + encrypt_and_send_rtp((owned) data); } else if (sink == send_rtcp) { encrypt_and_send_rtcp((owned) data); - } else { - warning("unknown sample"); } return Gst.FlowReturn.OK; } + private void encrypt_and_send_rtp(owned uint8[] data) { + Bytes bytes; + if (crypto_session.has_encrypt) { + bytes = new Bytes.take(crypto_session.encrypt_rtp(data)); + } else { + bytes = new Bytes.take(data); + } + on_send_rtp_data(bytes); + } + private void encrypt_and_send_rtcp(owned uint8[] data) { + Bytes bytes; if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtcp(data); + bytes = new Bytes.take(crypto_session.encrypt_rtcp(data)); + } else { + bytes = new Bytes.take(data); } if (rtcp_mux) { - on_send_rtp_data(new Bytes.take((owned) data)); + on_send_rtp_data(bytes); } else { - on_send_rtcp_data(new Bytes.take((owned) data)); + on_send_rtcp_data(bytes); } } @@ -300,41 +363,59 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { return Gst.PadProbeReturn.DROP; } + private static void on_eos_static(Gst.App.Sink sink, Stream self) { + debug("EOS on %s", sink.name); + if (sink == self.send_rtp) { + Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; }); + } else if (sink == self.send_rtcp) { + Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; }); + } + } + + private void on_send_rtp_eos() { + if (send_rtp_src_pad != null) { + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + send_rtp_src_pad = null; + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + pipe.remove(send_rtp); + send_rtp = null; + debug("Stopped sending RTP for %u", rtpid); + } + + private void on_send_rtcp_eos() { + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + pipe.remove(send_rtcp); + send_rtcp = null; + debug("Stopped sending RTCP for %u", rtpid); + } + public override void destroy() { // Stop network communication push_recv_data = false; - recv_rtp.end_of_stream(); - recv_rtcp.end_of_stream(); - send_rtp.new_sample.disconnect(on_new_sample); - send_rtcp.new_sample.disconnect(on_new_sample); + if (recv_rtp != null) recv_rtp.end_of_stream(); + if (recv_rtcp != null) recv_rtcp.end_of_stream(); + if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample); + if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample); // Disconnect input device if (input != null) { - input.unlink(encode); - input = null; + input_pad.unlink(send_rtp_sink_pad); + input.release_request_pad(input_pad); + input_pad = null; } if (this._input_device != null) { - if (!paused) this._input_device.unlink(); + if (!paused) this._input_device.unlink(input); this._input_device = null; + this.input = null; } - // Disconnect encode - encode.set_locked_state(true); - encode.set_state(Gst.State.NULL); - encode.get_static_pad("src").unlink(send_rtp_sink_pad); - pipe.remove(encode); - encode = null; - encode_pay = null; - - // Disconnect RTP sending - if (send_rtp_src_pad != null) { - send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); - send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + // Inject EOS + if (send_rtp_sink_pad != null) { + send_rtp_sink_pad.send_event(new Gst.Event.eos()); } - send_rtp.set_locked_state(true); - send_rtp.set_state(Gst.State.NULL); - pipe.remove(send_rtp); - send_rtp = null; // Disconnect decode if (recv_rtp_src_pad != null) { @@ -342,57 +423,63 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); } - // Disconnect RTP receiving - recv_rtp.set_locked_state(true); - recv_rtp.set_state(Gst.State.NULL); - recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); - pipe.remove(recv_rtp); - recv_rtp = null; - // Disconnect output if (output != null) { + decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); decode.unlink(output); } - decode.set_locked_state(true); - decode.set_state(Gst.State.NULL); - pipe.remove(decode); - decode = null; - decode_depay = null; - output = null; // Disconnect output device if (this._output_device != null) { - this._output_device.unlink(); + this._output_device.unlink(output); this._output_device = null; } + output = null; - // Disconnect RTCP receiving - recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); - recv_rtcp.set_locked_state(true); - recv_rtcp.set_state(Gst.State.NULL); - pipe.remove(recv_rtcp); - recv_rtcp = null; + // Destroy decode + if (decode != null) { + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + decode_depay = null; + } - // Disconnect RTCP sending - send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); - send_rtcp.set_locked_state(true); - send_rtcp.set_state(Gst.State.NULL); - pipe.remove(send_rtcp); - send_rtcp = null; + // Disconnect and remove RTP input + if (recv_rtp != null) { + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + pipe.remove(recv_rtp); + recv_rtp = null; + } - // Release rtp pads - rtpbin.release_request_pad(send_rtp_sink_pad); - send_rtp_sink_pad = null; - rtpbin.release_request_pad(recv_rtp_sink_pad); - recv_rtp_sink_pad = null; - rtpbin.release_request_pad(recv_rtcp_sink_pad); - recv_rtcp_sink_pad = null; - rtpbin.release_request_pad(send_rtcp_src_pad); - send_rtcp_src_pad = null; - send_rtp_src_pad = null; - recv_rtp_src_pad = null; + // Disconnect and remove RTCP input + if (recv_rtcp != null) { + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + } - session = null; + // Release rtp pads + if (send_rtp_sink_pad != null) { + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + } + if (recv_rtp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + } + if (send_rtcp_src_pad != null) { + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + } + if (recv_rtcp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + } } private void prepare_remote_crypto() { @@ -410,17 +497,38 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { on_recv_rtcp_data(bytes); return; } - prepare_remote_crypto(); - uint8[] data = bytes.get_data(); - if (crypto_session.has_decrypt) { - try { - data = crypto_session.decrypt_rtp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); +#if GST_1_16 + { + Gst.Buffer buffer = new Gst.Buffer.wrapped_bytes(bytes); + Gst.RTP.Buffer rtp_buffer; + uint buffer_ssrc = 0, buffer_seq = 0; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + buffer_ssrc = rtp_buffer.get_ssrc(); + buffer_seq = rtp_buffer.get_seq(); + rtp_buffer.unmap(); } + debug("Received RTP %s buffer seq %u with SSRC %u", media, buffer_seq, buffer_ssrc); } +#endif if (push_recv_data) { - Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { +#if GST_1_16 + buffer = new Gst.Buffer.wrapped_bytes(bytes); +#else + buffer = new Gst.Buffer.wrapped(bytes.get_data()); +#endif + } + Gst.RTP.Buffer rtp_buffer; if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { if (rtp_buffer.get_extension()) { @@ -448,11 +556,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { rtp_buffer.unmap(); } - // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() - // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of - // the underlying buffer, so and some point we should move over to the new version (once we require - // Vala >= 0.50) -#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI +#if VALA_0_50 recv_rtp.push_buffer((owned) buffer); #else Gst.FlowReturn ret; @@ -462,19 +566,26 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } public override void on_recv_rtcp_data(Bytes bytes) { - prepare_remote_crypto(); - uint8[] data = bytes.get_data(); - if (crypto_session.has_decrypt) { - try { - data = crypto_session.decrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - } - } if (push_recv_data) { - Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); - // See above -#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtcp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { +#if GST_1_16 + buffer = new Gst.Buffer.wrapped_bytes(bytes); +#else + buffer = new Gst.Buffer.wrapped(bytes.get_data()); +#endif + } + +#if VALA_0_50 recv_rtcp.push_buffer((owned) buffer); #else Gst.FlowReturn ret; @@ -502,10 +613,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); } - public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { - debug("New ssrc %s with pad %s", ssrc, pad.name); - if (participant_ssrc != null && participant_ssrc != ssrc) { - warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); + public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) { + debug("New ssrc %u with pad %s", ssrc, pad.name); + if (participant_ssrc != 0 && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc); return; } participant_ssrc = ssrc; @@ -534,7 +645,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void set_input_and_pause(Gst.Element? input, bool paused) { if (created && this.input != null) { - this.input.unlink(encode); + this.input_pad.unlink(send_rtp_sink_pad); + this.input.release_request_pad(this.input_pad); + this.input_pad = null; this.input = null; } @@ -543,28 +656,42 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (created && sending && !paused && input != null) { plugin.pause(); - input.link(encode); + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); plugin.unpause(); } } public void pause() { if (paused) return; + var input = this.input; set_input_and_pause(null, true); - if (input_device != null) input_device.unlink(); + if (input != null && input_device != null) input_device.unlink(input); } public void unpause() { if (!paused) return; - set_input_and_pause(input_device != null ? input_device.link_source() : null, false); + set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); + input_device.update_bitrate(payload_type, target_send_bitrate); + } + + public uint get_participant_ssrc(Xmpp.Jid participant) { + if (participant.equals(content.session.peer_full_jid)) { + return participant_ssrc; + } + return 0; } ulong block_probe_handler_id = 0; - public virtual void add_output(Gst.Element element) { + public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) { if (output != null) { critical("add_output() invoked more than once"); return; } + if (participant != null) { + critical("add_output() invoked with participant when not supported"); + return; + } this.output = element; if (created) { plugin.pause(); @@ -586,7 +713,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { decode.unlink(element); } if (this._output_device != null) { - this._output_device.unlink(); + this._output_device.unlink(element); this._output_device = null; } this.output = null; @@ -657,7 +784,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { disconnect(video_orientation_changed_handler); } - public override void add_output(Gst.Element element) { + public override void add_output(Gst.Element element, Xmpp.Jid? participant) { if (element == output_tee || element == rotate) { base.add_output(element); return; @@ -678,4 +805,4 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { output_tee.unlink(element); } } -}
\ No newline at end of file +} diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala index 351069a7..3daf5284 100644 --- a/plugins/rtp/src/video_widget.vala +++ b/plugins/rtp/src/video_widget.vala @@ -12,8 +12,9 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge private bool attached; private Device? connected_device; + private Gst.Element? connected_device_element; private Stream? connected_stream; - private Gst.Element convert; + private Gst.Element prepare; public VideoWidget(Plugin plugin) { this.plugin = plugin; @@ -24,26 +25,38 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge Gtk.Widget widget; element.@get("widget", out widget); element.@set("async", false); - element.@set("sync", false); + element.@set("sync", true); this.widget = widget; add(widget); widget.visible = true; - - // Listen for resolution changes - element.get_static_pad("sink").notify["caps"].connect(() => { - if (element.get_static_pad("sink").caps == null) return; - - int width, height; - element.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); - element.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); - resolution_changed(width, height); - }); } else { warning("Could not create GTK video sink. Won't display videos."); } + size_allocate.connect_after(after_size_allocate); + } + + public void input_caps_changed(GLib.Object pad, ParamSpec spec) { + Gst.Caps? caps = (pad as Gst.Pad).caps; + if (caps == null) return; + + int width, height; + caps.get_structure(0).get_int("width", out width); + caps.get_structure(0).get_int("height", out height); + resolution_changed(width, height); + } + + public void after_size_allocate(Gtk.Allocation allocation) { + if (prepare != null) { + Gst.Element crop = ((Gst.Bin)prepare).get_by_name(@"video_widget_$(id)_crop"); + if (crop != null) { + Value ratio = new Value(typeof(Gst.Fraction)); + Gst.Value.set_fraction(ref ratio, allocation.width, allocation.height); + crop.set_property("aspect-ratio", ratio); + } + } } - public void display_stream(Xmpp.Xep.JingleRtp.Stream stream) { + public void display_stream(Xmpp.Xep.JingleRtp.Stream stream, Xmpp.Jid jid) { if (element == null) return; detach(); if (stream.media != "video") return; @@ -51,11 +64,12 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_stream == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); - convert.name = @"video_widget_$(id)_prepare"; - pipe.add(convert); - convert.link(element); - connected_stream.add_output(convert); + prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + pipe.add(prepare); + connected_stream.add_output(prepare); + prepare.link(element); element.set_locked_state(false); plugin.unpause(); attached = true; @@ -68,11 +82,13 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_device == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); - convert.name = @"video_widget_$(id)_prepare"; - pipe.add(convert); - convert.link(element); - connected_device.link_source().link(convert); + prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + pipe.add(prepare); + connected_device_element = connected_device.link_source(); + connected_device_element.link(prepare); + prepare.link(element); element.set_locked_state(false); plugin.unpause(); attached = true; @@ -82,19 +98,19 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (element == null) return; if (attached) { if (connected_stream != null) { - connected_stream.remove_output(convert); + connected_stream.remove_output(prepare); connected_stream = null; } if (connected_device != null) { - connected_device.link_source().unlink(element); - connected_device.unlink(); // We get a new ref to recover the element, so unlink twice + connected_device_element.unlink(element); + connected_device_element = null; connected_device.unlink(); connected_device = null; } - convert.set_locked_state(true); - convert.set_state(Gst.State.NULL); - pipe.remove(convert); - convert = null; + prepare.set_locked_state(true); + prepare.set_state(Gst.State.NULL); + pipe.remove(prepare); + prepare = null; element.set_locked_state(true); element.set_state(Gst.State.NULL); pipe.remove(element); diff --git a/xmpp-vala/CMakeLists.txt b/xmpp-vala/CMakeLists.txt index bf8f0068..27f0d408 100644 --- a/xmpp-vala/CMakeLists.txt +++ b/xmpp-vala/CMakeLists.txt @@ -109,6 +109,8 @@ SOURCES "src/module/xep/0176_jingle_ice_udp/jingle_ice_udp_module.vala" "src/module/xep/0176_jingle_ice_udp/transport_parameters.vala" + "src/module/xep/0177_jingle_raw_udp.vala" + "src/module/xep/0384_omemo/omemo_encryptor.vala" "src/module/xep/0384_omemo/omemo_decryptor.vala" @@ -122,7 +124,9 @@ SOURCES "src/module/xep/0249_direct_muc_invitations.vala" "src/module/xep/0260_jingle_socks5_bytestreams.vala" "src/module/xep/0261_jingle_in_band_bytestreams.vala" + "src/module/xep/0272_muji.vala" "src/module/xep/0280_message_carbons.vala" + "src/module/xep/0298_coin.vala" "src/module/xep/0308_last_message_correction.vala" "src/module/xep/0313_message_archive_management.vala" "src/module/xep/0333_chat_markers.vala" @@ -133,6 +137,7 @@ SOURCES "src/module/xep/0380_explicit_encryption.vala" "src/module/xep/0391_jingle_encrypted_transports.vala" "src/module/xep/0410_muc_self_ping.vala" + "src/module/xep/muji_meta.vala" "src/module/xep/pixbuf_storage.vala" "src/util.vala" diff --git a/xmpp-vala/src/module/presence/flag.vala b/xmpp-vala/src/module/presence/flag.vala index 77bc0b5f..8e13d0ad 100644 --- a/xmpp-vala/src/module/presence/flag.vala +++ b/xmpp-vala/src/module/presence/flag.vala @@ -20,6 +20,17 @@ public class Flag : XmppStreamFlag { return presences[full_jid]; } + public Gee.List<Presence.Stanza> get_presences(Jid jid) { + Gee.List<Presence.Stanza> ret = new ArrayList<Presence.Stanza>(); + Gee.List<Jid>? jid_res = resources[jid]; + if (jid_res == null) return ret; + + foreach (Jid full_jid in jid_res) { + ret.add(presences[full_jid]); + } + return ret; + } + public void add_presence(Presence.Stanza presence) { if (!resources.has_key(presence.from)) { resources[presence.from] = new ArrayList<Jid>(Jid.equals_func); diff --git a/xmpp-vala/src/module/xep/0045_muc/module.vala b/xmpp-vala/src/module/xep/0045_muc/module.vala index 4cab9b6a..c9445f7e 100644 --- a/xmpp-vala/src/module/xep/0045_muc/module.vala +++ b/xmpp-vala/src/module/xep/0045_muc/module.vala @@ -58,6 +58,7 @@ public class JoinResult { public MucEnterError? muc_error; public string? stanza_error; public string? nick; + public bool newly_created = false; } public class Module : XmppStreamModule { @@ -80,7 +81,7 @@ public class Module : XmppStreamModule { received_pipeline_listener = new ReceivedPipelineListener(this); } - public async JoinResult? enter(XmppStream stream, Jid bare_jid, string nick, string? password, DateTime? history_since) { + public async JoinResult? enter(XmppStream stream, Jid bare_jid, string nick, string? password, DateTime? history_since, StanzaNode? additional_node) { try { Presence.Stanza presence = new Presence.Stanza(); presence.to = bare_jid.with_resource(nick); @@ -96,6 +97,10 @@ public class Module : XmppStreamModule { } presence.stanza.put_node(x_node); + if (additional_node != null) { + presence.stanza.put_node(additional_node); + } + stream.get_flag(Flag.IDENTITY).start_muc_enter(bare_jid, presence.id); query_room_info.begin(stream, bare_jid); @@ -210,11 +215,15 @@ public class Module : XmppStreamModule { stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); } - public void change_affiliation(XmppStream stream, Jid jid, string nick, string new_affiliation) { - StanzaNode query = new StanzaNode.build("query", NS_URI_ADMIN).add_self_xmlns(); - query.put_node(new StanzaNode.build("item", NS_URI_ADMIN).put_attribute("nick", nick, NS_URI_ADMIN).put_attribute("affiliation", new_affiliation, NS_URI_ADMIN)); - Iq.Stanza iq = new Iq.Stanza.set(query) { to=jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, iq); + public async void change_affiliation(XmppStream stream, Jid muc_jid, Jid? user_jid, string? nick, string new_affiliation) { + StanzaNode item_node = new StanzaNode.build("item", NS_URI_ADMIN) + .put_attribute("affiliation", new_affiliation, NS_URI_ADMIN); + if (user_jid != null) item_node.put_attribute("jid", user_jid.to_string(), NS_URI_ADMIN); + if (nick != null) item_node.put_attribute("nick", nick, NS_URI_ADMIN); + + StanzaNode query = new StanzaNode.build("query", NS_URI_ADMIN).add_self_xmlns().put_node(item_node); + Iq.Stanza iq = new Iq.Stanza.set(query) { to=muc_jid }; + yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, iq); } public async DataForms.DataForm? get_config_form(XmppStream stream, Jid jid) { @@ -229,19 +238,19 @@ public class Module : XmppStreamModule { return null; } - public void set_config_form(XmppStream stream, Jid jid, DataForms.DataForm data_form) { + public async void set_config_form(XmppStream stream, Jid jid, DataForms.DataForm data_form) { StanzaNode stanza_node = new StanzaNode.build("query", NS_URI_OWNER); stanza_node.add_self_xmlns().put_node(data_form.get_submit_node()); Iq.Stanza set_iq = new Iq.Stanza.set(stanza_node) { to=jid }; - stream.get_module(Iq.Module.IDENTITY).send_iq(stream, set_iq); + yield stream.get_module(Iq.Module.IDENTITY).send_iq_async(stream, set_iq); } public override void attach(XmppStream stream) { stream.add_flag(new Flag()); stream.get_module(MessageModule.IDENTITY).received_message.connect(on_received_message); stream.get_module(MessageModule.IDENTITY).received_pipeline.connect(received_pipeline_listener); - stream.get_module(Presence.Module.IDENTITY).received_presence.connect(check_for_enter_error); stream.get_module(Presence.Module.IDENTITY).received_available.connect(on_received_available); + stream.get_module(Presence.Module.IDENTITY).received_presence.connect(check_for_enter_error); stream.get_module(Presence.Module.IDENTITY).received_unavailable.connect(on_received_unavailable); stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); } @@ -249,8 +258,8 @@ public class Module : XmppStreamModule { public override void detach(XmppStream stream) { stream.get_module(MessageModule.IDENTITY).received_message.disconnect(on_received_message); stream.get_module(MessageModule.IDENTITY).received_pipeline.disconnect(received_pipeline_listener); - stream.get_module(Presence.Module.IDENTITY).received_presence.disconnect(check_for_enter_error); stream.get_module(Presence.Module.IDENTITY).received_available.disconnect(on_received_available); + stream.get_module(Presence.Module.IDENTITY).received_presence.disconnect(check_for_enter_error); stream.get_module(Presence.Module.IDENTITY).received_unavailable.disconnect(on_received_unavailable); stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); } @@ -339,7 +348,8 @@ public class Module : XmppStreamModule { query_affiliation.begin(stream, bare_jid, "owner"); flag.finish_muc_enter(bare_jid); - flag.enter_futures[bare_jid].set_value(new JoinResult() {nick=presence.from.resourcepart}); + var join_result = new JoinResult() { nick=presence.from.resourcepart, newly_created=status_codes.contains(StatusCode.NEW_ROOM_CREATED) }; + flag.enter_futures[bare_jid].set_value(join_result); } flag.set_muc_nick(presence.from); diff --git a/xmpp-vala/src/module/xep/0166_jingle/component.vala b/xmpp-vala/src/module/xep/0166_jingle/component.vala index 5d573522..e30175d5 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/component.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/component.vala @@ -2,6 +2,8 @@ namespace Xmpp.Xep.Jingle { public abstract class ComponentConnection : Object { public uint8 component_id { get; set; default = 0; } + public ulong bytes_sent { get; protected set; default=0; } + public ulong bytes_received { get; protected set; default=0; } public abstract async void terminate(bool we_terminated, string? reason_name = null, string? reason_text = null); public signal void connection_closed(); public signal void connection_error(IOError e); diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index b51bb26d..31d8f9fc 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -95,16 +95,27 @@ public class Xmpp.Xep.Jingle.Content : Object { } public void accept() { + if (state != State.PENDING) { + warning("accepting a non-pending content"); + return; + } state = State.WANTS_TO_BE_ACCEPTED; - session.accept_content(this); } public void reject() { + if (state != State.PENDING) { + warning("rejecting a non-pending content"); + return; + } session.reject_content(this); } public void terminate(bool we_terminated, string? reason_name, string? reason_text) { + if (state == State.PENDING) { + warning("terminating a pending call"); + return; + } content_params.terminate(we_terminated, reason_name, reason_text); transport_params.dispose(); @@ -137,7 +148,7 @@ public class Xmpp.Xep.Jingle.Content : Object { this.content_params.handle_accept(stream, this.session, this, content_node.description); } - private async void select_new_transport() { + public async void select_new_transport() { XmppStream stream = session.stream; Transport? new_transport = yield stream.get_module(Module.IDENTITY).select_transport(stream, transport.type_, transport_params.components, peer_full_jid, tried_transport_methods); if (new_transport == null) { diff --git a/xmpp-vala/src/module/xep/0166_jingle/session.vala b/xmpp-vala/src/module/xep/0166_jingle/session.vala index a45fc6db..af913aab 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/session.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/session.vala @@ -264,10 +264,7 @@ public class Xmpp.Xep.Jingle.Session : Object { warning("Received invalid session accept: %s", e.message); } } - // TODO(hrxi): more sanity checking, perhaps replace who we're talking to - if (!responder.is_full()) { - throw new IqError.BAD_REQUEST("invalid responder JID"); - } + foreach (ContentNode content_node in content_nodes) { handle_content_accept(content_node); } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index eadc1c8b..c4c299c5 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -21,6 +21,10 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public Gee.List<Crypto> remote_cryptos = new ArrayList<Crypto>(); public Crypto? local_crypto = null; public Crypto? remote_crypto = null; + public Jid? muji_muc = null; + + public bool rtp_ready { get; private set; default=false; } + public bool rtcp_ready { get; private set; default=false; } public weak Stream? stream { get; private set; } @@ -28,6 +32,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public Parameters(Module parent, string media, Gee.List<PayloadType> payload_types, + Jid? muji_muc, string? ssrc = null, bool rtcp_mux = false, string? bandwidth = null, string? bandwidth_type = null, bool encryption_required = false, Crypto? local_crypto = null @@ -41,6 +46,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.encryption_required = encryption_required; this.payload_types = payload_types; this.local_crypto = local_crypto; + this.muji_muc = muji_muc; } public Parameters.from_node(Module parent, StanzaNode node) throws Jingle.IqError { @@ -61,6 +67,10 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { foreach (StanzaNode subnode in node.get_subnodes(HeaderExtension.NAME, HeaderExtension.NS_URI)) { this.header_extensions.add(HeaderExtension.parse(subnode)); } + string? muji_muc_str = node.get_deep_attribute(Xep.Muji.NS_URI + ":muji", "muc"); + if (muji_muc_str != null) { + muji_muc = new Jid(muji_muc_str); + } } public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { @@ -95,6 +105,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ulong rtcp_ready_handler_id = 0; rtcp_ready_handler_id = rtcp_datagram.notify["ready"].connect((rtcp_datagram, _) => { this.stream.on_rtcp_ready(); + this.rtcp_ready = true; ((Jingle.DatagramConnection)rtcp_datagram).disconnect(rtcp_ready_handler_id); rtcp_ready_handler_id = 0; @@ -103,8 +114,10 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ulong rtp_ready_handler_id = 0; rtp_ready_handler_id = rtp_datagram.notify["ready"].connect((rtp_datagram, _) => { this.stream.on_rtp_ready(); + this.rtp_ready = true; if (rtcp_mux) { this.stream.on_rtcp_ready(); + this.rtcp_ready = true; } connection_ready(); @@ -138,6 +151,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } this.stream = parent.create_stream(content); + this.stream.weak_ref(() => this.stream = null); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); this.stream.on_send_rtp_data.connect(rtp_datagram.send_datagram); @@ -202,6 +216,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { if (rtcp_mux) { ret.put_node(new StanzaNode.build("rtcp-mux", NS_URI)); } + if (muji_muc != null) { + ret.put_node(new StanzaNode.build("muji", Xep.Muji.NS_URI).add_self_xmlns().put_attribute("muc", muji_muc.to_string())); + } return ret; } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 6b55cbe6..9dab5dc2 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -19,6 +19,7 @@ public abstract class Module : XmppStreamModule { } public abstract async Gee.List<PayloadType> get_supported_payloads(string media); + public abstract async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type); public abstract async PayloadType? pick_payload_type(string media, Gee.List<PayloadType> payloads); public abstract Crypto? generate_local_crypto(); public abstract Crypto? pick_remote_crypto(Gee.List<Crypto> cryptos); @@ -28,7 +29,7 @@ public abstract class Module : XmppStreamModule { public abstract Gee.List<HeaderExtension> get_suggested_header_extensions(string media); public abstract void close_stream(Stream stream); - public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error { + public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string sid, Jid? muji_muc) throws Jingle.Error { Jingle.Module jingle_module = stream.get_module(Jingle.Module.IDENTITY); @@ -40,7 +41,7 @@ public abstract class Module : XmppStreamModule { ArrayList<Jingle.Content> contents = new ArrayList<Jingle.Content>(); // Create audio content - Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); + Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio"), muji_muc); audio_content_parameters.local_crypto = generate_local_crypto(); audio_content_parameters.header_extensions.add_all(get_suggested_header_extensions("audio")); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); @@ -48,7 +49,7 @@ public abstract class Module : XmppStreamModule { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); } Jingle.TransportParameters audio_transport_params = audio_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); - Jingle.Content audio_content = new Jingle.Content.initiate_sent("voice", Jingle.Senders.BOTH, + Jingle.Content audio_content = new Jingle.Content.initiate_sent("audio", Jingle.Senders.BOTH, content_type, audio_content_parameters, audio_transport, audio_transport_params, null, null, @@ -58,7 +59,7 @@ public abstract class Module : XmppStreamModule { Jingle.Content? video_content = null; if (video) { // Create video content - Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video"), muji_muc); video_content_parameters.local_crypto = generate_local_crypto(); video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); @@ -66,7 +67,7 @@ public abstract class Module : XmppStreamModule { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); } Jingle.TransportParameters video_transport_params = video_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); - video_content = new Jingle.Content.initiate_sent("webcam", Jingle.Senders.BOTH, + video_content = new Jingle.Content.initiate_sent("video", Jingle.Senders.BOTH, content_type, video_content_parameters, video_transport, video_transport_params, null, null, @@ -83,7 +84,7 @@ public abstract class Module : XmppStreamModule { } } - public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) throws Jingle.Error { + public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session, Jid? muji_muc) throws Jingle.Error { Jid my_jid = session.local_full_jid; Jid receiver_full_jid = session.peer_full_jid; @@ -100,7 +101,7 @@ public abstract class Module : XmppStreamModule { if (content == null) { // Content for video does not yet exist -> create it - Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video"), muji_muc); video_content_parameters.local_crypto = generate_local_crypto(); video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); @@ -108,7 +109,7 @@ public abstract class Module : XmppStreamModule { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); } Jingle.TransportParameters video_transport_params = video_transport.create_transport_parameters(stream, content_type.required_components, my_jid, receiver_full_jid); - content = new Jingle.Content.initiate_sent("webcam", + content = new Jingle.Content.initiate_sent("video", session.we_initiated ? Jingle.Senders.INITIATOR : Jingle.Senders.RESPONDER, content_type, video_content_parameters, video_transport, video_transport_params, diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala index faba38c9..73bc9800 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala @@ -64,12 +64,27 @@ public class Xmpp.Xep.JingleRtp.PayloadType { } public static bool equals_func(PayloadType a, PayloadType b) { - return a.id == b.id && + bool simple = a.id == b.id && a.name == b.name && a.channels == b.channels && a.clockrate == b.clockrate && a.maxptime == b.maxptime && - a.ptime == b.ptime; + a.ptime == b.ptime && + a.parameters.size == b.parameters.size && + a.rtcp_fbs.size == b.rtcp_fbs.size; + if (!simple) return false; + foreach (string key in a.parameters.keys) { + if (!b.parameters.has_key(key)) return false; + if (a.parameters[key] != b.parameters[key]) return false; + } + foreach (RtcpFeedback fb in a.rtcp_fbs) { + if (!b.rtcp_fbs.any_match((it) => it.type_ == fb.type_ && it.subtype == fb.subtype)) return false; + } + return true; + } + + public static uint hash_func(PayloadType payload_type) { + return payload_type.to_xml().to_string().hash(); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 65be8a0a..031f0ad0 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -1,5 +1,4 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { - public Jingle.Content content { get; protected set; } public string name { get { @@ -54,6 +53,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { return false; }} + // Receiver Estimated Maximum Bitrate + public bool remb_enabled { get { + return payload_type != null ? payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb") : false; + }} + public uint target_receive_bitrate { get; set; default=256; } + public uint target_send_bitrate { get; set; default=256; } + protected Stream(Jingle.Content content) { this.content = content; } diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 07b599ee..680d7c80 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -28,6 +28,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T private bool connection_created = false; protected weak Jingle.Content? content = null; + protected bool use_raw = false; protected IceUdpTransportParameters(uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { this.components_ = components; @@ -80,10 +81,16 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T node.put_node(fingerprint_node); } - foreach (Candidate candidate in unsent_local_candidates) { + if (action_type != "transport-info") { + foreach (Candidate candidate in unsent_local_candidates) { + node.put_node(candidate.to_xml()); + } + unsent_local_candidates.clear(); + } else if (!unsent_local_candidates.is_empty) { + Candidate candidate = unsent_local_candidates.first(); node.put_node(candidate.to_xml()); + unsent_local_candidates.remove(candidate); } - unsent_local_candidates.clear(); return node; } @@ -128,15 +135,15 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T local_candidates.add(candidate); if (this.content != null && (this.connection_created || !this.incoming)) { - Timeout.add(50, () => { + Idle.add( () => { check_send_transport_info(); - return false; + return Source.REMOVE; }); } } private void check_send_transport_info() { - if (this.content != null && unsent_local_candidates.size > 0) { + if (this.content != null && !unsent_local_candidates.is_empty) { content.send_transport_info(to_transport_stanza_node("transport-info")); } } diff --git a/xmpp-vala/src/module/xep/0177_jingle_raw_udp.vala b/xmpp-vala/src/module/xep/0177_jingle_raw_udp.vala new file mode 100644 index 00000000..200cdfa9 --- /dev/null +++ b/xmpp-vala/src/module/xep/0177_jingle_raw_udp.vala @@ -0,0 +1,118 @@ +using Gee; +using Xmpp.Xep; +using Xmpp; + +namespace Xmpp.Xep.JingleRawUdp { + + public const string NS_URI = "urn:xmpp:jingle:transports:raw-udp:1"; + + public delegate Gee.List<string> GetLocalIpAddresses(); + + public class Module : XmppStreamModule, Jingle.Transport { + public static ModuleIdentity<Module> IDENTITY = new ModuleIdentity<Module>(NS_URI, "0177_jingle_raw_udp"); + + private GetLocalIpAddresses? get_local_ip_addresses_impl = null; + + public override void attach(XmppStream stream) { + stream.get_module(Jingle.Module.IDENTITY).register_transport(this); + stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); + } + public override void detach(XmppStream stream) { + stream.get_module(ServiceDiscovery.Module.IDENTITY).remove_feature(stream, NS_URI); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + + public async bool is_transport_available(XmppStream stream, uint8 components, Jid full_jid) { + return yield stream.get_module(ServiceDiscovery.Module.IDENTITY).has_entity_feature(stream, full_jid, NS_URI); + } + + public string ns_uri{ get { return NS_URI; } } + public Jingle.TransportType type_{ get { return Jingle.TransportType.DATAGRAM; } } + public int priority { get { return 1; } } + + public void set_local_ip_address_handler(owned GetLocalIpAddresses get_local_ip_addresses) { + get_local_ip_addresses_impl = (owned)get_local_ip_addresses; + } + + public Gee.List<string> get_local_ip_addresses() { + if (get_local_ip_addresses_impl == null) { + return Gee.List.empty(); + } + return get_local_ip_addresses_impl(); + } + + public Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { + return new TransportParameters(components, null); + } + + public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { + return new TransportParameters(components, transport); + } + } + + public class TransportParameters : Jingle.TransportParameters, Object { + public string ns_uri { get { return NS_URI; } } + public uint8 components { get; } + + public Gee.List<Candidate> remote_candidates = new ArrayList<Candidate>(); + public Gee.List<Candidate> own_candidates = new ArrayList<Candidate>(); + + public TransportParameters(uint8 components, StanzaNode? node = null) { +// this.components = components; + if (node != null) { + foreach (StanzaNode candidate_node in node.get_subnodes("candidate")) { + Candidate candidate = new Candidate(); + string component_str = candidate_node.get_attribute("component"); + candidate.component = int.parse(component_str); + string generation_str = candidate_node.get_attribute("generation"); + candidate.generation = int.parse(generation_str); + candidate.id = candidate_node.get_attribute("generation"); + string ip_str = candidate_node.get_attribute("ip"); + candidate.ip = new InetAddress.from_string(ip_str); + string port_str = candidate_node.get_attribute("port"); + candidate.port = int.parse(port_str); + + remote_candidates.add(candidate); + } + } + } + + public void set_content(Jingle.Content content) { + + } + + public StanzaNode to_transport_stanza_node(string action_type) { + StanzaNode transport_node = new StanzaNode.build("transport", NS_URI).add_self_xmlns(); + foreach (Candidate candidate in own_candidates) { + transport_node.put_node(new StanzaNode.build("candidate", NS_URI) + .put_attribute("generation", candidate.generation.to_string()) + .put_attribute("id", candidate.id) + .put_attribute("ip", candidate.ip.to_string()) + .put_attribute("port", candidate.port.to_string())); + } + return transport_node; + } + + public void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { + + } + + public void handle_transport_info(StanzaNode transport) throws Jingle.IqError { + + } + + public void create_transport_connection(XmppStream stream, Jingle.Content content) { + + } + } + + public class Candidate : Object { + public int component { get; set; } + public int generation { get; set; } + public string id { get; set; } + public InetAddress ip { get; set; } + public uint port { get; set; } + } +}
\ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala index 7b213ca5..f85049ac 100644 --- a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala +++ b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala @@ -100,7 +100,11 @@ public class Module : Jingle.ContentType, XmppStreamModule { yield; // Send the file data - Jingle.StreamingConnection connection = content.component_connections.values.to_array()[0] as Jingle.StreamingConnection; + Jingle.StreamingConnection connection = content.component_connections[1] as Jingle.StreamingConnection; + if (connection == null || connection.stream == null) { + warning("Connection or stream not null"); + return; + } IOStream io_stream = yield connection.stream.wait_async(); yield io_stream.input_stream.close_async(); yield io_stream.output_stream.splice_async(input_stream, OutputStreamSpliceFlags.CLOSE_SOURCE|OutputStreamSpliceFlags.CLOSE_TARGET); diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index 32ba8bb9..0fe9ce5f 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -759,6 +759,10 @@ class Parameters : Jingle.TransportParameters, Object { } private void content_set_transport_connection_error(Error e) { + Jingle.Content? strong_content = content; + if (strong_content == null) return; + + strong_content.select_new_transport.begin(); connection.set_error(e); } diff --git a/xmpp-vala/src/module/xep/0272_muji.vala b/xmpp-vala/src/module/xep/0272_muji.vala new file mode 100644 index 00000000..2bdc068e --- /dev/null +++ b/xmpp-vala/src/module/xep/0272_muji.vala @@ -0,0 +1,286 @@ +using Gee; +namespace Xmpp.Xep.Muji { + + public const string NS_URI = "http://telepathy.freedesktop.org/muji"; + + public class Module : XmppStreamModule { + public static ModuleIdentity<Module> IDENTITY = new ModuleIdentity<Module>(NS_URI, "0272_muji"); + + public async GroupCall? join_call(XmppStream stream, Jid muc_jid, bool video) { + StanzaNode initial_muji_node = new StanzaNode.build("muji", NS_URI).add_self_xmlns() + .put_node(new StanzaNode.build("preparing", NS_URI)); + + var group_call = new GroupCall(muc_jid); + stream.get_flag(Flag.IDENTITY).calls[muc_jid] = group_call; + + group_call.our_nick = "%08x".printf(Random.next_int()); + debug(@"[%s] MUJI joining as %s", stream.get_flag(Bind.Flag.IDENTITY).my_jid.to_string(), group_call.our_nick); + Xep.Muc.JoinResult? result = yield stream.get_module(Muc.Module.IDENTITY).enter(stream, muc_jid, group_call.our_nick, null, null, initial_muji_node); + if (result == null || result.nick == null) return null; + debug(@"[%s] MUJI joining as %s done", stream.get_flag(Bind.Flag.IDENTITY).my_jid.to_string(), group_call.our_nick); + + // Determine all participants that have finished preparation. Those are the ones we have to initiate the call with. + Gee.List<Presence.Stanza> other_presences = yield wait_for_preparing_peers(stream, muc_jid); + var finished_real_jids = new ArrayList<Jid>(Jid.equals_func); + foreach (Presence.Stanza presence in other_presences) { + if (presence.stanza.get_deep_subnode(NS_URI + ":muji", NS_URI + ":preparing") != null) continue; + Jid? real_jid = stream.get_flag(Muc.Flag.IDENTITY).get_real_jid(presence.from); + if (real_jid == null) { + warning("Don't know the real jid for %s", presence.from.to_string()); + continue; + } + finished_real_jids.add(real_jid); + } + group_call.peers_to_connect_to.add_all(finished_real_jids); + + // Build+send our own MUJI presence + StanzaNode muji_node = new StanzaNode.build("muji", NS_URI).add_self_xmlns(); + + foreach (string media in video ? new string[] { "audio", "video" } : new string[] { "audio" }) { + StanzaNode content_node = new StanzaNode.build("content", Xep.Jingle.NS_URI).add_self_xmlns() + .put_attribute("name", media); + StanzaNode description_node = new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns() + .put_attribute("media", media); + content_node.put_node(description_node); + + Gee.List<Xep.JingleRtp.PayloadType> payload_types = null; + if (other_presences.is_empty) { + payload_types = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).get_supported_payloads(media); + } else { + yield compute_payload_intersection(stream, group_call, media); + payload_types = group_call.current_payload_intersection[media]; + } + foreach (Xep.JingleRtp.PayloadType payload_type in payload_types) { + description_node.put_node(payload_type.to_xml().add_self_xmlns()); + } + muji_node.put_node(content_node); + } + + Presence.Stanza presence_stanza = new Presence.Stanza() { to=muc_jid.with_resource(group_call.our_nick) }; + presence_stanza.stanza.put_node(muji_node); + stream.get_module(Presence.Module.IDENTITY).send_presence(stream, presence_stanza); + + return group_call; + } + + private async Gee.List<Presence.Stanza> wait_for_preparing_peers(XmppStream stream, Jid muc_jid) { + var promise = new Promise<Gee.List<Presence.Stanza>>(); + + ArrayList<Jid> preparing_peers = new ArrayList<Jid>(Jid.equals_func); + + Gee.List<Presence.Stanza> presences = get_other_presences(stream, muc_jid); + + foreach (Presence.Stanza presence in presences) { + StanzaNode? preparing_node = presence.stanza.get_deep_subnode(NS_URI + ":muji", NS_URI + ":preparing"); + if (preparing_node != null) { + preparing_peers.add(presence.from); + } + } + + debug("[%s] MUJI waiting for %i/%i peers", stream.get_flag(Bind.Flag.IDENTITY).my_jid.to_string(), preparing_peers.size, presences.size); + + if (preparing_peers.is_empty) { + return presences; + } + + GroupCall group_call = stream.get_flag(Flag.IDENTITY).calls[muc_jid]; + group_call.waiting_for_finish_prepares[promise] = preparing_peers; + + return yield promise.future.wait_async(); + } + + private async void compute_payload_intersection(XmppStream stream, GroupCall group_call, string media) { + Gee.List<Presence.Stanza> presences = get_other_presences(stream, group_call.muc_jid); + if (presences.is_empty) return; + + Gee.List<Xep.JingleRtp.PayloadType> intersection = parse_payload_types(stream, media, presences[0]); + var remove_payloads = new ArrayList<Xep.JingleRtp.PayloadType>(); + + // Check if all peers support the payloads + foreach (Presence.Stanza presence in presences) { + Gee.List<Xep.JingleRtp.PayloadType> peer_payload_types = parse_payload_types(stream, media, presence); + + foreach (Xep.JingleRtp.PayloadType payload_type in intersection) { + if (!peer_payload_types.contains(payload_type)) { + remove_payloads.add(payload_type); + } + } + } + // Check if we support the payloads + foreach (Xep.JingleRtp.PayloadType payload_type in intersection) { + if (!yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).is_payload_supported(media, payload_type)) { + remove_payloads.add(payload_type); + } + } + // Remove payloads not supported by everyone + foreach (Xep.JingleRtp.PayloadType payload_type in remove_payloads) { + intersection.remove(payload_type); + } + + // Check if the payload intersection changed (if so: notify) + bool changed = !group_call.current_payload_intersection.has_key(media) || + !group_call.current_payload_intersection[media].contains_all(intersection) || + !intersection.contains_all(group_call.current_payload_intersection[media]); + + if (changed) { + group_call.current_payload_intersection[media] = intersection; + group_call.codecs_changed(intersection); + } + } + + private Gee.List<Xep.JingleRtp.PayloadType> parse_payload_types(XmppStream stream, string media, Presence.Stanza presence) { + Gee.List<Xep.JingleRtp.PayloadType> ret = new ArrayList<Xep.JingleRtp.PayloadType>(Xep.JingleRtp.PayloadType.equals_func); + + foreach (StanzaNode content_node in presence.stanza.get_deep_subnodes(NS_URI + ":muji", Xep.Jingle.NS_URI + ":content")) { + StanzaNode? description_node = content_node.get_subnode("description", Xep.JingleRtp.NS_URI); + if (description_node == null) continue; + + if (description_node.get_attribute("media") == media) { + Gee.List<StanzaNode> payload_nodes = description_node.get_subnodes("payload-type", Xep.JingleRtp.NS_URI); + foreach (StanzaNode payload_node in payload_nodes) { + Xep.JingleRtp.PayloadType payload_type = Xep.JingleRtp.PayloadType.parse(payload_node); + ret.add(payload_type); + } + } + } + return ret; + } + + private void on_received_available(XmppStream stream, Presence.Stanza presence) { + StanzaNode? muji_node = presence.stanza.get_subnode("muji", NS_URI); + if (muji_node == null) return; + + var flag = stream.get_flag(Flag.IDENTITY); + GroupCall? group_call = flag.calls.get(presence.from.bare_jid); + if (group_call == null) return; + + if (presence.from.resourcepart == group_call.our_nick) return; + + foreach (StanzaNode content_node in muji_node.get_subnodes("content", Xep.Jingle.NS_URI)) { + StanzaNode? description_node = content_node.get_subnode("description", Xep.JingleRtp.NS_URI); + if (description_node == null) continue; + + string? media = description_node.get_attribute("media"); + if (media == null) continue; + + compute_payload_intersection.begin(stream, group_call, media); + } + + StanzaNode? prepare_node = muji_node.get_subnode("preparing", NS_URI); + if (prepare_node == null) { + on_jid_finished_preparing(stream, presence.from, group_call); + + if (!group_call.peers.contains(presence.from)) { + // A new peer finished preparing + Jid? real_jid = stream.get_flag(Muc.Flag.IDENTITY).get_real_jid(presence.from); + if (real_jid == null) { + warning("Don't know the real jid for %s", presence.from.to_string()); + return; + } + debug("Muji peer joined %s / %s\n", real_jid.to_string(), presence.from.to_string()); + group_call.peers.add(presence.from); + group_call.real_jids[presence.from] = real_jid; + group_call.peer_joined(real_jid); + } + } + } + + private void on_received_unavailable(XmppStream stream, Presence.Stanza presence) { + Flag flag = stream.get_flag(Flag.IDENTITY); + GroupCall? group_call = flag.calls[presence.from.bare_jid]; + if (group_call == null) return; + + debug("Muji peer left %s / %s", group_call.real_jids.has_key(presence.from) ? group_call.real_jids[presence.from].to_string() : "Unknown real JID", presence.from.to_string()); + on_jid_finished_preparing(stream, presence.from, group_call); + group_call.peers.remove(presence.from); + group_call.peers_to_connect_to.remove(presence.from); + if (group_call.real_jids.has_key(presence.from)) { + group_call.peer_left(group_call.real_jids[presence.from]); + } + group_call.real_jids.remove(presence.from); + } + + private void on_jid_finished_preparing(XmppStream stream, Jid jid, GroupCall group_call) { + debug("Muji peer finished preparing %s", jid.to_string()); + foreach (Promise<Gee.List<Presence.Stanza>> promise in group_call.waiting_for_finish_prepares.keys) { + debug("Waiting for finish prepares %i", group_call.waiting_for_finish_prepares[promise].size); + Gee.List<Jid> outstanding_prepares = group_call.waiting_for_finish_prepares[promise]; + if (outstanding_prepares.contains(jid)) { + outstanding_prepares.remove(jid); + debug("Waiting for finish prepares %i", group_call.waiting_for_finish_prepares[promise].size); + + if (outstanding_prepares.is_empty) { + Gee.List<Presence.Stanza> presences = get_other_presences(stream, jid.bare_jid); + promise.set_value(presences); + } + } + } + } + + private Gee.List<Presence.Stanza> get_other_presences(XmppStream stream, Jid muc_jid) { + Gee.List<Presence.Stanza> presences = stream.get_flag(Presence.Flag.IDENTITY).get_presences(muc_jid); + string? own_nick = stream.get_flag(Flag.IDENTITY).calls[muc_jid].our_nick; + + var remove_presences = new ArrayList<Presence.Stanza>(); + foreach (Presence.Stanza presence in presences) { + if (presence.from.resourcepart == own_nick) { + remove_presences.add(presence); + } + StanzaNode? muji_node = presence.stanza.get_subnode("muji", NS_URI); + if (muji_node == null) { + remove_presences.add(presence); + } + } + presences.remove_all(remove_presences); + return presences; + } + + public override void attach(XmppStream stream) { + stream.add_flag(new Flag()); + stream.get_module(Presence.Module.IDENTITY).received_available.connect(on_received_available); + stream.get_module(Presence.Module.IDENTITY).received_unavailable.connect(on_received_unavailable); + } + + public override void detach(XmppStream stream) { } + + public override string get_ns() { + return NS_URI; + } + + public override string get_id() { + return IDENTITY.id; + } + } + + public class GroupCall { + public string our_nick; + public Jid muc_jid; + public ArrayList<Jid> peers_to_connect_to = new ArrayList<Jid>(Jid.equals_func); + public ArrayList<Jid> peers = new ArrayList<Jid>(Jid.equals_func); + public HashMap<Jid, Jid> real_jids = new HashMap<Jid, Jid>(Jid.hash_func, Jid.equals_func); + public HashMap<Promise, Gee.List<Jid>> waiting_for_finish_prepares = new HashMap<Promise, Gee.List<Jid>>(); + public HashMap<string, Gee.List<Xep.JingleRtp.PayloadType>> current_payload_intersection = new HashMap<string, Gee.List<Xep.JingleRtp.PayloadType>>(); + + public signal void peer_joined(Jid real_jid); + public signal void peer_left(Jid real_jid); + public signal void codecs_changed(Gee.List<Xep.JingleRtp.PayloadType> payload_types); + + public GroupCall(Jid muc_jid) { + this.muc_jid = muc_jid; + } + + public void leave(XmppStream stream) { + stream.get_module(Xep.Muc.Module.IDENTITY).exit(stream, muc_jid); + stream.get_flag(Flag.IDENTITY).calls.unset(muc_jid); + } + } + + public class Flag : XmppStreamFlag { + public static FlagIdentity<Flag> IDENTITY = new FlagIdentity<Flag>(NS_URI, "muji"); + + public HashMap<Jid, GroupCall> calls = new HashMap<Jid, GroupCall>(Jid.hash_bare_func, Jid.equals_bare_func); + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } +}
\ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0298_coin.vala b/xmpp-vala/src/module/xep/0298_coin.vala new file mode 100644 index 00000000..58b3b055 --- /dev/null +++ b/xmpp-vala/src/module/xep/0298_coin.vala @@ -0,0 +1,136 @@ +using Gee; + +namespace Xmpp.Xep.Coin { + private const string NS_RFC = "urn:ietf:params:xml:ns:conference-info"; + + public class Module : XmppStreamModule, Iq.Handler { + public static ModuleIdentity<Module> IDENTITY = new ModuleIdentity<Module>(NS_RFC, "0298_coin"); + + public signal void coin_info_received(Jid jid, ConferenceInfo info); + + public async override void on_iq_set(XmppStream stream, Iq.Stanza iq) { + ConferenceInfo? info = parse_node(iq.stanza.get_subnode("conference-info", NS_RFC), null); + if (info == null) return; + + coin_info_received(iq.from, info); + } + + public override void attach(XmppStream stream) { + stream.get_module(Iq.Module.IDENTITY).register_for_namespace(NS_RFC, this); + } + + public override void detach(XmppStream stream) { } + + public override string get_ns() { return NS_RFC; } + + public override string get_id() { return IDENTITY.id; } + } + + public ConferenceInfo? parse_node(StanzaNode conference_node, ConferenceInfo? previous_conference_info) { + string? version_str = conference_node.get_attribute("version"); + string? conference_state = conference_node.get_attribute("state"); + if (version_str == null || conference_state == null) return null; + + int version = int.parse(version_str); + if (previous_conference_info != null && version <= previous_conference_info.version) return null; + + ConferenceInfo conference_info = previous_conference_info ?? new ConferenceInfo(); + conference_info.version = version; + + Gee.List<StanzaNode> user_nodes = conference_node.get_deep_subnodes(NS_RFC + ":users", NS_RFC + ":user"); + foreach (StanzaNode user_node in user_nodes) { + string? jid_string = user_node.get_attribute("entity"); + if (jid_string == null) continue; +// if (!jid_string.has_prefix("xmpp:")) continue; // silk does this wrong + Jid? jid = null; + try { + jid = new Jid(jid_string.substring(4)); + } catch (Error e) { + continue; + } + string user_state = user_node.get_attribute("state"); + if (conference_state == "full" && user_state != "full") return null; + + if (user_state == "deleted") { + conference_info.users.unset(jid); + continue; + } + + ConferenceUser user = new ConferenceUser(); + user.jid = jid; + user.display_text = user_node.get_deep_string_content(NS_RFC + ":display-text"); + + Gee.List<StanzaNode> endpoint_nodes = user_node.get_subnodes("endpoint"); + foreach (StanzaNode entpoint_node in endpoint_nodes) { + Gee.List<StanzaNode> media_nodes = entpoint_node.get_subnodes("media"); + foreach (StanzaNode media_node in media_nodes) { + string? id = media_node.get_attribute("id"); + string? ty = media_node.get_deep_string_content(NS_RFC + ":type"); + string? src_id_str = media_node.get_deep_string_content(NS_RFC + ":src-id"); + + if (id == null) continue; + + ConferenceMedia media = new ConferenceMedia(); + media.id = id; + media.src_id = src_id_str != null ? int.parse(src_id_str) : -1; + media.ty = ty; + user.medias[id] = media; + } + + conference_info.users[user.jid] = user; + } + } + return conference_info; + } + + public class ConferenceInfo { + public int version = -1; + public HashMap<Jid, ConferenceUser> users = new HashMap<Jid, ConferenceUser>(Jid.hash_func, Jid.equals_func); + + public StanzaNode to_xml() { + StanzaNode ret = new StanzaNode.build("conference-info", NS_RFC).add_self_xmlns() + .put_attribute("version", this.version.to_string()) + .put_attribute("state", "full"); + StanzaNode users_node = new StanzaNode.build("users", NS_RFC); + + foreach (ConferenceUser user in this.users.values) { + users_node.put_node(user.to_xml()); + } + ret.put_node(users_node); + return ret; + } + } + + public class ConferenceUser { + public Jid jid; + public string? display_text; + public HashMap<string, ConferenceMedia> medias = new HashMap<string, ConferenceMedia>(); + + public StanzaNode to_xml() { + StanzaNode user_node = new StanzaNode.build("user", NS_RFC) + .put_attribute("entity", jid.to_string()); + foreach (ConferenceMedia media in medias.values) { + user_node.put_node(media.to_xml()); + } + return user_node; + } + } + + public class ConferenceMedia { + public string id; + public string? ty; + public int src_id = -1; + + public StanzaNode to_xml() { + StanzaNode media_node = new StanzaNode.build("media", NS_RFC) + .put_attribute("id", id); + if (ty != null) { + media_node.put_node(new StanzaNode.build("type", NS_RFC).put_node(new StanzaNode.text(ty))); + } + if (src_id != -1) { + media_node.put_node(new StanzaNode.build("src-id", NS_RFC).put_node(new StanzaNode.text(src_id.to_string()))); + } + return media_node; + } + } +}
\ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index 71e16a95..ac1d8329 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -8,7 +8,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { public signal void session_proposed(Jid from, Jid to, string sid, Gee.List<StanzaNode> descriptions); public signal void session_retracted(Jid from, Jid to, string sid); - public signal void session_accepted(Jid from, string sid); + public signal void session_accepted(Jid from, Jid to, string sid); public signal void session_rejected(Jid from, Jid to, string sid); public void send_session_propose_to_peer(XmppStream stream, Jid to, string sid, Gee.List<StanzaNode> descriptions) { @@ -65,7 +65,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { switch (mi_node.name) { case "accept": case "proceed": - session_accepted(message.from, mi_node.get_attribute("id")); + session_accepted(message.from, message.to, mi_node.get_attribute("id")); break; case "propose": ArrayList<StanzaNode> descriptions = new ArrayList<StanzaNode>(); diff --git a/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala b/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala index 8e3213ae..a8ca5016 100644 --- a/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala +++ b/xmpp-vala/src/module/xep/0384_omemo/omemo_decryptor.vala @@ -18,16 +18,25 @@ namespace Xmpp.Xep.Omemo { ParsedData ret = new ParsedData(); StanzaNode? header_node = encrypted_node.get_subnode("header"); - if (header_node == null) return null; + if (header_node == null) { + warning("Can't parse OMEMO node: No header node"); + return null; + } ret.sid = header_node.get_attribute_int("sid", -1); - if (ret.sid == -1) return null; + if (ret.sid == -1) { + warning("Can't parse OMEMO node: No sid"); + return null; + } string? payload_str = encrypted_node.get_deep_string_content("payload"); if (payload_str != null) ret.ciphertext = Base64.decode(payload_str); string? iv_str = header_node.get_deep_string_content("iv"); - if (iv_str == null) return null; + if (iv_str == null) { + warning("Can't parse OMEMO node: No iv"); + return null; + } ret.iv = Base64.decode(iv_str); foreach (StanzaNode key_node in header_node.get_subnodes("key")) { diff --git a/xmpp-vala/src/module/xep/muji_meta.vala b/xmpp-vala/src/module/xep/muji_meta.vala new file mode 100644 index 00000000..89a0e8de --- /dev/null +++ b/xmpp-vala/src/module/xep/muji_meta.vala @@ -0,0 +1,117 @@ +using Gee; +namespace Xmpp.Xep.MujiMeta { + + public const string NS_URI = "http://telepathy.freedesktop.org/muji"; + + public class Module : XmppStreamModule { + public static ModuleIdentity<Module> IDENTITY = new ModuleIdentity<Module>(NS_URI, "muji_meta"); + + public signal void call_proposed(Jid from, Jid to, Jid muc_jid, Gee.List<StanzaNode> descriptions, string message_type); + public signal void call_retracted(Jid from, Jid to, Jid muc_jid, string message_type); + public signal void call_accepted(Jid from, Jid muc_jid, string message_type); + public signal void call_rejected(Jid from, Jid to, Jid muc_jid, string message_type); + + public void send_invite(XmppStream stream, Jid invitee, Jid muc_jid, bool video, string? message_type = null) { + var invite_node = new StanzaNode.build("propose", NS_URI).put_attribute("muc", muc_jid.to_string()); + invite_node.put_node(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "audio")); + if (video) { + invite_node.put_node(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "video")); + } + var muji_node = new StanzaNode.build("muji", NS_URI).add_self_xmlns().put_node(invite_node); + MessageStanza invite_message = new MessageStanza() { to=invitee, type_=message_type }; + invite_message.stanza.put_node(muji_node); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, invite_message); + } + + public void send_invite_retract_to_peer(XmppStream stream, Jid invitee, Jid muc_jid, string? message_type = null) { + send_jmi_message(stream, "retract", invitee, muc_jid, message_type); + } + + public void send_invite_accept_to_peer(XmppStream stream, Jid invitor, Jid muc_jid, string? message_type = null) { + send_jmi_message(stream, "accept", invitor, muc_jid, message_type); + } + + public void send_invite_accept_to_self(XmppStream stream, Jid muc_jid) { + send_jmi_message(stream, "accept", Bind.Flag.get_my_jid(stream).bare_jid, muc_jid); + } + + public void send_invite_reject_to_peer(XmppStream stream, Jid invitor, Jid muc_jid, string? message_type = null) { + send_jmi_message(stream, "reject", invitor, muc_jid, message_type); + } + + public void send_invite_reject_to_self(XmppStream stream, Jid muc_jid) { + send_jmi_message(stream, "reject", Bind.Flag.get_my_jid(stream).bare_jid, muc_jid); + } + + private void send_jmi_message(XmppStream stream, string name, Jid to, Jid muc, string? message_type = null) { + var jmi_node = new StanzaNode.build(name, NS_URI).add_self_xmlns().put_attribute("muc", muc.to_string()); + var muji_node = new StanzaNode.build("muji", NS_URI).add_self_xmlns().put_node(jmi_node); + + MessageStanza accepted_message = new MessageStanza() { to=to, type_= message_type ?? MessageStanza.TYPE_CHAT }; + accepted_message.stanza.put_node(muji_node); + stream.get_module(MessageModule.IDENTITY).send_message.begin(stream, accepted_message); + } + + private void on_received_message(XmppStream stream, MessageStanza message) { + Xep.MessageArchiveManagement.MessageFlag? mam_flag = Xep.MessageArchiveManagement.MessageFlag.get_flag(message); + if (mam_flag != null) return; + + var muji_node = message.stanza.get_subnode("muji", NS_URI); + if (muji_node == null) return; + + StanzaNode? mi_node = null; + foreach (StanzaNode node in muji_node.sub_nodes) { + if (node.ns_uri == NS_URI) { + mi_node = node; + } + } + if (mi_node == null) return; + + string? jid_str = mi_node.get_attribute("muc"); + if (jid_str == null) return; + + Jid muc_jid = null; + try { + muc_jid = new Jid(jid_str); + } catch (Error e) { + return; + } + + switch (mi_node.name) { + case "accept": + case "proceed": + call_accepted(message.from, muc_jid, message.type_); + break; + case "propose": + ArrayList<StanzaNode> descriptions = new ArrayList<StanzaNode>(); + + foreach (StanzaNode node in mi_node.sub_nodes) { + if (node.name != "description") continue; + descriptions.add(node); + } + + if (descriptions.size > 0) { + call_proposed(message.from, message.to, muc_jid, descriptions, message.type_); + } + break; + case "retract": + call_retracted(message.from, message.to, muc_jid, message.type_); + break; + case "reject": + call_rejected(message.from, message.to, muc_jid, message.type_); + break; + } + } + + public override void attach(XmppStream stream) { + stream.get_module(MessageModule.IDENTITY).received_message.connect(on_received_message); + } + + public override void detach(XmppStream stream) { + stream.get_module(MessageModule.IDENTITY).received_message.disconnect(on_received_message); + } + + public override string get_ns() { return NS_URI; } + public override string get_id() { return IDENTITY.id; } + } +}
\ No newline at end of file |
