From 26d10d1dcb95f11b65611473c9840e13683cb5ec Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 4 Nov 2021 17:33:08 +0100 Subject: Add multiparty call support to libdino and xmpp-vala --- libdino/src/service/call_peer_state.vala | 453 +++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 libdino/src/service/call_peer_state.vala (limited to 'libdino/src/service/call_peer_state.vala') diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala new file mode 100644 index 00000000..ddd0d8dd --- /dev/null +++ b/libdino/src/service/call_peer_state.vala @@ -0,0 +1,453 @@ +using Dino.Entities; +using Gee; +using Xmpp; + +public class Dino.PeerState : Object { + public signal void counterpart_sends_video_updated(bool mute); + public signal void info_received(Xep.JingleRtp.CallSessionInfo session_info); + + public signal void connection_ready(); + public signal void session_terminated(bool we_terminated, string? reason_name, string? reason_text); + public signal void encryption_updated(Xep.Jingle.ContentEncryption? audio_encryption, Xep.Jingle.ContentEncryption? video_encryption, bool same); + + public StreamInteractor stream_interactor; + public Calls calls; + public Call call; + public Jid jid; + public Xep.Jingle.Session session; + public string sid; + public string internal_id = Xmpp.random_uuid(); + + public Xep.JingleRtp.Parameters? audio_content_parameter = null; + public Xep.JingleRtp.Parameters? video_content_parameter = null; + public Xep.Jingle.Content? audio_content = null; + public Xep.Jingle.Content? video_content = null; + public Xep.Jingle.ContentEncryption? video_encryption = null; + public Xep.Jingle.ContentEncryption? audio_encryption = null; + public bool encryption_keys_same = false; + public HashMap? video_encryptions = null; + public HashMap? audio_encryptions = null; + + public bool first_peer = false; + public bool accepted_jmi = false; + public bool waiting_for_inbound_muji_connection = false; + public Xep.Muji.GroupCall? group_call { get; set; } + + public bool counterpart_sends_video = false; + public bool we_should_send_audio { get; set; default=false; } + public bool we_should_send_video { get; set; default=false; } + + public PeerState(Jid jid, Call call, StreamInteractor stream_interactor) { + this.jid = jid; + this.call = call; + this.stream_interactor = stream_interactor; + this.calls = stream_interactor.get_module(Calls.IDENTITY); + + var session_info_type = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type; + session_info_type.mute_update_received.connect((session,mute, name) => { + if (this.sid != session.sid) return; + + foreach (Xep.Jingle.Content content in session.contents) { + if (name == null || content.content_name == name) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + on_counterpart_mute_update(mute, rtp_content_parameter.media); + } + } + } + }); + session_info_type.info_received.connect((session, session_info) => { + if (this.sid != session.sid) return; + + info_received(session_info); + }); + } + + public async void initiate_call(Jid counterpart) { + Gee.List call_resources = yield calls.get_call_resources(call.account, counterpart); + + bool do_jmi = false; + Jid? jid_for_direct = null; + if (yield calls.contains_jmi_resources(call.account, call_resources)) { + do_jmi = true; + } else if (!call_resources.is_empty) { + jid_for_direct = call_resources[0]; + } else if (calls.has_jmi_resources(jid)) { + do_jmi = true; + } + + sid = Xmpp.random_uuid(); + + if (do_jmi) { + XmppStream? stream = stream_interactor.get_stream(call.account); + + calls.current_jmi_request_call[call.account] = calls.call_states[call]; + calls.current_jmi_request_peer[call.account] = this; + + var descriptions = new ArrayList(); + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "audio")); + if (we_should_send_video) { + descriptions.add(new StanzaNode.build("description", Xep.JingleRtp.NS_URI).add_self_xmlns().put_attribute("media", "video")); + } + + stream.get_module(Xmpp.Xep.JingleMessageInitiation.Module.IDENTITY).send_session_propose_to_peer(stream, jid, sid, descriptions); + } else if (jid_for_direct != null) { + yield call_resource(jid_for_direct); + } + } + + public async void call_resource(Jid full_jid) { + XmppStream? stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + if (sid == null) sid = Xmpp.random_uuid(); + + Xep.Jingle.Session session = yield stream.get_module(Xep.JingleRtp.Module.IDENTITY).start_call(stream, full_jid, we_should_send_video, sid, group_call != null ? group_call.muc_jid : null); + set_session(session); + } + + public void accept() { + if (session != null) { + foreach (Xep.Jingle.Content content in session.contents) { + content.accept(); + } + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + accepted_jmi = true; + + calls.current_jmi_request_call[call.account] = calls.call_states[call]; + calls.current_jmi_request_peer[call.account] = this; + + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_accept_to_self(stream, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_proceed_to_peer(stream, jid, sid); + } + } + + public void reject() { + call.state = Call.State.DECLINED; + + if (session != null) { + foreach (Xep.Jingle.Content content in session.contents) { + content.reject(); + } + } else { + // Only a JMI so far + XmppStream stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_peer(stream, jid, sid); + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_reject_to_self(stream, sid); + } + } + + public void end(string terminate_reason, string? reason_text = null) { + switch (terminate_reason) { + case Xep.Jingle.ReasonElement.SUCCESS: + if (session != null) { + session.terminate(terminate_reason, reason_text, "success"); + } + break; + case Xep.Jingle.ReasonElement.CANCEL: + if (session != null) { + session.terminate(terminate_reason, reason_text, "cancel"); + } else if (group_call != null) { + // We don't have to do anything (?) + } else { + // Only a JMI so far + XmppStream? stream = stream_interactor.get_stream(call.account); + if (stream == null) return; + stream.get_module(Xep.JingleMessageInitiation.Module.IDENTITY).send_session_retract_to_peer(stream, jid, sid); + } + break; + } + } + + internal void mute_own_audio(bool mute) { + // Call isn't fully established yet. Audio will be muted once the stream is created. + if (session == null || audio_content_parameter == null || audio_content_parameter.stream == null) return; + + Xep.JingleRtp.Stream stream = audio_content_parameter.stream; + + // Inform our counterpart that we (un)muted our audio + stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY).session_info_type.send_mute(session, mute, "audio"); + + // Start/Stop sending audio data + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + internal void mute_own_video(bool mute) { + + if (session == null) { + // Call hasn't been established yet + return; + } + + Xep.JingleRtp.Module rtp_module = stream_interactor.module_manager.get_module(call.account, Xep.JingleRtp.Module.IDENTITY); + + if (video_content_parameter != null && + video_content_parameter.stream != null && + session.senders_include_us(video_content.senders)) { + // A video content already exists + + // Start/Stop sending video data + Xep.JingleRtp.Stream stream = video_content_parameter.stream; + if (stream != null) { + Application.get_default().plugin_registry.video_call_plugin.set_pause(stream, mute); + } + + // Inform our counterpart that we started/stopped our video + rtp_module.session_info_type.send_mute(session, mute, "video"); + } else if (!mute) { + // Add a new video content + XmppStream stream = stream_interactor.get_stream(call.account); + rtp_module.add_outgoing_video_content.begin(stream, session, group_call != null ? group_call.muc_jid : null, (_, res) => { + if (video_content_parameter == null) { + Xep.Jingle.Content content = rtp_module.add_outgoing_video_content.end(res); + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter != null) { + connect_content_signals(content, rtp_content_parameter); + } + } + }); + } + // If video_content_parameter == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. + } + + public Xep.JingleRtp.Stream? get_video_stream(Call call) { + if (video_content_parameter != null) { + return video_content_parameter.stream; + } + return null; + } + + public Xep.JingleRtp.Stream? get_audio_stream(Call call) { + if (audio_content_parameter != null) { + return audio_content_parameter.stream; + } + return null; + } + + internal void set_session(Xep.Jingle.Session session) { + this.session = session; + this.sid = session.sid; + + session.terminated.connect((stream, we_terminated, reason_name, reason_text) => + session_terminated(we_terminated, reason_name, reason_text) + ); + session.additional_content_add_incoming.connect((session,stream, content) => + on_incoming_content_add(stream, session, content) + ); + + foreach (Xep.Jingle.Content content in session.contents) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + if (rtp_content_parameter == null) continue; + + connect_content_signals(content, rtp_content_parameter); + } + } + + public PeerInfo get_info() { + var ret = new PeerInfo(); + + if (audio_content_parameter != null) { + ret.audio_rtcp_ready = audio_content_parameter.rtcp_ready; + ret.audio_rtp_ready = audio_content_parameter.rtp_ready; + + if (audio_content_parameter.agreed_payload_type != null) { + ret.audio_codec = audio_content_parameter.agreed_payload_type.name; + ret.audio_clockrate = audio_content_parameter.agreed_payload_type.clockrate; + } + } + + if (audio_content != null) { + Xmpp.Xep.Jingle.ComponentConnection? component0 = audio_content.get_transport_connection(1); + if (component0 != null) { + ret.audio_bytes_received = component0.bytes_received; + ret.audio_bytes_sent = component0.bytes_sent; + } + } + + if (video_content_parameter != null) { + ret.video_content_exists = true; + ret.video_rtcp_ready = video_content_parameter.rtcp_ready; + ret.video_rtp_ready = video_content_parameter.rtp_ready; + + if (video_content_parameter.agreed_payload_type != null) { + ret.video_codec = video_content_parameter.agreed_payload_type.name; + } + } + + if (video_content != null) { + Xmpp.Xep.Jingle.ComponentConnection? component0 = video_content.get_transport_connection(1); + if (component0 != null) { + ret.video_bytes_received = component0.bytes_received; + ret.video_bytes_sent = component0.bytes_sent; + } + } + return ret; + } + + private void connect_content_signals(Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { + if (rtp_content_parameter.media == "audio") { + audio_content = content; + audio_content_parameter = rtp_content_parameter; + } else if (rtp_content_parameter.media == "video") { + video_content = content; + video_content_parameter = rtp_content_parameter; + } + + debug(@"[%s] %s connecting content signals %s", call.account.bare_jid.to_string(), jid.to_string(), rtp_content_parameter.media); + rtp_content_parameter.stream_created.connect((stream) => on_stream_created(rtp_content_parameter.media, stream)); + rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(content, rtp_content_parameter.media)); + + content.senders_modify_incoming.connect((content, proposed_senders) => { + if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { + warning("counterpart set us to (not)sending %s. ignoring", content.content_name); + return; + } + + if (!content.session.senders_include_counterpart(content.senders) && content.session.senders_include_counterpart(proposed_senders)) { + // Counterpart wants to start sending. Ok. + content.accept_content_modify(proposed_senders); + on_counterpart_mute_update(false, "video"); + } + }); + } + + private void on_incoming_content_add(XmppStream stream, Xep.Jingle.Session session, Xep.Jingle.Content content) { + Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; + + if (rtp_content_parameter == null) { + content.reject(); + return; + } + + // Our peer shouldn't tell us to start sending, that's for us to initiate + if (session.senders_include_us(content.senders)) { + if (session.senders_include_counterpart(content.senders)) { + // If our peer wants to send, let them + content.modify(session.we_initiated ? Xep.Jingle.Senders.RESPONDER : Xep.Jingle.Senders.INITIATOR); + } else { + // If only we're supposed to send, reject + content.reject(); + } + } + + connect_content_signals(content, rtp_content_parameter); + content.accept(); + } + + private void on_stream_created(string media, Xep.JingleRtp.Stream stream) { + if (media == "video" && stream.receiving) { + counterpart_sends_video = true; + video_content_parameter.connection_ready.connect((status) => { + counterpart_sends_video_updated(false); + }); + } + + // Outgoing audio/video might have been muted in the meanwhile. + if (media == "video" && !we_should_send_video) { + mute_own_video(true); + } else if (media == "audio" && !we_should_send_audio) { + mute_own_audio(true); + } + } + + private void on_counterpart_mute_update(bool mute, string? media) { + if (!call.equals(call)) return; + + if (media == "video") { + counterpart_sends_video = !mute; + debug(@"[%s] %s video muted %s", call.account.bare_jid.to_string(), jid.to_string(), mute.to_string()); + counterpart_sends_video_updated(mute); + } + } + + private void on_connection_ready(Xep.Jingle.Content content, string media) { + debug("[%s] %s on_connection_ready", call.account.bare_jid.to_string(), jid.to_string()); + connection_ready(); + + if (call.state == Call.State.RINGING || call.state == Call.State.ESTABLISHING) { + call.state = Call.State.IN_PROGRESS; + } + + if (media == "audio") { + audio_encryptions = content.encryptions; + } else if (media == "video") { + video_encryptions = content.encryptions; + } + + if ((audio_encryptions != null && audio_encryptions.is_empty) || (video_encryptions != null && video_encryptions.is_empty)) { + call.encryption = Encryption.NONE; + encryption_updated(null, null, true); + return; + } + + HashMap encryptions = audio_encryptions ?? video_encryptions; + + Xep.Jingle.ContentEncryption? omemo_encryption = null, dtls_encryption = null, srtp_encryption = null; + foreach (string encr_name in encryptions.keys) { + if (video_encryptions != null && !video_encryptions.has_key(encr_name)) continue; + + var encryption = encryptions[encr_name]; + if (encryption.encryption_ns == "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification") { + omemo_encryption = encryption; + } else if (encryption.encryption_ns == Xep.JingleIceUdp.DTLS_NS_URI) { + dtls_encryption = encryption; + } else if (encryption.encryption_name == "SRTP") { + srtp_encryption = encryption; + } + } + + if (omemo_encryption != null && dtls_encryption != null) { + call.encryption = Encryption.OMEMO; + omemo_encryption.peer_key = dtls_encryption.peer_key; + omemo_encryption.our_key = dtls_encryption.our_key; + audio_encryption = omemo_encryption; + encryption_keys_same = true; + video_encryption = video_encryptions != null ? video_encryptions["http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"] : null; + } else if (dtls_encryption != null) { + call.encryption = Encryption.DTLS_SRTP; + audio_encryption = dtls_encryption; + video_encryption = video_encryptions != null ? video_encryptions[Xep.JingleIceUdp.DTLS_NS_URI] : null; + encryption_keys_same = true; + if (video_encryption != null && dtls_encryption.peer_key.length == video_encryption.peer_key.length) { + for (int i = 0; i < dtls_encryption.peer_key.length; i++) { + if (dtls_encryption.peer_key[i] != video_encryption.peer_key[i]) { + encryption_keys_same = false; + break; + } + } + } + } else if (srtp_encryption != null) { + call.encryption = Encryption.SRTP; + audio_encryption = srtp_encryption; + video_encryption = video_encryptions != null ? video_encryptions["SRTP"] : null; + encryption_keys_same = false; + } else { + call.encryption = Encryption.NONE; + encryption_keys_same = true; + } + + encryption_updated(audio_encryption, video_encryption, encryption_keys_same); + } +} + +public class Dino.PeerInfo { + public bool audio_rtp_ready { get; set; } + public bool audio_rtcp_ready { get; set; } + public ulong? audio_bytes_sent { get; set; default=0; } + public ulong? audio_bytes_received { get; set; default=0; } + public string? audio_codec { get; set; } + public uint32 audio_clockrate { get; set; } + + public bool video_content_exists { get; set; } + public bool video_rtp_ready { get; set; } + public bool video_rtcp_ready { get; set; } + public ulong? video_bytes_sent { get; set; default=0; } + public ulong? video_bytes_received { get; set; default=0; } + public string? video_codec { get; set; } +} \ No newline at end of file -- cgit v1.2.3-54-g00ecf From e205743f0cb71e32eecf183d067a4f34a7a89d48 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 11 Nov 2021 21:54:55 +0100 Subject: Display target bitrates in connection details UI --- libdino/src/service/call_peer_state.vala | 12 ++++++++ .../call_connection_details_window.vala | 36 +++++++++++++++++----- plugins/rtp/src/stream.vala | 17 +++++----- .../src/module/xep/0167_jingle_rtp/stream.vala | 8 ++++- 4 files changed, 55 insertions(+), 18 deletions(-) (limited to 'libdino/src/service/call_peer_state.vala') diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala index ddd0d8dd..c7bcd201 100644 --- a/libdino/src/service/call_peer_state.vala +++ b/libdino/src/service/call_peer_state.vala @@ -260,6 +260,10 @@ public class Dino.PeerState : Object { ret.audio_codec = audio_content_parameter.agreed_payload_type.name; ret.audio_clockrate = audio_content_parameter.agreed_payload_type.clockrate; } + if (audio_content_parameter.stream != null && audio_content_parameter.stream.remb_enabled) { + ret.audio_target_receive_bitrate = audio_content_parameter.stream.target_receive_bitrate; + ret.audio_target_send_bitrate = audio_content_parameter.stream.target_send_bitrate; + } } if (audio_content != null) { @@ -278,6 +282,10 @@ public class Dino.PeerState : Object { if (video_content_parameter.agreed_payload_type != null) { ret.video_codec = video_content_parameter.agreed_payload_type.name; } + if (video_content_parameter.stream != null && video_content_parameter.stream.remb_enabled) { + ret.video_target_receive_bitrate = video_content_parameter.stream.target_receive_bitrate; + ret.video_target_send_bitrate = video_content_parameter.stream.target_send_bitrate; + } } if (video_content != null) { @@ -443,6 +451,8 @@ public class Dino.PeerInfo { public ulong? audio_bytes_received { get; set; default=0; } public string? audio_codec { get; set; } public uint32 audio_clockrate { get; set; } + public uint audio_target_receive_bitrate { get; set; default=0; } + public uint audio_target_send_bitrate { get; set; default=0; } public bool video_content_exists { get; set; } public bool video_rtp_ready { get; set; } @@ -450,4 +460,6 @@ public class Dino.PeerInfo { public ulong? video_bytes_sent { get; set; default=0; } public ulong? video_bytes_received { get; set; default=0; } public string? video_codec { get; set; } + public uint video_target_receive_bitrate { get; set; default=0; } + public uint video_target_send_bitrate { get; set; default=0; } } \ No newline at end of file diff --git a/main/src/ui/call_window/call_connection_details_window.vala b/main/src/ui/call_window/call_connection_details_window.vala index b292b971..95e00296 100644 --- a/main/src/ui/call_window/call_connection_details_window.vala +++ b/main/src/ui/call_window/call_connection_details_window.vala @@ -8,15 +8,19 @@ namespace Dino.Ui { public Label audio_rtp_ready = new Label("?") { xalign=0, visible=true }; public Label audio_rtcp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_sent_bps = new Label("?") { xalign=0, visible=true }; - public Label audio_recv_bps = new Label("?") { xalign=0, visible=true }; + public Label audio_sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label audio_recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; public Label audio_codec = new Label("?") { xalign=0, visible=true }; + public Label audio_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Label audio_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; public Label video_rtp_ready = new Label("") { xalign=0, visible=true }; public Label video_rtcp_ready = new Label("") { xalign=0, visible=true }; - public Label video_sent_bps = new Label("") { xalign=0, visible=true }; - public Label video_recv_bps = new Label("") { xalign=0, visible=true }; + public Label video_sent_bps = new Label("") { use_markup=true, xalign=0, visible=true }; + public Label video_recv_bps = new Label("") { use_markup=true, xalign=0, visible=true }; public Label video_codec = new Label("") { xalign=0, visible=true }; + public Label video_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Label video_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; private int row_at = 0; private bool video_added = false; @@ -34,6 +38,10 @@ namespace Dino.Ui { grid.attach(audio_recv_bps, 1, row_at++, 1, 1); put_row("Codec"); grid.attach(audio_codec, 1, row_at++, 1, 1); + put_row("Target receive bitrate"); + grid.attach(audio_target_receive_bitrate, 1, row_at++, 1, 1); + put_row("Target send bitrate"); + grid.attach(audio_target_send_bitrate, 1, row_at++, 1, 1); this.child = grid; } @@ -46,18 +54,26 @@ namespace Dino.Ui { audio_rtp_ready.label = peer_info.audio_rtp_ready.to_string(); audio_rtcp_ready.label = peer_info.audio_rtcp_ready.to_string(); audio_codec.label = peer_info.audio_codec + " " + peer_info.audio_clockrate.to_string(); + audio_target_receive_bitrate.label = peer_info.audio_target_receive_bitrate.to_string(); + audio_target_send_bitrate.label = peer_info.audio_target_send_bitrate.to_string(); video_rtp_ready.label = peer_info.video_rtp_ready.to_string(); video_rtcp_ready.label = peer_info.video_rtcp_ready.to_string(); video_codec.label = peer_info.video_codec; + video_target_receive_bitrate.label = peer_info.video_target_receive_bitrate.to_string(); + video_target_send_bitrate.label = peer_info.video_target_send_bitrate.to_string(); if (peer_info.video_content_exists) add_video_widgets(); if (prev_peer_info != null) { - audio_sent_bps.label = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent).to_string(); - audio_recv_bps.label = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received).to_string(); - video_sent_bps.label = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent).to_string(); - video_recv_bps.label = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received).to_string(); + ulong audio_sent_kbps = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent) * 8 / 1000; + audio_sent_bps.label = "%lu kbps".printf(audio_sent_kbps); + ulong audio_recv_kbps = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received) * 8 / 1000; + audio_recv_bps.label = "%lu kbps".printf(audio_recv_kbps); + ulong video_sent_kbps = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent) * 8 / 1000; + video_sent_bps.label = "%lu kbps".printf(video_sent_kbps); + ulong video_recv_kbps = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received) * 8 / 1000; + video_recv_bps.label = "%lu kbps".printf(video_recv_kbps); } prev_peer_info = peer_info; } @@ -76,6 +92,10 @@ namespace Dino.Ui { grid.attach(video_recv_bps, 1, row_at++, 1, 1); put_row("Codec"); grid.attach(video_codec, 1, row_at++, 1, 1); + put_row("Target receive bitrate"); + grid.attach(video_target_receive_bitrate, 1, row_at++, 1, 1); + put_row("Target send bitrate"); + grid.attach(video_target_send_bitrate, 1, row_at++, 1, 1); video_added = true; } diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 17c955a5..bfe5ae28 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -153,7 +153,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { plugin.unpause(); GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); - if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + if (session != null && remb_enabled) { Object internal_session; session.@get("internal-session", out internal_session); if (internal_session != null) { @@ -166,7 +166,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } - private uint remb = 256; private int last_packets_lost = -1; private uint64 last_packets_received; private uint64 last_octets_received; @@ -212,12 +211,12 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); if (new_lost <= 0 || loss_rate < 0.02) { - remb = (uint)(1.08 * (double)remb); + target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { - remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); } - remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); - remb = uint.max(16, remb); // Never go below 16 + target_receive_bitrate = uint.max(target_receive_bitrate, (uint)((new_octets * 8) / 1000)); + target_receive_bitrate = uint.max(16, target_receive_bitrate); // Never go below 16 uint8[] data = new uint8[] { 143, 206, 0, 5, 0, 0, 0, 0, @@ -231,7 +230,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { data[6] = (uint8)((our_ssrc >> 8) & 0xff); data[7] = (uint8)(our_ssrc & 0xff); uint8 br_exp = 0; - uint32 br_mant = remb * 1000; + uint32 br_mant = target_receive_bitrate * 1000; uint8 bits = (uint8)Math.log2(br_mant); if (bits > 16) { br_exp = (uint8)bits - 16; @@ -258,8 +257,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; uint8 br_exp = data[5] >> 2; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; - uint bitrate = (br_mant << br_exp) / 1000; - self.input_device.update_bitrate(self.payload_type, bitrate); + self.target_send_bitrate = (br_mant << br_exp) / 1000; + self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 65be8a0a..031f0ad0 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -1,5 +1,4 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { - public Jingle.Content content { get; protected set; } public string name { get { @@ -54,6 +53,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { return false; }} + // Receiver Estimated Maximum Bitrate + public bool remb_enabled { get { + return payload_type != null ? payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb") : false; + }} + public uint target_receive_bitrate { get; set; default=256; } + public uint target_send_bitrate { get; set; default=256; } + protected Stream(Jingle.Content content) { this.content = content; } -- cgit v1.2.3-54-g00ecf From 2b3d150949fe1b3c4107e497be7dac8e2ba734aa Mon Sep 17 00:00:00 2001 From: fiaxh Date: Mon, 15 Nov 2021 13:29:13 +0100 Subject: Improve call details dialog + small multi-party call fixes --- libdino/src/entity/call.vala | 2 +- libdino/src/service/call_peer_state.vala | 86 +++++----- libdino/src/service/calls.vala | 11 +- .../call_connection_details_window.vala | 175 +++++++++++---------- .../ui/conversation_content_view/call_widget.vala | 2 +- plugins/rtp/src/stream.vala | 2 +- .../xep/0167_jingle_rtp/content_parameters.vala | 1 + .../module/xep/0353_jingle_message_initiation.vala | 4 +- 8 files changed, 139 insertions(+), 144 deletions(-) (limited to 'libdino/src/service/call_peer_state.vala') diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index a5ab672e..5221a807 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -11,7 +11,7 @@ namespace Dino.Entities { RINGING, ESTABLISHING, IN_PROGRESS, - OTHER_DEVICE_ACCEPTED, + OTHER_DEVICE, ENDED, DECLINED, MISSED, diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala index c7bcd201..8c4b0930 100644 --- a/libdino/src/service/call_peer_state.vala +++ b/libdino/src/service/call_peer_state.vala @@ -251,53 +251,43 @@ public class Dino.PeerState : Object { public PeerInfo get_info() { var ret = new PeerInfo(); - - if (audio_content_parameter != null) { - ret.audio_rtcp_ready = audio_content_parameter.rtcp_ready; - ret.audio_rtp_ready = audio_content_parameter.rtp_ready; - - if (audio_content_parameter.agreed_payload_type != null) { - ret.audio_codec = audio_content_parameter.agreed_payload_type.name; - ret.audio_clockrate = audio_content_parameter.agreed_payload_type.clockrate; - } - if (audio_content_parameter.stream != null && audio_content_parameter.stream.remb_enabled) { - ret.audio_target_receive_bitrate = audio_content_parameter.stream.target_receive_bitrate; - ret.audio_target_send_bitrate = audio_content_parameter.stream.target_send_bitrate; - } + if (audio_content != null || audio_content_parameter != null) { + ret.audio = get_content_info(audio_content, audio_content_parameter); } - - if (audio_content != null) { - Xmpp.Xep.Jingle.ComponentConnection? component0 = audio_content.get_transport_connection(1); - if (component0 != null) { - ret.audio_bytes_received = component0.bytes_received; - ret.audio_bytes_sent = component0.bytes_sent; - } + if (video_content != null || video_content_parameter != null) { + ret.video = get_content_info(video_content, video_content_parameter); } + return ret; + } - if (video_content_parameter != null) { - ret.video_content_exists = true; - ret.video_rtcp_ready = video_content_parameter.rtcp_ready; - ret.video_rtp_ready = video_content_parameter.rtp_ready; + private PeerContentInfo get_content_info(Xep.Jingle.Content? content, Xep.JingleRtp.Parameters? parameter) { + PeerContentInfo ret = new PeerContentInfo(); + if (parameter != null) { + ret.rtcp_ready = parameter.rtcp_ready; + ret.rtp_ready = parameter.rtp_ready; - if (video_content_parameter.agreed_payload_type != null) { - ret.video_codec = video_content_parameter.agreed_payload_type.name; + if (parameter.agreed_payload_type != null) { + ret.codec = parameter.agreed_payload_type.name; + ret.clockrate = parameter.agreed_payload_type.clockrate; } - if (video_content_parameter.stream != null && video_content_parameter.stream.remb_enabled) { - ret.video_target_receive_bitrate = video_content_parameter.stream.target_receive_bitrate; - ret.video_target_send_bitrate = video_content_parameter.stream.target_send_bitrate; + if (parameter.stream != null && parameter.stream.remb_enabled) { + ret.target_receive_bytes = parameter.stream.target_receive_bitrate; + ret.target_send_bytes = parameter.stream.target_send_bitrate; } } - if (video_content != null) { - Xmpp.Xep.Jingle.ComponentConnection? component0 = video_content.get_transport_connection(1); + if (content != null) { + Xmpp.Xep.Jingle.ComponentConnection? component0 = content.get_transport_connection(1); if (component0 != null) { - ret.video_bytes_received = component0.bytes_received; - ret.video_bytes_sent = component0.bytes_sent; + ret.bytes_received = component0.bytes_received; + ret.bytes_sent = component0.bytes_sent; } } return ret; } + + private void connect_content_signals(Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { if (rtp_content_parameter.media == "audio") { audio_content = content; @@ -444,22 +434,18 @@ public class Dino.PeerState : Object { } } +public class Dino.PeerContentInfo { + public bool rtp_ready { get; set; } + public bool rtcp_ready { get; set; } + public ulong? bytes_sent { get; set; default=0; } + public ulong? bytes_received { get; set; default=0; } + public string? codec { get; set; } + public uint32 clockrate { get; set; } + public uint target_receive_bytes { get; set; default=-1; } + public uint target_send_bytes { get; set; default=-1; } +} + public class Dino.PeerInfo { - public bool audio_rtp_ready { get; set; } - public bool audio_rtcp_ready { get; set; } - public ulong? audio_bytes_sent { get; set; default=0; } - public ulong? audio_bytes_received { get; set; default=0; } - public string? audio_codec { get; set; } - public uint32 audio_clockrate { get; set; } - public uint audio_target_receive_bitrate { get; set; default=0; } - public uint audio_target_send_bitrate { get; set; default=0; } - - public bool video_content_exists { get; set; } - public bool video_rtp_ready { get; set; } - public bool video_rtcp_ready { get; set; } - public ulong? video_bytes_sent { get; set; default=0; } - public ulong? video_bytes_received { get; set; default=0; } - public string? video_codec { get; set; } - public uint video_target_receive_bitrate { get; set; default=0; } - public uint video_target_send_bitrate { get; set; default=0; } + public PeerContentInfo? audio = null; + public PeerContentInfo? video = null; } \ No newline at end of file diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 51ed6e78..3d1ed7e8 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -202,16 +202,17 @@ namespace Dino { // Call requested by another of our devices call.direction = Call.DIRECTION_OUTGOING; call.ourpart = from; + call.state = Call.State.OTHER_DEVICE; counterpart = to; } else { call.direction = Call.DIRECTION_INCOMING; call.ourpart = account.full_jid; + call.state = Call.State.RINGING; counterpart = from; } call.add_peer(counterpart); call.account = account; call.time = call.local_time = call.end_time = new DateTime.now_utc(); - call.state = Call.State.RINGING; Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(counterpart.bare_jid, account, Conversation.Type.CHAT); @@ -329,7 +330,7 @@ namespace Dino { current_jmi_request_peer[account] = peer_state; current_jmi_request_call[account] = call_states[peer_state.call]; }); - mi_module.session_accepted.connect((from, sid) => { + mi_module.session_accepted.connect((from, to, sid) => { if (!current_jmi_request_peer.has_key(account) || current_jmi_request_peer[account].sid != sid) return; if (from.equals_bare(account.bare_jid)) { // Carboned message from our account @@ -337,9 +338,9 @@ namespace Dino { if (from.equals(account.full_jid)) return; Call call = current_jmi_request_peer[account].call; - call.state = Call.State.OTHER_DEVICE_ACCEPTED; + call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call); - } else if (from.equals_bare(current_jmi_request_peer[account].jid)) { // Message from our peer + } else if (from.equals_bare(current_jmi_request_peer[account].jid) && to.equals(account.full_jid)) { // Message from our peer // We proposed the call // We know the full jid of our peer now current_jmi_request_call[account].rename_peer(current_jmi_request_peer[account].jid, from); @@ -383,7 +384,7 @@ namespace Dino { CallState? call_state = get_call_state_for_groupcall(account, muc_jid); if (call_state == null) return; - call_state.call.state = Call.State.OTHER_DEVICE_ACCEPTED; + call_state.call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call_state.call); }); muji_meta_module.call_retracted.connect((from_jid, muc_jid) => { diff --git a/main/src/ui/call_window/call_connection_details_window.vala b/main/src/ui/call_window/call_connection_details_window.vala index 95e00296..1d5265c9 100644 --- a/main/src/ui/call_window/call_connection_details_window.vala +++ b/main/src/ui/call_window/call_connection_details_window.vala @@ -4,100 +4,107 @@ namespace Dino.Ui { public class CallConnectionDetailsWindow : Gtk.Window { - public Grid grid = new Grid() { column_spacing=5, margin=10, halign=Align.CENTER, valign=Align.CENTER, visible=true }; - - public Label audio_rtp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_rtcp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; - public Label audio_recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; - public Label audio_codec = new Label("?") { xalign=0, visible=true }; - public Label audio_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; - public Label audio_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; - - public Label video_rtp_ready = new Label("") { xalign=0, visible=true }; - public Label video_rtcp_ready = new Label("") { xalign=0, visible=true }; - public Label video_sent_bps = new Label("") { use_markup=true, xalign=0, visible=true }; - public Label video_recv_bps = new Label("") { use_markup=true, xalign=0, visible=true }; - public Label video_codec = new Label("") { xalign=0, visible=true }; - public Label video_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; - public Label video_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Box box = new Box(Orientation.VERTICAL, 15) { margin=10, halign=Align.CENTER, valign=Align.CENTER, visible=true }; - private int row_at = 0; private bool video_added = false; - private PeerInfo? prev_peer_info = null; + private CallContentDetails audio_details = new CallContentDetails("Audio") { visible=true }; + private CallContentDetails video_details = new CallContentDetails("Video"); public CallConnectionDetailsWindow() { - grid.attach(new Label("Audio") { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); - put_row("RTP"); - grid.attach(audio_rtp_ready, 1, row_at++, 1, 1); - put_row("RTCP"); - grid.attach(audio_rtcp_ready, 1, row_at++, 1, 1); - put_row("Sent bp/s"); - grid.attach(audio_sent_bps, 1, row_at++, 1, 1); - put_row("Received bp/s"); - grid.attach(audio_recv_bps, 1, row_at++, 1, 1); - put_row("Codec"); - grid.attach(audio_codec, 1, row_at++, 1, 1); - put_row("Target receive bitrate"); - grid.attach(audio_target_receive_bitrate, 1, row_at++, 1, 1); - put_row("Target send bitrate"); - grid.attach(audio_target_send_bitrate, 1, row_at++, 1, 1); - - this.child = grid; - } - - private void put_row(string label) { - grid.attach(new Label(label) { xalign=0, visible=true }, 0, row_at, 1, 1); + box.add(audio_details); + box.add(video_details); + add(box); } public void update_content(PeerInfo peer_info) { - audio_rtp_ready.label = peer_info.audio_rtp_ready.to_string(); - audio_rtcp_ready.label = peer_info.audio_rtcp_ready.to_string(); - audio_codec.label = peer_info.audio_codec + " " + peer_info.audio_clockrate.to_string(); - audio_target_receive_bitrate.label = peer_info.audio_target_receive_bitrate.to_string(); - audio_target_send_bitrate.label = peer_info.audio_target_send_bitrate.to_string(); - - video_rtp_ready.label = peer_info.video_rtp_ready.to_string(); - video_rtcp_ready.label = peer_info.video_rtcp_ready.to_string(); - video_codec.label = peer_info.video_codec; - video_target_receive_bitrate.label = peer_info.video_target_receive_bitrate.to_string(); - video_target_send_bitrate.label = peer_info.video_target_send_bitrate.to_string(); - - if (peer_info.video_content_exists) add_video_widgets(); - - if (prev_peer_info != null) { - ulong audio_sent_kbps = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent) * 8 / 1000; - audio_sent_bps.label = "%lu kbps".printf(audio_sent_kbps); - ulong audio_recv_kbps = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received) * 8 / 1000; - audio_recv_bps.label = "%lu kbps".printf(audio_recv_kbps); - ulong video_sent_kbps = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent) * 8 / 1000; - video_sent_bps.label = "%lu kbps".printf(video_sent_kbps); - ulong video_recv_kbps = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received) * 8 / 1000; - video_recv_bps.label = "%lu kbps".printf(video_recv_kbps); - } - prev_peer_info = peer_info; + if (peer_info.audio != null) { + audio_details.update_content(peer_info.audio); + } + if (peer_info.video != null) { + add_video_widgets(); + video_details.update_content(peer_info.video); + } } private void add_video_widgets() { - if (video_added) return; - - grid.attach(new Label("Video") { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); - put_row("RTP"); - grid.attach(video_rtp_ready, 1, row_at++, 1, 1); - put_row("RTCP"); - grid.attach(video_rtcp_ready, 1, row_at++, 1, 1); - put_row("Sent bp/s"); - grid.attach(video_sent_bps, 1, row_at++, 1, 1); - put_row("Received bp/s"); - grid.attach(video_recv_bps, 1, row_at++, 1, 1); - put_row("Codec"); - grid.attach(video_codec, 1, row_at++, 1, 1); - put_row("Target receive bitrate"); - grid.attach(video_target_receive_bitrate, 1, row_at++, 1, 1); - put_row("Target send bitrate"); - grid.attach(video_target_send_bitrate, 1, row_at++, 1, 1); - - video_added = true; + if (video_added) return; + + video_details.visible = true; + video_added = true; + } + } + + public class CallContentDetails : Gtk.Grid { + + public Label rtp_title = new Label("RTP") { xalign=0, visible=true }; + public Label rtcp_title = new Label("RTCP") { xalign=0, visible=true }; + public Label target_recv_title = new Label("Target receive bitrate") { xalign=0, visible=true }; + public Label target_send_title = new Label("Target send bitrate") { xalign=0, visible=true }; + + public Label rtp_ready = new Label("?") { xalign=0, visible=true }; + public Label rtcp_ready = new Label("?") { xalign=0, visible=true }; + public Label sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label codec = new Label("?") { xalign=0, visible=true }; + public Label target_receive_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + public Label target_send_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + + private PeerContentInfo? prev_info = null; + private int row_at = 0; + + public CallContentDetails(string headline) { + attach(new Label("%s".printf(headline)) { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); + attach(rtp_title, 0, row_at, 1, 1); + attach(rtp_ready, 1, row_at++, 1, 1); + attach(rtcp_title, 0, row_at, 1, 1); + attach(rtcp_ready, 1, row_at++, 1, 1); + put_row("Sent"); + attach(sent_bps, 1, row_at++, 1, 1); + put_row("Received"); + attach(recv_bps, 1, row_at++, 1, 1); + put_row("Codec"); + attach(codec, 1, row_at++, 1, 1); + attach(target_recv_title, 0, row_at, 1, 1); + attach(target_receive_bitrate, 1, row_at++, 1, 1); + attach(target_send_title, 0, row_at, 1, 1); + attach(target_send_bitrate, 1, row_at++, 1, 1); + + this.column_spacing = 5; + } + + public void update_content(PeerContentInfo info) { + if (!info.rtp_ready) { + rtp_ready.visible = rtcp_ready.visible = true; + rtp_title.visible = rtcp_title.visible = true; + rtp_ready.label = info.rtp_ready.to_string(); + rtcp_ready.label = info.rtcp_ready.to_string(); + } else { + rtp_ready.visible = rtcp_ready.visible = false; + rtp_title.visible = rtcp_title.visible = false; + } + if (info.target_send_bytes != -1) { + target_receive_bitrate.visible = target_send_bitrate.visible = true; + target_recv_title.visible = target_send_title.visible = true; + target_receive_bitrate.label = "%u kbps".printf(info.target_receive_bytes); + target_send_bitrate.label = "%u kbps".printf(info.target_send_bytes); + } else { + target_receive_bitrate.visible = target_send_bitrate.visible = false; + target_recv_title.visible = target_send_title.visible = false; + } + + codec.label = info.codec + " " + info.clockrate.to_string(); + + if (prev_info != null) { + ulong audio_sent_kbps = (info.bytes_sent - prev_info.bytes_sent) * 8 / 1000; + sent_bps.label = "%lu kbps".printf(audio_sent_kbps); + ulong audio_recv_kbps = (info.bytes_received - prev_info.bytes_received) * 8 / 1000; + recv_bps.label = "%lu kbps".printf(audio_recv_kbps); + } + prev_info = info; + } + + private void put_row(string label) { + attach(new Label(label) { xalign=0, visible=true }, 0, row_at, 1, 1); } } } diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index aad1e6d8..a7d37afd 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -144,7 +144,7 @@ namespace Dino.Ui { }); break; - case Call.State.OTHER_DEVICE_ACCEPTED: + case Call.State.OTHER_DEVICE: image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); title_label.label = call.direction == Call.DIRECTION_INCOMING ? _("Incoming call") : _("Outgoing call"); subtitle_label.label = _("You handled this call on another device"); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 5e5a556b..85864022 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -99,7 +99,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); } if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.get_data("no_buffer_probe_timeout"); + uint id = pad.steal_data("no_buffer_probe_timeout"); if (id != 0) { Source.remove(id); } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index 9022547d..c4c299c5 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -151,6 +151,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } this.stream = parent.create_stream(content); + this.stream.weak_ref(() => this.stream = null); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); this.stream.on_send_rtp_data.connect(rtp_datagram.send_datagram); diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index 71e16a95..ac1d8329 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -8,7 +8,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { public signal void session_proposed(Jid from, Jid to, string sid, Gee.List descriptions); public signal void session_retracted(Jid from, Jid to, string sid); - public signal void session_accepted(Jid from, string sid); + public signal void session_accepted(Jid from, Jid to, string sid); public signal void session_rejected(Jid from, Jid to, string sid); public void send_session_propose_to_peer(XmppStream stream, Jid to, string sid, Gee.List descriptions) { @@ -65,7 +65,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { switch (mi_node.name) { case "accept": case "proceed": - session_accepted(message.from, mi_node.get_attribute("id")); + session_accepted(message.from, message.to, mi_node.get_attribute("id")); break; case "propose": ArrayList descriptions = new ArrayList(); -- cgit v1.2.3-54-g00ecf From f0c7dd0682fec8d72c644d8e54896de7bdc40ddb Mon Sep 17 00:00:00 2001 From: fiaxh Date: Mon, 20 Dec 2021 00:15:05 +0100 Subject: UI + libdino: Improve MUJI calls from MUC - Move calls from ICE-thead onto main thread - Identify Call.ourpart as MUC nick if in MUC - Keep track of the initiator of a call --- libdino/src/entity/call.vala | 9 ++++---- libdino/src/service/call_peer_state.vala | 14 +++++++++---- libdino/src/service/call_state.vala | 9 ++++++-- libdino/src/service/call_store.vala | 9 +++++--- libdino/src/service/calls.vala | 24 ++++++++++++---------- libdino/src/service/content_item_store.vala | 4 ++-- libdino/src/service/muc_manager.vala | 2 +- main/data/call_widget.ui | 2 +- main/src/ui/application.vala | 18 ++++++++++++---- main/src/ui/call_window/call_window.vala | 2 +- .../ui/conversation_content_view/call_widget.vala | 2 +- main/src/ui/conversation_titlebar/call_entry.vala | 18 +++++++++------- main/src/ui/notifier_freedesktop.vala | 6 ++++-- 13 files changed, 75 insertions(+), 44 deletions(-) (limited to 'libdino/src/service/call_peer_state.vala') diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index 3b48f664..8e5bc246 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -20,9 +20,12 @@ namespace Dino.Entities { public int id { get; set; default=-1; } public Account account { get; set; } - public Jid counterpart { get; set; } // For backwards compatibility with db version 21. Not to be used anymore. + public Jid counterpart { get; set; } public Gee.List counterparts = new Gee.ArrayList(Jid.equals_bare_func); public Jid ourpart { get; set; } + public Jid proposer { + get { return direction == DIRECTION_OUTGOING ? ourpart : counterpart; } + } public bool direction { get; set; } public DateTime time { get; set; } public DateTime local_time { get; set; } @@ -58,7 +61,6 @@ namespace Dino.Entities { if (!counterparts.contains(peer)) { // Legacy: The first peer is also in the `call` table. Don't add twice. counterparts.add(peer); } - if (counterpart == null) counterpart = peer; } counterpart = db.get_jid_by_id(row[db.call.counterpart_id]); @@ -66,7 +68,6 @@ namespace Dino.Entities { if (counterpart_resource != null) counterpart = counterpart.with_resource(counterpart_resource); if (counterparts.is_empty) { counterparts.add(counterpart); - counterpart = counterpart; } notify.connect(on_update); @@ -107,8 +108,6 @@ namespace Dino.Entities { } public void add_peer(Jid peer) { - if (counterpart == null) counterpart = peer; - if (counterparts.contains(peer)) return; counterparts.add(peer); diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala index 8c4b0930..09440371 100644 --- a/libdino/src/service/call_peer_state.vala +++ b/libdino/src/service/call_peer_state.vala @@ -127,8 +127,6 @@ public class Dino.PeerState : Object { } public void reject() { - call.state = Call.State.DECLINED; - if (session != null) { foreach (Xep.Jingle.Content content in session.contents) { content.reject(); @@ -299,7 +297,12 @@ public class Dino.PeerState : Object { debug(@"[%s] %s connecting content signals %s", call.account.bare_jid.to_string(), jid.to_string(), rtp_content_parameter.media); rtp_content_parameter.stream_created.connect((stream) => on_stream_created(rtp_content_parameter.media, stream)); - rtp_content_parameter.connection_ready.connect((status) => on_connection_ready(content, rtp_content_parameter.media)); + rtp_content_parameter.connection_ready.connect((status) => { + Idle.add(() => { + on_connection_ready(content, rtp_content_parameter.media); + return false; + }); + }); content.senders_modify_incoming.connect((content, proposed_senders) => { if (content.session.senders_include_us(content.senders) != content.session.senders_include_us(proposed_senders)) { @@ -342,7 +345,10 @@ public class Dino.PeerState : Object { if (media == "video" && stream.receiving) { counterpart_sends_video = true; video_content_parameter.connection_ready.connect((status) => { - counterpart_sends_video_updated(false); + Idle.add(() => { + counterpart_sends_video_updated(false); + return false; + }); }); } diff --git a/libdino/src/service/call_state.vala b/libdino/src/service/call_state.vala index 51563552..188a8321 100644 --- a/libdino/src/service/call_state.vala +++ b/libdino/src/service/call_state.vala @@ -126,7 +126,7 @@ public class Dino.CallState : Object { foreach (PeerState peer in peers_cpy) { peer.end(Xep.Jingle.ReasonElement.CANCEL); } - if (parent_muc != null) { + if (parent_muc != null && group_call != null) { XmppStream stream = stream_interactor.get_stream(call.account); if (stream == null) return; stream.get_module(Xep.MujiMeta.Module.IDENTITY).send_invite_retract_to_peer(stream, parent_muc, group_call.muc_jid, message_type); @@ -242,7 +242,12 @@ public class Dino.CallState : Object { XmppStream stream = stream_interactor.get_stream(call.account); if (stream == null) return; - Jid muc_jid = stream_interactor.get_module(MucManager.IDENTITY).default_muc_server[call.account] ?? new Jid("chat.jabberfr.org"); + Jid? muc_jid = null; + if (muc_jid == null) { + warning("Failed to initiate group call: MUC server not known."); + return; + } + muc_jid = new Jid("%08x@".printf(Random.next_int()) + muc_jid.to_string()); // TODO longer? debug("[%s] Converting call to groupcall %s", call.account.bare_jid.to_string(), muc_jid.to_string()); diff --git a/libdino/src/service/call_store.vala b/libdino/src/service/call_store.vala index fa6e63ee..bfc8255f 100644 --- a/libdino/src/service/call_store.vala +++ b/libdino/src/service/call_store.vala @@ -30,7 +30,7 @@ namespace Dino { cache_call(call); } - public Call? get_call_by_id(int id) { + public Call? get_call_by_id(int id, Conversation conversation) { Call? call = calls_by_db_id[id]; if (call != null) { return call; @@ -38,14 +38,17 @@ namespace Dino { RowOption row_option = db.call.select().with(db.call.id, "=", id).row(); - return create_call_from_row_opt(row_option); + return create_call_from_row_opt(row_option, conversation); } - private Call? create_call_from_row_opt(RowOption row_opt) { + private Call? create_call_from_row_opt(RowOption row_opt, Conversation conversation) { if (!row_opt.is_present()) return null; try { Call call = new Call.from_row(db, row_opt.inner); + if (conversation.type_.is_muc_semantic()) { + call.ourpart = conversation.counterpart.with_resource(call.ourpart.resourcepart); + } cache_call(call); return call; } catch (InvalidJidError e) { diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 629d8074..3e2181a5 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -39,9 +39,8 @@ namespace Dino { Call call = new Call(); call.direction = Call.DIRECTION_OUTGOING; call.account = conversation.account; - // TODO we should only do that for Conversation.Type.CHAT, but the database currently requires a counterpart from the start call.counterpart = conversation.counterpart; - call.ourpart = conversation.account.full_jid; + call.ourpart = stream_interactor.get_module(MucManager.IDENTITY).get_own_jid(conversation.counterpart, conversation.account) ?? conversation.account.full_jid; call.time = call.local_time = call.end_time = new DateTime.now_utc(); call.state = Call.State.RINGING; @@ -57,7 +56,7 @@ namespace Dino { PeerState peer_state = call_state.set_first_peer(conversation.counterpart); yield peer_state.initiate_call(conversation.counterpart); } else { - call_state.initiate_groupchat_call(conversation.counterpart); + call_state.initiate_groupchat_call.begin(conversation.counterpart); } conversation.last_active = call.time; @@ -213,24 +212,23 @@ namespace Dino { private PeerState create_received_call(Account account, Jid from, Jid to, bool video_requested) { Call call = new Call(); - Jid counterpart = null; if (from.equals_bare(account.bare_jid)) { // Call requested by another of our devices call.direction = Call.DIRECTION_OUTGOING; call.ourpart = from; call.state = Call.State.OTHER_DEVICE; - counterpart = to; + call.counterpart = to; } else { call.direction = Call.DIRECTION_INCOMING; call.ourpart = account.full_jid; call.state = Call.State.RINGING; - counterpart = from; + call.counterpart = from; } - call.add_peer(counterpart); + call.add_peer(call.counterpart); call.account = account; call.time = call.local_time = call.end_time = new DateTime.now_utc(); - Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(counterpart.bare_jid, account, Conversation.Type.CHAT); + Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(call.counterpart.bare_jid, account, Conversation.Type.CHAT); stream_interactor.get_module(CallStore.IDENTITY).add_call(call, conversation); @@ -238,7 +236,7 @@ namespace Dino { var call_state = new CallState(call, stream_interactor); connect_call_state_signals(call_state); - PeerState peer_state = call_state.set_first_peer(counterpart); + PeerState peer_state = call_state.set_first_peer(call.counterpart); call_state.we_should_send_video = video_requested; call_state.we_should_send_audio = true; @@ -283,7 +281,7 @@ namespace Dino { Call call = new Call(); call.direction = Call.DIRECTION_INCOMING; call.ourpart = account.full_jid; - call.add_peer(inviter_jid); // not rly + call.counterpart = inviter_jid; call.account = account; call.time = call.local_time = call.end_time = new DateTime.now_utc(); call.state = Call.State.RINGING; @@ -361,13 +359,14 @@ namespace Dino { if (from.equals(account.full_jid)) return; Call call = current_jmi_request_peer[account].call; + call.ourpart = from; call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call); } else if (from.equals_bare(current_jmi_request_peer[account].jid) && to.equals(account.full_jid)) { // Message from our peer // We proposed the call // We know the full jid of our peer now current_jmi_request_call[account].rename_peer(current_jmi_request_peer[account].jid, from); - current_jmi_request_peer[account].call_resource(from); + current_jmi_request_peer[account].call_resource.begin(from); } }); mi_module.session_rejected.connect((from, to, sid) => { @@ -378,6 +377,9 @@ namespace Dino { bool incoming_reject = call.direction == Call.DIRECTION_INCOMING && from.equals_bare(account.bare_jid); if (!outgoing_reject && !incoming_reject) return; + // We don't care if a single person in a group call rejected the call + if (incoming_reject && call_states[call].group_call != null) return; + call.state = Call.State.DECLINED; call_states[call].terminated(from, Xep.Jingle.ReasonElement.DECLINE, "JMI reject"); remove_call_from_datastructures(call); diff --git a/libdino/src/service/content_item_store.vala b/libdino/src/service/content_item_store.vala index e0102d24..7a8e38b8 100644 --- a/libdino/src/service/content_item_store.vala +++ b/libdino/src/service/content_item_store.vala @@ -69,7 +69,7 @@ public class ContentItemStore : StreamInteractionModule, Object { } break; case 3: - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(foreign_id); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(foreign_id, conversation); if (call != null) { var call_item = new CallItem(call, conversation, row[db.content_item.id]); items.add(call_item); @@ -321,7 +321,7 @@ public class CallItem : ContentItem { public Conversation conversation; public CallItem(Call call, Conversation conversation, int id) { - base(id, TYPE, call.direction == Call.DIRECTION_OUTGOING ? conversation.account.bare_jid : conversation.counterpart, call.time, call.encryption, Message.Marked.NONE); + base(id, TYPE, call.proposer, call.time, call.encryption, Message.Marked.NONE); this.call = call; this.conversation = conversation; diff --git a/libdino/src/service/muc_manager.vala b/libdino/src/service/muc_manager.vala index 8a317a08..05473647 100644 --- a/libdino/src/service/muc_manager.vala +++ b/libdino/src/service/muc_manager.vala @@ -424,7 +424,7 @@ public class MucManager : StreamInteractionModule, Object { foreach (Xep.ServiceDiscovery.Identity identity in identities) { if (identity.category == Xep.ServiceDiscovery.Identity.CATEGORY_CONFERENCE) { default_muc_server[account] = item.jid; - print(@"$(account.bare_jid) Default MUC: $(item.jid)\n"); + debug("[%s] Default MUC: %s", account.bare_jid.to_string(), item.jid.to_string()); return; } } diff --git a/main/data/call_widget.ui b/main/data/call_widget.ui index 8e2ee36c..0c5d8bfa 100644 --- a/main/data/call_widget.ui +++ b/main/data/call_widget.ui @@ -98,7 +98,7 @@ 10 - 5 + 7 True diff --git a/main/src/ui/application.vala b/main/src/ui/application.vala index 9f48caec..ecbea85e 100644 --- a/main/src/ui/application.vala +++ b/main/src/ui/application.vala @@ -206,9 +206,14 @@ public class Dino.Ui.Application : Gtk.Application, Dino.Application { }); add_action(open_shortcuts_action); - SimpleAction accept_call_action = new SimpleAction("accept-call", VariantType.INT32); + SimpleAction accept_call_action = new SimpleAction("accept-call", new VariantType.tuple(new VariantType[]{VariantType.INT32, VariantType.INT32})); accept_call_action.activate.connect((variant) => { - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + int conversation_id = variant.get_child_value(0).get_int32(); + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation_by_id(conversation_id); + if (conversation == null) return; + + int call_id = variant.get_child_value(1).get_int32(); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(call_id, conversation); CallState? call_state = stream_interactor.get_module(Calls.IDENTITY).call_states[call]; if (call_state == null) return; @@ -220,9 +225,14 @@ public class Dino.Ui.Application : Gtk.Application, Dino.Application { }); add_action(accept_call_action); - SimpleAction deny_call_action = new SimpleAction("reject-call", VariantType.INT32); + SimpleAction deny_call_action = new SimpleAction("reject-call", new VariantType.tuple(new VariantType[]{VariantType.INT32, VariantType.INT32})); deny_call_action.activate.connect((variant) => { - Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(variant.get_int32()); + int conversation_id = variant.get_child_value(0).get_int32(); + Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation_by_id(conversation_id); + if (conversation == null) return; + + int call_id = variant.get_child_value(1).get_int32(); + Call? call = stream_interactor.get_module(CallStore.IDENTITY).get_call_by_id(call_id, conversation); CallState? call_state = stream_interactor.get_module(Calls.IDENTITY).call_states[call]; if (call_state == null) return; diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala index fab424bf..c610444f 100644 --- a/main/src/ui/call_window/call_window.vala +++ b/main/src/ui/call_window/call_window.vala @@ -20,7 +20,7 @@ namespace Dino.Ui { public Revealer header_bar_revealer = new Revealer() { halign=Align.END, valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200, visible=true }; public Box own_video_box = new Box(Orientation.HORIZONTAL, 0) { halign=Align.END, valign=Align.END, visible=true }; public Revealer invite_button_revealer = new Revealer() { margin_top=50, margin_right=30, halign=Align.END, valign=Align.START, transition_type=RevealerTransitionType.CROSSFADE, transition_duration=200 }; - public Button invite_button = new Button.from_icon_name("dino-account-plus") { relief=ReliefStyle.NONE }; + public Button invite_button = new Button.from_icon_name("dino-account-plus") { relief=ReliefStyle.NONE, visible=false }; private Widget? own_video = null; private HashMap participant_widgets = new HashMap(); private ArrayList participants = new ArrayList(); diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index a7d37afd..a2c8c0c2 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -86,7 +86,7 @@ namespace Dino.Ui { private void update_counterparts() { if (call.state != Call.State.IN_PROGRESS && call.state != Call.State.ENDED) return; - if (call.counterparts.size <= 1) return; + if (call.counterparts.size <= 1 && conversation.type_ == Conversation.Type.CHAT) return; multiparty_peer_box.foreach((widget) => { multiparty_peer_box.remove(widget); }); diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index 3fa399a6..3b3a5b39 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -115,13 +115,17 @@ namespace Dino.Ui { return; } - Conversation conv_bak = conversation; - bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); - bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_video_calls_async(conversation); - if (conv_bak != conversation) return; - - visible = audio_works; - video_button.visible = video_works; + if (conversation.type_ == Conversation.Type.CHAT) { + Conversation conv_bak = conversation; + bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); + bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_video_calls_async(conversation); + if (conv_bak != conversation) return; + + visible = audio_works; + video_button.visible = video_works; + } else { + visible = false; + } } public new void unset_conversation() { } diff --git a/main/src/ui/notifier_freedesktop.vala b/main/src/ui/notifier_freedesktop.vala index 83e9bf57..e8e2ba1d 100644 --- a/main/src/ui/notifier_freedesktop.vala +++ b/main/src/ui/notifier_freedesktop.vala @@ -141,10 +141,12 @@ public class Dino.Ui.FreeDesktopNotifier : NotificationProvider, Object { GLib.Application.get_default().activate_action("open-conversation", new Variant.int32(conversation.id)); }); add_action_listener(notification_id, "reject", () => { - GLib.Application.get_default().activate_action("reject-call", new Variant.int32(call.id)); + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.int32(call.id)}); + GLib.Application.get_default().activate_action("reject-call", variant); }); add_action_listener(notification_id, "accept", () => { - GLib.Application.get_default().activate_action("accept-call", new Variant.int32(call.id)); + var variant = new Variant.tuple(new Variant[] {new Variant.int32(conversation.id), new Variant.int32(call.id)}); + GLib.Application.get_default().activate_action("accept-call", variant); }); } catch (Error e) { warning("Failed showing subscription request notification: %s", e.message); -- cgit v1.2.3-54-g00ecf