From aae13b9ea6a3ac3e35164704bb7966d646833686 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:45 +0100 Subject: Crop video to match widget ratio --- plugins/rtp/src/video_widget.vala | 74 ++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 29 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala index 351069a7..ccd86bb2 100644 --- a/plugins/rtp/src/video_widget.vala +++ b/plugins/rtp/src/video_widget.vala @@ -12,8 +12,9 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge private bool attached; private Device? connected_device; + private Gst.Element? connected_device_element; private Stream? connected_stream; - private Gst.Element convert; + private Gst.Element prepare; public VideoWidget(Plugin plugin) { this.plugin = plugin; @@ -28,22 +29,34 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge this.widget = widget; add(widget); widget.visible = true; - - // Listen for resolution changes - element.get_static_pad("sink").notify["caps"].connect(() => { - if (element.get_static_pad("sink").caps == null) return; - - int width, height; - element.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); - element.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); - resolution_changed(width, height); - }); } else { warning("Could not create GTK video sink. Won't display videos."); } + size_allocate.connect_after(after_size_allocate); + } + + public void input_caps_changed(GLib.Object pad, ParamSpec spec) { + Gst.Caps? caps = (pad as Gst.Pad).caps; + if (caps == null) return; + + int width, height; + caps.get_structure(0).get_int("width", out width); + caps.get_structure(0).get_int("height", out height); + resolution_changed(width, height); + } + + public void after_size_allocate(Gtk.Allocation allocation) { + if (prepare != null) { + Gst.Element crop = ((Gst.Bin)prepare).get_by_name(@"video_widget_$(id)_crop"); + if (crop != null) { + Value ratio = new Value(typeof(Gst.Fraction)); + Gst.Value.set_fraction(ref ratio, allocation.width, allocation.height); + crop.set_property("aspect-ratio", ratio); + } + } } - public void display_stream(Xmpp.Xep.JingleRtp.Stream stream) { + public void display_stream(Xmpp.Xep.JingleRtp.Stream stream, Xmpp.Jid jid) { if (element == null) return; detach(); if (stream.media != "video") return; @@ -51,11 +64,12 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_stream == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); - convert.name = @"video_widget_$(id)_prepare"; - pipe.add(convert); - convert.link(element); - connected_stream.add_output(convert); + prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + pipe.add(prepare); + connected_stream.add_output(prepare); + prepare.link(element); element.set_locked_state(false); plugin.unpause(); attached = true; @@ -68,11 +82,13 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_device == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); - convert.name = @"video_widget_$(id)_prepare"; - pipe.add(convert); - convert.link(element); - connected_device.link_source().link(convert); + prepare = Gst.parse_bin_from_description(@"aspectratiocrop aspect-ratio=4/3 name=video_widget_$(id)_crop ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + prepare.name = @"video_widget_$(id)_prepare"; + prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); + pipe.add(prepare); + connected_device_element = connected_device.link_source(); + connected_device_element.link(prepare); + prepare.link(element); element.set_locked_state(false); plugin.unpause(); attached = true; @@ -82,19 +98,19 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (element == null) return; if (attached) { if (connected_stream != null) { - connected_stream.remove_output(convert); + connected_stream.remove_output(prepare); connected_stream = null; } if (connected_device != null) { - connected_device.link_source().unlink(element); - connected_device.unlink(); // We get a new ref to recover the element, so unlink twice + connected_device_element.unlink(element); + connected_device_element = null; connected_device.unlink(); connected_device = null; } - convert.set_locked_state(true); - convert.set_state(Gst.State.NULL); - pipe.remove(convert); - convert = null; + prepare.set_locked_state(true); + prepare.set_state(Gst.State.NULL); + pipe.remove(prepare); + prepare = null; element.set_locked_state(true); element.set_state(Gst.State.NULL); pipe.remove(element); -- cgit v1.2.3-54-g00ecf From 72569ea52faf9895b4a897feffd37fa7fe3e2116 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:46 +0100 Subject: Improve codec support --- plugins/rtp/src/codec_util.vala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 417dc4be..8299fc51 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -146,7 +146,7 @@ public class Dino.Plugins.Rtp.CodecUtil { // VP8 if (encode == "msdkvp8enc") return " rate-control=vbr"; if (encode == "vaapivp8enc") return " rate-control=vbr"; - if (encode == "vp8enc") return " deadline=1 error-resilient=1"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30"; // OPUS if (encode == "opusenc") { @@ -159,7 +159,8 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 - if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse"; + if (media == "video" && codec == "h264") return " ! capsfilter caps=video/x-h264,profile=constrained-baseline ! h264parse"; + if (media == "video" && codec == "vp8" && encode == "vp8enc") return " ! capsfilter caps=video/x-vp8,profile=(string)1"; return null; } @@ -171,17 +172,18 @@ public class Dino.Plugins.Rtp.CodecUtil { if (encode_name == null) return 0; Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); - bitrate = uint.min(2048000, bitrate); - switch (encode_name) { case "msdkh264enc": case "vaapih264enc": case "x264enc": + case "msdkvp9enc": + case "vaapivp9enc": case "msdkvp8enc": case "vaapivp8enc": bitrate = uint.min(2048000, bitrate); encode.set("bitrate", bitrate); return bitrate; + case "vp9enc": case "vp8enc": bitrate = uint.min(2147483, bitrate); encode.set("target-bitrate", bitrate * 1000); @@ -198,6 +200,7 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + if (decode == "vp8dec") return " threads=8"; return null; } -- cgit v1.2.3-54-g00ecf From 083f73b0ca1f1a10b5773942841c0f9e3cc78f27 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:47 +0100 Subject: Split payloader off encoder chain --- plugins/rtp/src/codec_util.vala | 43 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 8299fc51..4c68985f 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -273,7 +273,7 @@ public class Dino.Plugins.Rtp.CodecUtil { string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; + return @"queue ! $depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -288,16 +288,29 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + string? desc1 = get_encode_bin_without_payloader_description(media, codec, payload_type, element_name, name); + string? desc2 = get_payloader_bin_description(media, codec, payload_type, name); + return @"$desc1 ! $desc2"; + } + + public string? get_payloader_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? name = null) { if (codec == null) return null; string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string? pay = get_pay_element_name(media, codec); + if (pay == null) return null; + return @"$pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + } + + public string? get_encode_bin_without_payloader_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string? encode = element_name ?? get_encode_element_name(media, codec); - if (pay == null || encode == null) return null; + if (encode == null) return null; string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + return @"$(media)convert name=$(base_name)_convert$resample ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -311,4 +324,26 @@ public class Dino.Plugins.Rtp.CodecUtil { return bin; } -} \ No newline at end of file + public Gst.Element? get_encode_bin_without_payloader(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_without_payloader_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s without payloader: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public Gst.Element? get_payloader_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_payloader_bin_description(media, codec, payload_type, base_name); + if (desc == null) return null; + debug("Pipeline to payload %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +} -- cgit v1.2.3-54-g00ecf From b593aa05efe667db934c818543ac01a6f0b3e7a4 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:47 +0100 Subject: RTP: Encode with device --- plugins/rtp/src/device.vala | 335 ++++++++++++++++++++++++++++++++++---------- plugins/rtp/src/stream.vala | 252 ++++++++++++++++++++------------- 2 files changed, 413 insertions(+), 174 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index e25271b1..3546ab94 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -1,5 +1,9 @@ +using Xmpp.Xep.JingleRtp; +using Gee; + public class Dino.Plugins.Rtp.Device : MediaDevice, Object { public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { return plugin.codec_util; } } public Gst.Device device { get; private set; } private string device_name; @@ -17,28 +21,45 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return plugin.pipe; }} public string? media { get { - if (device.device_class.has_prefix("Audio/")) { + if (device.has_classes("Audio")) { return "audio"; - } else if (device.device_class.has_prefix("Video/")) { + } else if (device.has_classes("Video")) { return "video"; } else { return null; } }} public bool is_source { get { - return device.device_class.has_suffix("/Source"); + return device.has_classes("Source"); }} public bool is_sink { get { - return device.device_class.has_suffix("/Sink"); + return device.has_classes("Sink"); }} private Gst.Element element; private Gst.Element tee; private Gst.Element dsp; - private Gst.Element mixer; + private Gst.Base.Aggregator mixer; private Gst.Element filter; - private Gst.Element rate; - private int links = 0; + private int links; + + // Codecs + private Gee.Map codecs = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map codec_tees = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloaders = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloader_tees = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> payloader_links = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + private Gee.Map> codec_bitrates = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + + private class CodecBitrate { + public uint bitrate; + public int64 timestamp; + + public CodecBitrate(uint bitrate) { + this.bitrate = bitrate; + this.timestamp = get_monotonic_time(); + } + } public Device(Plugin plugin, Gst.Device device) { this.plugin = plugin; @@ -57,25 +78,154 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } public Gst.Element? link_sink() { + if (!is_sink) return null; if (element == null) create(); links++; - if (mixer != null) return mixer; - if (is_sink && media == "audio") return filter; + if (mixer != null) { + Gst.Element rate = Gst.ElementFactory.make("audiorate", @"$(id)_rate_$(Random.next_int())"); + pipe.add(rate); + rate.link(mixer); + return rate; + } + if (media == "audio") return filter; return element; } - public Gst.Element? link_source() { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1) { + if (!is_source) return null; if (element == null) create(); links++; + if (payload_type != null && tee != null) { + bool new_codec = false; + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (!codecs.has_key(payload_type)) { + codecs[payload_type] = codec_util.get_encode_bin_without_payloader(media, payload_type, @"$(id)_$(codec)_encoder"); + pipe.add(codecs[payload_type]); + new_codec = true; + } + if (!codec_tees.has_key(payload_type)) { + codec_tees[payload_type] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_tee"); + codec_tees[payload_type].@set("allow-not-linked", true); + pipe.add(codec_tees[payload_type]); + codecs[payload_type].link(codec_tees[payload_type]); + } + if (!payloaders.has_key(payload_type)) { + payloaders[payload_type] = new HashMap(); + } + if (!payloaders[payload_type].has_key(ssrc)) { + payloaders[payload_type][ssrc] = codec_util.get_payloader_bin(media, payload_type, @"$(id)_$(codec)_$(ssrc)"); + var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); + payload.ssrc = ssrc; + payload.seqnum_offset = seqnum_offset; + pipe.add(payloaders[payload_type][ssrc]); + codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + } + if (!payloader_tees.has_key(payload_type)) { + payloader_tees[payload_type] = new HashMap(); + } + if (!payloader_tees[payload_type].has_key(ssrc)) { + payloader_tees[payload_type][ssrc] = Gst.ElementFactory.make("tee", @"$(id)_$(codec)_$(ssrc)_tee"); + payloader_tees[payload_type][ssrc].@set("allow-not-linked", true); + pipe.add(payloader_tees[payload_type][ssrc]); + payloaders[payload_type][ssrc].link(payloader_tees[payload_type][ssrc]); + } + if (!payloader_links.has_key(payload_type)) { + payloader_links[payload_type] = new HashMap(); + } + if (!payloader_links[payload_type].has_key(ssrc)) { + payloader_links[payload_type][ssrc] = 1; + } else { + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] + 1; + } + if (new_codec) { + tee.link(codecs[payload_type]); + } + return payloader_tees[payload_type][ssrc]; + } if (tee != null) return tee; return element; } - public void unlink() { + public void update_bitrate(PayloadType payload_type, uint bitrate) { + if (codecs.has_key(payload_type)) { + lock(codec_bitrates); + if (!codec_bitrates.has_key(payload_type)) { + codec_bitrates[payload_type] = new ArrayList(); + } + codec_bitrates[payload_type].add(new CodecBitrate(bitrate)); + var remove = new ArrayList(); + foreach (CodecBitrate rate in codec_bitrates[payload_type]) { + if (rate.timestamp < get_monotonic_time() - 5000000L) { + remove.add(rate); + continue; + } + if (rate.bitrate < bitrate) { + bitrate = rate.bitrate; + } + } + codec_bitrates[payload_type].remove_all(remove); + codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); + unlock(codec_bitrates); + } + } + + public void unlink(Gst.Element? link = null) { if (links <= 0) { critical("Link count below zero."); return; } + if (link != null && is_source && tee != null) { + PayloadType payload_type = payloader_tees.first_match((entry) => entry.value.any_match((entry) => entry.value == link)).key; + uint ssrc = payloader_tees[payload_type].first_match((entry) => entry.value == link).key; + payloader_links[payload_type][ssrc] = payloader_links[payload_type][ssrc] - 1; + if (payloader_links[payload_type][ssrc] == 0) { + plugin.pause(); + + codec_tees[payload_type].unlink(payloaders[payload_type][ssrc]); + payloaders[payload_type][ssrc].set_locked_state(true); + payloaders[payload_type][ssrc].set_state(Gst.State.NULL); + payloaders[payload_type][ssrc].unlink(payloader_tees[payload_type][ssrc]); + pipe.remove(payloaders[payload_type][ssrc]); + payloaders[payload_type].unset(ssrc); + payloader_tees[payload_type][ssrc].set_locked_state(true); + payloader_tees[payload_type][ssrc].set_state(Gst.State.NULL); + pipe.remove(payloader_tees[payload_type][ssrc]); + payloader_tees[payload_type].unset(ssrc); + + payloader_links[payload_type].unset(ssrc); + plugin.unpause(); + } + if (payloader_links[payload_type].size == 0) { + plugin.pause(); + + tee.unlink(codecs[payload_type]); + codecs[payload_type].set_locked_state(true); + codecs[payload_type].set_state(Gst.State.NULL); + codecs[payload_type].unlink(codec_tees[payload_type]); + pipe.remove(codecs[payload_type]); + codecs.unset(payload_type); + codec_tees[payload_type].set_locked_state(true); + codec_tees[payload_type].set_state(Gst.State.NULL); + pipe.remove(codec_tees[payload_type]); + codec_tees.unset(payload_type); + + payloaders.unset(payload_type); + payloader_tees.unset(payload_type); + payloader_links.unset(payload_type); + plugin.unpause(); + } + } + if (link != null && is_sink && mixer != null) { + plugin.pause(); + link.set_locked_state(true); + Gst.Base.AggregatorPad mixer_sink_pad = (Gst.Base.AggregatorPad) link.get_static_pad("src").get_peer(); + link.get_static_pad("src").unlink(mixer_sink_pad); + mixer_sink_pad.set_active(false); + link.set_state(Gst.State.NULL); + pipe.remove(link); + mixer.release_request_pad(mixer_sink_pad); + plugin.unpause(); + } links--; if (links == 0) { destroy(); @@ -150,15 +300,59 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return target; } + private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); + } + if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { + var event = info.get_event(); + if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; + if (event.type == Gst.EventType.QOS) { + Gst.QOSType qos_type; + double proportion; + Gst.ClockTimeDiff diff; + Gst.ClockTime timestamp; + event.parse_qos(out qos_type, out proportion, out diff, out timestamp); + debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); + } else { + debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); + } + } + if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { + debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.BUFFER) > 0) { + uint id = pad.get_data("no_buffer_probe_timeout"); + if (id != 0) { + Source.remove(id); + } + string name = @"$(pad.get_parent_element().name).$(pad.name)"; + id = Timeout.add_seconds(1, () => { + debug("%s probed no buffer for 1 second", name); + return Source.REMOVE; + }); + pad.set_data("no_buffer_probe_timeout", id); + } + return Gst.PadProbeReturn.PASS; + } + private void create() { debug("Creating device %s", id); plugin.pause(); element = device.create_element(id); + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } pipe.add(element); if (is_source) { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); + filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); #if WITH_VOICE_PROCESSOR @@ -174,22 +368,18 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(tee); (dsp ?? filter).link(tee); } - if (is_sink) { - element.@set("async", false); - element.@set("sync", false); - } if (is_sink && media == "audio") { - filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); - pipe.add(filter); - if (plugin.echoprobe != null) { - rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); - rate.@set("tolerance", 100000000); - pipe.add(rate); - filter.link(rate); - rate.link(plugin.echoprobe); + mixer = (Gst.Base.Aggregator) Gst.ElementFactory.make("audiomixer", @"mixer_$id"); + pipe.add(mixer); + mixer.link(pipe); + if (plugin.echoprobe != null && !plugin.echoprobe.get_static_pad("src").is_linked()) { + mixer.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + mixer.link(filter); filter.link(element); } } @@ -197,38 +387,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } private void destroy() { - if (mixer != null) { - if (is_sink && media == "audio" && plugin.echoprobe != null) { - plugin.echoprobe.unlink(mixer); - } - int linked_sink_pads = 0; - mixer.foreach_sink_pad((_, pad) => { - if (pad.is_linked()) linked_sink_pads++; - return true; - }); - if (linked_sink_pads > 0) { - warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + if (is_sink) { + if (mixer != null) { + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.unlink(plugin.echoprobe ?? element); } - mixer.set_locked_state(true); - mixer.set_state(Gst.State.NULL); - mixer.unlink(element); - pipe.remove(mixer); - mixer = null; - } else if (is_sink && media == "audio") { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); + filter.unlink(element); pipe.remove(filter); filter = null; } - if (rate != null) { - rate.set_locked_state(true); - rate.set_state(Gst.State.NULL); - rate.unlink(plugin.echoprobe); - pipe.remove(rate); - rate = null; - } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } @@ -239,34 +416,42 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { else if (is_source) element.unlink(tee); pipe.remove(element); element = null; - if (filter != null) { - filter.set_locked_state(true); - filter.set_state(Gst.State.NULL); - filter.unlink(dsp ?? tee); - pipe.remove(filter); - filter = null; - } - if (dsp != null) { - dsp.set_locked_state(true); - dsp.set_state(Gst.State.NULL); - dsp.unlink(tee); - pipe.remove(dsp); - dsp = null; + if (mixer != null) { + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + pipe.remove(mixer); + mixer = null; } - if (tee != null) { - int linked_src_pads = 0; - tee.foreach_src_pad((_, pad) => { - if (pad.is_linked()) linked_src_pads++; - return true; - }); - if (linked_src_pads != 0) { - warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + if (is_source) { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; } - tee.set_locked_state(true); - tee.set_state(Gst.State.NULL); - pipe.remove(tee); - tee = null; } debug("Destroyed device %s", id); } -} \ No newline at end of file +} diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index bd8a279f..86f52ee7 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -18,22 +18,19 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.App.Sink send_rtcp; private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtcp; - private Gst.Element encode; - private Gst.RTP.BasePayload encode_pay; private Gst.Element decode; private Gst.RTP.BaseDepayload decode_depay; private Gst.Element input; + private Gst.Pad input_pad; private Gst.Element output; private Gst.Element session; private Device _input_device; public Device input_device { get { return _input_device; } set { if (!paused) { - if (this._input_device != null) { - this._input_device.unlink(); - this._input_device = null; - } - set_input(value != null ? value.link_source() : null); + var input = this.input; + set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset) : null); + if (this._input_device != null) this._input_device.unlink(input); } this._input_device = value; }} @@ -47,7 +44,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public bool created { get; private set; default = false; } public bool paused { get; private set; default = false; } private bool push_recv_data = false; - private string participant_ssrc = null; + private uint our_ssrc = Random.next_int(); + private int next_seqnum_offset = -1; + private uint32 participant_ssrc = 0; private Gst.Pad recv_rtcp_sink_pad; private Gst.Pad recv_rtp_sink_pad; @@ -93,7 +92,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.emit_signals = true; send_rtp.sync = false; + send_rtp.drop = true; + send_rtp.wait_on_eos = false; send_rtp.new_sample.connect(on_new_sample); + send_rtp.connect("signal::eos", on_eos_static, this); pipe.add(send_rtp); send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; @@ -101,7 +103,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.emit_signals = true; send_rtcp.sync = false; + send_rtcp.drop = true; + send_rtcp.wait_on_eos = false; send_rtcp.new_sample.connect(on_new_sample); + send_rtcp.connect("signal::eos", on_eos_static, this); pipe.add(send_rtcp); recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; @@ -125,18 +130,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); // Connect input - encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid"); - encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay"); - pipe.add(encode); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); - encode.get_static_pad("src").link(send_rtp_sink_pad); if (input != null) { - input.link(encode); + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); } // Connect output decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); - decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)decode).get_by_name(@"decode_$(rtpid)_rtp_depay"); pipe.add(decode); if (output != null) { decode.link(output); @@ -159,8 +161,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } Timeout.add(1000, () => remb_adjust()); } - if (media == "video") { - codec_util.update_bitrate(media, payload_type, encode, 256); + if (input_device != null && media == "video") { + input_device.update_bitrate(payload_type, 256); } } @@ -185,11 +187,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { warning("No source-stats for session %u", rtpid); return Source.REMOVE; } + + if (input_device == null) return Source.CONTINUE; + foreach (Value value in source_stats.values) { unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); - uint ssrc; + uint32 ssrc; if (!source_stat.get_uint("ssrc", out ssrc)) continue; - if (ssrc.to_string() == participant_ssrc) { + if (ssrc == participant_ssrc) { int packets_lost; uint64 packets_received, octets_received; source_stat.get_int("packets-lost", out packets_lost); @@ -218,10 +223,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { 1, 0, 0, 0, 0, 0, 0, 0 }; - data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); - data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); - data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); - data[7] = (uint8)(encode_pay.ssrc & 0xff); + data[4] = (uint8)((our_ssrc >> 24) & 0xff); + data[5] = (uint8)((our_ssrc >> 16) & 0xff); + data[6] = (uint8)((our_ssrc >> 8) & 0xff); + data[7] = (uint8)(our_ssrc & 0xff); uint8 br_exp = 0; uint32 br_mant = remb * 1000; uint8 bits = (uint8)Math.log2(br_mant); @@ -243,7 +248,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { - if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { + if (self.input_device != null && self.media == "video" && type == 206 && fbtype == 15 && fci != null && sender_ssrc == self.participant_ssrc) { // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 uint8[] data; fci.extract_dup(0, fci.get_size(), out data); @@ -251,7 +256,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { uint8 br_exp = data[5] >> 2; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; uint bitrate = (br_mant << br_exp) / 1000; - self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); + self.input_device.update_bitrate(self.payload_type, bitrate * 8); } } @@ -267,20 +272,30 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("Sink is null"); return Gst.FlowReturn.EOS; } + if (sink != send_rtp && sink != send_rtcp) { + warning("unknown sample"); + return Gst.FlowReturn.NOT_SUPPORTED; + } Gst.Sample sample = sink.pull_sample(); Gst.Buffer buffer = sample.get_buffer(); uint8[] data; buffer.extract_dup(0, buffer.get_size(), out data); prepare_local_crypto(); if (sink == send_rtp) { + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + if (our_ssrc != rtp_buffer.get_ssrc()) { + warning("Sending buffer with SSRC %u when our ssrc is %u", rtp_buffer.get_ssrc(), our_ssrc); + } + next_seqnum_offset = rtp_buffer.get_seq() + 1; + rtp_buffer.unmap(); + } if (crypto_session.has_encrypt) { data = crypto_session.encrypt_rtp(data); } on_send_rtp_data(new Bytes.take((owned) data)); } else if (sink == send_rtcp) { encrypt_and_send_rtcp((owned) data); - } else { - warning("unknown sample"); } return Gst.FlowReturn.OK; } @@ -300,41 +315,59 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { return Gst.PadProbeReturn.DROP; } + private static void on_eos_static(Gst.App.Sink sink, Stream self) { + debug("EOS on %s", sink.name); + if (sink == self.send_rtp) { + Idle.add(() => { self.on_send_rtp_eos(); return Source.REMOVE; }); + } else if (sink == self.send_rtcp) { + Idle.add(() => { self.on_send_rtcp_eos(); return Source.REMOVE; }); + } + } + + private void on_send_rtp_eos() { + if (send_rtp_src_pad != null) { + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + send_rtp_src_pad = null; + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + pipe.remove(send_rtp); + send_rtp = null; + debug("Stopped sending RTP for %u", rtpid); + } + + private void on_send_rtcp_eos() { + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + pipe.remove(send_rtcp); + send_rtcp = null; + debug("Stopped sending RTCP for %u", rtpid); + } + public override void destroy() { // Stop network communication push_recv_data = false; - recv_rtp.end_of_stream(); - recv_rtcp.end_of_stream(); - send_rtp.new_sample.disconnect(on_new_sample); - send_rtcp.new_sample.disconnect(on_new_sample); + if (recv_rtp != null) recv_rtp.end_of_stream(); + if (recv_rtcp != null) recv_rtcp.end_of_stream(); + if (send_rtp != null) send_rtp.new_sample.disconnect(on_new_sample); + if (send_rtcp != null) send_rtcp.new_sample.disconnect(on_new_sample); // Disconnect input device if (input != null) { - input.unlink(encode); - input = null; + input_pad.unlink(send_rtp_sink_pad); + input.release_request_pad(input_pad); + input_pad = null; } if (this._input_device != null) { - if (!paused) this._input_device.unlink(); + if (!paused) this._input_device.unlink(input); this._input_device = null; + this.input = null; } - // Disconnect encode - encode.set_locked_state(true); - encode.set_state(Gst.State.NULL); - encode.get_static_pad("src").unlink(send_rtp_sink_pad); - pipe.remove(encode); - encode = null; - encode_pay = null; - - // Disconnect RTP sending - if (send_rtp_src_pad != null) { - send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); - send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + // Inject EOS + if (send_rtp_sink_pad != null) { + send_rtp_sink_pad.send_event(new Gst.Event.eos()); } - send_rtp.set_locked_state(true); - send_rtp.set_state(Gst.State.NULL); - pipe.remove(send_rtp); - send_rtp = null; // Disconnect decode if (recv_rtp_src_pad != null) { @@ -342,57 +375,63 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); } - // Disconnect RTP receiving - recv_rtp.set_locked_state(true); - recv_rtp.set_state(Gst.State.NULL); - recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); - pipe.remove(recv_rtp); - recv_rtp = null; - // Disconnect output if (output != null) { + decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); decode.unlink(output); } - decode.set_locked_state(true); - decode.set_state(Gst.State.NULL); - pipe.remove(decode); - decode = null; - decode_depay = null; - output = null; // Disconnect output device if (this._output_device != null) { - this._output_device.unlink(); + this._output_device.unlink(output); this._output_device = null; } + output = null; - // Disconnect RTCP receiving - recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); - recv_rtcp.set_locked_state(true); - recv_rtcp.set_state(Gst.State.NULL); - pipe.remove(recv_rtcp); - recv_rtcp = null; + // Destroy decode + if (decode != null) { + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + decode_depay = null; + } - // Disconnect RTCP sending - send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); - send_rtcp.set_locked_state(true); - send_rtcp.set_state(Gst.State.NULL); - pipe.remove(send_rtcp); - send_rtcp = null; + // Disconnect and remove RTP input + if (recv_rtp != null) { + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + pipe.remove(recv_rtp); + recv_rtp = null; + } - // Release rtp pads - rtpbin.release_request_pad(send_rtp_sink_pad); - send_rtp_sink_pad = null; - rtpbin.release_request_pad(recv_rtp_sink_pad); - recv_rtp_sink_pad = null; - rtpbin.release_request_pad(recv_rtcp_sink_pad); - recv_rtcp_sink_pad = null; - rtpbin.release_request_pad(send_rtcp_src_pad); - send_rtcp_src_pad = null; - send_rtp_src_pad = null; - recv_rtp_src_pad = null; + // Disconnect and remove RTCP input + if (recv_rtcp != null) { + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + } - session = null; + // Release rtp pads + if (send_rtp_sink_pad != null) { + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + } + if (recv_rtp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + } + if (send_rtcp_src_pad != null) { + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + } + if (recv_rtcp_sink_pad != null) { + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + } } private void prepare_remote_crypto() { @@ -502,10 +541,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); } - public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { - debug("New ssrc %s with pad %s", ssrc, pad.name); - if (participant_ssrc != null && participant_ssrc != ssrc) { - warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); + public void on_ssrc_pad_added(uint32 ssrc, Gst.Pad pad) { + debug("New ssrc %u with pad %s", ssrc, pad.name); + if (participant_ssrc != 0 && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %u, new: %u), ignoring", participant_ssrc, ssrc); return; } participant_ssrc = ssrc; @@ -534,7 +573,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void set_input_and_pause(Gst.Element? input, bool paused) { if (created && this.input != null) { - this.input.unlink(encode); + this.input_pad.unlink(send_rtp_sink_pad); + this.input.release_request_pad(this.input_pad); + this.input_pad = null; this.input = null; } @@ -543,28 +584,41 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (created && sending && !paused && input != null) { plugin.pause(); - input.link(encode); + input_pad = input.get_request_pad(@"src_$rtpid"); + input_pad.link(send_rtp_sink_pad); plugin.unpause(); } } public void pause() { if (paused) return; + var input = this.input; set_input_and_pause(null, true); - if (input_device != null) input_device.unlink(); + if (input != null && input_device != null) input_device.unlink(input); } public void unpause() { if (!paused) return; - set_input_and_pause(input_device != null ? input_device.link_source() : null, false); + set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset) : null, false); + } + + public uint get_participant_ssrc(Xmpp.Jid participant) { + if (participant.equals(content.session.peer_full_jid)) { + return participant_ssrc; + } + return 0; } ulong block_probe_handler_id = 0; - public virtual void add_output(Gst.Element element) { + public virtual void add_output(Gst.Element element, Xmpp.Jid? participant = null) { if (output != null) { critical("add_output() invoked more than once"); return; } + if (participant != null) { + critical("add_output() invoked with participant when not supported"); + return; + } this.output = element; if (created) { plugin.pause(); @@ -586,7 +640,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { decode.unlink(element); } if (this._output_device != null) { - this._output_device.unlink(); + this._output_device.unlink(element); this._output_device = null; } this.output = null; @@ -657,7 +711,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { disconnect(video_orientation_changed_handler); } - public override void add_output(Gst.Element element) { + public override void add_output(Gst.Element element, Xmpp.Jid? participant) { if (element == output_tee || element == rotate) { base.add_output(element); return; @@ -678,4 +732,4 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { output_tee.unlink(element); } } -} \ No newline at end of file +} -- cgit v1.2.3-54-g00ecf From ea19a9c5cbbb6a8b9ed99627c8732005bf8854b9 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:48 +0100 Subject: RTP: Only start gstreamer pipeline once needed --- plugins/rtp/src/plugin.vala | 114 +++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 48 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d79fc2aa..3b19318e 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -5,10 +5,10 @@ using Xmpp.Xep; public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Dino.Application app { get; private set; } public CodecUtil codec_util { get; private set; } - public Gst.DeviceMonitor device_monitor { get; private set; } - public Gst.Pipeline pipe { get; private set; } - public Gst.Bin rtpbin { get; private set; } - public Gst.Element echoprobe { get; private set; } + public Gst.DeviceMonitor? device_monitor { get; private set; } + public Gst.Pipeline? pipe { get; private set; } + public Gst.Bin? rtpbin { get; private set; } + public Gst.Element? echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -42,18 +42,22 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { if (pause_count < 0) warning("Pause count below zero!"); } - public void startup() { + private void init_device_monitor() { + if (device_monitor != null) return; device_monitor = new Gst.DeviceMonitor(); device_monitor.show_all = true; device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); device_monitor.start(); foreach (Gst.Device device in device_monitor.get_devices()) { - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) continue; + if (device.properties.has_name("pipewire-proplist") && device.has_classes("Audio")) continue; if (device.properties.get_string("device.class") == "monitor") continue; if (devices.any_match((it) => it.matches(device))) continue; devices.add(new Device(this, device)); } + } + private void init_call_pipe() { + if (pipe != null) return; pipe = new Gst.Pipeline(null); // RTP @@ -66,7 +70,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { rtpbin.pad_added.connect(on_rtp_pad_added); rtpbin.@set("latency", 100); rtpbin.@set("do-lost", true); - rtpbin.@set("do-sync-event", true); +// rtpbin.@set("do-sync-event", true); rtpbin.@set("drop-on-latency", true); rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); @@ -86,6 +90,20 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.set_state(Gst.State.PLAYING); } + private void destroy_call_pipe() { + if (pipe == null) return; + pipe.set_state(Gst.State.NULL); + rtpbin = null; +#if WITH_VOICE_PROCESSOR + echoprobe = null; +#endif + pipe = null; + } + + public void startup() { + init_device_monitor(); + } + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { debug("request-pt-map"); return null; @@ -98,7 +116,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { uint8 rtpid = (uint8)int.parse(split[3]); foreach (Stream stream in streams) { if (stream.rtpid == rtpid) { - stream.on_ssrc_pad_added(split[4], pad); + stream.on_ssrc_pad_added((uint32) split[4].to_uint64(), pad); } } } @@ -137,6 +155,15 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { break; case Gst.MessageType.STATE_CHANGED: // Ignore + { + unowned Gst.Structure struc = message.get_structure(); + if (struc != null && message.src is Gst.Element) { + Gst.State oldState, newState, pendingState; + message.parse_state_changed(out oldState, out newState, out pendingState); + debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState"); + } + } + break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; @@ -179,51 +206,39 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { - Gst.Device old_device = null; - Gst.Device device = null; - Device old = null; + Gst.Device? old_gst_device = null; + Gst.Device? gst_device = null; + Device? device = null; switch (message.type) { case Gst.MessageType.DEVICE_ADDED: - message.parse_device_added(out device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; - devices.add(new Device(this, device)); + message.parse_device_added(out gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + if (devices.any_match((it) => it.matches(gst_device))) return Source.CONTINUE; + device = new Device(this, gst_device); + devices.add(device); break; #if GST_1_16 case Gst.MessageType.DEVICE_CHANGED: - message.parse_device_changed(out device, out old_device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - old = devices.first_match((it) => it.matches(old_device)); - if (old != null) old.update(device); + message.parse_device_changed(out gst_device, out old_gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + device = devices.first_match((it) => it.matches(old_gst_device)); + if (device != null) device.update(gst_device); break; #endif case Gst.MessageType.DEVICE_REMOVED: - message.parse_device_removed(out device); - if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; - if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - old = devices.first_match((it) => it.matches(device)); - if (old != null) devices.remove(old); + message.parse_device_removed(out gst_device); + if (gst_device.properties.has_name("pipewire-proplist") && gst_device.has_classes("Audio")) return Source.CONTINUE; + if (gst_device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + device = devices.first_match((it) => it.matches(gst_device)); + if (device != null) devices.remove(device); break; default: break; } if (device != null) { - switch (device.device_class) { - case "Audio/Source": - devices_changed("audio", false); - break; - case "Audio/Sink": - devices_changed("audio", true); - break; - case "Video/Source": - devices_changed("video", false); - break; - case "Video/Sink": - devices_changed("video", true); - break; - } + devices_changed(device.media, device.is_sink); } return Source.CONTINUE; } @@ -253,6 +268,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { // } public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + init_call_pipe(); var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; if (content_params == null) return null; Stream stream; @@ -271,15 +287,15 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } public void shutdown() { - device_monitor.stop(); - pipe.set_state(Gst.State.NULL); - rtpbin = null; - pipe = null; + if (device_monitor != null) { + device_monitor.stop(); + } + destroy_call_pipe(); Gst.deinit(); } public bool supports(string media) { - if (rtpbin == null) return false; + if (!codec_util.is_element_supported("rtpbin")) return false; if (media == "audio") { if (get_devices("audio", false).is_empty) return false; @@ -287,7 +303,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } if (media == "video") { - if (Gst.ElementFactory.make("gtksink", null) == null) return false; + if (!codec_util.is_element_supported("gtksink")) return false; if (get_devices("video", false).is_empty) return false; } @@ -295,6 +311,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } public VideoCallWidget? create_widget(WidgetType type) { + init_call_pipe(); if (type == WidgetType.GTK) { return new VideoWidget(this); } @@ -422,10 +439,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } } - private void dump_dot() { + public void dump_dot() { + if (pipe == null) return; string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); - debug("Stored pipe details as %s", name); + print(@"Stored pipe details as $name\n"); } public void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause) { -- cgit v1.2.3-54-g00ecf From dfda2f25f0b1d0b7e0cecf269265ca81ae1d506b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:48 +0100 Subject: DTLS: throw exceptions from SRTP --- plugins/ice/src/dtls_srtp.vala | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index ca398e69..91c11eee 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -37,40 +37,30 @@ public class Handler { this.own_fingerprint = creds.own_fingerprint; } - public uint8[]? process_incoming_data(uint component_id, uint8[] data) { + public uint8[]? process_incoming_data(uint component_id, uint8[] data) throws Crypto.Error { if (srtp_session.has_decrypt) { - try { - if (component_id == 1) { - if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { - return srtp_session.decrypt_rtcp(data); - } - return srtp_session.decrypt_rtp(data); + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.decrypt_rtcp(data); } - if (component_id == 2) return srtp_session.decrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return null; + return srtp_session.decrypt_rtp(data); } + if (component_id == 2) return srtp_session.decrypt_rtcp(data); } else if (component_id == 1) { on_data_rec(data); } return null; } - public uint8[]? process_outgoing_data(uint component_id, uint8[] data) { + public uint8[]? process_outgoing_data(uint component_id, uint8[] data) throws Crypto.Error { if (srtp_session.has_encrypt) { - try { - if (component_id == 1) { - if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { - return srtp_session.encrypt_rtcp(data); - } - return srtp_session.encrypt_rtp(data); + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.encrypt_rtcp(data); } - if (component_id == 2) return srtp_session.encrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - return null; + return srtp_session.encrypt_rtp(data); } + if (component_id == 2) return srtp_session.encrypt_rtcp(data); } return null; } -- cgit v1.2.3-54-g00ecf From f398135bc8a21343058f7861abfc083337bc401d Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:49 +0100 Subject: RTP: Make opus mono-channel --- plugins/rtp/src/module.vala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index dfe224aa..4f48b6c4 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -63,7 +63,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return supported; } - private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { + public override async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; @@ -131,7 +131,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async Gee.List get_supported_payloads(string media) { Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); if (media == "audio") { - var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 }; + var opus = new JingleRtp.PayloadType() { channels = 1, clockrate = 48000, name = "opus", id = 99 }; opus.parameters["useinbandfec"] = "1"; var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 }; var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 }; -- cgit v1.2.3-54-g00ecf From 6fa5702e9c9f8201f923210a2a54790ba9bc3a69 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:49 +0100 Subject: ICE: Report transferred bytes for UI --- plugins/ice/src/transport_parameters.vala | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 62c04906..cb9cea07 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -16,10 +16,6 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private DtlsSrtp.Handler? dtls_srtp_handler; private uint stream_id; private string? error; - private ulong sent; - private ulong sent_reported; - private ulong recv; - private ulong recv_reported; private ulong datagram_received_id; public DatagramConnection(Nice.Agent agent, DtlsSrtp.Handler? dtls_srtp_handler, uint stream_id, uint8 component_id) { @@ -28,11 +24,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport this.stream_id = stream_id; this.component_id = component_id; this.datagram_received_id = this.datagram_received.connect((datagram) => { - recv += datagram.length; - if (recv > recv_reported + 100000) { - debug("Received %lu bytes via stream %u component %u", recv, stream_id, component_id); - recv_reported = recv; - } + bytes_received += datagram.length; }); } @@ -47,15 +39,15 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { uint8[] encrypted_data = null; if (dtls_srtp_handler != null) { - encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); - if (encrypted_data == null) return; + try { + encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); + if (encrypted_data == null) return; + } catch (Crypto.Error e) { + warning("%s while send_datagram stream %u component %u", e.message, stream_id, component_id); + } } agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); - sent += datagram.length; - if (sent > sent_reported + 100000) { - debug("Sent %lu bytes via stream %u component %u", sent, stream_id, component_id); - sent_reported = sent; - } + bytes_sent += datagram.length; } } } -- cgit v1.2.3-54-g00ecf From c9194973de629a74736d7b42add8b7f3e5c5085b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:50 +0100 Subject: Log when DTLS-SRTP has errors --- plugins/ice/src/transport_parameters.vala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index cb9cea07..a91cc538 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -252,8 +252,12 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (stream_id != this.stream_id) return; uint8[] decrypt_data = null; if (dtls_srtp_handler != null) { - decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); - if (decrypt_data == null) return; + try { + decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); + if (decrypt_data == null) return; + } catch (Crypto.Error e) { + warning("%s while on_recv stream %u component %u", e.message, stream_id, component_id); + } } may_consider_ready(stream_id, component_id); if (connections.has_key((uint8) component_id)) { -- cgit v1.2.3-54-g00ecf From 0fe24c433909840a7d812630639bc13ab24f1bad Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 9 Nov 2021 22:06:50 +0100 Subject: Register local ip address handler for raw udp --- plugins/ice/src/plugin.vala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/ice/src/plugin.vala b/plugins/ice/src/plugin.vala index f145dd6d..4abf042c 100644 --- a/plugins/ice/src/plugin.vala +++ b/plugins/ice/src/plugin.vala @@ -15,7 +15,12 @@ public class Dino.Plugins.Ice.Plugin : RootInterface, Object { list.add(new Module()); }); app.stream_interactor.stream_attached_modules.connect((account, stream) => { - stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + if (stream.get_module(Socks5Bytestreams.Module.IDENTITY) != null) { + stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + } + if (stream.get_module(JingleRawUdp.Module.IDENTITY) != null) { + stream.get_module(JingleRawUdp.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + } }); app.stream_interactor.stream_negotiated.connect(on_stream_negotiated); } -- cgit v1.2.3-54-g00ecf From cfe43de5d5bcc46691be0d0328fcbcb9f1a2e2af Mon Sep 17 00:00:00 2001 From: Marvin W Date: Wed, 10 Nov 2021 23:11:25 +0100 Subject: Make elements sync to get proper qos data --- plugins/rtp/src/stream.vala | 4 ++-- plugins/rtp/src/video_widget.vala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 86f52ee7..5d1cf6bf 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -91,7 +91,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtp.async = false; send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.emit_signals = true; - send_rtp.sync = false; + send_rtp.sync = true; send_rtp.drop = true; send_rtp.wait_on_eos = false; send_rtp.new_sample.connect(on_new_sample); @@ -102,7 +102,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp.async = false; send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.emit_signals = true; - send_rtcp.sync = false; + send_rtcp.sync = true; send_rtcp.drop = true; send_rtcp.wait_on_eos = false; send_rtcp.new_sample.connect(on_new_sample); diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala index ccd86bb2..3daf5284 100644 --- a/plugins/rtp/src/video_widget.vala +++ b/plugins/rtp/src/video_widget.vala @@ -25,7 +25,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge Gtk.Widget widget; element.@get("widget", out widget); element.@set("async", false); - element.@set("sync", false); + element.@set("sync", true); this.widget = widget; add(widget); widget.visible = true; -- cgit v1.2.3-54-g00ecf From 1b157a20ab728a010eb85292177bcb62cef6a839 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Wed, 10 Nov 2021 23:12:19 +0100 Subject: Fix REMB calculation --- plugins/rtp/src/stream.vala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 5d1cf6bf..17c955a5 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -201,12 +201,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { source_stat.get_uint64("packets-received", out packets_received); source_stat.get_uint64("octets-received", out octets_received); int new_lost = packets_lost - last_packets_lost; + if (new_lost < 0) new_lost = 0; uint64 new_received = packets_received - last_packets_received; + if (packets_received < last_packets_received) new_received = 0; uint64 new_octets = octets_received - last_octets_received; - if (new_received == 0) continue; + if (octets_received < last_octets_received) octets_received = 0; last_packets_lost = packets_lost; last_packets_received = packets_received; last_octets_received = octets_received; + if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); if (new_lost <= 0 || loss_rate < 0.02) { remb = (uint)(1.08 * (double)remb); @@ -256,7 +259,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { uint8 br_exp = data[5] >> 2; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; uint bitrate = (br_mant << br_exp) / 1000; - self.input_device.update_bitrate(self.payload_type, bitrate * 8); + self.input_device.update_bitrate(self.payload_type, bitrate); } } -- cgit v1.2.3-54-g00ecf From e205743f0cb71e32eecf183d067a4f34a7a89d48 Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 11 Nov 2021 21:54:55 +0100 Subject: Display target bitrates in connection details UI --- libdino/src/service/call_peer_state.vala | 12 ++++++++ .../call_connection_details_window.vala | 36 +++++++++++++++++----- plugins/rtp/src/stream.vala | 17 +++++----- .../src/module/xep/0167_jingle_rtp/stream.vala | 8 ++++- 4 files changed, 55 insertions(+), 18 deletions(-) (limited to 'plugins') diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala index ddd0d8dd..c7bcd201 100644 --- a/libdino/src/service/call_peer_state.vala +++ b/libdino/src/service/call_peer_state.vala @@ -260,6 +260,10 @@ public class Dino.PeerState : Object { ret.audio_codec = audio_content_parameter.agreed_payload_type.name; ret.audio_clockrate = audio_content_parameter.agreed_payload_type.clockrate; } + if (audio_content_parameter.stream != null && audio_content_parameter.stream.remb_enabled) { + ret.audio_target_receive_bitrate = audio_content_parameter.stream.target_receive_bitrate; + ret.audio_target_send_bitrate = audio_content_parameter.stream.target_send_bitrate; + } } if (audio_content != null) { @@ -278,6 +282,10 @@ public class Dino.PeerState : Object { if (video_content_parameter.agreed_payload_type != null) { ret.video_codec = video_content_parameter.agreed_payload_type.name; } + if (video_content_parameter.stream != null && video_content_parameter.stream.remb_enabled) { + ret.video_target_receive_bitrate = video_content_parameter.stream.target_receive_bitrate; + ret.video_target_send_bitrate = video_content_parameter.stream.target_send_bitrate; + } } if (video_content != null) { @@ -443,6 +451,8 @@ public class Dino.PeerInfo { public ulong? audio_bytes_received { get; set; default=0; } public string? audio_codec { get; set; } public uint32 audio_clockrate { get; set; } + public uint audio_target_receive_bitrate { get; set; default=0; } + public uint audio_target_send_bitrate { get; set; default=0; } public bool video_content_exists { get; set; } public bool video_rtp_ready { get; set; } @@ -450,4 +460,6 @@ public class Dino.PeerInfo { public ulong? video_bytes_sent { get; set; default=0; } public ulong? video_bytes_received { get; set; default=0; } public string? video_codec { get; set; } + public uint video_target_receive_bitrate { get; set; default=0; } + public uint video_target_send_bitrate { get; set; default=0; } } \ No newline at end of file diff --git a/main/src/ui/call_window/call_connection_details_window.vala b/main/src/ui/call_window/call_connection_details_window.vala index b292b971..95e00296 100644 --- a/main/src/ui/call_window/call_connection_details_window.vala +++ b/main/src/ui/call_window/call_connection_details_window.vala @@ -8,15 +8,19 @@ namespace Dino.Ui { public Label audio_rtp_ready = new Label("?") { xalign=0, visible=true }; public Label audio_rtcp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_sent_bps = new Label("?") { xalign=0, visible=true }; - public Label audio_recv_bps = new Label("?") { xalign=0, visible=true }; + public Label audio_sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label audio_recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; public Label audio_codec = new Label("?") { xalign=0, visible=true }; + public Label audio_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Label audio_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; public Label video_rtp_ready = new Label("") { xalign=0, visible=true }; public Label video_rtcp_ready = new Label("") { xalign=0, visible=true }; - public Label video_sent_bps = new Label("") { xalign=0, visible=true }; - public Label video_recv_bps = new Label("") { xalign=0, visible=true }; + public Label video_sent_bps = new Label("") { use_markup=true, xalign=0, visible=true }; + public Label video_recv_bps = new Label("") { use_markup=true, xalign=0, visible=true }; public Label video_codec = new Label("") { xalign=0, visible=true }; + public Label video_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Label video_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; private int row_at = 0; private bool video_added = false; @@ -34,6 +38,10 @@ namespace Dino.Ui { grid.attach(audio_recv_bps, 1, row_at++, 1, 1); put_row("Codec"); grid.attach(audio_codec, 1, row_at++, 1, 1); + put_row("Target receive bitrate"); + grid.attach(audio_target_receive_bitrate, 1, row_at++, 1, 1); + put_row("Target send bitrate"); + grid.attach(audio_target_send_bitrate, 1, row_at++, 1, 1); this.child = grid; } @@ -46,18 +54,26 @@ namespace Dino.Ui { audio_rtp_ready.label = peer_info.audio_rtp_ready.to_string(); audio_rtcp_ready.label = peer_info.audio_rtcp_ready.to_string(); audio_codec.label = peer_info.audio_codec + " " + peer_info.audio_clockrate.to_string(); + audio_target_receive_bitrate.label = peer_info.audio_target_receive_bitrate.to_string(); + audio_target_send_bitrate.label = peer_info.audio_target_send_bitrate.to_string(); video_rtp_ready.label = peer_info.video_rtp_ready.to_string(); video_rtcp_ready.label = peer_info.video_rtcp_ready.to_string(); video_codec.label = peer_info.video_codec; + video_target_receive_bitrate.label = peer_info.video_target_receive_bitrate.to_string(); + video_target_send_bitrate.label = peer_info.video_target_send_bitrate.to_string(); if (peer_info.video_content_exists) add_video_widgets(); if (prev_peer_info != null) { - audio_sent_bps.label = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent).to_string(); - audio_recv_bps.label = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received).to_string(); - video_sent_bps.label = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent).to_string(); - video_recv_bps.label = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received).to_string(); + ulong audio_sent_kbps = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent) * 8 / 1000; + audio_sent_bps.label = "%lu kbps".printf(audio_sent_kbps); + ulong audio_recv_kbps = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received) * 8 / 1000; + audio_recv_bps.label = "%lu kbps".printf(audio_recv_kbps); + ulong video_sent_kbps = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent) * 8 / 1000; + video_sent_bps.label = "%lu kbps".printf(video_sent_kbps); + ulong video_recv_kbps = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received) * 8 / 1000; + video_recv_bps.label = "%lu kbps".printf(video_recv_kbps); } prev_peer_info = peer_info; } @@ -76,6 +92,10 @@ namespace Dino.Ui { grid.attach(video_recv_bps, 1, row_at++, 1, 1); put_row("Codec"); grid.attach(video_codec, 1, row_at++, 1, 1); + put_row("Target receive bitrate"); + grid.attach(video_target_receive_bitrate, 1, row_at++, 1, 1); + put_row("Target send bitrate"); + grid.attach(video_target_send_bitrate, 1, row_at++, 1, 1); video_added = true; } diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 17c955a5..bfe5ae28 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -153,7 +153,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { plugin.unpause(); GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); - if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + if (session != null && remb_enabled) { Object internal_session; session.@get("internal-session", out internal_session); if (internal_session != null) { @@ -166,7 +166,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } - private uint remb = 256; private int last_packets_lost = -1; private uint64 last_packets_received; private uint64 last_octets_received; @@ -212,12 +211,12 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); if (new_lost <= 0 || loss_rate < 0.02) { - remb = (uint)(1.08 * (double)remb); + target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { - remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); } - remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); - remb = uint.max(16, remb); // Never go below 16 + target_receive_bitrate = uint.max(target_receive_bitrate, (uint)((new_octets * 8) / 1000)); + target_receive_bitrate = uint.max(16, target_receive_bitrate); // Never go below 16 uint8[] data = new uint8[] { 143, 206, 0, 5, 0, 0, 0, 0, @@ -231,7 +230,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { data[6] = (uint8)((our_ssrc >> 8) & 0xff); data[7] = (uint8)(our_ssrc & 0xff); uint8 br_exp = 0; - uint32 br_mant = remb * 1000; + uint32 br_mant = target_receive_bitrate * 1000; uint8 bits = (uint8)Math.log2(br_mant); if (bits > 16) { br_exp = (uint8)bits - 16; @@ -258,8 +257,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; uint8 br_exp = data[5] >> 2; uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; - uint bitrate = (br_mant << br_exp) / 1000; - self.input_device.update_bitrate(self.payload_type, bitrate); + self.target_send_bitrate = (br_mant << br_exp) / 1000; + self.input_device.update_bitrate(self.payload_type, self.target_send_bitrate); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 65be8a0a..031f0ad0 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -1,5 +1,4 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { - public Jingle.Content content { get; protected set; } public string name { get { @@ -54,6 +53,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { return false; }} + // Receiver Estimated Maximum Bitrate + public bool remb_enabled { get { + return payload_type != null ? payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb") : false; + }} + public uint target_receive_bitrate { get; set; default=256; } + public uint target_send_bitrate { get; set; default=256; } + protected Stream(Jingle.Content content) { this.content = content; } -- cgit v1.2.3-54-g00ecf From 9e5a3895ae6f59a3b57c453c61cc3744c6abae9c Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 11 Nov 2021 22:35:45 +0100 Subject: Limit REMB target bitrate to 2x maximum actually seen value --- plugins/rtp/src/stream.vala | 80 +++++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 32 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index bfe5ae28..a5b1482a 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -167,8 +167,10 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private int last_packets_lost = -1; - private uint64 last_packets_received; - private uint64 last_octets_received; + private uint64 last_packets_received = 0; + private uint64 last_octets_received = 0; + private uint max_target_receive_bitrate = 0; + private int64 last_remb_time = 0; private bool remb_adjust() { unowned Gst.Structure? stats; if (session == null) { @@ -210,40 +212,54 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { last_octets_received = octets_received; if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); + uint new_target_receive_bitrate = 256; if (new_lost <= 0 || loss_rate < 0.02) { - target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); + new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { - target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); + new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); } - target_receive_bitrate = uint.max(target_receive_bitrate, (uint)((new_octets * 8) / 1000)); - target_receive_bitrate = uint.max(16, target_receive_bitrate); // Never go below 16 - uint8[] data = new uint8[] { - 143, 206, 0, 5, - 0, 0, 0, 0, - 0, 0, 0, 0, - 'R', 'E', 'M', 'B', - 1, 0, 0, 0, - 0, 0, 0, 0 - }; - data[4] = (uint8)((our_ssrc >> 24) & 0xff); - data[5] = (uint8)((our_ssrc >> 16) & 0xff); - data[6] = (uint8)((our_ssrc >> 8) & 0xff); - data[7] = (uint8)(our_ssrc & 0xff); - uint8 br_exp = 0; - uint32 br_mant = target_receive_bitrate * 1000; - uint8 bits = (uint8)Math.log2(br_mant); - if (bits > 16) { - br_exp = (uint8)bits - 16; - br_mant = br_mant >> br_exp; + if (last_remb_time == 0) { + last_remb_time = get_monotonic_time(); + } else { + int64 time_now = get_monotonic_time(); + int64 time_diff = time_now - last_remb_time; + last_remb_time = time_now; + uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); + new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); + max_target_receive_bitrate = uint.max(actual_bitrate * 2, max_target_receive_bitrate); + new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); + } + new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 + if (new_target_receive_bitrate != target_receive_bitrate) { + target_receive_bitrate = new_target_receive_bitrate; + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((our_ssrc >> 24) & 0xff); + data[5] = (uint8)((our_ssrc >> 16) & 0xff); + data[6] = (uint8)((our_ssrc >> 8) & 0xff); + data[7] = (uint8)(our_ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = target_receive_bitrate * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); } - data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); - data[18] = (uint8)((br_mant >> 8) & 0xff); - data[19] = (uint8)(br_mant & 0xff); - data[20] = (uint8)((ssrc >> 24) & 0xff); - data[21] = (uint8)((ssrc >> 16) & 0xff); - data[22] = (uint8)((ssrc >> 8) & 0xff); - data[23] = (uint8)(ssrc & 0xff); - encrypt_and_send_rtcp(data); } } return Source.CONTINUE; -- cgit v1.2.3-54-g00ecf From 9958cfbe7b4467ec5a5fed4c7e5e06f7f8e9179b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 11 Nov 2021 22:49:48 +0100 Subject: Log probe for decode QOS --- plugins/rtp/src/stream.vala | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'plugins') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index a5b1482a..24adcb9a 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -74,6 +74,45 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } + private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); + } + if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { + var event = info.get_event(); + if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; + if (event.type == Gst.EventType.QOS) { + Gst.QOSType qos_type; + double proportion; + Gst.ClockTimeDiff diff; + Gst.ClockTime timestamp; + event.parse_qos(out qos_type, out proportion, out diff, out timestamp); + debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); + } else { + debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); + } + } + if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { + debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { + debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); + } + if ((info.type & Gst.PadProbeType.BUFFER) > 0) { + uint id = pad.get_data("no_buffer_probe_timeout"); + if (id != 0) { + Source.remove(id); + } + string name = @"$(pad.get_parent_element().name).$(pad.name)"; + id = Timeout.add_seconds(1, () => { + debug("%s probed no buffer for 1 second", name); + return Source.REMOVE; + }); + pad.set_data("no_buffer_probe_timeout", id); + } + return Gst.PadProbeReturn.PASS; + } + public override void create() { plugin.pause(); @@ -114,6 +153,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp.do_timestamp = true; recv_rtp.format = Gst.Format.TIME; recv_rtp.is_live = true; + recv_rtp.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(recv_rtp); recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; -- cgit v1.2.3-54-g00ecf From 0b828a0ae55b3c0eaf1206f0c54cfcee4f60e6af Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 15 Nov 2021 22:49:44 +0100 Subject: Add maximum bitrate and adjust video resolution based on bitrate --- plugins/rtp/src/codec_util.vala | 35 +++++++++++++---- plugins/rtp/src/device.vala | 85 ++++++++++++++++++++++++++++++++++++++++- plugins/rtp/src/plugin.vala | 1 - plugins/rtp/src/stream.vala | 2 +- 4 files changed, 111 insertions(+), 12 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 4c68985f..ec922f97 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -140,13 +140,16 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 if (encode == "msdkh264enc") return @" rate-control=vbr"; - if (encode == "vaapih264enc") return @" tune=low-power"; - if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; + if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency"; // VP8 - if (encode == "msdkvp8enc") return " rate-control=vbr"; - if (encode == "vaapivp8enc") return " rate-control=vbr"; - if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30"; + if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; + + // VP9 + if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; + if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; // OPUS if (encode == "opusenc") { @@ -186,13 +189,29 @@ public class Dino.Plugins.Rtp.CodecUtil { case "vp9enc": case "vp8enc": bitrate = uint.min(2147483, bitrate); - encode.set("target-bitrate", bitrate * 1000); + encode.set("target-bitrate", bitrate * 1024); return bitrate; } return 0; } + public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + rescale_caps.set("caps", caps); + } + + public Gst.Caps? get_rescale_caps(Gst.Element encode_element) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return null; + Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps"); + Gst.Caps caps; + rescale_caps.get("caps", out caps); + return caps; + } + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { return null; } @@ -309,8 +328,8 @@ public class Dino.Plugins.Rtp.CodecUtil { string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; - return @"$(media)convert name=$(base_name)_convert$resample ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; + string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps"; + return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3546ab94..89d499ed 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -36,6 +36,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return device.has_classes("Sink"); }} + private Gst.Caps device_caps; private Gst.Element element; private Gst.Element tee; private Gst.Element dsp; @@ -46,9 +47,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { // Codecs private Gee.Map codecs = new HashMap(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map codec_tees = new HashMap(PayloadType.hash_func, PayloadType.equals_func); + + // Payloaders private Gee.Map> payloaders = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map> payloader_tees = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private Gee.Map> payloader_links = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); + + // Bitrate private Gee.Map> codec_bitrates = new HashMap>(PayloadType.hash_func, PayloadType.equals_func); private class CodecBitrate { @@ -146,6 +151,51 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } + private static double get_target_bitrate(Gst.Caps caps) { + if (caps == null || caps.get_size() == 0) return uint.MAX; + unowned Gst.Structure? that = caps.get_structure(0); + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX; + if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX; + if (!that.has_field("framerate")) return uint.MAX; + Value framerate = that.get_value("framerate"); + if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX; + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + double pxs = ((double)num/(double)den) * (double)width * (double)height; + double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0; + if (br < 128.0) return 128.0; + return br; + } + + private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840}; + private Gst.Caps get_active_caps(PayloadType payload_type) { + return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; + } + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { + plugin.pause(); + debug("Set scaled caps to %s", caps.to_string()); + codec_util.update_rescale_caps(codecs[payload_type], caps); + plugin.unpause(); + } + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { + int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; + device_caps.get_structure(0).get_int("width", out device_caps_width); + device_caps.get_structure(0).get_int("height", out device_caps_height); + device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den); + Gst.Caps active_caps = get_active_caps(payload_type); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (new_width == active_caps_width) return; + int new_height = device_caps_height * new_width / device_caps_width; + Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); + double required_bitrate = get_target_bitrate(new_caps); + if (bitrate < required_bitrate) return; + apply_caps(payload_type, new_caps); + } public void update_bitrate(PayloadType payload_type, uint bitrate) { if (codecs.has_key(payload_type)) { lock(codec_bitrates); @@ -164,6 +214,36 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { } } codec_bitrates[payload_type].remove_all(remove); + if (media == "video") { + if (bitrate < 128) bitrate = 128; + Gst.Caps active_caps = get_active_caps(payload_type); + double max_bitrate = get_target_bitrate(device_caps) * 2; + double current_target_bitrate = get_target_bitrate(active_caps); + int device_caps_width, active_caps_width; + device_caps.get_structure(0).get_int("width", out device_caps_width); + if (active_caps != null && active_caps.get_size() > 0) { + active_caps.get_structure(0).get_int("width", out active_caps_width); + } else { + active_caps_width = device_caps_width; + } + if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { + // Lower video resolution + int i = 1; + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++); + apply_width(payload_type, common_widths[i-1], bitrate); + } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { + // Higher video resolution + int i = 0; + for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] > device_caps_width) { + // We never scale up, so just stick with what the device gives + apply_width(payload_type, device_caps_width, bitrate); + } else if (common_widths[i] != active_caps_width) { + apply_width(payload_type, common_widths[i], bitrate); + } + } + if (bitrate > max_bitrate) bitrate = (uint) max_bitrate; + } codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate); unlock(codec_bitrates); } @@ -348,10 +428,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } pipe.add(element); + device_caps = get_best_caps(); if (is_source) { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); + filter.@set("caps", device_caps); filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); @@ -377,7 +458,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { plugin.echoprobe.link(element); } else { filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); - filter.@set("caps", get_best_caps()); + filter.@set("caps", device_caps); pipe.add(filter); mixer.link(filter); filter.link(element); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 3b19318e..d10303a6 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -163,7 +163,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState"); } } - break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 24adcb9a..5e5a556b 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -266,7 +266,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { last_remb_time = time_now; uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0); new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate)); - max_target_receive_bitrate = uint.max(actual_bitrate * 2, max_target_receive_bitrate); + max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate); new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate); } new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16 -- cgit v1.2.3-54-g00ecf From ec6541518684d7c61c6475498c4ddf25d8f96b55 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 15 Nov 2021 23:48:47 +0100 Subject: Optimize encoder for low cpu usage --- plugins/rtp/src/codec_util.vala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index ec922f97..f9f7f7ec 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -97,7 +97,7 @@ public class Dino.Plugins.Rtp.CodecUtil { case "h264": return new string[] {/*"msdkh264enc", */"vaapih264enc", "x264enc"}; case "vp9": - return new string[] {/*"msdkvp9enc", */"vaapivp9enc" /*, "vp9enc" */}; + return new string[] {/*"msdkvp9enc", */"vaapivp9enc", "vp9enc"}; case "vp8": return new string[] {/*"msdkvp8enc", */"vaapivp8enc", "vp8enc"}; } @@ -141,15 +141,15 @@ public class Dino.Plugins.Rtp.CodecUtil { // H264 if (encode == "msdkh264enc") return @" rate-control=vbr"; if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power"; - if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency"; + if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency bframes=0 cabac=false dct8x8=false"; // VP8 if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90"; - if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; + if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; // VP9 if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90"; - if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr"; + if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr cpu-used=4"; // OPUS if (encode == "opusenc") { @@ -219,7 +219,7 @@ public class Dino.Plugins.Rtp.CodecUtil { public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; - if (decode == "vp8dec") return " threads=8"; + if (decode == "vp8dec" || decode == "vp9dec") return " threads=8"; return null; } -- cgit v1.2.3-54-g00ecf From 2b3d150949fe1b3c4107e497be7dac8e2ba734aa Mon Sep 17 00:00:00 2001 From: fiaxh Date: Mon, 15 Nov 2021 13:29:13 +0100 Subject: Improve call details dialog + small multi-party call fixes --- libdino/src/entity/call.vala | 2 +- libdino/src/service/call_peer_state.vala | 86 +++++----- libdino/src/service/calls.vala | 11 +- .../call_connection_details_window.vala | 175 +++++++++++---------- .../ui/conversation_content_view/call_widget.vala | 2 +- plugins/rtp/src/stream.vala | 2 +- .../xep/0167_jingle_rtp/content_parameters.vala | 1 + .../module/xep/0353_jingle_message_initiation.vala | 4 +- 8 files changed, 139 insertions(+), 144 deletions(-) (limited to 'plugins') diff --git a/libdino/src/entity/call.vala b/libdino/src/entity/call.vala index a5ab672e..5221a807 100644 --- a/libdino/src/entity/call.vala +++ b/libdino/src/entity/call.vala @@ -11,7 +11,7 @@ namespace Dino.Entities { RINGING, ESTABLISHING, IN_PROGRESS, - OTHER_DEVICE_ACCEPTED, + OTHER_DEVICE, ENDED, DECLINED, MISSED, diff --git a/libdino/src/service/call_peer_state.vala b/libdino/src/service/call_peer_state.vala index c7bcd201..8c4b0930 100644 --- a/libdino/src/service/call_peer_state.vala +++ b/libdino/src/service/call_peer_state.vala @@ -251,53 +251,43 @@ public class Dino.PeerState : Object { public PeerInfo get_info() { var ret = new PeerInfo(); - - if (audio_content_parameter != null) { - ret.audio_rtcp_ready = audio_content_parameter.rtcp_ready; - ret.audio_rtp_ready = audio_content_parameter.rtp_ready; - - if (audio_content_parameter.agreed_payload_type != null) { - ret.audio_codec = audio_content_parameter.agreed_payload_type.name; - ret.audio_clockrate = audio_content_parameter.agreed_payload_type.clockrate; - } - if (audio_content_parameter.stream != null && audio_content_parameter.stream.remb_enabled) { - ret.audio_target_receive_bitrate = audio_content_parameter.stream.target_receive_bitrate; - ret.audio_target_send_bitrate = audio_content_parameter.stream.target_send_bitrate; - } + if (audio_content != null || audio_content_parameter != null) { + ret.audio = get_content_info(audio_content, audio_content_parameter); } - - if (audio_content != null) { - Xmpp.Xep.Jingle.ComponentConnection? component0 = audio_content.get_transport_connection(1); - if (component0 != null) { - ret.audio_bytes_received = component0.bytes_received; - ret.audio_bytes_sent = component0.bytes_sent; - } + if (video_content != null || video_content_parameter != null) { + ret.video = get_content_info(video_content, video_content_parameter); } + return ret; + } - if (video_content_parameter != null) { - ret.video_content_exists = true; - ret.video_rtcp_ready = video_content_parameter.rtcp_ready; - ret.video_rtp_ready = video_content_parameter.rtp_ready; + private PeerContentInfo get_content_info(Xep.Jingle.Content? content, Xep.JingleRtp.Parameters? parameter) { + PeerContentInfo ret = new PeerContentInfo(); + if (parameter != null) { + ret.rtcp_ready = parameter.rtcp_ready; + ret.rtp_ready = parameter.rtp_ready; - if (video_content_parameter.agreed_payload_type != null) { - ret.video_codec = video_content_parameter.agreed_payload_type.name; + if (parameter.agreed_payload_type != null) { + ret.codec = parameter.agreed_payload_type.name; + ret.clockrate = parameter.agreed_payload_type.clockrate; } - if (video_content_parameter.stream != null && video_content_parameter.stream.remb_enabled) { - ret.video_target_receive_bitrate = video_content_parameter.stream.target_receive_bitrate; - ret.video_target_send_bitrate = video_content_parameter.stream.target_send_bitrate; + if (parameter.stream != null && parameter.stream.remb_enabled) { + ret.target_receive_bytes = parameter.stream.target_receive_bitrate; + ret.target_send_bytes = parameter.stream.target_send_bitrate; } } - if (video_content != null) { - Xmpp.Xep.Jingle.ComponentConnection? component0 = video_content.get_transport_connection(1); + if (content != null) { + Xmpp.Xep.Jingle.ComponentConnection? component0 = content.get_transport_connection(1); if (component0 != null) { - ret.video_bytes_received = component0.bytes_received; - ret.video_bytes_sent = component0.bytes_sent; + ret.bytes_received = component0.bytes_received; + ret.bytes_sent = component0.bytes_sent; } } return ret; } + + private void connect_content_signals(Xep.Jingle.Content content, Xep.JingleRtp.Parameters rtp_content_parameter) { if (rtp_content_parameter.media == "audio") { audio_content = content; @@ -444,22 +434,18 @@ public class Dino.PeerState : Object { } } +public class Dino.PeerContentInfo { + public bool rtp_ready { get; set; } + public bool rtcp_ready { get; set; } + public ulong? bytes_sent { get; set; default=0; } + public ulong? bytes_received { get; set; default=0; } + public string? codec { get; set; } + public uint32 clockrate { get; set; } + public uint target_receive_bytes { get; set; default=-1; } + public uint target_send_bytes { get; set; default=-1; } +} + public class Dino.PeerInfo { - public bool audio_rtp_ready { get; set; } - public bool audio_rtcp_ready { get; set; } - public ulong? audio_bytes_sent { get; set; default=0; } - public ulong? audio_bytes_received { get; set; default=0; } - public string? audio_codec { get; set; } - public uint32 audio_clockrate { get; set; } - public uint audio_target_receive_bitrate { get; set; default=0; } - public uint audio_target_send_bitrate { get; set; default=0; } - - public bool video_content_exists { get; set; } - public bool video_rtp_ready { get; set; } - public bool video_rtcp_ready { get; set; } - public ulong? video_bytes_sent { get; set; default=0; } - public ulong? video_bytes_received { get; set; default=0; } - public string? video_codec { get; set; } - public uint video_target_receive_bitrate { get; set; default=0; } - public uint video_target_send_bitrate { get; set; default=0; } + public PeerContentInfo? audio = null; + public PeerContentInfo? video = null; } \ No newline at end of file diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index 51ed6e78..3d1ed7e8 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -202,16 +202,17 @@ namespace Dino { // Call requested by another of our devices call.direction = Call.DIRECTION_OUTGOING; call.ourpart = from; + call.state = Call.State.OTHER_DEVICE; counterpart = to; } else { call.direction = Call.DIRECTION_INCOMING; call.ourpart = account.full_jid; + call.state = Call.State.RINGING; counterpart = from; } call.add_peer(counterpart); call.account = account; call.time = call.local_time = call.end_time = new DateTime.now_utc(); - call.state = Call.State.RINGING; Conversation conversation = stream_interactor.get_module(ConversationManager.IDENTITY).create_conversation(counterpart.bare_jid, account, Conversation.Type.CHAT); @@ -329,7 +330,7 @@ namespace Dino { current_jmi_request_peer[account] = peer_state; current_jmi_request_call[account] = call_states[peer_state.call]; }); - mi_module.session_accepted.connect((from, sid) => { + mi_module.session_accepted.connect((from, to, sid) => { if (!current_jmi_request_peer.has_key(account) || current_jmi_request_peer[account].sid != sid) return; if (from.equals_bare(account.bare_jid)) { // Carboned message from our account @@ -337,9 +338,9 @@ namespace Dino { if (from.equals(account.full_jid)) return; Call call = current_jmi_request_peer[account].call; - call.state = Call.State.OTHER_DEVICE_ACCEPTED; + call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call); - } else if (from.equals_bare(current_jmi_request_peer[account].jid)) { // Message from our peer + } else if (from.equals_bare(current_jmi_request_peer[account].jid) && to.equals(account.full_jid)) { // Message from our peer // We proposed the call // We know the full jid of our peer now current_jmi_request_call[account].rename_peer(current_jmi_request_peer[account].jid, from); @@ -383,7 +384,7 @@ namespace Dino { CallState? call_state = get_call_state_for_groupcall(account, muc_jid); if (call_state == null) return; - call_state.call.state = Call.State.OTHER_DEVICE_ACCEPTED; + call_state.call.state = Call.State.OTHER_DEVICE; remove_call_from_datastructures(call_state.call); }); muji_meta_module.call_retracted.connect((from_jid, muc_jid) => { diff --git a/main/src/ui/call_window/call_connection_details_window.vala b/main/src/ui/call_window/call_connection_details_window.vala index 95e00296..1d5265c9 100644 --- a/main/src/ui/call_window/call_connection_details_window.vala +++ b/main/src/ui/call_window/call_connection_details_window.vala @@ -4,100 +4,107 @@ namespace Dino.Ui { public class CallConnectionDetailsWindow : Gtk.Window { - public Grid grid = new Grid() { column_spacing=5, margin=10, halign=Align.CENTER, valign=Align.CENTER, visible=true }; - - public Label audio_rtp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_rtcp_ready = new Label("?") { xalign=0, visible=true }; - public Label audio_sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; - public Label audio_recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; - public Label audio_codec = new Label("?") { xalign=0, visible=true }; - public Label audio_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; - public Label audio_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; - - public Label video_rtp_ready = new Label("") { xalign=0, visible=true }; - public Label video_rtcp_ready = new Label("") { xalign=0, visible=true }; - public Label video_sent_bps = new Label("") { use_markup=true, xalign=0, visible=true }; - public Label video_recv_bps = new Label("") { use_markup=true, xalign=0, visible=true }; - public Label video_codec = new Label("") { xalign=0, visible=true }; - public Label video_target_receive_bitrate = new Label("n/a") { xalign=0, visible=true }; - public Label video_target_send_bitrate = new Label("n/a") { xalign=0, visible=true }; + public Box box = new Box(Orientation.VERTICAL, 15) { margin=10, halign=Align.CENTER, valign=Align.CENTER, visible=true }; - private int row_at = 0; private bool video_added = false; - private PeerInfo? prev_peer_info = null; + private CallContentDetails audio_details = new CallContentDetails("Audio") { visible=true }; + private CallContentDetails video_details = new CallContentDetails("Video"); public CallConnectionDetailsWindow() { - grid.attach(new Label("Audio") { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); - put_row("RTP"); - grid.attach(audio_rtp_ready, 1, row_at++, 1, 1); - put_row("RTCP"); - grid.attach(audio_rtcp_ready, 1, row_at++, 1, 1); - put_row("Sent bp/s"); - grid.attach(audio_sent_bps, 1, row_at++, 1, 1); - put_row("Received bp/s"); - grid.attach(audio_recv_bps, 1, row_at++, 1, 1); - put_row("Codec"); - grid.attach(audio_codec, 1, row_at++, 1, 1); - put_row("Target receive bitrate"); - grid.attach(audio_target_receive_bitrate, 1, row_at++, 1, 1); - put_row("Target send bitrate"); - grid.attach(audio_target_send_bitrate, 1, row_at++, 1, 1); - - this.child = grid; - } - - private void put_row(string label) { - grid.attach(new Label(label) { xalign=0, visible=true }, 0, row_at, 1, 1); + box.add(audio_details); + box.add(video_details); + add(box); } public void update_content(PeerInfo peer_info) { - audio_rtp_ready.label = peer_info.audio_rtp_ready.to_string(); - audio_rtcp_ready.label = peer_info.audio_rtcp_ready.to_string(); - audio_codec.label = peer_info.audio_codec + " " + peer_info.audio_clockrate.to_string(); - audio_target_receive_bitrate.label = peer_info.audio_target_receive_bitrate.to_string(); - audio_target_send_bitrate.label = peer_info.audio_target_send_bitrate.to_string(); - - video_rtp_ready.label = peer_info.video_rtp_ready.to_string(); - video_rtcp_ready.label = peer_info.video_rtcp_ready.to_string(); - video_codec.label = peer_info.video_codec; - video_target_receive_bitrate.label = peer_info.video_target_receive_bitrate.to_string(); - video_target_send_bitrate.label = peer_info.video_target_send_bitrate.to_string(); - - if (peer_info.video_content_exists) add_video_widgets(); - - if (prev_peer_info != null) { - ulong audio_sent_kbps = (peer_info.audio_bytes_sent - prev_peer_info.audio_bytes_sent) * 8 / 1000; - audio_sent_bps.label = "%lu kbps".printf(audio_sent_kbps); - ulong audio_recv_kbps = (peer_info.audio_bytes_received - prev_peer_info.audio_bytes_received) * 8 / 1000; - audio_recv_bps.label = "%lu kbps".printf(audio_recv_kbps); - ulong video_sent_kbps = (peer_info.video_bytes_sent - prev_peer_info.video_bytes_sent) * 8 / 1000; - video_sent_bps.label = "%lu kbps".printf(video_sent_kbps); - ulong video_recv_kbps = (peer_info.video_bytes_received - prev_peer_info.video_bytes_received) * 8 / 1000; - video_recv_bps.label = "%lu kbps".printf(video_recv_kbps); - } - prev_peer_info = peer_info; + if (peer_info.audio != null) { + audio_details.update_content(peer_info.audio); + } + if (peer_info.video != null) { + add_video_widgets(); + video_details.update_content(peer_info.video); + } } private void add_video_widgets() { - if (video_added) return; - - grid.attach(new Label("Video") { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); - put_row("RTP"); - grid.attach(video_rtp_ready, 1, row_at++, 1, 1); - put_row("RTCP"); - grid.attach(video_rtcp_ready, 1, row_at++, 1, 1); - put_row("Sent bp/s"); - grid.attach(video_sent_bps, 1, row_at++, 1, 1); - put_row("Received bp/s"); - grid.attach(video_recv_bps, 1, row_at++, 1, 1); - put_row("Codec"); - grid.attach(video_codec, 1, row_at++, 1, 1); - put_row("Target receive bitrate"); - grid.attach(video_target_receive_bitrate, 1, row_at++, 1, 1); - put_row("Target send bitrate"); - grid.attach(video_target_send_bitrate, 1, row_at++, 1, 1); - - video_added = true; + if (video_added) return; + + video_details.visible = true; + video_added = true; + } + } + + public class CallContentDetails : Gtk.Grid { + + public Label rtp_title = new Label("RTP") { xalign=0, visible=true }; + public Label rtcp_title = new Label("RTCP") { xalign=0, visible=true }; + public Label target_recv_title = new Label("Target receive bitrate") { xalign=0, visible=true }; + public Label target_send_title = new Label("Target send bitrate") { xalign=0, visible=true }; + + public Label rtp_ready = new Label("?") { xalign=0, visible=true }; + public Label rtcp_ready = new Label("?") { xalign=0, visible=true }; + public Label sent_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label recv_bps = new Label("?") { use_markup=true, xalign=0, visible=true }; + public Label codec = new Label("?") { xalign=0, visible=true }; + public Label target_receive_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + public Label target_send_bitrate = new Label("n/a") { use_markup=true, xalign=0, visible=true }; + + private PeerContentInfo? prev_info = null; + private int row_at = 0; + + public CallContentDetails(string headline) { + attach(new Label("%s".printf(headline)) { use_markup=true, xalign=0, visible=true }, 0, row_at++, 1, 1); + attach(rtp_title, 0, row_at, 1, 1); + attach(rtp_ready, 1, row_at++, 1, 1); + attach(rtcp_title, 0, row_at, 1, 1); + attach(rtcp_ready, 1, row_at++, 1, 1); + put_row("Sent"); + attach(sent_bps, 1, row_at++, 1, 1); + put_row("Received"); + attach(recv_bps, 1, row_at++, 1, 1); + put_row("Codec"); + attach(codec, 1, row_at++, 1, 1); + attach(target_recv_title, 0, row_at, 1, 1); + attach(target_receive_bitrate, 1, row_at++, 1, 1); + attach(target_send_title, 0, row_at, 1, 1); + attach(target_send_bitrate, 1, row_at++, 1, 1); + + this.column_spacing = 5; + } + + public void update_content(PeerContentInfo info) { + if (!info.rtp_ready) { + rtp_ready.visible = rtcp_ready.visible = true; + rtp_title.visible = rtcp_title.visible = true; + rtp_ready.label = info.rtp_ready.to_string(); + rtcp_ready.label = info.rtcp_ready.to_string(); + } else { + rtp_ready.visible = rtcp_ready.visible = false; + rtp_title.visible = rtcp_title.visible = false; + } + if (info.target_send_bytes != -1) { + target_receive_bitrate.visible = target_send_bitrate.visible = true; + target_recv_title.visible = target_send_title.visible = true; + target_receive_bitrate.label = "%u kbps".printf(info.target_receive_bytes); + target_send_bitrate.label = "%u kbps".printf(info.target_send_bytes); + } else { + target_receive_bitrate.visible = target_send_bitrate.visible = false; + target_recv_title.visible = target_send_title.visible = false; + } + + codec.label = info.codec + " " + info.clockrate.to_string(); + + if (prev_info != null) { + ulong audio_sent_kbps = (info.bytes_sent - prev_info.bytes_sent) * 8 / 1000; + sent_bps.label = "%lu kbps".printf(audio_sent_kbps); + ulong audio_recv_kbps = (info.bytes_received - prev_info.bytes_received) * 8 / 1000; + recv_bps.label = "%lu kbps".printf(audio_recv_kbps); + } + prev_info = info; + } + + private void put_row(string label) { + attach(new Label(label) { xalign=0, visible=true }, 0, row_at, 1, 1); } } } diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index aad1e6d8..a7d37afd 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -144,7 +144,7 @@ namespace Dino.Ui { }); break; - case Call.State.OTHER_DEVICE_ACCEPTED: + case Call.State.OTHER_DEVICE: image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); title_label.label = call.direction == Call.DIRECTION_INCOMING ? _("Incoming call") : _("Outgoing call"); subtitle_label.label = _("You handled this call on another device"); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 5e5a556b..85864022 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -99,7 +99,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); } if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.get_data("no_buffer_probe_timeout"); + uint id = pad.steal_data("no_buffer_probe_timeout"); if (id != 0) { Source.remove(id); } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index 9022547d..c4c299c5 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -151,6 +151,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } this.stream = parent.create_stream(content); + this.stream.weak_ref(() => this.stream = null); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); this.stream.on_send_rtp_data.connect(rtp_datagram.send_datagram); diff --git a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala index 71e16a95..ac1d8329 100644 --- a/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala +++ b/xmpp-vala/src/module/xep/0353_jingle_message_initiation.vala @@ -8,7 +8,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { public signal void session_proposed(Jid from, Jid to, string sid, Gee.List descriptions); public signal void session_retracted(Jid from, Jid to, string sid); - public signal void session_accepted(Jid from, string sid); + public signal void session_accepted(Jid from, Jid to, string sid); public signal void session_rejected(Jid from, Jid to, string sid); public void send_session_propose_to_peer(XmppStream stream, Jid to, string sid, Gee.List descriptions) { @@ -65,7 +65,7 @@ namespace Xmpp.Xep.JingleMessageInitiation { switch (mi_node.name) { case "accept": case "proceed": - session_accepted(message.from, mi_node.get_attribute("id")); + session_accepted(message.from, message.to, mi_node.get_attribute("id")); break; case "propose": ArrayList descriptions = new ArrayList(); -- cgit v1.2.3-54-g00ecf From 52698a23d321089810a0e2a10d59c461a00c83e0 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 22:35:52 +0100 Subject: ICE: Run receiver in own thread --- plugins/ice/src/transport_parameters.vala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'plugins') diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index a91cc538..f42fad64 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -10,6 +10,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport private bool remote_credentials_set; private Map connections = new HashMap(); private DtlsSrtp.Handler? dtls_srtp_handler; + private MainContext thread_context; + private MainLoop thread_loop; private class DatagramConnection : Jingle.DatagramConnection { private Nice.Agent agent; @@ -85,6 +87,14 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent.controlling_mode = !incoming; stream_id = agent.add_stream(components); + thread_context = new MainContext(); + new Thread(@"ice-thread-$stream_id", () => { + thread_context.push_thread_default(); + thread_loop = new MainLoop(thread_context, false); + thread_loop.run(); + thread_context.pop_thread_default(); + return null; + }); if (turn_ip != null) { for (uint8 component_id = 1; component_id <= components; component_id++) { @@ -99,7 +109,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport for (uint8 component_id = 1; component_id <= components; component_id++) { // We don't properly get local candidates before this call - agent.attach_recv(stream_id, component_id, MainContext.@default(), on_recv); + agent.attach_recv(stream_id, component_id, thread_context, on_recv); } agent.gather_candidates(stream_id); @@ -337,5 +347,8 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport agent = null; dtls_srtp_handler = null; connections.clear(); + if (thread_loop != null) { + thread_loop.quit(); + } } } -- cgit v1.2.3-54-g00ecf From 09cd060889c27f51d43e1e2dc127c2a13595234d Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:39:19 +0100 Subject: ICE: Use non-blocking send --- plugins/ice/src/transport_parameters.vala | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index f42fad64..9aa3dda1 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -39,17 +39,23 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport public override void send_datagram(Bytes datagram) { if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { - uint8[] encrypted_data = null; - if (dtls_srtp_handler != null) { - try { - encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); + try { + if (dtls_srtp_handler != null) { + uint8[] encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); if (encrypted_data == null) return; - } catch (Crypto.Error e) { - warning("%s while send_datagram stream %u component %u", e.message, stream_id, component_id); + // TODO: Nonblocking might require certain libnice versions? + GLib.OutputVector[] vectors = {{ encrypted_data, encrypted_data.length }}; + Nice.OutputMessage[] messages = {{ vectors }}; + agent.send_messages_nonblocking(stream_id, component_id, messages); + } else { + GLib.OutputVector[] vectors = {{ datagram.get_data(), datagram.get_size() }}; + Nice.OutputMessage[] messages = {{ vectors }}; + agent.send_messages_nonblocking(stream_id, component_id, messages); } + bytes_sent += datagram.length; + } catch (GLib.Error e) { + warning("%s while send_datagram stream %u component %u", e.message, stream_id, component_id); } - agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); - bytes_sent += datagram.length; } } } -- cgit v1.2.3-54-g00ecf From 4f80a9f5ccea5eaf6a78a841af417c597eb7db73 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:43:12 +0100 Subject: RTP: Correctly handle timestamp after re-enabling a stream --- plugins/rtp/src/device.vala | 6 +++++- plugins/rtp/src/stream.vala | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 89d499ed..de46bea4 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -96,7 +96,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } - public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1) { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1, uint32 timestamp_offset = 0) { if (!is_source) return null; if (element == null) create(); links++; @@ -122,8 +122,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { var payload = (Gst.RTP.BasePayload) ((Gst.Bin) payloaders[payload_type][ssrc]).get_by_name(@"$(id)_$(codec)_$(ssrc)_rtp_pay"); payload.ssrc = ssrc; payload.seqnum_offset = seqnum_offset; + if (timestamp_offset != 0) { + payload.timestamp_offset = timestamp_offset; + } pipe.add(payloaders[payload_type][ssrc]); codec_tees[payload_type].link(payloaders[payload_type][ssrc]); + debug("Payload for %s with %s using ssrc %u, seqnum_offset %u, timestamp_offset %u", media, codec, ssrc, seqnum_offset, timestamp_offset); } if (!payloader_tees.has_key(payload_type)) { payloader_tees[payload_type] = new HashMap(); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 85864022..2cc40783 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -29,7 +29,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public Device input_device { get { return _input_device; } set { if (!paused) { var input = this.input; - set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset) : null); + set_input(value != null ? value.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null); if (this._input_device != null) this._input_device.unlink(input); } this._input_device = value; @@ -46,6 +46,13 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private bool push_recv_data = false; private uint our_ssrc = Random.next_int(); private int next_seqnum_offset = -1; + private uint32 next_timestamp_offset_base = 0; + private int64 next_timestamp_offset_stamp = 0; + private uint32 next_timestamp_offset { get { + if (next_timestamp_offset_base == 0) return 0; + int64 monotonic_diff = get_monotonic_time() - next_timestamp_offset_stamp; + return next_timestamp_offset_base + (uint32)((double)monotonic_diff / 1000000.0 * payload_type.clockrate); + } } private uint32 participant_ssrc = 0; private Gst.Pad recv_rtcp_sink_pad; @@ -657,7 +664,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public void unpause() { if (!paused) return; - set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset) : null, false); + set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); } public uint get_participant_ssrc(Xmpp.Jid participant) { -- cgit v1.2.3-54-g00ecf From 9aeff4bf9e00624187931c0d358c166f5a6860a1 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:45:36 +0100 Subject: SRTP: Do not continue processing data after encrypt/decrypt failed RTP: Copy less --- plugins/ice/src/transport_parameters.vala | 1 + plugins/rtp/CMakeLists.txt | 4 + plugins/rtp/src/stream.vala | 119 ++++++++++++++++++++---------- 3 files changed, 87 insertions(+), 37 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala index 9aa3dda1..f684e411 100644 --- a/plugins/ice/src/transport_parameters.vala +++ b/plugins/ice/src/transport_parameters.vala @@ -273,6 +273,7 @@ public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransport if (decrypt_data == null) return; } catch (Crypto.Error e) { warning("%s while on_recv stream %u component %u", e.message, stream_id, component_id); + return; } } may_consider_ready(stream_id, component_id); diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 3264e24a..fa4f367c 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -16,6 +16,10 @@ if(GstRtp_VERSION VERSION_GREATER "1.16") set(RTP_DEFINITIONS GST_1_16) endif() +if(Vala_VERSION VERSION_GREATER "0.50") + set(RTP_DEFINITIONS VALA_0_50) +endif() + if(WebRTCAudioProcessing_VERSION GREATER "0.4") message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") unset(WebRTCAudioProcessing_FOUND) diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 2cc40783..2c8b6356 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -343,36 +343,57 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } Gst.Sample sample = sink.pull_sample(); Gst.Buffer buffer = sample.get_buffer(); - uint8[] data; - buffer.extract_dup(0, buffer.get_size(), out data); - prepare_local_crypto(); if (sink == send_rtp) { + uint buffer_ssrc = 0, buffer_seq = 0; Gst.RTP.Buffer rtp_buffer; if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { - if (our_ssrc != rtp_buffer.get_ssrc()) { - warning("Sending buffer with SSRC %u when our ssrc is %u", rtp_buffer.get_ssrc(), our_ssrc); - } + buffer_ssrc = rtp_buffer.get_ssrc(); + buffer_seq = rtp_buffer.get_seq(); next_seqnum_offset = rtp_buffer.get_seq() + 1; + next_timestamp_offset_base = rtp_buffer.get_timestamp(); + next_timestamp_offset_stamp = get_monotonic_time(); rtp_buffer.unmap(); } - if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtp(data); + if (our_ssrc != buffer_ssrc) { + warning("Sending RTP %s buffer seq %u with SSRC %u when our ssrc is %u", media, buffer_seq, buffer_ssrc, our_ssrc); + } else { + debug("Sending RTP %s buffer seq %u with SSRC %u", media, buffer_seq, buffer_ssrc); } - on_send_rtp_data(new Bytes.take((owned) data)); + } + + prepare_local_crypto(); + + uint8[] data; + buffer.extract_dup(0, buffer.get_size(), out data); + if (sink == send_rtp) { + encrypt_and_send_rtp((owned) data); } else if (sink == send_rtcp) { encrypt_and_send_rtcp((owned) data); } return Gst.FlowReturn.OK; } + private void encrypt_and_send_rtp(owned uint8[] data) { + Bytes bytes; + if (crypto_session.has_encrypt) { + bytes = new Bytes.take(crypto_session.encrypt_rtp(data)); + } else { + bytes = new Bytes.take(data); + } + on_send_rtp_data(bytes); + } + private void encrypt_and_send_rtcp(owned uint8[] data) { + Bytes bytes; if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtcp(data); + bytes = new Bytes.take(crypto_session.encrypt_rtcp(data)); + } else { + bytes = new Bytes.take(data); } if (rtcp_mux) { - on_send_rtp_data(new Bytes.take((owned) data)); + on_send_rtp_data(bytes); } else { - on_send_rtcp_data(new Bytes.take((owned) data)); + on_send_rtcp_data(bytes); } } @@ -514,17 +535,38 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { on_recv_rtcp_data(bytes); return; } - prepare_remote_crypto(); - uint8[] data = bytes.get_data(); - if (crypto_session.has_decrypt) { - try { - data = crypto_session.decrypt_rtp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); +#if GST_1_16 + { + Gst.Buffer buffer = new Gst.Buffer.wrapped_bytes(bytes); + Gst.RTP.Buffer rtp_buffer; + uint buffer_ssrc = 0, buffer_seq = 0; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + buffer_ssrc = rtp_buffer.get_ssrc(); + buffer_seq = rtp_buffer.get_seq(); + rtp_buffer.unmap(); } + debug("Received RTP %s buffer seq %u with SSRC %u", media, buffer_seq, buffer_ssrc); } +#endif if (push_recv_data) { - Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { +#if GST_1_16 + buffer = new Gst.Buffer.wrapped_bytes(bytes); +#else + buffer = new Gst.Buffer.wrapped(bytes.get_data()); +#endif + } + Gst.RTP.Buffer rtp_buffer; if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { if (rtp_buffer.get_extension()) { @@ -552,11 +594,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { rtp_buffer.unmap(); } - // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() - // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of - // the underlying buffer, so and some point we should move over to the new version (once we require - // Vala >= 0.50) -#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI +#if VALA_0_50 recv_rtp.push_buffer((owned) buffer); #else Gst.FlowReturn ret; @@ -566,19 +604,26 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } public override void on_recv_rtcp_data(Bytes bytes) { - prepare_remote_crypto(); - uint8[] data = bytes.get_data(); - if (crypto_session.has_decrypt) { - try { - data = crypto_session.decrypt_rtcp(data); - } catch (Error e) { - warning("%s (%d)", e.message, e.code); - } - } if (push_recv_data) { - Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); - // See above -#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + prepare_remote_crypto(); + + Gst.Buffer buffer; + if (crypto_session.has_decrypt) { + try { + buffer = new Gst.Buffer.wrapped(crypto_session.decrypt_rtcp(bytes.get_data())); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return; + } + } else { +#if GST_1_16 + buffer = new Gst.Buffer.wrapped_bytes(bytes); +#else + buffer = new Gst.Buffer.wrapped(bytes.get_data()); +#endif + } + +#if VALA_0_50 recv_rtcp.push_buffer((owned) buffer); #else Gst.FlowReturn ret; -- cgit v1.2.3-54-g00ecf From b07c4187ef40295c17cc09fea6dc5ba4137fd73b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 18 Dec 2021 21:47:50 +0100 Subject: RTP: Less log spam --- plugins/rtp/src/device.vala | 40 ---------------------------------------- plugins/rtp/src/plugin.vala | 8 -------- plugins/rtp/src/stream.vala | 40 ---------------------------------------- 3 files changed, 88 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index de46bea4..368356ae 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -384,45 +384,6 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return target; } - private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { - if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); - } - if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { - var event = info.get_event(); - if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; - if (event.type == Gst.EventType.QOS) { - Gst.QOSType qos_type; - double proportion; - Gst.ClockTimeDiff diff; - Gst.ClockTime timestamp; - event.parse_qos(out qos_type, out proportion, out diff, out timestamp); - debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); - } else { - debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); - } - } - if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { - debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.get_data("no_buffer_probe_timeout"); - if (id != 0) { - Source.remove(id); - } - string name = @"$(pad.get_parent_element().name).$(pad.name)"; - id = Timeout.add_seconds(1, () => { - debug("%s probed no buffer for 1 second", name); - return Source.REMOVE; - }); - pad.set_data("no_buffer_probe_timeout", id); - } - return Gst.PadProbeReturn.PASS; - } - private void create() { debug("Creating device %s", id); plugin.pause(); @@ -437,7 +398,6 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("do-timestamp", true); filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", device_caps); - filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(filter); element.link(filter); #if WITH_VOICE_PROCESSOR diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d10303a6..6d6da79a 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -155,14 +155,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { break; case Gst.MessageType.STATE_CHANGED: // Ignore - { - unowned Gst.Structure struc = message.get_structure(); - if (struc != null && message.src is Gst.Element) { - Gst.State oldState, newState, pendingState; - message.parse_state_changed(out oldState, out newState, out pendingState); - debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState"); - } - } break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 2c8b6356..385a590c 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -81,45 +81,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } - private static Gst.PadProbeReturn log_probe(Gst.Pad pad, Gst.PadProbeInfo info) { - if ((info.type & Gst.PadProbeType.EVENT_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream event %s", pad.get_parent_element().name, pad.name, info.get_event().type.get_name()); - } - if ((info.type & Gst.PadProbeType.EVENT_UPSTREAM) > 0) { - var event = info.get_event(); - if (event.type == Gst.EventType.RECONFIGURE) return Gst.PadProbeReturn.DROP; - if (event.type == Gst.EventType.QOS) { - Gst.QOSType qos_type; - double proportion; - Gst.ClockTimeDiff diff; - Gst.ClockTime timestamp; - event.parse_qos(out qos_type, out proportion, out diff, out timestamp); - debug("%s.%s probed qos event: type: %s, proportion: %f, diff: %lli, timestamp: %llu", pad.get_parent_element().name, pad.name, @"$qos_type", proportion, diff, timestamp); - } else { - debug("%s.%s probed upstream event %s", pad.get_parent_element().name, pad.name, event.type.get_name()); - } - } - if ((info.type & Gst.PadProbeType.QUERY_DOWNSTREAM) > 0) { - debug("%s.%s probed downstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.QUERY_UPSTREAM) > 0) { - debug("%s.%s probed upstream query %s", pad.get_parent_element().name, pad.name, info.get_query().type.get_name()); - } - if ((info.type & Gst.PadProbeType.BUFFER) > 0) { - uint id = pad.steal_data("no_buffer_probe_timeout"); - if (id != 0) { - Source.remove(id); - } - string name = @"$(pad.get_parent_element().name).$(pad.name)"; - id = Timeout.add_seconds(1, () => { - debug("%s probed no buffer for 1 second", name); - return Source.REMOVE; - }); - pad.set_data("no_buffer_probe_timeout", id); - } - return Gst.PadProbeReturn.PASS; - } - public override void create() { plugin.pause(); @@ -160,7 +121,6 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp.do_timestamp = true; recv_rtp.format = Gst.Format.TIME; recv_rtp.is_live = true; - recv_rtp.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe); pipe.add(recv_rtp); recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; -- cgit v1.2.3-54-g00ecf From b1c1751cc42948ae76a363691fad575348207396 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 19 Dec 2021 22:36:26 +0100 Subject: DTLS: Use own thread for connection establishment --- plugins/ice/src/dtls_srtp.vala | 52 ++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 22 deletions(-) (limited to 'plugins') diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index 91c11eee..c5b2eb75 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -112,6 +112,19 @@ public class Handler { } public async Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection() { + MainContext context = MainContext.current_source().get_context(); + var thread = new Thread("dtls-connection", () => { + var res = setup_dtls_connection_thread(); + Source source = new IdleSource(); + source.set_callback(setup_dtls_connection.callback); + source.attach(context); + return res; + }); + yield; + return thread.join(); + } + + private Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection_thread() { buffer_mutex.lock(); if (stop) { restart = true; @@ -146,28 +159,23 @@ public class Handler { session.set_push_function(push_function); session.set_verify_function(verify_function); - Thread thread = new Thread (null, () => { - DateTime maximum_time = new DateTime.now_utc().add_seconds(20); - do { - err = session.handshake(); + DateTime maximum_time = new DateTime.now_utc().add_seconds(20); + do { + err = session.handshake(); + + DateTime current_time = new DateTime.now_utc(); + if (maximum_time.compare(current_time) < 0) { + warning("DTLS handshake timeouted"); + err = ErrorCode.APPLICATION_ERROR_MIN + 1; + break; + } + if (stop) { + debug("DTLS handshake stopped"); + err = ErrorCode.APPLICATION_ERROR_MIN + 2; + break; + } + } while (err < 0 && !((ErrorCode)err).is_fatal()); - DateTime current_time = new DateTime.now_utc(); - if (maximum_time.compare(current_time) < 0) { - warning("DTLS handshake timeouted"); - err = ErrorCode.APPLICATION_ERROR_MIN + 1; - break; - } - if (stop) { - debug("DTLS handshake stopped"); - err = ErrorCode.APPLICATION_ERROR_MIN + 2; - break; - } - } while (err < 0 && !((ErrorCode)err).is_fatal()); - Idle.add(setup_dtls_connection.callback); - return err; - }); - yield; - err = thread.join(); buffer_mutex.lock(); if (stop) { stop = false; @@ -176,7 +184,7 @@ public class Handler { buffer_mutex.unlock(); if (restart) { debug("Restarting DTLS handshake"); - return yield setup_dtls_connection(); + return setup_dtls_connection_thread(); } return null; } -- cgit v1.2.3-54-g00ecf From cd6d501c23c8550e7b2d880f8e60e3df8e887c2a Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 19 Dec 2021 22:38:00 +0100 Subject: RTP: Improve screen resolution update logic --- plugins/rtp/src/device.vala | 46 +++++++++++++++++++++++---------------------- plugins/rtp/src/stream.vala | 9 ++++++--- 2 files changed, 30 insertions(+), 25 deletions(-) (limited to 'plugins') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 368356ae..97258d0c 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -2,24 +2,19 @@ using Xmpp.Xep.JingleRtp; using Gee; public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + private const int[] common_widths = {320, 360, 400, 480, 640, 960, 1280, 1920, 2560, 3840}; + public Plugin plugin { get; private set; } public CodecUtil codec_util { get { return plugin.codec_util; } } public Gst.Device device { get; private set; } - private string device_name; - public string id { get { - return device_name; - }} - private string device_display_name; - public string display_name { get { - return device_display_name; - }} + public string id { get { return device_name; }} + public string display_name { get { return device_display_name; }} public string detail_name { get { return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; }} - public Gst.Pipeline pipe { get { - return plugin.pipe; - }} + + public Gst.Pipeline pipe { get { return plugin.pipe; }} public string? media { get { if (device.has_classes("Audio")) { return "audio"; @@ -29,12 +24,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return null; } }} - public bool is_source { get { - return device.has_classes("Source"); - }} - public bool is_sink { get { - return device.has_classes("Sink"); - }} + public bool is_source { get { return device.has_classes("Source"); }} + public bool is_sink { get { return device.has_classes("Sink"); }} + + private string device_name; + private string device_display_name; private Gst.Caps device_caps; private Gst.Element element; @@ -96,11 +90,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return element; } - public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = Random.next_int(), int seqnum_offset = -1, uint32 timestamp_offset = 0) { + public Gst.Element? link_source(PayloadType? payload_type = null, uint ssrc = 0, int seqnum_offset = -1, uint32 timestamp_offset = 0) { if (!is_source) return null; if (element == null) create(); links++; - if (payload_type != null && tee != null) { + if (payload_type != null && ssrc != 0 && tee != null) { bool new_codec = false; string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (!codecs.has_key(payload_type)) { @@ -172,16 +166,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return br; } - private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840}; private Gst.Caps get_active_caps(PayloadType payload_type) { return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps; } + private void apply_caps(PayloadType payload_type, Gst.Caps caps) { plugin.pause(); debug("Set scaled caps to %s", caps.to_string()); codec_util.update_rescale_caps(codecs[payload_type], caps); plugin.unpause(); } + private void apply_width(PayloadType payload_type, int new_width, uint bitrate) { int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den; device_caps.get_structure(0).get_int("width", out device_caps_width); @@ -197,9 +192,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { int new_height = device_caps_height * new_width / device_caps_width; Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null); double required_bitrate = get_target_bitrate(new_caps); - if (bitrate < required_bitrate) return; + debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate); + if (bitrate < required_bitrate && new_width > active_caps_width) return; apply_caps(payload_type, new_caps); } + public void update_bitrate(PayloadType payload_type, uint bitrate) { if (codecs.has_key(payload_type)) { lock(codec_bitrates); @@ -233,12 +230,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) { // Lower video resolution int i = 1; - for(; i < common_widths.length && common_widths[i] < active_caps_width; i++); + for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);if (common_widths[i] != active_caps_width) { + debug("Decrease resolution to ensure target bitrate (%u) is in reach (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } apply_width(payload_type, common_widths[i-1], bitrate); } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) { // Higher video resolution int i = 0; for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++); + if (common_widths[i] != active_caps_width) { + debug("Increase resolution to make use of available bandwidth of target bitrate (%u) (current resolution target bitrate is %f)", bitrate, current_target_bitrate); + } if (common_widths[i] > device_caps_width) { // We never scale up, so just stick with what the device gives apply_width(payload_type, device_caps_width, bitrate); diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 385a590c..dc712b61 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -169,7 +169,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { Timeout.add(1000, () => remb_adjust()); } if (input_device != null && media == "video") { - input_device.update_bitrate(payload_type, 256); + input_device.update_bitrate(payload_type, target_send_bitrate); } } @@ -214,16 +214,18 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (packets_received < last_packets_received) new_received = 0; uint64 new_octets = octets_received - last_octets_received; if (octets_received < last_octets_received) octets_received = 0; + if (new_received == 0) continue; last_packets_lost = packets_lost; last_packets_received = packets_received; last_octets_received = octets_received; - if (new_received == 0) continue; double loss_rate = (double)new_lost / (double)(new_lost + new_received); - uint new_target_receive_bitrate = 256; + uint new_target_receive_bitrate; if (new_lost <= 0 || loss_rate < 0.02) { new_target_receive_bitrate = (uint)(1.08 * (double)target_receive_bitrate); } else if (loss_rate > 0.1) { new_target_receive_bitrate = (uint)((1.0 - 0.5 * loss_rate) * (double)target_receive_bitrate); + } else { + new_target_receive_bitrate = target_receive_bitrate; } if (last_remb_time == 0) { last_remb_time = get_monotonic_time(); @@ -670,6 +672,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public void unpause() { if (!paused) return; set_input_and_pause(input_device != null ? input_device.link_source(payload_type, our_ssrc, next_seqnum_offset, next_timestamp_offset) : null, false); + input_device.update_bitrate(payload_type, target_send_bitrate); } public uint get_participant_ssrc(Xmpp.Jid participant) { -- cgit v1.2.3-54-g00ecf From 8e99ed1e9c19beef2536880c7d46505ec18e44c1 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 19 Dec 2021 22:38:27 +0100 Subject: RTP: Disable VP9 --- plugins/rtp/src/codec_util.vala | 1 + 1 file changed, 1 insertion(+) (limited to 'plugins') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index f9f7f7ec..6fb5e7aa 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -264,6 +264,7 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_decode_element_name(string media, string? codec) { + if (codec == "vp9") return null; // Temporary unsupport VP9 if (get_depay_element_name(media, codec) == null) return null; foreach (string candidate in get_decode_candidates(media, codec)) { if (is_element_supported(candidate)) return candidate; -- cgit v1.2.3-54-g00ecf