From ef2e3c774cab82a94a5e34399f2013d64c3cf03b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 12:41:38 +0100 Subject: Add RTP implementation as plugin --- plugins/rtp/src/codec_util.vala | 245 ++++++++++++++++++++ plugins/rtp/src/device.vala | 206 +++++++++++++++++ plugins/rtp/src/module.vala | 264 +++++++++++++++++++++ plugins/rtp/src/participant.vala | 39 ++++ plugins/rtp/src/plugin.vala | 413 +++++++++++++++++++++++++++++++++ plugins/rtp/src/register_plugin.vala | 3 + plugins/rtp/src/stream.vala | 432 +++++++++++++++++++++++++++++++++++ plugins/rtp/src/video_widget.vala | 110 +++++++++ 8 files changed, 1712 insertions(+) create mode 100644 plugins/rtp/src/codec_util.vala create mode 100644 plugins/rtp/src/device.vala create mode 100644 plugins/rtp/src/module.vala create mode 100644 plugins/rtp/src/participant.vala create mode 100644 plugins/rtp/src/plugin.vala create mode 100644 plugins/rtp/src/register_plugin.vala create mode 100644 plugins/rtp/src/stream.vala create mode 100644 plugins/rtp/src/video_widget.vala (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala new file mode 100644 index 00000000..e419b5ad --- /dev/null +++ b/plugins/rtp/src/codec_util.vala @@ -0,0 +1,245 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.CodecUtil { + private Set supported_elements = new HashSet(); + private Set unsupported_elements = new HashSet(); + + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type) { + Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", + "media", typeof(string), media, + "payload", typeof(int), payload_type.id); + //"channels", typeof(int), payloadType.channels, + //"max-ptime", typeof(int), payloadType.maxptime); + unowned Gst.Structure s = caps.get_structure(0); + if (payload_type.clockrate != 0) { + s.set("clock-rate", typeof(int), payload_type.clockrate); + } + if (payload_type.name != null) { + s.set("encoding-name", typeof(string), payload_type.name.up()); + } + return caps; + } + + public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) { + if (payload_type.name != null) return payload_type.name.down(); + if (media == "audio") { + switch (payload_type.id) { + case 0: + return "pcmu"; + case 8: + return "pcma"; + } + } + return null; + } + + public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_media_type(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_media_type(string media, string? codec) { + if (codec == null) return null; + if (media == "audio") { + switch (codec) { + case "pcma": + return "audio/x-alaw"; + case "pcmu": + return "audio/x-mulaw"; + } + } + return @"$media/x-$codec"; + } + + public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_pay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_pay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)pay"; + } + + public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_depay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_depay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)depay"; + } + + public static string[] get_encode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusenc"}; + case "speex": + return new string[] {"speexenc"}; + case "pcma": + return new string[] {"alawenc"}; + case "pcmu": + return new string[] {"mulawenc"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264enc", */"vaapih264enc", "x264enc"}; + case "vp9": + return new string[] {/*"msdkvp9enc", */"vaapivp9enc" /*, "vp9enc" */}; + case "vp8": + return new string[] {/*"msdkvp8enc", */"vaapivp8enc", "vp8enc"}; + } + } + return new string[0]; + } + + public static string[] get_decode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusdec"}; + case "speex": + return new string[] {"speexdec"}; + case "pcma": + return new string[] {"alawdec"}; + case "pcmu": + return new string[] {"mulawdec"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264dec", */"vaapih264dec"}; + case "vp9": + return new string[] {/*"msdkvp9dec", */"vaapivp9dec", "vp9dec"}; + case "vp8": + return new string[] {/*"msdkvp8dec", */"vaapivp8dec", "vp8dec"}; + } + } + return new string[0]; + } + + public static string? get_encode_prefix(string media, string codec, string encode) { + if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; + if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; + return null; + } + + public static string? get_encode_suffix(string media, string codec, string encode) { + // H264 + const string h264_suffix = " ! video/x-h264,profile=constrained-baseline ! h264parse"; + if (encode == "msdkh264enc") return @" bitrate=256 rate-control=vbr target-usage=7$h264_suffix"; + if (encode == "vaapih264enc") return @" bitrate=256 quality-level=7 tune=low-power$h264_suffix"; + if (encode == "x264enc") return @" byte-stream=1 bitrate=256 profile=baseline speed-preset=ultrafast tune=zerolatency$h264_suffix"; + if (media == "video" && codec == "h264") return h264_suffix; + + // VP8 + if (encode == "msdkvp8enc") return " bitrate=256 rate-control=vbr target-usage=7"; + if (encode == "vaapivp8enc") return " bitrate=256 rate-control=vbr quality-level=7"; + if (encode == "vp8enc") return " target-bitrate=256000 deadline=1 error-resilient=1"; + + // OPUS + if (encode == "opusenc") return " audio-type=voice"; + + return null; + } + + public static string? get_decode_prefix(string media, string codec, string decode) { + return null; + } + + public bool is_element_supported(string element_name) { + if (unsupported_elements.contains(element_name)) return false; + if (supported_elements.contains(element_name)) return true; + var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name"); + if (test_element != null) { + supported_elements.add(element_name); + return true; + } else { + debug("%s is not supported on this platform", element_name); + unsupported_elements.add(element_name); + return false; + } + } + + public string? get_encode_element_name(string media, string? codec) { + foreach (string candidate in get_encode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_pay_element_name(string media, string? codec) { + string candidate = get_pay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public string? get_decode_element_name(string media, string? codec) { + foreach (string candidate in get_decode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_depay_element_name(string media, string? codec) { + string candidate = get_depay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public void mark_element_unsupported(string element_name) { + unsupported_elements.add(element_name); + } + + public string? get_decode_bin_description(string media, string? codec, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string depay = get_depay_element_name(media, codec); + string decode = element_name ?? get_decode_element_name(media, codec); + if (depay == null || decode == null) return null; + string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; + return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert"; + } + + public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? desc = get_decode_bin_description(media, codec, null, base_name); + if (desc == null) return null; + debug("Pipeline to decode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public string? get_encode_bin_description(string media, string? codec, string? element_name = null, uint pt = 96, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string pay = get_pay_element_name(media, codec); + string encode = element_name ?? get_encode_element_name(media, codec); + if (pay == null || encode == null) return null; + string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; + if (media == "audio") { + return @"audioconvert name=$base_name-convert ! audioresample name=$base_name-resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + } else { + return @"$(media)convert name=$base_name-convert ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + } + } + + public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, null, payload_type.id, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +} \ No newline at end of file diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala new file mode 100644 index 00000000..796be203 --- /dev/null +++ b/plugins/rtp/src/device.vala @@ -0,0 +1,206 @@ +public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + public Plugin plugin { get; private set; } + public Gst.Device device { get; private set; } + + private string device_name; + public string id { get { + return device_name; + }} + private string device_display_name; + public string display_name { get { + return device_display_name; + }} + public string detail_name { get { + return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; + }} + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public string? media { get { + if (device.device_class.has_prefix("Audio/")) { + return "audio"; + } else if (device.device_class.has_prefix("Video/")) { + return "video"; + } else { + return null; + } + }} + public bool is_source { get { + return device.device_class.has_suffix("/Source"); + }} + public bool is_sink { get { + return device.device_class.has_suffix("/Sink"); + }} + + private Gst.Element element; + private Gst.Element tee; + private Gst.Element dsp; + private Gst.Element mixer; + private Gst.Element filter; + private int links = 0; + + public Device(Plugin plugin, Gst.Device device) { + this.plugin = plugin; + update(device); + } + + public bool matches(Gst.Device device) { + if (this.device.name == device.name) return true; + return false; + } + + public void update(Gst.Device device) { + this.device = device; + this.device_name = device.name; + this.device_display_name = device.display_name; + } + + public Gst.Element? link_sink() { + if (element == null) create(); + links++; + if (mixer != null) return mixer; + if (is_sink && media == "audio") return plugin.echoprobe; + return element; + } + + public Gst.Element? link_source() { + if (element == null) create(); + links++; + if (tee != null) return tee; + return element; + } + + public void unlink() { + if (links <= 0) { + critical("Link count below zero."); + return; + } + links--; + if (links == 0) { + destroy(); + } + } + + private Gst.Caps get_best_caps() { + if (media == "audio") { + return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); + } else if (media == "video" && device.caps.get_size() > 0) { + int best_index = 0; + int best_fps = 0; + int best_width = 0; + int best_height = 0; + for (int i = 0; i < device.caps.get_size(); i++) { + unowned Gst.Structure? that = device.caps.get_structure(i); + if (!that.has_name("video/x-raw")) continue; + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("framerate") || !that.get_fraction("framerate", out num, out den)) continue; + if (!that.has_field("width") || !that.get_int("width", out width)) continue; + if (!that.has_field("height") || !that.get_int("height", out height)) continue; + int fps = num/den; + if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) { + best_fps = fps; + best_width = width; + best_height = height; + best_index = i; + } + } + return device.caps.copy_nth(best_index); + } else if (device.caps.get_size() > 0) { + return device.caps.copy_nth(0); + } else { + return new Gst.Caps.any(); + } + } + + private void create() { + debug("Creating device %s", id); + plugin.pause(); + element = device.create_element(id); + pipe.add(element); + if (is_source) { + filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + element.link(filter); + if (media == "audio") { + dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); + dsp.@set("probe", plugin.echoprobe.name); + pipe.add(dsp); + filter.link(dsp); + } + tee = Gst.ElementFactory.make("tee", @"$id-tee"); + tee.@set("allow-not-linked", true); + pipe.add(tee); + (dsp ?? filter).link(tee); + } + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } + if (is_sink && media == "audio") { +// mixer = Gst.ElementFactory.make("audiomixer", @"$id-mixer"); +// pipe.add(mixer); +// mixer.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } + plugin.unpause(); + } + + private void destroy() { + if (mixer != null) { + if (is_sink && media == "audio") { + plugin.echoprobe.unlink(mixer); + } + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + mixer.unlink(element); + pipe.remove(mixer); + mixer = null; + } else if (is_sink && media == "audio") { + plugin.echoprobe.unlink(element); + } + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + if (filter != null) element.unlink(filter); + else if (is_source) element.unlink(tee); + pipe.remove(element); + element = null; + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; + } + debug("Destroyed device %s", id); + } +} \ No newline at end of file diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala new file mode 100644 index 00000000..577e9f53 --- /dev/null +++ b/plugins/rtp/src/module.vala @@ -0,0 +1,264 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Module : JingleRtp.Module { + private Set supported_codecs = new HashSet(); + private Set unsupported_codecs = new HashSet(); + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + + public Module(Plugin plugin) { + base(); + this.plugin = plugin; + } + + private async bool pipeline_works(string media, string element_desc) { + var supported = false; + string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output"; + try { + var pipeline = Gst.parse_launch(pipeline_desc); + var output = (pipeline as Gst.Bin).get_by_name("output") as Gst.App.Sink; + SourceFunc callback = pipeline_works.callback; + var finished = false; + output.emit_signals = true; + output.new_sample.connect(() => { + if (!finished) { + finished = true; + supported = true; + Idle.add(() => { + callback(); + return Source.REMOVE; + }); + } + return Gst.FlowReturn.EOS; + }); + pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => { + if (message.type == Gst.MessageType.ERROR && !finished) { + Error e; + string d; + message.parse_error(out e, out d); + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + debug(d); + finished = true; + callback(); + } + return true; + }); + Timeout.add(2000, () => { + if (!finished) { + finished = true; + callback(); + } + return Source.REMOVE; + }); + pipeline.set_state(Gst.State.PLAYING); + yield; + pipeline.set_state(Gst.State.NULL); + } catch (Error e) { + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + } + return supported; + } + + private async bool supports(string media, JingleRtp.PayloadType payload_type) { + string codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (codec == null) return false; + if (unsupported_codecs.contains(codec)) return false; + if (supported_codecs.contains(codec)) return true; + + string encode_element = codec_util.get_encode_element_name(media, codec); + string decode_element = codec_util.get_decode_element_name(media, codec); + if (encode_element == null || decode_element == null) { + debug("No suitable encoder or decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + + string encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + while (!(yield pipeline_works(media, encode_bin))) { + debug("%s not suited for encoding %s", encode_element, codec); + codec_util.mark_element_unsupported(encode_element); + encode_element = codec_util.get_encode_element_name(media, codec); + if (encode_element == null) { + debug("No suitable encoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + } + debug("using %s to encode %s", encode_element, codec); + + string decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { + debug("%s not suited for decoding %s", decode_element, codec); + codec_util.mark_element_unsupported(decode_element); + decode_element = codec_util.get_decode_element_name(media, codec); + if (decode_element == null) { + debug("No suitable decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + } + debug("using %s to decode %s", decode_element, codec); + + supported_codecs.add(codec); + return true; + } + + public async void add_if_supported(Gee.List list, string media, JingleRtp.PayloadType payload_type) { + if (yield supports(media, payload_type)) { + list.add(payload_type); + } + } + + public override async Gee.List get_supported_payloads(string media) { + Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); + if (media == "audio") { + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 2, + clockrate = 48000, + name = "opus", + id = 96 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 32000, + name = "speex", + id = 97 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 16000, + name = "speex", + id = 98 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "speex", + id = 99 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "PCMU", + id = 0 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + channels = 1, + clockrate = 8000, + name = "PCMA", + id = 8 + }); + } else if (media == "video") { + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "H264", + id = 96 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "VP9", + id = 97 + }); + yield add_if_supported(list, media, new JingleRtp.PayloadType() { + clockrate = 90000, + name = "VP8", + id = 98 + }); + } else { + warning("Unsupported media type: %s", media); + } + return list; + } + + public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List payloads) { + if (media == "audio") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield supports(media, type)) return type; + } + } else if (media == "video") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield supports(media, type)) return type; + } + } else { + warning("Unsupported media type: %s", media); + } + return null; + } + + public override JingleRtp.Stream create_stream(Jingle.Content content) { + return plugin.open_stream(content); + } + + public override void close_stream(JingleRtp.Stream stream) { + var rtp_stream = stream as Rtp.Stream; + plugin.close_stream(rtp_stream); + } + +// public uint32 get_session_id(string id) { +// return (uint32) id.split("-")[0].to_int(); +// } +// +// public string create_feed(string media, bool incoming) { +// init(); +// string id = random_uuid(); +// if (media == "audio") { +// id = "0-" + id; +// } else { +// id = "1-" + id; +// } +// MediaDevice? device = plugin.get_preferred_device(media, incoming); +// Feed feed; +// if (incoming) { +// if (media == "audio") { +// feed = new IncomingAudioFeed(id, this, device); +// } else if (media == "video") { +// feed = new IncomingVideoFeed(id, this, device); +// } else { +// critical("Incoming feed of media '%s' not supported", media); +// return id; +// } +// } else { +// if (media == "audio") { +// string? matching_incoming_feed_id = null; +// foreach (Feed match in plugin.feeds.values) { +// if (match is IncomingAudioFeed) { +// matching_incoming_feed_id = match.id; +// } +// } +// feed = new OutgoingAudioFeed(id, this, device); +// } else if (media == "video") { +// feed = new OutgoingVideoFeed(id, this, device); +// } else { +// critical("Outgoing feed of media '%s' not supported", media); +// return id; +// } +// } +// plugin.add_feed(id, feed); +// return id; +// } +// +// public void connect_feed(string id, JingleRtp.PayloadType payload, Jingle.DatagramConnection connection) { +// if (!plugin.feeds.has_key(id)) { +// critical("Tried to connect feed with id %s, but no such feed found", id); +// return; +// } +// Feed feed = plugin.feeds[id]; +// feed.connect(payload, connection); +// } +// +// public void destroy_feed(string id) { +// if (!plugin.feeds.has_key(id)) { +// critical("Tried to destroy feed with id %s, but no such feed found", id); +// return; +// } +// Feed feed = plugin.feeds[id]; +// feed.destroy(); +// plugin.feeds.remove(id); +// } +} \ No newline at end of file diff --git a/plugins/rtp/src/participant.vala b/plugins/rtp/src/participant.vala new file mode 100644 index 00000000..1ca13191 --- /dev/null +++ b/plugins/rtp/src/participant.vala @@ -0,0 +1,39 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Participant { + public Jid full_jid { get; private set; } + + protected Gst.Pipeline pipe; + private Map ssrcs = new HashMap(); + + public Participant(Gst.Pipeline pipe, Jid full_jid) { + this.pipe = pipe; + this.full_jid = full_jid; + } + + public uint32 get_ssrc(Stream stream) { + if (ssrcs.has_key(stream)) { + return ssrcs[stream]; + } + return 0; + } + + public void set_ssrc(Stream stream, uint32 ssrc) { + if (ssrcs.has_key(stream)) { + warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]); + } else { + stream.on_destroy.connect(unset_ssrc); + } + ssrcs[stream] = ssrc; + } + + public void unset_ssrc(Stream stream) { + ssrcs.unset(stream); + stream.on_destroy.disconnect(unset_ssrc); + } + + public string to_string() { + return @"participant $full_jid"; + } +} \ No newline at end of file diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala new file mode 100644 index 00000000..69b0f37a --- /dev/null +++ b/plugins/rtp/src/plugin.vala @@ -0,0 +1,413 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { + public Dino.Application app { get; private set; } + public CodecUtil codec_util { get; private set; } + public Gst.DeviceMonitor device_monitor { get; private set; } + public Gst.Pipeline pipe { get; private set; } + public Gst.Bin rtpbin { get; private set; } + public Gst.Element echoprobe { get; private set; } + + private Gee.List streams = new ArrayList(); + private Gee.List devices = new ArrayList(); + // private Gee.List participants = new ArrayList(); + + public void registered(Dino.Application app) { + this.app = app; + this.codec_util = new CodecUtil(); + app.startup.connect(startup); + app.add_option_group(Gst.init_get_option_group()); + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module(this)); + }); + app.plugin_registry.video_call_plugin = this; + } + + private int pause_count = 0; + public void pause() { +// if (pause_count == 0) { +// debug("Pausing pipe for modifications"); +// pipe.set_state(Gst.State.PAUSED); +// } + pause_count++; + } + public void unpause() { + pause_count--; + if (pause_count == 0) { + debug("Continue pipe after modifications"); + pipe.set_state(Gst.State.PLAYING); + } + if (pause_count < 0) warning("Pause count below zero!"); + } + + public void startup() { + device_monitor = new Gst.DeviceMonitor(); + device_monitor.show_all = true; + device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); + device_monitor.start(); + + pipe = new Gst.Pipeline(null); + + // RTP + rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin; + if (rtpbin == null) { + warning("RTP not supported"); + pipe = null; + return; + } + rtpbin.pad_added.connect(on_rtp_pad_added); + rtpbin.@set("latency", 100); + rtpbin.connect("signal::request-pt-map", request_pt_map, this); + pipe.add(rtpbin); + + // Audio echo probe + echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); + pipe.add(echoprobe); + + // Pipeline + pipe.auto_flush_bus = true; + pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => { + on_pipe_bus_message(message); + return true; + }); + pipe.set_state(Gst.State.PLAYING); + } + + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { + debug("request-pt-map"); + return null; + } + + private void on_rtp_pad_added(Gst.Pad pad) { + debug("pad added: %s", pad.name); + if (pad.name.has_prefix("recv_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_ssrc_pad_added(split[4], pad); + } + } + } + if (pad.name.has_prefix("send_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + debug("pad %s for stream %hhu", pad.name, rtpid); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_send_rtp_src_added(pad); + } + } + } + } + + private void on_pipe_bus_message(Gst.Message message) { + switch (message.type) { + case Gst.MessageType.ERROR: + Error error; + string str; + message.parse_error(out error, out str); + warning("Error in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.WARNING: + Error error; + string str; + message.parse_warning(out error, out str); + warning("Warning in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.CLOCK_LOST: + debug("Clock lost. Restarting"); + pipe.set_state(Gst.State.READY); + pipe.set_state(Gst.State.PLAYING); + break; + case Gst.MessageType.STATE_CHANGED: + Gst.State new_state; + message.parse_state_changed(null, out new_state, null); + if (message.src is Gst.Element) { + debug("%s changed state to %s", ((Gst.Element)message.src).name, new_state.to_string()); + } + break; + case Gst.MessageType.STREAM_STATUS: + Gst.StreamStatusType status; + Gst.Element owner; + message.parse_stream_status(out status, out owner); + if (owner != null) { + debug("%s stream changed status to %s", owner.name, status.to_string()); + } + break; + case Gst.MessageType.ELEMENT: + unowned Gst.Structure struc = message.get_structure(); + if (struc != null && message.src is Gst.Element) { + debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string()); + } + break; + case Gst.MessageType.NEW_CLOCK: + debug("New clock."); + break; + case Gst.MessageType.TAG: + // Ignore + break; + case Gst.MessageType.QOS: + // Ignore + break; + default: + debug("Pipe bus message: %s", message.type.to_string()); + break; + } + } + + private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { + Gst.Device old_device = null; + Gst.Device device = null; + switch (message.type) { + case Gst.MessageType.DEVICE_ADDED: + message.parse_device_added(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.add(new Device(this, device)); + break; + case Gst.MessageType.DEVICE_CHANGED: + message.parse_device_changed(out device, out old_device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.first_match((it) => it.matches(old_device)).update(device); + break; + case Gst.MessageType.DEVICE_REMOVED: + message.parse_device_removed(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + devices.remove(devices.first_match((it) => it.matches(device))); + break; + } + if (device != null) { + switch (device.device_class) { + case "Audio/Source": + devices_changed("audio", false); + break; + case "Audio/Sink": + devices_changed("audio", true); + break; + case "Video/Source": + devices_changed("video", false); + break; + case "Video/Sink": + devices_changed("video", true); + break; + } + } + return Source.CONTINUE; + } + + public uint8 next_free_id() { + uint8 rtpid = 0; + while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) { + rtpid++; + } + return rtpid; + } + + // public Participant get_participant(Jid full_jid, bool self) { +// foreach (Participant participant in participants) { +// if (participant.full_jid.equals(full_jid)) { +// return participant; +// } +// } +// Participant participant; +// if (self) { +// participant = new SelfParticipant(pipe, full_jid); +// } else { +// participant = new Participant(pipe, full_jid); +// } +// participants.add(participant); +// return participant; +// } + + public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; + if (content_params == null) return null; + Stream stream; + if (content_params.media == "video") { + stream = new VideoStream(this, content); + } else { + stream = new Stream(this, content); + } + streams.add(stream); + return stream; + } + + public void close_stream(Stream stream) { + streams.remove(stream); + stream.destroy(); + } + + public void shutdown() { + device_monitor.stop(); + pipe.set_state(Gst.State.NULL); + rtpbin = null; + pipe = null; + Gst.deinit(); + } + + public VideoCallWidget? create_widget(WidgetType type) { + if (type == WidgetType.GTK) { + return new VideoWidget(this); + } + return null; + } + + public Gee.List get_devices(string media, bool incoming) { + if (media == "video" && !incoming) { + return get_video_sources(); + } + + ArrayList result = new ArrayList(); + foreach (Device device in devices) { + if (device.media == media && (incoming && device.is_sink || !incoming && device.is_source)) { + result.add(device); + } + } + if (media == "audio") { + // Reorder sources + result.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + bool left_is_pipewire = left.device.properties.has_name("pipewire-proplist"); + bool right_is_pipewire = right.device.properties.has_name("pipewire-proplist"); + + bool left_is_default = false; + left.device.properties.get_boolean("is-default", out left_is_default); + bool right_is_default = false; + right.device.properties.get_boolean("is-default", out right_is_default); + + // Prefer pipewire + if (left_is_pipewire && !right_is_pipewire) return -1; + if (right_is_pipewire && !left_is_pipewire) return 1; + + // Prefer pulse audio default device + if (left_is_default && !right_is_default) return -1; + if (right_is_default && !left_is_default) return 1; + + + return 0; + }); + } + return result; + } + + public Gee.List get_video_sources() { + ArrayList pipewire_devices = new ArrayList(); + ArrayList other_devices = new ArrayList(); + + foreach (Device device in devices) { + if (device.media != "video") continue; + if (device.is_sink) continue; + + bool is_color = false; + for (int i = 0; i < device.device.caps.get_size(); i++) { + unowned Gst.Structure structure = device.device.caps.get_structure(i); + if (structure.has_field("format") && !structure.get_string("format").has_prefix("GRAY")) { + is_color = true; + } + } + + // Don't allow grey-scale devices + if (!is_color) continue; + + if (device.device.properties.has_name("pipewire-proplist")) { + pipewire_devices.add(device); + } else { + other_devices.add(device); + } + } + + // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux. + ArrayList devices = pipewire_devices.size > 0 ? pipewire_devices : other_devices; + + // Reorder sources + devices.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + int left_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) left_fps = int.max(left_fps, num / den); + } + + int right_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) right_fps = int.max(right_fps, num / den); + } + + // More FPS is better + if (left_fps > right_fps) return -1; + if (right_fps > left_fps) return 1; + + return 0; + }); + + return devices; + } + + public Device? get_preferred_device(string media, bool incoming) { + foreach (MediaDevice media_device in get_devices(media, incoming)) { + Device? device = media_device as Device; + if (device != null) return device; + } + warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media); + return null; + } + + public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream stream, bool incoming) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return null; + if (incoming) { + return plugin_stream.output_device ?? get_preferred_device(stream.media, incoming); + } else { + return plugin_stream.input_device ?? get_preferred_device(stream.media, incoming); + } + } + + private void dump_dot() { + string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; + Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); + debug("Stored pipe details as %s", name); + } + + public void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return; + if (pause) { + plugin_stream.pause(); + } else { + plugin_stream.unpause(); + Timeout.add_seconds(3, () => { + dump_dot(); + return false; + }); + } + } + + public void set_device(Xmpp.Xep.JingleRtp.Stream stream, MediaDevice? device) { + Device real_device = device as Device; + Stream plugin_stream = stream as Stream; + if (real_device == null || plugin_stream == null) return; + if (real_device.is_source) { + plugin_stream.input_device = real_device; + } else if (real_device.is_sink) { + plugin_stream.output_device = real_device; + } + } +} diff --git a/plugins/rtp/src/register_plugin.vala b/plugins/rtp/src/register_plugin.vala new file mode 100644 index 00000000..a80137ea --- /dev/null +++ b/plugins/rtp/src/register_plugin.vala @@ -0,0 +1,3 @@ +public Type register_plugin(Module module) { + return typeof (Dino.Plugins.Rtp.Plugin); +} diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala new file mode 100644 index 00000000..aea2fe85 --- /dev/null +++ b/plugins/rtp/src/stream.vala @@ -0,0 +1,432 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { + public uint8 rtpid { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public Gst.Element rtpbin { get { + return plugin.rtpbin; + }} + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + private Gst.App.Sink send_rtp; + private Gst.App.Sink send_rtcp; + private Gst.App.Src recv_rtp; + private Gst.App.Src recv_rtcp; + private Gst.Element encode; + private Gst.Element decode; + private Gst.Element input; + private Gst.Element output; + + private Device _input_device; + public Device input_device { get { return _input_device; } set { + if (!paused) { + if (this._input_device != null) { + this._input_device.unlink(); + this._input_device = null; + } + set_input(value != null ? value.link_source() : null); + } + this._input_device = value; + }} + private Device _output_device; + public Device output_device { get { return _output_device; } set { + if (output != null) remove_output(output); + if (value != null) add_output(value.link_sink()); + this._output_device = value; + }} + + public bool created { get; private set; default = false; } + public bool paused { get; private set; default = false; } + private bool push_recv_data = false; + private string participant_ssrc = null; + + private Gst.Pad recv_rtcp_sink_pad; + private Gst.Pad recv_rtp_sink_pad; + private Gst.Pad recv_rtp_src_pad; + private Gst.Pad send_rtcp_src_pad; + private Gst.Pad send_rtp_sink_pad; + private Gst.Pad send_rtp_src_pad; + + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(content); + this.plugin = plugin; + this.rtpid = plugin.next_free_id(); + + content.notify["senders"].connect_after(on_senders_changed); + } + + public void on_senders_changed() { + if (sending && input == null) { + input_device = plugin.get_preferred_device(media, false); + } + if (receiving && output == null) { + output_device = plugin.get_preferred_device(media, true); + } + } + + public override void create() { + plugin.pause(); + + // Create i/o if needed + + if (input == null && input_device == null && sending) { + input_device = plugin.get_preferred_device(media, false); + } + if (output == null && output_device == null && receiving && media == "audio") { + output_device = plugin.get_preferred_device(media, true); + } + + // Create app elements + send_rtp = Gst.ElementFactory.make("appsink", @"rtp-sink-$rtpid") as Gst.App.Sink; + send_rtp.async = false; + send_rtp.caps = CodecUtil.get_caps(media, payload_type); + send_rtp.emit_signals = true; + send_rtp.sync = false; + send_rtp.new_sample.connect(on_new_sample); + pipe.add(send_rtp); + + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp-sink-$rtpid") as Gst.App.Sink; + send_rtcp.async = false; + send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + send_rtcp.emit_signals = true; + send_rtcp.sync = false; + send_rtcp.new_sample.connect(on_new_sample); + pipe.add(send_rtcp); + + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp-src-$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type); + recv_rtp.do_timestamp = true; + recv_rtp.format = Gst.Format.TIME; + recv_rtp.is_live = true; + pipe.add(recv_rtp); + + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp-src-$rtpid") as Gst.App.Src; + recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + recv_rtcp.do_timestamp = true; + recv_rtcp.format = Gst.Format.TIME; + recv_rtcp.is_live = true; + pipe.add(recv_rtcp); + + // Connect RTCP + send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid"); + send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink")); + recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid"); + recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); + + // Connect input + encode = codec_util.get_encode_bin(media, payload_type, @"encode-$rtpid"); + pipe.add(encode); + send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); + encode.get_static_pad("src").link(send_rtp_sink_pad); + if (input != null) { + input.link(encode); + } + + // Connect output + decode = codec_util.get_decode_bin(media, payload_type, @"decode-$rtpid"); + pipe.add(decode); + if (output != null) { + decode.link(output); + } + + // Connect RTP + recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid"); + recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad); + + created = true; + push_recv_data = true; + plugin.unpause(); + } + + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { + if (sink == null) { + debug("Sink is null"); + return Gst.FlowReturn.EOS; + } + Gst.Sample sample = sink.pull_sample(); + Gst.Buffer buffer = sample.get_buffer(); + uint8[] data; + buffer.extract_dup(0, buffer.get_size(), out data); + if (sink == send_rtp) { + on_send_rtp_data(new Bytes.take(data)); + } else if (sink == send_rtcp) { + on_send_rtcp_data(new Bytes.take(data)); + } else { + warning("unknown sample"); + } + return Gst.FlowReturn.OK; + } + + private static Gst.PadProbeReturn drop_probe() { + return Gst.PadProbeReturn.DROP; + } + + public override void destroy() { + // Stop network communication + push_recv_data = false; + recv_rtp.end_of_stream(); + recv_rtcp.end_of_stream(); + send_rtp.new_sample.disconnect(on_new_sample); + send_rtcp.new_sample.disconnect(on_new_sample); + + // Disconnect input device + if (input != null) { + input.unlink(encode); + input = null; + } + if (this._input_device != null) { + if (!paused) this._input_device.unlink(); + this._input_device = null; + } + + // Disconnect encode + encode.set_locked_state(true); + encode.set_state(Gst.State.NULL); + encode.get_static_pad("src").unlink(send_rtp_sink_pad); + pipe.remove(encode); + encode = null; + + // Disconnect RTP sending + if (send_rtp_src_pad != null) { + send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + pipe.remove(send_rtp); + send_rtp = null; + + // Disconnect decode + if (recv_rtp_src_pad != null) { + recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); + } + + // Disconnect RTP receiving + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + pipe.remove(recv_rtp); + recv_rtp = null; + + // Disconnect output + if (output != null) { + decode.unlink(output); + } + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + output = null; + + // Disconnect output device + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + + // Disconnect RTCP receiving + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + + // Disconnect RTCP sending + send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + pipe.remove(send_rtcp); + send_rtcp = null; + + // Release rtp pads + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + send_rtp_src_pad = null; + recv_rtp_src_pad = null; + } + + public override void on_recv_rtp_data(Bytes bytes) { + if (push_recv_data) { + recv_rtp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + } + } + + public override void on_recv_rtcp_data(Bytes bytes) { + if (push_recv_data) { + recv_rtcp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + } + } + + public override void on_rtp_ready() { + // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame. + // Send a full frame to let the counterpart display our video asap + rtpbin.send_event(new Gst.Event.custom( + Gst.EventType.CUSTOM_UPSTREAM, + new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null)) + ); + } + + public override void on_rtcp_ready() { + int rtp_session_id = (int) rtpid; + uint64 max_delay = int.MAX; + Object rtp_session; + bool rtp_sent; + GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session); + GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent); + debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); + } + + public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { + participant_ssrc = ssrc; + recv_rtp_src_pad = pad; + if (decode != null) { + plugin.pause(); + debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name); + recv_rtp_src_pad.link(decode.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void on_send_rtp_src_added(Gst.Pad pad) { + send_rtp_src_pad = pad; + if (send_rtp != null) { + plugin.pause(); + debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name); + send_rtp_src_pad.link(send_rtp.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void set_input(Gst.Element? input) { + set_input_and_pause(input, paused); + } + + private void set_input_and_pause(Gst.Element? input, bool paused) { + if (created && this.input != null) { + this.input.unlink(encode); + this.input = null; + } + + this.input = input; + this.paused = paused; + + if (created && sending && !paused && input != null) { + plugin.pause(); + input.link(encode); + plugin.unpause(); + } + } + + public void pause() { + if (paused) return; + set_input_and_pause(null, true); + if (input_device != null) input_device.unlink(); + } + + public void unpause() { + if (!paused) return; + set_input_and_pause(input_device != null ? input_device.link_source() : null, false); + } + + ulong block_probe_handler_id = 0; + public virtual void add_output(Gst.Element element) { + if (output != null) { + critical("add_output() invoked more than once"); + return; + } + this.output = element; + if (created) { + plugin.pause(); + decode.link(element); + if (block_probe_handler_id != 0) { + decode.get_static_pad("src").remove_probe(block_probe_handler_id); + } + plugin.unpause(); + } + } + + public virtual void remove_output(Gst.Element element) { + if (output != element) { + critical("remove_output() invoked without prior add_output()"); + return; + } + if (created) { + block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); + decode.unlink(element); + } + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + this.output = null; + } +} + +public class Dino.Plugins.Rtp.VideoStream : Stream { + private Gee.List outputs = new ArrayList(); + private Gst.Element output_tee; + + public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(plugin, content); + if (media != "video") critical("VideoStream created for non-video media"); + } + + public override void create() { + plugin.pause(); + output_tee = Gst.ElementFactory.make("tee", null); + output_tee.@set("allow-not-linked", true); + pipe.add(output_tee); + add_output(output_tee); + base.create(); + foreach (Gst.Element output in outputs) { + output_tee.link(output); + } + plugin.unpause(); + } + + public override void destroy() { + foreach (Gst.Element output in outputs) { + output_tee.unlink(output); + } + base.destroy(); + output_tee.set_locked_state(true); + output_tee.set_state(Gst.State.NULL); + pipe.remove(output_tee); + output_tee = null; + } + + public override void add_output(Gst.Element element) { + if (element == output_tee) { + base.add_output(element); + return; + } + outputs.add(element); + if (output_tee != null) { + output_tee.link(element); + } + } + + public override void remove_output(Gst.Element element) { + if (element == output_tee) { + base.remove_output(element); + return; + } + outputs.remove(element); + if (output_tee != null) { + output_tee.unlink(element); + } + } +} \ No newline at end of file diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala new file mode 100644 index 00000000..fa5ba138 --- /dev/null +++ b/plugins/rtp/src/video_widget.vala @@ -0,0 +1,110 @@ +public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidget { + private static uint last_id = 0; + + public uint id { get; private set; } + public Gst.Element element { get; private set; } + public Gtk.Widget widget { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + + private bool attached; + private Device? connected_device; + private Stream? connected_stream; + private Gst.Element convert; + + public VideoWidget(Plugin plugin) { + this.plugin = plugin; + + id = last_id++; + element = Gst.ElementFactory.make("gtksink", @"video-widget-$id"); + if (element != null) { + Gtk.Widget widget; + element.@get("widget", out widget); + element.@set("async", false); + element.@set("sync", false); + this.widget = widget; + add(widget); + widget.visible = true; + + // Listen for resolution changes + element.get_static_pad("sink").notify["caps"].connect(() => { + if (element.get_static_pad("sink").caps == null) return; + + int width, height; + element.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + element.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + resolution_changed(width, height); + }); + } else { + warning("Could not create GTK video sink. Won't display videos."); + } + } + + public void display_stream(Xmpp.Xep.JingleRtp.Stream stream) { + if (element == null) return; + detach(); + if (stream.media != "video") return; + connected_stream = stream as Stream; + if (connected_stream == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoconvert name=video-widget-$id-convert", true); + convert.name = @"video-widget-$id-prepare"; + pipe.add(convert); + convert.link(element); + connected_stream.add_output(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void display_device(MediaDevice media_device) { + if (element == null) return; + detach(); + connected_device = media_device as Device; + if (connected_device == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video-widget-$id-flip ! videoconvert name=video-widget-$id-convert", true); + convert.name = @"video-widget-$id-prepare"; + pipe.add(convert); + convert.link(element); + connected_device.link_source().link(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void detach() { + if (element == null) return; + if (attached) { + if (connected_stream != null) { + connected_stream.remove_output(convert); + connected_stream = null; + } + if (connected_device != null) { + connected_device.link_source().unlink(element); + connected_device.unlink(); // We get a new ref to recover the element, so unlink twice + connected_device.unlink(); + connected_device = null; + } + convert.set_locked_state(true); + convert.set_state(Gst.State.NULL); + pipe.remove(convert); + convert = null; + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + pipe.remove(element); + attached = false; + } + } + + public override void dispose() { + detach(); + widget = null; + element = null; + } +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From cde1e38f5d56269addff93b36c57299cbf546279 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 21 Mar 2021 15:42:58 +0100 Subject: RTP: Backport gst_caps_copy_nth from GStreamer 1.16 --- plugins/rtp/src/device.vala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 796be203..570c6667 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -104,14 +104,22 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { best_index = i; } } - return device.caps.copy_nth(best_index); + return caps_copy_nth(device.caps, best_index); } else if (device.caps.get_size() > 0) { - return device.caps.copy_nth(0); + return caps_copy_nth(device.caps, 0); } else { return new Gst.Caps.any(); } } + // Backport from gst_caps_copy_nth added in GStreamer 1.16 + private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) { + Gst.Caps target = new Gst.Caps.empty(); + target.flags = source.flags; + target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy()); + return target; + } + private void create() { debug("Creating device %s", id); plugin.pause(); -- cgit v1.2.3-70-g09d2 From b393d4160182873ea2acd9fbc6421f7e1a3adb9e Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:05:50 +0100 Subject: Add support for SRTP --- plugins/crypto-vala/CMakeLists.txt | 1 + plugins/crypto-vala/src/random.vala | 5 + plugins/rtp/CMakeLists.txt | 8 +- plugins/rtp/src/module.vala | 80 +- plugins/rtp/src/srtp.c | 836 +++++++++++++++++++++ plugins/rtp/src/srtp.h | 82 ++ plugins/rtp/src/srtp.vapi | 103 +++ plugins/rtp/src/stream.vala | 60 +- .../xep/0167_jingle_rtp/content_parameters.vala | 46 +- .../xep/0167_jingle_rtp/jingle_rtp_module.vala | 132 +++- .../src/module/xep/0167_jingle_rtp/stream.vala | 14 + 11 files changed, 1286 insertions(+), 81 deletions(-) create mode 100644 plugins/crypto-vala/src/random.vala create mode 100644 plugins/rtp/src/srtp.c create mode 100644 plugins/rtp/src/srtp.h create mode 100644 plugins/rtp/src/srtp.vapi (limited to 'plugins/rtp/src') diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 2c9f790a..036e45f6 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -10,6 +10,7 @@ SOURCES "src/cipher.vala" "src/cipher_converter.vala" "src/error.vala" + "src/random.vala" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" PACKAGES diff --git a/plugins/crypto-vala/src/random.vala b/plugins/crypto-vala/src/random.vala new file mode 100644 index 00000000..3f5d3ba9 --- /dev/null +++ b/plugins/crypto-vala/src/random.vala @@ -0,0 +1,5 @@ +namespace Crypto { +public static void randomize(uint8[] buffer) { + GCrypt.Random.randomize(buffer); +} +} \ No newline at end of file diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 2b66f6ff..ef2f7698 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -18,18 +18,20 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi PACKAGES ${RTP_PACKAGES} OPTIONS --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) -add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp") -add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino ${RTP_PACKAGES}) +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) +add_library(rtp SHARED ${RTP_VALA_C} src/srtp.c) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 577e9f53..ecf7b658 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -200,65 +200,23 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { plugin.close_stream(rtp_stream); } -// public uint32 get_session_id(string id) { -// return (uint32) id.split("-")[0].to_int(); -// } -// -// public string create_feed(string media, bool incoming) { -// init(); -// string id = random_uuid(); -// if (media == "audio") { -// id = "0-" + id; -// } else { -// id = "1-" + id; -// } -// MediaDevice? device = plugin.get_preferred_device(media, incoming); -// Feed feed; -// if (incoming) { -// if (media == "audio") { -// feed = new IncomingAudioFeed(id, this, device); -// } else if (media == "video") { -// feed = new IncomingVideoFeed(id, this, device); -// } else { -// critical("Incoming feed of media '%s' not supported", media); -// return id; -// } -// } else { -// if (media == "audio") { -// string? matching_incoming_feed_id = null; -// foreach (Feed match in plugin.feeds.values) { -// if (match is IncomingAudioFeed) { -// matching_incoming_feed_id = match.id; -// } -// } -// feed = new OutgoingAudioFeed(id, this, device); -// } else if (media == "video") { -// feed = new OutgoingVideoFeed(id, this, device); -// } else { -// critical("Outgoing feed of media '%s' not supported", media); -// return id; -// } -// } -// plugin.add_feed(id, feed); -// return id; -// } -// -// public void connect_feed(string id, JingleRtp.PayloadType payload, Jingle.DatagramConnection connection) { -// if (!plugin.feeds.has_key(id)) { -// critical("Tried to connect feed with id %s, but no such feed found", id); -// return; -// } -// Feed feed = plugin.feeds[id]; -// feed.connect(payload, connection); -// } -// -// public void destroy_feed(string id) { -// if (!plugin.feeds.has_key(id)) { -// critical("Tried to destroy feed with id %s, but no such feed found", id); -// return; -// } -// Feed feed = plugin.feeds[id]; -// feed.destroy(); -// plugin.feeds.remove(id); -// } + public override JingleRtp.Crypto? generate_local_crypto() { + uint8[] keyAndSalt = new uint8[30]; + Crypto.randomize(keyAndSalt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, keyAndSalt); + } + + public override JingleRtp.Crypto? pick_remote_crypto(Gee.List cryptos) { + foreach (JingleRtp.Crypto crypto in cryptos) { + if (crypto.is_valid) return crypto; + } + return null; + } + + public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { + if (remote == null || !remote.is_valid) return null; + uint8[] keyAndSalt = new uint8[30]; + Crypto.randomize(keyAndSalt); + return remote.rekey(keyAndSalt); + } } \ No newline at end of file diff --git a/plugins/rtp/src/srtp.c b/plugins/rtp/src/srtp.c new file mode 100644 index 00000000..708244d9 --- /dev/null +++ b/plugins/rtp/src/srtp.c @@ -0,0 +1,836 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* TODO: + * Useless stuff (because nothing depends on it): + * - non-nul key derivation rate + * - MKI payload + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include + +#include "srtp.h" + +#include +#include +#include +#include + +#include + +#ifdef _WIN32 +# include +#else +# include +#endif + +#define debug( ... ) (void)0 + +typedef struct srtp_proto_t +{ + gcry_cipher_hd_t cipher; + gcry_md_hd_t mac; + uint64_t window; + uint32_t salt[4]; +} srtp_proto_t; + +struct srtp_session_t +{ + srtp_proto_t rtp; + srtp_proto_t rtcp; + unsigned flags; + unsigned kdr; + uint32_t rtcp_index; + uint32_t rtp_roc; + uint16_t rtp_seq; + uint16_t rtp_rcc; + uint8_t tag_len; +}; + +enum +{ + SRTP_CRYPT, + SRTP_AUTH, + SRTP_SALT, + SRTCP_CRYPT, + SRTCP_AUTH, + SRTCP_SALT +}; + + +static inline unsigned rcc_mode (const srtp_session_t *s) +{ + return (s->flags >> 4) & 3; +} + + +static void proto_destroy (srtp_proto_t *p) +{ + gcry_md_close (p->mac); + gcry_cipher_close (p->cipher); +} + + +/** + * Releases all resources associated with a Secure RTP session. + */ +void srtp_destroy (srtp_session_t *s) +{ + assert (s != NULL); + + proto_destroy (&s->rtcp); + proto_destroy (&s->rtp); + free (s); +} + + +static int proto_create (srtp_proto_t *p, int gcipher, int gmd) +{ + if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) + { + if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) + return 0; + gcry_cipher_close (p->cipher); + } + return -1; +} + + +/** + * Allocates a Secure RTP one-way session. + * The same session cannot be used both ways because this would confuse + * internal cryptographic counters; it is however of course feasible to open + * multiple simultaneous sessions with the same master key. + * + * @param encr encryption algorithm number + * @param auth authentication algortihm number + * @param tag_len authentication tag byte length (NOT including RCC) + * @param flags OR'ed optional flags. + * + * @return NULL in case of error + */ +srtp_session_t * +srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) +{ + if ((flags & ~SRTP_FLAGS_MASK)) + return NULL; + + int cipher, md; + switch (encr) + { + case SRTP_ENCR_NULL: + cipher = GCRY_CIPHER_NONE; + break; + + case SRTP_ENCR_AES_CM: + cipher = GCRY_CIPHER_AES; + break; + + default: + return NULL; + } + + switch (auth) + { + case SRTP_AUTH_NULL: + md = GCRY_MD_NONE; + break; + + case SRTP_AUTH_HMAC_SHA1: + md = GCRY_MD_SHA1; + break; + + default: + return NULL; + } + + if (tag_len > gcry_md_get_algo_dlen (md)) + return NULL; + + if (prf != SRTP_PRF_AES_CM) + return NULL; + + srtp_session_t *s = malloc (sizeof (*s)); + if (s == NULL) + return NULL; + + memset (s, 0, sizeof (*s)); + s->flags = flags; + s->tag_len = tag_len; + s->rtp_rcc = 1; /* Default RCC rate */ + if (rcc_mode (s)) + { + if (tag_len < 4) + goto error; + } + + if (proto_create (&s->rtp, cipher, md) == 0) + { + if (proto_create (&s->rtcp, cipher, md) == 0) + return s; + proto_destroy (&s->rtp); + } + + error: + free (s); + return NULL; +} + + +/** + * Counter Mode encryption/decryption (ctr length = 16 bytes) + * with non-padded (truncated) text + */ +static int +do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) +{ + const size_t ctrlen = 16; + div_t d = div (len, ctrlen); + + if (gcry_cipher_setctr (hd, ctr, ctrlen) + || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) + return -1; + + if (d.rem) + { + /* Truncated last block */ + uint8_t dummy[ctrlen]; + data += d.quot * ctrlen; + memcpy (dummy, data, d.rem); + memset (dummy + d.rem, 0, ctrlen - d.rem); + + if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) + return -1; + memcpy (data, dummy, d.rem); + } + + return 0; +} + + +/** + * AES-CM key derivation (saltlen = 14 bytes) + */ +static int +do_derive (gcry_cipher_hd_t prf, const void *salt, + const uint8_t *r, size_t rlen, uint8_t label, + void *out, size_t outlen) +{ + uint8_t iv[16]; + + memcpy (iv, salt, 14); + iv[14] = iv[15] = 0; + + assert (rlen < 14); + iv[13 - rlen] ^= label; + for (size_t i = 0; i < rlen; i++) + iv[sizeof (iv) - rlen + i] ^= r[i]; + + memset (out, 0, outlen); + return do_ctr_crypt (prf, iv, out, outlen); +} + + +/** + * Sets (or resets) the master key and master salt for a SRTP session. + * This must be done at least once before using srtp_send(), srtp_recv(), + * srtcp_send() or srtcp_recv(). Also, rekeying is required every + * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), + * otherwise the protocol security might be broken. + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen) +{ + /* SRTP/SRTCP cipher/salt/MAC keys derivation */ + gcry_cipher_hd_t prf; + uint8_t r[6], keybuf[20]; + + if (saltlen != 14) + return EINVAL; + + if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) + || gcry_cipher_setkey (prf, key, keylen)) + return EINVAL; + + /* SRTP key derivation */ +#if 0 + if (s->kdr != 0) + { + uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; + index /= s->kdr; + + for (int i = sizeof (r) - 1; i >= 0; i--) + { + r[i] = index & 0xff; + index = index >> 8; + } + } + else +#endif + memset (r, 0, sizeof (r)); + if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtp.mac, keybuf, 20) + || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) + return -1; + + /* SRTCP key derivation */ + memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); + if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtcp.mac, keybuf, 20) + || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) + return -1; + + (void)gcry_cipher_close (prf); + return 0; +} + +static int hexdigit (char c) +{ + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'A') && (c <= 'F')) + return c - 'A' + 0xA; + if ((c >= 'a') && (c <= 'f')) + return c - 'a' + 0xa; + return -1; +} + +static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) +{ + size_t inlen = strlen (in); + + if ((inlen > (2 * outlen)) || (inlen & 1)) + return -1; + + for (size_t i = 0; i < inlen; i += 2) + { + int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); + if ((a == -1) || (b == -1)) + return -1; + out[i / 2] = (a << 4) | b; + } + return inlen / 2; +} + +/** + * Sets (or resets) the master key and master salt for a SRTP session + * from hexadecimal strings. See also srtp_setkey(). + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) +{ + uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ + uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ + ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); + ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); + + if ((bkeylen == -1) || (bsaltlen == -1)) + return EINVAL; + return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; +} + +/** + * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not + * specified (through this function), the default rate of ONE is assumed + * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none + * of the RCC mode has been selected. + * + * The RCC mode is selected through one of these flags for srtp_create(): + * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets + * SRTP_RCC_MODE2: integrity protection for all packets + * SRTP_RCC_MODE3: no integrity protection + * + * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality + * (through encryption) but is much more prone to DoS. It can only be used if + * anti-spoofing protection is provided by lower network layers (e.g. IPsec, + * or trusted routers and proper source address filtering). + * + * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. + * + * @param rate RoC Carry rate (MUST NOT be zero) + */ +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) +{ + assert (rate != 0); + s->rtp_rcc = rate; +} + + +/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ +static int +rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, + const uint32_t *salt, uint8_t *data, size_t len) +{ + /* Determines cryptographic counter (IV) */ + uint32_t counter[4]; + counter[0] = salt[0]; + counter[1] = salt[1] ^ ssrc; + counter[2] = salt[2] ^ htonl (roc); + counter[3] = salt[3] ^ htonl (seq << 16); + + /* Encryption */ + return do_ctr_crypt (hd, counter, data, len); +} + + +/** Determines SRTP Roll-Over-Counter (in host-byte order) */ +static uint32_t +srtp_compute_roc (const srtp_session_t *s, uint16_t seq) +{ + uint32_t roc = s->rtp_roc; + + if (((seq - s->rtp_seq) & 0xffff) < 0x8000) + { + /* Sequence is ahead, good */ + if (seq < s->rtp_seq) + roc++; /* Sequence number wrap */ + } + else + { + /* Sequence is late, bad */ + if (seq > s->rtp_seq) + roc--; /* Wrap back */ + } + return roc; +} + + +/** Returns RTP sequence (in host-byte order) */ +static inline uint16_t rtp_seq (const uint8_t *buf) +{ + return (buf[2] << 8) | buf[3]; +} + + +/** Message Authentication and Integrity for RTP */ +static const uint8_t * +rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, + uint32_t roc) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTP packet and updates SRTP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTP packet to be en-/decrypted + * @param len RTP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet + * EACCES replayed packet or out-of-window or sync lost + */ +static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + assert (len >= 12u); + + if ((buf[0] >> 6) != 2) + return EINVAL; + + /* Computes encryption offset */ + uint16_t offset = 12; + offset += (buf[0] & 0xf) * 4; // skips CSRC + + if (buf[0] & 0x10) + { + uint16_t extlen; + + offset += 4; + if (len < offset) + return EINVAL; + + memcpy (&extlen, buf + offset - 2, 2); + offset += htons (extlen); // skips RTP extension header + } + + if (len < offset) + return EINVAL; + + /* Determines RTP 48-bits counter and SSRC */ + uint16_t seq = rtp_seq (buf); + uint32_t roc = srtp_compute_roc (s, seq), ssrc; + memcpy (&ssrc, buf + 8, 4); + + /* Updates ROC and sequence (it's safe now) */ + int16_t diff = seq - s->rtp_seq; + if (diff > 0) + { + /* Sequence in the future, good */ + s->rtp.window = s->rtp.window << diff; + s->rtp.window |= UINT64_C(1); + s->rtp_seq = seq, s->rtp_roc = roc; + } + else + { + /* Sequence in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) + return EACCES; /* Replay attack */ + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Encrypt/Decrypt */ + if (s->flags & SRTP_UNENCRYPTED) + return 0; + + if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, + buf + offset, len - offset)) + return EINVAL; + + return 0; +} + + +/** + * Turns a RTP packet into a SRTP packet: encrypt it, then computes + * the authentication tag and appends it. + * Note that you can encrypt packet in disorder. + * + * @param buf RTP packet to be encrypted/digested + * @param lenp pointer to the RTP packet length on entry, + * set to the SRTP length on exit (undefined on non-ENOSPC error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet or internal error + * ENOSPC bufsize is too small to add authentication tag + * ( will hold the required byte size) + * EACCES packet would trigger a replay error on receiver + */ +int +srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + size_t tag_len; + size_t roc_len = 0; + + /* Compute required buffer size */ + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + tag_len = s->tag_len; + + if (rcc_mode (s)) + { + assert (tag_len >= 4); + assert (s->rtp_rcc != 0); + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; /* RCC mode 3 -> no auth*/ + else + tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; /* RCC mode 1 or 3 -> no auth */ + } + } + + *lenp = len + roc_len + tag_len; + } + else + tag_len = 0; + + if (bufsize < *lenp) + return ENOSPC; + + /* Encrypt payload */ + int val = srtp_crypt (s, buf, len); + if (val) + return val; + + /* Authenticate payload */ + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); + + if (roc_len) + { + memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); + len += 4; + } + memcpy (buf + len, tag, tag_len); +#if 0 + printf ("Sent : 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + puts (""); +#endif + } + + return 0; +} + + +/** + * Turns a SRTP packet into a RTP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTP packet to be digested/decrypted + * @param lenp pointer to the SRTP packet length on entry, + * set to the RTP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + size_t tag_len = s->tag_len, roc_len = 0; + if (rcc_mode (s)) + { + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; + else + tag_len -= 4; + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; // RCC mode 1 or 3: no auth + } + } + + if (len < (12u + roc_len + tag_len)) + return EINVAL; + len -= roc_len + tag_len; + + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; + if (roc_len) + { + assert (roc_len == 4); + memcpy (&rcc, buf + len, 4); + rcc = ntohl (rcc); + } + else + rcc = roc; + + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); +#if 0 + printf ("Computed: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + printf ("\nReceived: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", buf[len + roc_len + i]); + puts (""); +#endif + if (memcmp (buf + len + roc_len, tag, tag_len)) + return EACCES; + + if (roc_len) + { + /* Authenticated packet carried a Roll-Over-Counter */ + s->rtp_roc += rcc - roc; + assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); + } + *lenp = len; + } + + return srtp_crypt (s, buf, len); +} + + +/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ +static int +rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, + const uint32_t *salt, uint8_t *data, size_t len) +{ + return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); +} + + +/** Message Authentication and Integrity for RTCP */ +static const uint8_t * +rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTCP packet and updates SRTCP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTCP packet to be en-/decrypted + * @param len RTCP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet + */ +static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + + /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ + if ((len < 12) || ((buf[0] >> 6) != 2)) + return EINVAL; + + uint32_t index; + memcpy (&index, buf + len, 4); + index = ntohl (index); + if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) + return EINVAL; // E-bit mismatch + + index &= ~(1 << 31); // clear E-bit for counter + + /* Updates SRTCP index (safe here) */ + int32_t diff = index - s->rtcp_index; + if (diff > 0) + { + /* Packet in the future, good */ + s->rtcp.window = s->rtcp.window << diff; + s->rtcp.window |= UINT64_C(1); + s->rtcp_index = index; + } + else + { + /* Packet in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) + return EACCES; // replay attack! + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Crypts SRTCP */ + if (s->flags & SRTCP_UNENCRYPTED) + return 0; + + uint32_t ssrc; + memcpy (&ssrc, buf + 4, 4); + + if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, + buf + 8, len - 8)) + return EINVAL; + return 0; +} + + +/** + * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes + * the authentication tag and appends it. + * + * @param buf RTCP packet to be encrypted/digested + * @param lenp pointer to the RTCP packet length on entry, + * set to the SRTCP length on exit (undefined in case of error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet or internal error + * ENOSPC bufsize is too small (to add index and authentication tag) + */ +int +srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + if (bufsize < (len + 4 + s->tag_len)) + return ENOSPC; + + uint32_t index = ++s->rtcp_index; + if (index >> 31) + s->rtcp_index = index = 0; /* 31-bit wrap */ + + if ((s->flags & SRTCP_UNENCRYPTED) == 0) + index |= 0x80000000; /* Set Encrypted bit */ + memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); + + int val = srtcp_crypt (s, buf, len); + if (val) + return val; + + len += 4; /* Digests SRTCP index too */ + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + memcpy (buf + len, tag, s->tag_len); + *lenp = len + s->tag_len; + return 0; +} + + +/** + * Turns a SRTCP packet into a RTCP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTCP packet to be digested/decrypted + * @param lenp pointer to the SRTCP packet length on entry, + * set to the RTCP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTCP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + + if (len < (4u + s->tag_len)) + return EINVAL; + len -= s->tag_len; + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + if (memcmp (buf + len, tag, s->tag_len)) + return EACCES; + + len -= 4; /* Remove SRTCP index before decryption */ + *lenp = len; + return srtcp_crypt (s, buf, len); +} \ No newline at end of file diff --git a/plugins/rtp/src/srtp.h b/plugins/rtp/src/srtp.h new file mode 100644 index 00000000..abca6988 --- /dev/null +++ b/plugins/rtp/src/srtp.h @@ -0,0 +1,82 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + ****************************************************************************/ + +#ifndef LIBVLC_SRTP_H +# define LIBVLC_SRTP_H 1 +#include + +typedef struct srtp_session_t srtp_session_t; + +enum +{ + SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets + SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets + SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets + + SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 + SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 + SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) + + SRTP_FLAGS_MASK=0x37 //< mask for valid flags +}; + +/** SRTP encryption algorithms (ciphers); same values as MIKEY */ +enum +{ + SRTP_ENCR_NULL=0, //< no encryption + SRTP_ENCR_AES_CM=1, //< AES counter mode + SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) +}; + +/** SRTP authenticaton algorithms; same values as MIKEY */ +enum +{ + SRTP_AUTH_NULL=0, //< no authentication code + SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 +}; + +/** SRTP pseudo random function; same values as MIKEY */ +enum +{ + SRTP_PRF_AES_CM=0, //< AES counter mode +}; + +# ifdef __cplusplus +extern "C" { +# endif + +srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, + unsigned flags); +void srtp_destroy (srtp_session_t *s); + +int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen); +int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); + +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); + +int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); +int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); +int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); +int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); + +# ifdef __cplusplus +} +# endif +#endif \ No newline at end of file diff --git a/plugins/rtp/src/srtp.vapi b/plugins/rtp/src/srtp.vapi new file mode 100644 index 00000000..c5ce7fec --- /dev/null +++ b/plugins/rtp/src/srtp.vapi @@ -0,0 +1,103 @@ +[Compact] +[CCode (cname = "srtp_session_t", free_function = "srtp_destroy", cheader_filename="srtp.h")] +public class Dino.Plugins.Rtp.SrtpSession { + [CCode (cname = "srtp_create")] + public SrtpSession(SrtpEncryption encr, SrtpAuthentication auth, uint tag_len, SrtpPrf prf, SrtpFlags flags); + [CCode (cname = "srtp_setkey")] + public int setkey(uint8[] key, uint8[] salt); + [CCode (cname = "srtp_setkeystring")] + public int setkeystring(string key, string salt); + [CCode (cname = "srtp_setrcc_rate")] + public void setrcc_rate(uint16 rate); + + [CCode (cname = "srtp_send")] + private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtcp_send")] + private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtp_recv")] + private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + [CCode (cname = "srtcp_recv")] + private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + + public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length+tag_len]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length+tag_len+4]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } +} + +[Flags] +[CCode (cname = "unsigned", cprefix = "", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpFlags { + SRTP_UNENCRYPTED, + SRTCP_UNENCRYPTED, + SRTP_UNAUTHENTICATED, + + SRTP_RCC_MODE1, + SRTP_RCC_MODE2, + SRTP_RCC_MODE3 +} + +[CCode (cname = "int", cprefix = "SRTP_ENCR_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpEncryption { + NULL, + AES_CM, + AES_F8 +} + +[CCode (cname = "int", cprefix = "SRTP_AUTH_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpAuthentication { + NULL, + HMAC_SHA1 +} + +[CCode (cname = "int", cprefix = "SRTP_PRF_", cheader_filename="srtp.h", has_type_id = false)] +public enum Dino.Plugins.Rtp.SrtpPrf { + AES_CM +} \ No newline at end of file diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index aea2fe85..362e2d16 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,6 +53,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; + private SrtpSession? local_crypto_session; + private SrtpSession? remote_crypto_session; + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); this.plugin = plugin; @@ -144,6 +147,20 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { plugin.unpause(); } + private void prepare_local_crypto() { + if (local_crypto != null && local_crypto_session == null) { + local_crypto_session = new SrtpSession( + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, + SrtpAuthentication.HMAC_SHA1, + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, + SrtpPrf.AES_CM, + 0 + ); + local_crypto_session.setkey(local_crypto.key, local_crypto.salt); + debug("Setting up encryption with key params %s", local_crypto.key_params); + } + } + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { if (sink == null) { debug("Sink is null"); @@ -153,9 +170,16 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { Gst.Buffer buffer = sample.get_buffer(); uint8[] data; buffer.extract_dup(0, buffer.get_size(), out data); + prepare_local_crypto(); if (sink == send_rtp) { + if (local_crypto_session != null) { + data = local_crypto_session.encrypt_rtp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + } on_send_rtp_data(new Bytes.take(data)); } else if (sink == send_rtcp) { + if (local_crypto_session != null) { + data = local_crypto_session.encrypt_rtcp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + } on_send_rtcp_data(new Bytes.take(data)); } else { warning("unknown sample"); @@ -258,15 +282,47 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtp_src_pad = null; } + private void prepare_remote_crypto() { + if (remote_crypto != null && remote_crypto_session == null) { + remote_crypto_session = new SrtpSession( + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, + SrtpAuthentication.HMAC_SHA1, + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, + SrtpPrf.AES_CM, + 0 + ); + remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); + debug("Setting up decryption with key params %s", remote_crypto.key_params); + } + } + public override void on_recv_rtp_data(Bytes bytes) { + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (remote_crypto_session != null) { + try { + data = remote_crypto_session.decrypt_rtp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } if (push_recv_data) { - recv_rtp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + recv_rtp.push_buffer(new Gst.Buffer.wrapped((owned) data)); } } public override void on_recv_rtcp_data(Bytes bytes) { + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (remote_crypto_session != null) { + try { + data = remote_crypto_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } if (push_recv_data) { - recv_rtcp.push_buffer(new Gst.Buffer.wrapped_bytes(bytes)); + recv_rtcp.push_buffer(new Gst.Buffer.wrapped((owned) data)); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index 8a3668b2..cca03543 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -17,7 +17,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public bool encryption_required { get; private set; default = false; } public PayloadType? agreed_payload_type { get; private set; } public Gee.List payload_types = new ArrayList(PayloadType.equals_func); - public Gee.List cryptos = new ArrayList(); + public Gee.List remote_cryptos = new ArrayList(); + public Crypto? local_crypto = null; + public Crypto? remote_crypto = null; public weak Stream? stream { get; private set; } @@ -27,7 +29,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { string media, Gee.List payload_types, string? ssrc = null, bool rtcp_mux = false, string? bandwidth = null, string? bandwidth_type = null, - bool encryption_required = false, Gee.List cryptos = new ArrayList() + bool encryption_required = false, Crypto? local_crypto = null ) { this.parent = parent; this.media = media; @@ -37,7 +39,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.bandwidth_type = bandwidth_type; this.encryption_required = encryption_required; this.payload_types = payload_types; - this.cryptos = cryptos; + this.local_crypto = local_crypto; } public Parameters.from_node(Module parent, StanzaNode node) throws Jingle.IqError { @@ -49,7 +51,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { if (encryption != null) { this.encryption_required = encryption.get_attribute_bool("required", this.encryption_required); foreach (StanzaNode crypto in encryption.get_subnodes("crypto")) { - this.cryptos.add(Crypto.parse(crypto)); + this.remote_cryptos.add(Crypto.parse(crypto)); } } foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { @@ -64,6 +66,15 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { content.reject(); return; } + remote_crypto = parent.pick_remote_crypto(remote_cryptos); + if (local_crypto == null && remote_crypto != null) { + local_crypto = parent.pick_local_crypto(remote_crypto); + } + if ((local_crypto == null || remote_crypto == null) && encryption_required) { + debug("no usable encryption, but encryption required"); + content.reject(); + return; + } } public void accept(XmppStream stream, Jingle.Session session, Jingle.Content content) { @@ -97,6 +108,15 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } }); + if (remote_crypto == null || local_crypto == null) { + if (encryption_required) { + warning("Encryption required but not provided in both directions"); + return; + } + remote_crypto = null; + local_crypto = null; + } + this.stream = parent.create_stream(content); rtp_datagram.datagram_received.connect(this.stream.on_recv_rtp_data); rtcp_datagram.datagram_received.connect(this.stream.on_recv_rtcp_data); @@ -118,6 +138,20 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } agreed_payload_type = preferred_payload_type; + Gee.List crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto"); + if (crypto_nodes.size == 0) { + warning("Counterpart didn't include any cryptos"); + if (encryption_required) { + return; + } + } else { + Crypto preferred_crypto = Crypto.parse(crypto_nodes[0]); + if (local_crypto.crypto_suite != preferred_crypto.crypto_suite) { + warning("Counterpart's crypto suite doesn't match any of our sent ones"); + } + remote_crypto = preferred_crypto; + } + accept(stream, session, content); } @@ -137,6 +171,10 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ret.put_node(payload_type.to_xml()); } } + if (local_crypto != null) { + ret.put_node(new StanzaNode.build("encryption", NS_URI) + .put_node(local_crypto.to_xml())); + } return ret; } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 35e03168..23aee6c9 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -20,6 +20,9 @@ public abstract class Module : XmppStreamModule { public abstract async Gee.List get_supported_payloads(string media); public abstract async PayloadType? pick_payload_type(string media, Gee.List payloads); + public abstract Crypto? generate_local_crypto(); + public abstract Crypto? pick_remote_crypto(Gee.List cryptos); + public abstract Crypto? pick_local_crypto(Crypto? remote); public abstract Stream create_stream(Jingle.Content content); public abstract void close_stream(Stream stream); @@ -36,6 +39,7 @@ public abstract class Module : XmppStreamModule { // Create audio content Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); + audio_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (audio_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); @@ -52,6 +56,7 @@ public abstract class Module : XmppStreamModule { if (video) { // Create video content Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + video_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -92,6 +97,7 @@ public abstract class Module : XmppStreamModule { if (content == null) { // Content for video does not yet exist -> create it Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); + video_content_parameters.local_crypto = generate_local_crypto(); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -148,26 +154,130 @@ public abstract class Module : XmppStreamModule { } public class Crypto { - public string cryptoSuite { get; private set; } - public string keyParams { get; private set; } - public string? sessionParams { get; private set; } - public string? tag { get; private set; } + public const string AES_CM_128_HMAC_SHA1_80 = "AES_CM_128_HMAC_SHA1_80"; + public const string AES_CM_128_HMAC_SHA1_32 = "AES_CM_128_HMAC_SHA1_32"; + public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; + + public string crypto_suite { get; private set; } + public string key_params { get; private set; } + public string? session_params { get; private set; } + public string tag { get; private set; } + + public uint8[] key_and_salt { owned get { + if (!key_params.has_prefix("inline:")) return null; + int endIndex = key_params.index_of("|"); + if (endIndex < 0) endIndex = key_params.length; + string sub = key_params.substring(7, endIndex - 7); + return Base64.decode(sub); + }} + + public string? lifetime { owned get { + if (!key_params.has_prefix("inline:")) return null; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return null; + int endIndex = key_params.index_of("|", firstIndex + 1); + if (endIndex < 0) { + if (key_params.index_of(":", firstIndex) > 0) return null; // Is MKI + endIndex = key_params.length; + } + return key_params.substring(firstIndex + 1, endIndex); + }} + + public int mki { get { + if (!key_params.has_prefix("inline:")) return -1; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return -1; + int splitIndex = key_params.index_of(":", firstIndex); + if (splitIndex < 0) return -1; + int secondIndex = key_params.index_of("|", firstIndex + 1); + if (secondIndex < 0) { + return int.parse(key_params.substring(firstIndex + 1, splitIndex)); + } else if (splitIndex > secondIndex) { + return int.parse(key_params.substring(secondIndex + 1, splitIndex)); + } + return -1; + }} + + public int mki_length { get { + if (!key_params.has_prefix("inline:")) return -1; + int firstIndex = key_params.index_of("|"); + if (firstIndex < 0) return -1; + int splitIndex = key_params.index_of(":", firstIndex); + if (splitIndex < 0) return -1; + int secondIndex = key_params.index_of("|", firstIndex + 1); + if (secondIndex < 0 || splitIndex > secondIndex) { + return int.parse(key_params.substring(splitIndex + 1, key_params.length)); + } + return -1; + }} + + public bool is_valid { get { + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + return key_and_salt.length == 30; + } + return false; + }} + + public uint8[] key { owned get { + uint8[] key_and_salt = key_and_salt; + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + if (key_and_salt.length >= 16) return key_and_salt[0:16]; + break; + } + return null; + }} + + public uint8[] salt { owned get { + uint8[] keyAndSalt = key_and_salt; + switch(crypto_suite) { + case AES_CM_128_HMAC_SHA1_80: + case AES_CM_128_HMAC_SHA1_32: + case F8_128_HMAC_SHA1_80: + if (keyAndSalt.length >= 30) return keyAndSalt[16:30]; + break; + } + return null; + }} + + public static Crypto create(string crypto_suite, uint8[] key_and_salt, string? session_params = null, string tag = "1") { + Crypto crypto = new Crypto(); + crypto.crypto_suite = crypto_suite; + crypto.key_params = "inline:" + Base64.encode(key_and_salt); + crypto.session_params = session_params; + crypto.tag = tag; + return crypto; + } + + public Crypto rekey(uint8[] key_and_salt) { + Crypto crypto = new Crypto(); + crypto.crypto_suite = crypto_suite; + crypto.key_params = "inline:" + Base64.encode(key_and_salt); + crypto.session_params = session_params; + crypto.tag = tag; + return crypto; + } public static Crypto parse(StanzaNode node) { Crypto crypto = new Crypto(); - crypto.cryptoSuite = node.get_attribute("crypto-suite"); - crypto.keyParams = node.get_attribute("key-params"); - crypto.sessionParams = node.get_attribute("session-params"); + crypto.crypto_suite = node.get_attribute("crypto-suite"); + crypto.key_params = node.get_attribute("key-params"); + crypto.session_params = node.get_attribute("session-params"); crypto.tag = node.get_attribute("tag"); return crypto; } public StanzaNode to_xml() { StanzaNode node = new StanzaNode.build("crypto", NS_URI) - .put_attribute("crypto-suite", cryptoSuite) - .put_attribute("key-params", keyParams); - if (sessionParams != null) node.put_attribute("session-params", sessionParams); - if (tag != null) node.put_attribute("tag", tag); + .put_attribute("crypto-suite", crypto_suite) + .put_attribute("key-params", key_params) + .put_attribute("tag", tag); + if (session_params != null) node.put_attribute("session-params", session_params); return node; } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index 62d85dec..2fc29291 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -17,6 +17,20 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { } return null; }} + public JingleRtp.Crypto? local_crypto { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).local_crypto; + } + return null; + }} + public JingleRtp.Crypto? remote_crypto { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).remote_crypto; + } + return null; + }} public bool sending { get { return content.session.senders_include_us(content.senders); }} -- cgit v1.2.3-70-g09d2 From b01f6f9ef7c0e8b0a15149426a47a238a5f6f1ce Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 15:09:06 +0100 Subject: Resample audio data for common 48k sample rate --- plugins/rtp/src/codec_util.vala | 10 ++++------ plugins/rtp/src/device.vala | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index e419b5ad..6bd465c1 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -202,7 +202,8 @@ public class Dino.Plugins.Rtp.CodecUtil { string decode = element_name ?? get_decode_element_name(media, codec); if (depay == null || decode == null) return null; string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; - return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert"; + string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; + return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { @@ -224,11 +225,8 @@ public class Dino.Plugins.Rtp.CodecUtil { if (pay == null || encode == null) return null; string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; - if (media == "audio") { - return @"audioconvert name=$base_name-convert ! audioresample name=$base_name-resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; - } else { - return @"$(media)convert name=$base_name-convert ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; - } + string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; + return @"$(media)convert name=$base_name-convert$resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 570c6667..20762f77 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -59,7 +59,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (element == null) create(); links++; if (mixer != null) return mixer; - if (is_sink && media == "audio") return plugin.echoprobe; + if (is_sink && media == "audio") return filter; return element; } @@ -146,9 +146,10 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } if (is_sink && media == "audio") { -// mixer = Gst.ElementFactory.make("audiomixer", @"$id-mixer"); -// pipe.add(mixer); -// mixer.link(plugin.echoprobe); + filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + filter.link(plugin.echoprobe); plugin.echoprobe.link(element); } plugin.unpause(); @@ -173,6 +174,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.remove(mixer); mixer = null; } else if (is_sink && media == "audio") { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(plugin.echoprobe); + pipe.remove(filter); + filter = null; + } plugin.echoprobe.unlink(element); } element.set_locked_state(true); -- cgit v1.2.3-70-g09d2 From 4b230808b9566322fae8d1ef0d1a5cb3e8027d3b Mon Sep 17 00:00:00 2001 From: Marvin W Date: Tue, 23 Mar 2021 20:04:28 +0100 Subject: Move SRTP implementation into crypto library for reuse --- plugins/crypto-vala/CMakeLists.txt | 27 +- plugins/crypto-vala/src/srtp.c | 836 +++++++++++++++++++++++++++++++++++++ plugins/crypto-vala/src/srtp.h | 82 ++++ plugins/crypto-vala/src/srtp.vapi | 107 +++++ plugins/rtp/CMakeLists.txt | 5 +- plugins/rtp/src/srtp.c | 836 ------------------------------------- plugins/rtp/src/srtp.h | 82 ---- plugins/rtp/src/srtp.vapi | 103 ----- plugins/rtp/src/stream.vala | 20 +- 9 files changed, 1063 insertions(+), 1035 deletions(-) create mode 100644 plugins/crypto-vala/src/srtp.c create mode 100644 plugins/crypto-vala/src/srtp.h create mode 100644 plugins/crypto-vala/src/srtp.vapi delete mode 100644 plugins/rtp/src/srtp.c delete mode 100644 plugins/rtp/src/srtp.h delete mode 100644 plugins/rtp/src/srtp.vapi (limited to 'plugins/rtp/src') diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 036e45f6..f97b0d31 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -11,6 +11,7 @@ SOURCES "src/cipher_converter.vala" "src/error.vala" "src/random.vala" + "src/srtp.vapi" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" PACKAGES @@ -21,9 +22,33 @@ GENERATE_HEADER crypto-vala ) +add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/exports/srtp.h" +COMMAND + cp "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" "${CMAKE_BINARY_DIR}/exports/srtp.h" +DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" +COMMENT + Copy header file srtp.h +) + +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/exports/crypto.vapi +COMMAND + cat "${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi" "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi" > "${CMAKE_BINARY_DIR}/exports/crypto.vapi" +DEPENDS + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi +) + +add_custom_target(crypto-vapi +DEPENDS + ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/srtp.h +) + set(CFLAGS ${VALA_CFLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_definitions(${CFLAGS}) -add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) +add_library(crypto-vala STATIC ${CRYPTO_VALA_C} src/srtp.c) +add_dependencies(crypto-vala crypto-vapi) target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/crypto-vala/src/srtp.c b/plugins/crypto-vala/src/srtp.c new file mode 100644 index 00000000..708244d9 --- /dev/null +++ b/plugins/crypto-vala/src/srtp.c @@ -0,0 +1,836 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* TODO: + * Useless stuff (because nothing depends on it): + * - non-nul key derivation rate + * - MKI payload + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include +#include + +#include "srtp.h" + +#include +#include +#include +#include + +#include + +#ifdef _WIN32 +# include +#else +# include +#endif + +#define debug( ... ) (void)0 + +typedef struct srtp_proto_t +{ + gcry_cipher_hd_t cipher; + gcry_md_hd_t mac; + uint64_t window; + uint32_t salt[4]; +} srtp_proto_t; + +struct srtp_session_t +{ + srtp_proto_t rtp; + srtp_proto_t rtcp; + unsigned flags; + unsigned kdr; + uint32_t rtcp_index; + uint32_t rtp_roc; + uint16_t rtp_seq; + uint16_t rtp_rcc; + uint8_t tag_len; +}; + +enum +{ + SRTP_CRYPT, + SRTP_AUTH, + SRTP_SALT, + SRTCP_CRYPT, + SRTCP_AUTH, + SRTCP_SALT +}; + + +static inline unsigned rcc_mode (const srtp_session_t *s) +{ + return (s->flags >> 4) & 3; +} + + +static void proto_destroy (srtp_proto_t *p) +{ + gcry_md_close (p->mac); + gcry_cipher_close (p->cipher); +} + + +/** + * Releases all resources associated with a Secure RTP session. + */ +void srtp_destroy (srtp_session_t *s) +{ + assert (s != NULL); + + proto_destroy (&s->rtcp); + proto_destroy (&s->rtp); + free (s); +} + + +static int proto_create (srtp_proto_t *p, int gcipher, int gmd) +{ + if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) + { + if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) + return 0; + gcry_cipher_close (p->cipher); + } + return -1; +} + + +/** + * Allocates a Secure RTP one-way session. + * The same session cannot be used both ways because this would confuse + * internal cryptographic counters; it is however of course feasible to open + * multiple simultaneous sessions with the same master key. + * + * @param encr encryption algorithm number + * @param auth authentication algortihm number + * @param tag_len authentication tag byte length (NOT including RCC) + * @param flags OR'ed optional flags. + * + * @return NULL in case of error + */ +srtp_session_t * +srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) +{ + if ((flags & ~SRTP_FLAGS_MASK)) + return NULL; + + int cipher, md; + switch (encr) + { + case SRTP_ENCR_NULL: + cipher = GCRY_CIPHER_NONE; + break; + + case SRTP_ENCR_AES_CM: + cipher = GCRY_CIPHER_AES; + break; + + default: + return NULL; + } + + switch (auth) + { + case SRTP_AUTH_NULL: + md = GCRY_MD_NONE; + break; + + case SRTP_AUTH_HMAC_SHA1: + md = GCRY_MD_SHA1; + break; + + default: + return NULL; + } + + if (tag_len > gcry_md_get_algo_dlen (md)) + return NULL; + + if (prf != SRTP_PRF_AES_CM) + return NULL; + + srtp_session_t *s = malloc (sizeof (*s)); + if (s == NULL) + return NULL; + + memset (s, 0, sizeof (*s)); + s->flags = flags; + s->tag_len = tag_len; + s->rtp_rcc = 1; /* Default RCC rate */ + if (rcc_mode (s)) + { + if (tag_len < 4) + goto error; + } + + if (proto_create (&s->rtp, cipher, md) == 0) + { + if (proto_create (&s->rtcp, cipher, md) == 0) + return s; + proto_destroy (&s->rtp); + } + + error: + free (s); + return NULL; +} + + +/** + * Counter Mode encryption/decryption (ctr length = 16 bytes) + * with non-padded (truncated) text + */ +static int +do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) +{ + const size_t ctrlen = 16; + div_t d = div (len, ctrlen); + + if (gcry_cipher_setctr (hd, ctr, ctrlen) + || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) + return -1; + + if (d.rem) + { + /* Truncated last block */ + uint8_t dummy[ctrlen]; + data += d.quot * ctrlen; + memcpy (dummy, data, d.rem); + memset (dummy + d.rem, 0, ctrlen - d.rem); + + if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) + return -1; + memcpy (data, dummy, d.rem); + } + + return 0; +} + + +/** + * AES-CM key derivation (saltlen = 14 bytes) + */ +static int +do_derive (gcry_cipher_hd_t prf, const void *salt, + const uint8_t *r, size_t rlen, uint8_t label, + void *out, size_t outlen) +{ + uint8_t iv[16]; + + memcpy (iv, salt, 14); + iv[14] = iv[15] = 0; + + assert (rlen < 14); + iv[13 - rlen] ^= label; + for (size_t i = 0; i < rlen; i++) + iv[sizeof (iv) - rlen + i] ^= r[i]; + + memset (out, 0, outlen); + return do_ctr_crypt (prf, iv, out, outlen); +} + + +/** + * Sets (or resets) the master key and master salt for a SRTP session. + * This must be done at least once before using srtp_send(), srtp_recv(), + * srtcp_send() or srtcp_recv(). Also, rekeying is required every + * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), + * otherwise the protocol security might be broken. + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen) +{ + /* SRTP/SRTCP cipher/salt/MAC keys derivation */ + gcry_cipher_hd_t prf; + uint8_t r[6], keybuf[20]; + + if (saltlen != 14) + return EINVAL; + + if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) + || gcry_cipher_setkey (prf, key, keylen)) + return EINVAL; + + /* SRTP key derivation */ +#if 0 + if (s->kdr != 0) + { + uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; + index /= s->kdr; + + for (int i = sizeof (r) - 1; i >= 0; i--) + { + r[i] = index & 0xff; + index = index >> 8; + } + } + else +#endif + memset (r, 0, sizeof (r)); + if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtp.mac, keybuf, 20) + || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) + return -1; + + /* SRTCP key derivation */ + memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); + if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) + || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) + || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) + || gcry_md_setkey (s->rtcp.mac, keybuf, 20) + || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) + return -1; + + (void)gcry_cipher_close (prf); + return 0; +} + +static int hexdigit (char c) +{ + if ((c >= '0') && (c <= '9')) + return c - '0'; + if ((c >= 'A') && (c <= 'F')) + return c - 'A' + 0xA; + if ((c >= 'a') && (c <= 'f')) + return c - 'a' + 0xa; + return -1; +} + +static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) +{ + size_t inlen = strlen (in); + + if ((inlen > (2 * outlen)) || (inlen & 1)) + return -1; + + for (size_t i = 0; i < inlen; i += 2) + { + int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); + if ((a == -1) || (b == -1)) + return -1; + out[i / 2] = (a << 4) | b; + } + return inlen / 2; +} + +/** + * Sets (or resets) the master key and master salt for a SRTP session + * from hexadecimal strings. See also srtp_setkey(). + * + * @return 0 on success, in case of error: + * EINVAL invalid or unsupported key/salt sizes combination + */ +int +srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) +{ + uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ + uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ + ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); + ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); + + if ((bkeylen == -1) || (bsaltlen == -1)) + return EINVAL; + return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; +} + +/** + * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not + * specified (through this function), the default rate of ONE is assumed + * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none + * of the RCC mode has been selected. + * + * The RCC mode is selected through one of these flags for srtp_create(): + * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets + * SRTP_RCC_MODE2: integrity protection for all packets + * SRTP_RCC_MODE3: no integrity protection + * + * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality + * (through encryption) but is much more prone to DoS. It can only be used if + * anti-spoofing protection is provided by lower network layers (e.g. IPsec, + * or trusted routers and proper source address filtering). + * + * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. + * + * @param rate RoC Carry rate (MUST NOT be zero) + */ +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) +{ + assert (rate != 0); + s->rtp_rcc = rate; +} + + +/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ +static int +rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, + const uint32_t *salt, uint8_t *data, size_t len) +{ + /* Determines cryptographic counter (IV) */ + uint32_t counter[4]; + counter[0] = salt[0]; + counter[1] = salt[1] ^ ssrc; + counter[2] = salt[2] ^ htonl (roc); + counter[3] = salt[3] ^ htonl (seq << 16); + + /* Encryption */ + return do_ctr_crypt (hd, counter, data, len); +} + + +/** Determines SRTP Roll-Over-Counter (in host-byte order) */ +static uint32_t +srtp_compute_roc (const srtp_session_t *s, uint16_t seq) +{ + uint32_t roc = s->rtp_roc; + + if (((seq - s->rtp_seq) & 0xffff) < 0x8000) + { + /* Sequence is ahead, good */ + if (seq < s->rtp_seq) + roc++; /* Sequence number wrap */ + } + else + { + /* Sequence is late, bad */ + if (seq > s->rtp_seq) + roc--; /* Wrap back */ + } + return roc; +} + + +/** Returns RTP sequence (in host-byte order) */ +static inline uint16_t rtp_seq (const uint8_t *buf) +{ + return (buf[2] << 8) | buf[3]; +} + + +/** Message Authentication and Integrity for RTP */ +static const uint8_t * +rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, + uint32_t roc) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTP packet and updates SRTP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTP packet to be en-/decrypted + * @param len RTP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet + * EACCES replayed packet or out-of-window or sync lost + */ +static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + assert (len >= 12u); + + if ((buf[0] >> 6) != 2) + return EINVAL; + + /* Computes encryption offset */ + uint16_t offset = 12; + offset += (buf[0] & 0xf) * 4; // skips CSRC + + if (buf[0] & 0x10) + { + uint16_t extlen; + + offset += 4; + if (len < offset) + return EINVAL; + + memcpy (&extlen, buf + offset - 2, 2); + offset += htons (extlen); // skips RTP extension header + } + + if (len < offset) + return EINVAL; + + /* Determines RTP 48-bits counter and SSRC */ + uint16_t seq = rtp_seq (buf); + uint32_t roc = srtp_compute_roc (s, seq), ssrc; + memcpy (&ssrc, buf + 8, 4); + + /* Updates ROC and sequence (it's safe now) */ + int16_t diff = seq - s->rtp_seq; + if (diff > 0) + { + /* Sequence in the future, good */ + s->rtp.window = s->rtp.window << diff; + s->rtp.window |= UINT64_C(1); + s->rtp_seq = seq, s->rtp_roc = roc; + } + else + { + /* Sequence in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) + return EACCES; /* Replay attack */ + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Encrypt/Decrypt */ + if (s->flags & SRTP_UNENCRYPTED) + return 0; + + if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, + buf + offset, len - offset)) + return EINVAL; + + return 0; +} + + +/** + * Turns a RTP packet into a SRTP packet: encrypt it, then computes + * the authentication tag and appends it. + * Note that you can encrypt packet in disorder. + * + * @param buf RTP packet to be encrypted/digested + * @param lenp pointer to the RTP packet length on entry, + * set to the SRTP length on exit (undefined on non-ENOSPC error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTP packet or internal error + * ENOSPC bufsize is too small to add authentication tag + * ( will hold the required byte size) + * EACCES packet would trigger a replay error on receiver + */ +int +srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + size_t tag_len; + size_t roc_len = 0; + + /* Compute required buffer size */ + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + tag_len = s->tag_len; + + if (rcc_mode (s)) + { + assert (tag_len >= 4); + assert (s->rtp_rcc != 0); + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; /* RCC mode 3 -> no auth*/ + else + tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; /* RCC mode 1 or 3 -> no auth */ + } + } + + *lenp = len + roc_len + tag_len; + } + else + tag_len = 0; + + if (bufsize < *lenp) + return ENOSPC; + + /* Encrypt payload */ + int val = srtp_crypt (s, buf, len); + if (val) + return val; + + /* Authenticate payload */ + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); + + if (roc_len) + { + memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); + len += 4; + } + memcpy (buf + len, tag, tag_len); +#if 0 + printf ("Sent : 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + puts (""); +#endif + } + + return 0; +} + + +/** + * Turns a SRTP packet into a RTP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTP packet to be digested/decrypted + * @param lenp pointer to the SRTP packet length on entry, + * set to the RTP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + if (len < 12u) + return EINVAL; + + if (!(s->flags & SRTP_UNAUTHENTICATED)) + { + size_t tag_len = s->tag_len, roc_len = 0; + if (rcc_mode (s)) + { + if ((rtp_seq (buf) % s->rtp_rcc) == 0) + { + roc_len = 4; + if (rcc_mode (s) == 3) + tag_len = 0; + else + tag_len -= 4; + } + else + { + if (rcc_mode (s) & 1) + tag_len = 0; // RCC mode 1 or 3: no auth + } + } + + if (len < (12u + roc_len + tag_len)) + return EINVAL; + len -= roc_len + tag_len; + + uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; + if (roc_len) + { + assert (roc_len == 4); + memcpy (&rcc, buf + len, 4); + rcc = ntohl (rcc); + } + else + rcc = roc; + + const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); +#if 0 + printf ("Computed: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", tag[i]); + printf ("\nReceived: 0x"); + for (unsigned i = 0; i < tag_len; i++) + printf ("%02x", buf[len + roc_len + i]); + puts (""); +#endif + if (memcmp (buf + len + roc_len, tag, tag_len)) + return EACCES; + + if (roc_len) + { + /* Authenticated packet carried a Roll-Over-Counter */ + s->rtp_roc += rcc - roc; + assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); + } + *lenp = len; + } + + return srtp_crypt (s, buf, len); +} + + +/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ +static int +rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, + const uint32_t *salt, uint8_t *data, size_t len) +{ + return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); +} + + +/** Message Authentication and Integrity for RTCP */ +static const uint8_t * +rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) +{ + gcry_md_reset (md); + gcry_md_write (md, data, len); + return gcry_md_read (md, 0); +} + + +/** + * Encrypts/decrypts a RTCP packet and updates SRTCP context + * (CTR block cypher mode of operation has identical encryption and + * decryption function). + * + * @param buf RTCP packet to be en-/decrypted + * @param len RTCP packet length + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet + */ +static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) +{ + assert (s != NULL); + + /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ + if ((len < 12) || ((buf[0] >> 6) != 2)) + return EINVAL; + + uint32_t index; + memcpy (&index, buf + len, 4); + index = ntohl (index); + if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) + return EINVAL; // E-bit mismatch + + index &= ~(1 << 31); // clear E-bit for counter + + /* Updates SRTCP index (safe here) */ + int32_t diff = index - s->rtcp_index; + if (diff > 0) + { + /* Packet in the future, good */ + s->rtcp.window = s->rtcp.window << diff; + s->rtcp.window |= UINT64_C(1); + s->rtcp_index = index; + } + else + { + /* Packet in the past/present, bad */ + diff = -diff; + if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) + return EACCES; // replay attack! + s->rtp.window |= UINT64_C(1) << diff; + } + + /* Crypts SRTCP */ + if (s->flags & SRTCP_UNENCRYPTED) + return 0; + + uint32_t ssrc; + memcpy (&ssrc, buf + 4, 4); + + if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, + buf + 8, len - 8)) + return EINVAL; + return 0; +} + + +/** + * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes + * the authentication tag and appends it. + * + * @param buf RTCP packet to be encrypted/digested + * @param lenp pointer to the RTCP packet length on entry, + * set to the SRTCP length on exit (undefined in case of error) + * @param bufsize size (bytes) of the packet buffer + * + * @return 0 on success, in case of error: + * EINVAL malformatted RTCP packet or internal error + * ENOSPC bufsize is too small (to add index and authentication tag) + */ +int +srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) +{ + size_t len = *lenp; + if (bufsize < (len + 4 + s->tag_len)) + return ENOSPC; + + uint32_t index = ++s->rtcp_index; + if (index >> 31) + s->rtcp_index = index = 0; /* 31-bit wrap */ + + if ((s->flags & SRTCP_UNENCRYPTED) == 0) + index |= 0x80000000; /* Set Encrypted bit */ + memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); + + int val = srtcp_crypt (s, buf, len); + if (val) + return val; + + len += 4; /* Digests SRTCP index too */ + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + memcpy (buf + len, tag, s->tag_len); + *lenp = len + s->tag_len; + return 0; +} + + +/** + * Turns a SRTCP packet into a RTCP packet: authenticates the packet, + * then decrypts it. + * + * @param buf RTCP packet to be digested/decrypted + * @param lenp pointer to the SRTCP packet length on entry, + * set to the RTCP length on exit (undefined in case of error) + * + * @return 0 on success, in case of error: + * EINVAL malformatted SRTCP packet + * EACCES authentication failed (spoofed packet or out-of-sync) + */ +int +srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) +{ + size_t len = *lenp; + + if (len < (4u + s->tag_len)) + return EINVAL; + len -= s->tag_len; + + const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); + if (memcmp (buf + len, tag, s->tag_len)) + return EACCES; + + len -= 4; /* Remove SRTCP index before decryption */ + *lenp = len; + return srtcp_crypt (s, buf, len); +} \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.h b/plugins/crypto-vala/src/srtp.h new file mode 100644 index 00000000..abca6988 --- /dev/null +++ b/plugins/crypto-vala/src/srtp.h @@ -0,0 +1,82 @@ +/* + * Secure RTP with libgcrypt + * Copyright (C) 2007 Rémi Denis-Courmont + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation; either version 2.1 + * of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + ****************************************************************************/ + +#ifndef LIBVLC_SRTP_H +# define LIBVLC_SRTP_H 1 +#include + +typedef struct srtp_session_t srtp_session_t; + +enum +{ + SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets + SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets + SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets + + SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 + SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 + SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) + + SRTP_FLAGS_MASK=0x37 //< mask for valid flags +}; + +/** SRTP encryption algorithms (ciphers); same values as MIKEY */ +enum +{ + SRTP_ENCR_NULL=0, //< no encryption + SRTP_ENCR_AES_CM=1, //< AES counter mode + SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) +}; + +/** SRTP authenticaton algorithms; same values as MIKEY */ +enum +{ + SRTP_AUTH_NULL=0, //< no authentication code + SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 +}; + +/** SRTP pseudo random function; same values as MIKEY */ +enum +{ + SRTP_PRF_AES_CM=0, //< AES counter mode +}; + +# ifdef __cplusplus +extern "C" { +# endif + +srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, + unsigned flags); +void srtp_destroy (srtp_session_t *s); + +int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, + const void *salt, size_t saltlen); +int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); + +void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); + +int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); +int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); +int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); +int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); + +# ifdef __cplusplus +} +# endif +#endif \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vapi b/plugins/crypto-vala/src/srtp.vapi new file mode 100644 index 00000000..0fe825c3 --- /dev/null +++ b/plugins/crypto-vala/src/srtp.vapi @@ -0,0 +1,107 @@ +[CCode (cheader_filename="srtp.h")] +namespace Crypto.Srtp { + +[Compact] +[CCode (cname = "srtp_session_t", free_function = "srtp_destroy")] +public class Session { + [CCode (cname = "srtp_create")] + public Session(Encryption encr, Authentication auth, uint tag_len, Prf prf, Flags flags); + [CCode (cname = "srtp_setkey")] + public int setkey(uint8[] key, uint8[] salt); + [CCode (cname = "srtp_setkeystring")] + public int setkeystring(string key, string salt); + [CCode (cname = "srtp_setrcc_rate")] + public void setrcc_rate(uint16 rate); + + [CCode (cname = "srtp_send")] + private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtcp_send")] + private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); + [CCode (cname = "srtp_recv")] + private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + [CCode (cname = "srtcp_recv")] + private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); + + public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length + tag_len]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { + uint8[] buf = new uint8[input.length + tag_len + 4]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_send(buf, ref buf_use, buf.length); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP encrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { + uint8[] buf = new uint8[input.length]; + GLib.Memory.copy(buf, input, input.length); + size_t buf_use = input.length; + int res = rtcp_recv(buf, ref buf_use); + if (res != 0) { + throw new GLib.Error(-1, res, "RTCP decrypt failed"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } +} + +[Flags] +[CCode (cname = "unsigned", cprefix = "", has_type_id = false)] +public enum Flags { + SRTP_UNENCRYPTED, + SRTCP_UNENCRYPTED, + SRTP_UNAUTHENTICATED, + SRTP_RCC_MODE1, + SRTP_RCC_MODE2, + SRTP_RCC_MODE3 +} + +[CCode (cname = "int", cprefix = "SRTP_ENCR_", has_type_id = false)] +public enum Encryption { + NULL, + AES_CM, + AES_F8 +} + +[CCode (cname = "int", cprefix = "SRTP_AUTH_", has_type_id = false)] +public enum Authentication { + NULL, + HMAC_SHA1 +} + +[CCode (cname = "int", cprefix = "SRTP_PRF_", has_type_id = false)] +public enum Prf { + AES_CM +} + +} \ No newline at end of file diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index ef2f7698..5311fac3 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -18,11 +18,10 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS - ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_BINARY_DIR}/exports/crypto.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi - ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi PACKAGES ${RTP_PACKAGES} OPTIONS @@ -30,7 +29,7 @@ OPTIONS ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) -add_library(rtp SHARED ${RTP_VALA_C} src/srtp.c) +add_library(rtp SHARED ${RTP_VALA_C}) target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/srtp.c b/plugins/rtp/src/srtp.c deleted file mode 100644 index 708244d9..00000000 --- a/plugins/rtp/src/srtp.c +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/* TODO: - * Useless stuff (because nothing depends on it): - * - non-nul key derivation rate - * - MKI payload - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include - -#include "srtp.h" - -#include -#include -#include -#include - -#include - -#ifdef _WIN32 -# include -#else -# include -#endif - -#define debug( ... ) (void)0 - -typedef struct srtp_proto_t -{ - gcry_cipher_hd_t cipher; - gcry_md_hd_t mac; - uint64_t window; - uint32_t salt[4]; -} srtp_proto_t; - -struct srtp_session_t -{ - srtp_proto_t rtp; - srtp_proto_t rtcp; - unsigned flags; - unsigned kdr; - uint32_t rtcp_index; - uint32_t rtp_roc; - uint16_t rtp_seq; - uint16_t rtp_rcc; - uint8_t tag_len; -}; - -enum -{ - SRTP_CRYPT, - SRTP_AUTH, - SRTP_SALT, - SRTCP_CRYPT, - SRTCP_AUTH, - SRTCP_SALT -}; - - -static inline unsigned rcc_mode (const srtp_session_t *s) -{ - return (s->flags >> 4) & 3; -} - - -static void proto_destroy (srtp_proto_t *p) -{ - gcry_md_close (p->mac); - gcry_cipher_close (p->cipher); -} - - -/** - * Releases all resources associated with a Secure RTP session. - */ -void srtp_destroy (srtp_session_t *s) -{ - assert (s != NULL); - - proto_destroy (&s->rtcp); - proto_destroy (&s->rtp); - free (s); -} - - -static int proto_create (srtp_proto_t *p, int gcipher, int gmd) -{ - if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) - { - if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) - return 0; - gcry_cipher_close (p->cipher); - } - return -1; -} - - -/** - * Allocates a Secure RTP one-way session. - * The same session cannot be used both ways because this would confuse - * internal cryptographic counters; it is however of course feasible to open - * multiple simultaneous sessions with the same master key. - * - * @param encr encryption algorithm number - * @param auth authentication algortihm number - * @param tag_len authentication tag byte length (NOT including RCC) - * @param flags OR'ed optional flags. - * - * @return NULL in case of error - */ -srtp_session_t * -srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) -{ - if ((flags & ~SRTP_FLAGS_MASK)) - return NULL; - - int cipher, md; - switch (encr) - { - case SRTP_ENCR_NULL: - cipher = GCRY_CIPHER_NONE; - break; - - case SRTP_ENCR_AES_CM: - cipher = GCRY_CIPHER_AES; - break; - - default: - return NULL; - } - - switch (auth) - { - case SRTP_AUTH_NULL: - md = GCRY_MD_NONE; - break; - - case SRTP_AUTH_HMAC_SHA1: - md = GCRY_MD_SHA1; - break; - - default: - return NULL; - } - - if (tag_len > gcry_md_get_algo_dlen (md)) - return NULL; - - if (prf != SRTP_PRF_AES_CM) - return NULL; - - srtp_session_t *s = malloc (sizeof (*s)); - if (s == NULL) - return NULL; - - memset (s, 0, sizeof (*s)); - s->flags = flags; - s->tag_len = tag_len; - s->rtp_rcc = 1; /* Default RCC rate */ - if (rcc_mode (s)) - { - if (tag_len < 4) - goto error; - } - - if (proto_create (&s->rtp, cipher, md) == 0) - { - if (proto_create (&s->rtcp, cipher, md) == 0) - return s; - proto_destroy (&s->rtp); - } - - error: - free (s); - return NULL; -} - - -/** - * Counter Mode encryption/decryption (ctr length = 16 bytes) - * with non-padded (truncated) text - */ -static int -do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) -{ - const size_t ctrlen = 16; - div_t d = div (len, ctrlen); - - if (gcry_cipher_setctr (hd, ctr, ctrlen) - || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) - return -1; - - if (d.rem) - { - /* Truncated last block */ - uint8_t dummy[ctrlen]; - data += d.quot * ctrlen; - memcpy (dummy, data, d.rem); - memset (dummy + d.rem, 0, ctrlen - d.rem); - - if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) - return -1; - memcpy (data, dummy, d.rem); - } - - return 0; -} - - -/** - * AES-CM key derivation (saltlen = 14 bytes) - */ -static int -do_derive (gcry_cipher_hd_t prf, const void *salt, - const uint8_t *r, size_t rlen, uint8_t label, - void *out, size_t outlen) -{ - uint8_t iv[16]; - - memcpy (iv, salt, 14); - iv[14] = iv[15] = 0; - - assert (rlen < 14); - iv[13 - rlen] ^= label; - for (size_t i = 0; i < rlen; i++) - iv[sizeof (iv) - rlen + i] ^= r[i]; - - memset (out, 0, outlen); - return do_ctr_crypt (prf, iv, out, outlen); -} - - -/** - * Sets (or resets) the master key and master salt for a SRTP session. - * This must be done at least once before using srtp_send(), srtp_recv(), - * srtcp_send() or srtcp_recv(). Also, rekeying is required every - * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), - * otherwise the protocol security might be broken. - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen) -{ - /* SRTP/SRTCP cipher/salt/MAC keys derivation */ - gcry_cipher_hd_t prf; - uint8_t r[6], keybuf[20]; - - if (saltlen != 14) - return EINVAL; - - if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) - || gcry_cipher_setkey (prf, key, keylen)) - return EINVAL; - - /* SRTP key derivation */ -#if 0 - if (s->kdr != 0) - { - uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; - index /= s->kdr; - - for (int i = sizeof (r) - 1; i >= 0; i--) - { - r[i] = index & 0xff; - index = index >> 8; - } - } - else -#endif - memset (r, 0, sizeof (r)); - if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtp.mac, keybuf, 20) - || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) - return -1; - - /* SRTCP key derivation */ - memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); - if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtcp.mac, keybuf, 20) - || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) - return -1; - - (void)gcry_cipher_close (prf); - return 0; -} - -static int hexdigit (char c) -{ - if ((c >= '0') && (c <= '9')) - return c - '0'; - if ((c >= 'A') && (c <= 'F')) - return c - 'A' + 0xA; - if ((c >= 'a') && (c <= 'f')) - return c - 'a' + 0xa; - return -1; -} - -static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) -{ - size_t inlen = strlen (in); - - if ((inlen > (2 * outlen)) || (inlen & 1)) - return -1; - - for (size_t i = 0; i < inlen; i += 2) - { - int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); - if ((a == -1) || (b == -1)) - return -1; - out[i / 2] = (a << 4) | b; - } - return inlen / 2; -} - -/** - * Sets (or resets) the master key and master salt for a SRTP session - * from hexadecimal strings. See also srtp_setkey(). - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) -{ - uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ - uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ - ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); - ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); - - if ((bkeylen == -1) || (bsaltlen == -1)) - return EINVAL; - return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; -} - -/** - * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not - * specified (through this function), the default rate of ONE is assumed - * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none - * of the RCC mode has been selected. - * - * The RCC mode is selected through one of these flags for srtp_create(): - * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets - * SRTP_RCC_MODE2: integrity protection for all packets - * SRTP_RCC_MODE3: no integrity protection - * - * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality - * (through encryption) but is much more prone to DoS. It can only be used if - * anti-spoofing protection is provided by lower network layers (e.g. IPsec, - * or trusted routers and proper source address filtering). - * - * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. - * - * @param rate RoC Carry rate (MUST NOT be zero) - */ -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) -{ - assert (rate != 0); - s->rtp_rcc = rate; -} - - -/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ -static int -rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, - const uint32_t *salt, uint8_t *data, size_t len) -{ - /* Determines cryptographic counter (IV) */ - uint32_t counter[4]; - counter[0] = salt[0]; - counter[1] = salt[1] ^ ssrc; - counter[2] = salt[2] ^ htonl (roc); - counter[3] = salt[3] ^ htonl (seq << 16); - - /* Encryption */ - return do_ctr_crypt (hd, counter, data, len); -} - - -/** Determines SRTP Roll-Over-Counter (in host-byte order) */ -static uint32_t -srtp_compute_roc (const srtp_session_t *s, uint16_t seq) -{ - uint32_t roc = s->rtp_roc; - - if (((seq - s->rtp_seq) & 0xffff) < 0x8000) - { - /* Sequence is ahead, good */ - if (seq < s->rtp_seq) - roc++; /* Sequence number wrap */ - } - else - { - /* Sequence is late, bad */ - if (seq > s->rtp_seq) - roc--; /* Wrap back */ - } - return roc; -} - - -/** Returns RTP sequence (in host-byte order) */ -static inline uint16_t rtp_seq (const uint8_t *buf) -{ - return (buf[2] << 8) | buf[3]; -} - - -/** Message Authentication and Integrity for RTP */ -static const uint8_t * -rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, - uint32_t roc) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTP packet and updates SRTP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTP packet to be en-/decrypted - * @param len RTP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet - * EACCES replayed packet or out-of-window or sync lost - */ -static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - assert (len >= 12u); - - if ((buf[0] >> 6) != 2) - return EINVAL; - - /* Computes encryption offset */ - uint16_t offset = 12; - offset += (buf[0] & 0xf) * 4; // skips CSRC - - if (buf[0] & 0x10) - { - uint16_t extlen; - - offset += 4; - if (len < offset) - return EINVAL; - - memcpy (&extlen, buf + offset - 2, 2); - offset += htons (extlen); // skips RTP extension header - } - - if (len < offset) - return EINVAL; - - /* Determines RTP 48-bits counter and SSRC */ - uint16_t seq = rtp_seq (buf); - uint32_t roc = srtp_compute_roc (s, seq), ssrc; - memcpy (&ssrc, buf + 8, 4); - - /* Updates ROC and sequence (it's safe now) */ - int16_t diff = seq - s->rtp_seq; - if (diff > 0) - { - /* Sequence in the future, good */ - s->rtp.window = s->rtp.window << diff; - s->rtp.window |= UINT64_C(1); - s->rtp_seq = seq, s->rtp_roc = roc; - } - else - { - /* Sequence in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) - return EACCES; /* Replay attack */ - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Encrypt/Decrypt */ - if (s->flags & SRTP_UNENCRYPTED) - return 0; - - if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, - buf + offset, len - offset)) - return EINVAL; - - return 0; -} - - -/** - * Turns a RTP packet into a SRTP packet: encrypt it, then computes - * the authentication tag and appends it. - * Note that you can encrypt packet in disorder. - * - * @param buf RTP packet to be encrypted/digested - * @param lenp pointer to the RTP packet length on entry, - * set to the SRTP length on exit (undefined on non-ENOSPC error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet or internal error - * ENOSPC bufsize is too small to add authentication tag - * ( will hold the required byte size) - * EACCES packet would trigger a replay error on receiver - */ -int -srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - size_t tag_len; - size_t roc_len = 0; - - /* Compute required buffer size */ - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - tag_len = s->tag_len; - - if (rcc_mode (s)) - { - assert (tag_len >= 4); - assert (s->rtp_rcc != 0); - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; /* RCC mode 3 -> no auth*/ - else - tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; /* RCC mode 1 or 3 -> no auth */ - } - } - - *lenp = len + roc_len + tag_len; - } - else - tag_len = 0; - - if (bufsize < *lenp) - return ENOSPC; - - /* Encrypt payload */ - int val = srtp_crypt (s, buf, len); - if (val) - return val; - - /* Authenticate payload */ - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); - - if (roc_len) - { - memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); - len += 4; - } - memcpy (buf + len, tag, tag_len); -#if 0 - printf ("Sent : 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - puts (""); -#endif - } - - return 0; -} - - -/** - * Turns a SRTP packet into a RTP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTP packet to be digested/decrypted - * @param lenp pointer to the SRTP packet length on entry, - * set to the RTP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - size_t tag_len = s->tag_len, roc_len = 0; - if (rcc_mode (s)) - { - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; - else - tag_len -= 4; - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; // RCC mode 1 or 3: no auth - } - } - - if (len < (12u + roc_len + tag_len)) - return EINVAL; - len -= roc_len + tag_len; - - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; - if (roc_len) - { - assert (roc_len == 4); - memcpy (&rcc, buf + len, 4); - rcc = ntohl (rcc); - } - else - rcc = roc; - - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); -#if 0 - printf ("Computed: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - printf ("\nReceived: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", buf[len + roc_len + i]); - puts (""); -#endif - if (memcmp (buf + len + roc_len, tag, tag_len)) - return EACCES; - - if (roc_len) - { - /* Authenticated packet carried a Roll-Over-Counter */ - s->rtp_roc += rcc - roc; - assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); - } - *lenp = len; - } - - return srtp_crypt (s, buf, len); -} - - -/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ -static int -rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, - const uint32_t *salt, uint8_t *data, size_t len) -{ - return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); -} - - -/** Message Authentication and Integrity for RTCP */ -static const uint8_t * -rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTCP packet and updates SRTCP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTCP packet to be en-/decrypted - * @param len RTCP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet - */ -static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - - /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ - if ((len < 12) || ((buf[0] >> 6) != 2)) - return EINVAL; - - uint32_t index; - memcpy (&index, buf + len, 4); - index = ntohl (index); - if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) - return EINVAL; // E-bit mismatch - - index &= ~(1 << 31); // clear E-bit for counter - - /* Updates SRTCP index (safe here) */ - int32_t diff = index - s->rtcp_index; - if (diff > 0) - { - /* Packet in the future, good */ - s->rtcp.window = s->rtcp.window << diff; - s->rtcp.window |= UINT64_C(1); - s->rtcp_index = index; - } - else - { - /* Packet in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) - return EACCES; // replay attack! - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Crypts SRTCP */ - if (s->flags & SRTCP_UNENCRYPTED) - return 0; - - uint32_t ssrc; - memcpy (&ssrc, buf + 4, 4); - - if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, - buf + 8, len - 8)) - return EINVAL; - return 0; -} - - -/** - * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes - * the authentication tag and appends it. - * - * @param buf RTCP packet to be encrypted/digested - * @param lenp pointer to the RTCP packet length on entry, - * set to the SRTCP length on exit (undefined in case of error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet or internal error - * ENOSPC bufsize is too small (to add index and authentication tag) - */ -int -srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - if (bufsize < (len + 4 + s->tag_len)) - return ENOSPC; - - uint32_t index = ++s->rtcp_index; - if (index >> 31) - s->rtcp_index = index = 0; /* 31-bit wrap */ - - if ((s->flags & SRTCP_UNENCRYPTED) == 0) - index |= 0x80000000; /* Set Encrypted bit */ - memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); - - int val = srtcp_crypt (s, buf, len); - if (val) - return val; - - len += 4; /* Digests SRTCP index too */ - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - memcpy (buf + len, tag, s->tag_len); - *lenp = len + s->tag_len; - return 0; -} - - -/** - * Turns a SRTCP packet into a RTCP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTCP packet to be digested/decrypted - * @param lenp pointer to the SRTCP packet length on entry, - * set to the RTCP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTCP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - - if (len < (4u + s->tag_len)) - return EINVAL; - len -= s->tag_len; - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - if (memcmp (buf + len, tag, s->tag_len)) - return EACCES; - - len -= 4; /* Remove SRTCP index before decryption */ - *lenp = len; - return srtcp_crypt (s, buf, len); -} \ No newline at end of file diff --git a/plugins/rtp/src/srtp.h b/plugins/rtp/src/srtp.h deleted file mode 100644 index abca6988..00000000 --- a/plugins/rtp/src/srtp.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - ****************************************************************************/ - -#ifndef LIBVLC_SRTP_H -# define LIBVLC_SRTP_H 1 -#include - -typedef struct srtp_session_t srtp_session_t; - -enum -{ - SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets - SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets - SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets - - SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 - SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 - SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) - - SRTP_FLAGS_MASK=0x37 //< mask for valid flags -}; - -/** SRTP encryption algorithms (ciphers); same values as MIKEY */ -enum -{ - SRTP_ENCR_NULL=0, //< no encryption - SRTP_ENCR_AES_CM=1, //< AES counter mode - SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) -}; - -/** SRTP authenticaton algorithms; same values as MIKEY */ -enum -{ - SRTP_AUTH_NULL=0, //< no authentication code - SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 -}; - -/** SRTP pseudo random function; same values as MIKEY */ -enum -{ - SRTP_PRF_AES_CM=0, //< AES counter mode -}; - -# ifdef __cplusplus -extern "C" { -# endif - -srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, - unsigned flags); -void srtp_destroy (srtp_session_t *s); - -int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen); -int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); - -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); - -int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); -int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); -int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); -int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); - -# ifdef __cplusplus -} -# endif -#endif \ No newline at end of file diff --git a/plugins/rtp/src/srtp.vapi b/plugins/rtp/src/srtp.vapi deleted file mode 100644 index c5ce7fec..00000000 --- a/plugins/rtp/src/srtp.vapi +++ /dev/null @@ -1,103 +0,0 @@ -[Compact] -[CCode (cname = "srtp_session_t", free_function = "srtp_destroy", cheader_filename="srtp.h")] -public class Dino.Plugins.Rtp.SrtpSession { - [CCode (cname = "srtp_create")] - public SrtpSession(SrtpEncryption encr, SrtpAuthentication auth, uint tag_len, SrtpPrf prf, SrtpFlags flags); - [CCode (cname = "srtp_setkey")] - public int setkey(uint8[] key, uint8[] salt); - [CCode (cname = "srtp_setkeystring")] - public int setkeystring(string key, string salt); - [CCode (cname = "srtp_setrcc_rate")] - public void setrcc_rate(uint16 rate); - - [CCode (cname = "srtp_send")] - private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtcp_send")] - private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtp_recv")] - private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - [CCode (cname = "srtcp_recv")] - private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - - public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length+tag_len]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length+tag_len+4]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } -} - -[Flags] -[CCode (cname = "unsigned", cprefix = "", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpFlags { - SRTP_UNENCRYPTED, - SRTCP_UNENCRYPTED, - SRTP_UNAUTHENTICATED, - - SRTP_RCC_MODE1, - SRTP_RCC_MODE2, - SRTP_RCC_MODE3 -} - -[CCode (cname = "int", cprefix = "SRTP_ENCR_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpEncryption { - NULL, - AES_CM, - AES_F8 -} - -[CCode (cname = "int", cprefix = "SRTP_AUTH_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpAuthentication { - NULL, - HMAC_SHA1 -} - -[CCode (cname = "int", cprefix = "SRTP_PRF_", cheader_filename="srtp.h", has_type_id = false)] -public enum Dino.Plugins.Rtp.SrtpPrf { - AES_CM -} \ No newline at end of file diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 362e2d16..77080a09 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,8 +53,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; - private SrtpSession? local_crypto_session; - private SrtpSession? remote_crypto_session; + private Crypto.Srtp.Session? local_crypto_session; + private Crypto.Srtp.Session? remote_crypto_session; public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); @@ -149,11 +149,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void prepare_local_crypto() { if (local_crypto != null && local_crypto_session == null) { - local_crypto_session = new SrtpSession( - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, - SrtpAuthentication.HMAC_SHA1, + local_crypto_session = new Crypto.Srtp.Session( + local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, + Crypto.Srtp.Authentication.HMAC_SHA1, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - SrtpPrf.AES_CM, + Crypto.Srtp.Prf.AES_CM, 0 ); local_crypto_session.setkey(local_crypto.key, local_crypto.salt); @@ -284,11 +284,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private void prepare_remote_crypto() { if (remote_crypto != null && remote_crypto_session == null) { - remote_crypto_session = new SrtpSession( - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? SrtpEncryption.AES_F8 : SrtpEncryption.AES_CM, - SrtpAuthentication.HMAC_SHA1, + remote_crypto_session = new Crypto.Srtp.Session( + remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, + Crypto.Srtp.Authentication.HMAC_SHA1, remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - SrtpPrf.AES_CM, + Crypto.Srtp.Prf.AES_CM, 0 ); remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); -- cgit v1.2.3-70-g09d2 From fc3263d49e5a5c737742eb7e591498ade830b685 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 25 Mar 2021 13:06:41 +0100 Subject: Fix device manager usage for GStreamer 1.16 --- plugins/rtp/src/plugin.vala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 69b0f37a..0f3cb10d 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -47,6 +47,12 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { device_monitor.show_all = true; device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); device_monitor.start(); + foreach (Gst.Device device in device_monitor.get_devices()) { + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) continue; + if (device.properties.get_string("device.class") == "monitor") continue; + if (devices.any_match((it) => it.matches(device))) continue; + devices.add(new Device(this, device)); + } pipe = new Gst.Pipeline(null); @@ -163,24 +169,28 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { Gst.Device old_device = null; Gst.Device device = null; + Device old = null; switch (message.type) { case Gst.MessageType.DEVICE_ADDED: message.parse_device_added(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; devices.add(new Device(this, device)); break; case Gst.MessageType.DEVICE_CHANGED: message.parse_device_changed(out device, out old_device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - devices.first_match((it) => it.matches(old_device)).update(device); + old = devices.first_match((it) => it.matches(old_device)); + if (old != null) old.update(device); break; case Gst.MessageType.DEVICE_REMOVED: message.parse_device_removed(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; - devices.remove(devices.first_match((it) => it.matches(device))); + old = devices.first_match((it) => it.matches(device)); + if (old != null) devices.remove(old); break; } if (device != null) { -- cgit v1.2.3-70-g09d2 From 9520a81b814103c5549982a7e9d4e9ec6d9035f6 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:14:37 +0200 Subject: Don't reuse PTs for different media types --- plugins/rtp/src/module.vala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index ecf7b658..231a9dde 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -122,25 +122,25 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { channels = 2, clockrate = 48000, name = "opus", - id = 96 + id = 99 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", - id = 97 + id = 100 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", - id = 98 + id = 101 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", - id = 99 + id = 102 }); yield add_if_supported(list, media, new JingleRtp.PayloadType() { channels = 1, -- cgit v1.2.3-70-g09d2 From 5e58f2988382fffb70602cf308f6686b4731f0da Mon Sep 17 00:00:00 2001 From: Marvin W Date: Mon, 29 Mar 2021 13:20:12 +0200 Subject: Migrate to libsrtp2 --- cmake/FindSrtp2.cmake | 12 + plugins/crypto-vala/CMakeLists.txt | 34 +- plugins/crypto-vala/src/error.vala | 4 +- plugins/crypto-vala/src/srtp.c | 836 --------------------------------- plugins/crypto-vala/src/srtp.h | 82 ---- plugins/crypto-vala/src/srtp.vala | 122 +++++ plugins/crypto-vala/src/srtp.vapi | 107 ----- plugins/crypto-vala/vapi/libsrtp2.vapi | 115 +++++ plugins/ice/CMakeLists.txt | 2 +- plugins/ice/src/dtls_srtp.vala | 49 +- plugins/rtp/CMakeLists.txt | 2 +- plugins/rtp/src/stream.vala | 51 +- 12 files changed, 314 insertions(+), 1102 deletions(-) create mode 100644 cmake/FindSrtp2.cmake delete mode 100644 plugins/crypto-vala/src/srtp.c delete mode 100644 plugins/crypto-vala/src/srtp.h create mode 100644 plugins/crypto-vala/src/srtp.vala delete mode 100644 plugins/crypto-vala/src/srtp.vapi create mode 100644 plugins/crypto-vala/vapi/libsrtp2.vapi (limited to 'plugins/rtp/src') diff --git a/cmake/FindSrtp2.cmake b/cmake/FindSrtp2.cmake new file mode 100644 index 00000000..40b0ed97 --- /dev/null +++ b/cmake/FindSrtp2.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(Srtp2 + PKG_CONFIG_NAME libsrtp2 + LIB_NAMES srtp2 + INCLUDE_NAMES srtp2/srtp.h + INCLUDE_DIR_SUFFIXES srtp2 srtp2/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Srtp2 + REQUIRED_VARS Srtp2_LIBRARY + VERSION_VAR Srtp2_VERSION) \ No newline at end of file diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index f97b0d31..4a8da241 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -3,6 +3,7 @@ find_packages(CRYPTO_VALA_PACKAGES REQUIRED GLib GObject GIO + Srtp2 ) vala_precompile(CRYPTO_VALA_C @@ -11,44 +12,23 @@ SOURCES "src/cipher_converter.vala" "src/error.vala" "src/random.vala" - "src/srtp.vapi" + "src/srtp.vala" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" + "${CMAKE_CURRENT_SOURCE_DIR}/vapi/libsrtp2.vapi" PACKAGES ${CRYPTO_VALA_PACKAGES} +OPTIONS + --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi GENERATE_VAPI crypto-vala GENERATE_HEADER crypto-vala ) -add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/exports/srtp.h" -COMMAND - cp "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" "${CMAKE_BINARY_DIR}/exports/srtp.h" -DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.h" -COMMENT - Copy header file srtp.h -) - -add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/exports/crypto.vapi -COMMAND - cat "${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi" "${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi" > "${CMAKE_BINARY_DIR}/exports/crypto.vapi" -DEPENDS - ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi - ${CMAKE_CURRENT_SOURCE_DIR}/src/srtp.vapi -) - -add_custom_target(crypto-vapi -DEPENDS - ${CMAKE_BINARY_DIR}/exports/crypto.vapi - ${CMAKE_BINARY_DIR}/exports/srtp.h -) - -set(CFLAGS ${VALA_CFLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}/src) +set(CFLAGS ${VALA_CFLAGS}) add_definitions(${CFLAGS}) -add_library(crypto-vala STATIC ${CRYPTO_VALA_C} src/srtp.c) -add_dependencies(crypto-vala crypto-vapi) +add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/crypto-vala/src/error.vala b/plugins/crypto-vala/src/error.vala index bae4ad08..5007d725 100644 --- a/plugins/crypto-vala/src/error.vala +++ b/plugins/crypto-vala/src/error.vala @@ -2,7 +2,9 @@ namespace Crypto { public errordomain Error { ILLEGAL_ARGUMENTS, - GCRYPT + GCRYPT, + AUTHENTICATION_FAILED, + UNKNOWN } internal void may_throw_gcrypt_error(GCrypt.Error e) throws Error { diff --git a/plugins/crypto-vala/src/srtp.c b/plugins/crypto-vala/src/srtp.c deleted file mode 100644 index 708244d9..00000000 --- a/plugins/crypto-vala/src/srtp.c +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/* TODO: - * Useless stuff (because nothing depends on it): - * - non-nul key derivation rate - * - MKI payload - */ - -#ifdef HAVE_CONFIG_H -# include -#endif - -#include -#include - -#include "srtp.h" - -#include -#include -#include -#include - -#include - -#ifdef _WIN32 -# include -#else -# include -#endif - -#define debug( ... ) (void)0 - -typedef struct srtp_proto_t -{ - gcry_cipher_hd_t cipher; - gcry_md_hd_t mac; - uint64_t window; - uint32_t salt[4]; -} srtp_proto_t; - -struct srtp_session_t -{ - srtp_proto_t rtp; - srtp_proto_t rtcp; - unsigned flags; - unsigned kdr; - uint32_t rtcp_index; - uint32_t rtp_roc; - uint16_t rtp_seq; - uint16_t rtp_rcc; - uint8_t tag_len; -}; - -enum -{ - SRTP_CRYPT, - SRTP_AUTH, - SRTP_SALT, - SRTCP_CRYPT, - SRTCP_AUTH, - SRTCP_SALT -}; - - -static inline unsigned rcc_mode (const srtp_session_t *s) -{ - return (s->flags >> 4) & 3; -} - - -static void proto_destroy (srtp_proto_t *p) -{ - gcry_md_close (p->mac); - gcry_cipher_close (p->cipher); -} - - -/** - * Releases all resources associated with a Secure RTP session. - */ -void srtp_destroy (srtp_session_t *s) -{ - assert (s != NULL); - - proto_destroy (&s->rtcp); - proto_destroy (&s->rtp); - free (s); -} - - -static int proto_create (srtp_proto_t *p, int gcipher, int gmd) -{ - if (gcry_cipher_open (&p->cipher, gcipher, GCRY_CIPHER_MODE_CTR, 0) == 0) - { - if (gcry_md_open (&p->mac, gmd, GCRY_MD_FLAG_HMAC) == 0) - return 0; - gcry_cipher_close (p->cipher); - } - return -1; -} - - -/** - * Allocates a Secure RTP one-way session. - * The same session cannot be used both ways because this would confuse - * internal cryptographic counters; it is however of course feasible to open - * multiple simultaneous sessions with the same master key. - * - * @param encr encryption algorithm number - * @param auth authentication algortihm number - * @param tag_len authentication tag byte length (NOT including RCC) - * @param flags OR'ed optional flags. - * - * @return NULL in case of error - */ -srtp_session_t * -srtp_create (int encr, int auth, unsigned tag_len, int prf, unsigned flags) -{ - if ((flags & ~SRTP_FLAGS_MASK)) - return NULL; - - int cipher, md; - switch (encr) - { - case SRTP_ENCR_NULL: - cipher = GCRY_CIPHER_NONE; - break; - - case SRTP_ENCR_AES_CM: - cipher = GCRY_CIPHER_AES; - break; - - default: - return NULL; - } - - switch (auth) - { - case SRTP_AUTH_NULL: - md = GCRY_MD_NONE; - break; - - case SRTP_AUTH_HMAC_SHA1: - md = GCRY_MD_SHA1; - break; - - default: - return NULL; - } - - if (tag_len > gcry_md_get_algo_dlen (md)) - return NULL; - - if (prf != SRTP_PRF_AES_CM) - return NULL; - - srtp_session_t *s = malloc (sizeof (*s)); - if (s == NULL) - return NULL; - - memset (s, 0, sizeof (*s)); - s->flags = flags; - s->tag_len = tag_len; - s->rtp_rcc = 1; /* Default RCC rate */ - if (rcc_mode (s)) - { - if (tag_len < 4) - goto error; - } - - if (proto_create (&s->rtp, cipher, md) == 0) - { - if (proto_create (&s->rtcp, cipher, md) == 0) - return s; - proto_destroy (&s->rtp); - } - - error: - free (s); - return NULL; -} - - -/** - * Counter Mode encryption/decryption (ctr length = 16 bytes) - * with non-padded (truncated) text - */ -static int -do_ctr_crypt (gcry_cipher_hd_t hd, const void *ctr, uint8_t *data, size_t len) -{ - const size_t ctrlen = 16; - div_t d = div (len, ctrlen); - - if (gcry_cipher_setctr (hd, ctr, ctrlen) - || gcry_cipher_encrypt (hd, data, d.quot * ctrlen, NULL, 0)) - return -1; - - if (d.rem) - { - /* Truncated last block */ - uint8_t dummy[ctrlen]; - data += d.quot * ctrlen; - memcpy (dummy, data, d.rem); - memset (dummy + d.rem, 0, ctrlen - d.rem); - - if (gcry_cipher_encrypt (hd, dummy, ctrlen, data, ctrlen)) - return -1; - memcpy (data, dummy, d.rem); - } - - return 0; -} - - -/** - * AES-CM key derivation (saltlen = 14 bytes) - */ -static int -do_derive (gcry_cipher_hd_t prf, const void *salt, - const uint8_t *r, size_t rlen, uint8_t label, - void *out, size_t outlen) -{ - uint8_t iv[16]; - - memcpy (iv, salt, 14); - iv[14] = iv[15] = 0; - - assert (rlen < 14); - iv[13 - rlen] ^= label; - for (size_t i = 0; i < rlen; i++) - iv[sizeof (iv) - rlen + i] ^= r[i]; - - memset (out, 0, outlen); - return do_ctr_crypt (prf, iv, out, outlen); -} - - -/** - * Sets (or resets) the master key and master salt for a SRTP session. - * This must be done at least once before using srtp_send(), srtp_recv(), - * srtcp_send() or srtcp_recv(). Also, rekeying is required every - * 2^48 RTP packets or 2^31 RTCP packets (whichever comes first), - * otherwise the protocol security might be broken. - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen) -{ - /* SRTP/SRTCP cipher/salt/MAC keys derivation */ - gcry_cipher_hd_t prf; - uint8_t r[6], keybuf[20]; - - if (saltlen != 14) - return EINVAL; - - if (gcry_cipher_open (&prf, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_CTR, 0) - || gcry_cipher_setkey (prf, key, keylen)) - return EINVAL; - - /* SRTP key derivation */ -#if 0 - if (s->kdr != 0) - { - uint64_t index = (((uint64_t)s->rtp_roc) << 16) | s->rtp_seq; - index /= s->kdr; - - for (int i = sizeof (r) - 1; i >= 0; i--) - { - r[i] = index & 0xff; - index = index >> 8; - } - } - else -#endif - memset (r, 0, sizeof (r)); - if (do_derive (prf, salt, r, 6, SRTP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 6, SRTP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtp.mac, keybuf, 20) - || do_derive (prf, salt, r, 6, SRTP_SALT, s->rtp.salt, 14)) - return -1; - - /* SRTCP key derivation */ - memcpy (r, &(uint32_t){ htonl (s->rtcp_index) }, 4); - if (do_derive (prf, salt, r, 4, SRTCP_CRYPT, keybuf, 16) - || gcry_cipher_setkey (s->rtcp.cipher, keybuf, 16) - || do_derive (prf, salt, r, 4, SRTCP_AUTH, keybuf, 20) - || gcry_md_setkey (s->rtcp.mac, keybuf, 20) - || do_derive (prf, salt, r, 4, SRTCP_SALT, s->rtcp.salt, 14)) - return -1; - - (void)gcry_cipher_close (prf); - return 0; -} - -static int hexdigit (char c) -{ - if ((c >= '0') && (c <= '9')) - return c - '0'; - if ((c >= 'A') && (c <= 'F')) - return c - 'A' + 0xA; - if ((c >= 'a') && (c <= 'f')) - return c - 'a' + 0xa; - return -1; -} - -static ssize_t hexstring (const char *in, uint8_t *out, size_t outlen) -{ - size_t inlen = strlen (in); - - if ((inlen > (2 * outlen)) || (inlen & 1)) - return -1; - - for (size_t i = 0; i < inlen; i += 2) - { - int a = hexdigit (in[i]), b = hexdigit (in[i + 1]); - if ((a == -1) || (b == -1)) - return -1; - out[i / 2] = (a << 4) | b; - } - return inlen / 2; -} - -/** - * Sets (or resets) the master key and master salt for a SRTP session - * from hexadecimal strings. See also srtp_setkey(). - * - * @return 0 on success, in case of error: - * EINVAL invalid or unsupported key/salt sizes combination - */ -int -srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt) -{ - uint8_t bkey[16]; /* TODO/NOTE: hard-coded for AES */ - uint8_t bsalt[14]; /* TODO/NOTE: hard-coded for the PRF-AES-CM */ - ssize_t bkeylen = hexstring (key, bkey, sizeof (bkey)); - ssize_t bsaltlen = hexstring (salt, bsalt, sizeof (bsalt)); - - if ((bkeylen == -1) || (bsaltlen == -1)) - return EINVAL; - return srtp_setkey (s, bkey, bkeylen, bsalt, bsaltlen) ? EINVAL : 0; -} - -/** - * Sets Roll-over-Counter Carry (RCC) rate for the SRTP session. If not - * specified (through this function), the default rate of ONE is assumed - * (i.e. every RTP packets will carry the RoC). RCC rate is ignored if none - * of the RCC mode has been selected. - * - * The RCC mode is selected through one of these flags for srtp_create(): - * SRTP_RCC_MODE1: integrity protection only for RoC carrying packets - * SRTP_RCC_MODE2: integrity protection for all packets - * SRTP_RCC_MODE3: no integrity protection - * - * RCC mode 3 is insecure. Compared to plain RTP, it provides confidentiality - * (through encryption) but is much more prone to DoS. It can only be used if - * anti-spoofing protection is provided by lower network layers (e.g. IPsec, - * or trusted routers and proper source address filtering). - * - * If RCC rate is 1, RCC mode 1 and 2 are functionally identical. - * - * @param rate RoC Carry rate (MUST NOT be zero) - */ -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate) -{ - assert (rate != 0); - s->rtp_rcc = rate; -} - - -/** AES-CM for RTP (salt = 14 bytes + 2 nul bytes) */ -static int -rtp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t roc, uint16_t seq, - const uint32_t *salt, uint8_t *data, size_t len) -{ - /* Determines cryptographic counter (IV) */ - uint32_t counter[4]; - counter[0] = salt[0]; - counter[1] = salt[1] ^ ssrc; - counter[2] = salt[2] ^ htonl (roc); - counter[3] = salt[3] ^ htonl (seq << 16); - - /* Encryption */ - return do_ctr_crypt (hd, counter, data, len); -} - - -/** Determines SRTP Roll-Over-Counter (in host-byte order) */ -static uint32_t -srtp_compute_roc (const srtp_session_t *s, uint16_t seq) -{ - uint32_t roc = s->rtp_roc; - - if (((seq - s->rtp_seq) & 0xffff) < 0x8000) - { - /* Sequence is ahead, good */ - if (seq < s->rtp_seq) - roc++; /* Sequence number wrap */ - } - else - { - /* Sequence is late, bad */ - if (seq > s->rtp_seq) - roc--; /* Wrap back */ - } - return roc; -} - - -/** Returns RTP sequence (in host-byte order) */ -static inline uint16_t rtp_seq (const uint8_t *buf) -{ - return (buf[2] << 8) | buf[3]; -} - - -/** Message Authentication and Integrity for RTP */ -static const uint8_t * -rtp_digest (gcry_md_hd_t md, const uint8_t *data, size_t len, - uint32_t roc) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - gcry_md_write (md, &(uint32_t){ htonl (roc) }, 4); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTP packet and updates SRTP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTP packet to be en-/decrypted - * @param len RTP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet - * EACCES replayed packet or out-of-window or sync lost - */ -static int srtp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - assert (len >= 12u); - - if ((buf[0] >> 6) != 2) - return EINVAL; - - /* Computes encryption offset */ - uint16_t offset = 12; - offset += (buf[0] & 0xf) * 4; // skips CSRC - - if (buf[0] & 0x10) - { - uint16_t extlen; - - offset += 4; - if (len < offset) - return EINVAL; - - memcpy (&extlen, buf + offset - 2, 2); - offset += htons (extlen); // skips RTP extension header - } - - if (len < offset) - return EINVAL; - - /* Determines RTP 48-bits counter and SSRC */ - uint16_t seq = rtp_seq (buf); - uint32_t roc = srtp_compute_roc (s, seq), ssrc; - memcpy (&ssrc, buf + 8, 4); - - /* Updates ROC and sequence (it's safe now) */ - int16_t diff = seq - s->rtp_seq; - if (diff > 0) - { - /* Sequence in the future, good */ - s->rtp.window = s->rtp.window << diff; - s->rtp.window |= UINT64_C(1); - s->rtp_seq = seq, s->rtp_roc = roc; - } - else - { - /* Sequence in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtp.window >> diff) & 1)) - return EACCES; /* Replay attack */ - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Encrypt/Decrypt */ - if (s->flags & SRTP_UNENCRYPTED) - return 0; - - if (rtp_crypt (s->rtp.cipher, ssrc, roc, seq, s->rtp.salt, - buf + offset, len - offset)) - return EINVAL; - - return 0; -} - - -/** - * Turns a RTP packet into a SRTP packet: encrypt it, then computes - * the authentication tag and appends it. - * Note that you can encrypt packet in disorder. - * - * @param buf RTP packet to be encrypted/digested - * @param lenp pointer to the RTP packet length on entry, - * set to the SRTP length on exit (undefined on non-ENOSPC error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTP packet or internal error - * ENOSPC bufsize is too small to add authentication tag - * ( will hold the required byte size) - * EACCES packet would trigger a replay error on receiver - */ -int -srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - size_t tag_len; - size_t roc_len = 0; - - /* Compute required buffer size */ - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - tag_len = s->tag_len; - - if (rcc_mode (s)) - { - assert (tag_len >= 4); - assert (s->rtp_rcc != 0); - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; /* RCC mode 3 -> no auth*/ - else - tag_len -= 4; /* RCC mode 1 or 2 -> auth*/ - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; /* RCC mode 1 or 3 -> no auth */ - } - } - - *lenp = len + roc_len + tag_len; - } - else - tag_len = 0; - - if (bufsize < *lenp) - return ENOSPC; - - /* Encrypt payload */ - int val = srtp_crypt (s, buf, len); - if (val) - return val; - - /* Authenticate payload */ - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)); - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, roc); - - if (roc_len) - { - memcpy (buf + len, &(uint32_t){ htonl (s->rtp_roc) }, 4); - len += 4; - } - memcpy (buf + len, tag, tag_len); -#if 0 - printf ("Sent : 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - puts (""); -#endif - } - - return 0; -} - - -/** - * Turns a SRTP packet into a RTP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTP packet to be digested/decrypted - * @param lenp pointer to the SRTP packet length on entry, - * set to the RTP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - if (len < 12u) - return EINVAL; - - if (!(s->flags & SRTP_UNAUTHENTICATED)) - { - size_t tag_len = s->tag_len, roc_len = 0; - if (rcc_mode (s)) - { - if ((rtp_seq (buf) % s->rtp_rcc) == 0) - { - roc_len = 4; - if (rcc_mode (s) == 3) - tag_len = 0; - else - tag_len -= 4; - } - else - { - if (rcc_mode (s) & 1) - tag_len = 0; // RCC mode 1 or 3: no auth - } - } - - if (len < (12u + roc_len + tag_len)) - return EINVAL; - len -= roc_len + tag_len; - - uint32_t roc = srtp_compute_roc (s, rtp_seq (buf)), rcc; - if (roc_len) - { - assert (roc_len == 4); - memcpy (&rcc, buf + len, 4); - rcc = ntohl (rcc); - } - else - rcc = roc; - - const uint8_t *tag = rtp_digest (s->rtp.mac, buf, len, rcc); -#if 0 - printf ("Computed: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", tag[i]); - printf ("\nReceived: 0x"); - for (unsigned i = 0; i < tag_len; i++) - printf ("%02x", buf[len + roc_len + i]); - puts (""); -#endif - if (memcmp (buf + len + roc_len, tag, tag_len)) - return EACCES; - - if (roc_len) - { - /* Authenticated packet carried a Roll-Over-Counter */ - s->rtp_roc += rcc - roc; - assert (srtp_compute_roc (s, rtp_seq (buf)) == rcc); - } - *lenp = len; - } - - return srtp_crypt (s, buf, len); -} - - -/** AES-CM for RTCP (salt = 14 bytes + 2 nul bytes) */ -static int -rtcp_crypt (gcry_cipher_hd_t hd, uint32_t ssrc, uint32_t index, - const uint32_t *salt, uint8_t *data, size_t len) -{ - return rtp_crypt (hd, ssrc, index >> 16, index & 0xffff, salt, data, len); -} - - -/** Message Authentication and Integrity for RTCP */ -static const uint8_t * -rtcp_digest (gcry_md_hd_t md, const void *data, size_t len) -{ - gcry_md_reset (md); - gcry_md_write (md, data, len); - return gcry_md_read (md, 0); -} - - -/** - * Encrypts/decrypts a RTCP packet and updates SRTCP context - * (CTR block cypher mode of operation has identical encryption and - * decryption function). - * - * @param buf RTCP packet to be en-/decrypted - * @param len RTCP packet length - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet - */ -static int srtcp_crypt (srtp_session_t *s, uint8_t *buf, size_t len) -{ - assert (s != NULL); - - /* 8-bytes unencrypted header, and 4-bytes unencrypted footer */ - if ((len < 12) || ((buf[0] >> 6) != 2)) - return EINVAL; - - uint32_t index; - memcpy (&index, buf + len, 4); - index = ntohl (index); - if (((index >> 31) != 0) != ((s->flags & SRTCP_UNENCRYPTED) == 0)) - return EINVAL; // E-bit mismatch - - index &= ~(1 << 31); // clear E-bit for counter - - /* Updates SRTCP index (safe here) */ - int32_t diff = index - s->rtcp_index; - if (diff > 0) - { - /* Packet in the future, good */ - s->rtcp.window = s->rtcp.window << diff; - s->rtcp.window |= UINT64_C(1); - s->rtcp_index = index; - } - else - { - /* Packet in the past/present, bad */ - diff = -diff; - if ((diff >= 64) || ((s->rtcp.window >> diff) & 1)) - return EACCES; // replay attack! - s->rtp.window |= UINT64_C(1) << diff; - } - - /* Crypts SRTCP */ - if (s->flags & SRTCP_UNENCRYPTED) - return 0; - - uint32_t ssrc; - memcpy (&ssrc, buf + 4, 4); - - if (rtcp_crypt (s->rtcp.cipher, ssrc, index, s->rtp.salt, - buf + 8, len - 8)) - return EINVAL; - return 0; -} - - -/** - * Turns a RTCP packet into a SRTCP packet: encrypt it, then computes - * the authentication tag and appends it. - * - * @param buf RTCP packet to be encrypted/digested - * @param lenp pointer to the RTCP packet length on entry, - * set to the SRTCP length on exit (undefined in case of error) - * @param bufsize size (bytes) of the packet buffer - * - * @return 0 on success, in case of error: - * EINVAL malformatted RTCP packet or internal error - * ENOSPC bufsize is too small (to add index and authentication tag) - */ -int -srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t bufsize) -{ - size_t len = *lenp; - if (bufsize < (len + 4 + s->tag_len)) - return ENOSPC; - - uint32_t index = ++s->rtcp_index; - if (index >> 31) - s->rtcp_index = index = 0; /* 31-bit wrap */ - - if ((s->flags & SRTCP_UNENCRYPTED) == 0) - index |= 0x80000000; /* Set Encrypted bit */ - memcpy (buf + len, &(uint32_t){ htonl (index) }, 4); - - int val = srtcp_crypt (s, buf, len); - if (val) - return val; - - len += 4; /* Digests SRTCP index too */ - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - memcpy (buf + len, tag, s->tag_len); - *lenp = len + s->tag_len; - return 0; -} - - -/** - * Turns a SRTCP packet into a RTCP packet: authenticates the packet, - * then decrypts it. - * - * @param buf RTCP packet to be digested/decrypted - * @param lenp pointer to the SRTCP packet length on entry, - * set to the RTCP length on exit (undefined in case of error) - * - * @return 0 on success, in case of error: - * EINVAL malformatted SRTCP packet - * EACCES authentication failed (spoofed packet or out-of-sync) - */ -int -srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp) -{ - size_t len = *lenp; - - if (len < (4u + s->tag_len)) - return EINVAL; - len -= s->tag_len; - - const uint8_t *tag = rtcp_digest (s->rtcp.mac, buf, len); - if (memcmp (buf + len, tag, s->tag_len)) - return EACCES; - - len -= 4; /* Remove SRTCP index before decryption */ - *lenp = len; - return srtcp_crypt (s, buf, len); -} \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.h b/plugins/crypto-vala/src/srtp.h deleted file mode 100644 index abca6988..00000000 --- a/plugins/crypto-vala/src/srtp.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Secure RTP with libgcrypt - * Copyright (C) 2007 Rémi Denis-Courmont - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation; either version 2.1 - * of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - ****************************************************************************/ - -#ifndef LIBVLC_SRTP_H -# define LIBVLC_SRTP_H 1 -#include - -typedef struct srtp_session_t srtp_session_t; - -enum -{ - SRTP_UNENCRYPTED=0x1, //< do not encrypt SRTP packets - SRTCP_UNENCRYPTED=0x2, //< do not encrypt SRTCP packets - SRTP_UNAUTHENTICATED=0x4, //< authenticate only SRTCP packets - - SRTP_RCC_MODE1=0x10, //< use Roll-over-Counter Carry mode 1 - SRTP_RCC_MODE2=0x20, //< use Roll-over-Counter Carry mode 2 - SRTP_RCC_MODE3=0x30, //< use Roll-over-Counter Carry mode 3 (insecure) - - SRTP_FLAGS_MASK=0x37 //< mask for valid flags -}; - -/** SRTP encryption algorithms (ciphers); same values as MIKEY */ -enum -{ - SRTP_ENCR_NULL=0, //< no encryption - SRTP_ENCR_AES_CM=1, //< AES counter mode - SRTP_ENCR_AES_F8=2, //< AES F8 mode (not implemented) -}; - -/** SRTP authenticaton algorithms; same values as MIKEY */ -enum -{ - SRTP_AUTH_NULL=0, //< no authentication code - SRTP_AUTH_HMAC_SHA1=1, //< HMAC-SHA1 -}; - -/** SRTP pseudo random function; same values as MIKEY */ -enum -{ - SRTP_PRF_AES_CM=0, //< AES counter mode -}; - -# ifdef __cplusplus -extern "C" { -# endif - -srtp_session_t *srtp_create (int encr, int auth, unsigned tag_len, int prf, - unsigned flags); -void srtp_destroy (srtp_session_t *s); - -int srtp_setkey (srtp_session_t *s, const void *key, size_t keylen, - const void *salt, size_t saltlen); -int srtp_setkeystring (srtp_session_t *s, const char *key, const char *salt); - -void srtp_setrcc_rate (srtp_session_t *s, uint16_t rate); - -int srtp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsize); -int srtp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); -int srtcp_send (srtp_session_t *s, uint8_t *buf, size_t *lenp, size_t maxsiz); -int srtcp_recv (srtp_session_t *s, uint8_t *buf, size_t *lenp); - -# ifdef __cplusplus -} -# endif -#endif \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vala b/plugins/crypto-vala/src/srtp.vala new file mode 100644 index 00000000..77b5acde --- /dev/null +++ b/plugins/crypto-vala/src/srtp.vala @@ -0,0 +1,122 @@ +using Srtp; + +public class Crypto.Srtp { + public const string AES_CM_128_HMAC_SHA1_80 = "AES_CM_128_HMAC_SHA1_80"; + public const string AES_CM_128_HMAC_SHA1_32 = "AES_CM_128_HMAC_SHA1_32"; + public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; + + public class Session { + public bool has_encrypt { get; private set; } + public bool has_decrypt { get; private set; } + + private Context encrypt_context; + private Context decrypt_context; + + static construct { + init(); + install_log_handler(log); + } + + private static void log(LogLevel level, string msg) { + print(@"SRTP[$level]: $msg\n"); + } + + public Session() { + Context.create(out encrypt_context, null); + Context.create(out decrypt_context, null); + } + + public uint8[] encrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN + 4]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect_rtcp(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTCP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect_rtcp(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTCP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + private Policy create_policy(string profile) { + Policy policy = Policy(); + switch (profile) { + case AES_CM_128_HMAC_SHA1_80: + policy.rtp.set_aes_cm_128_hmac_sha1_80(); + policy.rtcp.set_aes_cm_128_hmac_sha1_80(); + break; + } + return policy; + } + + public void set_encryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_outbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + encrypt_context.add_stream(ref policy); + has_encrypt = true; + } + + public void set_decryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_inbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + decrypt_context.add_stream(ref policy); + has_decrypt = true; + } + } +} \ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vapi b/plugins/crypto-vala/src/srtp.vapi deleted file mode 100644 index 0fe825c3..00000000 --- a/plugins/crypto-vala/src/srtp.vapi +++ /dev/null @@ -1,107 +0,0 @@ -[CCode (cheader_filename="srtp.h")] -namespace Crypto.Srtp { - -[Compact] -[CCode (cname = "srtp_session_t", free_function = "srtp_destroy")] -public class Session { - [CCode (cname = "srtp_create")] - public Session(Encryption encr, Authentication auth, uint tag_len, Prf prf, Flags flags); - [CCode (cname = "srtp_setkey")] - public int setkey(uint8[] key, uint8[] salt); - [CCode (cname = "srtp_setkeystring")] - public int setkeystring(string key, string salt); - [CCode (cname = "srtp_setrcc_rate")] - public void setrcc_rate(uint16 rate); - - [CCode (cname = "srtp_send")] - private int rtp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtcp_send")] - private int rtcp_send([CCode (array_length = false)] uint8[] buf, ref size_t len, size_t maxsize); - [CCode (cname = "srtp_recv")] - private int rtp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - [CCode (cname = "srtcp_recv")] - private int rtcp_recv([CCode (array_length = false)] uint8[] buf, ref size_t len); - - public uint8[] encrypt_rtp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length + tag_len]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] encrypt_rtcp(uint8[] input, uint tag_len = 10) throws GLib.Error { - uint8[] buf = new uint8[input.length + tag_len + 4]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_send(buf, ref buf_use, buf.length); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP encrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } - - public uint8[] decrypt_rtcp(uint8[] input) throws GLib.Error { - uint8[] buf = new uint8[input.length]; - GLib.Memory.copy(buf, input, input.length); - size_t buf_use = input.length; - int res = rtcp_recv(buf, ref buf_use); - if (res != 0) { - throw new GLib.Error(-1, res, "RTCP decrypt failed"); - } - uint8[] ret = new uint8[buf_use]; - GLib.Memory.copy(ret, buf, buf_use); - return ret; - } -} - -[Flags] -[CCode (cname = "unsigned", cprefix = "", has_type_id = false)] -public enum Flags { - SRTP_UNENCRYPTED, - SRTCP_UNENCRYPTED, - SRTP_UNAUTHENTICATED, - SRTP_RCC_MODE1, - SRTP_RCC_MODE2, - SRTP_RCC_MODE3 -} - -[CCode (cname = "int", cprefix = "SRTP_ENCR_", has_type_id = false)] -public enum Encryption { - NULL, - AES_CM, - AES_F8 -} - -[CCode (cname = "int", cprefix = "SRTP_AUTH_", has_type_id = false)] -public enum Authentication { - NULL, - HMAC_SHA1 -} - -[CCode (cname = "int", cprefix = "SRTP_PRF_", has_type_id = false)] -public enum Prf { - AES_CM -} - -} \ No newline at end of file diff --git a/plugins/crypto-vala/vapi/libsrtp2.vapi b/plugins/crypto-vala/vapi/libsrtp2.vapi new file mode 100644 index 00000000..5ceedced --- /dev/null +++ b/plugins/crypto-vala/vapi/libsrtp2.vapi @@ -0,0 +1,115 @@ +[CCode (cheader_filename = "srtp2/srtp.h")] +namespace Srtp { +public const uint MAX_TRAILER_LEN; + +public static ErrorStatus init(); +public static ErrorStatus shutdown(); + +[Compact] +[CCode (cname = "srtp_ctx_t", cprefix = "srtp_", free_function = "srtp_dealloc")] +public class Context { + public static ErrorStatus create(out Context session, Policy? policy); + + public ErrorStatus protect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + public ErrorStatus unprotect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + + public ErrorStatus protect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + public ErrorStatus unprotect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + + public ErrorStatus add_stream(ref Policy policy); + public ErrorStatus update_stream(ref Policy policy); + public ErrorStatus remove_stream(uint ssrc); + public ErrorStatus update(ref Policy policy); +} + +[CCode (cname = "srtp_ssrc_t")] +public struct Ssrc { + public SsrcType type; + public uint value; +} + +[CCode (cname = "srtp_ssrc_type_t", cprefix = "ssrc_")] +public enum SsrcType { + undefined, specific, any_inbound, any_outbound +} + +[CCode (cname = "srtp_policy_t", destroy_function = "")] +public struct Policy { + public Ssrc ssrc; + public CryptoPolicy rtp; + public CryptoPolicy rtcp; + [CCode (array_length = false)] + public uint8[] key; + public ulong num_master_keys; + public ulong window_size; + public int allow_repeat_tx; + [CCode (array_length_cname = "enc_xtn_hdr_count")] + public int[] enc_xtn_hdr; +} + +[CCode (cname = "srtp_crypto_policy_t")] +public struct CryptoPolicy { + public CipherType cipher_type; + public int cipher_key_len; + public AuthType auth_type; + public int auth_key_len; + public int auth_tag_len; + public SecurityServices sec_serv; + + public void set_aes_cm_128_hmac_sha1_80(); + public void set_aes_cm_128_hmac_sha1_32(); + public void set_aes_cm_128_null_auth(); + public void set_aes_cm_192_hmac_sha1_32(); + public void set_aes_cm_192_hmac_sha1_80(); + public void set_aes_cm_192_null_auth(); + public void set_aes_cm_256_hmac_sha1_32(); + public void set_aes_cm_256_hmac_sha1_80(); + public void set_aes_cm_256_null_auth(); + public void set_aes_gcm_128_16_auth(); + public void set_aes_gcm_128_8_auth(); + public void set_aes_gcm_128_8_only_auth(); + public void set_aes_gcm_256_16_auth(); + public void set_aes_gcm_256_8_auth(); + public void set_aes_gcm_256_8_only_auth(); + public void set_null_cipher_hmac_null(); + public void set_null_cipher_hmac_sha1_80(); + + public void set_rtp_default(); + public void set_rtcp_default(); + + public void set_from_profile_for_rtp(Profile profile); + public void set_from_profile_for_rtcp(Profile profile); +} + +[CCode (cname = "srtp_profile_t", cprefix = "srtp_profile_")] +public enum Profile { + reserved, aes128_cm_sha1_80, aes128_cm_sha1_32, null_sha1_80, null_sha1_32, aead_aes_128_gcm, aead_aes_256_gcm +} + +[CCode (cname = "srtp_cipher_type_id_t")] +public struct CipherType : uint32 {} + +[CCode (cname = "srtp_auth_type_id_t")] +public struct AuthType : uint32 {} + +[CCode (cname = "srtp_sec_serv_t", cprefix = "sec_serv_")] +public enum SecurityServices { + none, conf, auth, conf_and_auth; +} + +[CCode (cname = "srtp_err_status_t", cprefix = "srtp_err_status_", has_type_id = false)] +public enum ErrorStatus { + ok, fail, bad_param, alloc_fail, dealloc_fail, init_fail, terminus, auth_fail, cipher_fail, replay_fail, algo_fail, no_such_op, no_ctx, cant_check, key_expired, socket_err, signal_err, nonce_bad, encode_err, semaphore_err, pfkey_err, bad_mki, pkt_idx_old, pkt_idx_adv +} + +[CCode (cname = "srtp_log_level_t", cprefix = "srtp_log_level_", has_type_id = false)] +public enum LogLevel { + error, warning, info, debug +} + +[CCode (cname = "srtp_log_handler_func_t")] +public delegate void LogHandler(LogLevel level, string msg); + +public static ErrorStatus install_log_handler(LogHandler func); + +} \ No newline at end of file diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt index 38025aa0..392a202f 100644 --- a/plugins/ice/CMakeLists.txt +++ b/plugins/ice/CMakeLists.txt @@ -20,7 +20,7 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi - ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi PACKAGES ${ICE_PACKAGES} OPTIONS diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala index a21c242b..b742ccab 100644 --- a/plugins/ice/src/dtls_srtp.vala +++ b/plugins/ice/src/dtls_srtp.vala @@ -12,8 +12,7 @@ public class DtlsSrtp { private uint pull_timeout = uint.MAX; private string peer_fingerprint; - private Crypto.Srtp.Session encrypt_session; - private Crypto.Srtp.Session decrypt_session; + private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); public static DtlsSrtp setup() throws GLib.Error { var obj = new DtlsSrtp(); @@ -30,9 +29,19 @@ public class DtlsSrtp { } public uint8[] process_incoming_data(uint component_id, uint8[] data) { - if (decrypt_session != null) { - if (component_id == 1) return decrypt_session.decrypt_rtp(data); - if (component_id == 2) return decrypt_session.decrypt_rtcp(data); + if (srtp_session.has_decrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.decrypt_rtcp(data); + } + return srtp_session.decrypt_rtp(data); + } + if (component_id == 2) return srtp_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } } else if (component_id == 1) { on_data_rec(data); } @@ -40,9 +49,19 @@ public class DtlsSrtp { } public uint8[] process_outgoing_data(uint component_id, uint8[] data) { - if (encrypt_session != null) { - if (component_id == 1) return encrypt_session.encrypt_rtp(data); - if (component_id == 2) return encrypt_session.encrypt_rtcp(data); + if (srtp_session.has_encrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.encrypt_rtcp(data); + } + return srtp_session.encrypt_rtp(data); + } + if (component_id == 2) return srtp_session.encrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } } return null; } @@ -123,19 +142,13 @@ public class DtlsSrtp { warning("SRTP client/server key/salt null"); } - Crypto.Srtp.Session encrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); - Crypto.Srtp.Session decrypt_session = new Crypto.Srtp.Session(Crypto.Srtp.Encryption.AES_CM, Crypto.Srtp.Authentication.HMAC_SHA1, 10, Crypto.Srtp.Prf.AES_CM, 0); - if (server) { - encrypt_session.setkey(server_key.extract(), server_salt.extract()); - decrypt_session.setkey(client_key.extract(), client_salt.extract()); + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); } else { - encrypt_session.setkey(client_key.extract(), client_salt.extract()); - decrypt_session.setkey(server_key.extract(), server_salt.extract()); + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); } - - this.encrypt_session = (owned)encrypt_session; - this.decrypt_session = (owned)decrypt_session; } private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 8ce2a7c6..c6888459 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -19,7 +19,7 @@ SOURCES src/video_widget.vala src/register_plugin.vala CUSTOM_VAPIS - ${CMAKE_BINARY_DIR}/exports/crypto.vapi + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 77080a09..bedd6f8a 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -53,8 +53,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.Pad send_rtp_sink_pad; private Gst.Pad send_rtp_src_pad; - private Crypto.Srtp.Session? local_crypto_session; - private Crypto.Srtp.Session? remote_crypto_session; + private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session(); public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(content); @@ -148,15 +147,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_local_crypto() { - if (local_crypto != null && local_crypto_session == null) { - local_crypto_session = new Crypto.Srtp.Session( - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, - Crypto.Srtp.Authentication.HMAC_SHA1, - local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - Crypto.Srtp.Prf.AES_CM, - 0 - ); - local_crypto_session.setkey(local_crypto.key, local_crypto.salt); + if (local_crypto != null && !crypto_session.has_encrypt) { + crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); debug("Setting up encryption with key params %s", local_crypto.key_params); } } @@ -172,15 +164,19 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { buffer.extract_dup(0, buffer.get_size(), out data); prepare_local_crypto(); if (sink == send_rtp) { - if (local_crypto_session != null) { - data = local_crypto_session.encrypt_rtp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtp(data); } on_send_rtp_data(new Bytes.take(data)); } else if (sink == send_rtcp) { - if (local_crypto_session != null) { - data = local_crypto_session.encrypt_rtcp(data, local_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10); + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtcp(data); + } + if (rtcp_mux) { + on_send_rtp_data(new Bytes.take(data)); + } else { + on_send_rtcp_data(new Bytes.take(data)); } - on_send_rtcp_data(new Bytes.take(data)); } else { warning("unknown sample"); } @@ -283,25 +279,22 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && remote_crypto_session == null) { - remote_crypto_session = new Crypto.Srtp.Session( - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.F8_128_HMAC_SHA1_80 ? Crypto.Srtp.Encryption.AES_F8 : Crypto.Srtp.Encryption.AES_CM, - Crypto.Srtp.Authentication.HMAC_SHA1, - remote_crypto.crypto_suite == Xep.JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_32 ? 4 : 10, - Crypto.Srtp.Prf.AES_CM, - 0 - ); - remote_crypto_session.setkey(remote_crypto.key, remote_crypto.salt); + if (remote_crypto != null && crypto_session.has_decrypt) { + crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } } public override void on_recv_rtp_data(Bytes bytes) { + if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { + on_recv_rtcp_data(bytes); + return; + } prepare_remote_crypto(); uint8[] data = bytes.get_data(); - if (remote_crypto_session != null) { + if (crypto_session.has_decrypt) { try { - data = remote_crypto_session.decrypt_rtp(data); + data = crypto_session.decrypt_rtp(data); } catch (Error e) { warning("%s (%d)", e.message, e.code); } @@ -314,9 +307,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public override void on_recv_rtcp_data(Bytes bytes) { prepare_remote_crypto(); uint8[] data = bytes.get_data(); - if (remote_crypto_session != null) { + if (crypto_session.has_decrypt) { try { - data = remote_crypto_session.decrypt_rtcp(data); + data = crypto_session.decrypt_rtcp(data); } catch (Error e) { warning("%s (%d)", e.message, e.code); } -- cgit v1.2.3-70-g09d2 From c5cb43350af15e99d7304935ccf5fe84c2acdfc9 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 1 Apr 2021 11:51:12 +0200 Subject: Remove unnecessary debug code --- plugins/rtp/src/plugin.vala | 4 ---- 1 file changed, 4 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 0f3cb10d..62e0d411 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -403,10 +403,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { plugin_stream.pause(); } else { plugin_stream.unpause(); - Timeout.add_seconds(3, () => { - dump_dot(); - return false; - }); } } -- cgit v1.2.3-70-g09d2 From c5ab4fed87d2dedb5ccbf671c8b2742a11251a0f Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 1 Apr 2021 11:51:35 +0200 Subject: Fix bug in legacy SRTP decryption --- plugins/rtp/src/stream.vala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index bedd6f8a..efa1b497 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -279,7 +279,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && crypto_session.has_decrypt) { + if (remote_crypto != null && !crypto_session.has_decrypt) { crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } @@ -339,6 +339,11 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { + debug("New ssrc %s with pad %s", ssrc, pad.name); + if (participant_ssrc != null && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); + return; + } participant_ssrc = ssrc; recv_rtp_src_pad = pad; if (decode != null) { -- cgit v1.2.3-70-g09d2 From 6ebdec1d78a7ad1b8668a2ba6eceb34515c75384 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 12:31:03 +0200 Subject: GStreamer compat --- plugins/rtp/CMakeLists.txt | 6 ++++++ plugins/rtp/src/device.vala | 30 +++++++++++++++++++++--------- plugins/rtp/src/plugin.vala | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index c6888459..0925ff0c 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -9,6 +9,10 @@ find_packages(RTP_PACKAGES REQUIRED GstApp ) +if(Gst_VERSION VERSION_GREATER "1.16") + set(RTP_DEFINITIONS GST_1_16) +endif() + vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala @@ -25,6 +29,8 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/qlite.vapi PACKAGES ${RTP_PACKAGES} +DEFINITIONS + ${RTP_DEFINITIONS} OPTIONS --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 20762f77..3c9a38d2 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -130,11 +130,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); - if (media == "audio") { + if (media == "audio" && plugin.echoprobe != null) { dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); - dsp.@set("probe", plugin.echoprobe.name); - pipe.add(dsp); - filter.link(dsp); + if (dsp != null) { + dsp.@set("probe", plugin.echoprobe.name); + pipe.add(dsp); + filter.link(dsp); + } } tee = Gst.ElementFactory.make("tee", @"$id-tee"); tee.@set("allow-not-linked", true); @@ -149,15 +151,19 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); filter.@set("caps", get_best_caps()); pipe.add(filter); - filter.link(plugin.echoprobe); - plugin.echoprobe.link(element); + if (plugin.echoprobe != null) { + filter.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } else { + filter.link(element); + } } plugin.unpause(); } private void destroy() { if (mixer != null) { - if (is_sink && media == "audio") { + if (is_sink && media == "audio" && plugin.echoprobe != null) { plugin.echoprobe.unlink(mixer); } int linked_sink_pads = 0; @@ -177,11 +183,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - filter.unlink(plugin.echoprobe); + if (plugin.echoprobe != null) { + filter.unlink(plugin.echoprobe); + } else { + filter.unlink(element); + } pipe.remove(filter); filter = null; } - plugin.echoprobe.unlink(element); + if (plugin.echoprobe != null) { + plugin.echoprobe.unlink(element); + } } element.set_locked_state(true); element.set_state(Gst.State.NULL); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 62e0d411..40ad1e0f 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -70,7 +70,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { // Audio echo probe echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); - pipe.add(echoprobe); + if (echoprobe != null) pipe.add(echoprobe); // Pipeline pipe.auto_flush_bus = true; @@ -178,6 +178,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; devices.add(new Device(this, device)); break; +#if GST_1_16 case Gst.MessageType.DEVICE_CHANGED: message.parse_device_changed(out device, out old_device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; @@ -185,6 +186,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { old = devices.first_match((it) => it.matches(old_device)); if (old != null) old.update(device); break; +#endif case Gst.MessageType.DEVICE_REMOVED: message.parse_device_removed(out device); if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; -- cgit v1.2.3-70-g09d2 From fe160d94ba8a08a806dc755d918dc3de0a645d7c Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 11 Apr 2021 15:57:53 +0200 Subject: Handle broken VAPI in older vala --- plugins/rtp/src/stream.vala | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index efa1b497..3a63f3fa 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -300,7 +300,17 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } if (push_recv_data) { - recv_rtp.push_buffer(new Gst.Buffer.wrapped((owned) data)); + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() + // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of + // the underlying buffer, so and some point we should move over to the new version (once we require + // Vala >= 0.50) +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtp, "push-buffer", buffer, out ret); +#endif } } @@ -315,7 +325,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } if (push_recv_data) { - recv_rtcp.push_buffer(new Gst.Buffer.wrapped((owned) data)); + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + // See above +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtcp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtcp, "push-buffer", buffer, out ret); +#endif } } -- cgit v1.2.3-70-g09d2 From 3880628de4785db4c0a03a79a0c486507fe9b1a8 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Thu, 29 Apr 2021 15:46:06 +0200 Subject: Video optimizations --- cmake/FindGstRtp.cmake | 14 + plugins/rtp/CMakeLists.txt | 4 +- plugins/rtp/src/codec_util.vala | 115 +++- plugins/rtp/src/device.vala | 9 +- plugins/rtp/src/module.vala | 133 +++-- plugins/rtp/src/plugin.vala | 14 + plugins/rtp/src/stream.vala | 220 +++++++- plugins/rtp/src/video_widget.vala | 10 +- plugins/rtp/vapi/gstreamer-rtp-1.0.vapi | 625 +++++++++++++++++++++ .../xep/0167_jingle_rtp/content_parameters.vala | 40 +- .../xep/0167_jingle_rtp/jingle_rtp_module.vala | 5 + .../module/xep/0167_jingle_rtp/payload_type.vala | 49 +- .../src/module/xep/0167_jingle_rtp/stream.vala | 7 + 13 files changed, 1126 insertions(+), 119 deletions(-) create mode 100644 cmake/FindGstRtp.cmake create mode 100644 plugins/rtp/vapi/gstreamer-rtp-1.0.vapi (limited to 'plugins/rtp/src') diff --git a/cmake/FindGstRtp.cmake b/cmake/FindGstRtp.cmake new file mode 100644 index 00000000..0756a985 --- /dev/null +++ b/cmake/FindGstRtp.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstRtp + PKG_CONFIG_NAME gstreamer-rtp-1.0 + LIB_NAMES gstrtp + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/rtp/rtp.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-rtp-1.0 gstreamer-rtp-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstRtp + REQUIRED_VARS GstRtp_LIBRARY + VERSION_VAR GstRtp_VERSION) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 76d6e66d..92ec1b97 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,3 +1,4 @@ +find_package(GstRtp REQUIRED) find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -27,6 +28,7 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/gstreamer-rtp-1.0.vapi PACKAGES ${RTP_PACKAGES} DEFINITIONS @@ -35,7 +37,7 @@ DEFINITIONS add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 6bd465c1..7537c11d 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -6,7 +6,7 @@ public class Dino.Plugins.Rtp.CodecUtil { private Set supported_elements = new HashSet(); private Set unsupported_elements = new HashSet(); - public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type) { + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) { Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", "media", typeof(string), media, "payload", typeof(int), payload_type.id); @@ -19,6 +19,15 @@ public class Dino.Plugins.Rtp.CodecUtil { if (payload_type.name != null) { s.set("encoding-name", typeof(string), payload_type.name.up()); } + if (incoming) { + foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) { + if (rtcp_fb.subtype == null) { + s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true); + } else { + s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true); + } + } + } return caps; } @@ -122,32 +131,82 @@ public class Dino.Plugins.Rtp.CodecUtil { return new string[0]; } - public static string? get_encode_prefix(string media, string codec, string encode) { + public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; return null; } - public static string? get_encode_suffix(string media, string codec, string encode) { + public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { // H264 - const string h264_suffix = " ! video/x-h264,profile=constrained-baseline ! h264parse"; - if (encode == "msdkh264enc") return @" bitrate=256 rate-control=vbr target-usage=7$h264_suffix"; - if (encode == "vaapih264enc") return @" bitrate=256 quality-level=7 tune=low-power$h264_suffix"; - if (encode == "x264enc") return @" byte-stream=1 bitrate=256 profile=baseline speed-preset=ultrafast tune=zerolatency$h264_suffix"; - if (media == "video" && codec == "h264") return h264_suffix; + if (encode == "msdkh264enc") return @" rate-control=vbr"; + if (encode == "vaapih264enc") return @" tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; // VP8 - if (encode == "msdkvp8enc") return " bitrate=256 rate-control=vbr target-usage=7"; - if (encode == "vaapivp8enc") return " bitrate=256 rate-control=vbr quality-level=7"; - if (encode == "vp8enc") return " target-bitrate=256000 deadline=1 error-resilient=1"; + if (encode == "msdkvp8enc") return " rate-control=vbr"; + if (encode == "vaapivp8enc") return " rate-control=vbr"; + if (encode == "vp8enc") return " deadline=1 error-resilient=1"; // OPUS - if (encode == "opusenc") return " audio-type=voice"; + if (encode == "opusenc") { + if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true"; + return " audio-type=voice"; + } return null; } - public static string? get_decode_prefix(string media, string codec, string decode) { + public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse"; + return null; + } + + public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return 0; + string? codec = get_codec_from_payload(media, payload_type); + string? encode_name = get_encode_element_name(media, codec); + if (encode_name == null) return 0; + Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); + + bitrate = uint.min(2048000, bitrate); + + switch (encode_name) { + case "msdkh264enc": + case "vaapih264enc": + case "x264enc": + case "msdkvp8enc": + case "vaapivp8enc": + bitrate = uint.min(2048000, bitrate); + encode.set("bitrate", bitrate); + return bitrate; + case "vp8enc": + bitrate = uint.min(2147483, bitrate); + encode.set("target-bitrate", bitrate * 1000); + return bitrate; + } + + return 0; + } + + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; + if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + return null; + } + + public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (codec == "vp8") return " wait-for-keyframe=true"; return null; } @@ -195,21 +254,24 @@ public class Dino.Plugins.Rtp.CodecUtil { unsupported_elements.add(element_name); } - public string? get_decode_bin_description(string media, string? codec, string? element_name = null, string? name = null) { + public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { if (codec == null) return null; string base_name = name ?? @"encode-$codec-$(Random.next_int())"; string depay = get_depay_element_name(media, codec); string decode = element_name ?? get_decode_element_name(media, codec); if (depay == null || decode == null) return null; - string decode_prefix = get_decode_prefix(media, codec, decode) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; - return @"$depay name=$base_name-rtp-depay ! $decode_prefix$decode name=$base_name-decode ! $(media)convert name=$base_name-convert$resample"; + string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? ""; + string decode_args = get_decode_args(media, codec, decode, payload_type) ?? ""; + string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; + string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; } public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { string? codec = get_codec_from_payload(media, payload_type); string base_name = name ?? @"encode-$codec-$(Random.next_int())"; - string? desc = get_decode_bin_description(media, codec, null, base_name); + string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name); if (desc == null) return null; debug("Pipeline to decode %s %s: %s", media, codec, desc); Gst.Element bin = Gst.parse_bin_from_description(desc, true); @@ -217,22 +279,23 @@ public class Dino.Plugins.Rtp.CodecUtil { return bin; } - public string? get_encode_bin_description(string media, string? codec, string? element_name = null, uint pt = 96, string? name = null) { + public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { if (codec == null) return null; - string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; string pay = get_pay_element_name(media, codec); string encode = element_name ?? get_encode_element_name(media, codec); if (pay == null || encode == null) return null; - string encode_prefix = get_encode_prefix(media, codec, encode) ?? ""; - string encode_suffix = get_encode_suffix(media, codec, encode) ?? ""; - string resample = media == "audio" ? @" ! audioresample name=$base_name-resample" : ""; - return @"$(media)convert name=$base_name-convert$resample ! $encode_prefix$encode$encode_suffix ! $pay pt=$pt name=$base_name-rtp-pay"; + string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; + string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; } public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { string? codec = get_codec_from_payload(media, payload_type); - string base_name = name ?? @"encode-$codec-$(Random.next_int())"; - string? desc = get_encode_bin_description(media, codec, null, payload_type.id, base_name); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name); if (desc == null) return null; debug("Pipeline to encode %s %s: %s", media, codec, desc); Gst.Element bin = Gst.parse_bin_from_description(desc, true); diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3c9a38d2..785f853a 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -126,19 +126,20 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element = device.create_element(id); pipe.add(element); if (is_source) { - filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + element.@set("do-timestamp", true); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); if (media == "audio" && plugin.echoprobe != null) { - dsp = Gst.ElementFactory.make("webrtcdsp", @"$id-dsp"); + dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); if (dsp != null) { dsp.@set("probe", plugin.echoprobe.name); pipe.add(dsp); filter.link(dsp); } } - tee = Gst.ElementFactory.make("tee", @"$id-tee"); + tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); pipe.add(tee); (dsp ?? filter).link(tee); @@ -148,7 +149,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { element.@set("sync", false); } if (is_sink && media == "audio") { - filter = Gst.ElementFactory.make("capsfilter", @"$id-caps-filter"); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); filter.@set("caps", get_best_caps()); pipe.add(filter); if (plugin.echoprobe != null) { diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 231a9dde..52cc1880 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -63,7 +63,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return supported; } - private async bool supports(string media, JingleRtp.PayloadType payload_type) { + private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { string codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; @@ -77,7 +77,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return false; } - string encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); while (!(yield pipeline_works(media, encode_bin))) { debug("%s not suited for encoding %s", encode_element, codec); codec_util.mark_element_unsupported(encode_element); @@ -87,11 +87,11 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { unsupported_codecs.add(codec); return false; } - encode_bin = codec_util.get_encode_bin_description(media, codec, encode_element); + encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); } debug("using %s to encode %s", encode_element, codec); - string decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { debug("%s not suited for decoding %s", decode_element, codec); codec_util.mark_element_unsupported(decode_element); @@ -101,7 +101,7 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { unsupported_codecs.add(codec); return false; } - decode_bin = codec_util.get_decode_bin_description(media, codec, decode_element); + decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); } debug("using %s to decode %s", decode_element, codec); @@ -109,8 +109,21 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return true; } + public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) { + if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true; + return false; + } + + public override Gee.List get_suggested_header_extensions(string media) { + Gee.List exts = new ArrayList(); + if (media == "video") { + exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation")); + } + return exts; + } + public async void add_if_supported(Gee.List list, string media, JingleRtp.PayloadType payload_type) { - if (yield supports(media, payload_type)) { + if (yield is_payload_supported(media, payload_type)) { list.add(payload_type); } } @@ -118,58 +131,34 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async Gee.List get_supported_payloads(string media) { Gee.List list = new ArrayList(JingleRtp.PayloadType.equals_func); if (media == "audio") { - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 2, - clockrate = 48000, - name = "opus", - id = 99 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 32000, - name = "speex", - id = 100 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 16000, - name = "speex", - id = 101 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "speex", - id = 102 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "PCMU", - id = 0 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - channels = 1, - clockrate = 8000, - name = "PCMA", - id = 8 - }); + var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 }; + opus.parameters["useinbandfec"] = "1"; + var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 }; + var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 }; + var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 102 }; + var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 }; + var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 }; + yield add_if_supported(list, media, opus); + yield add_if_supported(list, media, speex32); + yield add_if_supported(list, media, speex16); + yield add_if_supported(list, media, speex8); + yield add_if_supported(list, media, pcmu); + yield add_if_supported(list, media, pcma); } else if (media == "video") { - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "H264", - id = 96 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "VP9", - id = 97 - }); - yield add_if_supported(list, media, new JingleRtp.PayloadType() { - clockrate = 90000, - name = "VP8", - id = 98 - }); + var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 }; + var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 }; + var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 }; + var rtcp_fbs = new ArrayList(); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli")); + h264.rtcp_fbs.add_all(rtcp_fbs); + vp9.rtcp_fbs.add_all(rtcp_fbs); + vp8.rtcp_fbs.add_all(rtcp_fbs); + yield add_if_supported(list, media, h264); + yield add_if_supported(list, media, vp9); + yield add_if_supported(list, media, vp8); } else { warning("Unsupported media type: %s", media); } @@ -179,11 +168,15 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List payloads) { if (media == "audio") { foreach (JingleRtp.PayloadType type in payloads) { - if (yield supports(media, type)) return type; + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); } } else if (media == "video") { + // We prefer H.264 (best support for hardware acceleration and good overall codec quality) + JingleRtp.PayloadType? h264 = payloads.first_match((it) => it.name.up() == "H264"); + if (h264 != null && yield is_payload_supported(media, h264)) return adjust_payload_type(media, h264.clone()); + // Take first of the list that we do support otherwise foreach (JingleRtp.PayloadType type in payloads) { - if (yield supports(media, type)) return type; + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); } } else { warning("Unsupported media type: %s", media); @@ -191,6 +184,28 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { return null; } + public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) { + var iter = type.rtcp_fbs.iterator(); + while (iter.next()) { + var fb = iter.@get(); + switch (fb.type_) { + case "goog-remb": + if (fb.subtype != null) iter.remove(); + break; + case "ccm": + if (fb.subtype != "fir") iter.remove(); + break; + case "nack": + if (fb.subtype != null && fb.subtype != "pli") iter.remove(); + break; + default: + iter.remove(); + break; + } + } + return type; + } + public override JingleRtp.Stream create_stream(Jingle.Content content) { return plugin.open_stream(content); } diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index 40ad1e0f..f0ad7db2 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -65,6 +65,9 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { } rtpbin.pad_added.connect(on_rtp_pad_added); rtpbin.@set("latency", 100); + rtpbin.@set("do-lost", true); + rtpbin.@set("do-sync-event", true); + rtpbin.@set("drop-on-latency", true); rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); @@ -160,6 +163,17 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { case Gst.MessageType.QOS: // Ignore break; + case Gst.MessageType.LATENCY: + if (message.src != null && message.src.name != null && message.src is Gst.Element) { + Gst.Query latency_query = new Gst.Query.latency(); + if (((Gst.Element)message.src).query(latency_query)) { + bool live; + Gst.ClockTime min_latency, max_latency; + latency_query.parse_latency(out live, out min_latency, out max_latency); + debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string()); + } + } + break; default: debug("Pipe bus message: %s", message.type.to_string()); break; diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 3a63f3fa..23634aa3 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -19,9 +19,12 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { private Gst.App.Src recv_rtp; private Gst.App.Src recv_rtcp; private Gst.Element encode; + private Gst.RTP.BasePayload encode_pay; private Gst.Element decode; + private Gst.RTP.BaseDepayload decode_depay; private Gst.Element input; private Gst.Element output; + private Gst.Element session; private Device _input_device; public Device input_device { get { return _input_device; } set { @@ -85,15 +88,15 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } // Create app elements - send_rtp = Gst.ElementFactory.make("appsink", @"rtp-sink-$rtpid") as Gst.App.Sink; + send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink; send_rtp.async = false; - send_rtp.caps = CodecUtil.get_caps(media, payload_type); + send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); send_rtp.emit_signals = true; send_rtp.sync = false; send_rtp.new_sample.connect(on_new_sample); pipe.add(send_rtp); - send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp-sink-$rtpid") as Gst.App.Sink; + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; send_rtcp.async = false; send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); send_rtcp.emit_signals = true; @@ -101,14 +104,14 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp.new_sample.connect(on_new_sample); pipe.add(send_rtcp); - recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp-src-$rtpid") as Gst.App.Src; - recv_rtp.caps = CodecUtil.get_caps(media, payload_type); + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true); recv_rtp.do_timestamp = true; recv_rtp.format = Gst.Format.TIME; recv_rtp.is_live = true; pipe.add(recv_rtp); - recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp-src-$rtpid") as Gst.App.Src; + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); recv_rtcp.do_timestamp = true; recv_rtcp.format = Gst.Format.TIME; @@ -122,7 +125,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); // Connect input - encode = codec_util.get_encode_bin(media, payload_type, @"encode-$rtpid"); + encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid"); + encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay"); pipe.add(encode); send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); encode.get_static_pad("src").link(send_rtp_sink_pad); @@ -131,7 +135,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } // Connect output - decode = codec_util.get_decode_bin(media, payload_type, @"decode-$rtpid"); + decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); pipe.add(decode); if (output != null) { decode.link(output); @@ -144,6 +149,110 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { created = true; push_recv_data = true; plugin.unpause(); + + GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); + if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + Object internal_session; + session.@get("internal-session", out internal_session); + if (internal_session != null) { + internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this); + } + Timeout.add(1000, () => remb_adjust()); + } + if (media == "video") { + codec_util.update_bitrate(media, payload_type, encode, 256); + } + } + + private uint remb = 256; + private int last_packets_lost = -1; + private uint64 last_packets_received; + private uint64 last_octets_received; + private bool remb_adjust() { + unowned Gst.Structure? stats; + if (session == null) { + debug("Session for %u finished, turning off remb adjustment", rtpid); + return Source.REMOVE; + } + session.get("stats", out stats); + if (stats == null) { + warning("No stats for session %u", rtpid); + return Source.REMOVE; + } + unowned ValueArray? source_stats; + stats.get("source-stats", typeof(ValueArray), out source_stats); + if (source_stats == null) { + warning("No source-stats for session %u", rtpid); + return Source.REMOVE; + } + foreach (Value value in source_stats.values) { + unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); + uint ssrc; + if (!source_stat.get_uint("ssrc", out ssrc)) continue; + if (ssrc.to_string() == participant_ssrc) { + int packets_lost; + uint64 packets_received, octets_received; + source_stat.get_int("packets-lost", out packets_lost); + source_stat.get_uint64("packets-received", out packets_received); + source_stat.get_uint64("octets-received", out octets_received); + int new_lost = packets_lost - last_packets_lost; + uint64 new_received = packets_received - last_packets_received; + uint64 new_octets = octets_received - last_octets_received; + if (new_received == 0) continue; + last_packets_lost = packets_lost; + last_packets_received = packets_received; + last_octets_received = octets_received; + double loss_rate = (double)new_lost / (double)(new_lost + new_received); + if (new_lost <= 0 || loss_rate < 0.02) { + remb = (uint)(1.08 * (double)remb); + } else if (loss_rate > 0.1) { + remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + } + remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); + remb = uint.max(16, remb); // Never go below 16 + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); + data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); + data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); + data[7] = (uint8)(encode_pay.ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = remb * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); + } + } + return Source.CONTINUE; + } + + private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { + if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { + // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 + uint8[] data; + fci.extract_dup(0, fci.get_size(), out data); + if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; + uint8 br_exp = data[5] >> 2; + uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; + uint bitrate = (br_mant << br_exp) / 1000; + self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); + } } private void prepare_local_crypto() { @@ -167,22 +276,26 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { if (crypto_session.has_encrypt) { data = crypto_session.encrypt_rtp(data); } - on_send_rtp_data(new Bytes.take(data)); + on_send_rtp_data(new Bytes.take((owned) data)); } else if (sink == send_rtcp) { - if (crypto_session.has_encrypt) { - data = crypto_session.encrypt_rtcp(data); - } - if (rtcp_mux) { - on_send_rtp_data(new Bytes.take(data)); - } else { - on_send_rtcp_data(new Bytes.take(data)); - } + encrypt_and_send_rtcp((owned) data); } else { warning("unknown sample"); } return Gst.FlowReturn.OK; } + private void encrypt_and_send_rtcp(owned uint8[] data) { + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtcp(data); + } + if (rtcp_mux) { + on_send_rtp_data(new Bytes.take((owned) data)); + } else { + on_send_rtcp_data(new Bytes.take((owned) data)); + } + } + private static Gst.PadProbeReturn drop_probe() { return Gst.PadProbeReturn.DROP; } @@ -211,6 +324,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { encode.get_static_pad("src").unlink(send_rtp_sink_pad); pipe.remove(encode); encode = null; + encode_pay = null; // Disconnect RTP sending if (send_rtp_src_pad != null) { @@ -243,6 +357,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { decode.set_state(Gst.State.NULL); pipe.remove(decode); decode = null; + decode_depay = null; output = null; // Disconnect output device @@ -276,6 +391,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { send_rtcp_src_pad = null; send_rtp_src_pad = null; recv_rtp_src_pad = null; + + session = null; } private void prepare_remote_crypto() { @@ -285,6 +402,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } } + private uint16 previous_video_orientation_degree = uint16.MAX; + public signal void video_orientation_changed(uint16 degree); + public override void on_recv_rtp_data(Bytes bytes) { if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { on_recv_rtcp_data(bytes); @@ -301,6 +421,33 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } if (push_recv_data) { Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + if (rtp_buffer.get_extension()) { + Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); + if (ext != null) { + unowned uint8[] extension_data; + if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) { + bool camera = (extension_data[0] & 0x8) > 0; + bool flip = (extension_data[0] & 0x4) > 0; + uint8 rotation = extension_data[0] & 0x3; + uint16 rotation_degree = uint16.MAX; + switch(rotation) { + case 0: rotation_degree = 0; break; + case 1: rotation_degree = 90; break; + case 2: rotation_degree = 180; break; + case 3: rotation_degree = 270; break; + } + if (rotation_degree != previous_video_orientation_degree) { + video_orientation_changed(rotation_degree); + previous_video_orientation_degree = rotation_degree; + } + } + } + } + rtp_buffer.unmap(); + } + // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of // the underlying buffer, so and some point we should move over to the new version (once we require @@ -449,6 +596,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { public class Dino.Plugins.Rtp.VideoStream : Stream { private Gee.List outputs = new ArrayList(); private Gst.Element output_tee; + private Gst.Element rotate; + private ulong video_orientation_changed_handler; public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { base(plugin, content); @@ -456,11 +605,15 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { } public override void create() { + video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed); plugin.pause(); - output_tee = Gst.ElementFactory.make("tee", null); + rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); + pipe.add(rotate); + output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid"); output_tee.@set("allow-not-linked", true); pipe.add(output_tee); - add_output(output_tee); + rotate.link(output_tee); + add_output(rotate); base.create(); foreach (Gst.Element output in outputs) { output_tee.link(output); @@ -468,19 +621,44 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { plugin.unpause(); } + private void on_video_orientation_changed(uint16 degree) { + if (rotate != null) { + switch (degree) { + case 0: + rotate.@set("method", 0); + break; + case 90: + rotate.@set("method", 1); + break; + case 180: + rotate.@set("method", 2); + break; + case 270: + rotate.@set("method", 3); + break; + } + } + } + public override void destroy() { foreach (Gst.Element output in outputs) { output_tee.unlink(output); } base.destroy(); + rotate.set_locked_state(true); + rotate.set_state(Gst.State.NULL); + rotate.unlink(output_tee); + pipe.remove(rotate); + rotate = null; output_tee.set_locked_state(true); output_tee.set_state(Gst.State.NULL); pipe.remove(output_tee); output_tee = null; + disconnect(video_orientation_changed_handler); } public override void add_output(Gst.Element element) { - if (element == output_tee) { + if (element == output_tee || element == rotate) { base.add_output(element); return; } @@ -491,7 +669,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream { } public override void remove_output(Gst.Element element) { - if (element == output_tee) { + if (element == output_tee || element == rotate) { base.remove_output(element); return; } diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala index fa5ba138..351069a7 100644 --- a/plugins/rtp/src/video_widget.vala +++ b/plugins/rtp/src/video_widget.vala @@ -19,7 +19,7 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge this.plugin = plugin; id = last_id++; - element = Gst.ElementFactory.make("gtksink", @"video-widget-$id"); + element = Gst.ElementFactory.make("gtksink", @"video_widget_$id"); if (element != null) { Gtk.Widget widget; element.@get("widget", out widget); @@ -51,8 +51,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_stream == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoconvert name=video-widget-$id-convert", true); - convert.name = @"video-widget-$id-prepare"; + convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; pipe.add(convert); convert.link(element); connected_stream.add_output(convert); @@ -68,8 +68,8 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidge if (connected_device == null) return; plugin.pause(); pipe.add(element); - convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video-widget-$id-flip ! videoconvert name=video-widget-$id-convert", true); - convert.name = @"video-widget-$id-prepare"; + convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; pipe.add(convert); convert.link(element); connected_device.link_source().link(convert); diff --git a/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi new file mode 100644 index 00000000..30490896 --- /dev/null +++ b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi @@ -0,0 +1,625 @@ +// Fixme: This is fetched from development code of Vala upstream which fixed a few bugs. +/* gstreamer-rtp-1.0.vapi generated by vapigen, do not modify. */ + +[CCode (cprefix = "Gst", gir_namespace = "GstRtp", gir_version = "1.0", lower_case_cprefix = "gst_")] +namespace Gst { + namespace RTCP { + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public bool add_packet (Gst.RTCP.Type type, Gst.RTCP.Packet packet); + public bool get_first_packet (Gst.RTCP.Packet packet); + public uint get_packet_count (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTCP.Buffer rtcp); + public static Gst.Buffer @new (uint mtu); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] owned uint8[] data); + public bool unmap (); + public static bool validate (Gst.Buffer buffer); + public static bool validate_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_data_reduced ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_reduced (Gst.Buffer buffer); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPPacket")] + public struct Packet { + public weak Gst.RTCP.Buffer? rtcp; + public uint offset; + [Version (since = "1.10")] + public bool add_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public bool add_rb (uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + [Version (since = "1.10")] + public uint8 app_get_data (); + [Version (since = "1.10")] + public uint16 app_get_data_length (); + [Version (since = "1.10")] + public unowned string app_get_name (); + [Version (since = "1.10")] + public uint32 app_get_ssrc (); + [Version (since = "1.10")] + public uint8 app_get_subtype (); + [Version (since = "1.10")] + public bool app_set_data_length (uint16 wordlen); + [Version (since = "1.10")] + public void app_set_name (string name); + [Version (since = "1.10")] + public void app_set_ssrc (uint32 ssrc); + [Version (since = "1.10")] + public void app_set_subtype (uint8 subtype); + public bool bye_add_ssrc (uint32 ssrc); + public bool bye_add_ssrcs ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint32[] ssrc); + public uint32 bye_get_nth_ssrc (uint nth); + public string bye_get_reason (); + public uint8 bye_get_reason_len (); + public uint bye_get_ssrc_count (); + public bool bye_set_reason (string reason); + [Version (since = "1.10")] + public bool copy_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out uint8[] data); + public uint8 fb_get_fci (); + public uint16 fb_get_fci_length (); + public uint32 fb_get_media_ssrc (); + public uint32 fb_get_sender_ssrc (); + public Gst.RTCP.FBType fb_get_type (); + public bool fb_set_fci_length (uint16 wordlen); + public void fb_set_media_ssrc (uint32 ssrc); + public void fb_set_sender_ssrc (uint32 ssrc); + public void fb_set_type (Gst.RTCP.FBType type); + public uint8 get_count (); + public uint16 get_length (); + public bool get_padding (); + [Version (since = "1.10")] + public bool get_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.10")] + public uint16 get_profile_specific_ext_length (); + public void get_rb (uint nth, out uint32 ssrc, out uint8 fractionlost, out int32 packetslost, out uint32 exthighestseq, out uint32 jitter, out uint32 lsr, out uint32 dlsr); + public uint get_rb_count (); + public Gst.RTCP.Type get_type (); + public bool move_to_next (); + public bool remove (); + public uint32 rr_get_ssrc (); + public void rr_set_ssrc (uint32 ssrc); + public bool sdes_add_entry (Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] uint8[] data); + public bool sdes_add_item (uint32 ssrc); + public bool sdes_copy_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out uint8[] data); + public bool sdes_first_entry (); + public bool sdes_first_item (); + public bool sdes_get_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out unowned uint8[] data); + public uint sdes_get_item_count (); + public uint32 sdes_get_ssrc (); + public bool sdes_next_entry (); + public bool sdes_next_item (); + public void set_rb (uint nth, uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + public void sr_get_sender_info (out uint32 ssrc, out uint64 ntptime, out uint32 rtptime, out uint32 packet_count, out uint32 octet_count); + public void sr_set_sender_info (uint32 ssrc, uint64 ntptime, uint32 rtptime, uint32 packet_count, uint32 octet_count); + [Version (since = "1.16")] + public bool xr_first_rb (); + [Version (since = "1.16")] + public uint16 xr_get_block_length (); + [Version (since = "1.16")] + public Gst.RTCP.XRType xr_get_block_type (); + [Version (since = "1.16")] + public bool xr_get_dlrr_block (uint nth, out uint32 ssrc, out uint32 last_rr, out uint32 delay); + [Version (since = "1.16")] + public bool xr_get_prt_by_seq (uint16 seq, out uint32 receipt_time); + [Version (since = "1.16")] + public bool xr_get_prt_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_rle_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq, out uint32 chunk_count); + [Version (since = "1.16")] + public bool xr_get_rle_nth_chunk (uint nth, out uint16 chunk); + [Version (since = "1.16")] + public bool xr_get_rrt (out uint64 timestamp); + [Version (since = "1.16")] + public uint32 xr_get_ssrc (); + [Version (since = "1.16")] + public bool xr_get_summary_info (out uint32 ssrc, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_summary_jitter (out uint32 min_jitter, out uint32 max_jitter, out uint32 mean_jitter, out uint32 dev_jitter); + [Version (since = "1.16")] + public bool xr_get_summary_pkt (out uint32 lost_packets, out uint32 dup_packets); + [Version (since = "1.16")] + public bool xr_get_summary_ttl (out bool is_ipv4, out uint8 min_ttl, out uint8 max_ttl, out uint8 mean_ttl, out uint8 dev_ttl); + [Version (since = "1.16")] + public bool xr_get_voip_burst_metrics (out uint8 burst_density, out uint8 gap_density, out uint16 burst_duration, out uint16 gap_duration); + [Version (since = "1.16")] + public bool xr_get_voip_configuration_params (out uint8 gmin, out uint8 rx_config); + [Version (since = "1.16")] + public bool xr_get_voip_delay_metrics (out uint16 roundtrip_delay, out uint16 end_system_delay); + [Version (since = "1.16")] + public bool xr_get_voip_jitter_buffer_params (out uint16 jb_nominal, out uint16 jb_maximum, out uint16 jb_abs_max); + [Version (since = "1.16")] + public bool xr_get_voip_metrics_ssrc (out uint32 ssrc); + [Version (since = "1.16")] + public bool xr_get_voip_packet_metrics (out uint8 loss_rate, out uint8 discard_rate); + [Version (since = "1.16")] + public bool xr_get_voip_quality_metrics (out uint8 r_factor, out uint8 ext_r_factor, out uint8 mos_lq, out uint8 mos_cq); + [Version (since = "1.16")] + public bool xr_get_voip_signal_metrics (out uint8 signal_level, out uint8 noise_level, out uint8 rerl, out uint8 gmin); + [Version (since = "1.16")] + public bool xr_next_rb (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_", type_id = "gst_rtcpfb_type_get_type ()")] + [GIR (name = "RTCPFBType")] + public enum FBType { + FB_TYPE_INVALID, + RTPFB_TYPE_NACK, + RTPFB_TYPE_TMMBR, + RTPFB_TYPE_TMMBN, + RTPFB_TYPE_RTCP_SR_REQ, + RTPFB_TYPE_TWCC, + PSFB_TYPE_PLI, + PSFB_TYPE_SLI, + PSFB_TYPE_RPSI, + PSFB_TYPE_AFB, + PSFB_TYPE_FIR, + PSFB_TYPE_TSTR, + PSFB_TYPE_TSTN, + PSFB_TYPE_VBCN + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_SDES_", type_id = "gst_rtcpsdes_type_get_type ()")] + [GIR (name = "RTCPSDESType")] + public enum SDESType { + INVALID, + END, + CNAME, + NAME, + EMAIL, + PHONE, + LOC, + TOOL, + NOTE, + PRIV; + [CCode (cname = "gst_rtcp_sdes_name_to_type")] + public static Gst.RTCP.SDESType from_string (string name); + [CCode (cname = "gst_rtcp_sdes_type_to_name")] + public unowned string to_string (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_TYPE_", type_id = "gst_rtcp_type_get_type ()")] + [GIR (name = "RTCPType")] + public enum Type { + INVALID, + SR, + RR, + SDES, + BYE, + APP, + RTPFB, + PSFB, + XR + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_XR_TYPE_", type_id = "gst_rtcpxr_type_get_type ()")] + [GIR (name = "RTCPXRType")] + [Version (since = "1.16")] + public enum XRType { + INVALID, + LRLE, + DRLE, + PRT, + RRT, + DLRR, + SSUMM, + VOIP_METRICS + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_BYE_SSRC_COUNT")] + public const int MAX_BYE_SSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_RB_COUNT")] + public const int MAX_RB_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES")] + public const int MAX_SDES; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES_ITEM_COUNT")] + public const int MAX_SDES_ITEM_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_REDUCED_SIZE_VALID_MASK")] + public const int REDUCED_SIZE_VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_MASK")] + public const int VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_VALUE")] + public const int VALID_VALUE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 ntp_to_unix (uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 unix_to_ntp (uint64 unixtime); + } + namespace RTP { + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_audio_payload_get_type ()")] + [GIR (name = "RTPBaseAudioPayload")] + public class BaseAudioPayload : Gst.RTP.BasePayload { + public Gst.ClockTime base_ts; + public int frame_duration; + public int frame_size; + public int sample_size; + [CCode (has_construct_function = false)] + protected BaseAudioPayload (); + public Gst.FlowReturn flush (uint payload_len, Gst.ClockTime timestamp); + public Gst.Base.Adapter get_adapter (); + public Gst.FlowReturn push ([CCode (array_length_cname = "payload_len", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, Gst.ClockTime timestamp); + public void set_frame_based (); + public void set_frame_options (int frame_duration, int frame_size); + public void set_sample_based (); + public void set_sample_options (int sample_size); + public void set_samplebits_options (int sample_size); + [NoAccessorMethod] + public bool buffer_list { get; set; } + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_depayload_get_type ()")] + [GIR (name = "RTPBaseDepayload")] + public abstract class BaseDepayload : Gst.Element { + public uint clock_rate; + public bool need_newsegment; + public weak Gst.Segment segment; + public weak Gst.Pad sinkpad; + public weak Gst.Pad srcpad; + [CCode (has_construct_function = false)] + protected BaseDepayload (); + [NoWrapper] + public virtual bool handle_event (Gst.Event event); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + [NoWrapper] + public virtual bool packet_lost (Gst.Event event); + [NoWrapper] + public virtual Gst.Buffer process (Gst.Buffer @in); + [NoWrapper] + public virtual Gst.Buffer process_rtp_packet (Gst.RTP.Buffer rtp_buffer); + public Gst.FlowReturn push (Gst.Buffer out_buf); + public Gst.FlowReturn push_list (Gst.BufferList out_list); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public int max_reorder { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string? ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_payload_get_type ()")] + [GIR (name = "RTPBasePayload")] + public abstract class BasePayload : Gst.Element { + [CCode (has_construct_function = false)] + protected BasePayload (); + [Version (since = "1.16")] + public Gst.Buffer allocate_output_buffer (uint payload_len, uint8 pad_len, uint8 csrc_count); + [NoWrapper] + public virtual Gst.Caps get_caps (Gst.Pad pad, Gst.Caps filter); + [Version (since = "1.16")] + public uint get_source_count (Gst.Buffer buffer); + [NoWrapper] + public virtual Gst.FlowReturn handle_buffer (Gst.Buffer buffer); + public bool is_filled (uint size, Gst.ClockTime duration); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + public Gst.FlowReturn push (Gst.Buffer buffer); + public Gst.FlowReturn push_list (Gst.BufferList list); + [NoWrapper] + public virtual bool query (Gst.Pad pad, Gst.Query query); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + public void set_options (string media, bool @dynamic, string encoding_name, uint32 clock_rate); + [Version (since = "1.20")] + public bool set_outcaps_structure (Gst.Structure? s); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoWrapper] + public virtual bool sink_event (Gst.Event event); + [NoWrapper] + public virtual bool src_event (Gst.Event event); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + public int64 max_ptime { get; set; } + [NoAccessorMethod] + public int64 min_ptime { get; set; } + [NoAccessorMethod] + public uint mtu { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool onvif_no_rate_control { get; set; } + [NoAccessorMethod] + public bool perfect_rtptime { get; set; } + [NoAccessorMethod] + public uint pt { get; set; } + [NoAccessorMethod] + public int64 ptime_multiple { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public bool scale_rtptime { get; set; } + [NoAccessorMethod] + public uint seqnum { get; } + [NoAccessorMethod] + public int seqnum_offset { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public uint ssrc { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [NoAccessorMethod] + public uint timestamp { get; } + [NoAccessorMethod] + public uint timestamp_offset { get; set; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_header_extension_get_type ()")] + [GIR (name = "RTPHeaderExtension")] + [Version (since = "1.20")] + public abstract class HeaderExtension : Gst.Element { + public uint ext_id; + [CCode (has_construct_function = false)] + protected HeaderExtension (); + public static Gst.RTP.HeaderExtension? create_from_uri (string uri); + public uint get_id (); + public virtual size_t get_max_size (Gst.Buffer input_meta); + public string get_sdp_caps_field_name (); + public virtual Gst.RTP.HeaderExtensionFlags get_supported_flags (); + public unowned string get_uri (); + public virtual bool read (Gst.RTP.HeaderExtensionFlags read_flags, [CCode (array_length_cname = "size", array_length_pos = 2.5, array_length_type = "gsize", type = "const guint8*")] uint8[] data, Gst.Buffer buffer); + public virtual bool set_attributes_from_caps (Gst.Caps caps); + public bool set_attributes_from_caps_simple_sdp (Gst.Caps caps); + public virtual bool set_caps_from_attributes (Gst.Caps caps); + public bool set_caps_from_attributes_simple_sdp (Gst.Caps caps); + public void set_id (uint ext_id); + public virtual bool set_non_rtp_sink_caps (Gst.Caps caps); + [CCode (cname = "gst_rtp_header_extension_class_set_uri")] + public class void set_uri (string uri); + public void set_wants_update_non_rtp_src_caps (bool state); + public virtual bool update_non_rtp_src_caps (Gst.Caps caps); + public virtual size_t write (Gst.Buffer input_meta, Gst.RTP.HeaderExtensionFlags write_flags, Gst.Buffer output, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "gsize", type = "guint8*")] uint8[] data); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public uint state; + [CCode (array_length = false)] + public weak void* data[4]; + [CCode (array_length = false)] + public weak size_t size[4]; + public bool add_extension_onebyte_header (uint8 id, [CCode (array_length_cname = "size", array_length_pos = 2.1, array_length_type = "guint")] uint8[] data); + public bool add_extension_twobytes_header (uint8 appbits, uint8 id, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] uint8[] data); + [CCode (cname = "gst_buffer_add_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? add_rtp_source_meta (Gst.Buffer buffer, uint32? ssrc, uint32? csrc, uint csrc_count); + public static void allocate_data (Gst.Buffer buffer, uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_header_len (uint8 csrc_count); + public static uint calc_packet_len (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_payload_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static int compare_seqnum (uint16 seqnum1, uint16 seqnum2); + public static uint32 default_clock_rate (uint8 payload_type); + public static uint64 ext_timestamp (ref uint64 exttimestamp, uint32 timestamp); + public uint32 get_csrc (uint8 idx); + public uint8 get_csrc_count (); + public bool get_extension (); + [Version (since = "1.2")] + public GLib.Bytes get_extension_bytes (out uint16 bits); + public bool get_extension_data (out uint16 bits, [CCode (array_length = false)] out unowned uint8[] data, out uint wordlen); + public bool get_extension_onebyte_header (uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.18")] + public static bool get_extension_onebyte_header_from_bytes (GLib.Bytes bytes, uint16 bit_pattern, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 5.1, array_length_type = "guint")] out unowned uint8[] data); + public bool get_extension_twobytes_header (out uint8 appbits, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "guint")] out unowned uint8[] data); + public uint get_header_len (); + public bool get_marker (); + public uint get_packet_len (); + public bool get_padding (); + [CCode (array_length = false)] + public unowned uint8[] get_payload (); + public Gst.Buffer get_payload_buffer (); + [Version (since = "1.2")] + public GLib.Bytes get_payload_bytes (); + public uint get_payload_len (); + public Gst.Buffer get_payload_subbuffer (uint offset, uint len); + public uint8 get_payload_type (); + [CCode (cname = "gst_buffer_get_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? get_rtp_source_meta (Gst.Buffer buffer); + public uint16 get_seq (); + public uint32 get_ssrc (); + public uint32 get_timestamp (); + public uint8 get_version (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTP.Buffer rtp); + public static Gst.Buffer new_allocate (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_allocate_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] owned uint8[] data); + public void pad_to (uint len); + public void set_csrc (uint8 idx, uint32 csrc); + public void set_extension (bool extension); + public bool set_extension_data (uint16 bits, uint16 length); + public void set_marker (bool marker); + public void set_packet_len (uint len); + public void set_padding (bool padding); + public void set_payload_type (uint8 payload_type); + public void set_seq (uint16 seq); + public void set_ssrc (uint32 ssrc); + public void set_timestamp (uint32 timestamp); + public void set_version (uint8 version); + public void unmap (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPPayloadInfo")] + public struct PayloadInfo { + public uint8 payload_type; + public weak string media; + public weak string encoding_name; + public uint clock_rate; + public weak string encoding_parameters; + public uint bitrate; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPSourceMeta")] + [Version (since = "1.16")] + public struct SourceMeta { + public Gst.Meta meta; + public uint32 ssrc; + public bool ssrc_valid; + [CCode (array_length = false)] + public weak uint32 csrc[15]; + public uint csrc_count; + public bool append_csrc ([CCode (array_length_cname = "csrc_count", array_length_pos = 1.1, array_length_type = "guint", type = "const guint32*")] uint32[] csrc); + public uint get_source_count (); + public bool set_ssrc (uint32? ssrc); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_FLAG_", type_id = "gst_rtp_buffer_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferFlags")] + [Version (since = "1.10")] + public enum BufferFlags { + RETRANSMISSION, + REDUNDANT, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_MAP_FLAG_", type_id = "gst_rtp_buffer_map_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferMapFlags")] + [Version (since = "1.6.1")] + public enum BufferMapFlags { + SKIP_PADDING, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_HEADER_EXTENSION_", type_id = "gst_rtp_header_extension_flags_get_type ()")] + [Flags] + [GIR (name = "RTPHeaderExtensionFlags")] + [Version (since = "1.20")] + public enum HeaderExtensionFlags { + ONE_BYTE, + TWO_BYTE + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PAYLOAD_", type_id = "gst_rtp_payload_get_type ()")] + [GIR (name = "RTPPayload")] + public enum Payload { + PCMU, + @1016, + G721, + GSM, + G723, + DVI4_8000, + DVI4_16000, + LPC, + PCMA, + G722, + L16_STEREO, + L16_MONO, + QCELP, + CN, + MPA, + G728, + DVI4_11025, + DVI4_22050, + G729, + CELLB, + JPEG, + NV, + H261, + MPV, + MP2T, + H263; + public const string @1016_STRING; + public const string CELLB_STRING; + public const string CN_STRING; + public const string DVI4_11025_STRING; + public const string DVI4_16000_STRING; + public const string DVI4_22050_STRING; + public const string DVI4_8000_STRING; + public const string DYNAMIC_STRING; + public const string G721_STRING; + public const string G722_STRING; + public const int G723_53; + public const string G723_53_STRING; + public const int G723_63; + public const string G723_63_STRING; + public const string G723_STRING; + public const string G728_STRING; + public const string G729_STRING; + public const string GSM_STRING; + public const string H261_STRING; + public const string H263_STRING; + public const string JPEG_STRING; + public const string L16_MONO_STRING; + public const string L16_STEREO_STRING; + public const string LPC_STRING; + public const string MP2T_STRING; + public const string MPA_STRING; + public const string MPV_STRING; + public const string NV_STRING; + public const string PCMA_STRING; + public const string PCMU_STRING; + public const string QCELP_STRING; + public const int TS41; + public const string TS41_STRING; + public const int TS48; + public const string TS48_STRING; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PROFILE_", type_id = "gst_rtp_profile_get_type ()")] + [GIR (name = "RTPProfile")] + [Version (since = "1.6")] + public enum Profile { + UNKNOWN, + AVP, + SAVP, + AVPF, + SAVPF + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_BASE")] + public const string HDREXT_BASE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_ELEMENT_CLASS")] + [Version (since = "1.20")] + public const string HDREXT_ELEMENT_CLASS; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56")] + public const string HDREXT_NTP_56; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56_SIZE")] + public const int HDREXT_NTP_56_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64")] + public const string HDREXT_NTP_64; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64_SIZE")] + public const int HDREXT_NTP_64_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HEADER_EXTENSION_URI_METADATA_KEY")] + [Version (since = "1.20")] + public const string HEADER_EXTENSION_URI_METADATA_KEY; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_SOURCE_META_MAX_CSRC_COUNT")] + public const int SOURCE_META_MAX_CSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + [Version (since = "1.20")] + public static GLib.List get_header_extension_list (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_56 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_64 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_56 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_64 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_name (string media, string encoding_name); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_pt (uint8 payload_type); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static GLib.Type source_meta_api_get_type (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.MetaInfo? source_meta_get_info (); + } +} diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala index d6f1acd2..d4440169 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/content_parameters.vala @@ -17,6 +17,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { public bool encryption_required { get; private set; default = false; } public PayloadType? agreed_payload_type { get; private set; } public Gee.List payload_types = new ArrayList(PayloadType.equals_func); + public Gee.List header_extensions = new ArrayList(); public Gee.List remote_cryptos = new ArrayList(); public Crypto? local_crypto = null; public Crypto? remote_crypto = null; @@ -54,9 +55,12 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { this.remote_cryptos.add(Crypto.parse(crypto)); } } - foreach (StanzaNode payloadType in node.get_subnodes("payload-type")) { + foreach (StanzaNode payloadType in node.get_subnodes(PayloadType.NAME)) { this.payload_types.add(PayloadType.parse(payloadType)); } + foreach (StanzaNode subnode in node.get_subnodes(HeaderExtension.NAME, HeaderExtension.NS_URI)) { + this.header_extensions.add(HeaderExtension.parse(subnode)); + } } public async void handle_proposed_content(XmppStream stream, Jingle.Session session, Jingle.Content content) { @@ -66,6 +70,11 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { content.reject(); return; } + // Drop unsupported header extensions + var iter = header_extensions.iterator(); + while(iter.next()) { + if (!parent.is_header_extension_supported(media, iter.@get())) iter.remove(); + } remote_crypto = parent.pick_remote_crypto(remote_cryptos); if (local_crypto == null && remote_crypto != null) { local_crypto = parent.pick_local_crypto(remote_crypto); @@ -151,7 +160,7 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { Gee.List crypto_nodes = description_node.get_deep_subnodes("encryption", "crypto"); if (crypto_nodes.size == 0) { - warning("Counterpart didn't include any cryptos"); + debug("Counterpart didn't include any cryptos"); if (encryption_required) { return; } @@ -182,6 +191,9 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { ret.put_node(payload_type.to_xml()); } } + foreach (HeaderExtension ext in header_extensions) { + ret.put_node(ext.to_xml()); + } if (local_crypto != null) { ret.put_node(new StanzaNode.build("encryption", NS_URI) .put_node(local_crypto.to_xml())); @@ -191,4 +203,28 @@ public class Xmpp.Xep.JingleRtp.Parameters : Jingle.ContentParameters, Object { } return ret; } +} + +public class Xmpp.Xep.JingleRtp.HeaderExtension { + public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtp-hdrext:0"; + public const string NAME = "rtp-hdrext"; + + public uint8 id { get; private set; } + public string uri { get; private set; } + + public HeaderExtension(uint8 id, string uri) { + this.id = id; + this.uri = uri; + } + + public static HeaderExtension parse(StanzaNode node) { + return new HeaderExtension((uint8) node.get_attribute_int("id"), node.get_attribute("uri")); + } + + public StanzaNode to_xml() { + return new StanzaNode.build(NAME, NS_URI) + .add_self_xmlns() + .put_attribute("id", id.to_string()) + .put_attribute("uri", uri); + } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 3adad114..6eb6289b 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -24,6 +24,8 @@ public abstract class Module : XmppStreamModule { public abstract Crypto? pick_remote_crypto(Gee.List cryptos); public abstract Crypto? pick_local_crypto(Crypto? remote); public abstract Stream create_stream(Jingle.Content content); + public abstract bool is_header_extension_supported(string media, HeaderExtension ext); + public abstract Gee.List get_suggested_header_extensions(string media); public abstract void close_stream(Stream stream); public async Jingle.Session start_call(XmppStream stream, Jid receiver_full_jid, bool video, string? sid = null) throws Jingle.Error { @@ -40,6 +42,7 @@ public abstract class Module : XmppStreamModule { // Create audio content Parameters audio_content_parameters = new Parameters(this, "audio", yield get_supported_payloads("audio")); audio_content_parameters.local_crypto = generate_local_crypto(); + audio_content_parameters.header_extensions.add_all(get_suggested_header_extensions("audio")); Jingle.Transport? audio_transport = yield jingle_module.select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (audio_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable audio transports"); @@ -57,6 +60,7 @@ public abstract class Module : XmppStreamModule { // Create video content Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); video_content_parameters.local_crypto = generate_local_crypto(); + video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); @@ -98,6 +102,7 @@ public abstract class Module : XmppStreamModule { // Content for video does not yet exist -> create it Parameters video_content_parameters = new Parameters(this, "video", yield get_supported_payloads("video")); video_content_parameters.local_crypto = generate_local_crypto(); + video_content_parameters.header_extensions.add_all(get_suggested_header_extensions("video")); Jingle.Transport? video_transport = yield stream.get_module(Jingle.Module.IDENTITY).select_transport(stream, content_type.required_transport_type, content_type.required_components, receiver_full_jid, Set.empty()); if (video_transport == null) { throw new Jingle.Error.NO_SHARED_PROTOCOLS("No suitable video transports"); diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala index 452f1d65..faba38c9 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/payload_type.vala @@ -3,6 +3,8 @@ using Xmpp; using Xmpp.Xep; public class Xmpp.Xep.JingleRtp.PayloadType { + public const string NAME = "payload-type"; + public uint8 id { get; set; } public string? name { get; set; } public uint8 channels { get; set; default = 1; } @@ -10,6 +12,7 @@ public class Xmpp.Xep.JingleRtp.PayloadType { public uint32 maxptime { get; set; } public uint32 ptime { get; set; } public Map parameters = new HashMap(); + public Gee.List rtcp_fbs = new ArrayList(); public static PayloadType parse(StanzaNode node) { PayloadType payloadType = new PayloadType(); @@ -22,11 +25,14 @@ public class Xmpp.Xep.JingleRtp.PayloadType { foreach (StanzaNode parameter in node.get_subnodes("parameter")) { payloadType.parameters[parameter.get_attribute("name")] = parameter.get_attribute("value"); } + foreach (StanzaNode subnode in node.get_subnodes(RtcpFeedback.NAME, RtcpFeedback.NS_URI)) { + payloadType.rtcp_fbs.add(RtcpFeedback.parse(subnode)); + } return payloadType; } public StanzaNode to_xml() { - StanzaNode node = new StanzaNode.build("payload-type", NS_URI) + StanzaNode node = new StanzaNode.build(NAME, NS_URI) .put_attribute("id", id.to_string()); if (channels != 1) node.put_attribute("channels", channels.to_string()); if (clockrate != 0) node.put_attribute("clockrate", clockrate.to_string()); @@ -38,9 +44,25 @@ public class Xmpp.Xep.JingleRtp.PayloadType { .put_attribute("name", parameter) .put_attribute("value", parameters[parameter])); } + foreach (RtcpFeedback rtcp_fb in rtcp_fbs) { + node.put_node(rtcp_fb.to_xml()); + } return node; } + public PayloadType clone() { + PayloadType clone = new PayloadType(); + clone.id = id; + clone.name = name; + clone.channels = channels; + clone.clockrate = clockrate; + clone.maxptime = maxptime; + clone.ptime = ptime; + clone.parameters.set_all(parameters); + clone.rtcp_fbs.add_all(rtcp_fbs); + return clone; + } + public static bool equals_func(PayloadType a, PayloadType b) { return a.id == b.id && a.name == b.name && @@ -49,4 +71,29 @@ public class Xmpp.Xep.JingleRtp.PayloadType { a.maxptime == b.maxptime && a.ptime == b.ptime; } +} + +public class Xmpp.Xep.JingleRtp.RtcpFeedback { + public const string NS_URI = "urn:xmpp:jingle:apps:rtp:rtcp-fb:0"; + public const string NAME = "rtcp-fb"; + + public string type_ { get; private set; } + public string? subtype { get; private set; } + + public RtcpFeedback(string type, string? subtype = null) { + this.type_ = type; + this.subtype = subtype; + } + + public static RtcpFeedback parse(StanzaNode node) { + return new RtcpFeedback(node.get_attribute("type"), node.get_attribute("subtype")); + } + + public StanzaNode to_xml() { + StanzaNode node = new StanzaNode.build(NAME, NS_URI) + .add_self_xmlns() + .put_attribute("type", type_); + if (subtype != null) node.put_attribute("subtype", subtype); + return node; + } } \ No newline at end of file diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala index adae11f5..65be8a0a 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/stream.vala @@ -33,6 +33,13 @@ public abstract class Xmpp.Xep.JingleRtp.Stream : Object { } return null; }} + public Gee.List? header_extensions { get { + var content_params = content.content_params; + if (content_params is Parameters) { + return ((Parameters)content_params).header_extensions; + } + return null; + }} public bool sending { get { return content.session.senders_include_us(content.senders); }} -- cgit v1.2.3-70-g09d2 From 5d85b6cdb0165d863aadd25d9a73707b8f5cc83e Mon Sep 17 00:00:00 2001 From: fiaxh Date: Sat, 17 Apr 2021 14:50:31 +0200 Subject: Handle non-existant call support --- libdino/src/plugin/interfaces.vala | 2 ++ libdino/src/service/calls.vala | 38 ++++++++++++++++++++--- main/src/ui/conversation_titlebar/call_entry.vala | 13 +++++--- plugins/rtp/src/codec_util.vala | 1 + plugins/rtp/src/module.vala | 6 ++-- plugins/rtp/src/plugin.vala | 16 ++++++++++ 6 files changed, 65 insertions(+), 11 deletions(-) (limited to 'plugins/rtp/src') diff --git a/libdino/src/plugin/interfaces.vala b/libdino/src/plugin/interfaces.vala index 8be77895..97951850 100644 --- a/libdino/src/plugin/interfaces.vala +++ b/libdino/src/plugin/interfaces.vala @@ -85,6 +85,8 @@ public abstract interface ConversationAdditionPopulator : ConversationItemPopula } public abstract interface VideoCallPlugin : Object { + + public abstract bool supports(string media); // Video widget public abstract VideoCallWidget? create_widget(WidgetType type); diff --git a/libdino/src/service/calls.vala b/libdino/src/service/calls.vala index b7374607..1d47823d 100644 --- a/libdino/src/service/calls.vala +++ b/libdino/src/service/calls.vala @@ -83,7 +83,7 @@ namespace Dino { we_should_send_video[call] = video; we_should_send_audio[call] = true; - if (yield has_jmi_resources(conversation)) { + if (has_jmi_resources(conversation)) { XmppStream? stream = stream_interactor.get_stream(conversation.account); jmi_call[conversation.account] = call; jmi_video[conversation.account] = video; @@ -245,8 +245,28 @@ namespace Dino { // If video_feed == null && !mute we're trying to mute a non-existant feed. It will be muted as soon as it is created. } - public async bool can_do_calls(Conversation conversation) { - return (yield get_call_resources(conversation)).size > 0 || yield has_jmi_resources(conversation); + public async bool can_do_audio_calls_async(Conversation conversation) { + if (!can_do_audio_calls()) return false; + return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); + } + + private bool can_do_audio_calls() { + Plugins.VideoCallPlugin? plugin = Application.get_default().plugin_registry.video_call_plugin; + if (plugin == null) return false; + + return plugin.supports("audio"); + } + + public async bool can_do_video_calls_async(Conversation conversation) { + if (!can_do_video_calls()) return false; + return (yield get_call_resources(conversation)).size > 0 || has_jmi_resources(conversation); + } + + private bool can_do_video_calls() { + Plugins.VideoCallPlugin? plugin = Application.get_default().plugin_registry.video_call_plugin; + if (plugin == null) return false; + + return plugin.supports("video"); } private async Gee.List get_call_resources(Conversation conversation) { @@ -266,7 +286,7 @@ namespace Dino { return ret; } - private async bool has_jmi_resources(Conversation conversation) { + private bool has_jmi_resources(Conversation conversation) { int64 jmi_resources = db.entity.select() .with(db.entity.jid_id, "=", db.get_jid_id(conversation.counterpart)) .join_with(db.entity_feature, db.entity.caps_hash, db.entity_feature.entity) @@ -289,6 +309,11 @@ namespace Dino { } private void on_incoming_call(Account account, Xep.Jingle.Session session) { + if (!can_do_audio_calls()) { + warning("Incoming call but no call support detected. Ignoring."); + return; + } + bool counterpart_wants_video = false; foreach (Xep.Jingle.Content content in session.contents) { Xep.JingleRtp.Parameters? rtp_content_parameter = content.content_params as Xep.JingleRtp.Parameters; @@ -550,6 +575,11 @@ namespace Dino { Xep.JingleMessageInitiation.Module mi_module = stream_interactor.module_manager.get_module(account, Xep.JingleMessageInitiation.Module.IDENTITY); mi_module.session_proposed.connect((from, to, sid, descriptions) => { + if (!can_do_audio_calls()) { + warning("Incoming call but no call support detected. Ignoring."); + return; + } + bool audio_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "audio"); bool video_requested = descriptions.any_match((description) => description.ns_uri == Xep.JingleRtp.NS_URI && description.get_attribute("media") == "video"); if (!audio_requested && !video_requested) return; diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index 5e28ecbe..e1d10e5c 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -34,6 +34,9 @@ namespace Dino.Ui { private StreamInteractor stream_interactor; private Conversation conversation; + private ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; + private ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + public CallButton(StreamInteractor stream_interactor) { this.stream_interactor = stream_interactor; @@ -42,7 +45,6 @@ namespace Dino.Ui { Gtk.PopoverMenu popover_menu = new Gtk.PopoverMenu(); Box box = new Box(Orientation.VERTICAL, 0) { margin=10, visible=true }; - ModelButton audio_button = new ModelButton() { text="Audio call", visible=true }; audio_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, false, (_, res) => { Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); @@ -50,7 +52,7 @@ namespace Dino.Ui { }); }); box.add(audio_button); - ModelButton video_button = new ModelButton() { text="Video call", visible=true }; + video_button.clicked.connect(() => { stream_interactor.get_module(Calls.IDENTITY).initiate_call.begin(conversation, true, (_, res) => { Call call = stream_interactor.get_module(Calls.IDENTITY).initiate_call.end(res); @@ -116,9 +118,12 @@ namespace Dino.Ui { private async void update_visibility() { if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; - bool can_do_calls = yield stream_interactor.get_module(Calls.IDENTITY).can_do_calls(conversation); + bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); + bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); if (conv_bak != conversation) return; - visible = can_do_calls; + + visible = audio_works; + video_button.visible = video_works; } else { visible = false; } diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala index 7537c11d..6a2438f1 100644 --- a/plugins/rtp/src/codec_util.vala +++ b/plugins/rtp/src/codec_util.vala @@ -225,6 +225,7 @@ public class Dino.Plugins.Rtp.CodecUtil { } public string? get_encode_element_name(string media, string? codec) { + if (!is_element_supported(get_pay_element_name(media, codec))) return null; foreach (string candidate in get_encode_candidates(media, codec)) { if (is_element_supported(candidate)) return candidate; } diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 52cc1880..13a21cd8 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -64,13 +64,13 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { } private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { - string codec = CodecUtil.get_codec_from_payload(media, payload_type); + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); if (codec == null) return false; if (unsupported_codecs.contains(codec)) return false; if (supported_codecs.contains(codec)) return true; - string encode_element = codec_util.get_encode_element_name(media, codec); - string decode_element = codec_util.get_decode_element_name(media, codec); + string? encode_element = codec_util.get_encode_element_name(media, codec); + string? decode_element = codec_util.get_decode_element_name(media, codec); if (encode_element == null || decode_element == null) { debug("No suitable encoder or decoder found for %s", codec); unsupported_codecs.add(codec); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index f0ad7db2..d43588b4 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -278,6 +278,22 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { Gst.deinit(); } + public bool supports(string media) { + if (rtpbin == null) return false; + + if (media == "audio") { + if (get_devices("audio", false).is_empty) return false; + if (get_devices("audio", true).is_empty) return false; + } + + if (media == "video") { + if (Gst.ElementFactory.make("gtksink", null) == null) return false; + if (get_devices("video", false).is_empty) return false; + } + + return true; + } + public VideoCallWidget? create_widget(WidgetType type) { if (type == WidgetType.GTK) { return new VideoWidget(this); -- cgit v1.2.3-70-g09d2 From 23ffd37dded3bf872e42d7a00727ab3c4d105a97 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 15:19:05 +0200 Subject: Echo Cancellation --- CMakeLists.txt | 6 +- cmake/FindGstAudio.cmake | 14 +++ cmake/FindWebRTCAudioProcessing.cmake | 12 ++ plugins/rtp/CMakeLists.txt | 20 +++- plugins/rtp/src/device.vala | 30 +++-- plugins/rtp/src/plugin.vala | 5 +- plugins/rtp/src/voice_processor.vala | 176 +++++++++++++++++++++++++++++ plugins/rtp/src/voice_processor_native.cpp | 141 +++++++++++++++++++++++ 8 files changed, 385 insertions(+), 19 deletions(-) create mode 100644 cmake/FindGstAudio.cmake create mode 100644 cmake/FindWebRTCAudioProcessing.cmake create mode 100644 plugins/rtp/src/voice_processor.vala create mode 100644 plugins/rtp/src/voice_processor_native.cpp (limited to 'plugins/rtp/src') diff --git a/CMakeLists.txt b/CMakeLists.txt index f480b0b2..b3bd35cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,11 +2,11 @@ cmake_minimum_required(VERSION 3.3) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) include(ComputeVersion) if (NOT VERSION_FOUND) - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) elseif (VERSION_IS_RELEASE) - project(Dino VERSION ${VERSION_FULL} LANGUAGES C) + project(Dino VERSION ${VERSION_FULL} LANGUAGES C CXX) else () - project(Dino LANGUAGES C) + project(Dino LANGUAGES C CXX) set(PROJECT_VERSION ${VERSION_FULL}) endif () diff --git a/cmake/FindGstAudio.cmake b/cmake/FindGstAudio.cmake new file mode 100644 index 00000000..d5fc5dfb --- /dev/null +++ b/cmake/FindGstAudio.cmake @@ -0,0 +1,14 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(GstAudio + PKG_CONFIG_NAME gstreamer-audio-1.0 + LIB_NAMES gstaudio + LIB_DIR_HINTS gstreamer-1.0 + INCLUDE_NAMES gst/audio/audio.h + INCLUDE_DIR_SUFFIXES gstreamer-1.0 gstreamer-1.0/include gstreamer-audio-1.0 gstreamer-audio-1.0/include + DEPENDS Gst +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GstAudio + REQUIRED_VARS GstAudio_LIBRARY + VERSION_VAR GstAudio_VERSION) diff --git a/cmake/FindWebRTCAudioProcessing.cmake b/cmake/FindWebRTCAudioProcessing.cmake new file mode 100644 index 00000000..5f17805d --- /dev/null +++ b/cmake/FindWebRTCAudioProcessing.cmake @@ -0,0 +1,12 @@ +include(PkgConfigWithFallback) +find_pkg_config_with_fallback(WebRTCAudioProcessing + PKG_CONFIG_NAME webrtc-audio-processing + LIB_NAMES webrtc_audio_processing + INCLUDE_NAMES webrtc/modules/audio_processing/include/audio_processing.h + INCLUDE_DIR_SUFFIXES webrtc-audio-processing webrtc_audio_processing +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(WebRTCAudioProcessing + REQUIRED_VARS WebRTCAudioProcessing_LIBRARY + VERSION_VAR WebRTCAudioProcessing_VERSION) diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index 92ec1b97..b19c8a8f 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(GstRtp REQUIRED) +find_package(WebRTCAudioProcessing 0.2) find_packages(RTP_PACKAGES REQUIRED Gee GLib @@ -8,12 +9,26 @@ find_packages(RTP_PACKAGES REQUIRED GTK3 Gst GstApp + GstAudio ) if(Gst_VERSION VERSION_GREATER "1.16") set(RTP_DEFINITIONS GST_1_16) endif() +if(WebRTCAudioProcessing_VERSION GREATER "0.4") + message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + unset(WebRTCAudioProcessing_FOUND) +endif() + +if(WebRTCAudioProcessing_FOUND) + set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) + set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) + set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) +else() + message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") +endif() + vala_precompile(RTP_VALA_C SOURCES src/codec_util.vala @@ -23,6 +38,7 @@ SOURCES src/stream.vala src/video_widget.vala src/register_plugin.vala + ${RTP_VOICE_PROCESSOR_VALA} CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi @@ -36,8 +52,8 @@ DEFINITIONS ) add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) -add_library(rtp SHARED ${RTP_VALA_C}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0) +add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 785f853a..f8894502 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -37,6 +37,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { private Gst.Element dsp; private Gst.Element mixer; private Gst.Element filter; + private Gst.Element rate; private int links = 0; public Device(Plugin plugin, Gst.Device device) { @@ -132,12 +133,10 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { pipe.add(filter); element.link(filter); if (media == "audio" && plugin.echoprobe != null) { - dsp = Gst.ElementFactory.make("webrtcdsp", @"dsp_$id"); - if (dsp != null) { - dsp.@set("probe", plugin.echoprobe.name); - pipe.add(dsp); - filter.link(dsp); - } + dsp = new VoiceProcessor(plugin.echoprobe, element as Gst.Audio.StreamVolume); + dsp.name = @"dsp_$id"; + pipe.add(dsp); + filter.link(dsp); } tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); @@ -153,7 +152,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); if (plugin.echoprobe != null) { - filter.link(plugin.echoprobe); + rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); + rate.@set("tolerance", 100000000); + pipe.add(rate); + filter.link(rate); + rate.link(plugin.echoprobe); plugin.echoprobe.link(element); } else { filter.link(element); @@ -184,14 +187,17 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { if (filter != null) { filter.set_locked_state(true); filter.set_state(Gst.State.NULL); - if (plugin.echoprobe != null) { - filter.unlink(plugin.echoprobe); - } else { - filter.unlink(element); - } + filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); pipe.remove(filter); filter = null; } + if (rate != null) { + rate.set_locked_state(true); + rate.set_state(Gst.State.NULL); + rate.unlink(plugin.echoprobe); + pipe.remove(rate); + rate = null; + } if (plugin.echoprobe != null) { plugin.echoprobe.unlink(element); } diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index d43588b4..e3d5ee41 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -8,7 +8,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Gst.DeviceMonitor device_monitor { get; private set; } public Gst.Pipeline pipe { get; private set; } public Gst.Bin rtpbin { get; private set; } - public Gst.Element echoprobe { get; private set; } + public EchoProbe echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -72,7 +72,8 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.add(rtpbin); // Audio echo probe - echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); +// echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); + echoprobe = new EchoProbe(); if (echoprobe != null) pipe.add(echoprobe); // Pipeline diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala new file mode 100644 index 00000000..e6dc7e8f --- /dev/null +++ b/plugins/rtp/src/voice_processor.vala @@ -0,0 +1,176 @@ +using Gst; + +namespace Dino.Plugins.Rtp { +public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); +} + +public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + public signal void on_new_buffer(Buffer buffer); + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team "); + } + + construct { + set_passthrough(true); + } + + public override bool setup(Audio.Info info) { + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + return true; + } + + + public override FlowReturn transform_ip(Buffer buf) { + lock (adapter) { + adapter.push(adjust_to_running_time(this, buf)); + while (adapter.available() > period_size) { + on_new_buffer(adapter.take_buffer(period_size)); + } + } + return FlowReturn.OK; + } + + public override bool stop() { + adapter.clear(); + return true; + } +} + +public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + private ulong process_outgoing_buffer_handler_id; + private uint adjust_delay_timeout_id; + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + private EchoProbe? echo_probe; + private Audio.StreamVolume? stream_volume; + private ClockTime last_reverse; + private void* native; + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team "); + } + + construct { + set_passthrough(false); + } + + public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { + this.echo_probe = echo_probe; + this.stream_volume = stream_volume; + } + + private static extern void* init_native(int stream_delay); + private static extern void setup_native(void* native); + private static extern void destroy_native(void* native); + private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void adjust_stream_delay(void* native); + private static extern void notify_gain_level(void* native, int gain_level); + private static extern int get_suggested_gain_level(void* native); + private static extern bool get_stream_has_voice(void* native); + + public override bool setup(Audio.Info info) { + debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + adapter.clear(); + setup_native(native); + return true; + } + + public override bool start() { + native = init_native(150); + if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { + process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); + } + if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { + stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; + } + return true; + } + + private bool adjust_delay() { + if (native != null) { + adjust_stream_delay(native); + return Source.CONTINUE; + } else { + adjust_delay_timeout_id = 0; + return Source.REMOVE; + } + } + + private void process_outgoing_buffer(Buffer buffer) { + if (buffer.pts != uint64.MAX) { + last_reverse = buffer.pts; + } + analyze_reverse_stream(native, echo_probe.audio_info, buffer); + if (adjust_delay_timeout_id == 0 && echo_probe != null) { + adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); + } + } + + public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { + lock (adapter) { + if (is_discont) { + adapter.clear(); + } + adapter.push(adjust_to_running_time(this, input)); + } + return FlowReturn.OK; + } + + public override FlowReturn generate_output(out Buffer output_buffer) { + lock (adapter) { + if (adapter.available() >= period_size) { + output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); + int old_gain_level = 0; + if (stream_volume != null) { + old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); + notify_gain_level(native, old_gain_level); + } + process_stream(native, audio_info, output_buffer); + if (stream_volume != null) { + int new_gain_level = get_suggested_gain_level(native); + if (old_gain_level != new_gain_level) { + debug("Gain: %i -> %i", old_gain_level, new_gain_level); + stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); + } + } + } + } + return FlowReturn.OK; + } + + public override bool stop() { + if (process_outgoing_buffer_handler_id != 0) { + echo_probe.disconnect(process_outgoing_buffer_handler_id); + process_outgoing_buffer_handler_id = 0; + } + if (adjust_delay_timeout_id != 0) { + Source.remove(adjust_delay_timeout_id); + adjust_delay_timeout_id = 0; + } + adapter.clear(); + destroy_native(native); + native = null; + return true; + } +} \ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp new file mode 100644 index 00000000..9b3292b8 --- /dev/null +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include + +#define SAMPLE_RATE 48000 +#define SAMPLE_CHANNELS 1 + +struct _DinoPluginsRtpVoiceProcessorNative { + webrtc::AudioProcessing *apm; + gint stream_delay; +}; + +extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { + GstBuffer *copy = gst_buffer_copy(buffer); + GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); + return copy; +} + +extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { + _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative(); + webrtc::Config config; + config.Set(new webrtc::ExtendedFilter(true)); + config.Set(new webrtc::ExperimentalAgc(true, 85)); + native->apm = webrtc::AudioProcessing::Create(config); + native->stream_delay = stream_delay; + return native; +} + +extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + webrtc::ProcessingConfig pconfig; + pconfig.streams[webrtc::ProcessingConfig::kInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + apm->Initialize(pconfig); + apm->high_pass_filter()->Enable(true); + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); + apm->echo_cancellation()->enable_delay_logging(true); + apm->echo_cancellation()->Enable(true); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); + apm->noise_suppression()->Enable(true); + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); + apm->gain_control()->set_target_level_dbfs(3); + apm->gain_control()->set_compression_gain_db(9); + apm->gain_control()->enable_limiter(true); + apm->gain_control()->Enable(true); + apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); + apm->voice_detection()->Enable(true); +} + +extern "C" void +dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + int err = apm->AnalyzeReverseStream(&frame); + if (err < 0) g_warning("ProcessReverseStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + apm->gain_control()->set_stream_analog_level(gain_level); +} + +extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->gain_control()->stream_analog_level(); +} + +extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->voice_detection()->stream_has_voice(); +} + +extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + int median, std; + float fraction_poor_delays; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); + if (fraction_poor_delays < 0) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); + if (fraction_poor_delays > 0.5) { + native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); + g_debug("Adjusted stream delay %i", native->stream_delay); + } +} + +extern "C" void +dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstAudioBuffer audio_buffer; + gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = info->rate / 100; + memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms(native->stream_delay); + int err = apm->ProcessStream(&frame); + if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); + if (err < 0) g_warning("ProcessStream %i", err); + + gst_audio_buffer_unmap(&audio_buffer); +} + +extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + delete native; +} \ No newline at end of file -- cgit v1.2.3-70-g09d2 From d388525fc69ab688a90f19d2d2499e0f6f10f573 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 16:00:37 +0200 Subject: Correctly handle missing webrtc-audio-processing --- .github/workflows/build.yml | 2 +- plugins/rtp/src/device.vala | 4 +++- plugins/rtp/src/plugin.vala | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) (limited to 'plugins/rtp/src') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce12d441..60d587e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get update - - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev + - run: sudo apt-get install -y build-essential gettext cmake valac libgee-0.8-dev libsqlite3-dev libgtk-3-dev libnotify-dev libgpgme-dev libsoup2.4-dev libgcrypt20-dev libqrencode-dev libgspell-1-dev libnice-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libsrtp2-dev libwebrtc-audio-processing-dev - run: ./configure --with-tests --with-libsignal-in-tree - run: make - run: build/xmpp-vala-test diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index f8894502..3c650ad6 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -132,12 +132,14 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { filter.@set("caps", get_best_caps()); pipe.add(filter); element.link(filter); +#if WITH_VOICE_PROCESSOR if (media == "audio" && plugin.echoprobe != null) { - dsp = new VoiceProcessor(plugin.echoprobe, element as Gst.Audio.StreamVolume); + dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); dsp.name = @"dsp_$id"; pipe.add(dsp); filter.link(dsp); } +#endif tee = Gst.ElementFactory.make("tee", @"tee_$id"); tee.@set("allow-not-linked", true); pipe.add(tee); diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index e3d5ee41..f575a7d0 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -8,7 +8,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { public Gst.DeviceMonitor device_monitor { get; private set; } public Gst.Pipeline pipe { get; private set; } public Gst.Bin rtpbin { get; private set; } - public EchoProbe echoprobe { get; private set; } + public Gst.Element echoprobe { get; private set; } private Gee.List streams = new ArrayList(); private Gee.List devices = new ArrayList(); @@ -71,10 +71,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { rtpbin.connect("signal::request-pt-map", request_pt_map, this); pipe.add(rtpbin); +#if WITH_VOICE_PROCESSOR // Audio echo probe -// echoprobe = Gst.ElementFactory.make("webrtcechoprobe", "echo-probe"); echoprobe = new EchoProbe(); if (echoprobe != null) pipe.add(echoprobe); +#endif // Pipeline pipe.auto_flush_bus = true; -- cgit v1.2.3-70-g09d2 From 0409f554268c0e8f24e23e471a94de4d3a035ff1 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sat, 1 May 2021 17:27:55 +0200 Subject: Fix webcam framerate selection --- plugins/rtp/src/device.vala | 33 ++++++++++++++++++++++++++++-- plugins/rtp/src/plugin.vala | 6 +----- plugins/rtp/src/voice_processor_native.cpp | 6 +++--- 3 files changed, 35 insertions(+), 10 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala index 3c650ad6..e25271b1 100644 --- a/plugins/rtp/src/device.vala +++ b/plugins/rtp/src/device.vala @@ -87,6 +87,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); } else if (media == "video" && device.caps.get_size() > 0) { int best_index = 0; + Value? best_fraction = null; int best_fps = 0; int best_width = 0; int best_height = 0; @@ -94,7 +95,28 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { unowned Gst.Structure? that = device.caps.get_structure(i); if (!that.has_name("video/x-raw")) continue; int num = 0, den = 0, width = 0, height = 0; - if (!that.has_field("framerate") || !that.get_fraction("framerate", out num, out den)) continue; + if (!that.has_field("framerate")) continue; + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.Fraction)) { + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + } else if (framerate.type() == typeof(Gst.ValueList)) { + for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { + Value fraction = Gst.ValueList.get_value(framerate, j); + int in_num = Gst.Value.get_fraction_numerator(fraction); + int in_den = Gst.Value.get_fraction_denominator(fraction); + int fps = den > 0 ? (num/den) : 0; + int in_fps = in_den > 0 ? (in_num/in_den) : 0; + if (in_fps > fps) { + best_fraction = fraction; + num = in_num; + den = in_den; + } + } + } else { + debug("Unknown type for framerate: %s", framerate.type_name()); + } + if (den == 0) continue; if (!that.has_field("width") || !that.get_int("width", out width)) continue; if (!that.has_field("height") || !that.get_int("height", out height)) continue; int fps = num/den; @@ -105,7 +127,14 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object { best_index = i; } } - return caps_copy_nth(device.caps, best_index); + Gst.Caps res = caps_copy_nth(device.caps, best_index); + unowned Gst.Structure? that = res.get_structure(0); + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.ValueList)) { + that.set_value("framerate", best_fraction); + } + debug("Selected caps %s", res.to_string()); + return res; } else if (device.caps.get_size() > 0) { return caps_copy_nth(device.caps, 0); } else { diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala index f575a7d0..19a266b1 100644 --- a/plugins/rtp/src/plugin.vala +++ b/plugins/rtp/src/plugin.vala @@ -136,11 +136,7 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { pipe.set_state(Gst.State.PLAYING); break; case Gst.MessageType.STATE_CHANGED: - Gst.State new_state; - message.parse_state_changed(null, out new_state, null); - if (message.src is Gst.Element) { - debug("%s changed state to %s", ((Gst.Element)message.src).name, new_state.to_string()); - } + // Ignore break; case Gst.MessageType.STREAM_STATUS: Gst.StreamStatusType status; diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp index 9b3292b8..00f719e1 100644 --- a/plugins/rtp/src/voice_processor_native.cpp +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -75,7 +75,7 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); int err = apm->AnalyzeReverseStream(&frame); - if (err < 0) g_warning("ProcessReverseStream %i", err); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); gst_audio_buffer_unmap(&audio_buffer); } @@ -108,7 +108,7 @@ extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *nativ g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); if (fraction_poor_delays > 0.5) { native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); - g_debug("Adjusted stream delay %i", native->stream_delay); + g_debug("voice_processor_native.cpp: Adjusted stream delay %i", native->stream_delay); } } @@ -130,7 +130,7 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo * apm->set_stream_delay_ms(native->stream_delay); int err = apm->ProcessStream(&frame); if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); - if (err < 0) g_warning("ProcessStream %i", err); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); gst_audio_buffer_unmap(&audio_buffer); } -- cgit v1.2.3-70-g09d2 From 7d2e64769067c1b47e0500f6456dd7e6f4eb435a Mon Sep 17 00:00:00 2001 From: fiaxh Date: Thu, 29 Apr 2021 15:56:22 +0200 Subject: Improve call wording, cleanup --- main/src/ui/call_window/call_bottom_bar.vala | 1 - main/src/ui/call_window/call_window.vala | 4 +- .../src/ui/call_window/call_window_controller.vala | 30 +++--- .../ui/conversation_content_view/call_widget.vala | 2 +- main/src/ui/conversation_titlebar/call_entry.vala | 5 +- .../omemo/src/dtls_srtp_verification_draft.vala | 1 - plugins/rtp/src/module.vala | 12 +-- plugins/rtp/src/stream.vala | 4 +- .../src/module/xep/0166_jingle/component.vala | 12 ++- xmpp-vala/src/module/xep/0166_jingle/content.vala | 4 +- .../xep/0167_jingle_rtp/jingle_rtp_module.vala | 18 ++-- .../0176_jingle_ice_udp/transport_parameters.vala | 2 +- .../src/module/xep/0234_jingle_file_transfer.vala | 20 ++-- .../module/xep/0260_jingle_socks5_bytestreams.vala | 118 +++++++++++---------- .../xep/0261_jingle_in_band_bytestreams.vala | 2 +- 15 files changed, 125 insertions(+), 110 deletions(-) (limited to 'plugins/rtp/src') diff --git a/main/src/ui/call_window/call_bottom_bar.vala b/main/src/ui/call_window/call_bottom_bar.vala index a9fee8c3..a3e4b93b 100644 --- a/main/src/ui/call_window/call_bottom_bar.vala +++ b/main/src/ui/call_window/call_bottom_bar.vala @@ -25,7 +25,6 @@ public class Dino.Ui.CallBottomBar : Gtk.Box { private MenuButton video_settings_button = new MenuButton() { halign=Align.END, valign=Align.END }; public VideoSettingsPopover? video_settings_popover; - private EventBox encryption_event_box = new EventBox() { visible=true }; private MenuButton encryption_button = new MenuButton() { relief=ReliefStyle.NONE, height_request=30, width_request=30, margin_start=20, margin_bottom=25, halign=Align.START, valign=Align.END }; private Image encryption_image = new Image.from_icon_name("changes-allow-symbolic", IconSize.BUTTON) { visible=true }; diff --git a/main/src/ui/call_window/call_window.vala b/main/src/ui/call_window/call_window.vala index 572f73b6..3b3d4dc2 100644 --- a/main/src/ui/call_window/call_window.vala +++ b/main/src/ui/call_window/call_window.vala @@ -158,13 +158,13 @@ namespace Dino.Ui { public void set_status(string state) { switch (state) { case "requested": - header_bar.subtitle = _("Sending a call request…"); + header_bar.subtitle = _("Calling…"); break; case "ringing": header_bar.subtitle = _("Ringing…"); break; case "establishing": - header_bar.subtitle = _("Establishing a (peer-to-peer) connection…"); + header_bar.subtitle = _("Connecting…"); break; default: header_bar.subtitle = null; diff --git a/main/src/ui/call_window/call_window_controller.vala b/main/src/ui/call_window/call_window_controller.vala index 616e341d..0a223d72 100644 --- a/main/src/ui/call_window/call_window_controller.vala +++ b/main/src/ui/call_window/call_window_controller.vala @@ -3,8 +3,6 @@ using Gtk; public class Dino.Ui.CallWindowController : Object { - public signal void terminated(); - private CallWindow call_window; private Call call; private Conversation conversation; @@ -40,8 +38,16 @@ public class Dino.Ui.CallWindowController : Object { call_window.set_status("requested"); } - call_window.bottom_bar.hang_up.connect(end_call); - call_window.destroy.connect(end_call); + call_window.bottom_bar.hang_up.connect(() => { + calls.end_call(conversation, call); + call_window.close(); + call_window.destroy(); + this.dispose(); + }); + call_window.destroy.connect(() => { + calls.end_call(conversation, call); + this.dispose(); + }); call_window.bottom_bar.notify["audio-enabled"].connect(() => { calls.mute_own_audio(call, !call_window.bottom_bar.audio_enabled); @@ -116,16 +122,6 @@ public class Dino.Ui.CallWindowController : Object { this.window_width = this.call_window.get_allocated_width(); } - private void end_call() { - call.notify["state"].disconnect(on_call_state_changed); - calls.call_terminated.disconnect(on_call_terminated); - - calls.end_call(conversation, call); - call_window.close(); - call_window.destroy(); - terminated(); - } - private void on_call_state_changed() { if (call.state == Call.State.IN_PROGRESS) { call_window.set_status(""); @@ -234,4 +230,10 @@ public class Dino.Ui.CallWindowController : Object { call_window.unset_own_video(); } } + + public override void dispose() { + base.dispose(); + call.notify["state"].disconnect(on_call_state_changed); + calls.call_terminated.disconnect(on_call_terminated); + } } \ No newline at end of file diff --git a/main/src/ui/conversation_content_view/call_widget.vala b/main/src/ui/conversation_content_view/call_widget.vala index 66788e28..74525d11 100644 --- a/main/src/ui/conversation_content_view/call_widget.vala +++ b/main/src/ui/conversation_content_view/call_widget.vala @@ -154,7 +154,7 @@ namespace Dino.Ui { case Call.State.FAILED: image.set_from_icon_name("dino-phone-hangup-symbolic", IconSize.LARGE_TOOLBAR); title_label.label = _("Call failed"); - subtitle_label.label = "This call failed to establish"; + subtitle_label.label = "Call failed to establish"; break; } } diff --git a/main/src/ui/conversation_titlebar/call_entry.vala b/main/src/ui/conversation_titlebar/call_entry.vala index e1d10e5c..9353f631 100644 --- a/main/src/ui/conversation_titlebar/call_entry.vala +++ b/main/src/ui/conversation_titlebar/call_entry.vala @@ -92,9 +92,6 @@ namespace Dino.Ui { call_window.present(); update_button_state(); - call_controller.terminated.connect(() => { - update_button_state(); - }); } public new void set_conversation(Conversation conversation) { @@ -119,7 +116,7 @@ namespace Dino.Ui { if (conversation.type_ == Conversation.Type.CHAT) { Conversation conv_bak = conversation; bool audio_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); - bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_audio_calls_async(conversation); + bool video_works = yield stream_interactor.get_module(Calls.IDENTITY).can_do_video_calls_async(conversation); if (conv_bak != conversation) return; visible = audio_works; diff --git a/plugins/omemo/src/dtls_srtp_verification_draft.vala b/plugins/omemo/src/dtls_srtp_verification_draft.vala index e2441670..66a31954 100644 --- a/plugins/omemo/src/dtls_srtp_verification_draft.vala +++ b/plugins/omemo/src/dtls_srtp_verification_draft.vala @@ -65,7 +65,6 @@ namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); - if (session != null) print(@"$(session.contents_map.has_key(content_name))\n"); if (session == null || !session.contents_map.has_key(content_name)) return; var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], peer_device_id=device_id_by_jingle_sid[jingle_sid] }; session.contents_map[content_name].encryptions[NS_URI] = encryption; diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala index 13a21cd8..19a7501d 100644 --- a/plugins/rtp/src/module.vala +++ b/plugins/rtp/src/module.vala @@ -216,9 +216,9 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { } public override JingleRtp.Crypto? generate_local_crypto() { - uint8[] keyAndSalt = new uint8[30]; - Crypto.randomize(keyAndSalt); - return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, keyAndSalt); + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt); } public override JingleRtp.Crypto? pick_remote_crypto(Gee.List cryptos) { @@ -230,8 +230,8 @@ public class Dino.Plugins.Rtp.Module : JingleRtp.Module { public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { if (remote == null || !remote.is_valid) return null; - uint8[] keyAndSalt = new uint8[30]; - Crypto.randomize(keyAndSalt); - return remote.rekey(keyAndSalt); + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return remote.rekey(key_and_salt); } } \ No newline at end of file diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala index 23634aa3..bd8a279f 100644 --- a/plugins/rtp/src/stream.vala +++ b/plugins/rtp/src/stream.vala @@ -256,7 +256,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_local_crypto() { - if (local_crypto != null && !crypto_session.has_encrypt) { + if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) { crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); debug("Setting up encryption with key params %s", local_crypto.key_params); } @@ -396,7 +396,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { } private void prepare_remote_crypto() { - if (remote_crypto != null && !crypto_session.has_decrypt) { + if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) { crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); debug("Setting up decryption with key params %s", remote_crypto.key_params); } diff --git a/xmpp-vala/src/module/xep/0166_jingle/component.vala b/xmpp-vala/src/module/xep/0166_jingle/component.vala index 544bcd69..5d573522 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/component.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/component.vala @@ -31,7 +31,11 @@ namespace Xmpp.Xep.Jingle { protected Gee.Promise promise = new Gee.Promise(); private string? terminated = null; - public async void init(IOStream stream) { + public async void set_stream(IOStream? stream) { + if (stream == null) { + promise.set_exception(new IOError.FAILED("Jingle connection failed")); + return; + } assert(!this.stream.ready); promise.set_value(stream); if (terminated != null) { @@ -39,11 +43,17 @@ namespace Xmpp.Xep.Jingle { } } + public void set_error(GLib.Error? e) { + promise.set_exception(e); + } + public override async void terminate(bool we_terminated, string? reason_name = null, string? reason_text = null) { if (terminated == null) { terminated = (reason_name ?? "") + " - " + (reason_text ?? "") + @"we terminated? $we_terminated"; if (stream.ready) { yield stream.value.close_async(); + } else { + promise.set_exception(new IOError.FAILED("Jingle connection failed")); } } } diff --git a/xmpp-vala/src/module/xep/0166_jingle/content.vala b/xmpp-vala/src/module/xep/0166_jingle/content.vala index befe02f4..41310aeb 100644 --- a/xmpp-vala/src/module/xep/0166_jingle/content.vala +++ b/xmpp-vala/src/module/xep/0166_jingle/content.vala @@ -36,7 +36,7 @@ public class Xmpp.Xep.Jingle.Content : Object { public HashMap encryptions = new HashMap(); - public Set tried_transport_methods = new HashSet(); + private Set tried_transport_methods = new HashSet(); public Content.initiate_sent(string content_name, Senders senders, @@ -109,7 +109,7 @@ public class Xmpp.Xep.Jingle.Content : Object { transport_params.dispose(); foreach (ComponentConnection connection in component_connections.values) { - connection.terminate(we_terminated, reason_name, reason_text); + connection.terminate.begin(we_terminated, reason_name, reason_text); } } diff --git a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala index 6eb6289b..6b55cbe6 100644 --- a/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala +++ b/xmpp-vala/src/module/xep/0167_jingle_rtp/jingle_rtp_module.vala @@ -83,7 +83,7 @@ public abstract class Module : XmppStreamModule { } } - public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) { + public async Jingle.Content add_outgoing_video_content(XmppStream stream, Jingle.Session session) throws Jingle.Error { Jid my_jid = session.local_full_jid; Jid receiver_full_jid = session.peer_full_jid; @@ -168,7 +168,7 @@ public class Crypto { public string? session_params { get; private set; } public string tag { get; private set; } - public uint8[] key_and_salt { owned get { + public uint8[]? key_and_salt { owned get { if (!key_params.has_prefix("inline:")) return null; int endIndex = key_params.index_of("|"); if (endIndex < 0) endIndex = key_params.length; @@ -221,30 +221,30 @@ public class Crypto { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - return key_and_salt.length == 30; + return key_and_salt != null && key_and_salt.length == 30; } return false; }} - public uint8[] key { owned get { - uint8[] key_and_salt = key_and_salt; + public uint8[]? key { owned get { + uint8[]? key_and_salt = key_and_salt; switch(crypto_suite) { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - if (key_and_salt.length >= 16) return key_and_salt[0:16]; + if (key_and_salt != null && key_and_salt.length >= 16) return key_and_salt[0:16]; break; } return null; }} - public uint8[] salt { owned get { - uint8[] keyAndSalt = key_and_salt; + public uint8[]? salt { owned get { + uint8[]? key_and_salt = key_and_salt; switch(crypto_suite) { case AES_CM_128_HMAC_SHA1_80: case AES_CM_128_HMAC_SHA1_32: case F8_128_HMAC_SHA1_80: - if (keyAndSalt.length >= 30) return keyAndSalt[16:30]; + if (key_and_salt != null && key_and_salt.length >= 30) return key_and_salt[16:30]; break; } return null; diff --git a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala index 83da296b..07b599ee 100644 --- a/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala +++ b/xmpp-vala/src/module/xep/0176_jingle_ice_udp/transport_parameters.vala @@ -152,7 +152,7 @@ public abstract class Xmpp.Xep.JingleIceUdp.IceUdpTransportParameters : Jingle.T return sb.str; } - private uint8[] fingerprint_to_bytes(string? fingerprint_) { + private uint8[]? fingerprint_to_bytes(string? fingerprint_) { if (fingerprint_ == null) return null; string fingerprint = fingerprint_.replace(":", "").up(); diff --git a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala index 07b158bc..4581019f 100644 --- a/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala +++ b/xmpp-vala/src/module/xep/0234_jingle_file_transfer.vala @@ -268,13 +268,19 @@ public class FileTransfer : Object { content.accept(); Jingle.StreamingConnection connection = content.component_connections.values.to_array()[0] as Jingle.StreamingConnection; - IOStream? io_stream = yield connection.stream.wait_async(); - FileTransferInputStream ft_stream = new FileTransferInputStream(io_stream.input_stream, size); - io_stream.output_stream.close(); - ft_stream.closed.connect(() => { - session.terminate(Jingle.ReasonElement.SUCCESS, null, null); - }); - this.stream = ft_stream; + try { + IOStream io_stream = yield connection.stream.wait_async(); + FileTransferInputStream ft_stream = new FileTransferInputStream(io_stream.input_stream, size); + io_stream.output_stream.close(); + ft_stream.closed.connect(() => { + session.terminate(Jingle.ReasonElement.SUCCESS, null, null); + }); + this.stream = ft_stream; + } catch (FutureError.EXCEPTION e) { + warning("Error accepting Jingle file-transfer: %s", connection.stream.exception.message); + } catch (FutureError e) { + warning("FutureError accepting Jingle file-transfer: %s", e.message); + } } public void reject(XmppStream stream) { diff --git a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala index 6edebbbc..47c243e8 100644 --- a/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0260_jingle_socks5_bytestreams.vala @@ -28,6 +28,7 @@ public class Module : Jingle.Transport, XmppStreamModule { public string ns_uri { get { return NS_URI; } } public Jingle.TransportType type_ { get { return Jingle.TransportType.STREAMING; } } public int priority { get { return 1; } } + private Gee.List get_proxies(XmppStream stream) { Gee.List result = new ArrayList(); int i = 1 << 15; @@ -37,6 +38,7 @@ public class Module : Jingle.Transport, XmppStreamModule { } return result; } + private Gee.List start_local_listeners(XmppStream stream, Jid local_full_jid, string dstaddr, out LocalListener? local_listener) { Gee.List result = new ArrayList(); SocketListener listener = new SocketListener(); @@ -62,15 +64,17 @@ public class Module : Jingle.Transport, XmppStreamModule { } return result; } + private void select_candidates(XmppStream stream, Jid local_full_jid, string dstaddr, Parameters result) { result.local_candidates.add_all(get_proxies(stream)); - //result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); + result.local_candidates.add_all(start_local_listeners(stream, local_full_jid, dstaddr, out result.listener)); result.local_candidates.sort((c1, c2) => { if (c1.priority < c2.priority) { return 1; } if (c1.priority > c2.priority) { return -1; } return 0; }); } + public Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { assert(components == 1); Parameters result = new Parameters.create(local_full_jid, peer_full_jid, random_uuid()); @@ -78,6 +82,7 @@ public class Module : Jingle.Transport, XmppStreamModule { select_candidates(stream, local_full_jid, dstaddr, result); return result; } + public Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { Parameters result = Parameters.parse(local_full_jid, peer_full_jid, transport); string dstaddr = calculate_dstaddr(result.sid, local_full_jid, peer_full_jid); @@ -146,6 +151,7 @@ public class Candidate : Socks5Bytestreams.Proxy { public Candidate.build(string cid, string host, Jid jid, int port, int local_priority, CandidateType type) { this(cid, host, jid, port, type.type_preference() + local_priority, type); } + public Candidate.proxy(string cid, Socks5Bytestreams.Proxy proxy, int local_priority) { this.build(cid, proxy.host, proxy.jid, proxy.port, local_priority, CandidateType.PROXY); } @@ -170,6 +176,7 @@ public class Candidate : Socks5Bytestreams.Proxy { return new Candidate(cid, host, jid, port, priority, type); } + public StanzaNode to_xml() { return new StanzaNode.build("candidate", NS_URI) .put_attribute("cid", cid) @@ -210,6 +217,7 @@ class LocalListener { this.inner = inner; this.dstaddr = dstaddr; } + public LocalListener.empty() { this.inner = null; this.dstaddr = ""; @@ -233,6 +241,7 @@ class LocalListener { handle_conn.begin(((StringWrapper)cid).str, conn); } } + async void handle_conn(string cid, SocketConnection conn) { conn.socket.timeout = NEGOTIATION_TIMEOUT; size_t read; @@ -418,39 +427,39 @@ class Parameters : Jingle.TransportParameters, Object { } public void handle_transport_info(StanzaNode transport) throws Jingle.IqError { - StanzaNode? candidate_error = transport.get_subnode("candidate-error", NS_URI); - StanzaNode? candidate_used = transport.get_subnode("candidate-used", NS_URI); - StanzaNode? activated = transport.get_subnode("activated", NS_URI); - StanzaNode? proxy_error = transport.get_subnode("proxy-error", NS_URI); - int num_children = 0; - if (candidate_error != null) { num_children += 1; } - if (candidate_used != null) { num_children += 1; } - if (activated != null) { num_children += 1; } - if (proxy_error != null) { num_children += 1; } - if (num_children == 0) { - throw new Jingle.IqError.UNSUPPORTED_INFO("unknown transport-info"); - } else if (num_children > 1) { - throw new Jingle.IqError.BAD_REQUEST("transport-info with more than one child"); - } - if (candidate_error != null) { - handle_remote_candidate(null); - } - if (candidate_used != null) { - string? cid = candidate_used.get_attribute("cid"); - if (cid == null) { - throw new Jingle.IqError.BAD_REQUEST("missing cid"); - } - handle_remote_candidate(cid); - } - if (activated != null) { - string? cid = activated.get_attribute("cid"); - if (cid == null) { - throw new Jingle.IqError.BAD_REQUEST("missing cid"); - } - handle_activated(cid); + ArrayList socks5_nodes = new ArrayList(); + foreach (StanzaNode node in transport.sub_nodes) { + if (node.ns_uri == NS_URI) socks5_nodes.add(node); } - if (proxy_error != null) { - handle_proxy_error(); + if (socks5_nodes.is_empty) { warning("No socks5 subnodes in transport node"); return; } + if (socks5_nodes.size > 1) { warning("Too many socks5 subnodes in transport node"); return; } + + StanzaNode node = socks5_nodes[0]; + + switch (node.name) { + case "activated": + string? cid = node.get_attribute("cid"); + if (cid == null) { + throw new Jingle.IqError.BAD_REQUEST("missing cid"); + } + handle_activated(cid); + break; + case "candidate-used": + string? cid = node.get_attribute("cid"); + if (cid == null) { + throw new Jingle.IqError.BAD_REQUEST("missing cid"); + } + handle_remote_candidate(cid); + break; + case "candidate-error": + handle_remote_candidate(null); + break; + case "proxy-error": + handle_proxy_error(); + break; + default: + warning("Unknown transport-info: %s", transport.to_string()); + break; } } @@ -499,32 +508,22 @@ class Parameters : Jingle.TransportParameters, Object { return; } - Candidate? remote = remote_selected_candidate; - Candidate? local = local_selected_candidate; - - int num_candidates = 0; - if (remote != null) { num_candidates += 1; } - if (local != null) { num_candidates += 1; } - - if (num_candidates == 0) { - // Notify Jingle of the failed transport. - content_set_transport_connection(null); + if (remote_selected_candidate == null && local_selected_candidate == null) { + content_set_transport_connection_error(new IOError.FAILED("No candidates")); return; } bool remote_wins; - if (num_candidates == 1) { - remote_wins = remote != null; - } else { - if (local.priority < remote.priority) { - remote_wins = true; - } else if (local.priority > remote.priority) { - remote_wins = false; - } else { + if (remote_selected_candidate != null && local_selected_candidate != null) { + if (local_selected_candidate.priority == remote_selected_candidate.priority) { // equal priority -> XEP-0260 says that the candidate offered // by the initiator wins, so the one that the remote chose remote_wins = role == Jingle.Role.INITIATOR; + } else { + remote_wins = local_selected_candidate.priority < remote_selected_candidate.priority; } + } else { + remote_wins = remote_selected_candidate != null; } if (!remote_wins) { @@ -545,8 +544,7 @@ class Parameters : Jingle.TransportParameters, Object { } SocketConnection? conn = listener.get_connection(remote_selected_candidate.cid); if (conn == null) { - // Remote hasn't actually connected to us?! - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("Remote hasn't actually connected to us?!")); return; } content_set_transport_connection(conn); @@ -569,7 +567,7 @@ class Parameters : Jingle.TransportParameters, Object { if (!waiting_for_activation_error) { content_set_transport_connection(conn); } else { - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("waiting_for_activation_error")); } } @@ -620,7 +618,7 @@ class Parameters : Jingle.TransportParameters, Object { .put_attribute("sid", sid) .put_node(new StanzaNode.build("proxy-error", NS_URI)) ); - content_set_transport_connection(null); + content_set_transport_connection_error(new IOError.FAILED("Connect to local candidate error: %s", e.message)); } } @@ -745,15 +743,19 @@ class Parameters : Jingle.TransportParameters, Object { private Jingle.StreamingConnection connection = new Jingle.StreamingConnection(); - private void content_set_transport_connection(IOStream? ios) { - IOStream? iostream = ios; + private void content_set_transport_connection(IOStream ios) { + IOStream iostream = ios; Jingle.Content? strong_content = content; if (strong_content == null) return; if (strong_content.security_params != null) { iostream = strong_content.security_params.wrap_stream(iostream); } - connection.init.begin(iostream); + connection.set_stream.begin(iostream); + } + + private void content_set_transport_connection_error(Error e) { + connection.set_error(e); } public void create_transport_connection(XmppStream stream, Jingle.Content content) { diff --git a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala index f7c77544..09eaf711 100644 --- a/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala +++ b/xmpp-vala/src/module/xep/0261_jingle_in_band_bytestreams.vala @@ -98,7 +98,7 @@ class Parameters : Jingle.TransportParameters, Object { if (content.security_params != null) { iostream = content.security_params.wrap_stream(iostream); } - connection.init.begin(iostream); + connection.set_stream.begin(iostream); debug("set transport conn ibb"); content.set_transport_connection(connection, 1); } -- cgit v1.2.3-70-g09d2 From 8044b546d0ac15d34a3e6499b9c0d55d3d8f9c94 Mon Sep 17 00:00:00 2001 From: Marvin W Date: Sun, 2 May 2021 00:34:17 +0200 Subject: Support voice processing on GStreamer 0.14 --- plugins/rtp/CMakeLists.txt | 7 +++--- plugins/rtp/src/voice_processor.vala | 2 +- plugins/rtp/src/voice_processor_native.cpp | 37 ++++++++++++++++++------------ 3 files changed, 27 insertions(+), 19 deletions(-) (limited to 'plugins/rtp/src') diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt index b19c8a8f..52419425 100644 --- a/plugins/rtp/CMakeLists.txt +++ b/plugins/rtp/CMakeLists.txt @@ -17,7 +17,7 @@ if(Gst_VERSION VERSION_GREATER "1.16") endif() if(WebRTCAudioProcessing_VERSION GREATER "0.4") - message(WARNING "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") unset(WebRTCAudioProcessing_FOUND) endif() @@ -25,8 +25,9 @@ if(WebRTCAudioProcessing_FOUND) set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) + set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing) else() - message(WARNING "WebRTCAudioProcessing not found, build without voice pre-processing!") + message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!") endif() vala_precompile(RTP_VALA_C @@ -53,7 +54,7 @@ DEFINITIONS add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) -target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 webrtc-audio-processing) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB}) set_target_properties(rtp PROPERTIES PREFIX "") set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala index e6dc7e8f..66e95d72 100644 --- a/plugins/rtp/src/voice_processor.vala +++ b/plugins/rtp/src/voice_processor.vala @@ -123,7 +123,7 @@ public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { } analyze_reverse_stream(native, echo_probe.audio_info, buffer); if (adjust_delay_timeout_id == 0 && echo_probe != null) { - adjust_delay_timeout_id = Timeout.add(5000, adjust_delay); + adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); } } diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp index 00f719e1..8a052cf8 100644 --- a/plugins/rtp/src/voice_processor_native.cpp +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -11,6 +11,8 @@ struct _DinoPluginsRtpVoiceProcessorNative { webrtc::AudioProcessing *apm; gint stream_delay; + gint last_median; + gint last_poor_delays; }; extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { @@ -26,6 +28,8 @@ extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) config.Set(new webrtc::ExperimentalAgc(true, 85)); native->apm = webrtc::AudioProcessing::Create(config); native->stream_delay = stream_delay; + native->last_median = 0; + native->last_poor_delays = 0; return native; } @@ -65,19 +69,19 @@ dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAud webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::AudioProcessing *apm = native->apm; - GstAudioBuffer audio_buffer; - gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READ); + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READ); webrtc::AudioFrame frame; frame.num_channels_ = info->channels; frame.sample_rate_hz_ = info->rate; frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; - memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); int err = apm->AnalyzeReverseStream(&frame); if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); - gst_audio_buffer_unmap(&audio_buffer); + gst_buffer_unmap(buffer, &map); } extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { @@ -101,14 +105,17 @@ extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *nati extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; webrtc::AudioProcessing *apm = native->apm; - int median, std; + int median, std, poor_delays; float fraction_poor_delays; apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); - if (fraction_poor_delays < 0) return; - g_debug("voice_processor_native.cpp: Stream delay metrics: %i %i %f", median, std, fraction_poor_delays); - if (fraction_poor_delays > 0.5) { - native->stream_delay = std::max(0, native->stream_delay + std::min(-10, std::max(median, 10))); - g_debug("voice_processor_native.cpp: Adjusted stream delay %i", native->stream_delay); + poor_delays = (int)(fraction_poor_delays * 100.0); + if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); + native->last_median = median; + native->last_poor_delays = poor_delays; + if (poor_delays > 90) { + native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); + g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); } } @@ -118,21 +125,21 @@ dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo * webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); webrtc::AudioProcessing *apm = native->apm; - GstAudioBuffer audio_buffer; - gst_audio_buffer_map(&audio_buffer, info, buffer, GST_MAP_READWRITE); + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READWRITE); webrtc::AudioFrame frame; frame.num_channels_ = info->channels; frame.sample_rate_hz_ = info->rate; frame.samples_per_channel_ = info->rate / 100; - memcpy(frame.data_, audio_buffer.planes[0], frame.samples_per_channel_ * info->bpf); + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); apm->set_stream_delay_ms(native->stream_delay); int err = apm->ProcessStream(&frame); - if (err >= 0) memcpy(audio_buffer.planes[0], frame.data_, frame.samples_per_channel_ * info->bpf); + if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); - gst_audio_buffer_unmap(&audio_buffer); + gst_buffer_unmap(buffer, &map); } extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { -- cgit v1.2.3-70-g09d2