Add maximum bitrate and adjust video resolution based on bitrate

4 files changed, 111 insertions, 12 deletions

diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala
index 4c68985f..ec922f97 100644
--- a/plugins/rtp/src/codec_util.vala
+++ b/plugins/rtp/src/codec_util.vala
@@ -140,13 +140,16 @@ public class Dino.Plugins.Rtp.CodecUtil {
     public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
         // H264
         if (encode == "msdkh264enc") return @" rate-control=vbr";
-        if (encode == "vaapih264enc") return @" tune=low-power";
-        if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency";
+        if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power";
+        if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency";
 
         // VP8
-        if (encode == "msdkvp8enc") return " rate-control=vbr";
-        if (encode == "vaapivp8enc") return " rate-control=vbr";
-        if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30";
+        if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90";
+        if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr";
+
+        // VP9
+        if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90";
+        if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr";
 
         // OPUS
         if (encode == "opusenc") {
@@ -186,13 +189,29 @@ public class Dino.Plugins.Rtp.CodecUtil {
             case "vp9enc":
             case "vp8enc":
                 bitrate = uint.min(2147483, bitrate);
-                encode.set("target-bitrate", bitrate * 1000);
+                encode.set("target-bitrate", bitrate * 1024);
                 return bitrate;
         }
 
         return 0;
     }
 
+    public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) {
+        Gst.Bin? encode_bin = encode_element as Gst.Bin;
+        if (encode_bin == null) return;
+        Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+        rescale_caps.set("caps", caps);
+    }
+
+    public Gst.Caps? get_rescale_caps(Gst.Element encode_element) {
+        Gst.Bin? encode_bin = encode_element as Gst.Bin;
+        if (encode_bin == null) return null;
+        Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+        Gst.Caps caps;
+        rescale_caps.get("caps", out caps);
+        return caps;
+    }
+
     public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
         return null;
     }
@@ -309,8 +328,8 @@ public class Dino.Plugins.Rtp.CodecUtil {
         string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? "";
         string encode_args = get_encode_args(media, codec, encode, payload_type) ?? "";
         string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? "";
-        string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : "";
-        return @"$(media)convert name=$(base_name)_convert$resample ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix";
+        string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps";
+        return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix";
     }
 
     public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala
index 3546ab94..89d499ed 100644
--- a/plugins/rtp/src/device.vala
+++ b/plugins/rtp/src/device.vala
@@ -36,6 +36,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
         return device.has_classes("Sink");
     }}
 
+    private Gst.Caps device_caps;
     private Gst.Element element;
     private Gst.Element tee;
     private Gst.Element dsp;
@@ -46,9 +47,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
     // Codecs
     private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
     private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
+
+    // Payloaders
     private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
     private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
     private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func);
+
+    // Bitrate
     private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func);
 
     private class CodecBitrate {
@@ -146,6 +151,51 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
         return element;
     }
 
+    private static double get_target_bitrate(Gst.Caps caps) {
+        if (caps == null || caps.get_size() == 0) return uint.MAX;
+        unowned Gst.Structure? that = caps.get_structure(0);
+        int num = 0, den = 0, width = 0, height = 0;
+        if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX;
+        if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX;
+        if (!that.has_field("framerate")) return uint.MAX;
+        Value framerate = that.get_value("framerate");
+        if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX;
+        num = Gst.Value.get_fraction_numerator(framerate);
+        den = Gst.Value.get_fraction_denominator(framerate);
+        double pxs = ((double)num/(double)den) * (double)width * (double)height;
+        double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0;
+        if (br < 128.0) return 128.0;
+        return br;
+    }
+
+    private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840};
+    private Gst.Caps get_active_caps(PayloadType payload_type) {
+        return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps;
+    }
+    private void apply_caps(PayloadType payload_type, Gst.Caps caps) {
+        plugin.pause();
+        debug("Set scaled caps to %s", caps.to_string());
+        codec_util.update_rescale_caps(codecs[payload_type], caps);
+        plugin.unpause();
+    }
+    private void apply_width(PayloadType payload_type, int new_width, uint bitrate) {
+        int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den;
+        device_caps.get_structure(0).get_int("width", out device_caps_width);
+        device_caps.get_structure(0).get_int("height", out device_caps_height);
+        device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den);
+        Gst.Caps active_caps = get_active_caps(payload_type);
+        if (active_caps != null && active_caps.get_size() > 0) {
+            active_caps.get_structure(0).get_int("width", out active_caps_width);
+        } else {
+            active_caps_width = device_caps_width;
+        }
+        if (new_width == active_caps_width) return;
+        int new_height = device_caps_height * new_width / device_caps_width;
+        Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
+        double required_bitrate = get_target_bitrate(new_caps);
+        if (bitrate < required_bitrate) return;
+        apply_caps(payload_type, new_caps);
+    }
     public void update_bitrate(PayloadType payload_type, uint bitrate) {
         if (codecs.has_key(payload_type)) {
             lock(codec_bitrates);
@@ -164,6 +214,36 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
                 }
             }
             codec_bitrates[payload_type].remove_all(remove);
+            if (media == "video") {
+                if (bitrate < 128) bitrate = 128;
+                Gst.Caps active_caps = get_active_caps(payload_type);
+                double max_bitrate = get_target_bitrate(device_caps) * 2;
+                double current_target_bitrate = get_target_bitrate(active_caps);
+                int device_caps_width, active_caps_width;
+                device_caps.get_structure(0).get_int("width", out device_caps_width);
+                if (active_caps != null && active_caps.get_size() > 0) {
+                    active_caps.get_structure(0).get_int("width", out active_caps_width);
+                } else {
+                    active_caps_width = device_caps_width;
+                }
+                if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) {
+                    // Lower video resolution
+                    int i = 1;
+                    for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);
+                    apply_width(payload_type, common_widths[i-1], bitrate);
+                } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) {
+                    // Higher video resolution
+                    int i = 0;
+                    for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++);
+                    if (common_widths[i] > device_caps_width) {
+                        // We never scale up, so just stick with what the device gives
+                        apply_width(payload_type, device_caps_width, bitrate);
+                    } else if (common_widths[i] != active_caps_width) {
+                        apply_width(payload_type, common_widths[i], bitrate);
+                    }
+                }
+                if (bitrate > max_bitrate) bitrate = (uint) max_bitrate;
+            }
             codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate);
             unlock(codec_bitrates);
         }
@@ -348,10 +428,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
             element.@set("sync", false);
         }
         pipe.add(element);
+        device_caps = get_best_caps();
         if (is_source) {
             element.@set("do-timestamp", true);
             filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
-            filter.@set("caps", get_best_caps());
+            filter.@set("caps", device_caps);
             filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe);
             pipe.add(filter);
             element.link(filter);
@@ -377,7 +458,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
                 plugin.echoprobe.link(element);
             } else {
                 filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
-                filter.@set("caps", get_best_caps());
+                filter.@set("caps", device_caps);
                 pipe.add(filter);
                 mixer.link(filter);
                 filter.link(element);
diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala
index 3b19318e..d10303a6 100644
--- a/plugins/rtp/src/plugin.vala
+++ b/plugins/rtp/src/plugin.vala
@@ -163,7 +163,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
                     debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState");
                 }
             }
-
                 break;
             case Gst.MessageType.STREAM_STATUS:
                 Gst.StreamStatusType status;
diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala
index 24adcb9a..5e5a556b 100644
--- a/plugins/rtp/src/stream.vala
+++ b/plugins/rtp/src/stream.vala
@@ -266,7 +266,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
                     last_remb_time = time_now;
                     uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0);
                     new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate));
-                    max_target_receive_bitrate = uint.max(actual_bitrate * 2, max_target_receive_bitrate);
+                    max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate);
                     new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate);
                 }
                 new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16