aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarvin W <git@larma.de>2021-11-15 22:49:44 +0100
committerMarvin W <git@larma.de>2021-11-15 22:49:44 +0100
commit0b828a0ae55b3c0eaf1206f0c54cfcee4f60e6af (patch)
tree6985226eab65f05a8737b448c44b296470f46e6c
parent9958cfbe7b4467ec5a5fed4c7e5e06f7f8e9179b (diff)
downloaddino-0b828a0ae55b3c0eaf1206f0c54cfcee4f60e6af.tar.gz
dino-0b828a0ae55b3c0eaf1206f0c54cfcee4f60e6af.zip
Add maximum bitrate and adjust video resolution based on bitrate
-rw-r--r--plugins/rtp/src/codec_util.vala35
-rw-r--r--plugins/rtp/src/device.vala85
-rw-r--r--plugins/rtp/src/plugin.vala1
-rw-r--r--plugins/rtp/src/stream.vala2
4 files changed, 111 insertions, 12 deletions
diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala
index 4c68985f..ec922f97 100644
--- a/plugins/rtp/src/codec_util.vala
+++ b/plugins/rtp/src/codec_util.vala
@@ -140,13 +140,16 @@ public class Dino.Plugins.Rtp.CodecUtil {
public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) {
// H264
if (encode == "msdkh264enc") return @" rate-control=vbr";
- if (encode == "vaapih264enc") return @" tune=low-power";
- if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency";
+ if (encode == "vaapih264enc") return @" rate-control=vbr tune=low-power";
+ if (encode == "x264enc") return @" byte-stream=1 speed-preset=ultrafast tune=zerolatency";
// VP8
- if (encode == "msdkvp8enc") return " rate-control=vbr";
- if (encode == "vaapivp8enc") return " rate-control=vbr";
- if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30";
+ if (encode == "vaapivp8enc" || encode == "msdkvp8enc") return " rate-control=vbr target-percentage=90";
+ if (encode == "vp8enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr";
+
+ // VP9
+ if (encode == "msdkvp9enc" || encode == "vaapivp9enc") return " rate-control=vbr target-percentage=90";
+ if (encode == "vp9enc") return " deadline=1 error-resilient=3 lag-in-frames=0 resize-allowed=true threads=8 dropframe-threshold=30 end-usage=vbr";
// OPUS
if (encode == "opusenc") {
@@ -186,13 +189,29 @@ public class Dino.Plugins.Rtp.CodecUtil {
case "vp9enc":
case "vp8enc":
bitrate = uint.min(2147483, bitrate);
- encode.set("target-bitrate", bitrate * 1000);
+ encode.set("target-bitrate", bitrate * 1024);
return bitrate;
}
return 0;
}
+ public void update_rescale_caps(Gst.Element encode_element, Gst.Caps caps) {
+ Gst.Bin? encode_bin = encode_element as Gst.Bin;
+ if (encode_bin == null) return;
+ Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+ rescale_caps.set("caps", caps);
+ }
+
+ public Gst.Caps? get_rescale_caps(Gst.Element encode_element) {
+ Gst.Bin? encode_bin = encode_element as Gst.Bin;
+ if (encode_bin == null) return null;
+ Gst.Element rescale_caps = encode_bin.get_by_name(@"$(encode_bin.name)_rescale_caps");
+ Gst.Caps caps;
+ rescale_caps.get("caps", out caps);
+ return caps;
+ }
+
public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) {
return null;
}
@@ -309,8 +328,8 @@ public class Dino.Plugins.Rtp.CodecUtil {
string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? "";
string encode_args = get_encode_args(media, codec, encode, payload_type) ?? "";
string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? "";
- string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : "";
- return @"$(media)convert name=$(base_name)_convert$resample ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix";
+ string rescale = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : @" ! videoscale name=$(base_name)_rescale ! capsfilter name=$(base_name)_rescale_caps";
+ return @"$(media)convert name=$(base_name)_convert$rescale ! queue ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix";
}
public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) {
diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala
index 3546ab94..89d499ed 100644
--- a/plugins/rtp/src/device.vala
+++ b/plugins/rtp/src/device.vala
@@ -36,6 +36,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return device.has_classes("Sink");
}}
+ private Gst.Caps device_caps;
private Gst.Element element;
private Gst.Element tee;
private Gst.Element dsp;
@@ -46,9 +47,13 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
// Codecs
private Gee.Map<PayloadType, Gst.Element> codecs = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gst.Element> codec_tees = new HashMap<PayloadType, Gst.Element>(PayloadType.hash_func, PayloadType.equals_func);
+
+ // Payloaders
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloaders = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, Gst.Element>> payloader_tees = new HashMap<PayloadType, Gee.Map<uint, Gst.Element>>(PayloadType.hash_func, PayloadType.equals_func);
private Gee.Map<PayloadType, Gee.Map<uint, uint>> payloader_links = new HashMap<PayloadType, Gee.Map<uint, uint>>(PayloadType.hash_func, PayloadType.equals_func);
+
+ // Bitrate
private Gee.Map<PayloadType, Gee.List<CodecBitrate>> codec_bitrates = new HashMap<PayloadType, Gee.List<CodecBitrate>>(PayloadType.hash_func, PayloadType.equals_func);
private class CodecBitrate {
@@ -146,6 +151,51 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
return element;
}
+ private static double get_target_bitrate(Gst.Caps caps) {
+ if (caps == null || caps.get_size() == 0) return uint.MAX;
+ unowned Gst.Structure? that = caps.get_structure(0);
+ int num = 0, den = 0, width = 0, height = 0;
+ if (!that.has_field("width") || !that.get_int("width", out width)) return uint.MAX;
+ if (!that.has_field("height") || !that.get_int("height", out height)) return uint.MAX;
+ if (!that.has_field("framerate")) return uint.MAX;
+ Value framerate = that.get_value("framerate");
+ if (framerate.type() != typeof(Gst.Fraction)) return uint.MAX;
+ num = Gst.Value.get_fraction_numerator(framerate);
+ den = Gst.Value.get_fraction_denominator(framerate);
+ double pxs = ((double)num/(double)den) * (double)width * (double)height;
+ double br = Math.sqrt(Math.sqrt(pxs)) * 100.0 - 3700.0;
+ if (br < 128.0) return 128.0;
+ return br;
+ }
+
+ private const int[] common_widths = {320, 480, 640, 960, 1280, 1920, 2560, 3840};
+ private Gst.Caps get_active_caps(PayloadType payload_type) {
+ return codec_util.get_rescale_caps(codecs[payload_type]) ?? device_caps;
+ }
+ private void apply_caps(PayloadType payload_type, Gst.Caps caps) {
+ plugin.pause();
+ debug("Set scaled caps to %s", caps.to_string());
+ codec_util.update_rescale_caps(codecs[payload_type], caps);
+ plugin.unpause();
+ }
+ private void apply_width(PayloadType payload_type, int new_width, uint bitrate) {
+ int device_caps_width, device_caps_height, active_caps_width, device_caps_framerate_num, device_caps_framerate_den;
+ device_caps.get_structure(0).get_int("width", out device_caps_width);
+ device_caps.get_structure(0).get_int("height", out device_caps_height);
+ device_caps.get_structure(0).get_fraction("framerate", out device_caps_framerate_num, out device_caps_framerate_den);
+ Gst.Caps active_caps = get_active_caps(payload_type);
+ if (active_caps != null && active_caps.get_size() > 0) {
+ active_caps.get_structure(0).get_int("width", out active_caps_width);
+ } else {
+ active_caps_width = device_caps_width;
+ }
+ if (new_width == active_caps_width) return;
+ int new_height = device_caps_height * new_width / device_caps_width;
+ Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
+ double required_bitrate = get_target_bitrate(new_caps);
+ if (bitrate < required_bitrate) return;
+ apply_caps(payload_type, new_caps);
+ }
public void update_bitrate(PayloadType payload_type, uint bitrate) {
if (codecs.has_key(payload_type)) {
lock(codec_bitrates);
@@ -164,6 +214,36 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
}
}
codec_bitrates[payload_type].remove_all(remove);
+ if (media == "video") {
+ if (bitrate < 128) bitrate = 128;
+ Gst.Caps active_caps = get_active_caps(payload_type);
+ double max_bitrate = get_target_bitrate(device_caps) * 2;
+ double current_target_bitrate = get_target_bitrate(active_caps);
+ int device_caps_width, active_caps_width;
+ device_caps.get_structure(0).get_int("width", out device_caps_width);
+ if (active_caps != null && active_caps.get_size() > 0) {
+ active_caps.get_structure(0).get_int("width", out active_caps_width);
+ } else {
+ active_caps_width = device_caps_width;
+ }
+ if (bitrate < 0.75 * current_target_bitrate && active_caps_width > common_widths[0]) {
+ // Lower video resolution
+ int i = 1;
+ for(; i < common_widths.length && common_widths[i] < active_caps_width; i++);
+ apply_width(payload_type, common_widths[i-1], bitrate);
+ } else if (bitrate > 2 * current_target_bitrate && active_caps_width < device_caps_width) {
+ // Higher video resolution
+ int i = 0;
+ for(; i < common_widths.length && common_widths[i] <= active_caps_width; i++);
+ if (common_widths[i] > device_caps_width) {
+ // We never scale up, so just stick with what the device gives
+ apply_width(payload_type, device_caps_width, bitrate);
+ } else if (common_widths[i] != active_caps_width) {
+ apply_width(payload_type, common_widths[i], bitrate);
+ }
+ }
+ if (bitrate > max_bitrate) bitrate = (uint) max_bitrate;
+ }
codec_util.update_bitrate(media, payload_type, codecs[payload_type], bitrate);
unlock(codec_bitrates);
}
@@ -348,10 +428,11 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
element.@set("sync", false);
}
pipe.add(element);
+ device_caps = get_best_caps();
if (is_source) {
element.@set("do-timestamp", true);
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
- filter.@set("caps", get_best_caps());
+ filter.@set("caps", device_caps);
filter.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, log_probe);
pipe.add(filter);
element.link(filter);
@@ -377,7 +458,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
plugin.echoprobe.link(element);
} else {
filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id");
- filter.@set("caps", get_best_caps());
+ filter.@set("caps", device_caps);
pipe.add(filter);
mixer.link(filter);
filter.link(element);
diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala
index 3b19318e..d10303a6 100644
--- a/plugins/rtp/src/plugin.vala
+++ b/plugins/rtp/src/plugin.vala
@@ -163,7 +163,6 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
debug("State of %s changed. Old: %s, New: %s, Pending; %s", ((Gst.Element)message.src).name, @"$oldState", @"$newState", @"$pendingState");
}
}
-
break;
case Gst.MessageType.STREAM_STATUS:
Gst.StreamStatusType status;
diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala
index 24adcb9a..5e5a556b 100644
--- a/plugins/rtp/src/stream.vala
+++ b/plugins/rtp/src/stream.vala
@@ -266,7 +266,7 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
last_remb_time = time_now;
uint actual_bitrate = (uint)(((double)new_octets * 8.0) * (double)time_diff / 1000.0 / 1000000.0);
new_target_receive_bitrate = uint.max(new_target_receive_bitrate, (uint)(0.9 * (double)actual_bitrate));
- max_target_receive_bitrate = uint.max(actual_bitrate * 2, max_target_receive_bitrate);
+ max_target_receive_bitrate = uint.max((uint)(1.5 * (double)actual_bitrate), max_target_receive_bitrate);
new_target_receive_bitrate = uint.min(new_target_receive_bitrate, max_target_receive_bitrate);
}
new_target_receive_bitrate = uint.max(16, new_target_receive_bitrate); // Never go below 16