Fix video for cameras with rotated image

4 files changed, 122 insertions, 15 deletions

diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala
index 1db8c996..7fee7307 100644
--- a/plugins/rtp/src/device.vala
+++ b/plugins/rtp/src/device.vala
@@ -215,7 +215,12 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
         }
         if (new_width == active_caps_width) return;
         int new_height = device_caps_height * new_width / device_caps_width;
-        Gst.Caps new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
+        Gst.Caps new_caps;
+        if (device_caps_framerate_den != 0) {
+            new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, "framerate", typeof(Gst.Fraction), device_caps_framerate_num, device_caps_framerate_den, null);
+        } else {
+            new_caps = new Gst.Caps.simple("video/x-raw", "width", typeof(int), new_width, "height", typeof(int), new_height, null);
+        }
         double required_bitrate = get_target_bitrate(new_caps);
         debug("Changing resolution width from %d to %d (requires bitrate %f, current target is %u)", active_caps_width, new_width, required_bitrate, bitrate);
         if (bitrate < required_bitrate && new_width > active_caps_width) return;
@@ -347,7 +352,7 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
         if (media == "audio") {
             return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1");
         } else if (media == "video" && device.caps.get_size() > 0) {
-            int best_index = 0;
+            int best_index = -1;
             Value? best_fraction = null;
             int best_fps = 0;
             int best_width = 0;
@@ -390,10 +395,25 @@ public class Dino.Plugins.Rtp.Device : MediaDevice, Object {
                     best_fraction = best_fraction_now;
                 }
             }
+            if (best_index == -1) {
+                // No caps in first round, try without framerate
+                for (int i = 0; i < device.caps.get_size(); i++) {
+                    unowned Gst.Structure? that = device.caps.get_structure(i);
+                    if (!that.has_name("video/x-raw")) continue;
+                    int width = 0, height = 0;
+                    if (!that.has_field("width") || !that.get_int("width", out width)) continue;
+                    if (!that.has_field("height") || !that.get_int("height", out height)) continue;
+                    if (best_width < width || best_width == width && best_height < height) {
+                        best_width = width;
+                        best_height = height;
+                        best_index = i;
+                    }
+                }
+            }
             Gst.Caps res = caps_copy_nth(device.caps, best_index);
             unowned Gst.Structure? that = res.get_structure(0);
-            Value framerate = that.get_value("framerate");
-            if (framerate.type() == typeof(Gst.ValueList)) {
+            Value? framerate = that.get_value("framerate");
+            if (framerate != null && framerate.type() == typeof(Gst.ValueList) && best_fraction != null) {
                 that.set_value("framerate", best_fraction);
             }
             debug("Selected caps %s", res.to_string());
diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala
index aefe41ff..98b9717d 100644
--- a/plugins/rtp/src/plugin.vala
+++ b/plugins/rtp/src/plugin.vala
@@ -426,11 +426,11 @@ public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object {
 
         if (media == "video") {
             // Pick best FPS
-            int max_fps = 0;
+            int max_fps = -1;
             Device? max_fps_device = null;
             foreach (Device device in devices) {
                 int fps = get_max_fps(device);
-                if (fps > max_fps) {
+                if (fps > max_fps || max_fps_device == null) {
                     max_fps = fps;
                     max_fps_device = device;
                 }
diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala
index abdda776..94549856 100644
--- a/plugins/rtp/src/stream.vala
+++ b/plugins/rtp/src/stream.vala
@@ -102,6 +102,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
         send_rtp.drop = true;
         send_rtp.wait_on_eos = false;
         send_rtp.new_sample.connect(on_new_sample);
+#if GST_1_20
+        send_rtp.new_serialized_event.connect(on_new_event);
+#endif
         send_rtp.connect("signal::eos", on_eos_static, this);
         pipe.add(send_rtp);
 
@@ -294,6 +297,60 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
         }
     }
 
+    bool flip = false;
+    uint8 rotation = 0;
+#if GST_1_20
+    private bool on_new_event(Gst.App.Sink sink) {
+        if (sink == null || sink != send_rtp) {
+            return false;
+        }
+        Gst.MiniObject obj = sink.try_pull_object(0);
+        if (obj.type == typeof(Gst.Event)) {
+            unowned Gst.TagList tags;
+            if (((Gst.Event)obj).type == Gst.EventType.TAG) {
+                ((Gst.Event)obj).parse_tag(out tags);
+                Gst.Video.OrientationMethod orientation_method;
+                Gst.Video.Orientation.from_tag(tags, out orientation_method);
+                switch (orientation_method) {
+                    case Gst.Video.OrientationMethod.IDENTITY:
+                    case Gst.Video.OrientationMethod.VERT:
+                    default:
+                        rotation = 0;
+                        break;
+                    case Gst.Video.OrientationMethod.@90R:
+                    case Gst.Video.OrientationMethod.UL_LR:
+                        rotation = 1;
+                        break;
+                    case Gst.Video.OrientationMethod.@180:
+                    case Gst.Video.OrientationMethod.HORIZ:
+                        rotation = 2;
+                        break;
+                    case Gst.Video.OrientationMethod.@90L:
+                    case Gst.Video.OrientationMethod.UR_LL:
+                        rotation = 3;
+                        break;
+                }
+                switch (orientation_method) {
+                    case Gst.Video.OrientationMethod.IDENTITY:
+                    case Gst.Video.OrientationMethod.@90R:
+                    case Gst.Video.OrientationMethod.@180:
+                    case Gst.Video.OrientationMethod.@90L:
+                    default:
+                        flip = false;
+                        break;
+                    case Gst.Video.OrientationMethod.VERT:
+                    case Gst.Video.OrientationMethod.UL_LR:
+                    case Gst.Video.OrientationMethod.HORIZ:
+                    case Gst.Video.OrientationMethod.UR_LL:
+                        flip = true;
+                        break;
+                }
+            }
+        }
+        return false;
+    }
+#endif
+
     private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) {
         if (sink == null) {
             debug("Sink is null");
@@ -323,6 +380,24 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
 #endif
         }
 
+#if GST_1_20
+        if (sink == send_rtp) {
+            Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation");
+            if (ext != null) {
+                buffer = (Gst.Buffer) buffer.make_writable();
+                Gst.RTP.Buffer rtp_buffer;
+                if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.WRITE, out rtp_buffer)) {
+                    uint8[] extension_data = new uint8[1];
+                    bool camera = false;
+                    extension_data[0] = extension_data[0] | (rotation & 0x3);
+                    if (flip) extension_data[0] = extension_data[0] | 0x4;
+                    if (camera) extension_data[0] = extension_data[0] | 0x8;
+                    rtp_buffer.add_extension_onebyte_header(ext.id, extension_data);
+                }
+            }
+        }
+#endif
+
         prepare_local_crypto();
 
         uint8[] data;
@@ -489,8 +564,8 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
         }
     }
 
-    private uint16 previous_video_orientation_degree = uint16.MAX;
-    public signal void video_orientation_changed(uint16 degree);
+    private uint16 previous_incoming_video_orientation_degree = uint16.MAX;
+    public signal void incoming_video_orientation_changed(uint16 degree);
 
     public override void on_recv_rtp_data(Bytes bytes) {
         if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) {
@@ -545,9 +620,9 @@ public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream {
                                 case 2: rotation_degree = 180; break;
                                 case 3: rotation_degree = 270; break;
                             }
-                            if (rotation_degree != previous_video_orientation_degree) {
-                                video_orientation_changed(rotation_degree);
-                                previous_video_orientation_degree = rotation_degree;
+                            if (rotation_degree != previous_incoming_video_orientation_degree) {
+                                incoming_video_orientation_changed(rotation_degree);
+                                previous_incoming_video_orientation_degree = rotation_degree;
                             }
                         }
                     }
@@ -723,7 +798,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
     private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>();
     private Gst.Element output_tee;
     private Gst.Element rotate;
-    private ulong video_orientation_changed_handler;
+    private ulong incoming_video_orientation_changed_handler;
 
     public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) {
         base(plugin, content);
@@ -731,7 +806,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
     }
 
     public override void create() {
-        video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed);
+        incoming_video_orientation_changed_handler = incoming_video_orientation_changed.connect(on_video_orientation_changed);
         plugin.pause();
         rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid");
         pipe.add(rotate);
@@ -780,7 +855,7 @@ public class Dino.Plugins.Rtp.VideoStream : Stream {
         output_tee.set_state(Gst.State.NULL);
         pipe.remove(output_tee);
         output_tee = null;
-        disconnect(video_orientation_changed_handler);
+        disconnect(incoming_video_orientation_changed_handler);
     }
 
     public override void add_output(Gst.Element element, Xmpp.Jid? participant) {
diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala
index 0d66476b..20123c68 100644
--- a/plugins/rtp/src/video_widget.vala
+++ b/plugins/rtp/src/video_widget.vala
@@ -227,9 +227,21 @@ public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWi
         if (connected_device == null) return;
         plugin.pause();
         pipe.add(sink);
+#if GST_1_20
+        prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
+#else
         prepare = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true);
+#endif
         prepare.name = @"video_widget_$(id)_prepare";
-        prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+#if GST_1_20
+        if (prepare is Gst.Bin) {
+            ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+        } else {
+#endif
+            prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed);
+#if GST_1_20
+        }
+#endif
         pipe.add(prepare);
         connected_device_element = connected_device.link_source();
         connected_device_element.link(prepare);