private static extern unowned Gst.Video.Info gst_video_frame_get_video_info(Gst.Video.Frame frame); [CCode (array_length_type = "size_t", type = "void*")] private static extern unowned uint8[] gst_video_frame_get_data(Gst.Video.Frame frame); public class Dino.Plugins.Rtp.Paintable : Gdk.Paintable, Object { private Gdk.Paintable image; private double pixel_aspect_ratio; public override Gdk.PaintableFlags get_flags() { return 0; } public void snapshot(Gdk.Snapshot snapshot, double width, double height) { if (image != null) image.snapshot(snapshot, width, height); } public override Gdk.Paintable get_current_image() { if (image != null) return image; return Gdk.Paintable.new_empty(0, 0); } public override int get_intrinsic_width() { if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_width()); return 0; } public override int get_intrinsic_height() { if (image != null) return (int) (pixel_aspect_ratio * image.get_intrinsic_height()); return 0; } public override double get_intrinsic_aspect_ratio() { if (image != null) return pixel_aspect_ratio * image.get_intrinsic_aspect_ratio(); return 0.0; } public override void dispose() { image = null; base.dispose(); } private void set_paintable(Gdk.Paintable paintable, double pixel_aspect_ratio) { if (paintable == image) return; bool size_changed = image == null || this.pixel_aspect_ratio * image.get_intrinsic_width() != pixel_aspect_ratio * paintable.get_intrinsic_width() || image.get_intrinsic_height() != paintable.get_intrinsic_height() || image.get_intrinsic_aspect_ratio() != paintable.get_intrinsic_aspect_ratio(); if (image != null) this.image.dispose(); this.image = paintable; this.pixel_aspect_ratio = pixel_aspect_ratio; if (size_changed) invalidate_size(); invalidate_contents(); } public void queue_set_texture(Gdk.Texture texture, double pixel_aspect_ratio) { Idle.add(() => { set_paintable(texture, pixel_aspect_ratio); return Source.REMOVE; }, Priority.DEFAULT); } } public class Dino.Plugins.Rtp.Sink : Gst.Video.Sink { internal Paintable paintable = new Paintable(); private Gst.Video.Info info = new Gst.Video.Info(); class construct { set_metadata("Dino Gtk Video Sink", "Sink/Video", "The video sink used by Dino", "Dino Team "); add_pad_template(new Gst.PadTemplate("sink", Gst.PadDirection.SINK, Gst.PadPresence.ALWAYS, Gst.Caps.from_string(@"video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"))); } construct { set_drop_out_of_segment(false); } #if GST_1_20 public override bool set_info(Gst.Caps caps, Gst.Video.Info info) { this.info = info; return true; } #else public override bool set_caps(Gst.Caps caps) { base.set_caps(caps); return info.from_caps(caps); } #endif public override void get_times(Gst.Buffer buffer, out Gst.ClockTime start, out Gst.ClockTime end) { if (buffer.pts != -1) { start = buffer.pts; if (buffer.duration != -1) { end = start + buffer.duration; } else if (info.fps_n > 0) { end = start + Gst.Util.uint64_scale_int(Gst.SECOND, info.fps_d, info.fps_n); } } } public override Gst.Caps get_caps(Gst.Caps? filter) { Gst.Caps caps = Gst.Caps.from_string("video/x-raw, format={ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"); if (filter != null) { return filter.intersect(caps, Gst.CapsIntersectMode.FIRST); } else { return caps; } } private Gdk.MemoryFormat memory_format_from_video(Gst.Video.Format format) { switch (format) { case Gst.Video.Format.BGRA: return Gdk.MemoryFormat.B8G8R8A8; case Gst.Video.Format.ARGB: return Gdk.MemoryFormat.A8R8G8B8; case Gst.Video.Format.RGBA: return Gdk.MemoryFormat.R8G8B8A8; case Gst.Video.Format.ABGR: return Gdk.MemoryFormat.A8B8G8R8; case Gst.Video.Format.RGB: return Gdk.MemoryFormat.R8G8B8; case Gst.Video.Format.BGR: return Gdk.MemoryFormat.B8G8R8; default: warning("Unsupported video format: %s", format.to_string()); return Gdk.MemoryFormat.A8R8G8B8; } } private Gdk.Texture texture_from_buffer(Gst.Buffer buffer, out double pixel_aspect_ratio) { Gst.Video.Frame frame = Gst.Video.Frame(); Gdk.Texture texture; if (frame.map(info, buffer, Gst.MapFlags.READ)) { unowned Gst.Video.Info info = gst_video_frame_get_video_info(frame); Bytes bytes = new Bytes.take(gst_video_frame_get_data(frame)); texture = new Gdk.MemoryTexture(info.width, info.height, memory_format_from_video(info.finfo.format), bytes, info.stride[0]); pixel_aspect_ratio = ((double) info.par_n) / ((double) info.par_d); frame.unmap(); } else { texture = null; } return texture; } private void queue_buffer(Gst.Buffer buf) { double pixel_aspect_ratio; Gdk.Texture texture = texture_from_buffer(buf, out pixel_aspect_ratio); if (texture != null) { paintable.queue_set_texture(texture, pixel_aspect_ratio); } } public override Gst.FlowReturn show_frame(Gst.Buffer buf) { @lock.lock(); queue_buffer(buf); @lock.unlock(); return Gst.FlowReturn.OK; } } public class Dino.Plugins.Rtp.VideoWidget : Gtk.Widget, Dino.Plugins.VideoCallWidget { private const int RECAPS_AFTER_CHANGE = 5; private static uint last_id = 0; public uint id { get; private set; } public Plugin plugin { get; private set; } public Gst.Pipeline pipe { get { return plugin.pipe; }} private bool attached; private Device? connected_device; private Gst.Element? connected_device_element; private Stream? connected_stream; private Gst.Element prepare; private Gst.Caps last_input_caps; private Gst.Caps last_caps; private int recaps_since_change; private Sink sink; private Gtk.Picture widget; public VideoWidget(Plugin plugin) { this.plugin = plugin; this.layout_manager = new Gtk.BinLayout(); id = last_id++; sink = new Sink() { async = false, sync = true }; widget = new Gtk.Picture.for_paintable(sink.paintable); widget.insert_after(this, null); } public void input_caps_changed(GLib.Object pad, ParamSpec spec) { Gst.Caps? caps = ((Gst.Pad)pad).caps; if (caps == null) { debug("Input: No caps"); return; } int width, height; caps.get_structure(0).get_int("width", out width); caps.get_structure(0).get_int("height", out height); debug("Input resolution changed: %ix%i", width, height); // Invoke signal on GTK main loop as recipients are likely to use it for doing GTK operations Idle.add(() => { resolution_changed(width, height); return Source.REMOVE; }); last_input_caps = caps; } public void display_stream(Xmpp.Xep.JingleRtp.Stream? stream, Xmpp.Jid jid) { if (sink == null) return; detach(); if (stream.media != "video") return; connected_stream = stream as Stream?; if (connected_stream == null) return; plugin.pause(); pipe.add(sink); prepare = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); prepare.name = @"video_widget_$(id)_prepare"; prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); pipe.add(prepare); connected_stream.add_output(prepare); prepare.link(sink); sink.set_locked_state(false); plugin.unpause(); attached = true; } public void display_device(MediaDevice media_device) { if (sink == null) return; detach(); connected_device = media_device as Device; if (connected_device == null) return; plugin.pause(); pipe.add(sink); #if GST_1_20 prepare = Gst.parse_bin_from_description(@"videoflip video-direction=auto name=video_widget_$(id)_orientation ! videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); #else prepare = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); #endif prepare.name = @"video_widget_$(id)_prepare"; #if GST_1_20 if (prepare is Gst.Bin) { ((Gst.Bin) prepare).get_by_name(@"video_widget_$(id)_flip").get_static_pad("sink").notify["caps"].connect(input_caps_changed); } else { #endif prepare.get_static_pad("sink").notify["caps"].connect(input_caps_changed); #if GST_1_20 } #endif pipe.add(prepare); connected_device_element = connected_device.link_source(); connected_device_element.link(prepare); prepare.link(sink); sink.set_locked_state(false); plugin.unpause(); attached = true; } public void detach() { if (sink == null) return; if (attached) { debug("Detaching"); if (connected_stream != null) { connected_stream.remove_output(prepare); connected_stream = null; } if (connected_device != null) { connected_device_element.unlink(sink); connected_device_element = null; connected_device.unlink(); connected_device = null; } prepare.set_locked_state(true); prepare.set_state(Gst.State.NULL); pipe.remove(prepare); prepare = null; sink.set_locked_state(true); sink.set_state(Gst.State.NULL); pipe.remove(sink); attached = false; } } public override void dispose() { detach(); if (widget != null) widget.unparent(); widget = null; sink = null; } }