diff options
author | fiaxh <git@lightrise.org> | 2021-05-11 12:57:02 +0200 |
---|---|---|
committer | fiaxh <git@lightrise.org> | 2021-05-11 12:57:02 +0200 |
commit | d71604913dd5b3372a823320db83c37c845fac5c (patch) | |
tree | 2ffbff97a02c81d48d8aef4a4b7ee870507236e9 /plugins | |
parent | e92ed27317ae398c867c946cf7206b1f0b32f3b4 (diff) | |
parent | 90f9ecf62b2ebfef14de2874e7942552409632bf (diff) | |
download | dino-d71604913dd5b3372a823320db83c37c845fac5c.tar.gz dino-d71604913dd5b3372a823320db83c37c845fac5c.zip |
Merge remote-tracking branch 'origin/feature/calls'
Diffstat (limited to 'plugins')
40 files changed, 5720 insertions, 412 deletions
diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 6cccec3b..00bb6509 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -2,6 +2,14 @@ if(DINO_PLUGIN_ENABLED_http-files) add_subdirectory(http-files) endif(DINO_PLUGIN_ENABLED_http-files) +if(DINO_PLUGIN_ENABLED_ice) + add_subdirectory(ice) +endif(DINO_PLUGIN_ENABLED_ice) + +if(DINO_PLUGIN_ENABLED_rtp) + add_subdirectory(rtp) +endif(DINO_PLUGIN_ENABLED_rtp) + if(DINO_PLUGIN_ENABLED_openpgp) add_subdirectory(gpgme-vala) add_subdirectory(openpgp) diff --git a/plugins/crypto-vala/CMakeLists.txt b/plugins/crypto-vala/CMakeLists.txt index 2c9f790a..f615854c 100644 --- a/plugins/crypto-vala/CMakeLists.txt +++ b/plugins/crypto-vala/CMakeLists.txt @@ -1,4 +1,5 @@ find_package(GCrypt REQUIRED) +find_package(Srtp2 REQUIRED) find_packages(CRYPTO_VALA_PACKAGES REQUIRED GLib GObject @@ -10,8 +11,11 @@ SOURCES "src/cipher.vala" "src/cipher_converter.vala" "src/error.vala" + "src/random.vala" + "src/srtp.vala" CUSTOM_VAPIS "${CMAKE_CURRENT_SOURCE_DIR}/vapi/gcrypt.vapi" + "${CMAKE_CURRENT_SOURCE_DIR}/vapi/libsrtp2.vapi" PACKAGES ${CRYPTO_VALA_PACKAGES} GENERATE_VAPI @@ -20,9 +24,9 @@ GENERATE_HEADER crypto-vala ) -set(CFLAGS ${VALA_CFLAGS} -I${CMAKE_CURRENT_SOURCE_DIR}/src) +set(CFLAGS ${VALA_CFLAGS}) add_definitions(${CFLAGS}) add_library(crypto-vala STATIC ${CRYPTO_VALA_C}) -target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt) +target_link_libraries(crypto-vala ${CRYPTO_VALA_PACKAGES} gcrypt libsrtp2) set_property(TARGET crypto-vala PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/plugins/crypto-vala/src/error.vala b/plugins/crypto-vala/src/error.vala index bae4ad08..5007d725 100644 --- a/plugins/crypto-vala/src/error.vala +++ b/plugins/crypto-vala/src/error.vala @@ -2,7 +2,9 @@ namespace Crypto { public errordomain Error { ILLEGAL_ARGUMENTS, - GCRYPT + GCRYPT, + AUTHENTICATION_FAILED, + UNKNOWN } internal void may_throw_gcrypt_error(GCrypt.Error e) throws Error { diff --git a/plugins/crypto-vala/src/random.vala b/plugins/crypto-vala/src/random.vala new file mode 100644 index 00000000..3f5d3ba9 --- /dev/null +++ b/plugins/crypto-vala/src/random.vala @@ -0,0 +1,5 @@ +namespace Crypto { +public static void randomize(uint8[] buffer) { + GCrypt.Random.randomize(buffer); +} +}
\ No newline at end of file diff --git a/plugins/crypto-vala/src/srtp.vala b/plugins/crypto-vala/src/srtp.vala new file mode 100644 index 00000000..493afdb0 --- /dev/null +++ b/plugins/crypto-vala/src/srtp.vala @@ -0,0 +1,122 @@ +using Srtp; + +public class Crypto.Srtp { + public const string AES_CM_128_HMAC_SHA1_80 = "AES_CM_128_HMAC_SHA1_80"; + public const string AES_CM_128_HMAC_SHA1_32 = "AES_CM_128_HMAC_SHA1_32"; + public const string F8_128_HMAC_SHA1_80 = "F8_128_HMAC_SHA1_80"; + + public class Session { + public bool has_encrypt { get; private set; default = false; } + public bool has_decrypt { get; private set; default = false; } + + private Context encrypt_context; + private Context decrypt_context; + + static construct { + init(); + install_log_handler(log); + } + + private static void log(LogLevel level, string msg) { + print(@"SRTP[$level]: $msg\n"); + } + + public Session() { + Context.create(out encrypt_context, null); + Context.create(out decrypt_context, null); + } + + public uint8[] encrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] encrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length + MAX_TRAILER_LEN + 4]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = encrypt_context.protect_rtcp(buf, ref buf_use); + if (res != ErrorStatus.ok) { + throw new Error.UNKNOWN(@"SRTCP encrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + public uint8[] decrypt_rtcp(uint8[] data) throws Error { + uint8[] buf = new uint8[data.length]; + Memory.copy(buf, data, data.length); + int buf_use = data.length; + ErrorStatus res = decrypt_context.unprotect_rtcp(buf, ref buf_use); + switch (res) { + case ErrorStatus.auth_fail: + throw new Error.AUTHENTICATION_FAILED("SRTCP packet failed the message authentication check"); + case ErrorStatus.ok: + break; + default: + throw new Error.UNKNOWN(@"SRTP decrypt failed: $res"); + } + uint8[] ret = new uint8[buf_use]; + GLib.Memory.copy(ret, buf, buf_use); + return ret; + } + + private Policy create_policy(string profile) { + Policy policy = Policy(); + switch (profile) { + case AES_CM_128_HMAC_SHA1_80: + policy.rtp.set_aes_cm_128_hmac_sha1_80(); + policy.rtcp.set_aes_cm_128_hmac_sha1_80(); + break; + } + return policy; + } + + public void set_encryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_outbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + encrypt_context.add_stream(ref policy); + has_encrypt = true; + } + + public void set_decryption_key(string profile, uint8[] key, uint8[] salt) { + Policy policy = create_policy(profile); + policy.ssrc.type = SsrcType.any_inbound; + policy.key = new uint8[key.length + salt.length]; + Memory.copy(policy.key, key, key.length); + Memory.copy(((uint8*)policy.key) + key.length, salt, salt.length); + decrypt_context.add_stream(ref policy); + has_decrypt = true; + } + } +}
\ No newline at end of file diff --git a/plugins/crypto-vala/vapi/libsrtp2.vapi b/plugins/crypto-vala/vapi/libsrtp2.vapi new file mode 100644 index 00000000..5ceedced --- /dev/null +++ b/plugins/crypto-vala/vapi/libsrtp2.vapi @@ -0,0 +1,115 @@ +[CCode (cheader_filename = "srtp2/srtp.h")] +namespace Srtp { +public const uint MAX_TRAILER_LEN; + +public static ErrorStatus init(); +public static ErrorStatus shutdown(); + +[Compact] +[CCode (cname = "srtp_ctx_t", cprefix = "srtp_", free_function = "srtp_dealloc")] +public class Context { + public static ErrorStatus create(out Context session, Policy? policy); + + public ErrorStatus protect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + public ErrorStatus unprotect([CCode (type = "void*", array_length = false)] uint8[] rtp, ref int len); + + public ErrorStatus protect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + public ErrorStatus unprotect_rtcp([CCode (type = "void*", array_length = false)] uint8[] rtcp, ref int len); + + public ErrorStatus add_stream(ref Policy policy); + public ErrorStatus update_stream(ref Policy policy); + public ErrorStatus remove_stream(uint ssrc); + public ErrorStatus update(ref Policy policy); +} + +[CCode (cname = "srtp_ssrc_t")] +public struct Ssrc { + public SsrcType type; + public uint value; +} + +[CCode (cname = "srtp_ssrc_type_t", cprefix = "ssrc_")] +public enum SsrcType { + undefined, specific, any_inbound, any_outbound +} + +[CCode (cname = "srtp_policy_t", destroy_function = "")] +public struct Policy { + public Ssrc ssrc; + public CryptoPolicy rtp; + public CryptoPolicy rtcp; + [CCode (array_length = false)] + public uint8[] key; + public ulong num_master_keys; + public ulong window_size; + public int allow_repeat_tx; + [CCode (array_length_cname = "enc_xtn_hdr_count")] + public int[] enc_xtn_hdr; +} + +[CCode (cname = "srtp_crypto_policy_t")] +public struct CryptoPolicy { + public CipherType cipher_type; + public int cipher_key_len; + public AuthType auth_type; + public int auth_key_len; + public int auth_tag_len; + public SecurityServices sec_serv; + + public void set_aes_cm_128_hmac_sha1_80(); + public void set_aes_cm_128_hmac_sha1_32(); + public void set_aes_cm_128_null_auth(); + public void set_aes_cm_192_hmac_sha1_32(); + public void set_aes_cm_192_hmac_sha1_80(); + public void set_aes_cm_192_null_auth(); + public void set_aes_cm_256_hmac_sha1_32(); + public void set_aes_cm_256_hmac_sha1_80(); + public void set_aes_cm_256_null_auth(); + public void set_aes_gcm_128_16_auth(); + public void set_aes_gcm_128_8_auth(); + public void set_aes_gcm_128_8_only_auth(); + public void set_aes_gcm_256_16_auth(); + public void set_aes_gcm_256_8_auth(); + public void set_aes_gcm_256_8_only_auth(); + public void set_null_cipher_hmac_null(); + public void set_null_cipher_hmac_sha1_80(); + + public void set_rtp_default(); + public void set_rtcp_default(); + + public void set_from_profile_for_rtp(Profile profile); + public void set_from_profile_for_rtcp(Profile profile); +} + +[CCode (cname = "srtp_profile_t", cprefix = "srtp_profile_")] +public enum Profile { + reserved, aes128_cm_sha1_80, aes128_cm_sha1_32, null_sha1_80, null_sha1_32, aead_aes_128_gcm, aead_aes_256_gcm +} + +[CCode (cname = "srtp_cipher_type_id_t")] +public struct CipherType : uint32 {} + +[CCode (cname = "srtp_auth_type_id_t")] +public struct AuthType : uint32 {} + +[CCode (cname = "srtp_sec_serv_t", cprefix = "sec_serv_")] +public enum SecurityServices { + none, conf, auth, conf_and_auth; +} + +[CCode (cname = "srtp_err_status_t", cprefix = "srtp_err_status_", has_type_id = false)] +public enum ErrorStatus { + ok, fail, bad_param, alloc_fail, dealloc_fail, init_fail, terminus, auth_fail, cipher_fail, replay_fail, algo_fail, no_such_op, no_ctx, cant_check, key_expired, socket_err, signal_err, nonce_bad, encode_err, semaphore_err, pfkey_err, bad_mki, pkt_idx_old, pkt_idx_adv +} + +[CCode (cname = "srtp_log_level_t", cprefix = "srtp_log_level_", has_type_id = false)] +public enum LogLevel { + error, warning, info, debug +} + +[CCode (cname = "srtp_log_handler_func_t")] +public delegate void LogHandler(LogLevel level, string msg); + +public static ErrorStatus install_log_handler(LogHandler func); + +}
\ No newline at end of file diff --git a/plugins/http-files/src/file_sender.vala b/plugins/http-files/src/file_sender.vala index 25db49b9..a038e70f 100644 --- a/plugins/http-files/src/file_sender.vala +++ b/plugins/http-files/src/file_sender.vala @@ -81,12 +81,6 @@ public class HttpFileSender : FileSender, Object { } } - public async long get_max_file_size(Account account) { - lock (max_file_sizes) { - return max_file_sizes[account]; - } - } - private static void transfer_more_bytes(InputStream stream, Soup.MessageBody body) { uint8[] bytes = new uint8[4096]; ssize_t read = stream.read(bytes); diff --git a/plugins/ice/CMakeLists.txt b/plugins/ice/CMakeLists.txt new file mode 100644 index 00000000..4783cea6 --- /dev/null +++ b/plugins/ice/CMakeLists.txt @@ -0,0 +1,36 @@ +find_package(Nice 0.1.15 REQUIRED) +find_package(GnuTLS REQUIRED) +find_packages(ICE_PACKAGES REQUIRED + Gee + GLib + GModule + GObject + GTK3 +) + +vala_precompile(ICE_VALA_C +SOURCES + src/dtls_srtp.vala + src/module.vala + src/plugin.vala + src/transport_parameters.vala + src/util.vala + src/register_plugin.vala +CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi + ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/nice.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/gnutls.vapi +PACKAGES + ${ICE_PACKAGES} +) + +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="ice") +add_library(ice SHARED ${ICE_VALA_C}) +target_link_libraries(ice libdino crypto-vala ${ICE_PACKAGES} nice gnutls) +set_target_properties(ice PROPERTIES PREFIX "") +set_target_properties(ice PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) + +install(TARGETS ice ${PLUGIN_INSTALL}) diff --git a/plugins/ice/src/dtls_srtp.vala b/plugins/ice/src/dtls_srtp.vala new file mode 100644 index 00000000..0254351d --- /dev/null +++ b/plugins/ice/src/dtls_srtp.vala @@ -0,0 +1,356 @@ +using GnuTLS; + +namespace Dino.Plugins.Ice.DtlsSrtp { + +public class CredentialsCapsule { + public uint8[] own_fingerprint; + public X509.Certificate[] own_cert; + public X509.PrivateKey private_key; +} + +public class Handler { + + public signal void send_data(uint8[] data); + + public bool ready { get { + return srtp_session.has_encrypt && srtp_session.has_decrypt; + }} + + public Mode mode { get; set; default = Mode.CLIENT; } + public uint8[] own_fingerprint { get; private set; } + public uint8[] peer_fingerprint { get; set; } + public string peer_fp_algo { get; set; } + + private CredentialsCapsule credentials; + private Cond buffer_cond = Cond(); + private Mutex buffer_mutex = Mutex(); + private Gee.LinkedList<Bytes> buffer_queue = new Gee.LinkedList<Bytes>(); + + private bool running = false; + private bool stop = false; + private bool restart = false; + + private Crypto.Srtp.Session srtp_session = new Crypto.Srtp.Session(); + + public Handler.with_cert(CredentialsCapsule creds) { + this.credentials = creds; + this.own_fingerprint = creds.own_fingerprint; + } + + public uint8[]? process_incoming_data(uint component_id, uint8[] data) { + if (srtp_session.has_decrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.decrypt_rtcp(data); + } + return srtp_session.decrypt_rtp(data); + } + if (component_id == 2) return srtp_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } + } else if (component_id == 1) { + on_data_rec(data); + } + return null; + } + + public uint8[]? process_outgoing_data(uint component_id, uint8[] data) { + if (srtp_session.has_encrypt) { + try { + if (component_id == 1) { + if (data.length >= 2 && data[1] >= 192 && data[1] < 224) { + return srtp_session.encrypt_rtcp(data); + } + return srtp_session.encrypt_rtp(data); + } + if (component_id == 2) return srtp_session.encrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + return null; + } + } + return null; + } + + public void on_data_rec(owned uint8[] data) { + buffer_mutex.lock(); + buffer_queue.add(new Bytes.take(data)); + buffer_cond.signal(); + buffer_mutex.unlock(); + } + + internal static CredentialsCapsule generate_credentials() throws GLib.Error { + int err = 0; + + X509.PrivateKey private_key = X509.PrivateKey.create(); + err = private_key.generate(PKAlgorithm.RSA, 2048); + throw_if_error(err); + + var start_time = new DateTime.now_local().add_days(1); + var end_time = start_time.add_days(2); + + X509.Certificate cert = X509.Certificate.create(); + cert.set_key(private_key); + cert.set_version(1); + cert.set_activation_time ((time_t) start_time.to_unix ()); + cert.set_expiration_time ((time_t) end_time.to_unix ()); + + uint32 serial = 1; + cert.set_serial(&serial, sizeof(uint32)); + + cert.sign(cert, private_key); + + uint8[] own_fingerprint = get_fingerprint(cert, DigestAlgorithm.SHA256); + X509.Certificate[] own_cert = new X509.Certificate[] { (owned)cert }; + + var creds = new CredentialsCapsule(); + creds.own_fingerprint = own_fingerprint; + creds.own_cert = (owned) own_cert; + creds.private_key = (owned) private_key; + + return creds; + } + + public void stop_dtls_connection() { + buffer_mutex.lock(); + stop = true; + buffer_cond.signal(); + buffer_mutex.unlock(); + } + + public async Xmpp.Xep.Jingle.ContentEncryption? setup_dtls_connection() { + buffer_mutex.lock(); + if (stop) { + restart = true; + buffer_mutex.unlock(); + return null; + } + if (running || ready) { + buffer_mutex.unlock(); + return null; + } + running = true; + restart = false; + buffer_mutex.unlock(); + + InitFlags server_or_client = mode == Mode.SERVER ? InitFlags.SERVER : InitFlags.CLIENT; + debug("Setting up DTLS connection. We're %s", mode.to_string()); + + CertificateCredentials cert_cred = CertificateCredentials.create(); + int err = cert_cred.set_x509_key(credentials.own_cert, credentials.private_key); + throw_if_error(err); + + Session? session = Session.create(server_or_client | InitFlags.DATAGRAM); + session.enable_heartbeat(1); + session.set_srtp_profile_direct("SRTP_AES128_CM_HMAC_SHA1_80"); + session.set_credentials(GnuTLS.CredentialsType.CERTIFICATE, cert_cred); + session.server_set_request(CertificateRequest.REQUEST); + session.set_priority_from_string("NORMAL:!VERS-TLS-ALL:+VERS-DTLS-ALL:+CTYPE-CLI-X509"); + + session.set_transport_pointer(this); + session.set_pull_function(pull_function); + session.set_pull_timeout_function(pull_timeout_function); + session.set_push_function(push_function); + session.set_verify_function(verify_function); + + Thread<int> thread = new Thread<int> (null, () => { + DateTime maximum_time = new DateTime.now_utc().add_seconds(20); + do { + err = session.handshake(); + + DateTime current_time = new DateTime.now_utc(); + if (maximum_time.compare(current_time) < 0) { + warning("DTLS handshake timeouted"); + err = ErrorCode.APPLICATION_ERROR_MIN + 1; + break; + } + if (stop) { + debug("DTLS handshake stopped"); + err = ErrorCode.APPLICATION_ERROR_MIN + 2; + break; + } + } while (err < 0 && !((ErrorCode)err).is_fatal()); + Idle.add(setup_dtls_connection.callback); + return err; + }); + yield; + err = thread.join(); + buffer_mutex.lock(); + if (stop) { + stop = false; + running = false; + bool restart = restart; + buffer_mutex.unlock(); + if (restart) { + debug("Restarting DTLS handshake"); + return yield setup_dtls_connection(); + } + return null; + } + buffer_mutex.unlock(); + if (err != ErrorCode.SUCCESS) { + warning("DTLS handshake failed: %s", ((ErrorCode)err).to_string()); + return null; + } + + uint8[] km = new uint8[150]; + Datum? client_key, client_salt, server_key, server_salt; + session.get_srtp_keys(km, km.length, out client_key, out client_salt, out server_key, out server_salt); + if (client_key == null || client_salt == null || server_key == null || server_salt == null) { + warning("SRTP client/server key/salt null"); + } + + debug("Finished DTLS connection. We're %s", mode.to_string()); + if (mode == Mode.SERVER) { + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); + } else { + srtp_session.set_encryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, client_key.extract(), client_salt.extract()); + srtp_session.set_decryption_key(Crypto.Srtp.AES_CM_128_HMAC_SHA1_80, server_key.extract(), server_salt.extract()); + } + return new Xmpp.Xep.Jingle.ContentEncryption() { encryption_ns=Xmpp.Xep.JingleIceUdp.DTLS_NS_URI, encryption_name = "DTLS-SRTP", our_key=credentials.own_fingerprint, peer_key=peer_fingerprint }; + } + + private static ssize_t pull_function(void* transport_ptr, uint8[] buffer) { + Handler self = transport_ptr as Handler; + + self.buffer_mutex.lock(); + while (self.buffer_queue.size == 0) { + self.buffer_cond.wait(self.buffer_mutex); + if (self.stop) { + self.buffer_mutex.unlock(); + debug("DTLS handshake pull_function stopped"); + return -1; + } + } + Bytes data = self.buffer_queue.remove_at(0); + self.buffer_mutex.unlock(); + + uint8[] data_uint8 = Bytes.unref_to_data((owned) data); + Memory.copy(buffer, data_uint8, data_uint8.length); + + // The callback should return 0 on connection termination, a positive number indicating the number of bytes received, and -1 on error. + return (ssize_t)data_uint8.length; + } + + private static int pull_timeout_function(void* transport_ptr, uint ms) { + Handler self = transport_ptr as Handler; + + int64 end_time = get_monotonic_time() + ms * 1000; + + self.buffer_mutex.lock(); + while (self.buffer_queue.size == 0) { + self.buffer_cond.wait_until(self.buffer_mutex, end_time); + if (self.stop) { + self.buffer_mutex.unlock(); + debug("DTLS handshake pull_timeout_function stopped"); + return -1; + } + + if (get_monotonic_time() > end_time) { + self.buffer_mutex.unlock(); + return 0; + } + } + self.buffer_mutex.unlock(); + + // The callback should return 0 on timeout, a positive number if data can be received, and -1 on error. + return 1; + } + + private static ssize_t push_function(void* transport_ptr, uint8[] buffer) { + Handler self = transport_ptr as Handler; + self.send_data(buffer); + + // The callback should return a positive number indicating the bytes sent, and -1 on error. + return (ssize_t)buffer.length; + } + + private static int verify_function(Session session) { + Handler self = session.get_transport_pointer() as Handler; + try { + bool valid = self.verify_peer_cert(session); + if (!valid) { + warning("DTLS certificate invalid. Aborting handshake."); + return 1; + } + } catch (Error e) { + warning("Error during DTLS certificate validation: %s. Aborting handshake.", e.message); + return 1; + } + + // The callback function should return 0 for the handshake to continue or non-zero to terminate. + return 0; + } + + private bool verify_peer_cert(Session session) throws GLib.Error { + unowned Datum[] cert_datums = session.get_peer_certificates(); + if (cert_datums.length == 0) { + warning("No peer certs"); + return false; + } + if (cert_datums.length > 1) warning("More than one peer cert"); + + X509.Certificate peer_cert = X509.Certificate.create(); + peer_cert.import(ref cert_datums[0], CertificateFormat.DER); + + DigestAlgorithm algo; + switch (peer_fp_algo) { + case "sha-256": + algo = DigestAlgorithm.SHA256; + break; + default: + warning("Unkown peer fingerprint algorithm: %s", peer_fp_algo); + return false; + } + + uint8[] real_peer_fp = get_fingerprint(peer_cert, algo); + + if (real_peer_fp.length != this.peer_fingerprint.length) { + warning("Fingerprint lengths not equal %i vs %i", real_peer_fp.length, peer_fingerprint.length); + return false; + } + + for (int i = 0; i < real_peer_fp.length; i++) { + if (real_peer_fp[i] != this.peer_fingerprint[i]) { + warning("First cert in peer cert list doesn't equal advertised one: %s vs %s", format_fingerprint(real_peer_fp), format_fingerprint(peer_fingerprint)); + return false; + } + } + + return true; + } +} + +private uint8[] get_fingerprint(X509.Certificate certificate, DigestAlgorithm digest_algo) { + uint8[] buf = new uint8[512]; + size_t buf_out_size = 512; + certificate.get_fingerprint(digest_algo, buf, ref buf_out_size); + + uint8[] ret = new uint8[buf_out_size]; + for (int i = 0; i < buf_out_size; i++) { + ret[i] = buf[i]; + } + return ret; +} + +private string format_fingerprint(uint8[] fingerprint) { + var sb = new StringBuilder(); + for (int i = 0; i < fingerprint.length; i++) { + sb.append("%02x".printf(fingerprint[i])); + if (i < fingerprint.length - 1) { + sb.append(":"); + } + } + return sb.str; +} + + +public enum Mode { + CLIENT, SERVER +} + +} diff --git a/plugins/ice/src/module.vala b/plugins/ice/src/module.vala new file mode 100644 index 00000000..2645d7dc --- /dev/null +++ b/plugins/ice/src/module.vala @@ -0,0 +1,55 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Ice.Module : JingleIceUdp.Module { + + public string? stun_ip = null; + public uint stun_port = 0; + public string? turn_ip = null; + public Xep.ExternalServiceDiscovery.Service? turn_service = null; + + private weak Nice.Agent? agent; + private HashMap<string, DtlsSrtp.CredentialsCapsule> cerds = new HashMap<string, DtlsSrtp.CredentialsCapsule>(); + + private Nice.Agent get_agent() { + Nice.Agent? agent = this.agent; + if (agent == null) { + agent = new Nice.Agent(MainContext.@default(), Nice.Compatibility.RFC5245); + if (stun_ip != null) { + agent.stun_server = stun_ip; + agent.stun_server_port = stun_port; + } + agent.ice_tcp = false; + agent.set_software("Dino"); + agent.weak_ref(agent_unweak); + this.agent = agent; + debug("STUN server for libnice %s %u", agent.stun_server, agent.stun_server_port); + } + return agent; + } + + public override Jingle.TransportParameters create_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid) { + DtlsSrtp.CredentialsCapsule? cred = get_create_credentials(local_full_jid, peer_full_jid); + return new TransportParameters(get_agent(), cred, turn_service, turn_ip, components, local_full_jid, peer_full_jid); + } + + public override Jingle.TransportParameters parse_transport_parameters(XmppStream stream, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode transport) throws Jingle.IqError { + DtlsSrtp.CredentialsCapsule? cred = get_create_credentials(local_full_jid, peer_full_jid); + return new TransportParameters(get_agent(), cred, turn_service, turn_ip, components, local_full_jid, peer_full_jid, transport); + } + + private DtlsSrtp.CredentialsCapsule? get_create_credentials(Jid local_full_jid, Jid peer_full_jid) { + string from_to_id = local_full_jid.to_string() + peer_full_jid.to_string(); + try { + if (!cerds.has_key(from_to_id)) cerds[from_to_id] = DtlsSrtp.Handler.generate_credentials(); + } catch (Error e) { + warning("Error creating dtls credentials: %s", e.message); + } + return cerds[from_to_id]; + } + + private void agent_unweak() { + this.agent = null; + } +}
\ No newline at end of file diff --git a/plugins/ice/src/plugin.vala b/plugins/ice/src/plugin.vala new file mode 100644 index 00000000..3ee8a72a --- /dev/null +++ b/plugins/ice/src/plugin.vala @@ -0,0 +1,71 @@ +using Gee; +using Dino.Entities; +using Xmpp; +using Xmpp.Xep; + +private extern const size_t NICE_ADDRESS_STRING_LEN; + +public class Dino.Plugins.Ice.Plugin : RootInterface, Object { + public Dino.Application app; + + public void registered(Dino.Application app) { + Nice.debug_enable(true); + this.app = app; + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module()); + }); + app.stream_interactor.stream_attached_modules.connect((account, stream) => { + stream.get_module(Socks5Bytestreams.Module.IDENTITY).set_local_ip_address_handler(get_local_ip_addresses); + }); + app.stream_interactor.stream_negotiated.connect(on_stream_negotiated); + } + + private async void on_stream_negotiated(Account account, XmppStream stream) { + Module? ice_udp_module = stream.get_module(JingleIceUdp.Module.IDENTITY) as Module; + if (ice_udp_module == null) return; + Gee.List<Xep.ExternalServiceDiscovery.Service> services = yield ExternalServiceDiscovery.request_services(stream); + foreach (Xep.ExternalServiceDiscovery.Service service in services) { + if (service.transport == "udp" && (service.ty == "stun" || service.ty == "turn")) { + InetAddress ip = yield lookup_ipv4_addess(service.host); + if (ip == null) continue; + + if (service.ty == "stun") { + debug("Server offers STUN server: %s:%u, resolved to %s", service.host, service.port, ip.to_string()); + ice_udp_module.stun_ip = ip.to_string(); + ice_udp_module.stun_port = service.port; + } else if (service.ty == "turn") { + debug("Server offers TURN server: %s:%u, resolved to %s", service.host, service.port, ip.to_string()); + ice_udp_module.turn_ip = ip.to_string(); + ice_udp_module.turn_service = service; + } + } + } + if (ice_udp_module.stun_ip == null) { + InetAddress ip = yield lookup_ipv4_addess("stun.l.google.com"); + if (ip == null) return; + + debug("Using fallback STUN server: stun.l.google.com:19302, resolved to %s", ip.to_string()); + + ice_udp_module.stun_ip = ip.to_string(); + ice_udp_module.stun_port = 19302; + } + } + + public void shutdown() { + // Nothing to do + } + + private async InetAddress? lookup_ipv4_addess(string host) { + try { + Resolver resolver = Resolver.get_default(); + GLib.List<GLib.InetAddress>? ips = yield resolver.lookup_by_name_async(host); + foreach (GLib.InetAddress ina in ips) { + if (ina.get_family() != SocketFamily.IPV4) continue; + return ina; + } + } catch (Error e) { + warning("Failed looking up IP address of %s", host); + } + return null; + } +}
\ No newline at end of file diff --git a/plugins/ice/src/register_plugin.vala b/plugins/ice/src/register_plugin.vala new file mode 100644 index 00000000..b2ed56c1 --- /dev/null +++ b/plugins/ice/src/register_plugin.vala @@ -0,0 +1,3 @@ +public Type register_plugin(Module module) { + return typeof (Dino.Plugins.Ice.Plugin); +} diff --git a/plugins/ice/src/transport_parameters.vala b/plugins/ice/src/transport_parameters.vala new file mode 100644 index 00000000..62c04906 --- /dev/null +++ b/plugins/ice/src/transport_parameters.vala @@ -0,0 +1,345 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + + +public class Dino.Plugins.Ice.TransportParameters : JingleIceUdp.IceUdpTransportParameters { + private Nice.Agent agent; + private uint stream_id; + private bool we_want_connection; + private bool remote_credentials_set; + private Map<uint8, DatagramConnection> connections = new HashMap<uint8, DatagramConnection>(); + private DtlsSrtp.Handler? dtls_srtp_handler; + + private class DatagramConnection : Jingle.DatagramConnection { + private Nice.Agent agent; + private DtlsSrtp.Handler? dtls_srtp_handler; + private uint stream_id; + private string? error; + private ulong sent; + private ulong sent_reported; + private ulong recv; + private ulong recv_reported; + private ulong datagram_received_id; + + public DatagramConnection(Nice.Agent agent, DtlsSrtp.Handler? dtls_srtp_handler, uint stream_id, uint8 component_id) { + this.agent = agent; + this.dtls_srtp_handler = dtls_srtp_handler; + this.stream_id = stream_id; + this.component_id = component_id; + this.datagram_received_id = this.datagram_received.connect((datagram) => { + recv += datagram.length; + if (recv > recv_reported + 100000) { + debug("Received %lu bytes via stream %u component %u", recv, stream_id, component_id); + recv_reported = recv; + } + }); + } + + public override async void terminate(bool we_terminated, string? reason_string = null, string? reason_text = null) { + yield base.terminate(we_terminated, reason_string, reason_text); + this.disconnect(datagram_received_id); + agent = null; + dtls_srtp_handler = null; + } + + public override void send_datagram(Bytes datagram) { + if (this.agent != null && is_component_ready(agent, stream_id, component_id)) { + uint8[] encrypted_data = null; + if (dtls_srtp_handler != null) { + encrypted_data = dtls_srtp_handler.process_outgoing_data(component_id, datagram.get_data()); + if (encrypted_data == null) return; + } + agent.send(stream_id, component_id, encrypted_data ?? datagram.get_data()); + sent += datagram.length; + if (sent > sent_reported + 100000) { + debug("Sent %lu bytes via stream %u component %u", sent, stream_id, component_id); + sent_reported = sent; + } + } + } + } + + public TransportParameters(Nice.Agent agent, DtlsSrtp.CredentialsCapsule? credentials, Xep.ExternalServiceDiscovery.Service? turn_service, string? turn_ip, uint8 components, Jid local_full_jid, Jid peer_full_jid, StanzaNode? node = null) { + base(components, local_full_jid, peer_full_jid, node); + this.we_want_connection = (node == null); + this.agent = agent; + + if (this.peer_fingerprint != null || !incoming) { + dtls_srtp_handler = setup_dtls(this, credentials); + own_fingerprint = dtls_srtp_handler.own_fingerprint; + if (incoming) { + own_setup = "active"; + dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; + dtls_srtp_handler.peer_fingerprint = peer_fingerprint; + dtls_srtp_handler.peer_fp_algo = peer_fp_algo; + } else { + own_setup = "actpass"; + dtls_srtp_handler.mode = DtlsSrtp.Mode.SERVER; + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + var content_encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (content_encryption != null) { + this.content.encryptions[content_encryption.encryption_ns] = content_encryption; + } + }); + } + } + + agent.candidate_gathering_done.connect(on_candidate_gathering_done); + agent.initial_binding_request_received.connect(on_initial_binding_request_received); + agent.component_state_changed.connect(on_component_state_changed); + agent.new_selected_pair_full.connect(on_new_selected_pair_full); + agent.new_candidate_full.connect(on_new_candidate); + + agent.controlling_mode = !incoming; + stream_id = agent.add_stream(components); + + if (turn_ip != null) { + for (uint8 component_id = 1; component_id <= components; component_id++) { + agent.set_relay_info(stream_id, component_id, turn_ip, turn_service.port, turn_service.username, turn_service.password, Nice.RelayType.UDP); + debug("TURN info (component %i) %s:%u", component_id, turn_ip, turn_service.port); + } + } + string ufrag; + string pwd; + agent.get_local_credentials(stream_id, out ufrag, out pwd); + init(ufrag, pwd); + + for (uint8 component_id = 1; component_id <= components; component_id++) { + // We don't properly get local candidates before this call + agent.attach_recv(stream_id, component_id, MainContext.@default(), on_recv); + } + + agent.gather_candidates(stream_id); + } + + private static DtlsSrtp.Handler setup_dtls(TransportParameters tp, DtlsSrtp.CredentialsCapsule credentials) { + var weak_self = WeakRef(tp); + DtlsSrtp.Handler dtls_srtp = new DtlsSrtp.Handler.with_cert(credentials); + dtls_srtp.send_data.connect((data) => { + TransportParameters self = (TransportParameters) weak_self.get(); + if (self != null) self.agent.send(self.stream_id, 1, data); + }); + return dtls_srtp; + } + + private void on_candidate_gathering_done(uint stream_id) { + if (stream_id != this.stream_id) return; + debug("on_candidate_gathering_done in %u", stream_id); + + for (uint8 i = 1; i <= components; i++) { + foreach (unowned Nice.Candidate nc in agent.get_local_candidates(stream_id, i)) { + if (nc.transport == Nice.CandidateTransport.UDP) { + JingleIceUdp.Candidate? candidate = candidate_to_jingle(nc); + if (candidate == null) continue; + debug("Local candidate summary: %s", agent.generate_local_candidate_sdp(nc)); + } + } + } + } + + private void on_new_candidate(Nice.Candidate nc) { + if (nc.stream_id != stream_id) return; + JingleIceUdp.Candidate? candidate = candidate_to_jingle(nc); + if (candidate == null) return; + + if (nc.transport == Nice.CandidateTransport.UDP) { + // Execution was in the agent thread before + add_local_candidate_threadsafe(candidate); + } + } + + public override void handle_transport_accept(StanzaNode transport) throws Jingle.IqError { + debug("on_transport_accept from %s", peer_full_jid.to_string()); + base.handle_transport_accept(transport); + + if (dtls_srtp_handler != null && peer_fingerprint != null) { + dtls_srtp_handler.peer_fingerprint = peer_fingerprint; + dtls_srtp_handler.peer_fp_algo = peer_fp_algo; + if (peer_setup == "passive") { + dtls_srtp_handler.mode = DtlsSrtp.Mode.CLIENT; + dtls_srtp_handler.stop_dtls_connection(); + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + var content_encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (content_encryption != null) { + this.content.encryptions[content_encryption.encryption_ns] = content_encryption; + } + }); + } + } else { + dtls_srtp_handler = null; + } + } + + public override void handle_transport_info(StanzaNode transport) throws Jingle.IqError { + debug("on_transport_info from %s", peer_full_jid.to_string()); + base.handle_transport_info(transport); + + if (!we_want_connection) return; + + if (remote_ufrag != null && remote_pwd != null && !remote_credentials_set) { + agent.set_remote_credentials(stream_id, remote_ufrag, remote_pwd); + remote_credentials_set = true; + } + for (uint8 i = 1; i <= components; i++) { + SList<Nice.Candidate> candidates = new SList<Nice.Candidate>(); + foreach (JingleIceUdp.Candidate candidate in remote_candidates) { + if (candidate.component == i) { + candidates.append(candidate_to_nice(candidate)); + } + } + int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); + debug("Updated to %i remote candidates for candidate %u via transport info", new_candidates, i); + } + } + + public override void create_transport_connection(XmppStream stream, Jingle.Content content) { + debug("create_transport_connection: %s", content.session.sid); + debug("local_credentials: %s %s", local_ufrag, local_pwd); + debug("remote_credentials: %s %s", remote_ufrag, remote_pwd); + debug("expected incoming credentials: %s %s", local_ufrag + ":" + remote_ufrag, local_pwd); + debug("expected outgoing credentials: %s %s", remote_ufrag + ":" + local_ufrag, remote_pwd); + + we_want_connection = true; + + if (remote_ufrag != null && remote_pwd != null && !remote_credentials_set) { + agent.set_remote_credentials(stream_id, remote_ufrag, remote_pwd); + remote_credentials_set = true; + } + for (uint8 i = 1; i <= components; i++) { + SList<Nice.Candidate> candidates = new SList<Nice.Candidate>(); + foreach (JingleIceUdp.Candidate candidate in remote_candidates) { + if (candidate.ip.has_prefix("fe80::")) continue; + if (candidate.component == i) { + candidates.append(candidate_to_nice(candidate)); + debug("remote candidate: %s", agent.generate_local_candidate_sdp(candidate_to_nice(candidate))); + } + } + int new_candidates = agent.set_remote_candidates(stream_id, i, candidates); + debug("Initiated component %u with %i remote candidates", i, new_candidates); + + connections[i] = new DatagramConnection(agent, dtls_srtp_handler, stream_id, i); + content.set_transport_connection(connections[i], i); + } + + base.create_transport_connection(stream, content); + } + + private void on_component_state_changed(uint stream_id, uint component_id, uint state) { + if (stream_id != this.stream_id) return; + debug("stream %u component %u state changed to %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); + may_consider_ready(stream_id, component_id); + if (incoming && dtls_srtp_handler != null && !dtls_srtp_handler.ready && is_component_ready(agent, stream_id, component_id) && dtls_srtp_handler.mode == DtlsSrtp.Mode.CLIENT) { + dtls_srtp_handler.setup_dtls_connection.begin((_, res) => { + Jingle.ContentEncryption? encryption = dtls_srtp_handler.setup_dtls_connection.end(res); + if (encryption != null) { + this.content.encryptions[encryption.encryption_ns] = encryption; + } + }); + } + } + + private void may_consider_ready(uint stream_id, uint component_id) { + if (stream_id != this.stream_id) return; + if (connections.has_key((uint8) component_id) && !connections[(uint8)component_id].ready && is_component_ready(agent, stream_id, component_id) && (dtls_srtp_handler == null || dtls_srtp_handler.ready)) { + connections[(uint8)component_id].ready = true; + } + } + + private void on_initial_binding_request_received(uint stream_id) { + if (stream_id != this.stream_id) return; + debug("initial_binding_request_received"); + } + + private void on_new_selected_pair_full(uint stream_id, uint component_id, Nice.Candidate p1, Nice.Candidate p2) { + if (stream_id != this.stream_id) return; + debug("new_selected_pair_full %u [%s, %s]", component_id, agent.generate_local_candidate_sdp(p1), agent.generate_local_candidate_sdp(p2)); + } + + private void on_recv(Nice.Agent agent, uint stream_id, uint component_id, uint8[] data) { + if (stream_id != this.stream_id) return; + uint8[] decrypt_data = null; + if (dtls_srtp_handler != null) { + decrypt_data = dtls_srtp_handler.process_incoming_data(component_id, data); + if (decrypt_data == null) return; + } + may_consider_ready(stream_id, component_id); + if (connections.has_key((uint8) component_id)) { + if (!connections[(uint8) component_id].ready) { + debug("on_recv stream %u component %u when state %s", stream_id, component_id, agent.get_component_state(stream_id, component_id).to_string()); + } + connections[(uint8) component_id].datagram_received(new Bytes(decrypt_data ?? data)); + } else { + debug("on_recv stream %u component %u length %u", stream_id, component_id, data.length); + } + } + + private static Nice.Candidate candidate_to_nice(JingleIceUdp.Candidate c) { + Nice.CandidateType type; + switch (c.type_) { + case JingleIceUdp.Candidate.Type.HOST: type = Nice.CandidateType.HOST; break; + case JingleIceUdp.Candidate.Type.PRFLX: type = Nice.CandidateType.PEER_REFLEXIVE; break; + case JingleIceUdp.Candidate.Type.RELAY: type = Nice.CandidateType.RELAYED; break; + case JingleIceUdp.Candidate.Type.SRFLX: type = Nice.CandidateType.SERVER_REFLEXIVE; break; + default: assert_not_reached(); + } + + Nice.Candidate candidate = new Nice.Candidate(type); + candidate.component_id = c.component; + char[] foundation = new char[Nice.CANDIDATE_MAX_FOUNDATION]; + Memory.copy(foundation, c.foundation.data, size_t.min(c.foundation.length, Nice.CANDIDATE_MAX_FOUNDATION - 1)); + candidate.foundation = foundation; + candidate.addr = Nice.Address(); + candidate.addr.init(); + candidate.addr.set_from_string(c.ip); + candidate.addr.set_port(c.port); + candidate.priority = c.priority; + if (c.rel_addr != null) { + candidate.base_addr = Nice.Address(); + candidate.base_addr.init(); + candidate.base_addr.set_from_string(c.rel_addr); + candidate.base_addr.set_port(c.rel_port); + } + candidate.transport = Nice.CandidateTransport.UDP; + return candidate; + } + + private static JingleIceUdp.Candidate? candidate_to_jingle(Nice.Candidate nc) { + JingleIceUdp.Candidate candidate = new JingleIceUdp.Candidate(); + switch (nc.type) { + case Nice.CandidateType.HOST: candidate.type_ = JingleIceUdp.Candidate.Type.HOST; break; + case Nice.CandidateType.PEER_REFLEXIVE: candidate.type_ = JingleIceUdp.Candidate.Type.PRFLX; break; + case Nice.CandidateType.RELAYED: candidate.type_ = JingleIceUdp.Candidate.Type.RELAY; break; + case Nice.CandidateType.SERVER_REFLEXIVE: candidate.type_ = JingleIceUdp.Candidate.Type.SRFLX; break; + default: assert_not_reached(); + } + candidate.component = (uint8) nc.component_id; + candidate.foundation = ((string)nc.foundation).dup(); + candidate.generation = 0; + candidate.id = Random.next_int().to_string("%08x"); // TODO + + char[] res = new char[NICE_ADDRESS_STRING_LEN]; + nc.addr.to_string(res); + candidate.ip = (string) res; + candidate.network = 0; // TODO + candidate.port = (uint16) nc.addr.get_port(); + candidate.priority = nc.priority; + candidate.protocol = "udp"; + if (nc.base_addr.is_valid() && !nc.base_addr.equal(nc.addr)) { + res = new char[NICE_ADDRESS_STRING_LEN]; + nc.base_addr.to_string(res); + candidate.rel_addr = (string) res; + candidate.rel_port = (uint16) nc.base_addr.get_port(); + } + if (candidate.ip.has_prefix("fe80::")) return null; + + return candidate; + } + + public override void dispose() { + base.dispose(); + agent = null; + dtls_srtp_handler = null; + connections.clear(); + } +} diff --git a/plugins/ice/src/util.vala b/plugins/ice/src/util.vala new file mode 100644 index 00000000..dd89d2f4 --- /dev/null +++ b/plugins/ice/src/util.vala @@ -0,0 +1,18 @@ +using Gee; + +namespace Dino.Plugins.Ice { + +internal static bool is_component_ready(Nice.Agent agent, uint stream_id, uint component_id) { + var state = agent.get_component_state(stream_id, component_id); + return state == Nice.ComponentState.CONNECTED || state == Nice.ComponentState.READY; +} + +internal Gee.List<string> get_local_ip_addresses() { + Gee.List<string> result = new ArrayList<string>(); + foreach (string ip_address in Nice.interfaces_get_local_ips(false)) { + result.add(ip_address); + } + return result; +} + +}
\ No newline at end of file diff --git a/plugins/ice/vapi/gnutls.vapi b/plugins/ice/vapi/gnutls.vapi new file mode 100644 index 00000000..bc3f13d0 --- /dev/null +++ b/plugins/ice/vapi/gnutls.vapi @@ -0,0 +1,419 @@ +[CCode (cprefix = "gnutls_", lower_case_cprefix = "gnutls_", cheader_filename = "gnutls/gnutls.h")] +namespace GnuTLS { + + public int global_init(); + + [CCode (cname = "gnutls_pull_func", has_target = false)] + public delegate ssize_t PullFunc(void* transport_ptr, [CCode (ctype = "void*", array_length_type="size_t")] uint8[] array); + + [CCode (cname = "gnutls_pull_timeout_func", has_target = false)] + public delegate int PullTimeoutFunc(void* transport_ptr, uint ms); + + [CCode (cname = "gnutls_push_func", has_target = false)] + public delegate ssize_t PushFunc(void* transport_ptr, [CCode (ctype = "void*", array_length_type="size_t")] uint8[] array); + + [CCode (cname = "gnutls_certificate_verify_function", has_target = false)] + public delegate int VerifyFunc(Session session); + + [Compact] + [CCode (cname = "struct gnutls_session_int", free_function = "gnutls_deinit")] + public class Session { + + public static Session? create(int con_end) throws GLib.Error { + Session result; + var ret = init(out result, con_end); + throw_if_error(ret); + return result; + } + + [CCode (cname = "gnutls_init")] + private static int init(out Session session, int con_end); + + [CCode (cname = "gnutls_transport_set_push_function")] + public void set_push_function(PushFunc func); + + [CCode (cname = "gnutls_transport_set_pull_function")] + public void set_pull_function(PullFunc func); + + [CCode (cname = "gnutls_transport_set_pull_timeout_function")] + public void set_pull_timeout_function(PullTimeoutFunc func); + + [CCode (cname = "gnutls_transport_set_ptr")] + public void set_transport_pointer(void* ptr); + + [CCode (cname = "gnutls_transport_get_ptr")] + public void* get_transport_pointer(); + + [CCode (cname = "gnutls_heartbeat_enable")] + public int enable_heartbeat(uint type); + + [CCode (cname = "gnutls_certificate_server_set_request")] + public void server_set_request(CertificateRequest req); + + [CCode (cname = "gnutls_credentials_set")] + public int set_credentials_(CredentialsType type, void* cred); + [CCode (cname = "gnutls_credentials_set_")] + public void set_credentials(CredentialsType type, void* cred) throws GLib.Error { + int err = set_credentials_(type, cred); + throw_if_error(err); + } + + [CCode (cname = "gnutls_priority_set_direct")] + public int set_priority_from_string_(string priority, out unowned string err_pos = null); + [CCode (cname = "gnutls_priority_set_direct_")] + public void set_priority_from_string(string priority, out unowned string err_pos = null) throws GLib.Error { + int err = set_priority_from_string_(priority, out err_pos); + throw_if_error(err); + } + + [CCode (cname = "gnutls_srtp_set_profile_direct")] + public int set_srtp_profile_direct_(string profiles, out unowned string err_pos = null); + [CCode (cname = "gnutls_srtp_set_profile_direct_")] + public void set_srtp_profile_direct(string profiles, out unowned string err_pos = null) throws GLib.Error { + int err = set_srtp_profile_direct_(profiles, out err_pos); + throw_if_error(err); + } + + [CCode (cname = "gnutls_transport_set_int")] + public void transport_set_int(int fd); + + [CCode (cname = "gnutls_handshake")] + public int handshake(); + + [CCode (cname = "gnutls_srtp_get_keys")] + public int get_srtp_keys_(void *key_material, uint32 key_material_size, out Datum client_key, out Datum client_salt, out Datum server_key, out Datum server_salt); + [CCode (cname = "gnutls_srtp_get_keys_")] + public void get_srtp_keys(void *key_material, uint32 key_material_size, out Datum client_key, out Datum client_salt, out Datum server_key, out Datum server_salt) throws GLib.Error { + get_srtp_keys_(key_material, key_material_size, out client_key, out client_salt, out server_key, out server_salt); + } + + [CCode (cname = "gnutls_certificate_get_peers", array_length_type = "unsigned int")] + public unowned Datum[]? get_peer_certificates(); + + [CCode (cname = "gnutls_session_set_verify_function")] + public void set_verify_function(VerifyFunc func); + } + + [Compact] + [CCode (cname = "struct gnutls_certificate_credentials_st", free_function = "gnutls_certificate_free_credentials", cprefix = "gnutls_certificate_")] + public class CertificateCredentials { + + [CCode (cname = "gnutls_certificate_allocate_credentials")] + private static int allocate(out CertificateCredentials credentials); + + public static CertificateCredentials create() throws GLib.Error { + CertificateCredentials result; + var ret = allocate (out result); + throw_if_error(ret); + return result; + } + + public void get_x509_crt(uint index, [CCode (array_length_type = "unsigned int")] out unowned X509.Certificate[] x509_ca_list); + + public int set_x509_key(X509.Certificate[] cert_list, X509.PrivateKey key); + } + + [CCode (cheader_filename = "gnutls/x509.h", cprefix = "GNUTLS_")] + namespace X509 { + + [Compact] + [CCode (cname = "struct gnutls_x509_crt_int", cprefix = "gnutls_x509_crt_", free_function = "gnutls_x509_crt_deinit")] + public class Certificate { + + [CCode (cname = "gnutls_x509_crt_init")] + private static int init (out Certificate cert); + public static Certificate create() throws GLib.Error { + Certificate result; + var ret = init (out result); + throw_if_error(ret); + return result; + } + + [CCode (cname = "gnutls_x509_crt_import")] + public int import_(ref Datum data, CertificateFormat format); + [CCode (cname = "gnutls_x509_crt_import_")] + public void import(ref Datum data, CertificateFormat format) throws GLib.Error { + int err = import_(ref data, format); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_version")] + public int set_version_(uint version); + [CCode (cname = "gnutls_x509_crt_set_version_")] + public void set_version(uint version) throws GLib.Error { + int err = set_version_(version); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_key")] + public int set_key_(PrivateKey key); + [CCode (cname = "gnutls_x509_crt_set_key_")] + public void set_key(PrivateKey key) throws GLib.Error { + int err = set_key_(key); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_activation_time")] + public int set_activation_time_(time_t act_time); + [CCode (cname = "gnutls_x509_crt_set_activation_time_")] + public void set_activation_time(time_t act_time) throws GLib.Error { + int err = set_activation_time_(act_time); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_expiration_time")] + public int set_expiration_time_(time_t exp_time); + [CCode (cname = "gnutls_x509_crt_set_expiration_time_")] + public void set_expiration_time(time_t exp_time) throws GLib.Error { + int err = set_expiration_time_(exp_time); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_set_serial")] + public int set_serial_(void* serial, size_t serial_size); + [CCode (cname = "gnutls_x509_crt_set_serial_")] + public void set_serial(void* serial, size_t serial_size) throws GLib.Error { + int err = set_serial_(serial, serial_size); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_sign")] + public int sign_(Certificate issuer, PrivateKey issuer_key); + [CCode (cname = "gnutls_x509_crt_sign_")] + public void sign(Certificate issuer, PrivateKey issuer_key) throws GLib.Error { + int err = sign_(issuer, issuer_key); + throw_if_error(err); + } + + [CCode (cname = "gnutls_x509_crt_get_fingerprint")] + public int get_fingerprint_(DigestAlgorithm algo, void* buf, ref size_t buf_size); + [CCode (cname = "gnutls_x509_crt_get_fingerprint_")] + public void get_fingerprint(DigestAlgorithm algo, void* buf, ref size_t buf_size) throws GLib.Error { + int err = get_fingerprint_(algo, buf, ref buf_size); + throw_if_error(err); + } + } + + [Compact] + [CCode (cname = "struct gnutls_x509_privkey_int", cprefix = "gnutls_x509_privkey_", free_function = "gnutls_x509_privkey_deinit")] + public class PrivateKey { + private static int init (out PrivateKey key); + public static PrivateKey create () throws GLib.Error { + PrivateKey result; + var ret = init (out result); + throw_if_error(ret); + return result; + } + + public int generate(PKAlgorithm algo, uint bits, uint flags = 0); + } + + } + + [CCode (cname = "gnutls_certificate_request_t", cprefix = "GNUTLS_CERT_", has_type_id = false)] + public enum CertificateRequest { + IGNORE, + REQUEST, + REQUIRE + } + + [CCode (cname = "gnutls_pk_algorithm_t", cprefix = "GNUTLS_PK_", has_type_id = false)] + public enum PKAlgorithm { + UNKNOWN, + RSA, + DSA; + } + + [CCode (cname = "gnutls_digest_algorithm_t", cprefix = "GNUTLS_DIG_", has_type_id = false)] + public enum DigestAlgorithm { + NULL, + MD5, + SHA1, + RMD160, + MD2, + SHA224, + SHA256, + SHA384, + SHA512; + } + + [Flags] + [CCode (cname = "gnutls_init_flags_t", cprefix = "GNUTLS_", has_type_id = false)] + public enum InitFlags { + SERVER, + CLIENT, + DATAGRAM + } + + [CCode (cname = "gnutls_credentials_type_t", cprefix = "GNUTLS_CRD_", has_type_id = false)] + public enum CredentialsType { + CERTIFICATE, + ANON, + SRP, + PSK, + IA + } + + [CCode (cname = "gnutls_x509_crt_fmt_t", cprefix = "GNUTLS_X509_FMT_", has_type_id = false)] + public enum CertificateFormat { + DER, + PEM + } + + [Flags] + [CCode (cname = "gnutls_certificate_status_t", cprefix = "GNUTLS_CERT_", has_type_id = false)] + public enum CertificateStatus { + INVALID, // will be set if the certificate was not verified. + REVOKED, // in X.509 this will be set only if CRLs are checked + SIGNER_NOT_FOUND, + SIGNER_NOT_CA, + INSECURE_ALGORITHM + } + + [SimpleType] + [CCode (cname = "gnutls_datum_t", has_type_id = false)] + public struct Datum { + public uint8* data; + public uint size; + + public uint8[] extract() { + uint8[] ret = new uint8[size]; + for (int i = 0; i < size; i++) { + ret[i] = data[i]; + } + return ret; + } + } + + // Gnutls error codes. The mapping to a TLS alert is also shown in comments. + [CCode (cname = "int", cprefix = "GNUTLS_E_", lower_case_cprefix = "gnutls_error_", has_type_id = false)] + public enum ErrorCode { + SUCCESS, + UNKNOWN_COMPRESSION_ALGORITHM, + UNKNOWN_CIPHER_TYPE, + LARGE_PACKET, + UNSUPPORTED_VERSION_PACKET, // GNUTLS_A_PROTOCOL_VERSION + UNEXPECTED_PACKET_LENGTH, // GNUTLS_A_RECORD_OVERFLOW + INVALID_SESSION, + FATAL_ALERT_RECEIVED, + UNEXPECTED_PACKET, // GNUTLS_A_UNEXPECTED_MESSAGE + WARNING_ALERT_RECEIVED, + ERROR_IN_FINISHED_PACKET, + UNEXPECTED_HANDSHAKE_PACKET, + UNKNOWN_CIPHER_SUITE, // GNUTLS_A_HANDSHAKE_FAILURE + UNWANTED_ALGORITHM, + MPI_SCAN_FAILED, + DECRYPTION_FAILED, // GNUTLS_A_DECRYPTION_FAILED, GNUTLS_A_BAD_RECORD_MAC + MEMORY_ERROR, + DECOMPRESSION_FAILED, // GNUTLS_A_DECOMPRESSION_FAILURE + COMPRESSION_FAILED, + AGAIN, + EXPIRED, + DB_ERROR, + SRP_PWD_ERROR, + INSUFFICIENT_CREDENTIALS, + HASH_FAILED, + BASE64_DECODING_ERROR, + MPI_PRINT_FAILED, + REHANDSHAKE, // GNUTLS_A_NO_RENEGOTIATION + GOT_APPLICATION_DATA, + RECORD_LIMIT_REACHED, + ENCRYPTION_FAILED, + PK_ENCRYPTION_FAILED, + PK_DECRYPTION_FAILED, + PK_SIGN_FAILED, + X509_UNSUPPORTED_CRITICAL_EXTENSION, + KEY_USAGE_VIOLATION, + NO_CERTIFICATE_FOUND, // GNUTLS_A_BAD_CERTIFICATE + INVALID_REQUEST, + SHORT_MEMORY_BUFFER, + INTERRUPTED, + PUSH_ERROR, + PULL_ERROR, + RECEIVED_ILLEGAL_PARAMETER, // GNUTLS_A_ILLEGAL_PARAMETER + REQUESTED_DATA_NOT_AVAILABLE, + PKCS1_WRONG_PAD, + RECEIVED_ILLEGAL_EXTENSION, + INTERNAL_ERROR, + DH_PRIME_UNACCEPTABLE, + FILE_ERROR, + TOO_MANY_EMPTY_PACKETS, + UNKNOWN_PK_ALGORITHM, + // returned if libextra functionality was requested but + // gnutls_global_init_extra() was not called. + + INIT_LIBEXTRA, + LIBRARY_VERSION_MISMATCH, + // returned if you need to generate temporary RSA + // parameters. These are needed for export cipher suites. + + NO_TEMPORARY_RSA_PARAMS, + LZO_INIT_FAILED, + NO_COMPRESSION_ALGORITHMS, + NO_CIPHER_SUITES, + OPENPGP_GETKEY_FAILED, + PK_SIG_VERIFY_FAILED, + ILLEGAL_SRP_USERNAME, + SRP_PWD_PARSING_ERROR, + NO_TEMPORARY_DH_PARAMS, + // For certificate and key stuff + + ASN1_ELEMENT_NOT_FOUND, + ASN1_IDENTIFIER_NOT_FOUND, + ASN1_DER_ERROR, + ASN1_VALUE_NOT_FOUND, + ASN1_GENERIC_ERROR, + ASN1_VALUE_NOT_VALID, + ASN1_TAG_ERROR, + ASN1_TAG_IMPLICIT, + ASN1_TYPE_ANY_ERROR, + ASN1_SYNTAX_ERROR, + ASN1_DER_OVERFLOW, + OPENPGP_UID_REVOKED, + CERTIFICATE_ERROR, + CERTIFICATE_KEY_MISMATCH, + UNSUPPORTED_CERTIFICATE_TYPE, // GNUTLS_A_UNSUPPORTED_CERTIFICATE + X509_UNKNOWN_SAN, + OPENPGP_FINGERPRINT_UNSUPPORTED, + X509_UNSUPPORTED_ATTRIBUTE, + UNKNOWN_HASH_ALGORITHM, + UNKNOWN_PKCS_CONTENT_TYPE, + UNKNOWN_PKCS_BAG_TYPE, + INVALID_PASSWORD, + MAC_VERIFY_FAILED, // for PKCS #12 MAC + CONSTRAINT_ERROR, + WARNING_IA_IPHF_RECEIVED, + WARNING_IA_FPHF_RECEIVED, + IA_VERIFY_FAILED, + UNKNOWN_ALGORITHM, + BASE64_ENCODING_ERROR, + INCOMPATIBLE_CRYPTO_LIBRARY, + INCOMPATIBLE_LIBTASN1_LIBRARY, + OPENPGP_KEYRING_ERROR, + X509_UNSUPPORTED_OID, + RANDOM_FAILED, + BASE64_UNEXPECTED_HEADER_ERROR, + OPENPGP_SUBKEY_ERROR, + CRYPTO_ALREADY_REGISTERED, + HANDSHAKE_TOO_LARGE, + UNIMPLEMENTED_FEATURE, + APPLICATION_ERROR_MAX, // -65000 + APPLICATION_ERROR_MIN; // -65500 + + [CCode (cname = "gnutls_error_is_fatal")] + public bool is_fatal(); + + [CCode (cname = "gnutls_perror")] + public void print(); + + [CCode (cname = "gnutls_strerror")] + public unowned string to_string(); + } + + public void throw_if_error(int err_int) throws GLib.Error { + ErrorCode error = (ErrorCode)err_int; + if (error != ErrorCode.SUCCESS) { + throw new GLib.Error(-1, error, "%s%s", error.to_string(), error.is_fatal() ? " fatal" : ""); + } + } +}
\ No newline at end of file diff --git a/plugins/ice/vapi/metadata/Nice-0.1.metadata b/plugins/ice/vapi/metadata/Nice-0.1.metadata new file mode 100644 index 00000000..7fcf046a --- /dev/null +++ b/plugins/ice/vapi/metadata/Nice-0.1.metadata @@ -0,0 +1,11 @@ +Nice cheader_filename="nice.h" +Address.to_string.dst type="char[]" +Agent.new_reliable#constructor name="create_reliable" +Agent.attach_recv skip=false +Agent.send.buf type="uint8[]" array_length_idx=2 +AgentRecvFunc.buf type="uint8[]" array_length_idx=3 +PseudoTcpCallbacks#record skip +PseudoTcpSocket#class skip + +# Not yet supported by vapigen +# Candidate copy_function="nice_candidate_copy" free_function="nice_candidate_free" type_id="" diff --git a/plugins/ice/vapi/nice.vapi b/plugins/ice/vapi/nice.vapi new file mode 100644 index 00000000..540e2b4e --- /dev/null +++ b/plugins/ice/vapi/nice.vapi @@ -0,0 +1,386 @@ +/* nice.vapi generated by vapigen, do not modify. */ + +[CCode (cprefix = "Nice", gir_namespace = "Nice", gir_version = "0.1", lower_case_cprefix = "nice_")] +namespace Nice { + [CCode (cheader_filename = "nice.h", type_id = "nice_agent_get_type ()")] + public class Agent : GLib.Object { + [CCode (has_construct_function = false)] + public Agent (GLib.MainContext ctx, Nice.Compatibility compat); + public bool add_local_address (Nice.Address addr); + public uint add_stream (uint n_components); + public bool attach_recv (uint stream_id, uint component_id, GLib.MainContext ctx, Nice.AgentRecvFunc func); + [Version (since = "0.1.16")] + public async void close_async (); + [CCode (cname = "nice_agent_new_reliable", has_construct_function = false)] + [Version (since = "0.0.11")] + public Agent.create_reliable (GLib.MainContext ctx, Nice.Compatibility compat); + [Version (since = "0.1.6")] + public bool forget_relays (uint stream_id, uint component_id); + [CCode (has_construct_function = false)] + [Version (since = "0.1.15")] + public Agent.full (GLib.MainContext ctx, Nice.Compatibility compat, Nice.AgentOption flags); + public bool gather_candidates (uint stream_id); + [Version (since = "0.1.4")] + public string generate_local_candidate_sdp (Nice.Candidate candidate); + [Version (since = "0.1.4")] + public string generate_local_sdp (); + [Version (since = "0.1.4")] + public string generate_local_stream_sdp (uint stream_id, bool include_non_ice); + [Version (since = "0.1.8")] + public Nice.ComponentState get_component_state (uint stream_id, uint component_id); + public Nice.Candidate get_default_local_candidate (uint stream_id, uint component_id); + [Version (since = "0.1.5")] + public GLib.IOStream get_io_stream (uint stream_id, uint component_id); + public GLib.SList<Nice.Candidate> get_local_candidates (uint stream_id, uint component_id); + public bool get_local_credentials (uint stream_id, out string ufrag, out string pwd); + public GLib.SList<Nice.Candidate> get_remote_candidates (uint stream_id, uint component_id); + public bool get_selected_pair (uint stream_id, uint component_id, Nice.Candidate local, Nice.Candidate remote); + [Version (since = "0.1.5")] + public GLib.Socket? get_selected_socket (uint stream_id, uint component_id); + [Version (since = "0.1.4")] + public unowned string get_stream_name (uint stream_id); + [Version (since = "0.1.4")] + public Nice.Candidate parse_remote_candidate_sdp (uint stream_id, string sdp); + [Version (since = "0.1.4")] + public int parse_remote_sdp (string sdp); + [Version (since = "0.1.4")] + public GLib.SList<Nice.Candidate> parse_remote_stream_sdp (uint stream_id, string sdp, string ufrag, string pwd); + [Version (since = "0.1.16")] + public bool peer_candidate_gathering_done (uint stream_id); + [Version (since = "0.1.5")] + public ssize_t recv (uint stream_id, uint component_id, [CCode (array_length_cname = "buf_len", array_length_pos = 3.5, array_length_type = "gsize")] out unowned uint8[] buf, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public int recv_messages (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] out unowned Nice.InputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public int recv_messages_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] out unowned Nice.InputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + [Version (since = "0.1.5")] + public ssize_t recv_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "buf_len", array_length_pos = 3.5, array_length_type = "gsize")] out unowned uint8[] buf, GLib.Cancellable? cancellable = null) throws GLib.Error; + public void remove_stream (uint stream_id); + public bool restart (); + [Version (since = "0.1.6")] + public bool restart_stream (uint stream_id); + public int send (uint stream_id, uint component_id, [CCode (array_length_cname = "len", array_length_pos = 2.5, array_length_type = "guint", type = "const gchar*")] uint8[] buf); + [Version (since = "0.1.5")] + public int send_messages_nonblocking (uint stream_id, uint component_id, [CCode (array_length_cname = "n_messages", array_length_pos = 3.5, array_length_type = "guint")] Nice.OutputMessage[] messages, GLib.Cancellable? cancellable = null) throws GLib.Error; + public bool set_local_credentials (uint stream_id, string ufrag, string pwd); + public void set_port_range (uint stream_id, uint component_id, uint min_port, uint max_port); + public bool set_relay_info (uint stream_id, uint component_id, string server_ip, uint server_port, string username, string password, Nice.RelayType type); + public int set_remote_candidates (uint stream_id, uint component_id, GLib.SList<Nice.Candidate> candidates); + public bool set_remote_credentials (uint stream_id, string ufrag, string pwd); + public bool set_selected_pair (uint stream_id, uint component_id, string lfoundation, string rfoundation); + public bool set_selected_remote_candidate (uint stream_id, uint component_id, Nice.Candidate candidate); + [Version (since = "0.0.10")] + public void set_software (string software); + [Version (since = "0.1.4")] + public bool set_stream_name (uint stream_id, string name); + [Version (since = "0.0.9")] + public void set_stream_tos (uint stream_id, int tos); + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool bytestream_tcp { get; } + [NoAccessorMethod] + public uint compatibility { get; construct; } + [NoAccessorMethod] + public bool controlling_mode { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.14")] + public bool force_relay { get; set; } + [NoAccessorMethod] + public bool full_mode { get; construct; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool ice_tcp { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.16")] + public bool ice_trickle { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool ice_udp { get; set; } + [NoAccessorMethod] + [Version (since = "0.1.8")] + public bool keepalive_conncheck { get; set; } + [NoAccessorMethod] + public void* main_context { get; construct; } + [NoAccessorMethod] + public uint max_connectivity_checks { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_ip { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_password { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public uint proxy_port { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public uint proxy_type { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.4")] + public string proxy_username { owned get; set; } + [NoAccessorMethod] + [Version (since = "0.0.11")] + public bool reliable { get; construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_initial_timeout { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_max_retransmissions { get; set construct; } + [NoAccessorMethod] + public uint stun_pacing_timer { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.1.15")] + public uint stun_reliable_timeout { get; set construct; } + [NoAccessorMethod] + public string stun_server { owned get; set; } + [NoAccessorMethod] + public uint stun_server_port { get; set; } + [NoAccessorMethod] + public bool support_renomination { get; set; } + [NoAccessorMethod] + [Version (since = "0.0.7")] + public bool upnp { get; set construct; } + [NoAccessorMethod] + [Version (since = "0.0.7")] + public uint upnp_timeout { get; set construct; } + public signal void candidate_gathering_done (uint stream_id); + public signal void component_state_changed (uint stream_id, uint component_id, uint state); + public signal void initial_binding_request_received (uint stream_id); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_candidate (uint stream_id, uint component_id, string foundation); + [Version (since = "0.1.8")] + public signal void new_candidate_full (Nice.Candidate candidate); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_remote_candidate (uint stream_id, uint component_id, string foundation); + [Version (since = "0.1.8")] + public signal void new_remote_candidate_full (Nice.Candidate candidate); + [Version (deprecated = true, deprecated_since = "0.1.8")] + public signal void new_selected_pair (uint stream_id, uint component_id, string lfoundation, string rfoundation); + [Version (since = "0.1.8")] + public signal void new_selected_pair_full (uint stream_id, uint component_id, Nice.Candidate lcandidate, Nice.Candidate rcandidate); + [Version (since = "0.0.11")] + public signal void reliable_transport_writable (uint stream_id, uint component_id); + [Version (since = "0.1.5")] + public signal void streams_removed ([CCode (array_length = false, array_null_terminated = true)] uint[] stream_ids); + } + [CCode (cheader_filename = "nice.h", copy_function = "nice_candidate_copy", free_function = "nice_candidate_free")] + [Compact] + public class Candidate { + public Nice.Address addr; + public Nice.Address base_addr; + public uint component_id; + [CCode (array_length = false)] + public weak char foundation[33]; + public weak string password; + public uint32 priority; + public void* sockptr; + public uint stream_id; + public Nice.CandidateTransport transport; + public Nice.TurnServer turn; + public Nice.CandidateType type; + public weak string username; + [CCode (has_construct_function = false)] + public Candidate (Nice.CandidateType type); + public Nice.Candidate copy (); + [Version (since = "0.1.15")] + public bool equal_target (Nice.Candidate candidate2); + public void free (); + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + public struct Address { + [CCode (cname = "s.addr")] + public void* s_addr; + [CCode (cname = "s.ip4")] + public void* s_ip4; + [CCode (cname = "s.ip6")] + public void* s_ip6; + public void copy_to_sockaddr (void* sin); + public bool equal (Nice.Address b); + [Version (since = "0.1.8")] + public bool equal_no_port (Nice.Address b); + public void free (); + public uint get_port (); + public void init (); + public int ip_version (); + public bool is_private (); + public bool is_valid (); + public void set_from_sockaddr (void* sin); + public bool set_from_string (string str); + public void set_ipv4 (uint32 addr_ipv4); + public void set_ipv6 (uint8 addr_ipv6); + public void set_port (uint port); + public void to_string ([CCode (array_length = false, type = "gchar*")] char[] dst); + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + [Version (since = "0.1.5")] + public struct InputMessage { + [CCode (array_length_cname = "n_buffers")] + public weak GLib.InputVector[] buffers; + public int n_buffers; + public Nice.Address from; + public size_t length; + } + [CCode (cheader_filename = "nice.h", has_type_id = false)] + [Version (since = "0.1.5")] + public struct OutputMessage { + [CCode (array_length_cname = "n_buffers")] + public weak GLib.OutputVector[] buffers; + public int n_buffers; + } + [CCode (cheader_filename = "nice.h", cname = "TurnServer", has_type_id = false)] + public struct TurnServer { + public int ref_count; + public Nice.Address server; + public weak string username; + public weak string password; + public Nice.RelayType type; + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_AGENT_OPTION_", has_type_id = false)] + [Flags] + [Version (since = "0.1.15")] + public enum AgentOption { + REGULAR_NOMINATION, + RELIABLE, + LITE_MODE, + ICE_TRICKLE, + SUPPORT_RENOMINATION + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_CANDIDATE_TRANSPORT_", has_type_id = false)] + public enum CandidateTransport { + UDP, + TCP_ACTIVE, + TCP_PASSIVE, + TCP_SO + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_CANDIDATE_TYPE_", has_type_id = false)] + public enum CandidateType { + HOST, + SERVER_REFLEXIVE, + PEER_REFLEXIVE, + RELAYED + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPATIBILITY_", has_type_id = false)] + public enum Compatibility { + RFC5245, + DRAFT19, + GOOGLE, + MSN, + WLM2009, + OC2007, + OC2007R2, + LAST + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPONENT_STATE_", has_type_id = false)] + public enum ComponentState { + DISCONNECTED, + GATHERING, + CONNECTING, + CONNECTED, + READY, + FAILED, + LAST; + [Version (since = "0.1.6")] + public unowned string to_string (); + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_COMPONENT_TYPE_", has_type_id = false)] + public enum ComponentType { + RTP, + RTCP + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_NOMINATION_MODE_", has_type_id = false)] + [Version (since = "0.1.15")] + public enum NominationMode { + REGULAR, + AGGRESSIVE + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_PROXY_TYPE_", has_type_id = false)] + [Version (since = "0.0.4")] + public enum ProxyType { + NONE, + SOCKS5, + HTTP, + LAST + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpDebugLevel", cprefix = "PSEUDO_TCP_DEBUG_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpDebugLevel { + NONE, + NORMAL, + VERBOSE + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpShutdown", cprefix = "PSEUDO_TCP_SHUTDOWN_", has_type_id = false)] + [Version (since = "0.1.8")] + public enum PseudoTcpShutdown { + RD, + WR, + RDWR + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpState", cprefix = "PSEUDO_TCP_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpState { + LISTEN, + SYN_SENT, + SYN_RECEIVED, + ESTABLISHED, + CLOSED, + FIN_WAIT_1, + FIN_WAIT_2, + CLOSING, + TIME_WAIT, + CLOSE_WAIT, + LAST_ACK + } + [CCode (cheader_filename = "nice.h", cname = "PseudoTcpWriteResult", cprefix = "WR_", has_type_id = false)] + [Version (since = "0.0.11")] + public enum PseudoTcpWriteResult { + SUCCESS, + TOO_LARGE, + FAIL + } + [CCode (cheader_filename = "nice.h", cprefix = "NICE_RELAY_TYPE_TURN_", has_type_id = false)] + public enum RelayType { + UDP, + TCP, + TLS + } + [CCode (cheader_filename = "nice.h", instance_pos = 4.9)] + public delegate void AgentRecvFunc (Nice.Agent agent, uint stream_id, uint component_id, [CCode (array_length_cname = "len", array_length_pos = 3.5, array_length_type = "guint", type = "gchar*")] uint8[] buf); + [CCode (cheader_filename = "nice.h", cname = "NICE_AGENT_MAX_REMOTE_CANDIDATES")] + public const int AGENT_MAX_REMOTE_CANDIDATES; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_DIRECTION_MS_PREF_ACTIVE")] + public const int CANDIDATE_DIRECTION_MS_PREF_ACTIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_DIRECTION_MS_PREF_PASSIVE")] + public const int CANDIDATE_DIRECTION_MS_PREF_PASSIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_MAX_FOUNDATION")] + public const int CANDIDATE_MAX_FOUNDATION; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TRANSPORT_MS_PREF_TCP")] + public const int CANDIDATE_TRANSPORT_MS_PREF_TCP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TRANSPORT_MS_PREF_UDP")] + public const int CANDIDATE_TRANSPORT_MS_PREF_UDP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_HOST")] + public const int CANDIDATE_TYPE_PREF_HOST; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_NAT_ASSISTED")] + public const int CANDIDATE_TYPE_PREF_NAT_ASSISTED; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_PEER_REFLEXIVE")] + public const int CANDIDATE_TYPE_PREF_PEER_REFLEXIVE; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_RELAYED")] + public const int CANDIDATE_TYPE_PREF_RELAYED; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_RELAYED_UDP")] + public const int CANDIDATE_TYPE_PREF_RELAYED_UDP; + [CCode (cheader_filename = "nice.h", cname = "NICE_CANDIDATE_TYPE_PREF_SERVER_REFLEXIVE")] + public const int CANDIDATE_TYPE_PREF_SERVER_REFLEXIVE; + [CCode (cheader_filename = "nice.h")] + public static void debug_disable (bool with_stun); + [CCode (cheader_filename = "nice.h")] + public static void debug_enable (bool with_stun); + [CCode (cheader_filename = "nice.h")] + public static string? interfaces_get_ip_for_interface (string interface_name); + [CCode (cheader_filename = "nice.h")] + public static GLib.List<string> interfaces_get_local_interfaces (); + [CCode (cheader_filename = "nice.h")] + public static GLib.List<string> interfaces_get_local_ips (bool include_loopback); + [CCode (cheader_filename = "nice.h", cname = "pseudo_tcp_set_debug_level")] + [Version (since = "0.0.11")] + public static void pseudo_tcp_set_debug_level (Nice.PseudoTcpDebugLevel level); +} diff --git a/plugins/omemo/CMakeLists.txt b/plugins/omemo/CMakeLists.txt index 0f5a1521..195001cb 100644 --- a/plugins/omemo/CMakeLists.txt +++ b/plugins/omemo/CMakeLists.txt @@ -3,13 +3,13 @@ find_package(Gettext) include(${GETTEXT_USE_FILE}) gettext_compile(${GETTEXT_PACKAGE} SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/po TARGET_NAME ${GETTEXT_PACKAGE}-translations) +find_package(Qrencode REQUIRED) find_packages(OMEMO_PACKAGES REQUIRED Gee GLib GModule GObject GTK3 - Qrencode ) set(RESOURCE_LIST @@ -29,6 +29,7 @@ compile_gresources( vala_precompile(OMEMO_VALA_C SOURCES + src/dtls_srtp_verification_draft.vala src/plugin.vala src/register_plugin.vala src/trust_level.vala @@ -39,7 +40,8 @@ SOURCES src/jingle/jet_omemo.vala src/logic/database.vala - src/logic/encrypt_state.vala + src/logic/decrypt.vala + src/logic/encrypt.vala src/logic/manager.vala src/logic/pre_key_store.vala src/logic/session_store.vala @@ -53,6 +55,7 @@ SOURCES src/ui/account_settings_entry.vala src/ui/account_settings_widget.vala src/ui/bad_messages_populator.vala + src/ui/call_encryption_entry.vala src/ui/contact_details_provider.vala src/ui/contact_details_dialog.vala src/ui/device_notification_populator.vala @@ -66,18 +69,17 @@ CUSTOM_VAPIS ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi ${CMAKE_BINARY_DIR}/exports/qlite.vapi ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/libqrencode.vapi PACKAGES ${OMEMO_PACKAGES} GRESOURCES ${OMEMO_GRESOURCES_XML} -OPTIONS - --vapidir=${CMAKE_CURRENT_SOURCE_DIR}/vapi ) add_definitions(${VALA_CFLAGS} -DGETTEXT_PACKAGE=\"${GETTEXT_PACKAGE}\" -DLOCALE_INSTALL_DIR=\"${LOCALE_INSTALL_DIR}\" -DG_LOG_DOMAIN="OMEMO") add_library(omemo SHARED ${OMEMO_VALA_C} ${OMEMO_GRESOURCES_TARGET}) add_dependencies(omemo ${GETTEXT_PACKAGE}-translations) -target_link_libraries(omemo libdino signal-protocol-vala crypto-vala ${OMEMO_PACKAGES}) +target_link_libraries(omemo libdino signal-protocol-vala crypto-vala ${OMEMO_PACKAGES} libqrencode) set_target_properties(omemo PROPERTIES PREFIX "") set_target_properties(omemo PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) diff --git a/plugins/omemo/src/dtls_srtp_verification_draft.vala b/plugins/omemo/src/dtls_srtp_verification_draft.vala new file mode 100644 index 00000000..5fc9b339 --- /dev/null +++ b/plugins/omemo/src/dtls_srtp_verification_draft.vala @@ -0,0 +1,195 @@ +using Signal; +using Gee; +using Xmpp; + +namespace Dino.Plugins.Omemo.DtlsSrtpVerificationDraft { + public const string NS_URI = "http://gultsch.de/xmpp/drafts/omemo/dlts-srtp-verification"; + + public class StreamModule : XmppStreamModule { + + public static Xmpp.ModuleIdentity<StreamModule> IDENTITY = new Xmpp.ModuleIdentity<StreamModule>(NS_URI, "dtls_srtp_omemo_verification_draft"); + + private VerificationSendListener send_listener = new VerificationSendListener(); + private HashMap<string, int> device_id_by_jingle_sid = new HashMap<string, int>(); + private HashMap<string, Gee.List<string>> content_names_by_jingle_sid = new HashMap<string, Gee.List<string>>(); + + private void on_preprocess_incoming_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + Gee.List<StanzaNode> content_nodes = iq.stanza.get_deep_subnodes(Xep.Jingle.NS_URI + ":jingle", Xep.Jingle.NS_URI + ":content"); + if (content_nodes.size == 0) return; + + string? jingle_sid = iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "sid"); + if (jingle_sid == null) return; + + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); + + foreach (StanzaNode content_node in content_nodes) { + string? content_name = content_node.get_attribute("name"); + if (content_name == null) continue; + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", NS_URI); + if (fingerprint_node == null) continue; + StanzaNode? encrypted_node = fingerprint_node.get_subnode("encrypted", Omemo.NS_URI); + if (encrypted_node == null) continue; + + Xep.Omemo.ParsedData? parsed_data = decryptor.parse_node(encrypted_node); + if (parsed_data == null || parsed_data.ciphertext == null) continue; + + if (device_id_by_jingle_sid.has_key(jingle_sid) && device_id_by_jingle_sid[jingle_sid] != parsed_data.sid) { + warning("Expected DTLS fingerprint to be OMEMO encrypted from %s %d, but it was from %d", iq.from.to_string(), device_id_by_jingle_sid[jingle_sid], parsed_data.sid); + } + + foreach (Bytes encr_key in parsed_data.our_potential_encrypted_keys.keys) { + parsed_data.is_prekey = parsed_data.our_potential_encrypted_keys[encr_key]; + parsed_data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(parsed_data, iq.from.bare_jid); + string cleartext = decryptor.decrypt(parsed_data.ciphertext, key, parsed_data.iv); + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI).add_self_xmlns() + .put_node(new StanzaNode.text(cleartext)); + string? hash_attr = fingerprint_node.get_attribute("hash", NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + device_id_by_jingle_sid[jingle_sid] = parsed_data.sid; + if (!content_names_by_jingle_sid.has_key(content_name)) { + content_names_by_jingle_sid[content_name] = new ArrayList<string>(); + } + content_names_by_jingle_sid[content_name].add(content_name); + + stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.begin(jingle_sid, (_, res) => { + Xep.Jingle.Session? session = stream.get_flag(Xep.Jingle.Flag.IDENTITY).get_session.end(res); + if (session == null || !session.contents_map.has_key(content_name)) return; + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], sid=device_id_by_jingle_sid[jingle_sid], jid=iq.from.bare_jid }; + session.contents_map[content_name].encryptions[NS_URI] = encryption; + + if (iq.stanza.get_deep_attribute(Xep.Jingle.NS_URI + ":jingle", "action") == "session-accept") { + session.additional_content_add_incoming.connect(on_content_add_received); + } + }); + + break; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", iq.from.bare_jid.to_string(), parsed_data.sid, e.message); + } + } + } + } + + private void on_preprocess_outgoing_iq_set_get(XmppStream stream, Xmpp.Iq.Stanza iq) { + if (iq.type_ != Iq.Stanza.TYPE_SET) return; + + StanzaNode? jingle_node = iq.stanza.get_subnode("jingle", Xep.Jingle.NS_URI); + if (jingle_node == null) return; + + string? sid = jingle_node.get_attribute("sid", Xep.Jingle.NS_URI); + if (sid == null || !device_id_by_jingle_sid.has_key(sid)) return; + + Gee.List<StanzaNode> content_nodes = jingle_node.get_subnodes("content", Xep.Jingle.NS_URI); + if (content_nodes.size == 0) return; + + foreach (StanzaNode content_node in content_nodes) { + StanzaNode? transport_node = content_node.get_subnode("transport", Xep.JingleIceUdp.NS_URI); + if (transport_node == null) continue; + StanzaNode? fingerprint_node = transport_node.get_subnode("fingerprint", Xep.JingleIceUdp.DTLS_NS_URI); + if (fingerprint_node == null) continue; + string fingerprint = fingerprint_node.get_deep_string_content(); + + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + Xep.Omemo.EncryptionData enc_data = encryptor.encrypt_plaintext(fingerprint); + encryptor.encrypt_key(enc_data, iq.to.bare_jid, device_id_by_jingle_sid[sid]); + + StanzaNode new_fingerprint_node = new StanzaNode.build("fingerprint", NS_URI).add_self_xmlns().put_node(enc_data.get_encrypted_node()); + string? hash_attr = fingerprint_node.get_attribute("hash", Xep.JingleIceUdp.DTLS_NS_URI); + string? setup_attr = fingerprint_node.get_attribute("setup", Xep.JingleIceUdp.DTLS_NS_URI); + if (hash_attr != null) new_fingerprint_node.put_attribute("hash", hash_attr); + if (setup_attr != null) new_fingerprint_node.put_attribute("setup", setup_attr); + transport_node.put_node(new_fingerprint_node); + + transport_node.sub_nodes.remove(fingerprint_node); + } + } + + private void on_message_received(XmppStream stream, Xmpp.MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return; + + string? jingle_sid = proceed_node.get_attribute("id"); + if (jingle_sid == null) return; + + StanzaNode? device_node = proceed_node.get_subnode("device", NS_URI); + if (device_node == null) return; + + int device_id = device_node.get_attribute_int("id", -1); + if (device_id == -1) return; + + device_id_by_jingle_sid[jingle_sid] = device_id; + } + + private void on_session_initiate_received(XmppStream stream, Xep.Jingle.Session session) { + if (device_id_by_jingle_sid.has_key(session.sid)) { + foreach (Xep.Jingle.Content content in session.contents) { + on_content_add_received(stream, content); + } + } + session.additional_content_add_incoming.connect(on_content_add_received); + } + + private void on_content_add_received(XmppStream stream, Xep.Jingle.Content content) { + if (!content_names_by_jingle_sid.has_key(content.session.sid) || content_names_by_jingle_sid[content.session.sid].contains(content.content_name)) { + var encryption = new OmemoContentEncryption() { encryption_ns=NS_URI, encryption_name="OMEMO", our_key=new uint8[0], peer_key=new uint8[0], sid=device_id_by_jingle_sid[content.session.sid], jid=content.peer_full_jid.bare_jid }; + content.encryptions[encryption.encryption_ns] = encryption; + } + } + + public override void attach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.connect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.connect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.connect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.connect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.connect(on_session_initiate_received); + } + + public override void detach(XmppStream stream) { + stream.get_module(Xmpp.MessageModule.IDENTITY).received_message.disconnect(on_message_received); + stream.get_module(Xmpp.MessageModule.IDENTITY).send_pipeline.disconnect(send_listener); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_incoming_iq_set_get.disconnect(on_preprocess_incoming_iq_set_get); + stream.get_module(Xmpp.Iq.Module.IDENTITY).preprocess_outgoing_iq_set_get.disconnect(on_preprocess_outgoing_iq_set_get); + stream.get_module(Xep.Jingle.Module.IDENTITY).session_initiate_received.disconnect(on_session_initiate_received); + } + + public override string get_ns() { return NS_URI; } + + public override string get_id() { return IDENTITY.id; } + } + + public class VerificationSendListener : StanzaListener<MessageStanza> { + + private const string[] after_actions_const = {}; + + public override string action_group { get { return "REWRITE_NODES"; } } + public override string[] after_actions { get { return after_actions_const; } } + + public override async bool run(XmppStream stream, MessageStanza message) { + StanzaNode? proceed_node = message.stanza.get_subnode("proceed", Xep.JingleMessageInitiation.NS_URI); + if (proceed_node == null) return false; + + StanzaNode device_node = new StanzaNode.build("device", NS_URI).add_self_xmlns() + .put_attribute("id", stream.get_module(Omemo.StreamModule.IDENTITY).store.local_registration_id.to_string()); + proceed_node.put_node(device_node); + return false; + } + } + + public class OmemoContentEncryption : Xep.Jingle.ContentEncryption { + public Jid jid { get; set; } + public int sid { get; set; } + } +} + diff --git a/plugins/omemo/src/jingle/jet_omemo.vala b/plugins/omemo/src/jingle/jet_omemo.vala index 14307be2..afcdfcd6 100644 --- a/plugins/omemo/src/jingle/jet_omemo.vala +++ b/plugins/omemo/src/jingle/jet_omemo.vala @@ -7,18 +7,15 @@ using Xmpp; using Xmpp.Xep; namespace Dino.Plugins.JetOmemo { + private const string NS_URI = "urn:xmpp:jingle:jet-omemo:0"; private const string AES_128_GCM_URI = "urn:xmpp:ciphers:aes-128-gcm-nopadding"; + public class Module : XmppStreamModule, Jet.EnvelopEncoding { public static Xmpp.ModuleIdentity<Module> IDENTITY = new Xmpp.ModuleIdentity<Module>(NS_URI, "0396_jet_omemo"); - private Omemo.Plugin plugin; const uint KEY_SIZE = 16; const uint IV_SIZE = 12; - public Module(Omemo.Plugin plugin) { - this.plugin = plugin; - } - public override void attach(XmppStream stream) { if (stream.get_module(Jet.Module.IDENTITY) != null) { stream.get_module(ServiceDiscovery.Module.IDENTITY).add_feature(stream, NS_URI); @@ -44,71 +41,38 @@ public class Module : XmppStreamModule, Jet.EnvelopEncoding { } public Jet.TransportSecret decode_envolop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, StanzaNode security) throws Jingle.IqError { - Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store; StanzaNode? encrypted = security.get_subnode("encrypted", Omemo.NS_URI); if (encrypted == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing encrypted element"); - StanzaNode? header = encrypted.get_subnode("header", Omemo.NS_URI); - if (header == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing header element"); - string? iv_node = header.get_deep_string_content("iv"); - if (header == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: missing iv element"); - uint8[] iv = Base64.decode((!)iv_node); - foreach (StanzaNode key_node in header.get_subnodes("key")) { - if (key_node.get_attribute_int("rid") == store.local_registration_id) { - string? key_node_content = key_node.get_string_content(); - - uint8[] key; - Address address = new Address(peer_full_jid.bare_jid.to_string(), header.get_attribute_int("sid")); - if (key_node.get_attribute_bool("prekey")) { - PreKeySignalMessage msg = Omemo.Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_pre_key_signal_message(msg); - } else { - SignalMessage msg = Omemo.Plugin.get_context().deserialize_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_signal_message(msg); - } - address.device_id = 0; // TODO: Hack to have address obj live longer - - uint8[] authtag = null; - if (key.length >= 32) { - int authtaglength = key.length - 16; - authtag = new uint8[authtaglength]; - uint8[] new_key = new uint8[16]; - Memory.copy(authtag, (uint8*)key + 16, 16); - Memory.copy(new_key, key, 16); - key = new_key; - } - // TODO: authtag? - return new Jet.TransportSecret(key, iv); + + Xep.Omemo.OmemoDecryptor decryptor = stream.get_module(Xep.Omemo.OmemoDecryptor.IDENTITY); + + Xmpp.Xep.Omemo.ParsedData? data = decryptor.parse_node(encrypted); + if (data == null) throw new Jingle.IqError.BAD_REQUEST("Invalid JET-OMEMO envelop: bad encrypted element"); + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + + try { + uint8[] key = decryptor.decrypt_key(data, peer_full_jid.bare_jid); + return new Jet.TransportSecret(key, data.iv); + } catch (GLib.Error e) { + debug("Decrypting JET key from %s/%d failed: %s", peer_full_jid.bare_jid.to_string(), data.sid, e.message); } } throw new Jingle.IqError.NOT_ACCEPTABLE("Not encrypted for targeted device"); } public void encode_envelop(XmppStream stream, Jid local_full_jid, Jid peer_full_jid, Jet.SecurityParameters security_params, StanzaNode security) { - ArrayList<Account> accounts = plugin.app.stream_interactor.get_accounts(); Store store = stream.get_module(Omemo.StreamModule.IDENTITY).store; - Account? account = null; - foreach (Account compare in accounts) { - if (compare.bare_jid.equals_bare(local_full_jid)) { - account = compare; - break; - } - } - if (account == null) { - // TODO - critical("Sending from offline account %s", local_full_jid.to_string()); - } - StanzaNode header_node; - StanzaNode encrypted_node = new StanzaNode.build("encrypted", Omemo.NS_URI).add_self_xmlns() - .put_node(header_node = new StanzaNode.build("header", Omemo.NS_URI) - .put_attribute("sid", store.local_registration_id.to_string()) - .put_node(new StanzaNode.build("iv", Omemo.NS_URI) - .put_node(new StanzaNode.text(Base64.encode(security_params.secret.initialization_vector))))); + var encryption_data = new Xep.Omemo.EncryptionData(store.local_registration_id); + encryption_data.iv = security_params.secret.initialization_vector; + encryption_data.keytag = security_params.secret.transport_key; + Xep.Omemo.OmemoEncryptor encryptor = stream.get_module(Xep.Omemo.OmemoEncryptor.IDENTITY); + encryptor.encrypt_key_to_recipient(stream, encryption_data, peer_full_jid.bare_jid); - plugin.trust_manager.encrypt_key(header_node, security_params.secret.transport_key, local_full_jid.bare_jid, new ArrayList<Jid>.wrap(new Jid[] {peer_full_jid.bare_jid}), stream, account); - security.put_node(encrypted_node); + security.put_node(encryption_data.get_encrypted_node()); } public override string get_ns() { return NS_URI; } diff --git a/plugins/omemo/src/logic/decrypt.vala b/plugins/omemo/src/logic/decrypt.vala new file mode 100644 index 00000000..cfbb9c58 --- /dev/null +++ b/plugins/omemo/src/logic/decrypt.vala @@ -0,0 +1,211 @@ +using Dino.Entities; +using Qlite; +using Gee; +using Signal; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class OmemoDecryptor : Xep.Omemo.OmemoDecryptor { + + private Account account; + private Store store; + private Database db; + private StreamInteractor stream_interactor; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoDecryptor(Account account, StreamInteractor stream_interactor, TrustManager trust_manager, Database db, Store store) { + this.account = account; + this.stream_interactor = stream_interactor; + this.trust_manager = trust_manager; + this.db = db; + this.store = store; + } + + public bool decrypt_message(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + StanzaNode? encrypted_node = stanza.stanza.get_subnode("encrypted", NS_URI); + if (encrypted_node == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false; + + if (message.body == null && Xep.ExplicitEncryption.get_encryption_tag(stanza) == NS_URI) { + message.body = "[This message is OMEMO encrypted]"; // TODO temporary + } + if (!Plugin.ensure_context()) return false; + int identity_id = db.identity.get_id(conversation.account.id); + + MessageFlag flag = new MessageFlag(); + stanza.add_flag(flag); + + Xep.Omemo.ParsedData? data = parse_node(encrypted_node); + if (data == null || data.ciphertext == null) return false; + + + foreach (Bytes encr_key in data.our_potential_encrypted_keys.keys) { + data.is_prekey = data.our_potential_encrypted_keys[encr_key]; + data.encrypted_key = encr_key.get_data(); + Gee.List<Jid> possible_jids = get_potential_message_jids(message, data, identity_id); + if (possible_jids.size == 0) { + debug("Received message from unknown entity with device id %d", data.sid); + } + + foreach (Jid possible_jid in possible_jids) { + try { + uint8[] key = decrypt_key(data, possible_jid); + string cleartext = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, data.iv, data.ciphertext)); + + // If we figured out which real jid a message comes from due to decryption working, save it + if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) { + message.real_jid = possible_jid; + } + + message.body = cleartext; + message.encryption = Encryption.OMEMO; + + trust_manager.message_device_id_map[message] = data.sid; + return true; + } catch (Error e) { + debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), data.sid, e.message); + } + } + } + + if ( + encrypted_node.get_deep_string_content("payload") != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok + data.our_potential_encrypted_keys.size == 0 && // The message was not encrypted to us + stream_interactor.module_manager.get_module(message.account, StreamModule.IDENTITY).store.local_registration_id != data.sid // Message from this device. Never encrypted to itself. + ) { + db.identity_meta.update_last_message_undecryptable(identity_id, data.sid, message.time); + trust_manager.bad_message_state_updated(conversation.account, message.from, data.sid); + } + + debug("Received OMEMO encryped message that could not be decrypted."); + return false; + } + + public Gee.List<Jid> get_potential_message_jids(Entities.Message message, Xmpp.Xep.Omemo.ParsedData data, int identity_id) { + Gee.List<Jid> possible_jids = new ArrayList<Jid>(); + if (message.type_ == Message.Type.CHAT) { + possible_jids.add(message.from.bare_jid); + } else { + if (message.real_jid != null) { + possible_jids.add(message.real_jid.bare_jid); + } else if (data.is_prekey) { + // pre key messages do store the identity key, so we can use that to find the real jid + PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(data.encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) { + try { + possible_jids.add(new Jid(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } else { + // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id + foreach (Row row in db.identity_meta.get_with_device_id(identity_id, data.sid)) { + try { + possible_jids.add(new Jid(row[db.identity_meta.address_name])); + } catch (InvalidJidError e) { + warning("Ignoring invalid jid from database: %s", e.message); + } + } + } + } + return possible_jids; + } + + public override uint8[] decrypt_key(Xmpp.Xep.Omemo.ParsedData data, Jid from_jid) throws GLib.Error { + int sid = data.sid; + uint8[] ciphertext = data.ciphertext; + uint8[] encrypted_key = data.encrypted_key; + + Address address = new Address(from_jid.to_string(), sid); + uint8[] key; + + if (data.is_prekey) { + int identity_id = db.identity.get_id(account.id); + PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(encrypted_key); + string identity_key = Base64.encode(msg.identity_key.serialize()); + + bool ok = update_db_for_prekey(identity_id, identity_key, from_jid, sid); + if (!ok) return null; + + debug("Starting new session for decryption with device from %s/%d", from_jid.to_string(), sid); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_pre_key_signal_message(msg); + // TODO: Finish session + } else { + debug("Continuing session for decryption with device from %s/%d", from_jid.to_string(), sid); + SignalMessage msg = Plugin.get_context().deserialize_signal_message(encrypted_key); + SessionCipher cipher = store.create_session_cipher(address); + key = cipher.decrypt_signal_message(msg); + } + + if (key.length >= 32) { + int authtaglength = key.length - 16; + uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength]; + uint8[] new_key = new uint8[16]; + Memory.copy(new_ciphertext, ciphertext, ciphertext.length); + Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength); + Memory.copy(new_key, key, 16); + data.ciphertext = new_ciphertext; + key = new_key; + } + + return key; + } + + public override string decrypt(uint8[] ciphertext, uint8[] key, uint8[] iv) throws GLib.Error { + return arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext)); + } + + private bool update_db_for_prekey(int identity_id, string identity_key, Jid from_jid, int sid) { + Row? device = db.identity_meta.get_device(identity_id, from_jid.to_string(), sid); + if (device != null && device[db.identity_meta.identity_key_public_base64] != null) { + if (device[db.identity_meta.identity_key_public_base64] != identity_key) { + critical("Tried to use a different identity key for a known device id."); + return false; + } + } else { + debug("Learn new device from incoming message from %s/%d", from_jid.to_string(), sid); + bool blind_trust = db.trust.get_blind_trust(identity_id, from_jid.to_string(), true); + if (db.identity_meta.insert_device_session(identity_id, from_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) { + critical("Failed learning a device."); + return false; + } + + XmppStream? stream = stream_interactor.get_stream(account); + if (device == null && stream != null) { + stream.get_module(StreamModule.IDENTITY).request_user_devicelist.begin(stream, from_jid); + } + } + return true; + } + + private string arr_to_str(uint8[] arr) { + // null-terminate the array + uint8[] rarr = new uint8[arr.length+1]; + Memory.copy(rarr, arr, arr.length); + return (string)rarr; + } + } + + public class DecryptMessageListener : MessageListener { + public string[] after_actions_const = new string[]{ }; + public override string action_group { get { return "DECRYPT"; } } + public override string[] after_actions { get { return after_actions_const; } } + + private HashMap<Account, OmemoDecryptor> decryptors; + + public DecryptMessageListener(HashMap<Account, OmemoDecryptor> decryptors) { + this.decryptors = decryptors; + } + + public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { + decryptors[message.account].decrypt_message(message, stanza, conversation); + return false; + } + } +} + diff --git a/plugins/omemo/src/logic/encrypt.vala b/plugins/omemo/src/logic/encrypt.vala new file mode 100644 index 00000000..cd994c3a --- /dev/null +++ b/plugins/omemo/src/logic/encrypt.vala @@ -0,0 +1,131 @@ +using Gee; +using Signal; +using Dino.Entities; +using Xmpp; +using Xmpp.Xep.Omemo; + +namespace Dino.Plugins.Omemo { + + public class OmemoEncryptor : Xep.Omemo.OmemoEncryptor { + + private Account account; + private Store store; + private TrustManager trust_manager; + + public override uint32 own_device_id { get { return store.local_registration_id; }} + + public OmemoEncryptor(Account account, TrustManager trust_manager, Store store) { + this.account = account; + this.trust_manager = trust_manager; + this.store = store; + } + + public override Xep.Omemo.EncryptionData encrypt_plaintext(string plaintext) throws GLib.Error { + const uint KEY_SIZE = 16; + const uint IV_SIZE = 12; + + //Create a key and use it to encrypt the message + uint8[] key = new uint8[KEY_SIZE]; + Plugin.get_context().randomize(key); + uint8[] iv = new uint8[IV_SIZE]; + Plugin.get_context().randomize(iv); + + uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, plaintext.data); + uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length - 16]; + uint8[] tag = aes_encrypt_result[aes_encrypt_result.length - 16:aes_encrypt_result.length]; + uint8[] keytag = new uint8[key.length + tag.length]; + Memory.copy(keytag, key, key.length); + Memory.copy((uint8*)keytag + key.length, tag, tag.length); + + var ret = new Xep.Omemo.EncryptionData(own_device_id); + ret.ciphertext = ciphertext; + ret.keytag = keytag; + ret.iv = iv; + return ret; + } + + public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) { + + EncryptState status = new EncryptState(); + if (!Plugin.ensure_context()) return status; + if (message.to == null) return status; + + try { + EncryptionData enc_data = encrypt_plaintext(message.body); + status = encrypt_key_to_recipients(enc_data, self_jid, recipients, stream); + + message.stanza.put_node(enc_data.get_encrypted_node()); + Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO"); + message.body = "[This message is OMEMO encrypted]"; + status.encrypted = true; + } catch (Error e) { + warning(@"Signal error while encrypting message: $(e.message)\n"); + message.body = "[OMEMO encryption failed]"; + status.encrypted = false; + } + return status; + } + + internal EncryptState encrypt_key_to_recipients(EncryptionData enc_data, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream) throws Error { + EncryptState status = new EncryptState(); + + //Check we have the bundles and device lists needed to send the message + if (!trust_manager.is_known_address(account, self_jid)) return status; + status.own_list = true; + status.own_devices = trust_manager.get_trusted_devices(account, self_jid).size; + status.other_waiting_lists = 0; + status.other_devices = 0; + foreach (Jid recipient in recipients) { + if (!trust_manager.is_known_address(account, recipient)) { + status.other_waiting_lists++; + } + if (status.other_waiting_lists > 0) return status; + status.other_devices += trust_manager.get_trusted_devices(account, recipient).size; + } + if (status.own_devices == 0 || status.other_devices == 0) return status; + + + //Encrypt the key for each recipient's device individually + foreach (Jid recipient in recipients) { + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, recipient); + status.add_result(enc_res, false); + } + + // Encrypt the key for each own device + EncryptionResult enc_res = encrypt_key_to_recipient(stream, enc_data, self_jid); + status.add_result(enc_res, true); + + return status; + } + + public override EncryptionResult encrypt_key_to_recipient(XmppStream stream, Xep.Omemo.EncryptionData enc_data, Jid recipient) throws GLib.Error { + var result = new EncryptionResult(); + StreamModule module = stream.get_module(StreamModule.IDENTITY); + + foreach(int32 device_id in trust_manager.get_trusted_devices(account, recipient)) { + if (module.is_ignored_device(recipient, device_id)) { + result.lost++; + continue; + } + try { + encrypt_key(enc_data, recipient, device_id); + result.success++; + } catch (Error e) { + if (e.code == ErrorCode.UNKNOWN) result.unknown++; + else result.failure++; + } + } + return result; + } + + public override void encrypt_key(Xep.Omemo.EncryptionData encryption_data, Jid jid, int32 device_id) throws GLib.Error { + Address address = new Address(jid.to_string(), device_id); + SessionCipher cipher = store.create_session_cipher(address); + CiphertextMessage device_key = cipher.encrypt(encryption_data.keytag); + address.device_id = 0; + debug("Created encrypted key for %s/%d", jid.to_string(), device_id); + + encryption_data.add_device_key(device_id, device_key.serialized, device_key.type == CiphertextType.PREKEY); + } + } +}
\ No newline at end of file diff --git a/plugins/omemo/src/logic/encrypt_state.vala b/plugins/omemo/src/logic/encrypt_state.vala deleted file mode 100644 index fd72faf4..00000000 --- a/plugins/omemo/src/logic/encrypt_state.vala +++ /dev/null @@ -1,24 +0,0 @@ -namespace Dino.Plugins.Omemo { - -public class EncryptState { - public bool encrypted { get; internal set; } - public int other_devices { get; internal set; } - public int other_success { get; internal set; } - public int other_lost { get; internal set; } - public int other_unknown { get; internal set; } - public int other_failure { get; internal set; } - public int other_waiting_lists { get; internal set; } - - public int own_devices { get; internal set; } - public int own_success { get; internal set; } - public int own_lost { get; internal set; } - public int own_unknown { get; internal set; } - public int own_failure { get; internal set; } - public bool own_list { get; internal set; } - - public string to_string() { - return @"EncryptState (encrypted=$encrypted, other=(devices=$other_devices, success=$other_success, lost=$other_lost, unknown=$other_unknown, failure=$other_failure, waiting_lists=$other_waiting_lists, own=(devices=$own_devices, success=$own_success, lost=$own_lost, unknown=$own_unknown, failure=$own_failure, list=$own_list))"; - } -} - -} diff --git a/plugins/omemo/src/logic/manager.vala b/plugins/omemo/src/logic/manager.vala index 64b117c7..5552e212 100644 --- a/plugins/omemo/src/logic/manager.vala +++ b/plugins/omemo/src/logic/manager.vala @@ -13,11 +13,12 @@ public class Manager : StreamInteractionModule, Object { private StreamInteractor stream_interactor; private Database db; private TrustManager trust_manager; + private HashMap<Account, OmemoEncryptor> encryptors; private Map<Entities.Message, MessageState> message_states = new HashMap<Entities.Message, MessageState>(Entities.Message.hash_func, Entities.Message.equals_func); private class MessageState { public Entities.Message msg { get; private set; } - public EncryptState last_try { get; private set; } + public Xep.Omemo.EncryptState last_try { get; private set; } public int waiting_other_sessions { get; set; } public int waiting_own_sessions { get; set; } public bool waiting_own_devicelist { get; set; } @@ -26,11 +27,11 @@ public class Manager : StreamInteractionModule, Object { public bool will_send_now { get; private set; } public bool active_send_attempt { get; set; } - public MessageState(Entities.Message msg, EncryptState last_try) { + public MessageState(Entities.Message msg, Xep.Omemo.EncryptState last_try) { update_from_encrypt_status(msg, last_try); } - public void update_from_encrypt_status(Entities.Message msg, EncryptState new_try) { + public void update_from_encrypt_status(Entities.Message msg, Xep.Omemo.EncryptState new_try) { this.msg = msg; this.last_try = new_try; this.waiting_other_sessions = new_try.other_unknown; @@ -59,10 +60,11 @@ public class Manager : StreamInteractionModule, Object { } } - private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager) { + private Manager(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) { this.stream_interactor = stream_interactor; this.db = db; this.trust_manager = trust_manager; + this.encryptors = encryptors; stream_interactor.stream_negotiated.connect(on_stream_negotiated); stream_interactor.get_module(MessageProcessor.IDENTITY).pre_message_send.connect(on_pre_message_send); @@ -125,7 +127,7 @@ public class Manager : StreamInteractionModule, Object { } //Attempt to encrypt the message - EncryptState enc_state = trust_manager.encrypt(message_stanza, conversation.account.bare_jid, recipients, stream, conversation.account); + Xep.Omemo.EncryptState enc_state = encryptors[conversation.account].encrypt(message_stanza, conversation.account.bare_jid, recipients, stream); MessageState state; lock (message_states) { if (message_states.has_key(message)) { @@ -411,8 +413,8 @@ public class Manager : StreamInteractionModule, Object { return true; // TODO wait for stream? } - public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager) { - Manager m = new Manager(stream_interactor, db, trust_manager); + public static void start(StreamInteractor stream_interactor, Database db, TrustManager trust_manager, HashMap<Account, OmemoEncryptor> encryptors) { + Manager m = new Manager(stream_interactor, db, trust_manager, encryptors); stream_interactor.add_module(m); } } diff --git a/plugins/omemo/src/logic/trust_manager.vala b/plugins/omemo/src/logic/trust_manager.vala index 1e61b201..20076a43 100644 --- a/plugins/omemo/src/logic/trust_manager.vala +++ b/plugins/omemo/src/logic/trust_manager.vala @@ -12,18 +12,15 @@ public class TrustManager { private StreamInteractor stream_interactor; private Database db; - private DecryptMessageListener decrypt_message_listener; private TagMessageListener tag_message_listener; - private HashMap<Message, int> message_device_id_map = new HashMap<Message, int>(Message.hash_func, Message.equals_func); + public HashMap<Message, int> message_device_id_map = new HashMap<Message, int>(Message.hash_func, Message.equals_func); public TrustManager(StreamInteractor stream_interactor, Database db) { this.stream_interactor = stream_interactor; this.db = db; - decrypt_message_listener = new DecryptMessageListener(stream_interactor, this, db, message_device_id_map); tag_message_listener = new TagMessageListener(stream_interactor, this, db, message_device_id_map); - stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(decrypt_message_listener); stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(tag_message_listener); } @@ -69,127 +66,6 @@ public class TrustManager { } } - private StanzaNode create_encrypted_key_node(uint8[] key, Address address, Store store) throws GLib.Error { - SessionCipher cipher = store.create_session_cipher(address); - CiphertextMessage device_key = cipher.encrypt(key); - debug("Created encrypted key for %s/%d", address.name, address.device_id); - StanzaNode key_node = new StanzaNode.build("key", NS_URI) - .put_attribute("rid", address.device_id.to_string()) - .put_node(new StanzaNode.text(Base64.encode(device_key.serialized))); - if (device_key.type == CiphertextType.PREKEY) key_node.put_attribute("prekey", "true"); - return key_node; - } - - internal EncryptState encrypt_key(StanzaNode header_node, uint8[] keytag, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream, Account account) throws Error { - EncryptState status = new EncryptState(); - StreamModule module = stream.get_module(StreamModule.IDENTITY); - - //Check we have the bundles and device lists needed to send the message - if (!is_known_address(account, self_jid)) return status; - status.own_list = true; - status.own_devices = get_trusted_devices(account, self_jid).size; - status.other_waiting_lists = 0; - status.other_devices = 0; - foreach (Jid recipient in recipients) { - if (!is_known_address(account, recipient)) { - status.other_waiting_lists++; - } - if (status.other_waiting_lists > 0) return status; - status.other_devices += get_trusted_devices(account, recipient).size; - } - if (status.own_devices == 0 || status.other_devices == 0) return status; - - - //Encrypt the key for each recipient's device individually - Address address = new Address("", 0); - foreach (Jid recipient in recipients) { - foreach(int32 device_id in get_trusted_devices(account, recipient)) { - if (module.is_ignored_device(recipient, device_id)) { - status.other_lost++; - continue; - } - try { - address.name = recipient.bare_jid.to_string(); - address.device_id = (int) device_id; - StanzaNode key_node = create_encrypted_key_node(keytag, address, module.store); - header_node.put_node(key_node); - status.other_success++; - } catch (Error e) { - if (e.code == ErrorCode.UNKNOWN) status.other_unknown++; - else status.other_failure++; - } - } - } - - // Encrypt the key for each own device - address.name = self_jid.bare_jid.to_string(); - foreach(int32 device_id in get_trusted_devices(account, self_jid)) { - if (module.is_ignored_device(self_jid, device_id)) { - status.own_lost++; - continue; - } - if (device_id != module.store.local_registration_id) { - address.device_id = (int) device_id; - try { - StanzaNode key_node = create_encrypted_key_node(keytag, address, module.store); - header_node.put_node(key_node); - status.own_success++; - } catch (Error e) { - if (e.code == ErrorCode.UNKNOWN) status.own_unknown++; - else status.own_failure++; - } - } - } - - return status; - } - - public EncryptState encrypt(MessageStanza message, Jid self_jid, Gee.List<Jid> recipients, XmppStream stream, Account account) { - const uint KEY_SIZE = 16; - const uint IV_SIZE = 12; - EncryptState status = new EncryptState(); - if (!Plugin.ensure_context()) return status; - if (message.to == null) return status; - - StreamModule module = stream.get_module(StreamModule.IDENTITY); - - try { - //Create a key and use it to encrypt the message - uint8[] key = new uint8[KEY_SIZE]; - Plugin.get_context().randomize(key); - uint8[] iv = new uint8[IV_SIZE]; - Plugin.get_context().randomize(iv); - - uint8[] aes_encrypt_result = aes_encrypt(Cipher.AES_GCM_NOPADDING, key, iv, message.body.data); - uint8[] ciphertext = aes_encrypt_result[0:aes_encrypt_result.length-16]; - uint8[] tag = aes_encrypt_result[aes_encrypt_result.length-16:aes_encrypt_result.length]; - uint8[] keytag = new uint8[key.length + tag.length]; - Memory.copy(keytag, key, key.length); - Memory.copy((uint8*)keytag + key.length, tag, tag.length); - - StanzaNode header_node; - StanzaNode encrypted_node = new StanzaNode.build("encrypted", NS_URI).add_self_xmlns() - .put_node(header_node = new StanzaNode.build("header", NS_URI) - .put_attribute("sid", module.store.local_registration_id.to_string()) - .put_node(new StanzaNode.build("iv", NS_URI) - .put_node(new StanzaNode.text(Base64.encode(iv))))) - .put_node(new StanzaNode.build("payload", NS_URI) - .put_node(new StanzaNode.text(Base64.encode(ciphertext)))); - - status = encrypt_key(header_node, keytag, self_jid, recipients, stream, account); - - message.stanza.put_node(encrypted_node); - Xep.ExplicitEncryption.add_encryption_tag_to_message(message, NS_URI, "OMEMO"); - message.body = "[This message is OMEMO encrypted]"; - status.encrypted = true; - } catch (Error e) { - warning(@"Signal error while encrypting message: $(e.message)\n"); - message.body = "[OMEMO encryption failed]"; - status.encrypted = false; - } - return status; - } - public bool is_known_address(Account account, Jid jid) { int identity_id = db.identity.get_id(account.id); if (identity_id < 0) return false; @@ -260,182 +136,6 @@ public class TrustManager { return false; } } - - private class DecryptMessageListener : MessageListener { - public string[] after_actions_const = new string[]{ }; - public override string action_group { get { return "DECRYPT"; } } - public override string[] after_actions { get { return after_actions_const; } } - - private StreamInteractor stream_interactor; - private TrustManager trust_manager; - private Database db; - private HashMap<Message, int> message_device_id_map; - - public DecryptMessageListener(StreamInteractor stream_interactor, TrustManager trust_manager, Database db, HashMap<Message, int> message_device_id_map) { - this.stream_interactor = stream_interactor; - this.trust_manager = trust_manager; - this.db = db; - this.message_device_id_map = message_device_id_map; - } - - public override async bool run(Entities.Message message, Xmpp.MessageStanza stanza, Conversation conversation) { - StreamModule module = stream_interactor.module_manager.get_module(conversation.account, StreamModule.IDENTITY); - Store store = module.store; - - StanzaNode? _encrypted = stanza.stanza.get_subnode("encrypted", NS_URI); - if (_encrypted == null || MessageFlag.get_flag(stanza) != null || stanza.from == null) return false; - StanzaNode encrypted = (!)_encrypted; - if (message.body == null && Xep.ExplicitEncryption.get_encryption_tag(stanza) == NS_URI) { - message.body = "[This message is OMEMO encrypted]"; // TODO temporary - }; - if (!Plugin.ensure_context()) return false; - int identity_id = db.identity.get_id(conversation.account.id); - MessageFlag flag = new MessageFlag(); - stanza.add_flag(flag); - StanzaNode? _header = encrypted.get_subnode("header"); - if (_header == null) return false; - StanzaNode header = (!)_header; - int sid = header.get_attribute_int("sid"); - if (sid <= 0) return false; - - var our_nodes = new ArrayList<StanzaNode>(); - foreach (StanzaNode key_node in header.get_subnodes("key")) { - debug("Is ours? %d =? %u", key_node.get_attribute_int("rid"), store.local_registration_id); - if (key_node.get_attribute_int("rid") == store.local_registration_id) { - our_nodes.add(key_node); - } - } - - string? payload = encrypted.get_deep_string_content("payload"); - string? iv_node = header.get_deep_string_content("iv"); - - foreach (StanzaNode key_node in our_nodes) { - string? key_node_content = key_node.get_string_content(); - if (payload == null || iv_node == null || key_node_content == null) continue; - uint8[] key; - uint8[] ciphertext = Base64.decode((!)payload); - uint8[] iv = Base64.decode((!)iv_node); - Gee.List<Jid> possible_jids = new ArrayList<Jid>(); - if (conversation.type_ == Conversation.Type.CHAT) { - possible_jids.add(stanza.from.bare_jid); - } else { - Jid? real_jid = message.real_jid; - if (real_jid != null) { - possible_jids.add(real_jid.bare_jid); - } else if (key_node.get_attribute_bool("prekey")) { - // pre key messages do store the identity key, so we can use that to find the real jid - PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - string identity_key = Base64.encode(msg.identity_key.serialize()); - foreach (Row row in db.identity_meta.get_with_device_id(identity_id, sid).with(db.identity_meta.identity_key_public_base64, "=", identity_key)) { - try { - possible_jids.add(new Jid(row[db.identity_meta.address_name])); - } catch (InvalidJidError e) { - warning("Ignoring invalid jid from database: %s", e.message); - } - } - if (possible_jids.size != 1) { - continue; - } - } else { - // If we don't know the device name (MUC history w/o MAM), test decryption with all keys with fitting device id - foreach (Row row in db.identity_meta.get_with_device_id(identity_id, sid)) { - try { - possible_jids.add(new Jid(row[db.identity_meta.address_name])); - } catch (InvalidJidError e) { - warning("Ignoring invalid jid from database: %s", e.message); - } - } - } - } - - if (possible_jids.size == 0) { - debug("Received message from unknown entity with device id %d", sid); - } - - foreach (Jid possible_jid in possible_jids) { - try { - Address address = new Address(possible_jid.to_string(), sid); - if (key_node.get_attribute_bool("prekey")) { - Row? device = db.identity_meta.get_device(identity_id, possible_jid.to_string(), sid); - PreKeySignalMessage msg = Plugin.get_context().deserialize_pre_key_signal_message(Base64.decode((!)key_node_content)); - string identity_key = Base64.encode(msg.identity_key.serialize()); - if (device != null && device[db.identity_meta.identity_key_public_base64] != null) { - if (device[db.identity_meta.identity_key_public_base64] != identity_key) { - critical("Tried to use a different identity key for a known device id."); - continue; - } - } else { - debug("Learn new device from incoming message from %s/%d", possible_jid.to_string(), sid); - bool blind_trust = db.trust.get_blind_trust(identity_id, possible_jid.to_string(), true); - if (db.identity_meta.insert_device_session(identity_id, possible_jid.to_string(), sid, identity_key, blind_trust ? TrustLevel.TRUSTED : TrustLevel.UNKNOWN) < 0) { - critical("Failed learning a device."); - continue; - } - XmppStream? stream = stream_interactor.get_stream(conversation.account); - if (device == null && stream != null) { - module.request_user_devicelist.begin(stream, possible_jid); - } - } - debug("Starting new session for decryption with device from %s/%d", possible_jid.to_string(), sid); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_pre_key_signal_message(msg); - // TODO: Finish session - } else { - debug("Continuing session for decryption with device from %s/%d", possible_jid.to_string(), sid); - SignalMessage msg = Plugin.get_context().deserialize_signal_message(Base64.decode((!)key_node_content)); - SessionCipher cipher = store.create_session_cipher(address); - key = cipher.decrypt_signal_message(msg); - } - //address.device_id = 0; // TODO: Hack to have address obj live longer - - if (key.length >= 32) { - int authtaglength = key.length - 16; - uint8[] new_ciphertext = new uint8[ciphertext.length + authtaglength]; - uint8[] new_key = new uint8[16]; - Memory.copy(new_ciphertext, ciphertext, ciphertext.length); - Memory.copy((uint8*)new_ciphertext + ciphertext.length, (uint8*)key + 16, authtaglength); - Memory.copy(new_key, key, 16); - ciphertext = new_ciphertext; - key = new_key; - } - - message.body = arr_to_str(aes_decrypt(Cipher.AES_GCM_NOPADDING, key, iv, ciphertext)); - message_device_id_map[message] = address.device_id; - message.encryption = Encryption.OMEMO; - flag.decrypted = true; - } catch (Error e) { - debug("Decrypting message from %s/%d failed: %s", possible_jid.to_string(), sid, e.message); - continue; - } - - // If we figured out which real jid a message comes from due to decryption working, save it - if (conversation.type_ == Conversation.Type.GROUPCHAT && message.real_jid == null) { - message.real_jid = possible_jid; - } - return false; - } - } - - if ( - payload != null && // Ratchet forwarding doesn't contain payload and might not include us, which is ok - our_nodes.size == 0 && // The message was not encrypted to us - module.store.local_registration_id != sid // Message from this device. Never encrypted to itself. - ) { - db.identity_meta.update_last_message_undecryptable(identity_id, sid, message.time); - trust_manager.bad_message_state_updated(conversation.account, message.from, sid); - } - - debug("Received OMEMO encryped message that could not be decrypted."); - return false; - } - - private string arr_to_str(uint8[] arr) { - // null-terminate the array - uint8[] rarr = new uint8[arr.length+1]; - Memory.copy(rarr, arr, arr.length); - return (string)rarr; - } - } } } diff --git a/plugins/omemo/src/plugin.vala b/plugins/omemo/src/plugin.vala index e739fc4d..643428a8 100644 --- a/plugins/omemo/src/plugin.vala +++ b/plugins/omemo/src/plugin.vala @@ -1,3 +1,4 @@ +using Gee; using Dino.Entities; extern const string GETTEXT_PACKAGE; @@ -20,6 +21,7 @@ public class Plugin : RootInterface, Object { } return true; } catch (Error e) { + warning("Error initializing Signal Context %s", e.message); return false; } } @@ -33,6 +35,8 @@ public class Plugin : RootInterface, Object { public DeviceNotificationPopulator device_notification_populator; public OwnNotifications own_notifications; public TrustManager trust_manager; + public HashMap<Account, OmemoDecryptor> decryptors = new HashMap<Account, OmemoDecryptor>(Account.hash_func, Account.equals_func); + public HashMap<Account, OmemoEncryptor> encryptors = new HashMap<Account, OmemoEncryptor>(Account.hash_func, Account.equals_func); public void registered(Dino.Application app) { ensure_context(); @@ -43,22 +47,32 @@ public class Plugin : RootInterface, Object { this.contact_details_provider = new ContactDetailsProvider(this); this.device_notification_populator = new DeviceNotificationPopulator(this, this.app.stream_interactor); this.trust_manager = new TrustManager(this.app.stream_interactor, this.db); + this.app.plugin_registry.register_encryption_list_entry(list_entry); this.app.plugin_registry.register_account_settings_entry(settings_entry); this.app.plugin_registry.register_contact_details_entry(contact_details_provider); this.app.plugin_registry.register_notification_populator(device_notification_populator); this.app.plugin_registry.register_conversation_addition_populator(new BadMessagesPopulator(this.app.stream_interactor, this)); + this.app.plugin_registry.register_call_entryption_entry(DtlsSrtpVerificationDraft.NS_URI, new CallEncryptionEntry(db)); + this.app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { - list.add(new StreamModule()); - list.add(new JetOmemo.Module(this)); + Signal.Store signal_store = Plugin.get_context().create_store(); + list.add(new StreamModule(signal_store)); + decryptors[account] = new OmemoDecryptor(account, app.stream_interactor, trust_manager, db, signal_store); + list.add(decryptors[account]); + encryptors[account] = new OmemoEncryptor(account, trust_manager,signal_store); + list.add(encryptors[account]); + list.add(new JetOmemo.Module()); + list.add(new DtlsSrtpVerificationDraft.StreamModule()); this.own_notifications = new OwnNotifications(this, this.app.stream_interactor, account); }); + app.stream_interactor.get_module(MessageProcessor.IDENTITY).received_pipeline.connect(new DecryptMessageListener(decryptors)); app.stream_interactor.get_module(FileManager.IDENTITY).add_file_decryptor(new OmemoFileDecryptor()); app.stream_interactor.get_module(FileManager.IDENTITY).add_file_encryptor(new OmemoFileEncryptor()); JingleFileHelperRegistry.instance.add_encryption_helper(Encryption.OMEMO, new JetOmemo.EncryptionHelper(app.stream_interactor)); - Manager.start(this.app.stream_interactor, db, trust_manager); + Manager.start(this.app.stream_interactor, db, trust_manager, encryptors); SimpleAction own_keys_action = new SimpleAction("own-keys", VariantType.INT32); own_keys_action.activate.connect((variant) => { diff --git a/plugins/omemo/src/protocol/stream_module.vala b/plugins/omemo/src/protocol/stream_module.vala index e4a2733c..39d9c448 100644 --- a/plugins/omemo/src/protocol/stream_module.vala +++ b/plugins/omemo/src/protocol/stream_module.vala @@ -25,10 +25,8 @@ public class StreamModule : XmppStreamModule { public signal void bundle_fetched(Jid jid, int device_id, Bundle bundle); public signal void bundle_fetch_failed(Jid jid, int device_id); - public StreamModule() { - if (Plugin.ensure_context()) { - this.store = Plugin.get_context().create_store(); - } + public StreamModule(Store store) { + this.store = store; } public override void attach(XmppStream stream) { diff --git a/plugins/omemo/src/ui/call_encryption_entry.vala b/plugins/omemo/src/ui/call_encryption_entry.vala new file mode 100644 index 00000000..69b7b686 --- /dev/null +++ b/plugins/omemo/src/ui/call_encryption_entry.vala @@ -0,0 +1,57 @@ +using Dino.Entities; +using Gtk; +using Qlite; +using Xmpp; + +namespace Dino.Plugins.Omemo { + + public class CallEncryptionEntry : Plugins.CallEncryptionEntry, Object { + private Database db; + + public CallEncryptionEntry(Database db) { + this.db = db; + } + + public Plugins.CallEncryptionWidget? get_widget(Account account, Xmpp.Xep.Jingle.ContentEncryption encryption) { + DtlsSrtpVerificationDraft.OmemoContentEncryption? omemo_encryption = encryption as DtlsSrtpVerificationDraft.OmemoContentEncryption; + if (omemo_encryption == null) return null; + + int identity_id = db.identity.get_id(account.id); + Row? device = db.identity_meta.get_device(identity_id, omemo_encryption.jid.to_string(), omemo_encryption.sid); + if (device == null) return null; + TrustLevel trust = (TrustLevel) device[db.identity_meta.trust_level]; + + return new CallEncryptionWidget(trust); + } + } + + public class CallEncryptionWidget : Plugins.CallEncryptionWidget, Object { + + string? title = null; + string? icon = null; + bool should_show_keys = false; + + public CallEncryptionWidget(TrustLevel trust) { + if (trust == TrustLevel.VERIFIED) { + title = "This call is <b>encrypted and verified</b> with OMEMO."; + icon = "dino-security-high-symbolic"; + should_show_keys = false; + } else { + title = "This call is encrypted with OMEMO."; + should_show_keys = true; + } + } + + public string? get_title() { + return title; + } + + public string? get_icon_name() { + return icon; + } + + public bool show_keys() { + return should_show_keys; + } + } +} diff --git a/plugins/rtp/CMakeLists.txt b/plugins/rtp/CMakeLists.txt new file mode 100644 index 00000000..52419425 --- /dev/null +++ b/plugins/rtp/CMakeLists.txt @@ -0,0 +1,61 @@ +find_package(GstRtp REQUIRED) +find_package(WebRTCAudioProcessing 0.2) +find_packages(RTP_PACKAGES REQUIRED + Gee + GLib + GModule + GnuTLS + GObject + GTK3 + Gst + GstApp + GstAudio +) + +if(Gst_VERSION VERSION_GREATER "1.16") + set(RTP_DEFINITIONS GST_1_16) +endif() + +if(WebRTCAudioProcessing_VERSION GREATER "0.4") + message(STATUS "Ignoring WebRTCAudioProcessing, only versions < 0.4 supported so far") + unset(WebRTCAudioProcessing_FOUND) +endif() + +if(WebRTCAudioProcessing_FOUND) + set(RTP_DEFINITIONS ${RTP_DEFINITIONS} WITH_VOICE_PROCESSOR) + set(RTP_VOICE_PROCESSOR_VALA src/voice_processor.vala) + set(RTP_VOICE_PROCESSOR_CXX src/voice_processor_native.cpp) + set(RTP_VOICE_PROCESSOR_LIB webrtc-audio-processing) +else() + message(STATUS "WebRTCAudioProcessing not found, build without voice pre-processing!") +endif() + +vala_precompile(RTP_VALA_C +SOURCES + src/codec_util.vala + src/device.vala + src/module.vala + src/plugin.vala + src/stream.vala + src/video_widget.vala + src/register_plugin.vala + ${RTP_VOICE_PROCESSOR_VALA} +CUSTOM_VAPIS + ${CMAKE_BINARY_DIR}/exports/crypto-vala.vapi + ${CMAKE_BINARY_DIR}/exports/xmpp-vala.vapi + ${CMAKE_BINARY_DIR}/exports/dino.vapi + ${CMAKE_BINARY_DIR}/exports/qlite.vapi + ${CMAKE_CURRENT_SOURCE_DIR}/vapi/gstreamer-rtp-1.0.vapi +PACKAGES + ${RTP_PACKAGES} +DEFINITIONS + ${RTP_DEFINITIONS} +) + +add_definitions(${VALA_CFLAGS} -DG_LOG_DOMAIN="rtp" -I${CMAKE_CURRENT_SOURCE_DIR}/src) +add_library(rtp SHARED ${RTP_VALA_C} ${RTP_VOICE_PROCESSOR_CXX}) +target_link_libraries(rtp libdino crypto-vala ${RTP_PACKAGES} gstreamer-rtp-1.0 ${RTP_VOICE_PROCESSOR_LIB}) +set_target_properties(rtp PROPERTIES PREFIX "") +set_target_properties(rtp PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/plugins/) + +install(TARGETS rtp ${PLUGIN_INSTALL}) diff --git a/plugins/rtp/src/codec_util.vala b/plugins/rtp/src/codec_util.vala new file mode 100644 index 00000000..6a2438f1 --- /dev/null +++ b/plugins/rtp/src/codec_util.vala @@ -0,0 +1,307 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.CodecUtil { + private Set<string> supported_elements = new HashSet<string>(); + private Set<string> unsupported_elements = new HashSet<string>(); + + public static Gst.Caps get_caps(string media, JingleRtp.PayloadType payload_type, bool incoming) { + Gst.Caps caps = new Gst.Caps.simple("application/x-rtp", + "media", typeof(string), media, + "payload", typeof(int), payload_type.id); + //"channels", typeof(int), payloadType.channels, + //"max-ptime", typeof(int), payloadType.maxptime); + unowned Gst.Structure s = caps.get_structure(0); + if (payload_type.clockrate != 0) { + s.set("clock-rate", typeof(int), payload_type.clockrate); + } + if (payload_type.name != null) { + s.set("encoding-name", typeof(string), payload_type.name.up()); + } + if (incoming) { + foreach (JingleRtp.RtcpFeedback rtcp_fb in payload_type.rtcp_fbs) { + if (rtcp_fb.subtype == null) { + s.set(@"rtcp-fb-$(rtcp_fb.type_)", typeof(bool), true); + } else { + s.set(@"rtcp-fb-$(rtcp_fb.type_)-$(rtcp_fb.subtype)", typeof(bool), true); + } + } + } + return caps; + } + + public static string? get_codec_from_payload(string media, JingleRtp.PayloadType payload_type) { + if (payload_type.name != null) return payload_type.name.down(); + if (media == "audio") { + switch (payload_type.id) { + case 0: + return "pcmu"; + case 8: + return "pcma"; + } + } + return null; + } + + public static string? get_media_type_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_media_type(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_media_type(string media, string? codec) { + if (codec == null) return null; + if (media == "audio") { + switch (codec) { + case "pcma": + return "audio/x-alaw"; + case "pcmu": + return "audio/x-mulaw"; + } + } + return @"$media/x-$codec"; + } + + public static string? get_rtp_pay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_pay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_pay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)pay"; + } + + public static string? get_rtp_depay_element_name_from_payload(string media, JingleRtp.PayloadType payload_type) { + return get_depay_candidate(media, get_codec_from_payload(media, payload_type)); + } + + public static string? get_depay_candidate(string media, string? codec) { + if (codec == null) return null; + return @"rtp$(codec)depay"; + } + + public static string[] get_encode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusenc"}; + case "speex": + return new string[] {"speexenc"}; + case "pcma": + return new string[] {"alawenc"}; + case "pcmu": + return new string[] {"mulawenc"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264enc", */"vaapih264enc", "x264enc"}; + case "vp9": + return new string[] {/*"msdkvp9enc", */"vaapivp9enc" /*, "vp9enc" */}; + case "vp8": + return new string[] {/*"msdkvp8enc", */"vaapivp8enc", "vp8enc"}; + } + } + return new string[0]; + } + + public static string[] get_decode_candidates(string media, string? codec) { + if (codec == null) return new string[0]; + if (media == "audio") { + switch (codec) { + case "opus": + return new string[] {"opusdec"}; + case "speex": + return new string[] {"speexdec"}; + case "pcma": + return new string[] {"alawdec"}; + case "pcmu": + return new string[] {"mulawdec"}; + } + } else if (media == "video") { + switch (codec) { + case "h264": + return new string[] {/*"msdkh264dec", */"vaapih264dec"}; + case "vp9": + return new string[] {/*"msdkvp9dec", */"vaapivp9dec", "vp9dec"}; + case "vp8": + return new string[] {/*"msdkvp8dec", */"vaapivp8dec", "vp8dec"}; + } + } + return new string[0]; + } + + public static string? get_encode_prefix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (encode == "msdkh264enc") return "video/x-raw,format=NV12 ! "; + if (encode == "vaapih264enc") return "video/x-raw,format=NV12 ! "; + return null; + } + + public static string? get_encode_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (encode == "msdkh264enc") return @" rate-control=vbr"; + if (encode == "vaapih264enc") return @" tune=low-power"; + if (encode == "x264enc") return @" byte-stream=1 profile=baseline speed-preset=ultrafast tune=zerolatency"; + + // VP8 + if (encode == "msdkvp8enc") return " rate-control=vbr"; + if (encode == "vaapivp8enc") return " rate-control=vbr"; + if (encode == "vp8enc") return " deadline=1 error-resilient=1"; + + // OPUS + if (encode == "opusenc") { + if (payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " audio-type=voice inband-fec=true"; + return " audio-type=voice"; + } + + return null; + } + + public static string? get_encode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + // H264 + if (media == "video" && codec == "h264") return " ! video/x-h264,profile=constrained-baseline ! h264parse"; + return null; + } + + public uint update_bitrate(string media, JingleRtp.PayloadType payload_type, Gst.Element encode_element, uint bitrate) { + Gst.Bin? encode_bin = encode_element as Gst.Bin; + if (encode_bin == null) return 0; + string? codec = get_codec_from_payload(media, payload_type); + string? encode_name = get_encode_element_name(media, codec); + if (encode_name == null) return 0; + Gst.Element encode = encode_bin.get_by_name(@"$(encode_bin.name)_encode"); + + bitrate = uint.min(2048000, bitrate); + + switch (encode_name) { + case "msdkh264enc": + case "vaapih264enc": + case "x264enc": + case "msdkvp8enc": + case "vaapivp8enc": + bitrate = uint.min(2048000, bitrate); + encode.set("bitrate", bitrate); + return bitrate; + case "vp8enc": + bitrate = uint.min(2147483, bitrate); + encode.set("target-bitrate", bitrate * 1000); + return bitrate; + } + + return 0; + } + + public static string? get_decode_prefix(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_decode_args(string media, string codec, string decode, JingleRtp.PayloadType? payload_type) { + if (decode == "opusdec" && payload_type != null && payload_type.parameters.has("useinbandfec", "1")) return " use-inband-fec=true"; + if (decode == "vaapivp9dec" || decode == "vaapivp8dec" || decode == "vaapih264dec") return " max-errors=100"; + return null; + } + + public static string? get_decode_suffix(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + return null; + } + + public static string? get_depay_args(string media, string codec, string encode, JingleRtp.PayloadType? payload_type) { + if (codec == "vp8") return " wait-for-keyframe=true"; + return null; + } + + public bool is_element_supported(string element_name) { + if (unsupported_elements.contains(element_name)) return false; + if (supported_elements.contains(element_name)) return true; + var test_element = Gst.ElementFactory.make(element_name, @"test-$element_name"); + if (test_element != null) { + supported_elements.add(element_name); + return true; + } else { + debug("%s is not supported on this platform", element_name); + unsupported_elements.add(element_name); + return false; + } + } + + public string? get_encode_element_name(string media, string? codec) { + if (!is_element_supported(get_pay_element_name(media, codec))) return null; + foreach (string candidate in get_encode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_pay_element_name(string media, string? codec) { + string candidate = get_pay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public string? get_decode_element_name(string media, string? codec) { + foreach (string candidate in get_decode_candidates(media, codec)) { + if (is_element_supported(candidate)) return candidate; + } + return null; + } + + public string? get_depay_element_name(string media, string? codec) { + string candidate = get_depay_candidate(media, codec); + if (is_element_supported(candidate)) return candidate; + return null; + } + + public void mark_element_unsupported(string element_name) { + unsupported_elements.add(element_name); + } + + public string? get_decode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string depay = get_depay_element_name(media, codec); + string decode = element_name ?? get_decode_element_name(media, codec); + if (depay == null || decode == null) return null; + string decode_prefix = get_decode_prefix(media, codec, decode, payload_type) ?? ""; + string decode_args = get_decode_args(media, codec, decode, payload_type) ?? ""; + string decode_suffix = get_decode_suffix(media, codec, decode, payload_type) ?? ""; + string depay_args = get_depay_args(media, codec, decode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$depay$depay_args name=$(base_name)_rtp_depay ! $decode_prefix$decode$decode_args name=$(base_name)_$(codec)_decode$decode_suffix ! $(media)convert name=$(base_name)_convert$resample"; + } + + public Gst.Element? get_decode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode-$codec-$(Random.next_int())"; + string? desc = get_decode_bin_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to decode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + + public string? get_encode_bin_description(string media, string? codec, JingleRtp.PayloadType? payload_type, string? element_name = null, string? name = null) { + if (codec == null) return null; + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string pay = get_pay_element_name(media, codec); + string encode = element_name ?? get_encode_element_name(media, codec); + if (pay == null || encode == null) return null; + string encode_prefix = get_encode_prefix(media, codec, encode, payload_type) ?? ""; + string encode_args = get_encode_args(media, codec, encode, payload_type) ?? ""; + string encode_suffix = get_encode_suffix(media, codec, encode, payload_type) ?? ""; + string resample = media == "audio" ? @" ! audioresample name=$(base_name)_resample" : ""; + return @"$(media)convert name=$(base_name)_convert$resample ! $encode_prefix$encode$encode_args name=$(base_name)_encode$encode_suffix ! $pay pt=$(payload_type != null ? payload_type.id : 96) name=$(base_name)_rtp_pay"; + } + + public Gst.Element? get_encode_bin(string media, JingleRtp.PayloadType payload_type, string? name = null) { + string? codec = get_codec_from_payload(media, payload_type); + string base_name = name ?? @"encode_$(codec)_$(Random.next_int())"; + string? desc = get_encode_bin_description(media, codec, payload_type, null, base_name); + if (desc == null) return null; + debug("Pipeline to encode %s %s: %s", media, codec, desc); + Gst.Element bin = Gst.parse_bin_from_description(desc, true); + bin.name = name; + return bin; + } + +}
\ No newline at end of file diff --git a/plugins/rtp/src/device.vala b/plugins/rtp/src/device.vala new file mode 100644 index 00000000..e25271b1 --- /dev/null +++ b/plugins/rtp/src/device.vala @@ -0,0 +1,272 @@ +public class Dino.Plugins.Rtp.Device : MediaDevice, Object { + public Plugin plugin { get; private set; } + public Gst.Device device { get; private set; } + + private string device_name; + public string id { get { + return device_name; + }} + private string device_display_name; + public string display_name { get { + return device_display_name; + }} + public string detail_name { get { + return device.properties.get_string("alsa.card_name") ?? device.properties.get_string("alsa.id") ?? id; + }} + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public string? media { get { + if (device.device_class.has_prefix("Audio/")) { + return "audio"; + } else if (device.device_class.has_prefix("Video/")) { + return "video"; + } else { + return null; + } + }} + public bool is_source { get { + return device.device_class.has_suffix("/Source"); + }} + public bool is_sink { get { + return device.device_class.has_suffix("/Sink"); + }} + + private Gst.Element element; + private Gst.Element tee; + private Gst.Element dsp; + private Gst.Element mixer; + private Gst.Element filter; + private Gst.Element rate; + private int links = 0; + + public Device(Plugin plugin, Gst.Device device) { + this.plugin = plugin; + update(device); + } + + public bool matches(Gst.Device device) { + if (this.device.name == device.name) return true; + return false; + } + + public void update(Gst.Device device) { + this.device = device; + this.device_name = device.name; + this.device_display_name = device.display_name; + } + + public Gst.Element? link_sink() { + if (element == null) create(); + links++; + if (mixer != null) return mixer; + if (is_sink && media == "audio") return filter; + return element; + } + + public Gst.Element? link_source() { + if (element == null) create(); + links++; + if (tee != null) return tee; + return element; + } + + public void unlink() { + if (links <= 0) { + critical("Link count below zero."); + return; + } + links--; + if (links == 0) { + destroy(); + } + } + + private Gst.Caps get_best_caps() { + if (media == "audio") { + return Gst.Caps.from_string("audio/x-raw,rate=48000,channels=1"); + } else if (media == "video" && device.caps.get_size() > 0) { + int best_index = 0; + Value? best_fraction = null; + int best_fps = 0; + int best_width = 0; + int best_height = 0; + for (int i = 0; i < device.caps.get_size(); i++) { + unowned Gst.Structure? that = device.caps.get_structure(i); + if (!that.has_name("video/x-raw")) continue; + int num = 0, den = 0, width = 0, height = 0; + if (!that.has_field("framerate")) continue; + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.Fraction)) { + num = Gst.Value.get_fraction_numerator(framerate); + den = Gst.Value.get_fraction_denominator(framerate); + } else if (framerate.type() == typeof(Gst.ValueList)) { + for(uint j = 0; j < Gst.ValueList.get_size(framerate); j++) { + Value fraction = Gst.ValueList.get_value(framerate, j); + int in_num = Gst.Value.get_fraction_numerator(fraction); + int in_den = Gst.Value.get_fraction_denominator(fraction); + int fps = den > 0 ? (num/den) : 0; + int in_fps = in_den > 0 ? (in_num/in_den) : 0; + if (in_fps > fps) { + best_fraction = fraction; + num = in_num; + den = in_den; + } + } + } else { + debug("Unknown type for framerate: %s", framerate.type_name()); + } + if (den == 0) continue; + if (!that.has_field("width") || !that.get_int("width", out width)) continue; + if (!that.has_field("height") || !that.get_int("height", out height)) continue; + int fps = num/den; + if (best_fps < fps || best_fps == fps && best_width < width || best_fps == fps && best_width == width && best_height < height) { + best_fps = fps; + best_width = width; + best_height = height; + best_index = i; + } + } + Gst.Caps res = caps_copy_nth(device.caps, best_index); + unowned Gst.Structure? that = res.get_structure(0); + Value framerate = that.get_value("framerate"); + if (framerate.type() == typeof(Gst.ValueList)) { + that.set_value("framerate", best_fraction); + } + debug("Selected caps %s", res.to_string()); + return res; + } else if (device.caps.get_size() > 0) { + return caps_copy_nth(device.caps, 0); + } else { + return new Gst.Caps.any(); + } + } + + // Backport from gst_caps_copy_nth added in GStreamer 1.16 + private static Gst.Caps caps_copy_nth(Gst.Caps source, uint index) { + Gst.Caps target = new Gst.Caps.empty(); + target.flags = source.flags; + target.append_structure_full(source.get_structure(index).copy(), source.get_features(index).copy()); + return target; + } + + private void create() { + debug("Creating device %s", id); + plugin.pause(); + element = device.create_element(id); + pipe.add(element); + if (is_source) { + element.@set("do-timestamp", true); + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + element.link(filter); +#if WITH_VOICE_PROCESSOR + if (media == "audio" && plugin.echoprobe != null) { + dsp = new VoiceProcessor(plugin.echoprobe as EchoProbe, element as Gst.Audio.StreamVolume); + dsp.name = @"dsp_$id"; + pipe.add(dsp); + filter.link(dsp); + } +#endif + tee = Gst.ElementFactory.make("tee", @"tee_$id"); + tee.@set("allow-not-linked", true); + pipe.add(tee); + (dsp ?? filter).link(tee); + } + if (is_sink) { + element.@set("async", false); + element.@set("sync", false); + } + if (is_sink && media == "audio") { + filter = Gst.ElementFactory.make("capsfilter", @"caps_filter_$id"); + filter.@set("caps", get_best_caps()); + pipe.add(filter); + if (plugin.echoprobe != null) { + rate = Gst.ElementFactory.make("audiorate", @"rate_$id"); + rate.@set("tolerance", 100000000); + pipe.add(rate); + filter.link(rate); + rate.link(plugin.echoprobe); + plugin.echoprobe.link(element); + } else { + filter.link(element); + } + } + plugin.unpause(); + } + + private void destroy() { + if (mixer != null) { + if (is_sink && media == "audio" && plugin.echoprobe != null) { + plugin.echoprobe.unlink(mixer); + } + int linked_sink_pads = 0; + mixer.foreach_sink_pad((_, pad) => { + if (pad.is_linked()) linked_sink_pads++; + return true; + }); + if (linked_sink_pads > 0) { + warning("%s-mixer still has %i sink pads while being destroyed", id, linked_sink_pads); + } + mixer.set_locked_state(true); + mixer.set_state(Gst.State.NULL); + mixer.unlink(element); + pipe.remove(mixer); + mixer = null; + } else if (is_sink && media == "audio") { + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(rate ?? ((Gst.Element)plugin.echoprobe) ?? element); + pipe.remove(filter); + filter = null; + } + if (rate != null) { + rate.set_locked_state(true); + rate.set_state(Gst.State.NULL); + rate.unlink(plugin.echoprobe); + pipe.remove(rate); + rate = null; + } + if (plugin.echoprobe != null) { + plugin.echoprobe.unlink(element); + } + } + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + if (filter != null) element.unlink(filter); + else if (is_source) element.unlink(tee); + pipe.remove(element); + element = null; + if (filter != null) { + filter.set_locked_state(true); + filter.set_state(Gst.State.NULL); + filter.unlink(dsp ?? tee); + pipe.remove(filter); + filter = null; + } + if (dsp != null) { + dsp.set_locked_state(true); + dsp.set_state(Gst.State.NULL); + dsp.unlink(tee); + pipe.remove(dsp); + dsp = null; + } + if (tee != null) { + int linked_src_pads = 0; + tee.foreach_src_pad((_, pad) => { + if (pad.is_linked()) linked_src_pads++; + return true; + }); + if (linked_src_pads != 0) { + warning("%s-tee still has %d src pads while being destroyed", id, linked_src_pads); + } + tee.set_locked_state(true); + tee.set_state(Gst.State.NULL); + pipe.remove(tee); + tee = null; + } + debug("Destroyed device %s", id); + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/module.vala b/plugins/rtp/src/module.vala new file mode 100644 index 00000000..19a7501d --- /dev/null +++ b/plugins/rtp/src/module.vala @@ -0,0 +1,237 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Module : JingleRtp.Module { + private Set<string> supported_codecs = new HashSet<string>(); + private Set<string> unsupported_codecs = new HashSet<string>(); + public Plugin plugin { get; private set; } + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + + public Module(Plugin plugin) { + base(); + this.plugin = plugin; + } + + private async bool pipeline_works(string media, string element_desc) { + var supported = false; + string pipeline_desc = @"$(media)testsrc is-live=true ! $element_desc ! appsink name=output"; + try { + var pipeline = Gst.parse_launch(pipeline_desc); + var output = (pipeline as Gst.Bin).get_by_name("output") as Gst.App.Sink; + SourceFunc callback = pipeline_works.callback; + var finished = false; + output.emit_signals = true; + output.new_sample.connect(() => { + if (!finished) { + finished = true; + supported = true; + Idle.add(() => { + callback(); + return Source.REMOVE; + }); + } + return Gst.FlowReturn.EOS; + }); + pipeline.bus.add_watch(Priority.DEFAULT, (_, message) => { + if (message.type == Gst.MessageType.ERROR && !finished) { + Error e; + string d; + message.parse_error(out e, out d); + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + debug(d); + finished = true; + callback(); + } + return true; + }); + Timeout.add(2000, () => { + if (!finished) { + finished = true; + callback(); + } + return Source.REMOVE; + }); + pipeline.set_state(Gst.State.PLAYING); + yield; + pipeline.set_state(Gst.State.NULL); + } catch (Error e) { + debug("pipeline [%s] failed: %s", pipeline_desc, e.message); + } + return supported; + } + + private async bool is_payload_supported(string media, JingleRtp.PayloadType payload_type) { + string? codec = CodecUtil.get_codec_from_payload(media, payload_type); + if (codec == null) return false; + if (unsupported_codecs.contains(codec)) return false; + if (supported_codecs.contains(codec)) return true; + + string? encode_element = codec_util.get_encode_element_name(media, codec); + string? decode_element = codec_util.get_decode_element_name(media, codec); + if (encode_element == null || decode_element == null) { + debug("No suitable encoder or decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + + string encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); + while (!(yield pipeline_works(media, encode_bin))) { + debug("%s not suited for encoding %s", encode_element, codec); + codec_util.mark_element_unsupported(encode_element); + encode_element = codec_util.get_encode_element_name(media, codec); + if (encode_element == null) { + debug("No suitable encoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + encode_bin = codec_util.get_encode_bin_description(media, codec, null, encode_element); + } + debug("using %s to encode %s", encode_element, codec); + + string decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); + while (!(yield pipeline_works(media, @"$encode_bin ! $decode_bin"))) { + debug("%s not suited for decoding %s", decode_element, codec); + codec_util.mark_element_unsupported(decode_element); + decode_element = codec_util.get_decode_element_name(media, codec); + if (decode_element == null) { + debug("No suitable decoder found for %s", codec); + unsupported_codecs.add(codec); + return false; + } + decode_bin = codec_util.get_decode_bin_description(media, codec, null, decode_element); + } + debug("using %s to decode %s", decode_element, codec); + + supported_codecs.add(codec); + return true; + } + + public override bool is_header_extension_supported(string media, JingleRtp.HeaderExtension ext) { + if (media == "video" && ext.uri == "urn:3gpp:video-orientation") return true; + return false; + } + + public override Gee.List<JingleRtp.HeaderExtension> get_suggested_header_extensions(string media) { + Gee.List<JingleRtp.HeaderExtension> exts = new ArrayList<JingleRtp.HeaderExtension>(); + if (media == "video") { + exts.add(new JingleRtp.HeaderExtension(1, "urn:3gpp:video-orientation")); + } + return exts; + } + + public async void add_if_supported(Gee.List<JingleRtp.PayloadType> list, string media, JingleRtp.PayloadType payload_type) { + if (yield is_payload_supported(media, payload_type)) { + list.add(payload_type); + } + } + + public override async Gee.List<JingleRtp.PayloadType> get_supported_payloads(string media) { + Gee.List<JingleRtp.PayloadType> list = new ArrayList<JingleRtp.PayloadType>(JingleRtp.PayloadType.equals_func); + if (media == "audio") { + var opus = new JingleRtp.PayloadType() { channels = 2, clockrate = 48000, name = "opus", id = 99 }; + opus.parameters["useinbandfec"] = "1"; + var speex32 = new JingleRtp.PayloadType() { channels = 1, clockrate = 32000, name = "speex", id = 100 }; + var speex16 = new JingleRtp.PayloadType() { channels = 1, clockrate = 16000, name = "speex", id = 101 }; + var speex8 = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "speex", id = 102 }; + var pcmu = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMU", id = 0 }; + var pcma = new JingleRtp.PayloadType() { channels = 1, clockrate = 8000, name = "PCMA", id = 8 }; + yield add_if_supported(list, media, opus); + yield add_if_supported(list, media, speex32); + yield add_if_supported(list, media, speex16); + yield add_if_supported(list, media, speex8); + yield add_if_supported(list, media, pcmu); + yield add_if_supported(list, media, pcma); + } else if (media == "video") { + var h264 = new JingleRtp.PayloadType() { clockrate = 90000, name = "H264", id = 96 }; + var vp9 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP9", id = 97 }; + var vp8 = new JingleRtp.PayloadType() { clockrate = 90000, name = "VP8", id = 98 }; + var rtcp_fbs = new ArrayList<JingleRtp.RtcpFeedback>(); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("goog-remb")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("ccm", "fir")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack")); + rtcp_fbs.add(new JingleRtp.RtcpFeedback("nack", "pli")); + h264.rtcp_fbs.add_all(rtcp_fbs); + vp9.rtcp_fbs.add_all(rtcp_fbs); + vp8.rtcp_fbs.add_all(rtcp_fbs); + yield add_if_supported(list, media, h264); + yield add_if_supported(list, media, vp9); + yield add_if_supported(list, media, vp8); + } else { + warning("Unsupported media type: %s", media); + } + return list; + } + + public override async JingleRtp.PayloadType? pick_payload_type(string media, Gee.List<JingleRtp.PayloadType> payloads) { + if (media == "audio") { + foreach (JingleRtp.PayloadType type in payloads) { + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); + } + } else if (media == "video") { + // We prefer H.264 (best support for hardware acceleration and good overall codec quality) + JingleRtp.PayloadType? h264 = payloads.first_match((it) => it.name.up() == "H264"); + if (h264 != null && yield is_payload_supported(media, h264)) return adjust_payload_type(media, h264.clone()); + // Take first of the list that we do support otherwise + foreach (JingleRtp.PayloadType type in payloads) { + if (yield is_payload_supported(media, type)) return adjust_payload_type(media, type.clone()); + } + } else { + warning("Unsupported media type: %s", media); + } + return null; + } + + public JingleRtp.PayloadType adjust_payload_type(string media, JingleRtp.PayloadType type) { + var iter = type.rtcp_fbs.iterator(); + while (iter.next()) { + var fb = iter.@get(); + switch (fb.type_) { + case "goog-remb": + if (fb.subtype != null) iter.remove(); + break; + case "ccm": + if (fb.subtype != "fir") iter.remove(); + break; + case "nack": + if (fb.subtype != null && fb.subtype != "pli") iter.remove(); + break; + default: + iter.remove(); + break; + } + } + return type; + } + + public override JingleRtp.Stream create_stream(Jingle.Content content) { + return plugin.open_stream(content); + } + + public override void close_stream(JingleRtp.Stream stream) { + var rtp_stream = stream as Rtp.Stream; + plugin.close_stream(rtp_stream); + } + + public override JingleRtp.Crypto? generate_local_crypto() { + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return JingleRtp.Crypto.create(JingleRtp.Crypto.AES_CM_128_HMAC_SHA1_80, key_and_salt); + } + + public override JingleRtp.Crypto? pick_remote_crypto(Gee.List<JingleRtp.Crypto> cryptos) { + foreach (JingleRtp.Crypto crypto in cryptos) { + if (crypto.is_valid) return crypto; + } + return null; + } + + public override JingleRtp.Crypto? pick_local_crypto(JingleRtp.Crypto? remote) { + if (remote == null || !remote.is_valid) return null; + uint8[] key_and_salt = new uint8[30]; + Crypto.randomize(key_and_salt); + return remote.rekey(key_and_salt); + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/participant.vala b/plugins/rtp/src/participant.vala new file mode 100644 index 00000000..1ca13191 --- /dev/null +++ b/plugins/rtp/src/participant.vala @@ -0,0 +1,39 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Participant { + public Jid full_jid { get; private set; } + + protected Gst.Pipeline pipe; + private Map<Stream, uint32> ssrcs = new HashMap<Stream, uint32>(); + + public Participant(Gst.Pipeline pipe, Jid full_jid) { + this.pipe = pipe; + this.full_jid = full_jid; + } + + public uint32 get_ssrc(Stream stream) { + if (ssrcs.has_key(stream)) { + return ssrcs[stream]; + } + return 0; + } + + public void set_ssrc(Stream stream, uint32 ssrc) { + if (ssrcs.has_key(stream)) { + warning("Learning ssrc %ul for %s in %s when it is already known as %ul", ssrc, full_jid.to_string(), stream.to_string(), ssrcs[stream]); + } else { + stream.on_destroy.connect(unset_ssrc); + } + ssrcs[stream] = ssrc; + } + + public void unset_ssrc(Stream stream) { + ssrcs.unset(stream); + stream.on_destroy.disconnect(unset_ssrc); + } + + public string to_string() { + return @"participant $full_jid"; + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/plugin.vala b/plugins/rtp/src/plugin.vala new file mode 100644 index 00000000..19a266b1 --- /dev/null +++ b/plugins/rtp/src/plugin.vala @@ -0,0 +1,449 @@ +using Gee; +using Xmpp; +using Xmpp.Xep; + +public class Dino.Plugins.Rtp.Plugin : RootInterface, VideoCallPlugin, Object { + public Dino.Application app { get; private set; } + public CodecUtil codec_util { get; private set; } + public Gst.DeviceMonitor device_monitor { get; private set; } + public Gst.Pipeline pipe { get; private set; } + public Gst.Bin rtpbin { get; private set; } + public Gst.Element echoprobe { get; private set; } + + private Gee.List<Stream> streams = new ArrayList<Stream>(); + private Gee.List<Device> devices = new ArrayList<Device>(); + // private Gee.List<Participant> participants = new ArrayList<Participant>(); + + public void registered(Dino.Application app) { + this.app = app; + this.codec_util = new CodecUtil(); + app.startup.connect(startup); + app.add_option_group(Gst.init_get_option_group()); + app.stream_interactor.module_manager.initialize_account_modules.connect((account, list) => { + list.add(new Module(this)); + }); + app.plugin_registry.video_call_plugin = this; + } + + private int pause_count = 0; + public void pause() { +// if (pause_count == 0) { +// debug("Pausing pipe for modifications"); +// pipe.set_state(Gst.State.PAUSED); +// } + pause_count++; + } + public void unpause() { + pause_count--; + if (pause_count == 0) { + debug("Continue pipe after modifications"); + pipe.set_state(Gst.State.PLAYING); + } + if (pause_count < 0) warning("Pause count below zero!"); + } + + public void startup() { + device_monitor = new Gst.DeviceMonitor(); + device_monitor.show_all = true; + device_monitor.get_bus().add_watch(Priority.DEFAULT, on_device_monitor_message); + device_monitor.start(); + foreach (Gst.Device device in device_monitor.get_devices()) { + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) continue; + if (device.properties.get_string("device.class") == "monitor") continue; + if (devices.any_match((it) => it.matches(device))) continue; + devices.add(new Device(this, device)); + } + + pipe = new Gst.Pipeline(null); + + // RTP + rtpbin = Gst.ElementFactory.make("rtpbin", null) as Gst.Bin; + if (rtpbin == null) { + warning("RTP not supported"); + pipe = null; + return; + } + rtpbin.pad_added.connect(on_rtp_pad_added); + rtpbin.@set("latency", 100); + rtpbin.@set("do-lost", true); + rtpbin.@set("do-sync-event", true); + rtpbin.@set("drop-on-latency", true); + rtpbin.connect("signal::request-pt-map", request_pt_map, this); + pipe.add(rtpbin); + +#if WITH_VOICE_PROCESSOR + // Audio echo probe + echoprobe = new EchoProbe(); + if (echoprobe != null) pipe.add(echoprobe); +#endif + + // Pipeline + pipe.auto_flush_bus = true; + pipe.bus.add_watch(GLib.Priority.DEFAULT, (_, message) => { + on_pipe_bus_message(message); + return true; + }); + pipe.set_state(Gst.State.PLAYING); + } + + private static Gst.Caps? request_pt_map(Gst.Element rtpbin, uint session, uint pt, Plugin plugin) { + debug("request-pt-map"); + return null; + } + + private void on_rtp_pad_added(Gst.Pad pad) { + debug("pad added: %s", pad.name); + if (pad.name.has_prefix("recv_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_ssrc_pad_added(split[4], pad); + } + } + } + if (pad.name.has_prefix("send_rtp_src_")) { + string[] split = pad.name.split("_"); + uint8 rtpid = (uint8)int.parse(split[3]); + debug("pad %s for stream %hhu", pad.name, rtpid); + foreach (Stream stream in streams) { + if (stream.rtpid == rtpid) { + stream.on_send_rtp_src_added(pad); + } + } + } + } + + private void on_pipe_bus_message(Gst.Message message) { + switch (message.type) { + case Gst.MessageType.ERROR: + Error error; + string str; + message.parse_error(out error, out str); + warning("Error in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.WARNING: + Error error; + string str; + message.parse_warning(out error, out str); + warning("Warning in pipeline: %s", error.message); + debug(str); + break; + case Gst.MessageType.CLOCK_LOST: + debug("Clock lost. Restarting"); + pipe.set_state(Gst.State.READY); + pipe.set_state(Gst.State.PLAYING); + break; + case Gst.MessageType.STATE_CHANGED: + // Ignore + break; + case Gst.MessageType.STREAM_STATUS: + Gst.StreamStatusType status; + Gst.Element owner; + message.parse_stream_status(out status, out owner); + if (owner != null) { + debug("%s stream changed status to %s", owner.name, status.to_string()); + } + break; + case Gst.MessageType.ELEMENT: + unowned Gst.Structure struc = message.get_structure(); + if (struc != null && message.src is Gst.Element) { + debug("Message from %s in pipeline: %s", ((Gst.Element)message.src).name, struc.to_string()); + } + break; + case Gst.MessageType.NEW_CLOCK: + debug("New clock."); + break; + case Gst.MessageType.TAG: + // Ignore + break; + case Gst.MessageType.QOS: + // Ignore + break; + case Gst.MessageType.LATENCY: + if (message.src != null && message.src.name != null && message.src is Gst.Element) { + Gst.Query latency_query = new Gst.Query.latency(); + if (((Gst.Element)message.src).query(latency_query)) { + bool live; + Gst.ClockTime min_latency, max_latency; + latency_query.parse_latency(out live, out min_latency, out max_latency); + debug("Latency message from %s: live=%s, min_latency=%s, max_latency=%s", message.src.name, live.to_string(), min_latency.to_string(), max_latency.to_string()); + } + } + break; + default: + debug("Pipe bus message: %s", message.type.to_string()); + break; + } + } + + private bool on_device_monitor_message(Gst.Bus bus, Gst.Message message) { + Gst.Device old_device = null; + Gst.Device device = null; + Device old = null; + switch (message.type) { + case Gst.MessageType.DEVICE_ADDED: + message.parse_device_added(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + if (devices.any_match((it) => it.matches(device))) return Source.CONTINUE; + devices.add(new Device(this, device)); + break; +#if GST_1_16 + case Gst.MessageType.DEVICE_CHANGED: + message.parse_device_changed(out device, out old_device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + old = devices.first_match((it) => it.matches(old_device)); + if (old != null) old.update(device); + break; +#endif + case Gst.MessageType.DEVICE_REMOVED: + message.parse_device_removed(out device); + if (device.properties.has_name("pipewire-proplist") && device.device_class.has_prefix("Audio/")) return Source.CONTINUE; + if (device.properties.get_string("device.class") == "monitor") return Source.CONTINUE; + old = devices.first_match((it) => it.matches(device)); + if (old != null) devices.remove(old); + break; + } + if (device != null) { + switch (device.device_class) { + case "Audio/Source": + devices_changed("audio", false); + break; + case "Audio/Sink": + devices_changed("audio", true); + break; + case "Video/Source": + devices_changed("video", false); + break; + case "Video/Sink": + devices_changed("video", true); + break; + } + } + return Source.CONTINUE; + } + + public uint8 next_free_id() { + uint8 rtpid = 0; + while (streams.size < 100 && streams.any_match((stream) => stream.rtpid == rtpid)) { + rtpid++; + } + return rtpid; + } + + // public Participant get_participant(Jid full_jid, bool self) { +// foreach (Participant participant in participants) { +// if (participant.full_jid.equals(full_jid)) { +// return participant; +// } +// } +// Participant participant; +// if (self) { +// participant = new SelfParticipant(pipe, full_jid); +// } else { +// participant = new Participant(pipe, full_jid); +// } +// participants.add(participant); +// return participant; +// } + + public Stream open_stream(Xmpp.Xep.Jingle.Content content) { + var content_params = content.content_params as Xmpp.Xep.JingleRtp.Parameters; + if (content_params == null) return null; + Stream stream; + if (content_params.media == "video") { + stream = new VideoStream(this, content); + } else { + stream = new Stream(this, content); + } + streams.add(stream); + return stream; + } + + public void close_stream(Stream stream) { + streams.remove(stream); + stream.destroy(); + } + + public void shutdown() { + device_monitor.stop(); + pipe.set_state(Gst.State.NULL); + rtpbin = null; + pipe = null; + Gst.deinit(); + } + + public bool supports(string media) { + if (rtpbin == null) return false; + + if (media == "audio") { + if (get_devices("audio", false).is_empty) return false; + if (get_devices("audio", true).is_empty) return false; + } + + if (media == "video") { + if (Gst.ElementFactory.make("gtksink", null) == null) return false; + if (get_devices("video", false).is_empty) return false; + } + + return true; + } + + public VideoCallWidget? create_widget(WidgetType type) { + if (type == WidgetType.GTK) { + return new VideoWidget(this); + } + return null; + } + + public Gee.List<MediaDevice> get_devices(string media, bool incoming) { + if (media == "video" && !incoming) { + return get_video_sources(); + } + + ArrayList<MediaDevice> result = new ArrayList<MediaDevice>(); + foreach (Device device in devices) { + if (device.media == media && (incoming && device.is_sink || !incoming && device.is_source)) { + result.add(device); + } + } + if (media == "audio") { + // Reorder sources + result.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + bool left_is_pipewire = left.device.properties.has_name("pipewire-proplist"); + bool right_is_pipewire = right.device.properties.has_name("pipewire-proplist"); + + bool left_is_default = false; + left.device.properties.get_boolean("is-default", out left_is_default); + bool right_is_default = false; + right.device.properties.get_boolean("is-default", out right_is_default); + + // Prefer pipewire + if (left_is_pipewire && !right_is_pipewire) return -1; + if (right_is_pipewire && !left_is_pipewire) return 1; + + // Prefer pulse audio default device + if (left_is_default && !right_is_default) return -1; + if (right_is_default && !left_is_default) return 1; + + + return 0; + }); + } + return result; + } + + public Gee.List<MediaDevice> get_video_sources() { + ArrayList<MediaDevice> pipewire_devices = new ArrayList<MediaDevice>(); + ArrayList<MediaDevice> other_devices = new ArrayList<MediaDevice>(); + + foreach (Device device in devices) { + if (device.media != "video") continue; + if (device.is_sink) continue; + + bool is_color = false; + for (int i = 0; i < device.device.caps.get_size(); i++) { + unowned Gst.Structure structure = device.device.caps.get_structure(i); + if (structure.has_field("format") && !structure.get_string("format").has_prefix("GRAY")) { + is_color = true; + } + } + + // Don't allow grey-scale devices + if (!is_color) continue; + + if (device.device.properties.has_name("pipewire-proplist")) { + pipewire_devices.add(device); + } else { + other_devices.add(device); + } + } + + // If we have any pipewire devices, present only those. Don't want duplicated devices from pipewire and video for linux. + ArrayList<MediaDevice> devices = pipewire_devices.size > 0 ? pipewire_devices : other_devices; + + // Reorder sources + devices.sort((media_left, media_right) => { + Device left = media_left as Device; + Device right = media_right as Device; + if (left == null) return 1; + if (right == null) return -1; + + int left_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) left_fps = int.max(left_fps, num / den); + } + + int right_fps = 0; + for (int i = 0; i < left.device.caps.get_size(); i++) { + unowned Gst.Structure structure = left.device.caps.get_structure(i); + int num = 0, den = 0; + if (structure.has_field("framerate") && structure.get_fraction("framerate", out num, out den)) right_fps = int.max(right_fps, num / den); + } + + // More FPS is better + if (left_fps > right_fps) return -1; + if (right_fps > left_fps) return 1; + + return 0; + }); + + return devices; + } + + public Device? get_preferred_device(string media, bool incoming) { + foreach (MediaDevice media_device in get_devices(media, incoming)) { + Device? device = media_device as Device; + if (device != null) return device; + } + warning("No preferred device for %s %s. Media will not be processed.", incoming ? "incoming" : "outgoing", media); + return null; + } + + public MediaDevice? get_device(Xmpp.Xep.JingleRtp.Stream stream, bool incoming) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return null; + if (incoming) { + return plugin_stream.output_device ?? get_preferred_device(stream.media, incoming); + } else { + return plugin_stream.input_device ?? get_preferred_device(stream.media, incoming); + } + } + + private void dump_dot() { + string name = @"pipe-$(pipe.clock.get_time())-$(pipe.current_state)"; + Gst.Debug.bin_to_dot_file(pipe, Gst.DebugGraphDetails.ALL, name); + debug("Stored pipe details as %s", name); + } + + public void set_pause(Xmpp.Xep.JingleRtp.Stream stream, bool pause) { + Stream plugin_stream = stream as Stream; + if (plugin_stream == null) return; + if (pause) { + plugin_stream.pause(); + } else { + plugin_stream.unpause(); + } + } + + public void set_device(Xmpp.Xep.JingleRtp.Stream stream, MediaDevice? device) { + Device real_device = device as Device; + Stream plugin_stream = stream as Stream; + if (real_device == null || plugin_stream == null) return; + if (real_device.is_source) { + plugin_stream.input_device = real_device; + } else if (real_device.is_sink) { + plugin_stream.output_device = real_device; + } + } +} diff --git a/plugins/rtp/src/register_plugin.vala b/plugins/rtp/src/register_plugin.vala new file mode 100644 index 00000000..a80137ea --- /dev/null +++ b/plugins/rtp/src/register_plugin.vala @@ -0,0 +1,3 @@ +public Type register_plugin(Module module) { + return typeof (Dino.Plugins.Rtp.Plugin); +} diff --git a/plugins/rtp/src/stream.vala b/plugins/rtp/src/stream.vala new file mode 100644 index 00000000..bd8a279f --- /dev/null +++ b/plugins/rtp/src/stream.vala @@ -0,0 +1,681 @@ +using Gee; +using Xmpp; + +public class Dino.Plugins.Rtp.Stream : Xmpp.Xep.JingleRtp.Stream { + public uint8 rtpid { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + public Gst.Element rtpbin { get { + return plugin.rtpbin; + }} + public CodecUtil codec_util { get { + return plugin.codec_util; + }} + private Gst.App.Sink send_rtp; + private Gst.App.Sink send_rtcp; + private Gst.App.Src recv_rtp; + private Gst.App.Src recv_rtcp; + private Gst.Element encode; + private Gst.RTP.BasePayload encode_pay; + private Gst.Element decode; + private Gst.RTP.BaseDepayload decode_depay; + private Gst.Element input; + private Gst.Element output; + private Gst.Element session; + + private Device _input_device; + public Device input_device { get { return _input_device; } set { + if (!paused) { + if (this._input_device != null) { + this._input_device.unlink(); + this._input_device = null; + } + set_input(value != null ? value.link_source() : null); + } + this._input_device = value; + }} + private Device _output_device; + public Device output_device { get { return _output_device; } set { + if (output != null) remove_output(output); + if (value != null) add_output(value.link_sink()); + this._output_device = value; + }} + + public bool created { get; private set; default = false; } + public bool paused { get; private set; default = false; } + private bool push_recv_data = false; + private string participant_ssrc = null; + + private Gst.Pad recv_rtcp_sink_pad; + private Gst.Pad recv_rtp_sink_pad; + private Gst.Pad recv_rtp_src_pad; + private Gst.Pad send_rtcp_src_pad; + private Gst.Pad send_rtp_sink_pad; + private Gst.Pad send_rtp_src_pad; + + private Crypto.Srtp.Session? crypto_session = new Crypto.Srtp.Session(); + + public Stream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(content); + this.plugin = plugin; + this.rtpid = plugin.next_free_id(); + + content.notify["senders"].connect_after(on_senders_changed); + } + + public void on_senders_changed() { + if (sending && input == null) { + input_device = plugin.get_preferred_device(media, false); + } + if (receiving && output == null) { + output_device = plugin.get_preferred_device(media, true); + } + } + + public override void create() { + plugin.pause(); + + // Create i/o if needed + + if (input == null && input_device == null && sending) { + input_device = plugin.get_preferred_device(media, false); + } + if (output == null && output_device == null && receiving && media == "audio") { + output_device = plugin.get_preferred_device(media, true); + } + + // Create app elements + send_rtp = Gst.ElementFactory.make("appsink", @"rtp_sink_$rtpid") as Gst.App.Sink; + send_rtp.async = false; + send_rtp.caps = CodecUtil.get_caps(media, payload_type, false); + send_rtp.emit_signals = true; + send_rtp.sync = false; + send_rtp.new_sample.connect(on_new_sample); + pipe.add(send_rtp); + + send_rtcp = Gst.ElementFactory.make("appsink", @"rtcp_sink_$rtpid") as Gst.App.Sink; + send_rtcp.async = false; + send_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + send_rtcp.emit_signals = true; + send_rtcp.sync = false; + send_rtcp.new_sample.connect(on_new_sample); + pipe.add(send_rtcp); + + recv_rtp = Gst.ElementFactory.make("appsrc", @"rtp_src_$rtpid") as Gst.App.Src; + recv_rtp.caps = CodecUtil.get_caps(media, payload_type, true); + recv_rtp.do_timestamp = true; + recv_rtp.format = Gst.Format.TIME; + recv_rtp.is_live = true; + pipe.add(recv_rtp); + + recv_rtcp = Gst.ElementFactory.make("appsrc", @"rtcp_src_$rtpid") as Gst.App.Src; + recv_rtcp.caps = new Gst.Caps.empty_simple("application/x-rtcp"); + recv_rtcp.do_timestamp = true; + recv_rtcp.format = Gst.Format.TIME; + recv_rtcp.is_live = true; + pipe.add(recv_rtcp); + + // Connect RTCP + send_rtcp_src_pad = rtpbin.get_request_pad(@"send_rtcp_src_$rtpid"); + send_rtcp_src_pad.link(send_rtcp.get_static_pad("sink")); + recv_rtcp_sink_pad = rtpbin.get_request_pad(@"recv_rtcp_sink_$rtpid"); + recv_rtcp.get_static_pad("src").link(recv_rtcp_sink_pad); + + // Connect input + encode = codec_util.get_encode_bin(media, payload_type, @"encode_$rtpid"); + encode_pay = (Gst.RTP.BasePayload)((Gst.Bin)encode).get_by_name(@"encode_$(rtpid)_rtp_pay"); + pipe.add(encode); + send_rtp_sink_pad = rtpbin.get_request_pad(@"send_rtp_sink_$rtpid"); + encode.get_static_pad("src").link(send_rtp_sink_pad); + if (input != null) { + input.link(encode); + } + + // Connect output + decode = codec_util.get_decode_bin(media, payload_type, @"decode_$rtpid"); + decode_depay = (Gst.RTP.BaseDepayload)((Gst.Bin)encode).get_by_name(@"decode_$(rtpid)_rtp_depay"); + pipe.add(decode); + if (output != null) { + decode.link(output); + } + + // Connect RTP + recv_rtp_sink_pad = rtpbin.get_request_pad(@"recv_rtp_sink_$rtpid"); + recv_rtp.get_static_pad("src").link(recv_rtp_sink_pad); + + created = true; + push_recv_data = true; + plugin.unpause(); + + GLib.Signal.emit_by_name(rtpbin, "get-session", rtpid, out session); + if (session != null && payload_type.rtcp_fbs.any_match((it) => it.type_ == "goog-remb")) { + Object internal_session; + session.@get("internal-session", out internal_session); + if (internal_session != null) { + internal_session.connect("signal::on-feedback-rtcp", on_feedback_rtcp, this); + } + Timeout.add(1000, () => remb_adjust()); + } + if (media == "video") { + codec_util.update_bitrate(media, payload_type, encode, 256); + } + } + + private uint remb = 256; + private int last_packets_lost = -1; + private uint64 last_packets_received; + private uint64 last_octets_received; + private bool remb_adjust() { + unowned Gst.Structure? stats; + if (session == null) { + debug("Session for %u finished, turning off remb adjustment", rtpid); + return Source.REMOVE; + } + session.get("stats", out stats); + if (stats == null) { + warning("No stats for session %u", rtpid); + return Source.REMOVE; + } + unowned ValueArray? source_stats; + stats.get("source-stats", typeof(ValueArray), out source_stats); + if (source_stats == null) { + warning("No source-stats for session %u", rtpid); + return Source.REMOVE; + } + foreach (Value value in source_stats.values) { + unowned Gst.Structure source_stat = (Gst.Structure) value.get_boxed(); + uint ssrc; + if (!source_stat.get_uint("ssrc", out ssrc)) continue; + if (ssrc.to_string() == participant_ssrc) { + int packets_lost; + uint64 packets_received, octets_received; + source_stat.get_int("packets-lost", out packets_lost); + source_stat.get_uint64("packets-received", out packets_received); + source_stat.get_uint64("octets-received", out octets_received); + int new_lost = packets_lost - last_packets_lost; + uint64 new_received = packets_received - last_packets_received; + uint64 new_octets = octets_received - last_octets_received; + if (new_received == 0) continue; + last_packets_lost = packets_lost; + last_packets_received = packets_received; + last_octets_received = octets_received; + double loss_rate = (double)new_lost / (double)(new_lost + new_received); + if (new_lost <= 0 || loss_rate < 0.02) { + remb = (uint)(1.08 * (double)remb); + } else if (loss_rate > 0.1) { + remb = (uint)((1.0 - 0.5 * loss_rate) * (double)remb); + } + remb = uint.max(remb, (uint)((new_octets * 8) / 1000)); + remb = uint.max(16, remb); // Never go below 16 + uint8[] data = new uint8[] { + 143, 206, 0, 5, + 0, 0, 0, 0, + 0, 0, 0, 0, + 'R', 'E', 'M', 'B', + 1, 0, 0, 0, + 0, 0, 0, 0 + }; + data[4] = (uint8)((encode_pay.ssrc >> 24) & 0xff); + data[5] = (uint8)((encode_pay.ssrc >> 16) & 0xff); + data[6] = (uint8)((encode_pay.ssrc >> 8) & 0xff); + data[7] = (uint8)(encode_pay.ssrc & 0xff); + uint8 br_exp = 0; + uint32 br_mant = remb * 1000; + uint8 bits = (uint8)Math.log2(br_mant); + if (bits > 16) { + br_exp = (uint8)bits - 16; + br_mant = br_mant >> br_exp; + } + data[17] = (uint8)((br_exp << 2) | ((br_mant >> 16) & 0x3)); + data[18] = (uint8)((br_mant >> 8) & 0xff); + data[19] = (uint8)(br_mant & 0xff); + data[20] = (uint8)((ssrc >> 24) & 0xff); + data[21] = (uint8)((ssrc >> 16) & 0xff); + data[22] = (uint8)((ssrc >> 8) & 0xff); + data[23] = (uint8)(ssrc & 0xff); + encrypt_and_send_rtcp(data); + } + } + return Source.CONTINUE; + } + + private static void on_feedback_rtcp(Gst.Element session, uint type, uint fbtype, uint sender_ssrc, uint media_ssrc, Gst.Buffer? fci, Stream self) { + if (type == 206 && fbtype == 15 && fci != null && sender_ssrc.to_string() == self.participant_ssrc) { + // https://tools.ietf.org/html/draft-alvestrand-rmcat-remb-03 + uint8[] data; + fci.extract_dup(0, fci.get_size(), out data); + if (data[0] != 'R' || data[1] != 'E' || data[2] != 'M' || data[3] != 'B') return; + uint8 br_exp = data[5] >> 2; + uint32 br_mant = (((uint32)data[5] & 0x3) << 16) + ((uint32)data[6] << 8) + (uint32)data[7]; + uint bitrate = (br_mant << br_exp) / 1000; + self.codec_util.update_bitrate(self.media, self.payload_type, self.encode, bitrate * 8); + } + } + + private void prepare_local_crypto() { + if (local_crypto != null && local_crypto.is_valid && !crypto_session.has_encrypt) { + crypto_session.set_encryption_key(local_crypto.crypto_suite, local_crypto.key, local_crypto.salt); + debug("Setting up encryption with key params %s", local_crypto.key_params); + } + } + + private Gst.FlowReturn on_new_sample(Gst.App.Sink sink) { + if (sink == null) { + debug("Sink is null"); + return Gst.FlowReturn.EOS; + } + Gst.Sample sample = sink.pull_sample(); + Gst.Buffer buffer = sample.get_buffer(); + uint8[] data; + buffer.extract_dup(0, buffer.get_size(), out data); + prepare_local_crypto(); + if (sink == send_rtp) { + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtp(data); + } + on_send_rtp_data(new Bytes.take((owned) data)); + } else if (sink == send_rtcp) { + encrypt_and_send_rtcp((owned) data); + } else { + warning("unknown sample"); + } + return Gst.FlowReturn.OK; + } + + private void encrypt_and_send_rtcp(owned uint8[] data) { + if (crypto_session.has_encrypt) { + data = crypto_session.encrypt_rtcp(data); + } + if (rtcp_mux) { + on_send_rtp_data(new Bytes.take((owned) data)); + } else { + on_send_rtcp_data(new Bytes.take((owned) data)); + } + } + + private static Gst.PadProbeReturn drop_probe() { + return Gst.PadProbeReturn.DROP; + } + + public override void destroy() { + // Stop network communication + push_recv_data = false; + recv_rtp.end_of_stream(); + recv_rtcp.end_of_stream(); + send_rtp.new_sample.disconnect(on_new_sample); + send_rtcp.new_sample.disconnect(on_new_sample); + + // Disconnect input device + if (input != null) { + input.unlink(encode); + input = null; + } + if (this._input_device != null) { + if (!paused) this._input_device.unlink(); + this._input_device = null; + } + + // Disconnect encode + encode.set_locked_state(true); + encode.set_state(Gst.State.NULL); + encode.get_static_pad("src").unlink(send_rtp_sink_pad); + pipe.remove(encode); + encode = null; + encode_pay = null; + + // Disconnect RTP sending + if (send_rtp_src_pad != null) { + send_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + send_rtp_src_pad.unlink(send_rtp.get_static_pad("sink")); + } + send_rtp.set_locked_state(true); + send_rtp.set_state(Gst.State.NULL); + pipe.remove(send_rtp); + send_rtp = null; + + // Disconnect decode + if (recv_rtp_src_pad != null) { + recv_rtp_src_pad.add_probe(Gst.PadProbeType.BLOCK, drop_probe); + recv_rtp_src_pad.unlink(decode.get_static_pad("sink")); + } + + // Disconnect RTP receiving + recv_rtp.set_locked_state(true); + recv_rtp.set_state(Gst.State.NULL); + recv_rtp.get_static_pad("src").unlink(recv_rtp_sink_pad); + pipe.remove(recv_rtp); + recv_rtp = null; + + // Disconnect output + if (output != null) { + decode.unlink(output); + } + decode.set_locked_state(true); + decode.set_state(Gst.State.NULL); + pipe.remove(decode); + decode = null; + decode_depay = null; + output = null; + + // Disconnect output device + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + + // Disconnect RTCP receiving + recv_rtcp.get_static_pad("src").unlink(recv_rtcp_sink_pad); + recv_rtcp.set_locked_state(true); + recv_rtcp.set_state(Gst.State.NULL); + pipe.remove(recv_rtcp); + recv_rtcp = null; + + // Disconnect RTCP sending + send_rtcp_src_pad.unlink(send_rtcp.get_static_pad("sink")); + send_rtcp.set_locked_state(true); + send_rtcp.set_state(Gst.State.NULL); + pipe.remove(send_rtcp); + send_rtcp = null; + + // Release rtp pads + rtpbin.release_request_pad(send_rtp_sink_pad); + send_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtp_sink_pad); + recv_rtp_sink_pad = null; + rtpbin.release_request_pad(recv_rtcp_sink_pad); + recv_rtcp_sink_pad = null; + rtpbin.release_request_pad(send_rtcp_src_pad); + send_rtcp_src_pad = null; + send_rtp_src_pad = null; + recv_rtp_src_pad = null; + + session = null; + } + + private void prepare_remote_crypto() { + if (remote_crypto != null && remote_crypto.is_valid && !crypto_session.has_decrypt) { + crypto_session.set_decryption_key(remote_crypto.crypto_suite, remote_crypto.key, remote_crypto.salt); + debug("Setting up decryption with key params %s", remote_crypto.key_params); + } + } + + private uint16 previous_video_orientation_degree = uint16.MAX; + public signal void video_orientation_changed(uint16 degree); + + public override void on_recv_rtp_data(Bytes bytes) { + if (rtcp_mux && bytes.length >= 2 && bytes.get(1) >= 192 && bytes.get(1) < 224) { + on_recv_rtcp_data(bytes); + return; + } + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (crypto_session.has_decrypt) { + try { + data = crypto_session.decrypt_rtp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } + if (push_recv_data) { + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + Gst.RTP.Buffer rtp_buffer; + if (Gst.RTP.Buffer.map(buffer, Gst.MapFlags.READ, out rtp_buffer)) { + if (rtp_buffer.get_extension()) { + Xmpp.Xep.JingleRtp.HeaderExtension? ext = header_extensions.first_match((it) => it.uri == "urn:3gpp:video-orientation"); + if (ext != null) { + unowned uint8[] extension_data; + if (rtp_buffer.get_extension_onebyte_header(ext.id, 0, out extension_data) && extension_data.length == 1) { + bool camera = (extension_data[0] & 0x8) > 0; + bool flip = (extension_data[0] & 0x4) > 0; + uint8 rotation = extension_data[0] & 0x3; + uint16 rotation_degree = uint16.MAX; + switch(rotation) { + case 0: rotation_degree = 0; break; + case 1: rotation_degree = 90; break; + case 2: rotation_degree = 180; break; + case 3: rotation_degree = 270; break; + } + if (rotation_degree != previous_video_orientation_degree) { + video_orientation_changed(rotation_degree); + previous_video_orientation_degree = rotation_degree; + } + } + } + } + rtp_buffer.unmap(); + } + + // FIXME: VAPI file in Vala < 0.49.1 has a bug that results in broken ownership of buffer in push_buffer() + // We workaround by using the plain signal. The signal unfortunately will cause an unnecessary copy of + // the underlying buffer, so and some point we should move over to the new version (once we require + // Vala >= 0.50) +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtp, "push-buffer", buffer, out ret); +#endif + } + } + + public override void on_recv_rtcp_data(Bytes bytes) { + prepare_remote_crypto(); + uint8[] data = bytes.get_data(); + if (crypto_session.has_decrypt) { + try { + data = crypto_session.decrypt_rtcp(data); + } catch (Error e) { + warning("%s (%d)", e.message, e.code); + } + } + if (push_recv_data) { + Gst.Buffer buffer = new Gst.Buffer.wrapped((owned) data); + // See above +#if FIXED_APPSRC_PUSH_BUFFER_IN_VAPI + recv_rtcp.push_buffer((owned) buffer); +#else + Gst.FlowReturn ret; + GLib.Signal.emit_by_name(recv_rtcp, "push-buffer", buffer, out ret); +#endif + } + } + + public override void on_rtp_ready() { + // If full frame has been sent before the connection was ready, the counterpart would only display our video after the next full frame. + // Send a full frame to let the counterpart display our video asap + rtpbin.send_event(new Gst.Event.custom( + Gst.EventType.CUSTOM_UPSTREAM, + new Gst.Structure("GstForceKeyUnit", "all-headers", typeof(bool), true, null)) + ); + } + + public override void on_rtcp_ready() { + int rtp_session_id = (int) rtpid; + uint64 max_delay = int.MAX; + Object rtp_session; + bool rtp_sent; + GLib.Signal.emit_by_name(rtpbin, "get-internal-session", rtp_session_id, out rtp_session); + GLib.Signal.emit_by_name(rtp_session, "send-rtcp-full", max_delay, out rtp_sent); + debug("RTCP is ready, resending rtcp: %s", rtp_sent.to_string()); + } + + public void on_ssrc_pad_added(string ssrc, Gst.Pad pad) { + debug("New ssrc %s with pad %s", ssrc, pad.name); + if (participant_ssrc != null && participant_ssrc != ssrc) { + warning("Got second ssrc on stream (old: %s, new: %s), ignoring", participant_ssrc, ssrc); + return; + } + participant_ssrc = ssrc; + recv_rtp_src_pad = pad; + if (decode != null) { + plugin.pause(); + debug("Link %s to %s decode for %s", recv_rtp_src_pad.name, media, name); + recv_rtp_src_pad.link(decode.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void on_send_rtp_src_added(Gst.Pad pad) { + send_rtp_src_pad = pad; + if (send_rtp != null) { + plugin.pause(); + debug("Link %s to %s send_rtp for %s", send_rtp_src_pad.name, media, name); + send_rtp_src_pad.link(send_rtp.get_static_pad("sink")); + plugin.unpause(); + } + } + + public void set_input(Gst.Element? input) { + set_input_and_pause(input, paused); + } + + private void set_input_and_pause(Gst.Element? input, bool paused) { + if (created && this.input != null) { + this.input.unlink(encode); + this.input = null; + } + + this.input = input; + this.paused = paused; + + if (created && sending && !paused && input != null) { + plugin.pause(); + input.link(encode); + plugin.unpause(); + } + } + + public void pause() { + if (paused) return; + set_input_and_pause(null, true); + if (input_device != null) input_device.unlink(); + } + + public void unpause() { + if (!paused) return; + set_input_and_pause(input_device != null ? input_device.link_source() : null, false); + } + + ulong block_probe_handler_id = 0; + public virtual void add_output(Gst.Element element) { + if (output != null) { + critical("add_output() invoked more than once"); + return; + } + this.output = element; + if (created) { + plugin.pause(); + decode.link(element); + if (block_probe_handler_id != 0) { + decode.get_static_pad("src").remove_probe(block_probe_handler_id); + } + plugin.unpause(); + } + } + + public virtual void remove_output(Gst.Element element) { + if (output != element) { + critical("remove_output() invoked without prior add_output()"); + return; + } + if (created) { + block_probe_handler_id = decode.get_static_pad("src").add_probe(Gst.PadProbeType.BLOCK, drop_probe); + decode.unlink(element); + } + if (this._output_device != null) { + this._output_device.unlink(); + this._output_device = null; + } + this.output = null; + } +} + +public class Dino.Plugins.Rtp.VideoStream : Stream { + private Gee.List<Gst.Element> outputs = new ArrayList<Gst.Element>(); + private Gst.Element output_tee; + private Gst.Element rotate; + private ulong video_orientation_changed_handler; + + public VideoStream(Plugin plugin, Xmpp.Xep.Jingle.Content content) { + base(plugin, content); + if (media != "video") critical("VideoStream created for non-video media"); + } + + public override void create() { + video_orientation_changed_handler = video_orientation_changed.connect(on_video_orientation_changed); + plugin.pause(); + rotate = Gst.ElementFactory.make("videoflip", @"video_rotate_$rtpid"); + pipe.add(rotate); + output_tee = Gst.ElementFactory.make("tee", @"video_tee_$rtpid"); + output_tee.@set("allow-not-linked", true); + pipe.add(output_tee); + rotate.link(output_tee); + add_output(rotate); + base.create(); + foreach (Gst.Element output in outputs) { + output_tee.link(output); + } + plugin.unpause(); + } + + private void on_video_orientation_changed(uint16 degree) { + if (rotate != null) { + switch (degree) { + case 0: + rotate.@set("method", 0); + break; + case 90: + rotate.@set("method", 1); + break; + case 180: + rotate.@set("method", 2); + break; + case 270: + rotate.@set("method", 3); + break; + } + } + } + + public override void destroy() { + foreach (Gst.Element output in outputs) { + output_tee.unlink(output); + } + base.destroy(); + rotate.set_locked_state(true); + rotate.set_state(Gst.State.NULL); + rotate.unlink(output_tee); + pipe.remove(rotate); + rotate = null; + output_tee.set_locked_state(true); + output_tee.set_state(Gst.State.NULL); + pipe.remove(output_tee); + output_tee = null; + disconnect(video_orientation_changed_handler); + } + + public override void add_output(Gst.Element element) { + if (element == output_tee || element == rotate) { + base.add_output(element); + return; + } + outputs.add(element); + if (output_tee != null) { + output_tee.link(element); + } + } + + public override void remove_output(Gst.Element element) { + if (element == output_tee || element == rotate) { + base.remove_output(element); + return; + } + outputs.remove(element); + if (output_tee != null) { + output_tee.unlink(element); + } + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/video_widget.vala b/plugins/rtp/src/video_widget.vala new file mode 100644 index 00000000..351069a7 --- /dev/null +++ b/plugins/rtp/src/video_widget.vala @@ -0,0 +1,110 @@ +public class Dino.Plugins.Rtp.VideoWidget : Gtk.Bin, Dino.Plugins.VideoCallWidget { + private static uint last_id = 0; + + public uint id { get; private set; } + public Gst.Element element { get; private set; } + public Gtk.Widget widget { get; private set; } + + public Plugin plugin { get; private set; } + public Gst.Pipeline pipe { get { + return plugin.pipe; + }} + + private bool attached; + private Device? connected_device; + private Stream? connected_stream; + private Gst.Element convert; + + public VideoWidget(Plugin plugin) { + this.plugin = plugin; + + id = last_id++; + element = Gst.ElementFactory.make("gtksink", @"video_widget_$id"); + if (element != null) { + Gtk.Widget widget; + element.@get("widget", out widget); + element.@set("async", false); + element.@set("sync", false); + this.widget = widget; + add(widget); + widget.visible = true; + + // Listen for resolution changes + element.get_static_pad("sink").notify["caps"].connect(() => { + if (element.get_static_pad("sink").caps == null) return; + + int width, height; + element.get_static_pad("sink").caps.get_structure(0).get_int("width", out width); + element.get_static_pad("sink").caps.get_structure(0).get_int("height", out height); + resolution_changed(width, height); + }); + } else { + warning("Could not create GTK video sink. Won't display videos."); + } + } + + public void display_stream(Xmpp.Xep.JingleRtp.Stream stream) { + if (element == null) return; + detach(); + if (stream.media != "video") return; + connected_stream = stream as Stream; + if (connected_stream == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; + pipe.add(convert); + convert.link(element); + connected_stream.add_output(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void display_device(MediaDevice media_device) { + if (element == null) return; + detach(); + connected_device = media_device as Device; + if (connected_device == null) return; + plugin.pause(); + pipe.add(element); + convert = Gst.parse_bin_from_description(@"videoflip method=horizontal-flip name=video_widget_$(id)_flip ! videoconvert name=video_widget_$(id)_convert", true); + convert.name = @"video_widget_$(id)_prepare"; + pipe.add(convert); + convert.link(element); + connected_device.link_source().link(convert); + element.set_locked_state(false); + plugin.unpause(); + attached = true; + } + + public void detach() { + if (element == null) return; + if (attached) { + if (connected_stream != null) { + connected_stream.remove_output(convert); + connected_stream = null; + } + if (connected_device != null) { + connected_device.link_source().unlink(element); + connected_device.unlink(); // We get a new ref to recover the element, so unlink twice + connected_device.unlink(); + connected_device = null; + } + convert.set_locked_state(true); + convert.set_state(Gst.State.NULL); + pipe.remove(convert); + convert = null; + element.set_locked_state(true); + element.set_state(Gst.State.NULL); + pipe.remove(element); + attached = false; + } + } + + public override void dispose() { + detach(); + widget = null; + element = null; + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/voice_processor.vala b/plugins/rtp/src/voice_processor.vala new file mode 100644 index 00000000..66e95d72 --- /dev/null +++ b/plugins/rtp/src/voice_processor.vala @@ -0,0 +1,176 @@ +using Gst; + +namespace Dino.Plugins.Rtp { +public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf); +} + +public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + public signal void on_new_buffer(Buffer buffer); + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team <contact@dino.im>"); + } + + construct { + set_passthrough(true); + } + + public override bool setup(Audio.Info info) { + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + return true; + } + + + public override FlowReturn transform_ip(Buffer buf) { + lock (adapter) { + adapter.push(adjust_to_running_time(this, buf)); + while (adapter.available() > period_size) { + on_new_buffer(adapter.take_buffer(period_size)); + } + } + return FlowReturn.OK; + } + + public override bool stop() { + adapter.clear(); + return true; + } +} + +public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter { + private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}}; + public Audio.Info audio_info { get; private set; } + private ulong process_outgoing_buffer_handler_id; + private uint adjust_delay_timeout_id; + private uint period_samples; + private uint period_size; + private Base.Adapter adapter = new Base.Adapter(); + private EchoProbe? echo_probe; + private Audio.StreamVolume? stream_volume; + private ClockTime last_reverse; + private void* native; + + static construct { + add_static_pad_template(sink_template); + add_static_pad_template(src_template); + set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team <contact@dino.im>"); + } + + construct { + set_passthrough(false); + } + + public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) { + this.echo_probe = echo_probe; + this.stream_volume = stream_volume; + } + + private static extern void* init_native(int stream_delay); + private static extern void setup_native(void* native); + private static extern void destroy_native(void* native); + private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void process_stream(void* native, Audio.Info info, Buffer buffer); + private static extern void adjust_stream_delay(void* native); + private static extern void notify_gain_level(void* native, int gain_level); + private static extern int get_suggested_gain_level(void* native); + private static extern bool get_stream_has_voice(void* native); + + public override bool setup(Audio.Info info) { + debug("VoiceProcessor.setup(%s)", info.to_caps().to_string()); + audio_info = info; + period_samples = info.rate / 100; // 10ms buffers + period_size = period_samples * info.bpf; + adapter.clear(); + setup_native(native); + return true; + } + + public override bool start() { + native = init_native(150); + if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) { + process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer); + } + if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) { + stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume; + } + return true; + } + + private bool adjust_delay() { + if (native != null) { + adjust_stream_delay(native); + return Source.CONTINUE; + } else { + adjust_delay_timeout_id = 0; + return Source.REMOVE; + } + } + + private void process_outgoing_buffer(Buffer buffer) { + if (buffer.pts != uint64.MAX) { + last_reverse = buffer.pts; + } + analyze_reverse_stream(native, echo_probe.audio_info, buffer); + if (adjust_delay_timeout_id == 0 && echo_probe != null) { + adjust_delay_timeout_id = Timeout.add(1000, adjust_delay); + } + } + + public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) { + lock (adapter) { + if (is_discont) { + adapter.clear(); + } + adapter.push(adjust_to_running_time(this, input)); + } + return FlowReturn.OK; + } + + public override FlowReturn generate_output(out Buffer output_buffer) { + lock (adapter) { + if (adapter.available() >= period_size) { + output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable(); + int old_gain_level = 0; + if (stream_volume != null) { + old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0); + notify_gain_level(native, old_gain_level); + } + process_stream(native, audio_info, output_buffer); + if (stream_volume != null) { + int new_gain_level = get_suggested_gain_level(native); + if (old_gain_level != new_gain_level) { + debug("Gain: %i -> %i", old_gain_level, new_gain_level); + stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0); + } + } + } + } + return FlowReturn.OK; + } + + public override bool stop() { + if (process_outgoing_buffer_handler_id != 0) { + echo_probe.disconnect(process_outgoing_buffer_handler_id); + process_outgoing_buffer_handler_id = 0; + } + if (adjust_delay_timeout_id != 0) { + Source.remove(adjust_delay_timeout_id); + adjust_delay_timeout_id = 0; + } + adapter.clear(); + destroy_native(native); + native = null; + return true; + } +}
\ No newline at end of file diff --git a/plugins/rtp/src/voice_processor_native.cpp b/plugins/rtp/src/voice_processor_native.cpp new file mode 100644 index 00000000..8a052cf8 --- /dev/null +++ b/plugins/rtp/src/voice_processor_native.cpp @@ -0,0 +1,148 @@ +#include <algorithm> +#include <gst/gst.h> +#include <gst/audio/audio.h> +#include <webrtc/modules/audio_processing/include/audio_processing.h> +#include <webrtc/modules/interface/module_common_types.h> +#include <webrtc/system_wrappers/include/trace.h> + +#define SAMPLE_RATE 48000 +#define SAMPLE_CHANNELS 1 + +struct _DinoPluginsRtpVoiceProcessorNative { + webrtc::AudioProcessing *apm; + gint stream_delay; + gint last_median; + gint last_poor_delays; +}; + +extern "C" void *dino_plugins_rtp_adjust_to_running_time(GstBaseTransform *transform, GstBuffer *buffer) { + GstBuffer *copy = gst_buffer_copy(buffer); + GST_BUFFER_PTS(copy) = gst_segment_to_running_time(&transform->segment, GST_FORMAT_TIME, GST_BUFFER_PTS(buffer)); + return copy; +} + +extern "C" void *dino_plugins_rtp_voice_processor_init_native(gint stream_delay) { + _DinoPluginsRtpVoiceProcessorNative *native = new _DinoPluginsRtpVoiceProcessorNative(); + webrtc::Config config; + config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true)); + config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(true, 85)); + native->apm = webrtc::AudioProcessing::Create(config); + native->stream_delay = stream_delay; + native->last_median = 0; + native->last_poor_delays = 0; + return native; +} + +extern "C" void dino_plugins_rtp_voice_processor_setup_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + webrtc::ProcessingConfig pconfig; + pconfig.streams[webrtc::ProcessingConfig::kInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseInputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + pconfig.streams[webrtc::ProcessingConfig::kReverseOutputStream] = + webrtc::StreamConfig(SAMPLE_RATE, SAMPLE_CHANNELS, false); + apm->Initialize(pconfig); + apm->high_pass_filter()->Enable(true); + apm->echo_cancellation()->enable_drift_compensation(false); + apm->echo_cancellation()->set_suppression_level(webrtc::EchoCancellation::kModerateSuppression); + apm->echo_cancellation()->enable_delay_logging(true); + apm->echo_cancellation()->Enable(true); + apm->noise_suppression()->set_level(webrtc::NoiseSuppression::kModerate); + apm->noise_suppression()->Enable(true); + apm->gain_control()->set_analog_level_limits(0, 255); + apm->gain_control()->set_mode(webrtc::GainControl::kAdaptiveAnalog); + apm->gain_control()->set_target_level_dbfs(3); + apm->gain_control()->set_compression_gain_db(9); + apm->gain_control()->enable_limiter(true); + apm->gain_control()->Enable(true); + apm->voice_detection()->set_likelihood(webrtc::VoiceDetection::Likelihood::kLowLikelihood); + apm->voice_detection()->Enable(true); +} + +extern "C" void +dino_plugins_rtp_voice_processor_analyze_reverse_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READ); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = gst_buffer_get_size(buffer) / info->bpf; + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); + + int err = apm->AnalyzeReverseStream(&frame); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessReverseStream %i", err); + + gst_buffer_unmap(buffer, &map); +} + +extern "C" void dino_plugins_rtp_voice_processor_notify_gain_level(void *native_ptr, gint gain_level) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + apm->gain_control()->set_stream_analog_level(gain_level); +} + +extern "C" gint dino_plugins_rtp_voice_processor_get_suggested_gain_level(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->gain_control()->stream_analog_level(); +} + +extern "C" bool dino_plugins_rtp_voice_processor_get_stream_has_voice(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + return apm->voice_detection()->stream_has_voice(); +} + +extern "C" void dino_plugins_rtp_voice_processor_adjust_stream_delay(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::AudioProcessing *apm = native->apm; + int median, std, poor_delays; + float fraction_poor_delays; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, &fraction_poor_delays); + poor_delays = (int)(fraction_poor_delays * 100.0); + if (fraction_poor_delays < 0 || (native->last_median == median && native->last_poor_delays == poor_delays)) return; + g_debug("voice_processor_native.cpp: Stream delay metrics: median=%i std=%i poor_delays=%i%%", median, std, poor_delays); + native->last_median = median; + native->last_poor_delays = poor_delays; + if (poor_delays > 90) { + native->stream_delay = std::min(std::max(0, native->stream_delay + std::min(48, std::max(median, -48))), 384); + g_debug("voice_processor_native.cpp: set stream_delay=%i", native->stream_delay); + } +} + +extern "C" void +dino_plugins_rtp_voice_processor_process_stream(void *native_ptr, GstAudioInfo *info, GstBuffer *buffer) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + webrtc::StreamConfig config(SAMPLE_RATE, SAMPLE_CHANNELS, false); + webrtc::AudioProcessing *apm = native->apm; + + GstMapInfo map; + gst_buffer_map(buffer, &map, GST_MAP_READWRITE); + + webrtc::AudioFrame frame; + frame.num_channels_ = info->channels; + frame.sample_rate_hz_ = info->rate; + frame.samples_per_channel_ = info->rate / 100; + memcpy(frame.data_, map.data, frame.samples_per_channel_ * info->bpf); + + apm->set_stream_delay_ms(native->stream_delay); + int err = apm->ProcessStream(&frame); + if (err >= 0) memcpy(map.data, frame.data_, frame.samples_per_channel_ * info->bpf); + if (err < 0) g_warning("voice_processor_native.cpp: ProcessStream %i", err); + + gst_buffer_unmap(buffer, &map); +} + +extern "C" void dino_plugins_rtp_voice_processor_destroy_native(void *native_ptr) { + _DinoPluginsRtpVoiceProcessorNative *native = (_DinoPluginsRtpVoiceProcessorNative *) native_ptr; + delete native; +}
\ No newline at end of file diff --git a/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi new file mode 100644 index 00000000..30490896 --- /dev/null +++ b/plugins/rtp/vapi/gstreamer-rtp-1.0.vapi @@ -0,0 +1,625 @@ +// Fixme: This is fetched from development code of Vala upstream which fixed a few bugs. +/* gstreamer-rtp-1.0.vapi generated by vapigen, do not modify. */ + +[CCode (cprefix = "Gst", gir_namespace = "GstRtp", gir_version = "1.0", lower_case_cprefix = "gst_")] +namespace Gst { + namespace RTCP { + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public bool add_packet (Gst.RTCP.Type type, Gst.RTCP.Packet packet); + public bool get_first_packet (Gst.RTCP.Packet packet); + public uint get_packet_count (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTCP.Buffer rtcp); + public static Gst.Buffer @new (uint mtu); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] owned uint8[] data); + public bool unmap (); + public static bool validate (Gst.Buffer buffer); + public static bool validate_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_data_reduced ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + [Version (since = "1.6")] + public static bool validate_reduced (Gst.Buffer buffer); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTCPPacket")] + public struct Packet { + public weak Gst.RTCP.Buffer? rtcp; + public uint offset; + [Version (since = "1.10")] + public bool add_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint8[] data); + public bool add_rb (uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + [Version (since = "1.10")] + public uint8 app_get_data (); + [Version (since = "1.10")] + public uint16 app_get_data_length (); + [Version (since = "1.10")] + public unowned string app_get_name (); + [Version (since = "1.10")] + public uint32 app_get_ssrc (); + [Version (since = "1.10")] + public uint8 app_get_subtype (); + [Version (since = "1.10")] + public bool app_set_data_length (uint16 wordlen); + [Version (since = "1.10")] + public void app_set_name (string name); + [Version (since = "1.10")] + public void app_set_ssrc (uint32 ssrc); + [Version (since = "1.10")] + public void app_set_subtype (uint8 subtype); + public bool bye_add_ssrc (uint32 ssrc); + public bool bye_add_ssrcs ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] uint32[] ssrc); + public uint32 bye_get_nth_ssrc (uint nth); + public string bye_get_reason (); + public uint8 bye_get_reason_len (); + public uint bye_get_ssrc_count (); + public bool bye_set_reason (string reason); + [Version (since = "1.10")] + public bool copy_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out uint8[] data); + public uint8 fb_get_fci (); + public uint16 fb_get_fci_length (); + public uint32 fb_get_media_ssrc (); + public uint32 fb_get_sender_ssrc (); + public Gst.RTCP.FBType fb_get_type (); + public bool fb_set_fci_length (uint16 wordlen); + public void fb_set_media_ssrc (uint32 ssrc); + public void fb_set_sender_ssrc (uint32 ssrc); + public void fb_set_type (Gst.RTCP.FBType type); + public uint8 get_count (); + public uint16 get_length (); + public bool get_padding (); + [Version (since = "1.10")] + public bool get_profile_specific_ext ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.10")] + public uint16 get_profile_specific_ext_length (); + public void get_rb (uint nth, out uint32 ssrc, out uint8 fractionlost, out int32 packetslost, out uint32 exthighestseq, out uint32 jitter, out uint32 lsr, out uint32 dlsr); + public uint get_rb_count (); + public Gst.RTCP.Type get_type (); + public bool move_to_next (); + public bool remove (); + public uint32 rr_get_ssrc (); + public void rr_set_ssrc (uint32 ssrc); + public bool sdes_add_entry (Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] uint8[] data); + public bool sdes_add_item (uint32 ssrc); + public bool sdes_copy_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out uint8[] data); + public bool sdes_first_entry (); + public bool sdes_first_item (); + public bool sdes_get_entry (out Gst.RTCP.SDESType type, [CCode (array_length_cname = "len", array_length_pos = 1.5, array_length_type = "guint8")] out unowned uint8[] data); + public uint sdes_get_item_count (); + public uint32 sdes_get_ssrc (); + public bool sdes_next_entry (); + public bool sdes_next_item (); + public void set_rb (uint nth, uint32 ssrc, uint8 fractionlost, int32 packetslost, uint32 exthighestseq, uint32 jitter, uint32 lsr, uint32 dlsr); + public void sr_get_sender_info (out uint32 ssrc, out uint64 ntptime, out uint32 rtptime, out uint32 packet_count, out uint32 octet_count); + public void sr_set_sender_info (uint32 ssrc, uint64 ntptime, uint32 rtptime, uint32 packet_count, uint32 octet_count); + [Version (since = "1.16")] + public bool xr_first_rb (); + [Version (since = "1.16")] + public uint16 xr_get_block_length (); + [Version (since = "1.16")] + public Gst.RTCP.XRType xr_get_block_type (); + [Version (since = "1.16")] + public bool xr_get_dlrr_block (uint nth, out uint32 ssrc, out uint32 last_rr, out uint32 delay); + [Version (since = "1.16")] + public bool xr_get_prt_by_seq (uint16 seq, out uint32 receipt_time); + [Version (since = "1.16")] + public bool xr_get_prt_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_rle_info (out uint32 ssrc, out uint8 thinning, out uint16 begin_seq, out uint16 end_seq, out uint32 chunk_count); + [Version (since = "1.16")] + public bool xr_get_rle_nth_chunk (uint nth, out uint16 chunk); + [Version (since = "1.16")] + public bool xr_get_rrt (out uint64 timestamp); + [Version (since = "1.16")] + public uint32 xr_get_ssrc (); + [Version (since = "1.16")] + public bool xr_get_summary_info (out uint32 ssrc, out uint16 begin_seq, out uint16 end_seq); + [Version (since = "1.16")] + public bool xr_get_summary_jitter (out uint32 min_jitter, out uint32 max_jitter, out uint32 mean_jitter, out uint32 dev_jitter); + [Version (since = "1.16")] + public bool xr_get_summary_pkt (out uint32 lost_packets, out uint32 dup_packets); + [Version (since = "1.16")] + public bool xr_get_summary_ttl (out bool is_ipv4, out uint8 min_ttl, out uint8 max_ttl, out uint8 mean_ttl, out uint8 dev_ttl); + [Version (since = "1.16")] + public bool xr_get_voip_burst_metrics (out uint8 burst_density, out uint8 gap_density, out uint16 burst_duration, out uint16 gap_duration); + [Version (since = "1.16")] + public bool xr_get_voip_configuration_params (out uint8 gmin, out uint8 rx_config); + [Version (since = "1.16")] + public bool xr_get_voip_delay_metrics (out uint16 roundtrip_delay, out uint16 end_system_delay); + [Version (since = "1.16")] + public bool xr_get_voip_jitter_buffer_params (out uint16 jb_nominal, out uint16 jb_maximum, out uint16 jb_abs_max); + [Version (since = "1.16")] + public bool xr_get_voip_metrics_ssrc (out uint32 ssrc); + [Version (since = "1.16")] + public bool xr_get_voip_packet_metrics (out uint8 loss_rate, out uint8 discard_rate); + [Version (since = "1.16")] + public bool xr_get_voip_quality_metrics (out uint8 r_factor, out uint8 ext_r_factor, out uint8 mos_lq, out uint8 mos_cq); + [Version (since = "1.16")] + public bool xr_get_voip_signal_metrics (out uint8 signal_level, out uint8 noise_level, out uint8 rerl, out uint8 gmin); + [Version (since = "1.16")] + public bool xr_next_rb (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_", type_id = "gst_rtcpfb_type_get_type ()")] + [GIR (name = "RTCPFBType")] + public enum FBType { + FB_TYPE_INVALID, + RTPFB_TYPE_NACK, + RTPFB_TYPE_TMMBR, + RTPFB_TYPE_TMMBN, + RTPFB_TYPE_RTCP_SR_REQ, + RTPFB_TYPE_TWCC, + PSFB_TYPE_PLI, + PSFB_TYPE_SLI, + PSFB_TYPE_RPSI, + PSFB_TYPE_AFB, + PSFB_TYPE_FIR, + PSFB_TYPE_TSTR, + PSFB_TYPE_TSTN, + PSFB_TYPE_VBCN + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_SDES_", type_id = "gst_rtcpsdes_type_get_type ()")] + [GIR (name = "RTCPSDESType")] + public enum SDESType { + INVALID, + END, + CNAME, + NAME, + EMAIL, + PHONE, + LOC, + TOOL, + NOTE, + PRIV; + [CCode (cname = "gst_rtcp_sdes_name_to_type")] + public static Gst.RTCP.SDESType from_string (string name); + [CCode (cname = "gst_rtcp_sdes_type_to_name")] + public unowned string to_string (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_TYPE_", type_id = "gst_rtcp_type_get_type ()")] + [GIR (name = "RTCPType")] + public enum Type { + INVALID, + SR, + RR, + SDES, + BYE, + APP, + RTPFB, + PSFB, + XR + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTCP_XR_TYPE_", type_id = "gst_rtcpxr_type_get_type ()")] + [GIR (name = "RTCPXRType")] + [Version (since = "1.16")] + public enum XRType { + INVALID, + LRLE, + DRLE, + PRT, + RRT, + DLRR, + SSUMM, + VOIP_METRICS + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_BYE_SSRC_COUNT")] + public const int MAX_BYE_SSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_RB_COUNT")] + public const int MAX_RB_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES")] + public const int MAX_SDES; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_MAX_SDES_ITEM_COUNT")] + public const int MAX_SDES_ITEM_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_REDUCED_SIZE_VALID_MASK")] + public const int REDUCED_SIZE_VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_MASK")] + public const int VALID_MASK; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VALID_VALUE")] + public const int VALID_VALUE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTCP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 ntp_to_unix (uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static uint64 unix_to_ntp (uint64 unixtime); + } + namespace RTP { + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_audio_payload_get_type ()")] + [GIR (name = "RTPBaseAudioPayload")] + public class BaseAudioPayload : Gst.RTP.BasePayload { + public Gst.ClockTime base_ts; + public int frame_duration; + public int frame_size; + public int sample_size; + [CCode (has_construct_function = false)] + protected BaseAudioPayload (); + public Gst.FlowReturn flush (uint payload_len, Gst.ClockTime timestamp); + public Gst.Base.Adapter get_adapter (); + public Gst.FlowReturn push ([CCode (array_length_cname = "payload_len", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, Gst.ClockTime timestamp); + public void set_frame_based (); + public void set_frame_options (int frame_duration, int frame_size); + public void set_sample_based (); + public void set_sample_options (int sample_size); + public void set_samplebits_options (int sample_size); + [NoAccessorMethod] + public bool buffer_list { get; set; } + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_depayload_get_type ()")] + [GIR (name = "RTPBaseDepayload")] + public abstract class BaseDepayload : Gst.Element { + public uint clock_rate; + public bool need_newsegment; + public weak Gst.Segment segment; + public weak Gst.Pad sinkpad; + public weak Gst.Pad srcpad; + [CCode (has_construct_function = false)] + protected BaseDepayload (); + [NoWrapper] + public virtual bool handle_event (Gst.Event event); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + [NoWrapper] + public virtual bool packet_lost (Gst.Event event); + [NoWrapper] + public virtual Gst.Buffer process (Gst.Buffer @in); + [NoWrapper] + public virtual Gst.Buffer process_rtp_packet (Gst.RTP.Buffer rtp_buffer); + public Gst.FlowReturn push (Gst.Buffer out_buf); + public Gst.FlowReturn push_list (Gst.BufferList out_list); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public int max_reorder { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string? ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_base_payload_get_type ()")] + [GIR (name = "RTPBasePayload")] + public abstract class BasePayload : Gst.Element { + [CCode (has_construct_function = false)] + protected BasePayload (); + [Version (since = "1.16")] + public Gst.Buffer allocate_output_buffer (uint payload_len, uint8 pad_len, uint8 csrc_count); + [NoWrapper] + public virtual Gst.Caps get_caps (Gst.Pad pad, Gst.Caps filter); + [Version (since = "1.16")] + public uint get_source_count (Gst.Buffer buffer); + [NoWrapper] + public virtual Gst.FlowReturn handle_buffer (Gst.Buffer buffer); + public bool is_filled (uint size, Gst.ClockTime duration); + [Version (since = "1.16")] + public bool is_source_info_enabled (); + public Gst.FlowReturn push (Gst.Buffer buffer); + public Gst.FlowReturn push_list (Gst.BufferList list); + [NoWrapper] + public virtual bool query (Gst.Pad pad, Gst.Query query); + [NoWrapper] + public virtual bool set_caps (Gst.Caps caps); + public void set_options (string media, bool @dynamic, string encoding_name, uint32 clock_rate); + [Version (since = "1.20")] + public bool set_outcaps_structure (Gst.Structure? s); + [Version (since = "1.16")] + public void set_source_info_enabled (bool enable); + [NoWrapper] + public virtual bool sink_event (Gst.Event event); + [NoWrapper] + public virtual bool src_event (Gst.Event event); + [NoAccessorMethod] + [Version (since = "1.20")] + public bool auto_header_extension { get; set; } + [NoAccessorMethod] + public int64 max_ptime { get; set; } + [NoAccessorMethod] + public int64 min_ptime { get; set; } + [NoAccessorMethod] + public uint mtu { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool onvif_no_rate_control { get; set; } + [NoAccessorMethod] + public bool perfect_rtptime { get; set; } + [NoAccessorMethod] + public uint pt { get; set; } + [NoAccessorMethod] + public int64 ptime_multiple { get; set; } + [NoAccessorMethod] + [Version (since = "1.18")] + public bool scale_rtptime { get; set; } + [NoAccessorMethod] + public uint seqnum { get; } + [NoAccessorMethod] + public int seqnum_offset { get; set; } + [NoAccessorMethod] + [Version (since = "1.16")] + public bool source_info { get; set; } + [NoAccessorMethod] + public uint ssrc { get; set; } + [NoAccessorMethod] + public Gst.Structure stats { owned get; } + [NoAccessorMethod] + public uint timestamp { get; } + [NoAccessorMethod] + public uint timestamp_offset { get; set; } + [Version (since = "1.20")] + public signal void add_extension (owned Gst.RTP.HeaderExtension ext); + [Version (since = "1.20")] + public signal void clear_extensions (); + [Version (since = "1.20")] + public signal Gst.RTP.HeaderExtension request_extension (uint ext_id, string ext_uri); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", type_id = "gst_rtp_header_extension_get_type ()")] + [GIR (name = "RTPHeaderExtension")] + [Version (since = "1.20")] + public abstract class HeaderExtension : Gst.Element { + public uint ext_id; + [CCode (has_construct_function = false)] + protected HeaderExtension (); + public static Gst.RTP.HeaderExtension? create_from_uri (string uri); + public uint get_id (); + public virtual size_t get_max_size (Gst.Buffer input_meta); + public string get_sdp_caps_field_name (); + public virtual Gst.RTP.HeaderExtensionFlags get_supported_flags (); + public unowned string get_uri (); + public virtual bool read (Gst.RTP.HeaderExtensionFlags read_flags, [CCode (array_length_cname = "size", array_length_pos = 2.5, array_length_type = "gsize", type = "const guint8*")] uint8[] data, Gst.Buffer buffer); + public virtual bool set_attributes_from_caps (Gst.Caps caps); + public bool set_attributes_from_caps_simple_sdp (Gst.Caps caps); + public virtual bool set_caps_from_attributes (Gst.Caps caps); + public bool set_caps_from_attributes_simple_sdp (Gst.Caps caps); + public void set_id (uint ext_id); + public virtual bool set_non_rtp_sink_caps (Gst.Caps caps); + [CCode (cname = "gst_rtp_header_extension_class_set_uri")] + public class void set_uri (string uri); + public void set_wants_update_non_rtp_src_caps (bool state); + public virtual bool update_non_rtp_src_caps (Gst.Caps caps); + public virtual size_t write (Gst.Buffer input_meta, Gst.RTP.HeaderExtensionFlags write_flags, Gst.Buffer output, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "gsize", type = "guint8*")] uint8[] data); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPBuffer")] + public struct Buffer { + public weak Gst.Buffer buffer; + public uint state; + [CCode (array_length = false)] + public weak void* data[4]; + [CCode (array_length = false)] + public weak size_t size[4]; + public bool add_extension_onebyte_header (uint8 id, [CCode (array_length_cname = "size", array_length_pos = 2.1, array_length_type = "guint")] uint8[] data); + public bool add_extension_twobytes_header (uint8 appbits, uint8 id, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] uint8[] data); + [CCode (cname = "gst_buffer_add_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? add_rtp_source_meta (Gst.Buffer buffer, uint32? ssrc, uint32? csrc, uint csrc_count); + public static void allocate_data (Gst.Buffer buffer, uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_header_len (uint8 csrc_count); + public static uint calc_packet_len (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static uint calc_payload_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static int compare_seqnum (uint16 seqnum1, uint16 seqnum2); + public static uint32 default_clock_rate (uint8 payload_type); + public static uint64 ext_timestamp (ref uint64 exttimestamp, uint32 timestamp); + public uint32 get_csrc (uint8 idx); + public uint8 get_csrc_count (); + public bool get_extension (); + [Version (since = "1.2")] + public GLib.Bytes get_extension_bytes (out uint16 bits); + public bool get_extension_data (out uint16 bits, [CCode (array_length = false)] out unowned uint8[] data, out uint wordlen); + public bool get_extension_onebyte_header (uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 3.1, array_length_type = "guint")] out unowned uint8[] data); + [Version (since = "1.18")] + public static bool get_extension_onebyte_header_from_bytes (GLib.Bytes bytes, uint16 bit_pattern, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 5.1, array_length_type = "guint")] out unowned uint8[] data); + public bool get_extension_twobytes_header (out uint8 appbits, uint8 id, uint nth, [CCode (array_length_cname = "size", array_length_pos = 4.1, array_length_type = "guint")] out unowned uint8[] data); + public uint get_header_len (); + public bool get_marker (); + public uint get_packet_len (); + public bool get_padding (); + [CCode (array_length = false)] + public unowned uint8[] get_payload (); + public Gst.Buffer get_payload_buffer (); + [Version (since = "1.2")] + public GLib.Bytes get_payload_bytes (); + public uint get_payload_len (); + public Gst.Buffer get_payload_subbuffer (uint offset, uint len); + public uint8 get_payload_type (); + [CCode (cname = "gst_buffer_get_rtp_source_meta")] + [Version (since = "1.16")] + public static unowned Gst.RTP.SourceMeta? get_rtp_source_meta (Gst.Buffer buffer); + public uint16 get_seq (); + public uint32 get_ssrc (); + public uint32 get_timestamp (); + public uint8 get_version (); + public static bool map (Gst.Buffer buffer, Gst.MapFlags flags, out Gst.RTP.Buffer rtp); + public static Gst.Buffer new_allocate (uint payload_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_allocate_len (uint packet_len, uint8 pad_len, uint8 csrc_count); + public static Gst.Buffer new_copy_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] uint8[] data); + public static Gst.Buffer new_take_data ([CCode (array_length_cname = "len", array_length_pos = 1.1, array_length_type = "gsize")] owned uint8[] data); + public void pad_to (uint len); + public void set_csrc (uint8 idx, uint32 csrc); + public void set_extension (bool extension); + public bool set_extension_data (uint16 bits, uint16 length); + public void set_marker (bool marker); + public void set_packet_len (uint len); + public void set_padding (bool padding); + public void set_payload_type (uint8 payload_type); + public void set_seq (uint16 seq); + public void set_ssrc (uint32 ssrc); + public void set_timestamp (uint32 timestamp); + public void set_version (uint8 version); + public void unmap (); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPPayloadInfo")] + public struct PayloadInfo { + public uint8 payload_type; + public weak string media; + public weak string encoding_name; + public uint clock_rate; + public weak string encoding_parameters; + public uint bitrate; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", has_type_id = false)] + [GIR (name = "RTPSourceMeta")] + [Version (since = "1.16")] + public struct SourceMeta { + public Gst.Meta meta; + public uint32 ssrc; + public bool ssrc_valid; + [CCode (array_length = false)] + public weak uint32 csrc[15]; + public uint csrc_count; + public bool append_csrc ([CCode (array_length_cname = "csrc_count", array_length_pos = 1.1, array_length_type = "guint", type = "const guint32*")] uint32[] csrc); + public uint get_source_count (); + public bool set_ssrc (uint32? ssrc); + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_FLAG_", type_id = "gst_rtp_buffer_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferFlags")] + [Version (since = "1.10")] + public enum BufferFlags { + RETRANSMISSION, + REDUNDANT, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_BUFFER_MAP_FLAG_", type_id = "gst_rtp_buffer_map_flags_get_type ()")] + [Flags] + [GIR (name = "RTPBufferMapFlags")] + [Version (since = "1.6.1")] + public enum BufferMapFlags { + SKIP_PADDING, + LAST + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_HEADER_EXTENSION_", type_id = "gst_rtp_header_extension_flags_get_type ()")] + [Flags] + [GIR (name = "RTPHeaderExtensionFlags")] + [Version (since = "1.20")] + public enum HeaderExtensionFlags { + ONE_BYTE, + TWO_BYTE + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PAYLOAD_", type_id = "gst_rtp_payload_get_type ()")] + [GIR (name = "RTPPayload")] + public enum Payload { + PCMU, + @1016, + G721, + GSM, + G723, + DVI4_8000, + DVI4_16000, + LPC, + PCMA, + G722, + L16_STEREO, + L16_MONO, + QCELP, + CN, + MPA, + G728, + DVI4_11025, + DVI4_22050, + G729, + CELLB, + JPEG, + NV, + H261, + MPV, + MP2T, + H263; + public const string @1016_STRING; + public const string CELLB_STRING; + public const string CN_STRING; + public const string DVI4_11025_STRING; + public const string DVI4_16000_STRING; + public const string DVI4_22050_STRING; + public const string DVI4_8000_STRING; + public const string DYNAMIC_STRING; + public const string G721_STRING; + public const string G722_STRING; + public const int G723_53; + public const string G723_53_STRING; + public const int G723_63; + public const string G723_63_STRING; + public const string G723_STRING; + public const string G728_STRING; + public const string G729_STRING; + public const string GSM_STRING; + public const string H261_STRING; + public const string H263_STRING; + public const string JPEG_STRING; + public const string L16_MONO_STRING; + public const string L16_STEREO_STRING; + public const string LPC_STRING; + public const string MP2T_STRING; + public const string MPA_STRING; + public const string MPV_STRING; + public const string NV_STRING; + public const string PCMA_STRING; + public const string PCMU_STRING; + public const string QCELP_STRING; + public const int TS41; + public const string TS41_STRING; + public const int TS48; + public const string TS48_STRING; + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cprefix = "GST_RTP_PROFILE_", type_id = "gst_rtp_profile_get_type ()")] + [GIR (name = "RTPProfile")] + [Version (since = "1.6")] + public enum Profile { + UNKNOWN, + AVP, + SAVP, + AVPF, + SAVPF + } + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_BASE")] + public const string HDREXT_BASE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_ELEMENT_CLASS")] + [Version (since = "1.20")] + public const string HDREXT_ELEMENT_CLASS; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56")] + public const string HDREXT_NTP_56; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_56_SIZE")] + public const int HDREXT_NTP_56_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64")] + public const string HDREXT_NTP_64; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HDREXT_NTP_64_SIZE")] + public const int HDREXT_NTP_64_SIZE; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_HEADER_EXTENSION_URI_METADATA_KEY")] + [Version (since = "1.20")] + public const string HEADER_EXTENSION_URI_METADATA_KEY; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_SOURCE_META_MAX_CSRC_COUNT")] + public const int SOURCE_META_MAX_CSRC_COUNT; + [CCode (cheader_filename = "gst/rtp/rtp.h", cname = "GST_RTP_VERSION")] + public const int VERSION; + [CCode (cheader_filename = "gst/rtp/rtp.h")] + [Version (since = "1.20")] + public static GLib.List<Gst.RTP.HeaderExtension> get_header_extension_list (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_56 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_get_ntp_64 ([CCode (array_length_cname = "size", array_length_pos = 1.5, array_length_type = "guint")] uint8[] data, out uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_56 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static bool hdrext_set_ntp_64 (void* data, uint size, uint64 ntptime); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_name (string media, string encoding_name); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.RTP.PayloadInfo? payload_info_for_pt (uint8 payload_type); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static GLib.Type source_meta_api_get_type (); + [CCode (cheader_filename = "gst/rtp/rtp.h")] + public static unowned Gst.MetaInfo? source_meta_get_info (); + } +} |