Unable to get the sample from Appsink using C++
RK29
giri_2984 at yahoo.co.in
Fri Aug 21 04:16:10 UTC 2020
This is my C# working. Just trying to do the same in C++
using System;
using static System.Diagnostics.Debug;
using System.Threading;
using Gst;
using WebSocketSharp;
using Gst.WebRTC;
using Newtonsoft.Json;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using Gst.Sdp;
using System.Text;
using GLib;
using Gst.App;
namespace GstSamples
{
//
https://github.com/centricular/gstwebrtc-demos/blob/master/sendrecv/gst/webrtc-sendrecv.py
class WebRtcClient : IDisposable
{
// const string SERVER = "wss://127.0.0.1:8443";
const string SERVER = "wss://webrtc.nirbheek.in:8443";
public delegate void NewFrameHandler(WebRtcClient sender,
IGstVideoFrameContext frameContext);
const string PIPELINE_DESC0 = @"webrtcbin name=sendrecv
videotestsrc pattern=ball ! videoconvert ! queue ! vp8enc deadline=1 !
rtpvp8pay !
queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 !
sendrecv.
audiotestsrc wave=red-noise ! audioconvert ! audioresample ! queue !
opusenc ! rtpopuspay !
queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 !
sendrecv.";
const string PIPELINE_DESC = @"videotestsrc ! queue ! vp8enc !
rtpvp8pay !
application/x-rtp,media=video,encoding-name=VP8,payload=96 ! webrtcbin name
= sendrecv";
readonly int _id;
readonly int _peerId;
readonly string _server;
readonly WebSocket _conn;
Pipeline _pipe;
Element _webRtc;
private AppSink sink = null;
private Timer _renderTimer = null;
private Timer _messageTimer = null;
private long _sampleLock = 0;
private const int _renderTimerFrequency = 60;
private const int _messageTimerFrequency = 30;
public bool IsSynchronized { get; private set; }
public event NewFrameHandler NewFrame;
public WebRtcClient(int id, int peerId, string server = SERVER)
{
_id = id;
_peerId = peerId;
_server = server;
_conn = new WebSocket(_server);
_conn.SslConfiguration.ServerCertificateValidationCallback =
validatCert;
// _conn.WaitTime = new TimeSpan(0, 1, 0);
_conn.OnOpen += OnOpen;
_conn.OnError += OnError;
_conn.OnMessage += OnMessage;
_conn.OnClose += OnClose;
GLib.GType.Register(Promise.GType, typeof(Promise));
_pipe = (Pipeline)Parse.Launch(PIPELINE_DESC);
}
bool validatCert(object sender, X509Certificate certificate,
X509Chain chain, SslPolicyErrors sslPolicyErrors)
{
return true;
}
public void Connect()
{
_conn.ConnectAsync();
}
void SetupCall()
{
_conn.Send($"SESSION {_peerId}");
}
void OnClose(object sender, CloseEventArgs e)
{
Console.WriteLine("Closed: " + e.Reason);
}
void OnError(object sender, ErrorEventArgs e)
{
Console.WriteLine("Error " + e.Message);
}
void OnOpen(object sender, EventArgs e)
{
var ws = sender as WebSocket;
ws.SendAsync($"HELLO {_id}", (b) => Console.WriteLine($"Opened
{b}"));
}
void OnMessage(object sender, MessageEventArgs args)
{
var msg = args.Data;
switch (msg)
{
case "HELLO":
SetupCall();
break;
case "SESSION_OK":
StartPipeline();
break;
case var m when m.StartsWith("ERROR"):
msg.PrintErr();
Environment.Exit(1);
break;
default:
HandleSdp(msg);
break;
}
}
void StartPipeline()
{
_webRtc = _pipe.GetByName("sendrecv");
Assert(_webRtc != null);
_webRtc.Connect("on-negotiation-needed", OnNegotiationNeeded);
_webRtc.Connect("on-ice-candidate", OnIceCandidate);
_webRtc.Connect("pad-added", OnIncomingStream);
_pipe.SetState(State.Playing);
Console.WriteLine("Playing");
}
#region Webrtc signal handlers
#region Incoming stream
void OnIncomingStream(object o, GLib.SignalArgs args)
{
var pad = args.Args[0] as Pad;
if (pad.Direction != PadDirection.Src)
return;
var decodebin = ElementFactory.Make("decodebin");
decodebin.Connect("pad-added", OnIncomingDecodebinStream);
_pipe.Add(decodebin);
decodebin.SyncStateWithParent();
_webRtc.Link(decodebin);
}
void OnIncomingDecodebinStream(object o, SignalArgs args)
{
var pad = (Pad)args.Args[0];
if (!pad.HasCurrentCaps)
{
Console.WriteLine($"{pad.Name} has no caps, ignoring");
return;
}
var caps = pad.CurrentCaps;
Assert(!caps.IsEmpty);
Structure s = caps[0];
var name = s.Name;
if (name.StartsWith("video"))
{
var q = ElementFactory.Make("queue");
var conv = ElementFactory.Make("videoconvert");
var scale = ElementFactory.Make("videoscale");
var capsf = ElementFactory.Make("capsfilter");
// this.sink = ElementFactory.Make("autovideosink") as
AppSink;
this.sink = new AppSink("autovideosink");
capsf["caps"] = Caps.FromString("video/x-raw,format=BGR,
width= 1280, height=720");
this.sink.Drop = true;
this.sink.Qos = true;
this.sink.Sync = IsSynchronized;
//var sink = ElementFactory.Make("autovideosink");
//var sink = _pipe. as AppSink;
// var sink = ElementFactory.Make("appsink") as AppSink;
// var sink = new AppSink("videoSink");
// var sink = ElementFactory.Make("autovideosink");
sink.NewSample += NewVideoSample;
_pipe.Add(q, conv,scale,capsf,sink);
_pipe.SyncChildrenStates();
pad.Link(q.GetStaticPad("sink"));
Element.Link(q, conv, scale, capsf,sink);
_renderTimer = new Timer(RenderTimerProc, this, 0, 1000 /
_renderTimerFrequency);
_messageTimer = new Timer(MessageTimerProc, this, 0, 1000 /
_messageTimerFrequency);
}
else if (name.StartsWith("audio"))
{
var q = ElementFactory.Make("queue");
var conv = ElementFactory.Make("audioconvert");
var resample = ElementFactory.Make("audioresample");
var sink = ElementFactory.Make("autoaudiosink");
_pipe.Add(q, conv, resample, sink);
_pipe.SyncChildrenStates();
pad.Link(q.GetStaticPad("sink"));
Element.Link(q, conv, resample, sink);
}
}
#endregion
static void NewVideoSample(object sender, NewSampleArgs args)
{
var sink = (Gst.App.AppSink)sender;
// Retrieve the buffer
var sample = sink.PullSample();
if (sample != null)
{
Caps caps = sample.Caps;
var cap = caps[0];
string format;
int width = 0;
int height = 0;
int stride = 0;
int offset = 0;
int fpsNumerator = 0;
int fpsDenominator = 1;
format = cap.GetString("format");
cap.GetInt("width", out width);
cap.GetInt("height", out height);
cap.GetInt("stride", out stride);
cap.GetInt("offset", out offset);
cap.GetFraction("framerate", out fpsNumerator, out
fpsDenominator);
MapInfo map;
if (sample.Buffer.Map(out map, MapFlags.Read))
{
// TODO: work with your RGBA frame in map.Data or map
DataPtr or use map.CopyTo(IntPtr, long) to copy raw memory
var test = sample.Buffer.Handle;
//var off = sample.Buffer.Offset;
sample.Buffer.Offset = 1;
// var test1 =
sample.Buffer.AllMemory.Handle;//.OwnedCopy;
// var test1 = sample.Buffer.Pool.Handle;
// sample.Buffer.AllMemory.Align = 0;
//Bitmap videoBuffer = new Bitmap(width,height,3840,
PixelFormat.Format24bppRgb,test1);
//BitmapData imageLock = videoBuffer.LockBits(new
Rectangle(0, 0, videoBuffer.Width, videoBuffer.Height),
ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);
//DriverInterface.SetData(imageLock.Scan0, 3840, width,
height);
//var stride = Gst. GST_ROUND_UP_4(width * 3);
//var st = roundToMultiple(width * 3, 4);
//DriverInterface.SetData(map.DataPtr, (int)st, width,
height);
sample.Buffer.Unmap(map);
}
sample.Dispose();
}
}
private void PullAndProcessVideoSample()
{
if (sink != null)
{
Sample sample1 = sink.TryPullSample(0);
//if (sample != null)
//{
// using (sample)
// {
// using (var context = new
GstVideoFrameContext(sample))
// {
// NewFrame?.Invoke(this, context);
// }
// }
//}
// Retrieve the buffer
var sample = sink.PullSample();
if (sample != null)
{
Caps caps = sample.Caps;
var cap = caps[0];
string format;
int width = 0;
int height = 0;
int fpsNumerator = 0;
int fpsDenominator = 1;
format = cap.GetString("format");
cap.GetInt("width", out width);
cap.GetInt("height", out height);
cap.GetFraction("framerate", out fpsNumerator, out
fpsDenominator);
MapInfo map;
if (sample.Buffer.Map(out map, MapFlags.Read))
{
// TODO: work with your RGBA frame in map.Data or map
DataPtr or use map.CopyTo(IntPtr, long) to copy raw memory
var test = sample.Buffer.Handle;
sample.Buffer.Unmap(map);
}
sample.Dispose();
}
}
}
private void MessageTimerProc(object _)
{
if (_pipe != null)
{
using (var bus = _pipe.Bus)
{
var message = bus.Poll(MessageType.Any, 0);
if (message != null)
{
OnNewMessage(message);
message.Dispose();
}
}
}
}
private void OnNewMessage(Gst.Message message)
{
switch (message.Type)
{
case MessageType.Error:
GLib.GException ex;
string debug;
message.ParseError(out ex, out debug);
// Error?.Invoke(this, ex, debug);
break;
case MessageType.Eos:
// EndOfStream?.Invoke(this);
break;
case MessageType.StateChanged:
State oldState, newState, pendingState;
message.ParseStateChanged(out oldState, out newState,
out pendingState);
// StateChanged?.Invoke(this, oldState, newState,
pendingState);
break;
}
// Message?.Invoke(this, message);
}
void OnIceCandidate(object o, GLib.SignalArgs args)
{
var index = (uint)args.Args[0];
var cand = (string)args.Args[1];
var obj = new { ice = new { sdpMLineIndex = index, candidate =
cand } };
var iceMsg = JsonConvert.SerializeObject(obj);
_conn.SendAsync(iceMsg, (b) => { } );
}
private void RenderTimerProc(object _)
{
if (System.Threading.Interlocked.CompareExchange(ref
_sampleLock, 1, 0) == 0)
{
try
{
if (_pipe != null && sink != null)
{
PullAndProcessVideoSample();
}
}
finally
{
System.Threading.Interlocked.Decrement(ref _sampleLock);
}
}
}
void OnNegotiationNeeded(object o, GLib.SignalArgs args)
{
var webRtc = o as Element;
Assert(webRtc != null, "not a webrtc object");
Promise promise = new Promise(OnOfferCreated, _webRtc.Handle,
null); // webRtc.Handle, null);
Structure structure = new Structure("struct");
_webRtc.Emit("create-offer", structure, promise);
}
void OnOfferCreated(Promise promise)
{
var res = promise.Wait();
var reply = promise.RetrieveReply();
var gval = reply.GetValue("offer");
WebRTCSessionDescription offer =
(WebRTCSessionDescription)gval.Val;
promise = new Promise();
_webRtc.Emit("set-local-description", offer, promise);
promise.Interrupt();
SendSdpOffer(offer) ;
}
#endregion
void SendSdpOffer(WebRTCSessionDescription offer)
{
var text = offer.Sdp.AsText();
var obj = new { sdp = new { type = "offer", sdp = text } };
var json = JsonConvert.SerializeObject(obj);
json.PrintYellow();
_conn.SendAsync(json, (b) => Console.WriteLine($"Send offer
completed {b}"));
}
void HandleSdp(string message)
{
var msg = JsonConvert.DeserializeObject<dynamic>(message);
if (msg.sdp != null)
{
var sdp = msg.sdp;
if (sdp.type != null && sdp.type != "answer")
{
throw new Exception("Not an answer");
}
string sdpAns = sdp.sdp;
Console.WriteLine($"received answer:\n{sdpAns}");
var res = SDPMessage.New(out SDPMessage sdpMsg);
SDPMessage.ParseBuffer(ASCIIEncoding.Default.GetBytes(sdpAns),
(uint)sdpAns.Length, sdpMsg);
var answer =
WebRTCSessionDescription.New(WebRTCSDPType.Answer, sdpMsg);
var promise = new Promise();
_webRtc.Emit("set-remote-description", answer, promise);
}
else if (msg.ice != null)
{
var ice = msg.ice;
string candidate = ice.candidate;
uint sdpMLineIndex = ice.sdpMLineIndex;
_webRtc.Emit("add-ice-candidate", sdpMLineIndex, candidate);
}
}
public void Dispose()
{
((IDisposable)_conn).Dispose();
_pipe.SetState(State.Null);
_pipe.Dispose();
}
}
public interface IGstVideoFrameContext
{
/// Frame Width
int Width { get; }
/// Frame Height
int Height { get; }
/// Frame Format
string Format { get; }
/// Frame stride size (size in bytes of a single scan-line)
long Stride { get; }
/// Frame memory size in bytes
long Size { get; }
/// Frame raw data buffer
IntPtr Buffer { get; }
/// Copies frame's raw data to an unmanaged memory pointer
destination
void CopyTo(IntPtr destination, long destinationSizeInBytes = 0);
}
static class WebRtcSendRcv
{
const string SERVER = "wss://webrtc.nirbheek.in:8443";
//const string SERVER = "wss://127.0.0.1:8443";
static Random random = new Random();
public static void Run(string[] args)
{
//if (args.Length == 0)
// throw new Exception("need peerId");
// int peerId = Int32.Parse(args[0]);
int peerId = 1996;
var server = (args.Length > 1) ? args[1] : SERVER;
GtkSharp.GstreamerSharp.ObjectManager.Initialize();
GLib.GType.Register(WebRTCSessionDescription.GType,
typeof(WebRTCSessionDescription));
Gst.Application.Init();
GLib.GType.Register(WebRTCSessionDescription.GType,
typeof(WebRTCSessionDescription));
var ourId = random.Next(100, 10000);
Console.WriteLine($"PeerId:{peerId} OurId:{ourId} ");
var c = new WebRtcClient(ourId, peerId, server);
c.Connect();
Console.ReadKey();
c.Dispose();
}
}
}
--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
More information about the gstreamer-devel
mailing list