Python gstreamer/webrtc answer with browser offer

Thu Oct 12 13:29:30 UTC 2023

Hi, I've been trying to get the code below to work. In the past I've been
able to have my python code make an offer and the browser answer, but now I
want to make the relationship the other way for a couple of reasons

import boto3
import websocket
import ssl
import threading
import queue
import enum
import gi
import json
import base64

from sign_url import SigV4RequestSigner

gi.require_version("Gst", "1.0")
from gi.repository import Gst

gi.require_version("GstWebRTC", "1.0")
from gi.repository import GstWebRTC

gi.require_version("GstSdp", "1.0")
from gi.repository import GstSdp

class EventType(enum.Enum):
UNKNOWN = "UNKNOWN"
INIT = "INIT"
CONNECT_SIGNAL_CHANNEL = "CONNECT_SIGNAL_CHANNEL"
SIGNAL_MESSAGE_SDP_OFFER = "SIGNAL_MESSAGE_SDP_OFFER"
SIGNAL_ERROR = "SIGNAL_ERROR"
SIGNAL_CLOSE = "SIGNAL_CLOSE"
SIGNAL_OPEN = "SIGNAL_OPEN"
ANSWER_CREATED = "ANSWER_CREATED"
SEND_ICE_CANDIDATE = "SEND_ICE_CANDIDATE"
NEGOTIATION_NEEDED = "NEGOTIATION_NEEDED"

class VideoService:
def __init__(self, region, channel_arn, pipeline_description):
self.event_queue = queue.Queue()
self.client_id = None

self.region = region
self.channel_arn = channel_arn
self.pipeline_description = pipeline_description
self.pipeline = None
self.appsrc = None
self.webrtc = None

def connect_signal_channel(self):
client = boto3.client("kinesisvideo", region_name=self.region)

# Browser should be viewer, we will be master
response = client.get_signaling_channel_endpoint(
ChannelARN=self.channel_arn,
SingleMasterChannelEndpointConfiguration={
"Protocols": [
"WSS",
],
"Role": "MASTER",
},
)

endpoints = []

for e in response["ResourceEndpointList"]:
print(e["Protocol"])
print(e["ResourceEndpoint"])

endpoints.append(e["ResourceEndpoint"])

query_params = {
"X-Amz-ChannelARN": self.channel_arn,
}

credentials = boto3.Session().get_credentials()
signer = SigV4RequestSigner(credentials, self.region, "kinesisvideo")
signed_url = signer.get_signed_url(
endpoints[0],
query_params=query_params,
)

self.ws = websocket.WebSocketApp(
signed_url,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close,
on_open=self.on_open,
)

# TODO do SSL

# Start ws.run_forever() in a separate thread
self.ws_thread = threading.Thread(
target=self.ws.run_forever, kwargs={"sslopt": {"cert_reqs": ssl.CERT_NONE,
"check_hostname": False}}
)
self.ws_thread.start()

def on_message(self, ws, message):
if message:
received = json.loads(message)

if "messagePayload" in received and "messageType" in received:
payload = base64.b64decode(received["messagePayload"]).decode("utf-8")
payload = json.loads(payload)

if received["messageType"] == "SDP_OFFER":
print("Received SDP_OFFER")
client_id = received["senderClientId"]
self.event_queue.put(
(EventType.SIGNAL_MESSAGE_SDP_OFFER, {"client_id": client_id, "payload":
payload})
)

else:
print(received["messageType"])
print(payload)
elif "message" in received:
print(received["message"])

def on_error(self, ws, error):
self.event_queue.put((EventType.SIGNAL_ERROR, error))

def on_close(self, ws, close_status_code, close_msg):
self.event_queue.put((EventType.SIGNAL_CLOSE, None))

def on_open(self, ws):
self.event_queue.put((EventType.SIGNAL_OPEN, None))

def on_negotiation_needed(self, element, user_data=None):
print("on_negotiation_needed")

self.event_queue.put((EventType.NEGOTIATION_NEEDED, None))

def on_ice_gathering_state_notify(self, element, pspec):
# TODO
print("on_ice_gathering_state_notify")

def on_incoming_stream(self, webrtc, pad, pipe):
# TODO Do we need to handle incoming streams?

print("on_incoming_stream")

def send_ice_candidate_message(self, element, mlineindex, candidate):
print("send_ice_candidate_message")
self.event_queue.put(
(
EventType.SEND_ICE_CANDIDATE,
{
"sdpMLineIndex": mlineindex,
"candidate": candidate,
},
)
)

def get_ice_candidates(self):
# TODO
pass

def setup_gstreamer(self):
try:
# Create a new pipeline
self.pipeline = Gst.parse_launch(self.pipeline_description)
self.webrtc = self.pipeline.get_by_name("sink")

# Attach signals
self.webrtc.connect("on-negotiation-needed", self.on_negotiation_needed)
self.webrtc.connect("on-ice-candidate", self.send_ice_candidate_message)
self.webrtc.connect("notify::ice-gathering-state", self.
on_ice_gathering_state_notify)

self.pipeline.set_state(Gst.State.READY)

self.webrtc.connect("pad-added", self.on_incoming_stream, self.pipeline)

self.get_ice_candidates()

print("Starting pipeline")

except Exception as ex:
raise Exception("Failed to start pipeline", ex)

ret = self.pipeline.set_state(Gst.State.PLAYING)
if ret == Gst.StateChangeReturn.FAILURE:
raise Exception("Failed to start pipeline")

def send_sdp_ice_candidate(self, ice_candidate):
print("send_sdp_ice_candidate")

msg = json.dumps(ice_candidate)
print(msg)

base64_str = base64.b64encode(msg.encode("utf-8"))

payload = {
"action": "ICE_CANDIDATE",
"recipientClientId": self.client_id,
"messagePayload": base64_str.decode("utf-8"),
}

self.ws.send(json.dumps(payload))

def set_offer_on_gstreamer(self, sdp):
res, sdpmsg = GstSdp.SDPMessage.new()
GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg)
offer =
GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.OFFER, sdpmsg
)
promise = Gst.Promise.new()
self.webrtc.emit("set-remote-description", offer, promise)
promise.wait()

promise = Gst.Promise.new_with_change_func(self.on_answer_created, None,
None)
self.webrtc.emit("create-answer", None, promise)

def on_answer_created(self, promise, _, __):
reply = promise.get_reply()
answer = reply.get_value("answer")
self.webrtc.emit("set-local-description", answer, None)

answer_text = answer.sdp.as_text()
self.event_queue.put((EventType.ANSWER_CREATED, answer_text))

def send_sdp_answer(self, answer):
print("send_sdp_answer")

msg = json.dumps({"type": "answer", "sdp": answer})
print(msg)

base64_str = base64.b64encode(msg.encode("utf-8"))

payload = {
"action": "SDP_ANSWER",
"recipientClientId": self.client_id,
"messagePayload": base64_str.decode("utf-8"),
}

self.ws.send(json.dumps(payload))

def run(self):
# Start the queue with a request to connect
self.event_queue.put((EventType.INIT, None))

running = True

sdp_offer = None

while running:
# Get the next event from the queue
event_type, event_data = self.event_queue.get()

if event_type == EventType.INIT:
sdp_offer = None
Gst.init(None)

self.event_queue.put((EventType.CONNECT_SIGNAL_CHANNEL, None))

elif event_type == EventType.CONNECT_SIGNAL_CHANNEL:
self.connect_signal_channel()

elif event_type == EventType.SIGNAL_OPEN:
print("Waiting for peers")

elif event_type == EventType.SIGNAL_MESSAGE_SDP_OFFER:
sdp_offer = event_data["payload"]["sdp"]
client_id = event_data["client_id"]
self.client_id = client_id

self.setup_gstreamer()
self.set_offer_on_gstreamer(sdp_offer)

elif event_type == EventType.NEGOTIATION_NEEDED:
# TODO
pass

elif event_type == EventType.ANSWER_CREATED:
self.send_sdp_answer(event_data)

elif event_type == EventType.SEND_ICE_CANDIDATE:
self.send_sdp_ice_candidate(event_data)

elif event_type == EventType.SIGNAL_CLOSE:
print("Signal channel closed")

else:
print(f"Unexpected event: {event_type}")
print(event_data)
# TODO shutdown self.ws_thread and self.pipeline
running = False

if __name__ == "__main__":
video_parameters = {
"region": "us-west-2",
"channel_arn": "...",
"pipeline_description": "webrtcbin bundle-policy=max-bundle name=sink
stun-server=stun://stun.kinesisvideo.us-west-2.amazonaws.com:443\nvideotestsrc
pattern=ball ! videoconvert ! videoscale !
video/x-raw,width=1280,height=720 ! videorate ! video/x-raw,framerate=25/1
! queue ! x264enc speed-preset=ultrafast tune=zerolatency bframes=0
key-int-max=25 ! video/x-h264,profile=baseline ! rtph264pay !\nqueue !
application/x-rtp,media=video,encoding-name=H264 ! sink.",
}
vs = VideoService(**video_parameters)
vs.run()

My browser code will send an offer:

{"type":"offer","sdp":"v=0\r\no=- 6426415272463828937 2 IN IP4
127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0
1\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS\r\nm=audio 9
UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9
IN IP4
0.0.0.0\r\na=ice-ufrag:e9Aj\r\na=ice-pwd:o//T9KUkuruDaBLI4lUrt4Pt\r\na=ice-options:trickle\r\na=fingerprint:sha-256
9B:54:6D:8E:DF:AD:5E:A1:B6:5B:A2:CD:C5:67:22:9D:5F:90:0F:EF:5B:9B:FD:C9:0E:45:CF:3C:4A:B7:43:1D\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1
urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2
http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3
http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4
urn:ietf:params:rtp-hdrext:sdes:mid\r\na=recvonly\r\na=rtcp-mux\r\na=rtpmap:111
opus/48000/2\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111
minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63
111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8
PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:110
telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\nm=video 9
UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 35 36 37 38 102 103 104 105 106 107
108 109 127 125 39 40 41 42 43 44 45 46 47 48 112 113 114 115 116 117 118
49\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4
0.0.0.0\r\na=ice-ufrag:e9Aj\r\na=ice-pwd:o//T9KUkuruDaBLI4lUrt4Pt\r\na=ice-options:trickle\r\na=fingerprint:sha-256
9B:54:6D:8E:DF:AD:5E:A1:B6:5B:A2:CD:C5:67:22:9D:5F:90:0F:EF:5B:9B:FD:C9:0E:45:CF:3C:4A:B7:43:1D\r\na=setup:actpass\r\na=mid:1\r\na=extmap:14
urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:2
http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:13
urn:3gpp:video-orientation\r\na=extmap:3
http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:5
http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\na=extmap:6
http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\na=extmap:7
http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\na=extmap:8
http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\na=extmap:4
urn:ietf:params:rtp-hdrext:sdes:mid\r\na=extmap:10
urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\na=extmap:11
urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\na=recvonly\r\na=rtcp-mux\r\na=rtcp-rsize\r\na=rtpmap:96
VP8/90000\r\na=rtcp-fb:96 goog-remb\r\na=rtcp-fb:96
transport-cc\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96
nack pli\r\na=rtpmap:97 rtx/90000\r\na=fmtp:97 apt=96\r\na=rtpmap:98
VP9/90000\r\na=rtcp-fb:98 goog-remb\r\na=rtcp-fb:98
transport-cc\r\na=rtcp-fb:98 ccm fir\r\na=rtcp-fb:98 nack\r\na=rtcp-fb:98
nack pli\r\na=fmtp:98 profile-id=0\r\na=rtpmap:99 rtx/90000\r\na=fmtp:99
apt=98\r\na=rtpmap:100 VP9/90000\r\na=rtcp-fb:100
goog-remb\r\na=rtcp-fb:100 transport-cc\r\na=rtcp-fb:100 ccm
fir\r\na=rtcp-fb:100 nack\r\na=rtcp-fb:100 nack pli\r\na=fmtp:100
profile-id=2\r\na=rtpmap:101 rtx/90000\r\na=fmtp:101 apt=100\r\na=rtpmap:35
VP9/90000\r\na=rtcp-fb:35 goog-remb\r\na=rtcp-fb:35
transport-cc\r\na=rtcp-fb:35 ccm fir\r\na=rtcp-fb:35 nack\r\na=rtcp-fb:35
nack pli\r\na=fmtp:35 profile-id=1\r\na=rtpmap:36 rtx/90000\r\na=fmtp:36
apt=35\r\na=rtpmap:37 VP9/90000\r\na=rtcp-fb:37 goog-remb\r\na=rtcp-fb:37
transport-cc\r\na=rtcp-fb:37 ccm fir\r\na=rtcp-fb:37 nack\r\na=rtcp-fb:37
nack pli\r\na=fmtp:37 profile-id=3\r\na=rtpmap:38 rtx/90000\r\na=fmtp:38
apt=37\r\na=rtpmap:102 H264/90000\r\na=rtcp-fb:102
goog-remb\r\na=rtcp-fb:102 transport-cc\r\na=rtcp-fb:102 ccm
fir\r\na=rtcp-fb:102 nack\r\na=rtcp-fb:102 nack pli\r\na=fmtp:102
level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r\na=rtpmap:103
rtx/90000\r\na=fmtp:103 apt=102\r\na=rtpmap:104 H264/90000\r\na=rtcp-fb:104
goog-remb\r\na=rtcp-fb:104 transport-cc\r\na=rtcp-fb:104 ccm
fir\r\na=rtcp-fb:104 nack\r\na=rtcp-fb:104 nack pli\r\na=fmtp:104
level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r\na=rtpmap:105
rtx/90000\r\na=fmtp:105 apt=104\r\na=rtpmap:106 H264/90000\r\na=rtcp-fb:106
goog-remb\r\na=rtcp-fb:106 transport-cc\r\na=rtcp-fb:106 ccm
fir\r\na=rtcp-fb:106 nack\r\na=rtcp-fb:106 nack pli\r\na=fmtp:106
level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\na=rtpmap:107
rtx/90000\r\na=fmtp:107 apt=106\r\na=rtpmap:108 H264/90000\r\na=rtcp-fb:108
goog-remb\r\na=rtcp-fb:108 transport-cc\r\na=rtcp-fb:108 ccm
fir\r\na=rtcp-fb:108 nack\r\na=rtcp-fb:108 nack pli\r\na=fmtp:108
level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\r\na=rtpmap:109
rtx/90000\r\na=fmtp:109 apt=108\r\na=rtpmap:127 H264/90000\r\na=rtcp-fb:127
goog-remb\r\na=rtcp-fb:127 transport-cc\r\na=rtcp-fb:127 ccm
fir\r\na=rtcp-fb:127 nack\r\na=rtcp-fb:127 nack pli\r\na=fmtp:127
level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f\r\na=rtpmap:125
rtx/90000\r\na=fmtp:125 apt=127\r\na=rtpmap:39 H264/90000\r\na=rtcp-fb:39
goog-remb\r\na=rtcp-fb:39 transport-cc\r\na=rtcp-fb:39 ccm
fir\r\na=rtcp-fb:39 nack\r\na=rtcp-fb:39 nack pli\r\na=fmtp:39
level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f\r\na=rtpmap:40
rtx/90000\r\na=fmtp:40 apt=39\r\na=rtpmap:41 H264/90000\r\na=rtcp-fb:41
goog-remb\r\na=rtcp-fb:41 transport-cc\r\na=rtcp-fb:41 ccm
fir\r\na=rtcp-fb:41 nack\r\na=rtcp-fb:41 nack pli\r\na=fmtp:41
level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=f4001f\r\na=rtpmap:42
rtx/90000\r\na=fmtp:42 apt=41\r\na=rtpmap:43 H264/90000\r\na=rtcp-fb:43
goog-remb\r\na=rtcp-fb:43 transport-cc\r\na=rtcp-fb:43 ccm
fir\r\na=rtcp-fb:43 nack\r\na=rtcp-fb:43 nack pli\r\na=fmtp:43
level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=f4001f\r\na=rtpmap:44
rtx/90000\r\na=fmtp:44 apt=43\r\na=rtpmap:45 AV1/90000\r\na=rtcp-fb:45
goog-remb\r\na=rtcp-fb:45 transport-cc\r\na=rtcp-fb:45 ccm
fir\r\na=rtcp-fb:45 nack\r\na=rtcp-fb:45 nack pli\r\na=rtpmap:46
rtx/90000\r\na=fmtp:46 apt=45\r\na=rtpmap:47 AV1/90000\r\na=rtcp-fb:47
goog-remb\r\na=rtcp-fb:47 transport-cc\r\na=rtcp-fb:47 ccm
fir\r\na=rtcp-fb:47 nack\r\na=rtcp-fb:47 nack pli\r\na=fmtp:47
profile=1\r\na=rtpmap:48 rtx/90000\r\na=fmtp:48 apt=47\r\na=rtpmap:112
H264/90000\r\na=rtcp-fb:112 goog-remb\r\na=rtcp-fb:112
transport-cc\r\na=rtcp-fb:112 ccm fir\r\na=rtcp-fb:112
nack\r\na=rtcp-fb:112 nack pli\r\na=fmtp:112
level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f\r\na=rtpmap:113
rtx/90000\r\na=fmtp:113 apt=112\r\na=rtpmap:114 H264/90000\r\na=rtcp-fb:114
goog-remb\r\na=rtcp-fb:114 transport-cc\r\na=rtcp-fb:114 ccm
fir\r\na=rtcp-fb:114 nack\r\na=rtcp-fb:114 nack pli\r\na=fmtp:114
level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=64001f\r\na=rtpmap:115
rtx/90000\r\na=fmtp:115 apt=114\r\na=rtpmap:116 red/90000\r\na=rtpmap:117
rtx/90000\r\na=fmtp:117 apt=116\r\na=rtpmap:118 ulpfec/90000\r\na=rtpmap:49
flexfec-03/90000\r\na=rtcp-fb:49 goog-remb\r\na=rtcp-fb:49
transport-cc\r\na=fmtp:49 repair-window=10000000\r\n"}

and I am able to send a answer from Python:

{"type":"answer","sdp":"v=0\r\no=- 6426415272463828937 2 IN IP4
0.0.0.0\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0 1\r\nm=audio 9
UDP/TLS/RTP/SAVPF 111\r\nc=IN IP4
0.0.0.0\r\na=ice-ufrag:B1FuRGQ3FL4KnNY7C/h8BygmBRYbAEol\r\na=ice-pwd:3YJMhXnjRkWEC/ZMruapjMIRAOXWjcBd\r\na=mid:0\r\na=rtcp-mux\r\na=setup:active\r\na=rtpmap:111
OPUS/48000/2\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111
minptime=10;useinbandfec=1\r\na=inactive\r\na=fingerprint:sha-256
2E:6D:EE:5F:5B:0F:91:35:6A:B0:7C:D1:7D:D2:37:8B:68:73:E0:0D:A7:65:E1:5D:A1:6F:5F:3E:8A:44:15:85\r\nm=video
9 UDP/TLS/RTP/SAVPF 96\r\nc=IN IP4
0.0.0.0\r\na=ice-ufrag:B1FuRGQ3FL4KnNY7C/h8BygmBRYbAEol\r\na=ice-pwd:3YJMhXnjRkWEC/ZMruapjMIRAOXWjcBd\r\na=mid:1\r\na=rtcp-mux\r\na=setup:active\r\na=rtpmap:96
VP8/90000\r\na=rtcp-fb:96 nack pli\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96
transport-cc\r\na=inactive\r\na=fingerprint:sha-256
2E:6D:EE:5F:5B:0F:91:35:6A:B0:7C:D1:7D:D2:37:8B:68:73:E0:0D:A7:65:E1:5D:A1:6F:5F:3E:8A:44:15:85\r\n"}

But the connection is not made, note the answer is VP8 and my pipeline is
H.264:

webrtcbin bundle-policy=max-bundle name=sink stun-server=stun://
stun.kinesisvideo.us-west-2.amazonaws.com:443 videotestsrc pattern=ball !
videoconvert ! videoscale ! video/x-raw,width=1280,height=720 ! videorate !
video/x-raw,framerate=25/1 ! queue ! x264enc speed-preset=ultrafast
tune=zerolatency bframes=0 key-int-max=25 ! video/x-h264,profile=baseline !
rtph264pay ! queue ! application/x-rtp,media=video,encoding-name=H264 !
sink.

Any help would be most appreciated here.

Thanks, C
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.freedesktop.org/archives/gstreamer-devel/attachments/20231012/244d22e4/attachment-0001.htm>