python code for muxing klv and video

[Arturo Flores]

Hello,

I have some code for muxing klv and video (included at the end of this
email).

This code is a proof of concept to see if I am doing things correctly.  I
have an mpg video with klv and video streams.  I first dumped the klv into
a file using the following pipeline:

gst-launch-1.0 filesrc=input.ts ! tsdemux ! meta/x-klv ! filesink
location=klv.bin

By inspecting the klv, I determined how many klv packets are in the stream
and the klv frame rate.

I then extracted the video frames (using ffmpeg) into a folder.  I also
used ffprobe to determine things like the video frame rate.

What I attempted to do is remux the video frames and klv and end up with
something identical to the original file.  The code mostly works though
there are some minor differences that I am curious about.

This code uses the python bindings.  I used
http://gstreamer-devel.966125.n4.nabble.com/Example-code-for-muxing-klv-meta-x-klv-with-mpegtsmux-plugins-bad-and-GStreamer-1-8-3-td4684782.html
 and
https://github.com/tamaggo/gstreamer-examples/blob/master/test_gst_appsrc_testvideo_mp4mux.py
as
guidelines.

One difference I don't understand, is the ffprobe output of the original
file has:

$ ffprobe input.ts
[...]
  Duration: 00:01:05.15, start: 271.125322, bitrate: 5535 kb/s
[...]

Whereas the ffprobe output of the remuxed file has:

$ ffprobe remuxed.ts
[...]
  Duration: 00:01:04.57, start: 3600.033367, bitrate: 1745 kb/s
[...]

Note the difference in start time.  What controls the start time here?  I
tried setting the start-time property of x264enc, but that didn't do the
trick.

Is there anything in this code that jumps out as the wrong way to do
things, or is there a better way to mux together video and klv streams?

Thanks

Code:

import matplotlib.pyplot as plt
import os
import sys
import gi
import json

gi.require_version('Gst', '1.0')
from gi.repository import Gst, GObject


if len(sys.argv) != 2:
    print("Usage: python muxklv.py [config file]")
    sys.exit(1)

config_file = sys.argv[1]
fh = open(config_file)
config_json = json.load(fh)
fh.close()



vid_frames_folder = config_json["vid_frames_folder"]
vid_frame_rate = config_json["vid_frame_rate"]
vid_width = config_json["vid_width"]
vid_height = config_json["vid_height"]
klv_file = config_json["klv_file"]
klv_frame_rate = config_json["klv_frame_rate"]
klv_packet_size = config_json["klv_packet_size"]
out_file = config_json["out_file"]


GObject.threads_init()
Gst.init(None)

# video source elements
vsrc = Gst.ElementFactory.make("appsrc", "vidsrc")
vqueue = Gst.ElementFactory.make("queue")
vtee = Gst.ElementFactory.make("tee")

# klv source elements
appsrc = Gst.ElementFactory.make("appsrc")
queue_klv = Gst.ElementFactory.make("queue")

# display elements
queue_display = Gst.ElementFactory.make("queue")
vcvt = Gst.ElementFactory.make("videoconvert", "vidcvt")
vsink = Gst.ElementFactory.make("autovideosink", "vidsink")

# recording elements
queue_record = Gst.ElementFactory.make("queue")
vcvt_encoder = Gst.ElementFactory.make("videoconvert")
encoder = Gst.ElementFactory.make("x264enc")
muxer = Gst.ElementFactory.make("mpegtsmux")
filesink = Gst.ElementFactory.make("filesink")

# configure video element
caps_str = "video/x-raw"
caps_str +=
",format=(string)RGB,width={},height={}".format(vid_width,vid_height)
caps_str += ",framerate={}/1".format(int(vid_frame_rate))
vcaps = Gst.Caps.from_string(caps_str)
vsrc.set_property("caps", vcaps);
vsrc.set_property("format", Gst.Format.TIME)

# configure appsrc element
caps_str = "meta/x-klv"
caps_str += ",parsed=True"
caps = Gst.Caps.from_string(caps_str)
appsrc.set_property("caps", caps)
# appsrc.connect("need-data", klv_need_data)
appsrc.set_property("format", Gst.Format.TIME)

# configure encoder
encoder.set_property("noise-reduction", 1000)
encoder.set_property("threads", 4)
encoder.set_property("bitrate", 1755)

# configure filesink
filesink.set_property("location", out_file)
filesink.set_property("async", 0)

pipeline = Gst.Pipeline()
pipeline.add(vsrc)
pipeline.add(vqueue)
pipeline.add(vtee)
pipeline.add(appsrc)
pipeline.add(queue_klv)
pipeline.add(queue_display)
pipeline.add(vcvt)
pipeline.add(vsink)
pipeline.add(queue_record)
pipeline.add(vcvt_encoder)
pipeline.add(encoder)
pipeline.add(muxer)
pipeline.add(filesink)

# link video elements
vsrc.link(vqueue)
vqueue.link(vtee)

# link display elements
vtee.link(queue_display)
queue_display.link(vcvt)
vcvt.link(vsink)

# link recording elements
vtee.link(queue_record)
queue_record.link(vcvt_encoder)
vcvt_encoder.link(encoder)
encoder.link(muxer)
muxer.link(filesink)

# link klv elements
appsrc.link(queue_klv)
queue_klv.link(muxer)

pipeline.set_state(Gst.State.PLAYING)

fh = open(klv_file, 'rb')
timestamp = 0

klv_done = False
vid_done = False
vid_frame_counter = 0
klv_frame_counter = 0

t = 0
last_klv_t = 0
last_vid_t = 0
while True:
    if vid_done and klv_done:
        break
    if t - last_klv_t >= 1.0 / klv_frame_rate:
        if not klv_done:
            klv_bytes = fh.read(klv_packet_size)
            if klv_bytes:
                klvbuf = Gst.Buffer.new_allocate(None, klv_packet_size,
None)
                klvbuf.fill(0, klv_bytes)
                klvbuf.pts = int(t * 1e9)
                klvbuf.dts = int(t * 1e9)

                appsrc.emit("push-buffer", klvbuf)
                klv_frame_counter += 1
                last_klv_t = t
                print("klv {} {}".format(klv_frame_counter, last_klv_t))
            else:
                klv_done = True

    if t - last_vid_t >= 1.0 / vid_frame_rate:
        if not vid_done:
            frame_filename = '%s/%04d.jpg' % (vid_frames_folder,
vid_frame_counter + 1)
            if os.path.isfile(frame_filename):
                vid_frame = plt.imread(frame_filename)
                data = vid_frame.tostring()
                vidbuf = Gst.Buffer.new_allocate(None, len(data), None)
                vidbuf.fill(0, data)
                vidbuf.pts = int(t * 1e9)
                vidbuf.dts = int(t * 1e9)

                vsrc.emit("push-buffer", vidbuf)
                vid_frame_counter += 1
                last_vid_t = t
                print("vid {} {}".format(vid_frame_counter, last_vid_t))
            else:
                vid_done = True
                continue

    t += 0.000001
    #print(t)

vsrc.emit("end-of-stream")
appsrc.emit("end-of-stream")

bus = pipeline.get_bus()

while True:
    msg = bus.poll(Gst.MessageType.ANY, Gst.CLOCK_TIME_NONE)
    t = msg.type
    if t == Gst.MessageType.EOS:
        print("EOS")
        break
        pipeline.set_state(Gst.State.NULL)
    elif t == Gst.MessageType.ERROR:
        err, debug = msg.parse_error()
        # print("Error: %s" % err, debug)
        break
    elif t == Gst.MessageType.WARNING:
        err, debug = msg.parse_warning()
        # print("Warning: %s" % err, debug)
    elif t == Gst.MessageType.STATE_CHANGED:
        pass
    elif t == Gst.MessageType.STREAM_STATUS:
        pass
    else:
        pass
        # print(t)
        # print("Unknown message: %s" % msg)

pipeline.set_state(Gst.State.NULL)

print("Bye")
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.freedesktop.org/archives/gstreamer-devel/attachments/20200624/79262661/attachment.htm>