gstreamer逐帧处理视频图像
一般想要逐帧处理视频图像，可以用opencv，代码可以如下：
#!/usr/bin/env python
# _*_ coding: utf-8 _*_
#
#
# Copyright (C) 2020 Wei Keting<weikting@gmail.com>. All rights reserved.
# @Time : 2020-06-26 11:51
# @File : test_opencv.py
# @Description :
#
#
import sys
import cv2
import numpy as np
import os
f = sys.argv[1]
cap = cv2.VideoCapture(sys.argv[1])
fps = cap.get(cv2.CAP_PROP_FPS)
# create a window named `image`
cv2.imshow("image", np.random.randint(0, 255, (64, 64, 3), dtype='uint8'))
# set window's title
cv2.setWindowTitle("image", os.path.basename(f))
while True:
ret, frame = cap.read()
if ret is False:
break
cv2.imshow("image", frame)
# 控制帧率
cv2.waitKey(int(1000/fps))
运行：
python3 test_opencv.py path_of_the_video.webm
不过使用opencv不会播放声音，当逐帧处理图像的同时还需要播放声音时，可以考虑使用gstreamer。
python3代码：
#!/usr/bin/env python3
# _*_ coding: utf-8 _*_
#
#
# Copyright (C) 2020 Wei Keting<weikting@gmail.com>. All rights reserved.
# @Time : 2020-06-26 10:26
# @File : test_gst.py
# @Description :
#
#
import os
import sys
import gi
gi.require_version('Gst', '1.0')
gi.require_version("GstVideo", "1.0")
from gi.repository import Gst, GLib, GstVideo
import numpy as np
import cv2
# noinspection PyMethodMayBeStatic
class GstDemo:
def __init__(self):
self.frame_idx = 0
self.player = Gst.ElementFactory.make("playbin", "player")
appsink = Gst.ElementFactory.make("appsink", "sink")
appsink.set_property("emit-signals", True) # 发送new-sample信号
caps = Gst.Caps.new_empty_simple("video/x-raw")
caps.set_value('format', "BGRA") # BGR有点问题，所以不用
appsink.set_property("caps", caps) # 设置appsink接受BGRA的数据格式
appsink.connect("new-sample", self._on_new_sample)
self.player.set_property("video-sink", appsink)
bus = self.player.get_bus()
bus.add_signal_watch()
bus.connect("message", self.on_message)
self.loop = GLib.MainLoop()
def _on_new_sample(self, sink):
"""
读取并处理视频帧图像
:param sink:
:return:
"""
self.frame_idx += 1
sample = sink.emit('pull_sample')
if sample is None:
return Gst.FlowReturn.EOS
caps = sample.get_caps()
buffer = sample.get_buffer()
structure = caps.get_structure(0)
_, width = structure.get_int("width")
_, height = structure.get_int("height")
success, map_info = buffer.map(Gst.MapFlags.READ)
if self.frame_idx == 1:
print("caps: ", caps.to_string(), "size:", map_info.size)
img = np.ndarray(
shape=(height, width, 4), # BGRA 是4通道
dtype=np.uint8,
buffer=map_info.data)
img = img[:, :, :3] # 放弃Alpha通道
# 在主线程调用cv2.imshow，否则crash
GLib.idle_add(self._idle_show_image, img)
os.makedirs("/tmp/cv", exist_ok=True)
cv2.imwrite("/tmp/cv/{:06d}.jpg".format(self.frame_idx), img)
return Gst.FlowReturn.OK
def _idle_show_image(self, image):
cv2.imshow("test", image)
return GLib.SOURCE_REMOVE
def on_message(self, bus, message):
t = message.type
if t == Gst.MessageType.EOS:
self.player.set_state(Gst.State.NULL)
self.loop.quit()
elif t == Gst.MessageType.ERROR:
self.player.set_state(Gst.State.NULL)
err, debug = message.parse_error()
print("Error: %s" % err, debug)
self.loop.quit()
def start(self):
for filepath in sys.argv[1:]:
if os.path.isfile(filepath):
filepath = os.path.abspath(filepath)
self.player.set_property("uri", "file://" + filepath)
self.player.set_state(Gst.State.PLAYING)
break
Gst.init(None)
demo = GstDemo()
demo.start()
demo.loop.run()
运行测试：
python3 test_gst.py path_of_the_video.webm
C代码：
/**
*
* Copyright (C) 2020 Wei Keting<weikting@gmail.com>. All rights reserved.
* @Time : 2020-06-25 09:22
* @File : test_gst1.c
* @Description :
*
**/
#include <gst/gst.h>
#include <gst/app/app.h>
#include <gst/video/video.h>
#include <cairo.h>
#include <stdio.h>
#include <glib/gstdio.h>
#include <gtk/gtk.h>
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
#define FORMAT "BGRx" /* BGRA */
#else
#define FORMAT "xRGB" /* ARGB */
#endif
static GstFlowReturn
on_new_sample (GstAppSink *appsink, GtkWidget *image)
{
static gint frame_idx = 0;
GstSample *sample = gst_app_sink_pull_sample(appsink);
if(sample == NULL){
if(gst_app_sink_is_eos(appsink))
return GST_FLOW_EOS;
}
GstBuffer *buffer = gst_sample_get_buffer(sample);
GstCaps *caps = gst_sample_get_caps(sample);
GstMapInfo map_info;
gint width,height;
gst_buffer_map(buffer,&map_info,GST_MAP_READ);
GstStructure *structure = gst_caps_get_structure(caps,0);
gst_structure_get_int(structure,"width",&width);
gst_structure_get_int(structure,"height",&height);
frame_idx += 1;
cairo_format_t format;
cairo_surface_t *surface;
format = CAIRO_FORMAT_ARGB32;
surface = cairo_image_surface_create_for_data (map_info.data,
format, width, height, cairo_format_stride_for_width(format,width));
gtk_image_set_from_surface(GTK_IMAGE(image),surface);
char filename[128] = {0};
g_mkdir_with_parents("/tmp/pictures",0700);
snprintf(filename,sizeof(filename),"/tmp/pictures/%06d.png",frame_idx);
cairo_status_t st = cairo_surface_write_to_png(surface,filename);
if(st != CAIRO_STATUS_SUCCESS){
g_printerr("st:%s\n",cairo_status_to_string(st));
}
cairo_surface_destroy(surface);
gst_sample_unref(sample);
return GST_FLOW_OK;
}
static void
on_bus_message (GstBus *bus,
GstMessage *message,
GstElement *pipeline)
{
switch(GST_MESSAGE_TYPE(message)){
case GST_MESSAGE_EOS:
case GST_MESSAGE_ERROR:
gst_element_set_state(pipeline,GST_STATE_NULL);
gtk_main_quit();
break;
default:
break;
}
}
static void
on_win_destroy(GtkWidget*win,GstElement *pipeline)
{
gst_element_set_state(pipeline,GST_STATE_NULL);
gtk_main_quit();
}
int main(int argc, char *argv[]) {
GstElement *pipeline,*appsink;
GstCaps *caps;
GstBus *bus;
GtkWidget *win,*image;
gtk_init(&argc,&argv);
gst_init (&argc, &argv);
win = gtk_window_new(GTK_WINDOW_TOPLEVEL);
image = gtk_image_new();
gtk_container_add(GTK_CONTAINER(win),image);
gtk_widget_show_all(win);
gtk_window_present(GTK_WINDOW(win));
gchar *args = g_strdup_printf("playbin uri=file://%s",argv[1]);
pipeline = gst_parse_launch (args, NULL);
g_free(args);
appsink = gst_element_factory_make("appsink","sink");
caps = gst_caps_new_simple("video/x-raw","format", G_TYPE_STRING, FORMAT, NULL);
g_object_set(appsink,"emit-signals",TRUE,NULL);
g_object_set(appsink,"caps",caps,NULL);
g_signal_connect(appsink,"new-sample",G_CALLBACK(on_new_sample),image);
g_object_set(pipeline,"video-sink",appsink,NULL);
bus = gst_element_get_bus (pipeline);
gst_bus_add_signal_watch(bus);
g_signal_connect(bus,"message",G_CALLBACK(on_bus_message),pipeline);
g_signal_connect(win,"destroy",G_CALLBACK(on_win_destroy),pipeline);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
gtk_main();
gst_object_unref (bus);
gst_object_unref (pipeline);
return 0;
}运行测试：
gcc test_gst1.c -o test_gst1 pkg-config --cflags --libs gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0 cairo gtk+-3.0
./test_gst1 path_of_the_video.mp4