mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-05-18 00:12:46 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'h264decoder-corrupted-flag' into 'main'

Browse source 
codecs: Mark corrupted if IDR frame is not observed

Closes #2994

See merge request gstreamer/gstreamer!5380
This commit is contained in:
Seungha Yang 2024-05-03 20:24:56 +00:00
parent 73c64e8182 f91f15ff0e
commit 09230dd186
5 changed files with 215 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstcodecpicture.h
View file

@ -68,6 +68,8 @@ struct _GstCodecPicture
guint32 system_frame_number;
GstVideoCodecState *discont_state;
gboolean corrupted;
gpointer user_data;
GDestroyNotify notify;

114
subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth264decoder.c
View file

@ -162,6 +162,13 @@ struct _GstH264DecoderPrivate
guint32 last_reorder_frame_number;
gint fps_n;
gint fps_d;
gboolean have_idr;
guint32 missing_idr_count;
guint32 max_corrupted_count;
gboolean keyframe_requested;
gboolean recovery_point;
gboolean inter_predicted;
};
typedef struct
@ -418,7 +425,7 @@ gst_h264_decoder_finalize (GObject * object)
}
static void
gst_h264_decoder_reset_latency_infos (GstH264Decoder * self)
gst_h264_decoder_reset_latency_state (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
@ -428,6 +435,27 @@ gst_h264_decoder_reset_latency_infos (GstH264Decoder * self)
priv->fps_d = 1;
}
static void
gst_h264_decoder_reset_idr_state (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
priv->have_idr = FALSE;
priv->max_corrupted_count = 0;
priv->missing_idr_count = 0;
priv->keyframe_requested = FALSE;
}
static void
gst_h264_decoder_reset_frame_state (GstH264Decoder * self)
{
GstH264DecoderPrivate *priv = self->priv;
priv->recovery_point = FALSE;
priv->inter_predicted = FALSE;
priv->last_flow = GST_FLOW_OK;
}
static void
gst_h264_decoder_reset (GstH264Decoder * self)
{
@ -442,9 +470,10 @@ gst_h264_decoder_reset (GstH264Decoder * self)
priv->width = 0;
priv->height = 0;
priv->nal_length_size = 4;
priv->last_flow = GST_FLOW_OK;
gst_h264_decoder_reset_latency_infos (self);
gst_h264_decoder_reset_latency_state (self);
gst_h264_decoder_reset_idr_state (self);
gst_h264_decoder_reset_frame_state (self);
}
static gboolean
@ -554,8 +583,9 @@ gst_h264_decoder_handle_frame (GstVideoDecoder * decoder,
GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_PTS (in_buf)),
GST_TIME_ARGS (GST_BUFFER_DTS (in_buf)));
gst_h264_decoder_reset_frame_state (self);
priv->current_frame = frame;
priv->last_flow = GST_FLOW_OK;
gst_buffer_map (in_buf, &map, GST_MAP_READ);
if (priv->in_format == GST_H264_DECODER_FORMAT_AVC) {
@ -1292,6 +1322,20 @@ gst_h264_decoder_parse_slice (GstH264Decoder * self, GstH264NalUnit * nalu)
}
}
if (!priv->have_idr) {
if (priv->current_slice.nalu.idr_pic_flag) {
GST_DEBUG_OBJECT (self, "Found IDR picture");
priv->have_idr = TRUE;
} else if (priv->recovery_point) {
GST_DEBUG_OBJECT (self, "Found recovery point SEI");
priv->have_idr = TRUE;
} else if (!GST_H264_IS_I_SLICE (&priv->current_slice.header) &&
!GST_H264_IS_SI_SLICE (&priv->current_slice.header)) {
GST_DEBUG_OBJECT (self, "Current picture requires reference frames");
priv->inter_predicted = TRUE;
}
}
if (!priv->current_picture) {
GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self);
GstH264Picture *picture = NULL;
@ -1346,6 +1390,28 @@ gst_h264_decoder_parse_slice (GstH264Decoder * self, GstH264NalUnit * nalu)
return gst_h264_decoder_decode_slice (self);
}
static void
gst_h264_decoder_parse_sei (GstH264Decoder * self, GstH264NalUnit * nalu)
{
GstH264DecoderPrivate *priv = self->priv;
GArray *sei_array;
guint i;
/* Ignore error, it should not be critical from decoder point of view */
gst_h264_parser_parse_sei (priv->parser, nalu, &sei_array);
for (i = 0; i < sei_array->len; i++) {
GstH264SEIMessage *sei = &g_array_index (sei_array, GstH264SEIMessage, i);
if (sei->payloadType == GST_H264_SEI_RECOVERY_POINT) {
GST_DEBUG_OBJECT (self, "Found recovery point");
priv->recovery_point = TRUE;
break;
}
}
g_array_free (sei_array, TRUE);
}
static GstFlowReturn
gst_h264_decoder_decode_nal (GstH264Decoder * self, GstH264NalUnit * nalu)
{
@ -1369,6 +1435,9 @@ gst_h264_decoder_decode_nal (GstH264Decoder * self, GstH264NalUnit * nalu)
case GST_H264_NAL_SLICE_EXT:
ret = gst_h264_decoder_parse_slice (self, nalu);
break;
case GST_H264_NAL_SEI:
gst_h264_decoder_parse_sei (self, nalu);
break;
default:
break;
}
@ -1906,6 +1975,25 @@ gst_h264_decoder_finish_current_picture (GstH264Decoder * self,
return;
}
if (!priv->have_idr && priv->inter_predicted) {
GST_DEBUG_OBJECT (self,
"IDR frame has not been observed yet, marking corrupted");
GST_CODEC_PICTURE (priv->current_picture)->corrupted = TRUE;
if (!priv->keyframe_requested) {
/* Don't spam keyframe request */
priv->keyframe_requested = TRUE;
gst_video_decoder_request_sync_point (GST_VIDEO_DECODER (self),
priv->current_frame, 0);
}
priv->missing_idr_count++;
if (priv->missing_idr_count > priv->max_corrupted_count) {
GST_DEBUG_OBJECT (self, "Missing IDR count %u > threshold %u",
priv->missing_idr_count, priv->max_corrupted_count);
priv->have_idr = TRUE;
}
}
klass = GST_H264_DECODER_GET_CLASS (self);
if (klass->end_picture) {
@ -1924,6 +2012,14 @@ gst_h264_decoder_finish_current_picture (GstH264Decoder * self,
}
}
if (flow_ret == GST_FLOW_OK &&
GST_CODEC_PICTURE (priv->current_picture)->corrupted) {
/* If subclass didn't revert the corrupted flag, set codec frame flag
* accordingly */
GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame,
GST_VIDEO_CODEC_FRAME_FLAG_CORRUPTED);
}
/* We no longer need the per frame reference lists */
gst_h264_decoder_clear_ref_pic_lists (self);
@ -2500,7 +2596,15 @@ gst_h264_decoder_process_sps (GstH264Decoder * self, GstH264SPS * sps)
if (ret != GST_FLOW_OK)
return ret;
gst_h264_decoder_reset_latency_infos (self);
gst_h264_decoder_reset_latency_state (self);
gst_h264_decoder_reset_idr_state (self);
/* Gussing GOP size to decide max corrupted frame count, with hardcoded
* upper bound 64 frames */
priv->max_corrupted_count = MIN (64, sps->max_frame_num);
GST_DEBUG_OBJECT (self,
"Configured max-corrupted-frame-count: %u, MaxFrameNum: %u",
priv->max_corrupted_count, sps->max_frame_num);
g_assert (klass->new_sequence);

88
subprojects/gst-plugins-bad/gst-libs/gst/codecs/gsth265decoder.c
View file

@ -138,6 +138,13 @@ struct _GstH265DecoderPrivate
gboolean input_state_changed;
GstFlowReturn last_flow;
gboolean have_idr;
guint32 missing_idr_count;
guint32 max_corrupted_count;
gboolean keyframe_requested;
gboolean recovery_point;
gboolean inter_predicted;
};
typedef struct
@ -528,6 +535,17 @@ gst_h265_decoder_get_max_dpb_size_from_sps (GstH265Decoder * self,
return max_dpb_size;
}
static void
gst_h265_decoder_reset_idr_state (GstH265Decoder * self)
{
GstH265DecoderPrivate *priv = self->priv;
priv->have_idr = FALSE;
priv->max_corrupted_count = 0;
priv->missing_idr_count = 0;
priv->keyframe_requested = FALSE;
}
static GstFlowReturn
gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
{
@ -555,6 +573,8 @@ gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
priv->interlaced_source_flag != interlaced_source_flag ||
gst_h265_decoder_is_crop_rect_changed (self, sps)) {
GstH265DecoderClass *klass = GST_H265_DECODER_GET_CLASS (self);
guint32 MaxPicOrderCntLsb =
1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
GST_DEBUG_OBJECT (self,
"SPS updated, resolution: %dx%d -> %dx%d, dpb size: %d -> %d, "
@ -582,6 +602,15 @@ gst_h265_decoder_process_sps (GstH265Decoder * self, GstH265SPS * sps)
priv->preferred_output_delay = 0;
}
gst_h265_decoder_reset_idr_state (self);
/* Gussing GOP size to decide max corrupted frame count, with hardcoded
* upper bound 64 frames */
priv->max_corrupted_count = MIN (64, MaxPicOrderCntLsb);
GST_DEBUG_OBJECT (self,
"Configured max-corrupted-frame-count: %u, MaxPicOrderCntLsb: %u",
priv->max_corrupted_count, MaxPicOrderCntLsb);
g_assert (klass->new_sequence);
ret = klass->new_sequence (self,
sps, max_dpb_size + priv->preferred_output_delay);
@ -622,19 +651,11 @@ static GstH265ParserResult
gst_h265_decoder_parse_sei (GstH265Decoder * self, GstH265NalUnit * nalu)
{
GstH265DecoderPrivate *priv = self->priv;
GstH265ParserResult pres;
GArray *messages = NULL;
guint i;
pres = gst_h265_parser_parse_sei (priv->parser, nalu, &messages);
if (pres != GST_H265_PARSER_OK) {
GST_WARNING_OBJECT (self, "Failed to parse SEI, result %d", pres);
/* XXX: Ignore error from SEI parsing, it might be malformed bitstream,
* or our fault. But shouldn't be critical */
g_clear_pointer (&messages, g_array_unref);
return GST_H265_PARSER_OK;
}
/* Ignore error, it should not be critical from decoder point of view */
gst_h265_parser_parse_sei (priv->parser, nalu, &messages);
for (i = 0; i < messages->len; i++) {
GstH265SEIMessage *sei = &g_array_index (messages, GstH265SEIMessage, i);
@ -650,6 +671,10 @@ gst_h265_decoder_parse_sei (GstH265Decoder * self, GstH265NalUnit * nalu)
"duplicate_flag: %d", priv->cur_pic_struct,
priv->cur_source_scan_type, priv->cur_duplicate_flag);
break;
case GST_H265_SEI_RECOVERY_POINT:
GST_DEBUG_OBJECT (self, "Found recovery point");
priv->recovery_point = TRUE;
break;
default:
break;
}
@ -864,6 +889,19 @@ gst_h265_decoder_process_slice (GstH265Decoder * self, GstH265Slice * slice)
return ret;
}
if (!priv->have_idr) {
if (GST_H265_IS_NAL_TYPE_IRAP (slice->nalu.type)) {
GST_DEBUG_OBJECT (self, "Found IRAP picture");
priv->have_idr = TRUE;
} else if (priv->recovery_point) {
GST_DEBUG_OBJECT (self, "Found recovery point SEI");
priv->have_idr = TRUE;
} else if (!GST_H265_IS_I_SLICE (&slice->header)) {
GST_DEBUG_OBJECT (self, "Current picture requires reference frames");
priv->inter_predicted = TRUE;
}
}
priv->active_pps = priv->current_slice.header.pps;
priv->active_sps = priv->active_pps->sps;
@ -1987,6 +2025,25 @@ gst_h265_decoder_finish_current_picture (GstH265Decoder * self,
if (!priv->current_picture)
return;
if (!priv->have_idr && priv->inter_predicted) {
GST_DEBUG_OBJECT (self,
"IDR frame has not been observed yet, marking corrupted");
GST_CODEC_PICTURE (priv->current_picture)->corrupted = TRUE;
if (!priv->keyframe_requested) {
/* Don't spam keyframe request */
priv->keyframe_requested = TRUE;
gst_video_decoder_request_sync_point (GST_VIDEO_DECODER (self),
priv->current_frame, 0);
}
priv->missing_idr_count++;
if (priv->missing_idr_count > priv->max_corrupted_count) {
GST_DEBUG_OBJECT (self, "Missing IDR count %u > threshold %u",
priv->missing_idr_count, priv->max_corrupted_count);
priv->have_idr = TRUE;
}
}
klass = GST_H265_DECODER_GET_CLASS (self);
if (klass->end_picture) {
@ -1999,6 +2056,14 @@ gst_h265_decoder_finish_current_picture (GstH265Decoder * self,
}
}
if (flow_ret == GST_FLOW_OK &&
GST_CODEC_PICTURE (priv->current_picture)->corrupted) {
/* If subclass didn't revert the corrupted flag, set codec frame flag
* accordingly */
GST_VIDEO_CODEC_FRAME_FLAG_SET (priv->current_frame,
GST_VIDEO_CODEC_FRAME_FLAG_CORRUPTED);
}
/* finish picture takes ownership of the picture */
gst_h265_decoder_finish_picture (self, priv->current_picture, &flow_ret);
priv->current_picture = NULL;
@ -2017,6 +2082,8 @@ gst_h265_decoder_reset_frame_state (GstH265Decoder * self)
priv->cur_duplicate_flag = 0;
priv->no_output_of_prior_pics_flag = FALSE;
priv->current_frame = NULL;
priv->recovery_point = FALSE;
priv->inter_predicted = FALSE;
g_array_set_size (priv->nalu, 0);
}
@ -2100,7 +2167,6 @@ gst_h265_decoder_handle_frame (GstVideoDecoder * decoder,
}
gst_buffer_unmap (in_buf, &map);
gst_h265_decoder_reset_frame_state (self);
if (decode_ret != GST_FLOW_OK) {
if (decode_ret == GST_FLOW_ERROR) {

10
subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp8decoder.c
View file

@ -43,6 +43,7 @@ struct _GstVp8DecoderPrivate
gboolean had_sequence;
GstVp8Parser parser;
gboolean wait_keyframe;
gboolean keyframe_requested;
guint preferred_output_delay;
/* for delayed output */
GstQueueArray *output_queue;
@ -115,6 +116,7 @@ gst_vp8_decoder_start (GstVideoDecoder * decoder)
gst_vp8_parser_init (&priv->parser);
priv->wait_keyframe = TRUE;
priv->keyframe_requested = FALSE;
priv->output_queue =
gst_queue_array_new_for_struct (sizeof (GstVp8DecoderOutputFrame), 1);
@ -134,6 +136,7 @@ gst_vp8_decoder_reset (GstVp8Decoder * self)
gst_clear_vp8_picture (&self->alt_ref_picture);
priv->wait_keyframe = TRUE;
priv->keyframe_requested = FALSE;
gst_queue_array_clear (priv->output_queue);
}
@ -404,6 +407,12 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder,
GST_PTR_FORMAT, in_buf);
gst_buffer_unmap (in_buf, &map);
if (!priv->keyframe_requested) {
gst_video_decoder_request_sync_point (decoder, frame, 0);
priv->keyframe_requested = TRUE;
}
gst_video_decoder_release_frame (decoder, frame);
return GST_FLOW_OK;
@ -411,6 +420,7 @@ gst_vp8_decoder_handle_frame (GstVideoDecoder * decoder,
}
priv->wait_keyframe = FALSE;
priv->keyframe_requested = FALSE;
if (frame_hdr.key_frame) {
ret = gst_vp8_decoder_check_codec_change (self, &frame_hdr);

18
subprojects/gst-plugins-bad/gst-libs/gst/codecs/gstvp9decoder.c
View file

@ -80,6 +80,8 @@ struct _GstVp9DecoderPrivate
gboolean support_non_kf_change;
gboolean wait_keyframe;
gboolean keyframe_requested;
/* controls how many frames to delay when calling output_picture() */
guint preferred_output_delay;
GstQueueArray *output_queue;
@ -157,6 +159,7 @@ gst_vp9_decoder_start (GstVideoDecoder * decoder)
priv->parser = gst_vp9_stateful_parser_new ();
priv->dpb = gst_vp9_dpb_new ();
priv->wait_keyframe = TRUE;
priv->keyframe_requested = FALSE;
priv->profile = GST_VP9_PROFILE_UNDEFINED;
priv->frame_width = 0;
priv->frame_height = 0;
@ -305,6 +308,7 @@ gst_vp9_decoder_reset (GstVp9Decoder * self)
gst_vp9_dpb_clear (priv->dpb);
priv->wait_keyframe = TRUE;
priv->keyframe_requested = FALSE;
gst_queue_array_clear (priv->output_queue);
}
@ -434,7 +438,12 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder,
GST_DEBUG_OBJECT (self, "Drop frame before initial keyframe");
gst_buffer_unmap (in_buf, &map);
gst_video_decoder_release_frame (decoder, frame);;
if (!priv->keyframe_requested) {
gst_video_decoder_request_sync_point (decoder, frame, 0);
priv->keyframe_requested = TRUE;
}
gst_video_decoder_release_frame (decoder, frame);
return GST_FLOW_OK;
}
@ -450,6 +459,12 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder,
GST_DEBUG_OBJECT (self, "Drop frame on non-keyframe format change");
gst_buffer_unmap (in_buf, &map);
if (!priv->keyframe_requested) {
gst_video_decoder_request_sync_point (decoder, frame, 0);
priv->keyframe_requested = TRUE;
}
gst_video_decoder_release_frame (decoder, frame);
/* Drains frames if any and waits for keyframe again */
@ -462,6 +477,7 @@ gst_vp9_decoder_handle_frame (GstVideoDecoder * decoder,
}
priv->wait_keyframe = FALSE;
priv->keyframe_requested = FALSE;
if (frame_hdr.show_existing_frame) {
GstVp9Picture *pic_to_dup;