video - FFmpeg c++ H264 decoding error

Question

I have a program which capture video from webcam, encode with ffmpeg, encoded packet then write to buffer. At the receiver side, read from buffer decode with ffmpeg and play.

Now I merge sender and receiver in one program for testing. It works fine with AV_CODEC_ID_MPEG1VIDEO, but when I change the ffmpeg codec to AV_CODEC_ID_H264, at the decoding progress, it shows error:

The whole program is here FYI, I made a loop to let the whole progress run twice.

What is the cause of the error, is there anything special for H264? Thanks in advance!

#include <math.h>

extern "C" {

#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <libavutil/channel_layout.h>
#include <libavutil/common.h>
#include <libavutil/imgutils.h>
#include <libavutil/mathematics.h>
#include <libavutil/samplefmt.h>
#include <libswscale/swscale.h>
#include "v4l2.h"
}
#include "opencv2/highgui/highgui.hpp"
#include <iostream>

using namespace cv;
using namespace std;
#define INBUF_SIZE 4096
static uint8_t inbuf[INBUF_SIZE + FF_INPUT_BUFFER_PADDING_SIZE];



static AVCodec *codec;
static AVCodecContext *c= NULL;
static int  ret,   got_output;
static int frame_count;
static FILE *f;

static AVPacket pkt;
static AVFrame *frame;
static AVFrame *frameDecode;
static AVFrame *framergb;
static uint8_t endcode[] = { 0, 0, 1, 0xb7 };
static AVPacket avpkt;
int totalSize=0;

#define SUBSITY     3



static int decode_write_frame(AVCodecContext *avctx,
                          AVFrame *frame, int *frame_count, AVPacket *pkt, int last)
{
int len, got_frame;
char buf[1024];
struct SwsContext *convert_ctx;
Mat m;
AVFrame dst;

len = avcodec_decode_video2(avctx, frame, &got_frame, pkt);
if (len < 0) {
    fprintf(stderr, "Error while decoding frame %d
", *frame_count);
    return len;
}
if (got_frame) {
    printf("Saving %s frame %3d
", last ? "last " : "", *frame_count);
    fflush(stdout);

int w = avctx->width;
int h = avctx->height;

/*convert AVFrame to opencv Mat frame*/

m = cv::Mat(h, w, CV_8UC3);
dst.data[0] = (uint8_t *)m.data;
avpicture_fill( (AVPicture *)&dst, dst.data[0], PIX_FMT_BGR24, w, h);

enum PixelFormat src_pixfmt = (enum PixelFormat)frame->format;
enum PixelFormat dst_pixfmt = PIX_FMT_BGR24;
convert_ctx = sws_getContext(w, h, src_pixfmt, w, h, dst_pixfmt,
                    SWS_FAST_BILINEAR, NULL, NULL, NULL);

if(convert_ctx == NULL) {
    fprintf(stderr, "Cannot initialize the conversion context!
");
    exit(1);
}

sws_scale(convert_ctx, frame->data, frame->linesize, 0, h,
                    dst.data, dst.linesize);


    imshow("MyVideo", m);
    //video.write(m);
    waitKey(10); //wait next frame time


    (*frame_count)++;
}
if (pkt->data) {
    pkt->size -= len;
    pkt->data += len;
}
return 0;
}


static void video_decode_example(char *inbufout)
{
int bytes;
uint8_t *buffer;

av_init_packet(&avpkt);


memset(inbuf + INBUF_SIZE, 0, FF_INPUT_BUFFER_PADDING_SIZE);


codec = avcodec_find_decoder(AV_CODEC_ID_H264);
if (!codec) {
    fprintf(stderr, "Codec not found
");
    exit(1);
}

c = avcodec_alloc_context3(codec);
if (!c) {
    fprintf(stderr, "Could not allocate video codec context
");
    exit(1);
}

if(codec->capabilities&CODEC_CAP_TRUNCATED)
    c->flags|= CODEC_FLAG_TRUNCATED; /* we do not send complete frames */


/* open it */
if (avcodec_open2(c, codec, NULL) < 0) {
    fprintf(stderr, "Could not open codec
");
    exit(1);
}


frameDecode = avcodec_alloc_frame();
if (!frameDecode) {
    fprintf(stderr, "Could not allocate video frame
");
    exit(1);
}


bytes=avpicture_get_size(PIX_FMT_RGB24, CAMER_WIDTH, CAMER_HEIGHT);
buffer=(uint8_t *)av_malloc(bytes*sizeof(uint8_t));
avpicture_fill((AVPicture *)framergb, buffer, PIX_FMT_RGB24,
                CAMER_WIDTH, CAMER_HEIGHT);*/

frame_count = 0;

namedWindow("MyVideo",CV_WINDOW_AUTOSIZE); //create a window called "MyVideo"


int size1=0;
for(;;) {

    memcpy(inbuf,inbufout+size1,INBUF_SIZE);
    size1+=INBUF_SIZE;
    if (size1>(totalSize-INBUF_SIZE))
        break;
    avpkt.size=INBUF_SIZE;


    avpkt.data = inbuf;

    /*frame by frame process*/

    while (avpkt.size > 0)
        if (decode_write_frame(c, frameDecode, &frame_count, &avpkt, 0) < 0)
            exit(1);
}
avpkt.data = NULL;
avpkt.size = 0;
decode_write_frame(c, frameDecode, &frame_count, &avpkt, 1);
}
static void init_video_encode(const char *filename, AVCodecID codec_id, int max_f)
{

printf("Encode video file %s
", filename);

/* find the mpeg1 video encoder */
codec = avcodec_find_encoder(codec_id);
if (!codec) {
    fprintf(stderr, "Codec not found
");
    exit(1);
}

c = avcodec_alloc_context3(codec);
if (!c) {
    fprintf(stderr, "Could not allocate video codec context
");
    exit(1);
}

/* put sample parameters */
c->bit_rate = 400000;
/* resolution must be a multiple of two */
c->width = 640;
c->height = 480;
/* frames per second */
c->time_base= (AVRational){1,25};
c->gop_size = 10; /* emit one intra frame every ten frames */
c->max_b_frames=max_f;
c->pix_fmt = AV_PIX_FMT_YUV420P;

if(codec_id == AV_CODEC_ID_H264)
    av_opt_set(c->priv_data, "preset", "slow", 0);

/* open it */
if (avcodec_open2(c, codec, NULL) < 0) {
    fprintf(stderr, "Could not open codec
");
    exit(1);
}

frame = avcodec_alloc_frame();
if (!frame) {
    fprintf(stderr, "Could not allocate video frame
");
    exit(1);
}
frame->format = c->pix_fmt;
frame->width  = c->width;
frame->height = c->height;


ret = av_image_alloc(frame->data, frame->linesize, c->width, c->height,
                     c->pix_fmt, 32);

/* get the delayed frames */
if (ret < 0) {
    fprintf(stderr, "Could not allocate raw picture buffer
");
    exit(1);
}

printf("
");
}

int video_encode(int frameNo,char *inbufout)
{
static int count = 0;
static int i = 0;

/* encode 1 frame of video */
av_init_packet(&pkt);
pkt.data = NULL;    // packet data will be allocated by the encoder
pkt.size = 0;
//cout<<"
Before YUV
";
if(count == 0)
read_yuv420(frame->data[0]);
count ++;

if(count == SUBSITY) {
count = 0;
}

frame->pts = i++;

/* encode the image */
ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
if (ret < 0) {
     fprintf(stderr, "Error encoding frame
");
     return -1;
}

if (got_output) {
     printf("Write frame %3d (size=%5d)
", i, pkt.size);
     memcpy(inbufout+totalSize,pkt.data,pkt.size);
     totalSize+=pkt.size;
     fwrite(pkt.data, 1, pkt.size, f);

     av_free_packet(&pkt);
}
return 0;
}

void cancle_encode(void)
{
fclose(f);
avcodec_close(c);
av_free(c);
av_freep(&frame->data[0]);
avcodec_free_frame(&frame);
}


int main(int argc, char **argv)
{
int i;


char inbufout[25*50*(INBUF_SIZE + FF_INPUT_BUFFER_PADDING_SIZE)];
if(init_v4l2() < 0) {
printf("can't open camera
");
return 0;
}

/* register all the codecs */
avcodec_register_all();



for(int j=0;j<2;j++){
    //init_video_encode("test.mpg", AV_CODEC_ID_MPEG1VIDEO, 15);
    init_video_encode("test.mpg", AV_CODEC_ID_H264, 15);
    //for(i = 0;i< 10*15;i++ ) {
    for(i = 0;i< 25*10;i++ ) {
    if(video_encode(i,inbufout) < 0)
        return 0;
    }
    cout<<"
"<<totalSize<<"
"<<endl;
    video_decode_example(inbufout);
    cancle_encode();
    totalSize=0;
}
exit_v4l2();


return 0;
}

Answer 1

You need to include a parser. The ffmpeg mpeg1/2 decoders happen to work fine without a parser, but h264/mpeg4/vp9 need a parser, or you'll get errors like the above.

Note that if you use libavformat for demuxing and call avformat_read_frame(), it will automatically parse for you, but since you're doing buffer management yourself, you need to include the parser yourself also.