iphone - Performance issues when using AVCaptureVideoDataOutput and AVCaptureAudioDataOutput

Question

I'm having lag issues when I'm recording audio+video by using AVCaptureVideoDataOutput and AVCaptureAudioDataOutput. Sometimes the video blocks for a few milliseconds, sometimes the audio is not in sync with the video.

I inserted some logs and observed that first I get a lot of video buffers in captureOutput callback, and after some time I get the audio buffers(sometimes I don't receive the audio buffers at all, and the resulting video is without sound). If I comment the code that handles the video buffers, I get the audio buffers without problems.

This is the code I'm using:

-(void)initMovieOutput:(AVCaptureSession *)captureSessionLocal
{   
    AVCaptureVideoDataOutput *dataOutput = [[AVCaptureVideoDataOutput alloc] init];
    self._videoOutput = dataOutput;
    [dataOutput release];

    self._videoOutput.alwaysDiscardsLateVideoFrames = NO;
    self._videoOutput.videoSettings = [NSDictionary dictionaryWithObject: [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange]
                                                             forKey:(id)kCVPixelBufferPixelFormatTypeKey
                                  ];     
    AVCaptureAudioDataOutput *audioOutput =  [[AVCaptureAudioDataOutput alloc] init];
    self._audioOutput = audioOutput;
    [audioOutput release];

    [captureSessionLocal addOutput:self._videoOutput];
    [captureSessionLocal addOutput:self._audioOutput];


    // Setup the queue
    dispatch_queue_t queue = dispatch_queue_create("MyQueue", NULL);
    [self._videoOutput setSampleBufferDelegate:self queue:queue];
    [self._audioOutput setSampleBufferDelegate:self queue:queue];
    dispatch_release(queue);
}

Here I set up the writer:

-(BOOL) setupWriter:(NSURL *)videoURL session:(AVCaptureSession *)captureSessionLocal
{
    NSError *error = nil;
    self._videoWriter = [[AVAssetWriter alloc] initWithURL:videoURL fileType:AVFileTypeQuickTimeMovie
                                                         error:&error];
    NSParameterAssert(self._videoWriter);


    // Add video input  
    NSDictionary *videoSettings = [NSDictionary dictionaryWithObjectsAndKeys:
                                   AVVideoCodecH264, AVVideoCodecKey,
                                   [NSNumber numberWithInt:640], AVVideoWidthKey,
                                   [NSNumber numberWithInt:480], AVVideoHeightKey,
                                   nil];

    self._videoWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo
                                                                         outputSettings:videoSettings];


    NSParameterAssert(self._videoWriterInput);
    self._videoWriterInput.expectsMediaDataInRealTime = YES;
    self._videoWriterInput.transform = [self returnOrientation];

    // Add the audio input
    AudioChannelLayout acl;
    bzero( &acl, sizeof(acl));
    acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;


    NSDictionary* audioOutputSettings = nil;          
    // Both type of audio inputs causes output video file to be corrupted.

        // should work on any device requires more space
        audioOutputSettings = [ NSDictionary dictionaryWithObjectsAndKeys:                       
                               [ NSNumber numberWithInt: kAudioFormatAppleLossless ], AVFormatIDKey,
                               [ NSNumber numberWithInt: 16 ], AVEncoderBitDepthHintKey,
                               [ NSNumber numberWithFloat: 44100.0 ], AVSampleRateKey,
                               [ NSNumber numberWithInt: 1 ], AVNumberOfChannelsKey,                                      
                               [ NSData dataWithBytes: &acl length: sizeof( acl ) ], AVChannelLayoutKey,
                               nil ];

    self._audioWriterInput = [AVAssetWriterInput 
                                        assetWriterInputWithMediaType: AVMediaTypeAudio 
                                        outputSettings: audioOutputSettings ];

    self._audioWriterInput.expectsMediaDataInRealTime = YES;    

    // add input
    [self._videoWriter addInput:_videoWriterInput];
    [self._videoWriter addInput:_audioWriterInput];

    return YES;
}

And here is the callback:

- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
       fromConnection:(AVCaptureConnection *)connection
{

    if( !CMSampleBufferDataIsReady(sampleBuffer) )
    {
        NSLog( @"sample buffer is not ready. Skipping sample" );
        return;
    }
    if( _videoWriter.status !=  AVAssetWriterStatusCompleted )
    {
        if( _videoWriter.status != AVAssetWriterStatusWriting  )
        {               
            CMTime lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); 
            [_videoWriter startWriting];
            [_videoWriter startSessionAtSourceTime:lastSampleTime];
        }

        if( captureOutput == _videoOutput )
        {
            if( [self._videoWriterInput isReadyForMoreMediaData] )
            {

            [self newVideoSample:sampleBuffer];

            }
        }
        else if( captureOutput == _audioOutput )
        {
            if( [self._audioWriterInput isReadyForMoreMediaData] )
            {

                 [self newAudioSample:sampleBuffer];


            }
        }
    }

}

-(void) newAudioSample:(CMSampleBufferRef)sampleBuffer
{

        if( _videoWriter.status > AVAssetWriterStatusWriting )
        {

            [self NSLogPrint:[NSString stringWithFormat:@"Audio:Warning: writer status is %d", _videoWriter.status]];
            if( _videoWriter.status == AVAssetWriterStatusFailed )
                [self NSLogPrint:[NSString stringWithFormat:@"Audio:Error: %@", _videoWriter.error]];
            return;
        }

        if( ![_audioWriterInput appendSampleBuffer:sampleBuffer] )
            [self NSLogPrint:[NSString stringWithFormat:@"Unable to write to audio input"]];

}

-(void) newVideoSample:(CMSampleBufferRef)sampleBuffer
{
    if( _videoWriter.status > AVAssetWriterStatusWriting )
    {
        [self NSLogPrint:[NSString stringWithFormat:@"Video:Warning: writer status is %d", _videoWriter.status]];
        if( _videoWriter.status == AVAssetWriterStatusFailed )
            [self NSLogPrint:[NSString stringWithFormat:@"Video:Error: %@", _videoWriter.error]];
        return;
    }


    if( ![_videoWriterInput appendSampleBuffer:sampleBuffer] )
        [self NSLogPrint:[NSString stringWithFormat:@"Unable to write to video input"]];
}

Is there something wrong in my code, why does the video lag? (I'm testing it on a Iphone 4 ios 4.2.1)

Answer 1

It looks like you are using serial queues. The audio Output queue is right after the video output queue. Consider using concurrent queues.