AVCaptureSession 的設計是被動的:註冊成 delegate,系統準備好一帧就回呼你一次。對大多數應用這樣很合理,但如果你在做串流、手勢判斷或影格處理,你會希望「我要,就給我最新的一帧」——這種主動抓取模式下,帧率上限只取決於你的處理速度,而不是回呼的節奏。
這個工具類提供三件事:預覽畫面、自動依裝置方向旋轉、以及主動抓帧。
#import <Foundation/Foundation.h>
// 都是 4:3 的比例
typedef enum TKVideoFrameSize {
tkVideoFrame480x360 = 480 << 16 | 360,
tkVideoFrame720x540 = 720 << 16 | 540, // 這個解析度效率高很多
} TKVideoFrameSize;
@interface TKVideoCapture : NSObject
- (bool)create:(UIView *)preview frame:(TKVideoFrameSize)type;
- (bool)destory;
- (bool)start;
- (bool)stop;
// 回傳 BGRA 位元組序的影像資料
- (uint8_t *)get_image_rgb32:(uint32_t *)length;
@end
把寬高打包進一個 enum(高 16 位、寬 16 位)是個省字的小技巧,呼叫端寫起來也比傳兩個參數乾淨。
#import "TKVideoCapture.h"
#import <UIKit/UIKit.h>
#import <CoreGraphics/CoreGraphics.h>
#import <CoreVideo/CoreVideo.h>
#import <CoreMedia/CoreMedia.h>
#import "TKLock.h"
@interface TKVideoCapture ()
{
TKVideoFrameSize _frametype;
UIView *_preview;
AVCaptureSession *_captureSession;
AVCaptureVideoPreviewLayer *_capturePreview;
AVCaptureVideoDataOutput *_captureOutput;
AVCaptureDevice *_captureDevice;
AVCaptureDeviceInput *_captureInput;
uint8_t *_buffer_temp; // 每一帧都存在這裡
uint8_t *_buffer_obox; // 需要時從 temp 拷貝出來
CGRect _subImageRect; // 子圖(裁切)位置
TKLock *_buffer_lock;
}
@end
@implementation TKVideoCapture
- (void)do_create {
_captureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
_captureInput = [AVCaptureDeviceInput deviceInputWithDevice:_captureDevice error:nil];
_captureOutput = [[AVCaptureVideoDataOutput alloc] init];
if (_captureOutput) {
[_captureOutput setAlwaysDiscardsLateVideoFrames:true];
}
dispatch_queue_t queue = dispatch_queue_create("cameraQueue", NULL);
[_captureOutput setSampleBufferDelegate:self queue:queue];
dispatch_release(queue);
// 指定輸出格式為 32bit BGRA
NSString *key = (NSString *)kCVPixelBufferPixelFormatTypeKey;
NSNumber *value = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA];
NSDictionary *videoSettings = [NSDictionary dictionaryWithObject:value forKey:key];
[_captureOutput setVideoSettings:videoSettings];
_captureSession = [[AVCaptureSession alloc] init];
uint16_t width = (uint16_t)(((uint32_t)_frametype) >> 16);
uint16_t height = (uint16_t)(((uint32_t)_frametype) & 0xFFFF);
_buffer_temp = (uint8_t *)malloc(width * height * 4);
_buffer_obox = (uint8_t *)malloc(width * height * 4);
switch (_frametype) {
case tkVideoFrame480x360:
_captureSession.sessionPreset = AVCaptureSessionPreset640x480;
_subImageRect = CGRectMake((640 - 360) / 2, 0, 360, 480);
break;
case tkVideoFrame720x540:
_captureSession.sessionPreset = AVCaptureSessionPresetiFrame1280x720;
_subImageRect = CGRectMake((1280 - 540) / 2, 0, 540, 720);
break;
default:
break;
}
if (_captureInput != nil) {
[_captureSession addInput:_captureInput];
}
[_captureSession addOutput:_captureOutput];
_capturePreview = [AVCaptureVideoPreviewLayer layerWithSession:_captureSession];
_capturePreview.frame = _preview.bounds;
_capturePreview.videoGravity = AVLayerVideoGravityResizeAspectFill;
_capturePreview.connection.videoOrientation = [self getOrientation];
[_preview.layer addSublayer:_capturePreview];
_buffer_lock = [[TKLock alloc] init];
[_buffer_lock open];
}
- (bool)create:(UIView *)preview frame:(TKVideoFrameSize)type {
_frametype = type;
_preview = preview;
// AVCaptureSession 的設定必須在主執行緒
[self performSelectorOnMainThread:@selector(do_create) withObject:self waitUntilDone:true];
return true;
}
- (AVCaptureVideoOrientation)getOrientation {
UIInterfaceOrientation orientation = [UIApplication sharedApplication].statusBarOrientation;
switch (orientation) {
case UIInterfaceOrientationPortrait: return AVCaptureVideoOrientationPortrait;
case UIInterfaceOrientationPortraitUpsideDown: return AVCaptureVideoOrientationPortraitUpsideDown;
case UIInterfaceOrientationLandscapeLeft: return AVCaptureVideoOrientationLandscapeLeft;
case UIInterfaceOrientationLandscapeRight: return AVCaptureVideoOrientationLandscapeRight;
}
return AVCaptureVideoOrientationLandscapeLeft;
}
- (void)do_destory {
[_buffer_lock close];
[_buffer_lock release];
_buffer_lock = nil;
free(_buffer_temp);
free(_buffer_obox);
_buffer_temp = NULL;
_buffer_obox = NULL;
[_captureSession stopRunning];
[_capturePreview removeFromSuperlayer];
[_captureOutput release];
[_captureSession release];
_captureSession = nil;
_capturePreview = nil;
_captureOutput = nil;
_captureDevice = nil;
_captureInput = nil;
_preview = nil;
}
- (bool)destory {
[self performSelectorOnMainThread:@selector(do_destory) withObject:self waitUntilDone:true];
return true;
}
#pragma mark 系統回呼:把畫面轉成直向
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(imageBuffer, 0);
uint8_t *baseAddress = (uint8_t *)CVPixelBufferGetBaseAddress(imageBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
CGDataProviderRef provider = CGDataProviderCreateWithData(NULL, baseAddress,
bytesPerRow * height, NULL);
CGImageRef imageRef = CGImageCreate(width, height, 8, 32, bytesPerRow, colorSpace,
kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst,
provider, NULL, false, kCGRenderingIntentDefault);
// 從中間裁出 4:3 的範圍
CGImageRef subImageRef = CGImageCreateWithImageInRect(imageRef, _subImageRect);
size_t subWidth = _subImageRect.size.height;
size_t subHeight = _subImageRect.size.width;
CGContextRef context = CGBitmapContextCreate(NULL, subWidth, subHeight,
CGImageGetBitsPerComponent(subImageRef), 0,
CGImageGetColorSpace(subImageRef), CGImageGetBitmapInfo(subImageRef));
// 旋轉 90 度,把橫的轉成直的
CGContextTranslateCTM(context, 0, subHeight);
CGContextRotateCTM(context, -M_PI / 2);
CGContextDrawImage(context, CGRectMake(0, 0, subHeight, subWidth), subImageRef);
uint8_t *data = (uint8_t *)CGBitmapContextGetData(context);
[_buffer_lock lock];
memcpy(_buffer_temp, data, subWidth * subHeight * 4);
[_buffer_lock unlock];
CGContextRelease(context);
CGImageRelease(imageRef);
CGImageRelease(subImageRef);
CGDataProviderRelease(provider);
CGColorSpaceRelease(colorSpace);
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
}
#pragma mark 控制
- (void)do_start { [_captureSession startRunning]; }
- (void)do_stop { [_captureSession stopRunning]; }
- (bool)start {
[self performSelectorOnMainThread:@selector(do_start) withObject:self waitUntilDone:true];
return true;
}
- (bool)stop {
[self performSelectorOnMainThread:@selector(do_stop) withObject:self waitUntilDone:true];
return true;
}
#pragma mark 主動抓帧
- (uint8_t *)get_image_rgb32:(uint32_t *)length {
uint16_t width = (uint16_t)(((uint32_t)_frametype) >> 16);
uint16_t height = (uint16_t)(((uint32_t)_frametype) & 0xFFFF);
[_buffer_lock lock];
memcpy(_buffer_obox, _buffer_temp, width * height * 4);
[_buffer_lock unlock];
if (length) *length = width * height * 4;
return _buffer_obox;
}
@end
雙緩衝:_buffer_temp 由回呼寫入,_buffer_obox 由呼叫端讀取。少了這一層,呼叫端拿到的可能是寫到一半的畫面(畫面撕裂)。鎖只保護 memcpy 那一瞬間,所以抓帧的等待時間很短。
回呼佇列:delegate 的 queue 刻意用自訂的 serial queue 而不是主佇列,因為裡面做了裁切、旋轉、格式轉換這些 CPU 密集工作,放在主執行緒會直接讓 UI 卡住。
為什麼要旋轉:多數手機姿勢下 sensor 輸出是橫的,預覽層有 videoOrientation 可以自動處理,但你自己拿到的原始位圖不會跟著轉,所以必須手動轉一次,否則串流出去會是躺倒的。
這份程式碼示範的觀念(雙緩衝、序列佇列、格式指定)都還成立,但有幾處在今天是必須修的:第一,相機權限要在使用前呼叫 AVCaptureDevice requestAccessForMediaType: 並在 Info.plist 寫入 NSCameraUsageDescription,否則 App 會直接被終止。第二,[UIApplication sharedApplication].statusBarOrientation 已廢棄,方向應該從 windowScene.interfaceOrientation 或直接把 AVCaptureConnection.videoOrientation 交給系統處理。第三,反覆用 CoreGraphics 做旋轉與縮放非常耗 CPU,實務上應該改用 CVPixelBufferLockBaseAddress 直接讀 BGRA 交給 GPU,或在 AVCaptureVideoDataOutput 上設定 videoOrientation 讓系統代勞。第四,這裡的 ARC 相關 release 語法需要移除。
另外提醒,別忘了在 dealloc 中呼叫 destory,否則 session 一直在跑,電量會以你看得見的速度消失。