685 lines
25 KiB
Swift
685 lines
25 KiB
Swift
import Cocoa
|
|
import FlutterMacOS
|
|
import CoreGraphics
|
|
import ScreenCaptureKit
|
|
import CoreMedia
|
|
import CoreVideo
|
|
import VideoToolbox
|
|
|
|
@main
|
|
class AppDelegate: FlutterAppDelegate {
|
|
private var inputChannel: FlutterMethodChannel?
|
|
private var videoChannel: FlutterMethodChannel?
|
|
private var frameEventSink: FlutterEventSink?
|
|
|
|
// SCK objects (typed Any? to satisfy 10.15 deployment target)
|
|
private var scStream: Any?
|
|
private var screenDelegate: Any?
|
|
private var streamReady = false
|
|
|
|
// VTCompressionSession (Any? for same reason)
|
|
private var vtSession: Any?
|
|
private var encoderReady = false
|
|
|
|
// resolution last seen from SCK (to build config)
|
|
private var captureWidth = 1280
|
|
private var captureHeight = 720
|
|
|
|
// viewer decoder sessions keyed by texture id
|
|
private var decoders: [Int64: VideoDecoderSession] = [:]
|
|
private var textureRegistry: FlutterTextureRegistry?
|
|
|
|
override func applicationDidFinishLaunching(_ notification: Notification) {
|
|
guard let window = mainFlutterWindow,
|
|
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
|
|
|
let messenger = ctrl.engine.binaryMessenger
|
|
textureRegistry = ctrl.engine.registrar(forPlugin: "PulsarTextures").textures
|
|
|
|
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
|
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
|
if call.method == "injectInput",
|
|
let args = call.arguments as? [String: Any] {
|
|
self?.injectInput(args)
|
|
result(nil)
|
|
} else {
|
|
result(FlutterMethodNotImplemented)
|
|
}
|
|
}
|
|
|
|
// event channel for pushing encoded NAL units to Flutter
|
|
let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
|
|
frameChannel.setStreamHandler(self)
|
|
|
|
videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
|
|
videoChannel?.setMethodCallHandler { [weak self] call, result in
|
|
guard let self = self else { return }
|
|
switch call.method {
|
|
case "start":
|
|
self.startEncoder(result: result)
|
|
case "stop":
|
|
self.stopEncoder()
|
|
result(nil)
|
|
case "forceKeyframe":
|
|
self.requestKeyframe()
|
|
result(nil)
|
|
|
|
case "createDecoder":
|
|
guard let args = call.arguments as? [String: Any],
|
|
let spsPpsTyped = args["spsPps"] as? FlutterStandardTypedData,
|
|
let registry = self.textureRegistry else {
|
|
result(FlutterError(code: "INVALID_ARGS", message: nil, details: nil))
|
|
return
|
|
}
|
|
if let decoder = VideoDecoderSession(spsPps: spsPpsTyped.data, textureRegistry: registry) {
|
|
self.decoders[decoder.textureId] = decoder
|
|
result(["textureId": decoder.textureId])
|
|
} else {
|
|
result(FlutterError(code: "DECODER_FAILED", message: "Failed to create H264 decoder", details: nil))
|
|
}
|
|
|
|
case "feedNal":
|
|
guard let args = call.arguments as? [String: Any],
|
|
let textureId = (args["textureId"] as? NSNumber)?.int64Value,
|
|
let nalTyped = args["nal"] as? FlutterStandardTypedData,
|
|
let decoder = self.decoders[textureId] else {
|
|
result(nil)
|
|
return
|
|
}
|
|
decoder.feedNal(nalTyped.data)
|
|
result(nil)
|
|
|
|
case "stopDecoder":
|
|
if let args = call.arguments as? [String: Any],
|
|
let textureId = (args["textureId"] as? NSNumber)?.int64Value {
|
|
self.decoders[textureId]?.invalidate()
|
|
self.decoders.removeValue(forKey: textureId)
|
|
}
|
|
result(nil)
|
|
|
|
default:
|
|
result(FlutterMethodNotImplemented)
|
|
}
|
|
}
|
|
|
|
if #available(macOS 12.3, *) {
|
|
Task { await self.ensureSCKStream() }
|
|
}
|
|
}
|
|
|
|
// MARK: - SCK stream setup
|
|
|
|
@available(macOS 12.3, *)
|
|
private func ensureSCKStream() async {
|
|
guard scStream == nil else { return }
|
|
|
|
guard let content = try? await SCShareableContent.current else { return }
|
|
guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
|
|
|
|
let filter = SCContentFilter(display: display, excludingWindows: [])
|
|
|
|
let srcW = Double(display.width)
|
|
let srcH = Double(display.height)
|
|
let scale = min(1.0, 1280.0 / srcW)
|
|
let dstW = Int(srcW * scale)
|
|
let dstH = Int(srcH * scale)
|
|
|
|
captureWidth = dstW
|
|
captureHeight = dstH
|
|
|
|
let cfg = SCStreamConfiguration()
|
|
cfg.width = dstW
|
|
cfg.height = dstH
|
|
cfg.pixelFormat = kCVPixelFormatType_32BGRA
|
|
cfg.showsCursor = true
|
|
cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
|
|
|
|
let del = ScreenDelegate { [weak self] sampleBuffer in
|
|
self?.encodeFrame(sampleBuffer)
|
|
}
|
|
screenDelegate = del
|
|
|
|
let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
|
|
try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
|
|
|
|
do {
|
|
try await stream.startCapture()
|
|
scStream = stream
|
|
streamReady = true
|
|
} catch {}
|
|
}
|
|
|
|
// MARK: - VTCompressionSession
|
|
|
|
private func startEncoder(result: @escaping FlutterResult) {
|
|
let w = captureWidth
|
|
let h = captureHeight
|
|
|
|
var session: VTCompressionSession?
|
|
let status = VTCompressionSessionCreate(
|
|
allocator: kCFAllocatorDefault,
|
|
width: Int32(w),
|
|
height: Int32(h),
|
|
codecType: kCMVideoCodecType_H264,
|
|
encoderSpecification: nil,
|
|
imageBufferAttributes: nil,
|
|
compressedDataAllocator: nil,
|
|
outputCallback: vtOutputCallback,
|
|
refcon: Unmanaged.passUnretained(self).toOpaque(),
|
|
compressionSessionOut: &session
|
|
)
|
|
|
|
guard status == noErr, let session = session else {
|
|
result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
|
|
return
|
|
}
|
|
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
|
|
value: kVTProfileLevel_H264_Baseline_AutoLevel)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
|
|
value: 6_000_000 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
|
|
value: 60 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
|
|
value: 120 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
|
|
value: 0 as CFNumber)
|
|
|
|
VTCompressionSessionPrepareToEncodeFrames(session)
|
|
vtSession = VTSessionBox(session)
|
|
encoderReady = true
|
|
|
|
// extract SPS+PPS from a forced keyframe by encoding a dummy frame
|
|
// instead, return empty bytes and let the real SPS+PPS come through the callback
|
|
// host_screen will call forceKeyframe right after start
|
|
result(FlutterStandardTypedData(bytes: Data()))
|
|
}
|
|
|
|
private func stopEncoder() {
|
|
guard let box = vtSession as? VTSessionBox else { return }
|
|
VTCompressionSessionInvalidate(box.session)
|
|
vtSession = nil
|
|
encoderReady = false
|
|
}
|
|
|
|
func requestKeyframe() {
|
|
forceKeyframeNext = true
|
|
}
|
|
|
|
private var forceKeyframeNext = false
|
|
|
|
private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
|
|
guard encoderReady, let box = vtSession as? VTSessionBox else { return }
|
|
let s = box.session
|
|
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
|
|
|
var frameProps: CFDictionary? = nil
|
|
if forceKeyframeNext {
|
|
frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
|
|
forceKeyframeNext = false
|
|
}
|
|
|
|
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
|
VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
|
|
duration: .invalid, frameProperties: frameProps,
|
|
sourceFrameRefcon: nil, infoFlagsOut: nil)
|
|
}
|
|
|
|
// MARK: - VT output callback (C function)
|
|
|
|
// called by VideoToolbox on an internal thread — collect Data packets here,
|
|
// then hop to main before touching the event sink
|
|
func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
|
|
var packets: [Data] = []
|
|
|
|
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
|
|
let isKeyframe = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
|
|
|
|
if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
|
|
if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
|
|
}
|
|
|
|
if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
|
|
var totalLen = 0
|
|
var rawPtr: UnsafeMutablePointer<CChar>? = nil
|
|
if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
|
|
totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
|
|
let ptr = rawPtr {
|
|
|
|
let ts = Int64(Date().timeIntervalSince1970 * 1000)
|
|
var offset = 0
|
|
|
|
while offset + 4 <= totalLen {
|
|
// read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
|
|
let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
|
|
let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
|
|
let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
|
|
let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
|
|
let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
|
|
offset += 4
|
|
|
|
guard naluLen > 0, offset + naluLen <= totalLen else { break }
|
|
|
|
var packet = Data(capacity: 1 + 8 + 4 + naluLen)
|
|
packet.append(0x02)
|
|
withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
|
|
packets.append(packet)
|
|
|
|
offset += naluLen
|
|
}
|
|
}
|
|
}
|
|
|
|
guard !packets.isEmpty else { return }
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let sink = self?.frameEventSink else { return }
|
|
for packet in packets {
|
|
sink(FlutterStandardTypedData(bytes: packet))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
|
|
var spsCount = 0
|
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
|
parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
|
|
nalUnitHeaderLengthOut: nil)
|
|
var packet = Data()
|
|
packet.append(0x01)
|
|
|
|
for i in 0 ..< spsCount {
|
|
var ptr: UnsafePointer<UInt8>? = nil
|
|
var len = 0
|
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
|
|
parameterSetPointerOut: &ptr,
|
|
parameterSetSizeOut: &len,
|
|
parameterSetCountOut: nil,
|
|
nalUnitHeaderLengthOut: nil)
|
|
if let ptr = ptr {
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ptr, count: len))
|
|
}
|
|
}
|
|
|
|
// PPS sits at index spsCount
|
|
var ppsPtr: UnsafePointer<UInt8>? = nil
|
|
var ppsLen = 0
|
|
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
|
|
parameterSetPointerOut: &ppsPtr,
|
|
parameterSetSizeOut: &ppsLen,
|
|
parameterSetCountOut: nil,
|
|
nalUnitHeaderLengthOut: nil) == noErr,
|
|
let ppsPtr = ppsPtr {
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ppsPtr, count: ppsLen))
|
|
}
|
|
|
|
return packet.count > 1 ? packet : nil
|
|
}
|
|
|
|
// MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths — unused in normal operation)
|
|
|
|
private func captureCG() -> Data? {
|
|
let displayID = CGMainDisplayID()
|
|
guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
|
|
let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
|
|
let srcW = CGFloat(composited.width)
|
|
let scale = min(1.0, 1280.0 / srcW)
|
|
let dstW = Int(srcW * scale)
|
|
let dstH = Int(CGFloat(composited.height) * scale)
|
|
let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
|
guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
|
|
bytesPerRow: 0, space: colorSpace,
|
|
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
|
|
ctx.interpolationQuality = .high
|
|
ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
|
|
guard let scaled = ctx.makeImage() else { return nil }
|
|
return jpegData(from: scaled, quality: 0.4)
|
|
}
|
|
|
|
private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
|
|
let nsImage = NSImage(cgImage: cgImage, size: .zero)
|
|
guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
|
|
return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
|
|
}
|
|
|
|
private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
|
|
let imgW = cgImage.width
|
|
let imgH = cgImage.height
|
|
let mousePt = NSEvent.mouseLocation
|
|
let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
|
|
let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
|
|
let cursorX = mousePt.x * scale
|
|
let cursorY = (screenH - mousePt.y) * scale
|
|
let cursor = NSCursor.current
|
|
let cursorImg = cursor.image
|
|
let hotspot = cursor.hotSpot
|
|
let cw = cursorImg.size.width * scale
|
|
let ch = cursorImg.size.height * scale
|
|
let drawX = cursorX - hotspot.x * scale
|
|
let drawY = cursorY - hotspot.y * scale
|
|
let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
|
guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
|
|
bytesPerRow: 0, space: colorSpace,
|
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
|
|
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
|
|
var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
|
|
if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
|
|
ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
|
|
}
|
|
return ctx.makeImage()
|
|
}
|
|
|
|
// MARK: - Input injection
|
|
|
|
private func injectInput(_ ev: [String: Any]) {
|
|
guard let type = ev["type"] as? String else { return }
|
|
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
|
|
|
|
switch type {
|
|
case "move":
|
|
let nx = ev["x"] as? Double ?? 0
|
|
let ny = ev["y"] as? Double ?? 0
|
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
|
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
|
|
.post(tap: .cghidEventTap)
|
|
|
|
case "click":
|
|
let nx = ev["x"] as? Double ?? 0
|
|
let ny = ev["y"] as? Double ?? 0
|
|
let btn = ev["button"] as? Int ?? 0
|
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
|
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
|
|
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
|
|
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
|
.post(tap: .cghidEventTap)
|
|
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
|
.post(tap: .cghidEventTap)
|
|
|
|
case "keydown":
|
|
if let code = ev["keyCode"] as? Int {
|
|
CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(code & 0xFFFF), keyDown: true)?
|
|
.post(tap: .cghidEventTap)
|
|
}
|
|
|
|
case "keyup":
|
|
if let code = ev["keyCode"] as? Int {
|
|
CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(code & 0xFFFF), keyDown: false)?
|
|
.post(tap: .cghidEventTap)
|
|
}
|
|
|
|
default: break
|
|
}
|
|
}
|
|
|
|
// MARK: - App lifecycle
|
|
|
|
override func applicationWillTerminate(_ notification: Notification) {
|
|
stopEncoder()
|
|
if #available(macOS 12.3, *), let stream = scStream as? SCStream {
|
|
Task { try? await stream.stopCapture() }
|
|
}
|
|
}
|
|
|
|
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
|
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
|
|
}
|
|
|
|
// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
|
|
|
|
private class VTSessionBox {
|
|
let session: VTCompressionSession
|
|
init(_ s: VTCompressionSession) { session = s }
|
|
}
|
|
|
|
// MARK: - VT output C callback
|
|
|
|
private func vtOutputCallback(
|
|
outputCallbackRefCon: UnsafeMutableRawPointer?,
|
|
sourceFrameRefCon: UnsafeMutableRawPointer?,
|
|
status: OSStatus,
|
|
infoFlags: VTEncodeInfoFlags,
|
|
sampleBuffer: CMSampleBuffer?
|
|
) {
|
|
guard status == noErr, let sampleBuffer = sampleBuffer,
|
|
let refcon = outputCallbackRefCon else { return }
|
|
let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
|
|
delegate.handleEncodedFrame(sampleBuffer)
|
|
}
|
|
|
|
// MARK: - SCK delegate
|
|
|
|
@available(macOS 12.3, *)
|
|
class ScreenDelegate: NSObject, SCStreamOutput {
|
|
private let onFrame: (CMSampleBuffer) -> Void
|
|
|
|
init(onFrame: @escaping (CMSampleBuffer) -> Void) {
|
|
self.onFrame = onFrame
|
|
}
|
|
|
|
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
|
guard type == .screen else { return }
|
|
onFrame(sampleBuffer)
|
|
}
|
|
}
|
|
|
|
// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
|
|
|
|
extension AppDelegate: FlutterStreamHandler {
|
|
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
|
|
frameEventSink = events
|
|
return nil
|
|
}
|
|
|
|
func onCancel(withArguments arguments: Any?) -> FlutterError? {
|
|
frameEventSink = nil
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// MARK: - VT decompression C callback
|
|
|
|
private func vtDecompressOutputCallback(
|
|
refCon: UnsafeMutableRawPointer?,
|
|
frameRefCon: UnsafeMutableRawPointer?,
|
|
status: OSStatus,
|
|
infoFlags: VTDecodeInfoFlags,
|
|
imageBuffer: CVImageBuffer?,
|
|
pts: CMTime,
|
|
duration: CMTime
|
|
) {
|
|
guard let refCon = refCon, status == noErr, let imageBuffer = imageBuffer else { return }
|
|
let session = Unmanaged<VideoDecoderSession>.fromOpaque(refCon).takeUnretainedValue()
|
|
session.handleDecodedFrame(imageBuffer)
|
|
}
|
|
|
|
// MARK: - VideoDecoderSession
|
|
|
|
class VideoDecoderSession: NSObject, FlutterTexture {
|
|
private(set) var textureId: Int64 = -1
|
|
|
|
private var decompressionSession: VTDecompressionSession?
|
|
private var formatDescription: CMVideoFormatDescription?
|
|
private var latestPixelBuffer: CVPixelBuffer?
|
|
private let lock = NSLock()
|
|
private weak var textureRegistry: FlutterTextureRegistry?
|
|
|
|
init?(spsPps: Data, textureRegistry: FlutterTextureRegistry) {
|
|
self.textureRegistry = textureRegistry
|
|
super.init()
|
|
guard setupSession(spsPps: spsPps) else { return nil }
|
|
textureId = textureRegistry.register(self)
|
|
}
|
|
|
|
// MARK: Parse Annex-B SPS+PPS and create VTDecompressionSession
|
|
|
|
private func setupSession(spsPps: Data) -> Bool {
|
|
// Split on 4-byte Annex-B start codes to get raw NALU payloads
|
|
let bytes = [UInt8](spsPps)
|
|
var nalus: [Data] = []
|
|
var scanStart = 0
|
|
var i = 0
|
|
while i + 4 <= bytes.count {
|
|
if bytes[i] == 0 && bytes[i+1] == 0 && bytes[i+2] == 0 && bytes[i+3] == 1 {
|
|
if i > scanStart { nalus.append(Data(bytes[scanStart..<i])) }
|
|
scanStart = i + 4
|
|
i += 4
|
|
} else {
|
|
i += 1
|
|
}
|
|
}
|
|
if scanStart < bytes.count { nalus.append(Data(bytes[scanStart...])) }
|
|
guard nalus.count >= 2 else { return false }
|
|
|
|
let sps = nalus[0]
|
|
let pps = nalus[1]
|
|
|
|
var formatDesc: CMVideoFormatDescription?
|
|
let fmtStatus: OSStatus = sps.withUnsafeBytes { spsBuf in
|
|
pps.withUnsafeBytes { ppsBuf in
|
|
var ptrs: [UnsafePointer<UInt8>] = [
|
|
spsBuf.bindMemory(to: UInt8.self).baseAddress!,
|
|
ppsBuf.bindMemory(to: UInt8.self).baseAddress!
|
|
]
|
|
var sizes: [Int] = [sps.count, pps.count]
|
|
return CMVideoFormatDescriptionCreateFromH264ParameterSets(
|
|
allocator: kCFAllocatorDefault,
|
|
parameterSetCount: 2,
|
|
parameterSetPointers: &ptrs,
|
|
parameterSetSizes: &sizes,
|
|
nalUnitHeaderLength: 4,
|
|
formatDescriptionOut: &formatDesc
|
|
)
|
|
}
|
|
}
|
|
guard fmtStatus == noErr, let formatDesc = formatDesc else { return false }
|
|
self.formatDescription = formatDesc
|
|
|
|
let dims = CMVideoFormatDescriptionGetDimensions(formatDesc)
|
|
let imageAttrs: [CFString: Any] = [
|
|
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
|
|
kCVPixelBufferWidthKey: Int(dims.width),
|
|
kCVPixelBufferHeightKey: Int(dims.height),
|
|
kCVPixelBufferIOSurfacePropertiesKey: [:] as [String: Any]
|
|
]
|
|
|
|
var callback = VTDecompressionOutputCallbackRecord(
|
|
decompressionOutputCallback: vtDecompressOutputCallback,
|
|
decompressionOutputRefCon: Unmanaged.passUnretained(self).toOpaque()
|
|
)
|
|
|
|
var session: VTDecompressionSession?
|
|
let sessionStatus = VTDecompressionSessionCreate(
|
|
allocator: kCFAllocatorDefault,
|
|
formatDescription: formatDesc,
|
|
decoderSpecification: nil,
|
|
imageBufferAttributes: imageAttrs as CFDictionary,
|
|
outputCallback: &callback,
|
|
decompressionSessionOut: &session
|
|
)
|
|
guard sessionStatus == noErr, let session = session else { return false }
|
|
self.decompressionSession = session
|
|
return true
|
|
}
|
|
|
|
// MARK: Feed an Annex-B NAL unit for decoding
|
|
|
|
func feedNal(_ data: Data) {
|
|
guard let session = decompressionSession,
|
|
let formatDesc = formatDescription,
|
|
data.count > 4 else { return }
|
|
|
|
// Convert Annex-B start code to AVCC 4-byte big-endian length prefix
|
|
let naluLen = UInt32(data.count - 4).bigEndian
|
|
var avcc = Data(capacity: data.count)
|
|
withUnsafeBytes(of: naluLen) { avcc.append(contentsOf: $0) }
|
|
avcc.append(data.advanced(by: 4))
|
|
|
|
var blockBuffer: CMBlockBuffer?
|
|
guard CMBlockBufferCreateWithMemoryBlock(
|
|
allocator: kCFAllocatorDefault,
|
|
memoryBlock: nil,
|
|
blockLength: avcc.count,
|
|
blockAllocator: kCFAllocatorDefault,
|
|
customBlockSource: nil,
|
|
offsetToData: 0,
|
|
dataLength: avcc.count,
|
|
flags: 0,
|
|
blockBufferOut: &blockBuffer
|
|
) == kCMBlockBufferNoErr, let blockBuffer = blockBuffer else { return }
|
|
|
|
avcc.withUnsafeBytes { ptr in
|
|
_ = CMBlockBufferReplaceDataBytes(
|
|
with: ptr.baseAddress!,
|
|
blockBuffer: blockBuffer,
|
|
offsetIntoDestination: 0,
|
|
dataLength: avcc.count
|
|
)
|
|
}
|
|
|
|
var timingInfo = CMSampleTimingInfo(
|
|
duration: CMTime(value: 1, timescale: 60),
|
|
presentationTimeStamp: CMTime(seconds: CACurrentMediaTime(), preferredTimescale: 600),
|
|
decodeTimeStamp: .invalid
|
|
)
|
|
var sampleBuffer: CMSampleBuffer?
|
|
guard CMSampleBufferCreate(
|
|
allocator: kCFAllocatorDefault,
|
|
dataBuffer: blockBuffer,
|
|
dataReady: true,
|
|
makeDataReadyCallback: nil,
|
|
refcon: nil,
|
|
formatDescription: formatDesc,
|
|
sampleCount: 1,
|
|
sampleTimingEntryCount: 1,
|
|
sampleTimingArray: &timingInfo,
|
|
sampleSizeEntryCount: 0,
|
|
sampleSizeArray: nil,
|
|
sampleBufferOut: &sampleBuffer
|
|
) == noErr, let sampleBuffer = sampleBuffer else { return }
|
|
|
|
VTDecompressionSessionDecodeFrame(
|
|
session,
|
|
sampleBuffer: sampleBuffer,
|
|
flags: [],
|
|
frameRefcon: nil,
|
|
infoFlagsOut: nil
|
|
)
|
|
}
|
|
|
|
// MARK: Called from the VT C callback
|
|
|
|
func handleDecodedFrame(_ imageBuffer: CVImageBuffer) {
|
|
lock.lock()
|
|
latestPixelBuffer = imageBuffer as CVPixelBuffer
|
|
lock.unlock()
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self = self else { return }
|
|
self.textureRegistry?.textureFrameAvailable(self.textureId)
|
|
}
|
|
}
|
|
|
|
// MARK: FlutterTexture
|
|
|
|
func copyPixelBuffer() -> Unmanaged<CVPixelBuffer>? {
|
|
lock.lock()
|
|
defer { lock.unlock() }
|
|
guard let pb = latestPixelBuffer else { return nil }
|
|
return Unmanaged.passRetained(pb)
|
|
}
|
|
|
|
// MARK: Teardown
|
|
|
|
func invalidate() {
|
|
if let session = decompressionSession {
|
|
VTDecompressionSessionInvalidate(session)
|
|
}
|
|
decompressionSession = nil
|
|
latestPixelBuffer = nil
|
|
textureRegistry?.unregisterTexture(textureId)
|
|
}
|
|
}
|