443 lines
17 KiB
Swift
443 lines
17 KiB
Swift
import Cocoa
|
|
import FlutterMacOS
|
|
import CoreGraphics
|
|
import ScreenCaptureKit
|
|
import CoreMedia
|
|
import CoreVideo
|
|
import VideoToolbox
|
|
|
|
@main
|
|
class AppDelegate: FlutterAppDelegate {
|
|
private var inputChannel: FlutterMethodChannel?
|
|
private var videoChannel: FlutterMethodChannel?
|
|
private var frameEventSink: FlutterEventSink?
|
|
|
|
// SCK objects (typed Any? to satisfy 10.15 deployment target)
|
|
private var scStream: Any?
|
|
private var screenDelegate: Any?
|
|
private var streamReady = false
|
|
|
|
// VTCompressionSession (Any? for same reason)
|
|
private var vtSession: Any?
|
|
private var encoderReady = false
|
|
|
|
// resolution last seen from SCK (to build config)
|
|
private var captureWidth = 1280
|
|
private var captureHeight = 720
|
|
|
|
override func applicationDidFinishLaunching(_ notification: Notification) {
|
|
guard let window = mainFlutterWindow,
|
|
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
|
|
|
let messenger = ctrl.engine.binaryMessenger
|
|
|
|
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
|
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
|
if call.method == "injectInput",
|
|
let args = call.arguments as? [String: Any] {
|
|
self?.injectInput(args)
|
|
result(nil)
|
|
} else {
|
|
result(FlutterMethodNotImplemented)
|
|
}
|
|
}
|
|
|
|
// event channel for pushing encoded NAL units to Flutter
|
|
let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
|
|
frameChannel.setStreamHandler(self)
|
|
|
|
videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
|
|
videoChannel?.setMethodCallHandler { [weak self] call, result in
|
|
guard let self = self else { return }
|
|
switch call.method {
|
|
case "start":
|
|
self.startEncoder(result: result)
|
|
case "stop":
|
|
self.stopEncoder()
|
|
result(nil)
|
|
case "forceKeyframe":
|
|
self.requestKeyframe()
|
|
result(nil)
|
|
default:
|
|
result(FlutterMethodNotImplemented)
|
|
}
|
|
}
|
|
|
|
if #available(macOS 12.3, *) {
|
|
Task { await self.ensureSCKStream() }
|
|
}
|
|
}
|
|
|
|
// MARK: - SCK stream setup
|
|
|
|
@available(macOS 12.3, *)
|
|
private func ensureSCKStream() async {
|
|
guard scStream == nil else { return }
|
|
|
|
guard let content = try? await SCShareableContent.current else { return }
|
|
guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
|
|
|
|
let filter = SCContentFilter(display: display, excludingWindows: [])
|
|
|
|
let srcW = Double(display.width)
|
|
let srcH = Double(display.height)
|
|
let scale = min(1.0, 1280.0 / srcW)
|
|
let dstW = Int(srcW * scale)
|
|
let dstH = Int(srcH * scale)
|
|
|
|
captureWidth = dstW
|
|
captureHeight = dstH
|
|
|
|
let cfg = SCStreamConfiguration()
|
|
cfg.width = dstW
|
|
cfg.height = dstH
|
|
cfg.pixelFormat = kCVPixelFormatType_32BGRA
|
|
cfg.showsCursor = true
|
|
cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
|
|
|
|
let del = ScreenDelegate { [weak self] sampleBuffer in
|
|
self?.encodeFrame(sampleBuffer)
|
|
}
|
|
screenDelegate = del
|
|
|
|
let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
|
|
try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
|
|
|
|
do {
|
|
try await stream.startCapture()
|
|
scStream = stream
|
|
streamReady = true
|
|
} catch {}
|
|
}
|
|
|
|
// MARK: - VTCompressionSession
|
|
|
|
private func startEncoder(result: @escaping FlutterResult) {
|
|
let w = captureWidth
|
|
let h = captureHeight
|
|
|
|
var session: VTCompressionSession?
|
|
let status = VTCompressionSessionCreate(
|
|
allocator: kCFAllocatorDefault,
|
|
width: Int32(w),
|
|
height: Int32(h),
|
|
codecType: kCMVideoCodecType_H264,
|
|
encoderSpecification: nil,
|
|
imageBufferAttributes: nil,
|
|
compressedDataAllocator: nil,
|
|
outputCallback: vtOutputCallback,
|
|
refcon: Unmanaged.passUnretained(self).toOpaque(),
|
|
compressionSessionOut: &session
|
|
)
|
|
|
|
guard status == noErr, let session = session else {
|
|
result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
|
|
return
|
|
}
|
|
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
|
|
value: kVTProfileLevel_H264_Baseline_AutoLevel)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
|
|
value: 6_000_000 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
|
|
value: 60 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
|
|
value: 120 as CFNumber)
|
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
|
|
value: 0 as CFNumber)
|
|
|
|
VTCompressionSessionPrepareToEncodeFrames(session)
|
|
vtSession = VTSessionBox(session)
|
|
encoderReady = true
|
|
|
|
// extract SPS+PPS from a forced keyframe by encoding a dummy frame
|
|
// instead, return empty bytes and let the real SPS+PPS come through the callback
|
|
// host_screen will call forceKeyframe right after start
|
|
result(FlutterStandardTypedData(bytes: Data()))
|
|
}
|
|
|
|
private func stopEncoder() {
|
|
guard let box = vtSession as? VTSessionBox else { return }
|
|
VTCompressionSessionInvalidate(box.session)
|
|
vtSession = nil
|
|
encoderReady = false
|
|
}
|
|
|
|
func requestKeyframe() {
|
|
forceKeyframeNext = true
|
|
}
|
|
|
|
private var forceKeyframeNext = false
|
|
|
|
private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
|
|
guard encoderReady, let box = vtSession as? VTSessionBox else { return }
|
|
let s = box.session
|
|
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
|
|
|
var frameProps: CFDictionary? = nil
|
|
if forceKeyframeNext {
|
|
frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
|
|
forceKeyframeNext = false
|
|
}
|
|
|
|
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
|
VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
|
|
duration: .invalid, frameProperties: frameProps,
|
|
sourceFrameRefcon: nil, infoFlagsOut: nil)
|
|
}
|
|
|
|
// MARK: - VT output callback (C function)
|
|
|
|
// called by VideoToolbox on an internal thread — collect Data packets here,
|
|
// then hop to main before touching the event sink
|
|
func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
|
|
var packets: [Data] = []
|
|
|
|
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
|
|
let isKeyframe = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
|
|
|
|
if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
|
|
if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
|
|
}
|
|
|
|
if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
|
|
var totalLen = 0
|
|
var rawPtr: UnsafeMutablePointer<CChar>? = nil
|
|
if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
|
|
totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
|
|
let ptr = rawPtr {
|
|
|
|
let ts = Int64(Date().timeIntervalSince1970 * 1000)
|
|
var offset = 0
|
|
|
|
while offset + 4 <= totalLen {
|
|
// read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
|
|
let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
|
|
let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
|
|
let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
|
|
let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
|
|
let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
|
|
offset += 4
|
|
|
|
guard naluLen > 0, offset + naluLen <= totalLen else { break }
|
|
|
|
var packet = Data(capacity: 1 + 8 + 4 + naluLen)
|
|
packet.append(0x02)
|
|
withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
|
|
packets.append(packet)
|
|
|
|
offset += naluLen
|
|
}
|
|
}
|
|
}
|
|
|
|
guard !packets.isEmpty else { return }
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let sink = self?.frameEventSink else { return }
|
|
for packet in packets {
|
|
sink(FlutterStandardTypedData(bytes: packet))
|
|
}
|
|
}
|
|
}
|
|
|
|
private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
|
|
var spsCount = 0
|
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
|
parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
|
|
nalUnitHeaderLengthOut: nil)
|
|
var packet = Data()
|
|
packet.append(0x01)
|
|
|
|
for i in 0 ..< spsCount {
|
|
var ptr: UnsafePointer<UInt8>? = nil
|
|
var len = 0
|
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
|
|
parameterSetPointerOut: &ptr,
|
|
parameterSetSizeOut: &len,
|
|
parameterSetCountOut: nil,
|
|
nalUnitHeaderLengthOut: nil)
|
|
if let ptr = ptr {
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ptr, count: len))
|
|
}
|
|
}
|
|
|
|
// PPS sits at index spsCount
|
|
var ppsPtr: UnsafePointer<UInt8>? = nil
|
|
var ppsLen = 0
|
|
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
|
|
parameterSetPointerOut: &ppsPtr,
|
|
parameterSetSizeOut: &ppsLen,
|
|
parameterSetCountOut: nil,
|
|
nalUnitHeaderLengthOut: nil) == noErr,
|
|
let ppsPtr = ppsPtr {
|
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
|
packet.append(Data(bytes: ppsPtr, count: ppsLen))
|
|
}
|
|
|
|
return packet.count > 1 ? packet : nil
|
|
}
|
|
|
|
// MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths — unused in normal operation)
|
|
|
|
private func captureCG() -> Data? {
|
|
let displayID = CGMainDisplayID()
|
|
guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
|
|
let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
|
|
let srcW = CGFloat(composited.width)
|
|
let scale = min(1.0, 1280.0 / srcW)
|
|
let dstW = Int(srcW * scale)
|
|
let dstH = Int(CGFloat(composited.height) * scale)
|
|
let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
|
guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
|
|
bytesPerRow: 0, space: colorSpace,
|
|
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
|
|
ctx.interpolationQuality = .high
|
|
ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
|
|
guard let scaled = ctx.makeImage() else { return nil }
|
|
return jpegData(from: scaled, quality: 0.4)
|
|
}
|
|
|
|
private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
|
|
let nsImage = NSImage(cgImage: cgImage, size: .zero)
|
|
guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
|
|
return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
|
|
}
|
|
|
|
private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
|
|
let imgW = cgImage.width
|
|
let imgH = cgImage.height
|
|
let mousePt = NSEvent.mouseLocation
|
|
let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
|
|
let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
|
|
let cursorX = mousePt.x * scale
|
|
let cursorY = (screenH - mousePt.y) * scale
|
|
let cursor = NSCursor.current
|
|
let cursorImg = cursor.image
|
|
let hotspot = cursor.hotSpot
|
|
let cw = cursorImg.size.width * scale
|
|
let ch = cursorImg.size.height * scale
|
|
let drawX = cursorX - hotspot.x * scale
|
|
let drawY = cursorY - hotspot.y * scale
|
|
let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
|
guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
|
|
bytesPerRow: 0, space: colorSpace,
|
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
|
|
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
|
|
var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
|
|
if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
|
|
ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
|
|
}
|
|
return ctx.makeImage()
|
|
}
|
|
|
|
// MARK: - Input injection
|
|
|
|
private func injectInput(_ ev: [String: Any]) {
|
|
guard let type = ev["type"] as? String else { return }
|
|
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
|
|
|
|
switch type {
|
|
case "move":
|
|
let nx = ev["x"] as? Double ?? 0
|
|
let ny = ev["y"] as? Double ?? 0
|
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
|
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
|
|
.post(tap: .cghidEventTap)
|
|
|
|
case "click":
|
|
let nx = ev["x"] as? Double ?? 0
|
|
let ny = ev["y"] as? Double ?? 0
|
|
let btn = ev["button"] as? Int ?? 0
|
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
|
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
|
|
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
|
|
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
|
.post(tap: .cghidEventTap)
|
|
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
|
.post(tap: .cghidEventTap)
|
|
|
|
case "keydown":
|
|
if let code = ev["keyCode"] as? Int {
|
|
CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(code & 0xFFFF), keyDown: true)?
|
|
.post(tap: .cghidEventTap)
|
|
}
|
|
|
|
case "keyup":
|
|
if let code = ev["keyCode"] as? Int {
|
|
CGEvent(keyboardEventSource: nil, virtualKey: CGKeyCode(code & 0xFFFF), keyDown: false)?
|
|
.post(tap: .cghidEventTap)
|
|
}
|
|
|
|
default: break
|
|
}
|
|
}
|
|
|
|
// MARK: - App lifecycle
|
|
|
|
override func applicationWillTerminate(_ notification: Notification) {
|
|
stopEncoder()
|
|
if #available(macOS 12.3, *), let stream = scStream as? SCStream {
|
|
Task { try? await stream.stopCapture() }
|
|
}
|
|
}
|
|
|
|
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
|
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
|
|
}
|
|
|
|
// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
|
|
|
|
private class VTSessionBox {
|
|
let session: VTCompressionSession
|
|
init(_ s: VTCompressionSession) { session = s }
|
|
}
|
|
|
|
// MARK: - VT output C callback
|
|
|
|
private func vtOutputCallback(
|
|
outputCallbackRefCon: UnsafeMutableRawPointer?,
|
|
sourceFrameRefCon: UnsafeMutableRawPointer?,
|
|
status: OSStatus,
|
|
infoFlags: VTEncodeInfoFlags,
|
|
sampleBuffer: CMSampleBuffer?
|
|
) {
|
|
guard status == noErr, let sampleBuffer = sampleBuffer,
|
|
let refcon = outputCallbackRefCon else { return }
|
|
let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
|
|
delegate.handleEncodedFrame(sampleBuffer)
|
|
}
|
|
|
|
// MARK: - SCK delegate
|
|
|
|
@available(macOS 12.3, *)
|
|
class ScreenDelegate: NSObject, SCStreamOutput {
|
|
private let onFrame: (CMSampleBuffer) -> Void
|
|
|
|
init(onFrame: @escaping (CMSampleBuffer) -> Void) {
|
|
self.onFrame = onFrame
|
|
}
|
|
|
|
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
|
guard type == .screen else { return }
|
|
onFrame(sampleBuffer)
|
|
}
|
|
}
|
|
|
|
// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
|
|
|
|
extension AppDelegate: FlutterStreamHandler {
|
|
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
|
|
frameEventSink = events
|
|
return nil
|
|
}
|
|
|
|
func onCancel(withArguments arguments: Any?) -> FlutterError? {
|
|
frameEventSink = nil
|
|
return nil
|
|
}
|
|
}
|