inital

macos/Podfile.lock

@@ -1,9 +1,9 @@
 PODS:
-  - flutter_webrtc (0.9.36):
+  - flutter_webrtc (1.4.0):
     - FlutterMacOS
-    - WebRTC-SDK (= 114.5735.08)
+    - WebRTC-SDK (= 144.7559.01)
   - FlutterMacOS (1.0.0)
-  - WebRTC-SDK (114.5735.08)
+  - WebRTC-SDK (144.7559.01)
 
 DEPENDENCIES:
   - flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`)
@@ -20,9 +20,9 @@ EXTERNAL SOURCES:
     :path: Flutter/ephemeral
 
 SPEC CHECKSUMS:
-  flutter_webrtc: cf7dc44d26cbb5c5f1ae5f583dab545871f287f9
+  flutter_webrtc: dbacf72a9bd951ccfa5997198f0a6214bb848ede
   FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1
-  WebRTC-SDK: c24d2a6c9f571f2ed42297cb8ffba9557093142b
+  WebRTC-SDK: ab9b5319e458c2bfebdc92b3600740da35d5630d
 
 PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009
 

macos/Runner/AppDelegate.swift

@@ -1,20 +1,37 @@
 import Cocoa
 import FlutterMacOS
 import CoreGraphics
+import ScreenCaptureKit
+import CoreMedia
+import CoreVideo
+import VideoToolbox
 
 @main
 class AppDelegate: FlutterAppDelegate {
   private var inputChannel: FlutterMethodChannel?
+  private var videoChannel: FlutterMethodChannel?
+  private var frameEventSink: FlutterEventSink?
+
+  // SCK objects (typed Any? to satisfy 10.15 deployment target)
+  private var scStream: Any?
+  private var screenDelegate: Any?
+  private var streamReady = false
+
+  // VTCompressionSession (Any? for same reason)
+  private var vtSession: Any?
+  private var encoderReady = false
+
+  // resolution last seen from SCK (to build config)
+  private var captureWidth  = 1280
+  private var captureHeight = 720
 
   override func applicationDidFinishLaunching(_ notification: Notification) {
     guard let window = mainFlutterWindow,
           let ctrl = window.contentViewController as? FlutterViewController else { return }
 
-    inputChannel = FlutterMethodChannel(
-      name: "com.pulsar/input",
-      binaryMessenger: ctrl.engine.binaryMessenger
-    )
+    let messenger = ctrl.engine.binaryMessenger
 
+    inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
     inputChannel?.setMethodCallHandler { [weak self] call, result in
       if call.method == "injectInput",
          let args = call.arguments as? [String: Any] {
@@ -24,8 +41,302 @@ class AppDelegate: FlutterAppDelegate {
         result(FlutterMethodNotImplemented)
       }
     }
+
+    // event channel for pushing encoded NAL units to Flutter
+    let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
+    frameChannel.setStreamHandler(self)
+
+    videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
+    videoChannel?.setMethodCallHandler { [weak self] call, result in
+      guard let self = self else { return }
+      switch call.method {
+      case "start":
+        self.startEncoder(result: result)
+      case "stop":
+        self.stopEncoder()
+        result(nil)
+      case "forceKeyframe":
+        self.requestKeyframe()
+        result(nil)
+      default:
+        result(FlutterMethodNotImplemented)
+      }
+    }
+
+    if #available(macOS 12.3, *) {
+      Task { await self.ensureSCKStream() }
+    }
   }
 
+  // MARK: - SCK stream setup
+
+  @available(macOS 12.3, *)
+  private func ensureSCKStream() async {
+    guard scStream == nil else { return }
+
+    guard let content = try? await SCShareableContent.current else { return }
+    guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
+
+    let filter = SCContentFilter(display: display, excludingWindows: [])
+
+    let srcW   = Double(display.width)
+    let srcH   = Double(display.height)
+    let scale  = min(1.0, 1280.0 / srcW)
+    let dstW   = Int(srcW * scale)
+    let dstH   = Int(srcH * scale)
+
+    captureWidth  = dstW
+    captureHeight = dstH
+
+    let cfg = SCStreamConfiguration()
+    cfg.width  = dstW
+    cfg.height = dstH
+    cfg.pixelFormat = kCVPixelFormatType_32BGRA
+    cfg.showsCursor = true
+    cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
+
+    let del = ScreenDelegate { [weak self] sampleBuffer in
+      self?.encodeFrame(sampleBuffer)
+    }
+    screenDelegate = del
+
+    let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
+    try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
+
+    do {
+      try await stream.startCapture()
+      scStream   = stream
+      streamReady = true
+    } catch {}
+  }
+
+  // MARK: - VTCompressionSession
+
+  private func startEncoder(result: @escaping FlutterResult) {
+    let w = captureWidth
+    let h = captureHeight
+
+    var session: VTCompressionSession?
+    let status = VTCompressionSessionCreate(
+      allocator: kCFAllocatorDefault,
+      width:  Int32(w),
+      height: Int32(h),
+      codecType: kCMVideoCodecType_H264,
+      encoderSpecification: nil,
+      imageBufferAttributes: nil,
+      compressedDataAllocator: nil,
+      outputCallback: vtOutputCallback,
+      refcon: Unmanaged.passUnretained(self).toOpaque(),
+      compressionSessionOut: &session
+    )
+
+    guard status == noErr, let session = session else {
+      result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
+      return
+    }
+
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
+                         value: kVTProfileLevel_H264_Baseline_AutoLevel)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
+                         value: 6_000_000 as CFNumber)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
+                         value: 60 as CFNumber)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
+                         value: 120 as CFNumber)
+    VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
+                         value: 0 as CFNumber)
+
+    VTCompressionSessionPrepareToEncodeFrames(session)
+    vtSession    = VTSessionBox(session)
+    encoderReady = true
+
+    // extract SPS+PPS from a forced keyframe by encoding a dummy frame
+    // instead, return empty bytes and let the real SPS+PPS come through the callback
+    // host_screen will call forceKeyframe right after start
+    result(FlutterStandardTypedData(bytes: Data()))
+  }
+
+  private func stopEncoder() {
+    guard let box = vtSession as? VTSessionBox else { return }
+    VTCompressionSessionInvalidate(box.session)
+    vtSession    = nil
+    encoderReady = false
+  }
+
+  func requestKeyframe() {
+    forceKeyframeNext = true
+  }
+
+  private var forceKeyframeNext = false
+
+  private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
+    guard encoderReady, let box = vtSession as? VTSessionBox else { return }
+    let s = box.session
+    guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
+
+    var frameProps: CFDictionary? = nil
+    if forceKeyframeNext {
+      frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
+      forceKeyframeNext = false
+    }
+
+    let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+    VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
+                                    duration: .invalid, frameProperties: frameProps,
+                                    sourceFrameRefcon: nil, infoFlagsOut: nil)
+  }
+
+  // MARK: - VT output callback (C function)
+
+  // called by VideoToolbox on an internal thread — collect Data packets here,
+  // then hop to main before touching the event sink
+  func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
+    var packets: [Data] = []
+
+    let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
+    let isKeyframe  = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
+
+    if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
+      if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
+    }
+
+    if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
+      var totalLen = 0
+      var rawPtr: UnsafeMutablePointer<CChar>? = nil
+      if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
+                                     totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
+         let ptr = rawPtr {
+
+        let ts = Int64(Date().timeIntervalSince1970 * 1000)
+        var offset = 0
+
+        while offset + 4 <= totalLen {
+          // read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
+          let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
+          let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
+          let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
+          let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
+          let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
+          offset += 4
+
+          guard naluLen > 0, offset + naluLen <= totalLen else { break }
+
+          var packet = Data(capacity: 1 + 8 + 4 + naluLen)
+          packet.append(0x02)
+          withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
+          packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
+          packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
+          packets.append(packet)
+
+          offset += naluLen
+        }
+      }
+    }
+
+    guard !packets.isEmpty else { return }
+    DispatchQueue.main.async { [weak self] in
+      guard let sink = self?.frameEventSink else { return }
+      for packet in packets {
+        sink(FlutterStandardTypedData(bytes: packet))
+      }
+    }
+  }
+
+  private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
+    var spsCount = 0
+    CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
+                                                       parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
+                                                       nalUnitHeaderLengthOut: nil)
+    var packet = Data()
+    packet.append(0x01)
+
+    for i in 0 ..< spsCount {
+      var ptr: UnsafePointer<UInt8>? = nil
+      var len = 0
+      CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
+                                                         parameterSetPointerOut: &ptr,
+                                                         parameterSetSizeOut: &len,
+                                                         parameterSetCountOut: nil,
+                                                         nalUnitHeaderLengthOut: nil)
+      if let ptr = ptr {
+        packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
+        packet.append(Data(bytes: ptr, count: len))
+      }
+    }
+
+    // PPS sits at index spsCount
+    var ppsPtr: UnsafePointer<UInt8>? = nil
+    var ppsLen = 0
+    if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
+                                                          parameterSetPointerOut: &ppsPtr,
+                                                          parameterSetSizeOut: &ppsLen,
+                                                          parameterSetCountOut: nil,
+                                                          nalUnitHeaderLengthOut: nil) == noErr,
+       let ppsPtr = ppsPtr {
+      packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
+      packet.append(Data(bytes: ppsPtr, count: ppsLen))
+    }
+
+    return packet.count > 1 ? packet : nil
+  }
+
+  // MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths — unused in normal operation)
+
+  private func captureCG() -> Data? {
+    let displayID = CGMainDisplayID()
+    guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
+    let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
+    let srcW  = CGFloat(composited.width)
+    let scale = min(1.0, 1280.0 / srcW)
+    let dstW  = Int(srcW * scale)
+    let dstH  = Int(CGFloat(composited.height) * scale)
+    let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
+    guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
+                              bytesPerRow: 0, space: colorSpace,
+                              bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
+    ctx.interpolationQuality = .high
+    ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
+    guard let scaled = ctx.makeImage() else { return nil }
+    return jpegData(from: scaled, quality: 0.4)
+  }
+
+  private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
+    let nsImage = NSImage(cgImage: cgImage, size: .zero)
+    guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
+    return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
+  }
+
+  private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
+    let imgW = cgImage.width
+    let imgH = cgImage.height
+    let mousePt = NSEvent.mouseLocation
+    let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
+    let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
+    let cursorX = mousePt.x * scale
+    let cursorY = (screenH - mousePt.y) * scale
+    let cursor = NSCursor.current
+    let cursorImg = cursor.image
+    let hotspot = cursor.hotSpot
+    let cw = cursorImg.size.width * scale
+    let ch = cursorImg.size.height * scale
+    let drawX = cursorX - hotspot.x * scale
+    let drawY = cursorY - hotspot.y * scale
+    let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
+    guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
+                              bytesPerRow: 0, space: colorSpace,
+                              bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
+    ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
+    var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
+    if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
+      ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
+    }
+    return ctx.makeImage()
+  }
+
+  // MARK: - Input injection
+
   private func injectInput(_ ev: [String: Any]) {
     guard let type = ev["type"] as? String else { return }
     let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
@@ -34,24 +345,17 @@ class AppDelegate: FlutterAppDelegate {
     case "move":
       let nx = ev["x"] as? Double ?? 0
       let ny = ev["y"] as? Double ?? 0
-      // CoreGraphics origin is top-left on screen, NSScreen origin is bottom-left
-      let px = CGFloat(nx) * screen.width
-      let py = CGFloat(ny) * screen.height
-      let pt = CGPoint(x: px, y: py)
+      let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
       CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
         .post(tap: .cghidEventTap)
 
     case "click":
-      let nx = ev["x"] as? Double ?? 0
-      let ny = ev["y"] as? Double ?? 0
+      let nx  = ev["x"] as? Double ?? 0
+      let ny  = ev["y"] as? Double ?? 0
       let btn = ev["button"] as? Int ?? 0
-      let px = CGFloat(nx) * screen.width
-      let py = CGFloat(ny) * screen.height
-      let pt = CGPoint(x: px, y: py)
-
+      let pt  = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
       let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
         btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
-
       CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
         .post(tap: .cghidEventTap)
       CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
@@ -69,16 +373,71 @@ class AppDelegate: FlutterAppDelegate {
           .post(tap: .cghidEventTap)
       }
 
-    default:
-      break
+    default: break
     }
   }
 
-  override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
-    return true
+  // MARK: - App lifecycle
+
+  override func applicationWillTerminate(_ notification: Notification) {
+    stopEncoder()
+    if #available(macOS 12.3, *), let stream = scStream as? SCStream {
+      Task { try? await stream.stopCapture() }
+    }
   }
 
-  override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool {
-    return true
+  override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
+  override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
+}
+
+// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
+
+private class VTSessionBox {
+  let session: VTCompressionSession
+  init(_ s: VTCompressionSession) { session = s }
+}
+
+// MARK: - VT output C callback
+
+private func vtOutputCallback(
+  outputCallbackRefCon: UnsafeMutableRawPointer?,
+  sourceFrameRefCon: UnsafeMutableRawPointer?,
+  status: OSStatus,
+  infoFlags: VTEncodeInfoFlags,
+  sampleBuffer: CMSampleBuffer?
+) {
+  guard status == noErr, let sampleBuffer = sampleBuffer,
+        let refcon = outputCallbackRefCon else { return }
+  let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
+  delegate.handleEncodedFrame(sampleBuffer)
+}
+
+// MARK: - SCK delegate
+
+@available(macOS 12.3, *)
+class ScreenDelegate: NSObject, SCStreamOutput {
+  private let onFrame: (CMSampleBuffer) -> Void
+
+  init(onFrame: @escaping (CMSampleBuffer) -> Void) {
+    self.onFrame = onFrame
+  }
+
+  func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
+    guard type == .screen else { return }
+    onFrame(sampleBuffer)
+  }
+}
+
+// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
+
+extension AppDelegate: FlutterStreamHandler {
+  func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
+    frameEventSink = events
+    return nil
+  }
+
+  func onCancel(withArguments arguments: Any?) -> FlutterError? {
+    frameEventSink = nil
+    return nil
   }
 }