This commit is contained in:
ImBenji 2026-05-04 08:21:31 +01:00
parent 0ed740ad19
commit bfdaf4a801
9 changed files with 1081 additions and 149 deletions

View file

@ -1,4 +1,13 @@
<manifest xmlns:android="http://schemas.android.com/apk/res/android"> <manifest xmlns:android="http://schemas.android.com/apk/res/android">
<uses-permission android:name="android.permission.INTERNET"/>
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
<uses-permission android:name="android.permission.CHANGE_NETWORK_STATE"/>
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
<uses-permission android:name="android.permission.CAMERA"/>
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
<uses-permission android:name="android.permission.BLUETOOTH" android:maxSdkVersion="30"/>
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/>
<application <application
android:label="pulsar" android:label="pulsar"
android:name="${applicationName}" android:name="${applicationName}"

View file

@ -1,6 +1,255 @@
package net.imbenji.pulsar; package net.imbenji.pulsar;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.HandlerThread;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import io.flutter.embedding.android.FlutterActivity; import io.flutter.embedding.android.FlutterActivity;
import io.flutter.embedding.engine.FlutterEngine;
import io.flutter.plugin.common.MethodChannel;
import io.flutter.view.TextureRegistry;
public class MainActivity extends FlutterActivity { public class MainActivity extends FlutterActivity {
private static final String CHANNEL = "com.pulsar/video";
// one decoder entry per textureId
private static class DecoderEntry {
TextureRegistry.SurfaceTextureEntry textureEntry;
Surface surface;
MediaCodec codec;
HandlerThread outputThread;
Handler outputHandler;
HandlerThread inputThread;
Handler inputHandler;
AtomicBoolean running = new AtomicBoolean(true);
}
private final Map<Long, DecoderEntry> decoders = new HashMap<>();
@Override
public void configureFlutterEngine(FlutterEngine flutterEngine) {
super.configureFlutterEngine(flutterEngine);
TextureRegistry registry = flutterEngine.getRenderer();
new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), CHANNEL)
.setMethodCallHandler((call, result) -> {
switch (call.method) {
case "createDecoder": {
byte[] spsPps = call.argument("spsPps");
if (spsPps == null) {
result.error("BAD_ARGS", "spsPps required", null);
break;
}
TextureRegistry.SurfaceTextureEntry entry = registry.createSurfaceTexture();
long texId = entry.id();
// we don't know the video size yet configure with a resonable
// placeholder; MediaCodec will adapt once it sees the stream
int width = 1280;
int height = 720;
MediaFormat fmt = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
// csd-0 must include Annex B start codes pass the whole SPS+PPS blob
// (MediaCodec H264 accepts combined SPS+PPS with start codes in csd-0)
fmt.setByteBuffer("csd-0", ByteBuffer.wrap(spsPps));
entry.surfaceTexture().setDefaultBufferSize(width, height);
fmt.setInteger(MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
Surface surface = new Surface(entry.surfaceTexture());
MediaCodec codec;
try {
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
codec.configure(fmt, surface, null, 0);
codec.start();
} catch (IOException e) {
surface.release();
entry.release();
result.error("CODEC_FAIL", e.getMessage(), null);
break;
}
DecoderEntry dec = new DecoderEntry();
dec.textureEntry = entry;
dec.surface = surface;
dec.codec = codec;
HandlerThread ot = new HandlerThread("decoder-out-" + texId);
ot.start();
dec.outputThread = ot;
dec.outputHandler = new Handler(ot.getLooper());
HandlerThread it = new HandlerThread("decoder-in-" + texId);
it.start();
dec.inputThread = it;
dec.inputHandler = new Handler(it.getLooper());
drainOutputLoop(dec);
decoders.put(texId, dec);
Map<String, Long> res = new HashMap<>();
res.put("textureId", texId);
result.success(res);
break;
}
case "feedNal": {
long texId = ((Number) call.argument("textureId")).longValue();
byte[] nal = call.argument("nal");
DecoderEntry dec = decoders.get(texId);
if (dec == null || nal == null) {
result.error("NO_DECODER", "decoder not found", null);
break;
}
dec.inputHandler.post(() -> {
try {
int idx = dec.codec.dequeueInputBuffer(0);
if (idx >= 0) {
ByteBuffer buf = dec.codec.getInputBuffer(idx);
if (buf != null) {
buf.clear();
buf.put(nal);
dec.codec.queueInputBuffer(idx, 0, nal.length,
System.nanoTime() / 1000, 0);
}
}
// if idx < 0 the decoder is backed up drop this NAL silently
} catch (Exception ignored) {}
});
result.success(null);
break;
}
case "stopDecoder": {
long texId = ((Number) call.argument("textureId")).longValue();
DecoderEntry dec = decoders.remove(texId);
if (dec != null) releaseDecoder(dec);
result.success(null);
break;
}
case "start":
case "stop":
case "forceKeyframe":
result.success(null);
break;
default:
result.notImplemented();
}
});
}
private void drainOutputLoop(DecoderEntry dec) {
dec.outputHandler.post(() -> {
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
while (dec.running.get()) {
try {
// block up to 10ms waiting for a decoded frame
int idx = dec.codec.dequeueOutputBuffer(info, 10_000);
if (idx >= 0) {
dec.codec.releaseOutputBuffer(idx, true);
}
// idx == INFO_TRY_AGAIN_LATER / FORMAT_CHANGED / BUFFERS_CHANGED loop
} catch (Exception e) {
break;
}
}
});
}
private final Handler mainHandler = new Handler(android.os.Looper.getMainLooper());
private void releaseDecoder(DecoderEntry dec) {
dec.running.set(false);
// codec stop/release must happen on the decoder thread (after the drain loop exits)
// but entry.release() calls FlutterJNI.unregisterTexture which requires the main thread
dec.outputHandler.post(() -> {
try {
dec.codec.stop();
dec.codec.release();
} catch (Exception ignored) {}
dec.surface.release();
mainHandler.post(() -> dec.textureEntry.release());
});
dec.outputThread.quitSafely();
dec.inputThread.quitSafely();
}
// splits an Annex B blob into individual NAL unit ByteBuffers
// returns null if fewer than 2 NALUs found
private ByteBuffer[] splitAnnexB(byte[] data) {
java.util.List<ByteBuffer> units = new java.util.ArrayList<>();
int start = -1;
for (int i = 0; i < data.length - 3; i++) {
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1) {
if (start >= 0) {
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
units.add(bb);
}
start = i + 4;
i += 3;
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
if (start >= 0) {
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
units.add(bb);
}
start = i + 3;
i += 2;
}
}
if (start >= 0 && start < data.length) {
units.add(ByteBuffer.wrap(data, start, data.length - start));
}
return units.size() >= 2 ? units.toArray(new ByteBuffer[0]) : null;
}
// crude SPS parser reads width/height from the first SPS NALU
// returns null if it cant parse
private int[] dimsFromSps(ByteBuffer spsBuf) {
try {
byte[] sps = new byte[spsBuf.remaining()];
spsBuf.duplicate().get(sps);
// skip NAL header byte (forbidden_zero_bit + nal_ref_idc + nal_unit_type)
// a minimal parse isn't reliable without a full RBSP/Exp-Golomb parser,
// so just return null and let MediaCodec figure it out
return null;
} catch (Exception e) {
return null;
}
}
@Override
protected void onDestroy() {
for (DecoderEntry dec : decoders.values()) {
releaseDecoder(dec);
}
decoders.clear();
super.onDestroy();
}
} }

View file

@ -1,6 +1,7 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:math'; import 'dart:math';
import 'dart:typed_data';
import 'package:flutter/material.dart'; import 'package:flutter/material.dart';
import 'package:flutter/services.dart'; import 'package:flutter/services.dart';
@ -8,8 +9,9 @@ import 'package:flutter_webrtc/flutter_webrtc.dart';
import '../services/signalling.dart'; import '../services/signalling.dart';
// platform channel for injecting input into the OS
const _inputChannel = MethodChannel('com.pulsar/input'); const _inputChannel = MethodChannel('com.pulsar/input');
const _videoChannel = MethodChannel('com.pulsar/video');
const _frameEvents = EventChannel('com.pulsar/video/frames');
class HostScreen extends StatefulWidget { class HostScreen extends StatefulWidget {
final String serverUrl; final String serverUrl;
@ -23,13 +25,31 @@ class _HostScreenState extends State<HostScreen> {
final _sig = SignallingService(); final _sig = SignallingService();
RTCPeerConnection? _pc; RTCPeerConnection? _pc;
RTCDataChannel? _dataChannel; RTCDataChannel? _dataChannel;
MediaStream? _localStream;
String _roomId = ''; String _roomId = '';
String _status = 'Initialising...'; String _status = 'Initialising...';
bool _connected = false; bool _connected = false;
StreamSubscription? _sigSub; StreamSubscription? _sigSub;
StreamSubscription? _frameSub;
// stats
bool _showStats = false;
int _statFps = 0;
double _statBitrateMbps = 0;
int _statDropped = 0;
int _statBufferKb = 0;
double _statSendMs = 0;
int _frameCount = 0;
int _byteCount = 0;
int _droppedCount = 0;
double _sendMsSum = 0;
int _sendMsCount = 0;
Timer? _statsTimer;
// viewer stats received over data channel
Map<String, dynamic>? _viewerStats;
@override @override
void initState() { void initState() {
@ -67,7 +87,6 @@ class _HostScreenState extends State<HostScreen> {
switch (type) { switch (type) {
case 'created': case 'created':
_setState('Waiting for viewer...'); _setState('Waiting for viewer...');
await _startCapture();
break; break;
case 'peer_joined': case 'peer_joined':
@ -82,6 +101,7 @@ class _HostScreenState extends State<HostScreen> {
case 'peer_disconnected': case 'peer_disconnected':
_setState('Viewer disconnected'); _setState('Viewer disconnected');
_stopCapture();
setState(() => _connected = false); setState(() => _connected = false);
break; break;
@ -91,17 +111,6 @@ class _HostScreenState extends State<HostScreen> {
} }
} }
Future<void> _startCapture() async {
try {
_localStream = await navigator.mediaDevices.getDisplayMedia({
'video': {'cursor': 'always'},
'audio': false,
});
} catch (e) {
_setState('Screen capture failed: $e');
}
}
Future<void> _createOffer() async { Future<void> _createOffer() async {
final config = { final config = {
'iceServers': [ 'iceServers': [
@ -111,31 +120,34 @@ class _HostScreenState extends State<HostScreen> {
_pc = await createPeerConnection(config); _pc = await createPeerConnection(config);
// data channel for input events (host creates it as offerer) // unordered + unreliable drop stale frames, only the latest matters
_dataChannel = await _pc!.createDataChannel( final dcInit = RTCDataChannelInit()
'input', ..ordered = false
RTCDataChannelInit()..ordered = true, ..maxRetransmits = 0;
);
_dataChannel = await _pc!.createDataChannel('frames', dcInit);
_dataChannel!.onMessage = (msg) { _dataChannel!.onMessage = (msg) {
_handleInputEvent(msg.text); if (msg.isBinary) return;
}; try {
final parsed = jsonDecode(msg.text) as Map<String, dynamic>;
// add screen tracks if (parsed['type'] == 'stats') {
if (_localStream != null) { setState(() {
for (final track in _localStream!.getTracks()) { _viewerStats = parsed;
await _pc!.addTrack(track, _localStream!); });
} else {
_handleInputEvent(msg.text);
}
} catch (_) {
_handleInputEvent(msg.text);
} }
} };
_pc!.onIceCandidate = (candidate) { _pc!.onIceCandidate = (candidate) {
_sig.send({ _sig.send({
'type': 'signal', 'type': 'signal',
'roomId': _roomId, 'roomId': _roomId,
'data': { 'data': {'type': 'candidate', 'candidate': candidate.toMap()},
'type': 'candidate',
'candidate': candidate.toMap(),
},
}); });
}; };
@ -143,9 +155,11 @@ class _HostScreenState extends State<HostScreen> {
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) { if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
_setState('Connected'); _setState('Connected');
setState(() => _connected = true); setState(() => _connected = true);
_startCapture();
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed || } else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) { state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
_setState('Connection lost'); _setState('Connection lost');
_stopCapture();
setState(() => _connected = false); setState(() => _connected = false);
} }
}; };
@ -177,6 +191,58 @@ class _HostScreenState extends State<HostScreen> {
} }
} }
Future<void> _startCapture() async {
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
// subscribe BEFORE starting the encoder so the initial SPS+PPS+keyframe isn't dropped
_frameSub = _frameEvents.receiveBroadcastStream().listen((dynamic raw) {
final bytes = (raw as Uint8List);
final dc = _dataChannel;
if (dc == null) return;
final buffered = dc.bufferedAmount ?? 0;
if (buffered > 262144) {
_droppedCount++;
return;
}
try {
final sendStart = DateTime.now();
dc.send(RTCDataChannelMessage.fromBinary(bytes));
_sendMsSum += DateTime.now().difference(sendStart).inMicroseconds / 1000.0;
_sendMsCount++;
_frameCount++;
_byteCount += bytes.length;
} catch (_) {}
});
await _videoChannel.invokeMethod('start');
await _videoChannel.invokeMethod('forceKeyframe');
}
void _stopCapture() {
_frameSub?.cancel();
_frameSub = null;
_statsTimer?.cancel();
_statsTimer = null;
_videoChannel.invokeMethod('stop');
}
void _flushStats() {
final buf = _dataChannel?.bufferedAmount ?? 0;
setState(() {
_statFps = _frameCount;
_statBitrateMbps = _byteCount * 8 / 1e6;
_statDropped = _droppedCount;
_statBufferKb = (buf / 1024).round();
_statSendMs = _sendMsCount > 0 ? _sendMsSum / _sendMsCount : 0;
_frameCount = 0;
_byteCount = 0;
_droppedCount = 0;
_sendMsSum = 0; _sendMsCount = 0;
});
}
void _handleInputEvent(String json) async { void _handleInputEvent(String json) async {
try { try {
final ev = jsonDecode(json) as Map<String, dynamic>; final ev = jsonDecode(json) as Map<String, dynamic>;
@ -190,10 +256,10 @@ class _HostScreenState extends State<HostScreen> {
@override @override
void dispose() { void dispose() {
_stopCapture();
_sigSub?.cancel(); _sigSub?.cancel();
_sig.dispose(); _sig.dispose();
_pc?.close(); _pc?.close();
_localStream?.dispose();
super.dispose(); super.dispose();
} }
@ -204,37 +270,100 @@ class _HostScreenState extends State<HostScreen> {
title: const Text('Pulsar — Host'), title: const Text('Pulsar — Host'),
backgroundColor: Theme.of(context).colorScheme.inversePrimary, backgroundColor: Theme.of(context).colorScheme.inversePrimary,
), ),
body: Center( floatingActionButton: FloatingActionButton(
child: Column( mini: true,
mainAxisAlignment: MainAxisAlignment.center, tooltip: 'Stats',
children: [ onPressed: () => setState(() => _showStats = !_showStats),
const Icon(Icons.monitor, size: 80, color: Colors.indigo), child: const Icon(Icons.bar_chart),
const SizedBox(height: 24), ),
Text('Room Code', style: Theme.of(context).textTheme.titleMedium), body: Stack(
const SizedBox(height: 8), children: [
SelectableText( Center(
_roomId, child: Column(
style: Theme.of(context).textTheme.displaySmall?.copyWith( mainAxisAlignment: MainAxisAlignment.center,
fontFamily: 'monospace',
fontWeight: FontWeight.bold,
letterSpacing: 8,
),
),
const SizedBox(height: 32),
Row(
mainAxisSize: MainAxisSize.min,
children: [ children: [
Icon( const Icon(Icons.monitor, size: 80, color: Colors.indigo),
_connected ? Icons.circle : Icons.circle_outlined, const SizedBox(height: 24),
color: _connected ? Colors.green : Colors.orange, Text('Room Code', style: Theme.of(context).textTheme.titleMedium),
size: 14, const SizedBox(height: 8),
SelectableText(
_roomId,
style: Theme.of(context).textTheme.displaySmall?.copyWith(
fontFamily: 'monospace',
fontWeight: FontWeight.bold,
letterSpacing: 8,
),
),
const SizedBox(height: 32),
Row(
mainAxisSize: MainAxisSize.min,
children: [
Icon(
_connected ? Icons.circle : Icons.circle_outlined,
color: _connected ? Colors.green : Colors.orange,
size: 14,
),
const SizedBox(width: 8),
Text(_status),
],
), ),
const SizedBox(width: 8),
Text(_status),
], ],
), ),
], ),
),
if (_showStats)
Positioned(
top: 12,
left: 12,
child: _StatsOverlay(lines: [
'Host:',
' FPS: $_statFps',
' Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
' Dropped: $_statDropped/s',
' Buffer: $_statBufferKb KB',
' Send: ${_statSendMs.toStringAsFixed(1)}ms',
'',
if (_viewerStats == null)
'Viewer: no data'
else ...[
'Viewer:',
' FPS: ${_viewerStats!['fps']}',
' Bitrate: ${(_viewerStats!['bitrateMbps'] as num).toStringAsFixed(2)} Mbps',
' Dropped: ${_viewerStats!['dropped']}/s',
' Res: ${_viewerStats!['resolution']}',
' Latency: ${_viewerStats!['latencyMs']}ms',
' Submit: ${_viewerStats!['decodeMs'] ?? ''}ms',
],
]),
),
],
),
);
}
}
class _StatsOverlay extends StatelessWidget {
final List<String> lines;
const _StatsOverlay({required this.lines});
@override
Widget build(BuildContext context) {
return Container(
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
decoration: BoxDecoration(
color: Colors.black.withValues(alpha: 0.6),
borderRadius: BorderRadius.circular(6),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: lines.map((l) => Text(
l,
style: const TextStyle(
color: Colors.white,
fontFamily: 'monospace',
fontSize: 12,
),
)).toList(),
), ),
); );
} }

View file

@ -1,5 +1,6 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:typed_data';
import 'package:flutter/material.dart'; import 'package:flutter/material.dart';
import 'package:flutter/services.dart'; import 'package:flutter/services.dart';
@ -16,10 +17,11 @@ class ViewerScreen extends StatefulWidget {
} }
class _ViewerScreenState extends State<ViewerScreen> { class _ViewerScreenState extends State<ViewerScreen> {
static const _videoCh = MethodChannel("com.pulsar/video");
final _sig = SignallingService(); final _sig = SignallingService();
RTCPeerConnection? _pc; RTCPeerConnection? _pc;
RTCDataChannel? _dataChannel; RTCDataChannel? _dataChannel;
final _remoteRenderer = RTCVideoRenderer();
final _roomCodeCtrl = TextEditingController(); final _roomCodeCtrl = TextEditingController();
String _status = 'Enter a room code to connect'; String _status = 'Enter a room code to connect';
@ -28,23 +30,46 @@ class _ViewerScreenState extends State<ViewerScreen> {
StreamSubscription? _sigSub; StreamSubscription? _sigSub;
// size of the rendered video widget, used to normalise pointer coords int? _textureId;
Size _videoSize = Size.zero; bool _decoderReady = false;
final _videoKey = GlobalKey(); bool _settingUpDecoder = false;
@override // frames that arrive while createDecoder is in-flight are queued here and
void initState() { // drained once the decoder is ready, so we don't miss the initial keyframe
super.initState(); final _pendingNals = <(Uint8List, int)>[];
_remoteRenderer.initialize();
} // widget key for normalising input coords
final _videoKey = GlobalKey();
Size _videoSize = Size.zero;
// stats
bool _showStats = false;
int _statFps = 0;
double _statBitrateMbps = 0;
int _statDropped = 0;
String _statResolution = '';
int _statLatencyMs = 0;
double _statDecodeMs = 0; // actually submit time for feedNal
int _frameCount = 0;
int _byteCount = 0;
int _droppedCount = 0;
double _decodeMsSum = 0;
int _decodeMsCount = 0;
Timer? _statsTimer;
@override @override
void dispose() { void dispose() {
_statsTimer?.cancel();
_sigSub?.cancel(); _sigSub?.cancel();
_sig.dispose(); _sig.dispose();
_pc?.close(); _pc?.close();
_remoteRenderer.dispose();
_roomCodeCtrl.dispose(); _roomCodeCtrl.dispose();
if (_textureId != null) {
_videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
}
super.dispose(); super.dispose();
} }
@ -92,6 +117,7 @@ class _ViewerScreenState extends State<ViewerScreen> {
break; break;
case 'host_disconnected': case 'host_disconnected':
_statsTimer?.cancel();
setState(() { setState(() {
_status = 'Host disconnected'; _status = 'Host disconnected';
_streaming = false; _streaming = false;
@ -109,9 +135,9 @@ class _ViewerScreenState extends State<ViewerScreen> {
Future<void> _initPeerConnection(String roomId) async { Future<void> _initPeerConnection(String roomId) async {
final config = { final config = {
'iceServers': [ 'iceServers': [{'urls': 'stun:stun.l.google.com:19302'}],
{'urls': 'stun:stun.l.google.com:19302'}, 'sdpSemantics': 'unified-plan',
], 'encodedInsertableStreams': false,
}; };
_pc = await createPeerConnection(config); _pc = await createPeerConnection(config);
@ -120,18 +146,17 @@ class _ViewerScreenState extends State<ViewerScreen> {
_sig.send({ _sig.send({
'type': 'signal', 'type': 'signal',
'roomId': roomId, 'roomId': roomId,
'data': { 'data': {'type': 'candidate', 'candidate': candidate.toMap()},
'type': 'candidate',
'candidate': candidate.toMap(),
},
}); });
}; };
_pc!.onConnectionState = (state) { _pc!.onConnectionState = (state) {
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) { if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
setState(() => _status = 'Connected'); setState(() => _status = 'Connected');
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed || } else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) { state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
_statsTimer?.cancel();
setState(() { setState(() {
_status = 'Connection lost'; _status = 'Connection lost';
_streaming = false; _streaming = false;
@ -139,21 +164,133 @@ class _ViewerScreenState extends State<ViewerScreen> {
} }
}; };
_pc!.onTrack = (event) {
if (event.track.kind == 'video') {
setState(() {
_remoteRenderer.srcObject = event.streams[0];
_streaming = true;
_status = 'Streaming';
});
}
};
_pc!.onDataChannel = (channel) { _pc!.onDataChannel = (channel) {
_dataChannel = channel; _dataChannel = channel;
channel.onMessage = (msg) {
if (msg.isBinary) _onData(msg.binary);
};
}; };
} }
void _flushStats() {
setState(() {
_statFps = _frameCount;
_statBitrateMbps = _byteCount * 8 / 1e6;
_statDropped = _droppedCount;
_statDecodeMs = _decodeMsCount > 0 ? _decodeMsSum / _decodeMsCount : 0;
_frameCount = 0;
_byteCount = 0;
_droppedCount = 0;
_decodeMsSum = 0; _decodeMsCount = 0;
});
_sendInput({
'type': 'stats',
'fps': _statFps,
'bitrateMbps': _statBitrateMbps,
'dropped': _statDropped,
'resolution': _statResolution,
'latencyMs': _statLatencyMs,
'decodeMs': double.parse(_statDecodeMs.toStringAsFixed(1)),
'paintMs': 0.0,
});
}
void _onData(Uint8List bytes) {
if (bytes.isEmpty) return;
_byteCount += bytes.length;
final type = bytes[0];
if (type == 0x01) {
// config packet: SPS+PPS create/reconfigure the decoder
final spsPps = Uint8List.sublistView(bytes, 1);
_setupDecoder(spsPps);
} else if (type == 0x02) {
// frame packet: [0x02][8-byte ts][Annex B NALU]
if (bytes.length < 10) return;
final bd = ByteData.sublistView(bytes, 1, 9);
final ts = bd.getInt64(0, Endian.big);
final latency = DateTime.now().millisecondsSinceEpoch - ts;
final nal = Uint8List.sublistView(bytes, 9);
// drop frames that are already stale prevents burst playback after a lag spike
if (latency > 300) return;
if (!_decoderReady) {
_pendingNals.add((nal, latency));
} else {
_feedNal(nal, latency);
}
}
}
Future<void> _setupDecoder(Uint8List spsPps) async {
// a second 0x01 packet can arrive while createDecoder is still awaiting;
// ignore it the first decoder will be ready soon enough
if (_settingUpDecoder) return;
_settingUpDecoder = true;
_decoderReady = false;
_pendingNals.clear();
if (_textureId != null) {
await _videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
_textureId = null;
}
try {
final res = await _videoCh.invokeMethod<Map>('createDecoder', {'spsPps': spsPps});
final id = (res?['textureId'] as num?)?.toInt();
if (!mounted) return;
setState(() {
_textureId = id;
if (!_streaming) _streaming = true;
});
_decoderReady = true;
// drain frames that queued up while createDecoder was in-flight
final pending = List.of(_pendingNals);
_pendingNals.clear();
for (final (nal, latency) in pending) {
_feedNal(nal, latency);
}
} catch (_) {
_decoderReady = false;
} finally {
_settingUpDecoder = false;
}
}
Future<void> _feedNal(Uint8List nal, int latency) async {
if (_textureId == null) return;
try {
final t0 = DateTime.now();
await _videoCh.invokeMethod('feedNal', {
'textureId': _textureId,
'nal': nal,
});
final elapsed = DateTime.now().difference(t0).inMicroseconds / 1000.0;
if (!mounted) return;
_decodeMsSum += elapsed;
_decodeMsCount++;
_frameCount++;
setState(() {
_statLatencyMs = latency.clamp(0, 9999);
});
} catch (_) {}
}
Future<void> _handleSignalData(Map<String, dynamic> data, String roomId) async { Future<void> _handleSignalData(Map<String, dynamic> data, String roomId) async {
final sigType = data['type'] as String?; final sigType = data['type'] as String?;
@ -169,7 +306,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
'roomId': roomId, 'roomId': roomId,
'data': {'type': 'answer', 'sdp': answer.sdp}, 'data': {'type': 'answer', 'sdp': answer.sdp},
}); });
} else if (sigType == 'candidate') { } else if (sigType == 'candidate') {
final raw = data['candidate'] as Map<String, dynamic>; final raw = data['candidate'] as Map<String, dynamic>;
final candidate = RTCIceCandidate( final candidate = RTCIceCandidate(
@ -188,7 +324,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
} catch (_) {} } catch (_) {}
} }
// get video widget size via the global key
void _updateVideoSize() { void _updateVideoSize() {
final ctx = _videoKey.currentContext; final ctx = _videoKey.currentContext;
if (ctx != null) { if (ctx != null) {
@ -206,8 +341,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
); );
} }
// ---- input capture
void _onPointerMove(PointerMoveEvent e) { void _onPointerMove(PointerMoveEvent e) {
final n = _normalise(e.localPosition); final n = _normalise(e.localPosition);
_sendInput({'type': 'move', 'x': n.dx, 'y': n.dy}); _sendInput({'type': 'move', 'x': n.dx, 'y': n.dy});
@ -215,7 +348,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
void _onPointerDown(PointerDownEvent e) { void _onPointerDown(PointerDownEvent e) {
final n = _normalise(e.localPosition); final n = _normalise(e.localPosition);
// button 0 = left, 1 = right (PointerDeviceKind bit not directly available)
_sendInput({'type': 'click', 'x': n.dx, 'y': n.dy, 'button': 0}); _sendInput({'type': 'click', 'x': n.dx, 'y': n.dy, 'button': 0});
} }
@ -234,6 +366,14 @@ class _ViewerScreenState extends State<ViewerScreen> {
title: const Text('Pulsar — Viewer'), title: const Text('Pulsar — Viewer'),
backgroundColor: Theme.of(context).colorScheme.inversePrimary, backgroundColor: Theme.of(context).colorScheme.inversePrimary,
), ),
floatingActionButton: _streaming
? FloatingActionButton(
mini: true,
tooltip: 'Stats',
onPressed: () => setState(() => _showStats = !_showStats),
child: const Icon(Icons.bar_chart),
)
: null,
body: _streaming ? _buildStream() : _buildJoinView(), body: _streaming ? _buildStream() : _buildJoinView(),
); );
} }
@ -296,20 +436,66 @@ class _ViewerScreenState extends State<ViewerScreen> {
} }
Widget _buildStream() { Widget _buildStream() {
return Focus( return Stack(
autofocus: true, children: [
onKeyEvent: (_, e) { Focus(
_onKeyEvent(e); autofocus: true,
return KeyEventResult.handled; onKeyEvent: (_, e) {
}, _onKeyEvent(e);
child: Listener( return KeyEventResult.handled;
onPointerMove: _onPointerMove, },
onPointerDown: _onPointerDown, child: Listener(
child: RTCVideoView( onPointerMove: _onPointerMove,
_remoteRenderer, onPointerDown: _onPointerDown,
key: _videoKey, child: _textureId != null
objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, ? SizedBox.expand(
key: _videoKey,
child: Texture(textureId: _textureId!),
)
: const Center(child: CircularProgressIndicator()),
),
), ),
if (_showStats)
Positioned(
top: 12,
left: 12,
child: _StatsOverlay(lines: [
'FPS: $_statFps',
'Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
'Dropped: $_statDropped/s',
'Res: $_statResolution',
'Latency: ${_statLatencyMs}ms',
'Submit: ${_statDecodeMs.toStringAsFixed(1)}ms',
]),
),
],
);
}
}
class _StatsOverlay extends StatelessWidget {
final List<String> lines;
const _StatsOverlay({required this.lines});
@override
Widget build(BuildContext context) {
return Container(
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
decoration: BoxDecoration(
color: Colors.black.withValues(alpha: 0.6),
borderRadius: BorderRadius.circular(6),
),
child: Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: lines.map((l) => Text(
l,
style: const TextStyle(
color: Colors.white,
fontFamily: 'monospace',
fontSize: 12,
),
)).toList(),
), ),
); );
} }

View file

@ -20,10 +20,10 @@ class SignallingService {
} catch (_) {} } catch (_) {}
}, },
onError: (err) { onError: (err) {
_controller.addError(err); if (!_controller.isClosed) _controller.addError(err);
}, },
onDone: () { onDone: () {
_controller.addError(Exception('WebSocket connection closed')); if (!_controller.isClosed) _controller.addError(Exception('WebSocket connection closed'));
}, },
); );
} }

View file

@ -1,9 +1,9 @@
PODS: PODS:
- flutter_webrtc (0.9.36): - flutter_webrtc (1.4.0):
- FlutterMacOS - FlutterMacOS
- WebRTC-SDK (= 114.5735.08) - WebRTC-SDK (= 144.7559.01)
- FlutterMacOS (1.0.0) - FlutterMacOS (1.0.0)
- WebRTC-SDK (114.5735.08) - WebRTC-SDK (144.7559.01)
DEPENDENCIES: DEPENDENCIES:
- flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`) - flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`)
@ -20,9 +20,9 @@ EXTERNAL SOURCES:
:path: Flutter/ephemeral :path: Flutter/ephemeral
SPEC CHECKSUMS: SPEC CHECKSUMS:
flutter_webrtc: cf7dc44d26cbb5c5f1ae5f583dab545871f287f9 flutter_webrtc: dbacf72a9bd951ccfa5997198f0a6214bb848ede
FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1 FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1
WebRTC-SDK: c24d2a6c9f571f2ed42297cb8ffba9557093142b WebRTC-SDK: ab9b5319e458c2bfebdc92b3600740da35d5630d
PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009 PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009

View file

@ -1,20 +1,37 @@
import Cocoa import Cocoa
import FlutterMacOS import FlutterMacOS
import CoreGraphics import CoreGraphics
import ScreenCaptureKit
import CoreMedia
import CoreVideo
import VideoToolbox
@main @main
class AppDelegate: FlutterAppDelegate { class AppDelegate: FlutterAppDelegate {
private var inputChannel: FlutterMethodChannel? private var inputChannel: FlutterMethodChannel?
private var videoChannel: FlutterMethodChannel?
private var frameEventSink: FlutterEventSink?
// SCK objects (typed Any? to satisfy 10.15 deployment target)
private var scStream: Any?
private var screenDelegate: Any?
private var streamReady = false
// VTCompressionSession (Any? for same reason)
private var vtSession: Any?
private var encoderReady = false
// resolution last seen from SCK (to build config)
private var captureWidth = 1280
private var captureHeight = 720
override func applicationDidFinishLaunching(_ notification: Notification) { override func applicationDidFinishLaunching(_ notification: Notification) {
guard let window = mainFlutterWindow, guard let window = mainFlutterWindow,
let ctrl = window.contentViewController as? FlutterViewController else { return } let ctrl = window.contentViewController as? FlutterViewController else { return }
inputChannel = FlutterMethodChannel( let messenger = ctrl.engine.binaryMessenger
name: "com.pulsar/input",
binaryMessenger: ctrl.engine.binaryMessenger
)
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
inputChannel?.setMethodCallHandler { [weak self] call, result in inputChannel?.setMethodCallHandler { [weak self] call, result in
if call.method == "injectInput", if call.method == "injectInput",
let args = call.arguments as? [String: Any] { let args = call.arguments as? [String: Any] {
@ -24,8 +41,302 @@ class AppDelegate: FlutterAppDelegate {
result(FlutterMethodNotImplemented) result(FlutterMethodNotImplemented)
} }
} }
// event channel for pushing encoded NAL units to Flutter
let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
frameChannel.setStreamHandler(self)
videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
videoChannel?.setMethodCallHandler { [weak self] call, result in
guard let self = self else { return }
switch call.method {
case "start":
self.startEncoder(result: result)
case "stop":
self.stopEncoder()
result(nil)
case "forceKeyframe":
self.requestKeyframe()
result(nil)
default:
result(FlutterMethodNotImplemented)
}
}
if #available(macOS 12.3, *) {
Task { await self.ensureSCKStream() }
}
} }
// MARK: - SCK stream setup
@available(macOS 12.3, *)
private func ensureSCKStream() async {
guard scStream == nil else { return }
guard let content = try? await SCShareableContent.current else { return }
guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
let filter = SCContentFilter(display: display, excludingWindows: [])
let srcW = Double(display.width)
let srcH = Double(display.height)
let scale = min(1.0, 1280.0 / srcW)
let dstW = Int(srcW * scale)
let dstH = Int(srcH * scale)
captureWidth = dstW
captureHeight = dstH
let cfg = SCStreamConfiguration()
cfg.width = dstW
cfg.height = dstH
cfg.pixelFormat = kCVPixelFormatType_32BGRA
cfg.showsCursor = true
cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
let del = ScreenDelegate { [weak self] sampleBuffer in
self?.encodeFrame(sampleBuffer)
}
screenDelegate = del
let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
do {
try await stream.startCapture()
scStream = stream
streamReady = true
} catch {}
}
// MARK: - VTCompressionSession
private func startEncoder(result: @escaping FlutterResult) {
let w = captureWidth
let h = captureHeight
var session: VTCompressionSession?
let status = VTCompressionSessionCreate(
allocator: kCFAllocatorDefault,
width: Int32(w),
height: Int32(h),
codecType: kCMVideoCodecType_H264,
encoderSpecification: nil,
imageBufferAttributes: nil,
compressedDataAllocator: nil,
outputCallback: vtOutputCallback,
refcon: Unmanaged.passUnretained(self).toOpaque(),
compressionSessionOut: &session
)
guard status == noErr, let session = session else {
result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
return
}
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
value: kVTProfileLevel_H264_Baseline_AutoLevel)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
value: 6_000_000 as CFNumber)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
value: 60 as CFNumber)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
value: 120 as CFNumber)
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
value: 0 as CFNumber)
VTCompressionSessionPrepareToEncodeFrames(session)
vtSession = VTSessionBox(session)
encoderReady = true
// extract SPS+PPS from a forced keyframe by encoding a dummy frame
// instead, return empty bytes and let the real SPS+PPS come through the callback
// host_screen will call forceKeyframe right after start
result(FlutterStandardTypedData(bytes: Data()))
}
private func stopEncoder() {
guard let box = vtSession as? VTSessionBox else { return }
VTCompressionSessionInvalidate(box.session)
vtSession = nil
encoderReady = false
}
func requestKeyframe() {
forceKeyframeNext = true
}
private var forceKeyframeNext = false
private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
guard encoderReady, let box = vtSession as? VTSessionBox else { return }
let s = box.session
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
var frameProps: CFDictionary? = nil
if forceKeyframeNext {
frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
forceKeyframeNext = false
}
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
duration: .invalid, frameProperties: frameProps,
sourceFrameRefcon: nil, infoFlagsOut: nil)
}
// MARK: - VT output callback (C function)
// called by VideoToolbox on an internal thread collect Data packets here,
// then hop to main before touching the event sink
func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
var packets: [Data] = []
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
let isKeyframe = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
}
if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
var totalLen = 0
var rawPtr: UnsafeMutablePointer<CChar>? = nil
if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
let ptr = rawPtr {
let ts = Int64(Date().timeIntervalSince1970 * 1000)
var offset = 0
while offset + 4 <= totalLen {
// read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
offset += 4
guard naluLen > 0, offset + naluLen <= totalLen else { break }
var packet = Data(capacity: 1 + 8 + 4 + naluLen)
packet.append(0x02)
withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
packets.append(packet)
offset += naluLen
}
}
}
guard !packets.isEmpty else { return }
DispatchQueue.main.async { [weak self] in
guard let sink = self?.frameEventSink else { return }
for packet in packets {
sink(FlutterStandardTypedData(bytes: packet))
}
}
}
private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
var spsCount = 0
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
nalUnitHeaderLengthOut: nil)
var packet = Data()
packet.append(0x01)
for i in 0 ..< spsCount {
var ptr: UnsafePointer<UInt8>? = nil
var len = 0
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
parameterSetPointerOut: &ptr,
parameterSetSizeOut: &len,
parameterSetCountOut: nil,
nalUnitHeaderLengthOut: nil)
if let ptr = ptr {
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
packet.append(Data(bytes: ptr, count: len))
}
}
// PPS sits at index spsCount
var ppsPtr: UnsafePointer<UInt8>? = nil
var ppsLen = 0
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
parameterSetPointerOut: &ppsPtr,
parameterSetSizeOut: &ppsLen,
parameterSetCountOut: nil,
nalUnitHeaderLengthOut: nil) == noErr,
let ppsPtr = ppsPtr {
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
packet.append(Data(bytes: ppsPtr, count: ppsLen))
}
return packet.count > 1 ? packet : nil
}
// MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths unused in normal operation)
private func captureCG() -> Data? {
let displayID = CGMainDisplayID()
guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
let srcW = CGFloat(composited.width)
let scale = min(1.0, 1280.0 / srcW)
let dstW = Int(srcW * scale)
let dstH = Int(CGFloat(composited.height) * scale)
let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
bytesPerRow: 0, space: colorSpace,
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
ctx.interpolationQuality = .high
ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
guard let scaled = ctx.makeImage() else { return nil }
return jpegData(from: scaled, quality: 0.4)
}
private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
let nsImage = NSImage(cgImage: cgImage, size: .zero)
guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
}
private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
let imgW = cgImage.width
let imgH = cgImage.height
let mousePt = NSEvent.mouseLocation
let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
let cursorX = mousePt.x * scale
let cursorY = (screenH - mousePt.y) * scale
let cursor = NSCursor.current
let cursorImg = cursor.image
let hotspot = cursor.hotSpot
let cw = cursorImg.size.width * scale
let ch = cursorImg.size.height * scale
let drawX = cursorX - hotspot.x * scale
let drawY = cursorY - hotspot.y * scale
let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
bytesPerRow: 0, space: colorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
}
return ctx.makeImage()
}
// MARK: - Input injection
private func injectInput(_ ev: [String: Any]) { private func injectInput(_ ev: [String: Any]) {
guard let type = ev["type"] as? String else { return } guard let type = ev["type"] as? String else { return }
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080) let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
@ -34,24 +345,17 @@ class AppDelegate: FlutterAppDelegate {
case "move": case "move":
let nx = ev["x"] as? Double ?? 0 let nx = ev["x"] as? Double ?? 0
let ny = ev["y"] as? Double ?? 0 let ny = ev["y"] as? Double ?? 0
// CoreGraphics origin is top-left on screen, NSScreen origin is bottom-left let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
let px = CGFloat(nx) * screen.width
let py = CGFloat(ny) * screen.height
let pt = CGPoint(x: px, y: py)
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)? CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
.post(tap: .cghidEventTap) .post(tap: .cghidEventTap)
case "click": case "click":
let nx = ev["x"] as? Double ?? 0 let nx = ev["x"] as? Double ?? 0
let ny = ev["y"] as? Double ?? 0 let ny = ev["y"] as? Double ?? 0
let btn = ev["button"] as? Int ?? 0 let btn = ev["button"] as? Int ?? 0
let px = CGFloat(nx) * screen.width let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
let py = CGFloat(ny) * screen.height
let pt = CGPoint(x: px, y: py)
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) = let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left) btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)? CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
.post(tap: .cghidEventTap) .post(tap: .cghidEventTap)
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)? CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
@ -69,16 +373,71 @@ class AppDelegate: FlutterAppDelegate {
.post(tap: .cghidEventTap) .post(tap: .cghidEventTap)
} }
default: default: break
break
} }
} }
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { // MARK: - App lifecycle
return true
override func applicationWillTerminate(_ notification: Notification) {
stopEncoder()
if #available(macOS 12.3, *), let stream = scStream as? SCStream {
Task { try? await stream.stopCapture() }
}
} }
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
return true override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
}
// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
private class VTSessionBox {
let session: VTCompressionSession
init(_ s: VTCompressionSession) { session = s }
}
// MARK: - VT output C callback
private func vtOutputCallback(
outputCallbackRefCon: UnsafeMutableRawPointer?,
sourceFrameRefCon: UnsafeMutableRawPointer?,
status: OSStatus,
infoFlags: VTEncodeInfoFlags,
sampleBuffer: CMSampleBuffer?
) {
guard status == noErr, let sampleBuffer = sampleBuffer,
let refcon = outputCallbackRefCon else { return }
let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
delegate.handleEncodedFrame(sampleBuffer)
}
// MARK: - SCK delegate
@available(macOS 12.3, *)
class ScreenDelegate: NSObject, SCStreamOutput {
private let onFrame: (CMSampleBuffer) -> Void
init(onFrame: @escaping (CMSampleBuffer) -> Void) {
self.onFrame = onFrame
}
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .screen else { return }
onFrame(sampleBuffer)
}
}
// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
extension AppDelegate: FlutterStreamHandler {
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
frameEventSink = events
return nil
}
func onCancel(withArguments arguments: Any?) -> FlutterError? {
frameEventSink = nil
return nil
} }
} }

View file

@ -77,10 +77,10 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: dart_webrtc name: dart_webrtc
sha256: ac7ef077084b3e54004716f1d736fcd839e1b60bc3f21f4122a35a9bb5ca2e47 sha256: f6d615bddea5e458ce180a914f3055c234ffb52fb7397a51b3491e76d6d7edb2
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "1.4.8" version: "1.8.1"
fake_async: fake_async:
dependency: transitive dependency: transitive
description: description:
@ -127,10 +127,10 @@ packages:
dependency: "direct main" dependency: "direct main"
description: description:
name: flutter_webrtc name: flutter_webrtc
sha256: "2f17fb96e0c9c6ff75f6b1c36d94755461fc7f36a5c28386f5ee5a18b98688c8" sha256: c7b0a67ca2c878575fc5c146d801cd874f58f5f1ef5fa6e8eb0c93d413beb948
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.9.48+hotfix.1" version: "1.4.1"
glob: glob:
dependency: transitive dependency: transitive
description: description:
@ -203,6 +203,14 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "6.1.0" version: "6.1.0"
logger:
dependency: transitive
description:
name: logger
sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c"
url: "https://pub.dev"
source: hosted
version: "2.7.0"
logging: logging:
dependency: transitive dependency: transitive
description: description:
@ -323,14 +331,6 @@ packages:
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "3.1.6" version: "3.1.6"
platform_detect:
dependency: transitive
description:
name: platform_detect
sha256: "6e91e2158b9db7f6f4581a0f0b3a77171570b392108de196ba0c52e3c1976334"
url: "https://pub.dev"
source: hosted
version: "2.1.6"
plugin_platform_interface: plugin_platform_interface:
dependency: transitive dependency: transitive
description: description:
@ -444,18 +444,18 @@ packages:
dependency: transitive dependency: transitive
description: description:
name: web name: web
sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27" sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "0.5.1" version: "1.1.1"
web_socket_channel: web_socket_channel:
dependency: "direct main" dependency: "direct main"
description: description:
name: web_socket_channel name: web_socket_channel
sha256: "58c6666b342a38816b2e7e50ed0f1e261959630becd4c879c4f26bfa14aa5a42" sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b
url: "https://pub.dev" url: "https://pub.dev"
source: hosted source: hosted
version: "2.4.5" version: "2.4.0"
webrtc_interface: webrtc_interface:
dependency: transitive dependency: transitive
description: description:

View file

@ -34,7 +34,7 @@ dependencies:
# The following adds the Cupertino Icons font to your application. # The following adds the Cupertino Icons font to your application.
# Use with the CupertinoIcons class for iOS style icons. # Use with the CupertinoIcons class for iOS style icons.
cupertino_icons: ^1.0.8 cupertino_icons: ^1.0.8
flutter_webrtc: ^0.9.47 flutter_webrtc: ^1.4.1
web_socket_channel: ^2.4.0 web_socket_channel: ^2.4.0
dev_dependencies: dev_dependencies: