inital
This commit is contained in:
parent
0ed740ad19
commit
bfdaf4a801
9 changed files with 1081 additions and 149 deletions
|
|
@ -1,4 +1,13 @@
|
||||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||||
|
<uses-permission android:name="android.permission.INTERNET"/>
|
||||||
|
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
|
||||||
|
<uses-permission android:name="android.permission.CHANGE_NETWORK_STATE"/>
|
||||||
|
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
|
||||||
|
<uses-permission android:name="android.permission.CAMERA"/>
|
||||||
|
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
|
||||||
|
<uses-permission android:name="android.permission.BLUETOOTH" android:maxSdkVersion="30"/>
|
||||||
|
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/>
|
||||||
|
|
||||||
<application
|
<application
|
||||||
android:label="pulsar"
|
android:label="pulsar"
|
||||||
android:name="${applicationName}"
|
android:name="${applicationName}"
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,255 @@
|
||||||
package net.imbenji.pulsar;
|
package net.imbenji.pulsar;
|
||||||
|
|
||||||
|
import android.media.MediaCodec;
|
||||||
|
import android.media.MediaCodecInfo;
|
||||||
|
import android.media.MediaFormat;
|
||||||
|
import android.os.Handler;
|
||||||
|
import android.os.HandlerThread;
|
||||||
|
import android.view.Surface;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
import io.flutter.embedding.android.FlutterActivity;
|
import io.flutter.embedding.android.FlutterActivity;
|
||||||
|
import io.flutter.embedding.engine.FlutterEngine;
|
||||||
|
import io.flutter.plugin.common.MethodChannel;
|
||||||
|
import io.flutter.view.TextureRegistry;
|
||||||
|
|
||||||
public class MainActivity extends FlutterActivity {
|
public class MainActivity extends FlutterActivity {
|
||||||
|
|
||||||
|
private static final String CHANNEL = "com.pulsar/video";
|
||||||
|
|
||||||
|
// one decoder entry per textureId
|
||||||
|
private static class DecoderEntry {
|
||||||
|
TextureRegistry.SurfaceTextureEntry textureEntry;
|
||||||
|
Surface surface;
|
||||||
|
MediaCodec codec;
|
||||||
|
HandlerThread outputThread;
|
||||||
|
Handler outputHandler;
|
||||||
|
HandlerThread inputThread;
|
||||||
|
Handler inputHandler;
|
||||||
|
AtomicBoolean running = new AtomicBoolean(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Map<Long, DecoderEntry> decoders = new HashMap<>();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void configureFlutterEngine(FlutterEngine flutterEngine) {
|
||||||
|
super.configureFlutterEngine(flutterEngine);
|
||||||
|
|
||||||
|
TextureRegistry registry = flutterEngine.getRenderer();
|
||||||
|
|
||||||
|
new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), CHANNEL)
|
||||||
|
.setMethodCallHandler((call, result) -> {
|
||||||
|
switch (call.method) {
|
||||||
|
|
||||||
|
case "createDecoder": {
|
||||||
|
byte[] spsPps = call.argument("spsPps");
|
||||||
|
if (spsPps == null) {
|
||||||
|
result.error("BAD_ARGS", "spsPps required", null);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
TextureRegistry.SurfaceTextureEntry entry = registry.createSurfaceTexture();
|
||||||
|
long texId = entry.id();
|
||||||
|
|
||||||
|
// we don't know the video size yet — configure with a resonable
|
||||||
|
// placeholder; MediaCodec will adapt once it sees the stream
|
||||||
|
int width = 1280;
|
||||||
|
int height = 720;
|
||||||
|
|
||||||
|
MediaFormat fmt = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
|
||||||
|
|
||||||
|
// csd-0 must include Annex B start codes — pass the whole SPS+PPS blob
|
||||||
|
// (MediaCodec H264 accepts combined SPS+PPS with start codes in csd-0)
|
||||||
|
fmt.setByteBuffer("csd-0", ByteBuffer.wrap(spsPps));
|
||||||
|
|
||||||
|
entry.surfaceTexture().setDefaultBufferSize(width, height);
|
||||||
|
|
||||||
|
fmt.setInteger(MediaFormat.KEY_COLOR_FORMAT,
|
||||||
|
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
|
||||||
|
|
||||||
|
Surface surface = new Surface(entry.surfaceTexture());
|
||||||
|
|
||||||
|
MediaCodec codec;
|
||||||
|
try {
|
||||||
|
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
|
||||||
|
codec.configure(fmt, surface, null, 0);
|
||||||
|
codec.start();
|
||||||
|
} catch (IOException e) {
|
||||||
|
surface.release();
|
||||||
|
entry.release();
|
||||||
|
result.error("CODEC_FAIL", e.getMessage(), null);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
DecoderEntry dec = new DecoderEntry();
|
||||||
|
dec.textureEntry = entry;
|
||||||
|
dec.surface = surface;
|
||||||
|
dec.codec = codec;
|
||||||
|
|
||||||
|
HandlerThread ot = new HandlerThread("decoder-out-" + texId);
|
||||||
|
ot.start();
|
||||||
|
dec.outputThread = ot;
|
||||||
|
dec.outputHandler = new Handler(ot.getLooper());
|
||||||
|
|
||||||
|
HandlerThread it = new HandlerThread("decoder-in-" + texId);
|
||||||
|
it.start();
|
||||||
|
dec.inputThread = it;
|
||||||
|
dec.inputHandler = new Handler(it.getLooper());
|
||||||
|
|
||||||
|
drainOutputLoop(dec);
|
||||||
|
|
||||||
|
decoders.put(texId, dec);
|
||||||
|
|
||||||
|
Map<String, Long> res = new HashMap<>();
|
||||||
|
res.put("textureId", texId);
|
||||||
|
result.success(res);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "feedNal": {
|
||||||
|
long texId = ((Number) call.argument("textureId")).longValue();
|
||||||
|
byte[] nal = call.argument("nal");
|
||||||
|
|
||||||
|
DecoderEntry dec = decoders.get(texId);
|
||||||
|
if (dec == null || nal == null) {
|
||||||
|
result.error("NO_DECODER", "decoder not found", null);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
dec.inputHandler.post(() -> {
|
||||||
|
try {
|
||||||
|
int idx = dec.codec.dequeueInputBuffer(0);
|
||||||
|
if (idx >= 0) {
|
||||||
|
ByteBuffer buf = dec.codec.getInputBuffer(idx);
|
||||||
|
if (buf != null) {
|
||||||
|
buf.clear();
|
||||||
|
buf.put(nal);
|
||||||
|
dec.codec.queueInputBuffer(idx, 0, nal.length,
|
||||||
|
System.nanoTime() / 1000, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// if idx < 0 the decoder is backed up — drop this NAL silently
|
||||||
|
} catch (Exception ignored) {}
|
||||||
|
});
|
||||||
|
|
||||||
|
result.success(null);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "stopDecoder": {
|
||||||
|
long texId = ((Number) call.argument("textureId")).longValue();
|
||||||
|
DecoderEntry dec = decoders.remove(texId);
|
||||||
|
if (dec != null) releaseDecoder(dec);
|
||||||
|
result.success(null);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
case "start":
|
||||||
|
case "stop":
|
||||||
|
case "forceKeyframe":
|
||||||
|
result.success(null);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
result.notImplemented();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void drainOutputLoop(DecoderEntry dec) {
|
||||||
|
dec.outputHandler.post(() -> {
|
||||||
|
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||||
|
while (dec.running.get()) {
|
||||||
|
try {
|
||||||
|
// block up to 10ms waiting for a decoded frame
|
||||||
|
int idx = dec.codec.dequeueOutputBuffer(info, 10_000);
|
||||||
|
if (idx >= 0) {
|
||||||
|
dec.codec.releaseOutputBuffer(idx, true);
|
||||||
|
}
|
||||||
|
// idx == INFO_TRY_AGAIN_LATER / FORMAT_CHANGED / BUFFERS_CHANGED → loop
|
||||||
|
} catch (Exception e) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Handler mainHandler = new Handler(android.os.Looper.getMainLooper());
|
||||||
|
|
||||||
|
private void releaseDecoder(DecoderEntry dec) {
|
||||||
|
dec.running.set(false);
|
||||||
|
// codec stop/release must happen on the decoder thread (after the drain loop exits)
|
||||||
|
// but entry.release() calls FlutterJNI.unregisterTexture which requires the main thread
|
||||||
|
dec.outputHandler.post(() -> {
|
||||||
|
try {
|
||||||
|
dec.codec.stop();
|
||||||
|
dec.codec.release();
|
||||||
|
} catch (Exception ignored) {}
|
||||||
|
dec.surface.release();
|
||||||
|
mainHandler.post(() -> dec.textureEntry.release());
|
||||||
|
});
|
||||||
|
dec.outputThread.quitSafely();
|
||||||
|
dec.inputThread.quitSafely();
|
||||||
|
}
|
||||||
|
|
||||||
|
// splits an Annex B blob into individual NAL unit ByteBuffers
|
||||||
|
// returns null if fewer than 2 NALUs found
|
||||||
|
private ByteBuffer[] splitAnnexB(byte[] data) {
|
||||||
|
java.util.List<ByteBuffer> units = new java.util.ArrayList<>();
|
||||||
|
int start = -1;
|
||||||
|
|
||||||
|
for (int i = 0; i < data.length - 3; i++) {
|
||||||
|
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1) {
|
||||||
|
if (start >= 0) {
|
||||||
|
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
|
||||||
|
units.add(bb);
|
||||||
|
}
|
||||||
|
start = i + 4;
|
||||||
|
i += 3;
|
||||||
|
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
|
||||||
|
if (start >= 0) {
|
||||||
|
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
|
||||||
|
units.add(bb);
|
||||||
|
}
|
||||||
|
start = i + 3;
|
||||||
|
i += 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (start >= 0 && start < data.length) {
|
||||||
|
units.add(ByteBuffer.wrap(data, start, data.length - start));
|
||||||
|
}
|
||||||
|
|
||||||
|
return units.size() >= 2 ? units.toArray(new ByteBuffer[0]) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// crude SPS parser — reads width/height from the first SPS NALU
|
||||||
|
// returns null if it cant parse
|
||||||
|
private int[] dimsFromSps(ByteBuffer spsBuf) {
|
||||||
|
try {
|
||||||
|
byte[] sps = new byte[spsBuf.remaining()];
|
||||||
|
spsBuf.duplicate().get(sps);
|
||||||
|
|
||||||
|
// skip NAL header byte (forbidden_zero_bit + nal_ref_idc + nal_unit_type)
|
||||||
|
// a minimal parse isn't reliable without a full RBSP/Exp-Golomb parser,
|
||||||
|
// so just return null and let MediaCodec figure it out
|
||||||
|
return null;
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void onDestroy() {
|
||||||
|
for (DecoderEntry dec : decoders.values()) {
|
||||||
|
releaseDecoder(dec);
|
||||||
|
}
|
||||||
|
decoders.clear();
|
||||||
|
super.onDestroy();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import 'dart:async';
|
import 'dart:async';
|
||||||
import 'dart:convert';
|
import 'dart:convert';
|
||||||
import 'dart:math';
|
import 'dart:math';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
import 'package:flutter/material.dart';
|
import 'package:flutter/material.dart';
|
||||||
import 'package:flutter/services.dart';
|
import 'package:flutter/services.dart';
|
||||||
|
|
@ -8,8 +9,9 @@ import 'package:flutter_webrtc/flutter_webrtc.dart';
|
||||||
|
|
||||||
import '../services/signalling.dart';
|
import '../services/signalling.dart';
|
||||||
|
|
||||||
// platform channel for injecting input into the OS
|
|
||||||
const _inputChannel = MethodChannel('com.pulsar/input');
|
const _inputChannel = MethodChannel('com.pulsar/input');
|
||||||
|
const _videoChannel = MethodChannel('com.pulsar/video');
|
||||||
|
const _frameEvents = EventChannel('com.pulsar/video/frames');
|
||||||
|
|
||||||
class HostScreen extends StatefulWidget {
|
class HostScreen extends StatefulWidget {
|
||||||
final String serverUrl;
|
final String serverUrl;
|
||||||
|
|
@ -23,13 +25,31 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
final _sig = SignallingService();
|
final _sig = SignallingService();
|
||||||
RTCPeerConnection? _pc;
|
RTCPeerConnection? _pc;
|
||||||
RTCDataChannel? _dataChannel;
|
RTCDataChannel? _dataChannel;
|
||||||
MediaStream? _localStream;
|
|
||||||
|
|
||||||
String _roomId = '';
|
String _roomId = '';
|
||||||
String _status = 'Initialising...';
|
String _status = 'Initialising...';
|
||||||
bool _connected = false;
|
bool _connected = false;
|
||||||
|
|
||||||
StreamSubscription? _sigSub;
|
StreamSubscription? _sigSub;
|
||||||
|
StreamSubscription? _frameSub;
|
||||||
|
|
||||||
|
// stats
|
||||||
|
bool _showStats = false;
|
||||||
|
int _statFps = 0;
|
||||||
|
double _statBitrateMbps = 0;
|
||||||
|
int _statDropped = 0;
|
||||||
|
int _statBufferKb = 0;
|
||||||
|
double _statSendMs = 0;
|
||||||
|
|
||||||
|
int _frameCount = 0;
|
||||||
|
int _byteCount = 0;
|
||||||
|
int _droppedCount = 0;
|
||||||
|
double _sendMsSum = 0;
|
||||||
|
int _sendMsCount = 0;
|
||||||
|
Timer? _statsTimer;
|
||||||
|
|
||||||
|
// viewer stats received over data channel
|
||||||
|
Map<String, dynamic>? _viewerStats;
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void initState() {
|
void initState() {
|
||||||
|
|
@ -67,7 +87,6 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case 'created':
|
case 'created':
|
||||||
_setState('Waiting for viewer...');
|
_setState('Waiting for viewer...');
|
||||||
await _startCapture();
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'peer_joined':
|
case 'peer_joined':
|
||||||
|
|
@ -82,6 +101,7 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
|
|
||||||
case 'peer_disconnected':
|
case 'peer_disconnected':
|
||||||
_setState('Viewer disconnected');
|
_setState('Viewer disconnected');
|
||||||
|
_stopCapture();
|
||||||
setState(() => _connected = false);
|
setState(() => _connected = false);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
|
@ -91,17 +111,6 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> _startCapture() async {
|
|
||||||
try {
|
|
||||||
_localStream = await navigator.mediaDevices.getDisplayMedia({
|
|
||||||
'video': {'cursor': 'always'},
|
|
||||||
'audio': false,
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
_setState('Screen capture failed: $e');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> _createOffer() async {
|
Future<void> _createOffer() async {
|
||||||
final config = {
|
final config = {
|
||||||
'iceServers': [
|
'iceServers': [
|
||||||
|
|
@ -111,31 +120,34 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
|
|
||||||
_pc = await createPeerConnection(config);
|
_pc = await createPeerConnection(config);
|
||||||
|
|
||||||
// data channel for input events (host creates it as offerer)
|
// unordered + unreliable — drop stale frames, only the latest matters
|
||||||
_dataChannel = await _pc!.createDataChannel(
|
final dcInit = RTCDataChannelInit()
|
||||||
'input',
|
..ordered = false
|
||||||
RTCDataChannelInit()..ordered = true,
|
..maxRetransmits = 0;
|
||||||
);
|
|
||||||
|
_dataChannel = await _pc!.createDataChannel('frames', dcInit);
|
||||||
|
|
||||||
_dataChannel!.onMessage = (msg) {
|
_dataChannel!.onMessage = (msg) {
|
||||||
_handleInputEvent(msg.text);
|
if (msg.isBinary) return;
|
||||||
};
|
try {
|
||||||
|
final parsed = jsonDecode(msg.text) as Map<String, dynamic>;
|
||||||
// add screen tracks
|
if (parsed['type'] == 'stats') {
|
||||||
if (_localStream != null) {
|
setState(() {
|
||||||
for (final track in _localStream!.getTracks()) {
|
_viewerStats = parsed;
|
||||||
await _pc!.addTrack(track, _localStream!);
|
});
|
||||||
|
} else {
|
||||||
|
_handleInputEvent(msg.text);
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
_handleInputEvent(msg.text);
|
||||||
}
|
}
|
||||||
}
|
};
|
||||||
|
|
||||||
_pc!.onIceCandidate = (candidate) {
|
_pc!.onIceCandidate = (candidate) {
|
||||||
_sig.send({
|
_sig.send({
|
||||||
'type': 'signal',
|
'type': 'signal',
|
||||||
'roomId': _roomId,
|
'roomId': _roomId,
|
||||||
'data': {
|
'data': {'type': 'candidate', 'candidate': candidate.toMap()},
|
||||||
'type': 'candidate',
|
|
||||||
'candidate': candidate.toMap(),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -143,9 +155,11 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
||||||
_setState('Connected');
|
_setState('Connected');
|
||||||
setState(() => _connected = true);
|
setState(() => _connected = true);
|
||||||
|
_startCapture();
|
||||||
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
||||||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
||||||
_setState('Connection lost');
|
_setState('Connection lost');
|
||||||
|
_stopCapture();
|
||||||
setState(() => _connected = false);
|
setState(() => _connected = false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -177,6 +191,58 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> _startCapture() async {
|
||||||
|
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
|
||||||
|
|
||||||
|
// subscribe BEFORE starting the encoder so the initial SPS+PPS+keyframe isn't dropped
|
||||||
|
_frameSub = _frameEvents.receiveBroadcastStream().listen((dynamic raw) {
|
||||||
|
final bytes = (raw as Uint8List);
|
||||||
|
final dc = _dataChannel;
|
||||||
|
if (dc == null) return;
|
||||||
|
|
||||||
|
final buffered = dc.bufferedAmount ?? 0;
|
||||||
|
if (buffered > 262144) {
|
||||||
|
_droppedCount++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
final sendStart = DateTime.now();
|
||||||
|
dc.send(RTCDataChannelMessage.fromBinary(bytes));
|
||||||
|
_sendMsSum += DateTime.now().difference(sendStart).inMicroseconds / 1000.0;
|
||||||
|
_sendMsCount++;
|
||||||
|
_frameCount++;
|
||||||
|
_byteCount += bytes.length;
|
||||||
|
} catch (_) {}
|
||||||
|
});
|
||||||
|
|
||||||
|
await _videoChannel.invokeMethod('start');
|
||||||
|
await _videoChannel.invokeMethod('forceKeyframe');
|
||||||
|
}
|
||||||
|
|
||||||
|
void _stopCapture() {
|
||||||
|
_frameSub?.cancel();
|
||||||
|
_frameSub = null;
|
||||||
|
_statsTimer?.cancel();
|
||||||
|
_statsTimer = null;
|
||||||
|
_videoChannel.invokeMethod('stop');
|
||||||
|
}
|
||||||
|
|
||||||
|
void _flushStats() {
|
||||||
|
final buf = _dataChannel?.bufferedAmount ?? 0;
|
||||||
|
setState(() {
|
||||||
|
_statFps = _frameCount;
|
||||||
|
_statBitrateMbps = _byteCount * 8 / 1e6;
|
||||||
|
_statDropped = _droppedCount;
|
||||||
|
_statBufferKb = (buf / 1024).round();
|
||||||
|
_statSendMs = _sendMsCount > 0 ? _sendMsSum / _sendMsCount : 0;
|
||||||
|
_frameCount = 0;
|
||||||
|
_byteCount = 0;
|
||||||
|
_droppedCount = 0;
|
||||||
|
_sendMsSum = 0; _sendMsCount = 0;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
void _handleInputEvent(String json) async {
|
void _handleInputEvent(String json) async {
|
||||||
try {
|
try {
|
||||||
final ev = jsonDecode(json) as Map<String, dynamic>;
|
final ev = jsonDecode(json) as Map<String, dynamic>;
|
||||||
|
|
@ -190,10 +256,10 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void dispose() {
|
void dispose() {
|
||||||
|
_stopCapture();
|
||||||
_sigSub?.cancel();
|
_sigSub?.cancel();
|
||||||
_sig.dispose();
|
_sig.dispose();
|
||||||
_pc?.close();
|
_pc?.close();
|
||||||
_localStream?.dispose();
|
|
||||||
super.dispose();
|
super.dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -204,37 +270,100 @@ class _HostScreenState extends State<HostScreen> {
|
||||||
title: const Text('Pulsar — Host'),
|
title: const Text('Pulsar — Host'),
|
||||||
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
||||||
),
|
),
|
||||||
body: Center(
|
floatingActionButton: FloatingActionButton(
|
||||||
child: Column(
|
mini: true,
|
||||||
mainAxisAlignment: MainAxisAlignment.center,
|
tooltip: 'Stats',
|
||||||
children: [
|
onPressed: () => setState(() => _showStats = !_showStats),
|
||||||
const Icon(Icons.monitor, size: 80, color: Colors.indigo),
|
child: const Icon(Icons.bar_chart),
|
||||||
const SizedBox(height: 24),
|
),
|
||||||
Text('Room Code', style: Theme.of(context).textTheme.titleMedium),
|
body: Stack(
|
||||||
const SizedBox(height: 8),
|
children: [
|
||||||
SelectableText(
|
Center(
|
||||||
_roomId,
|
child: Column(
|
||||||
style: Theme.of(context).textTheme.displaySmall?.copyWith(
|
mainAxisAlignment: MainAxisAlignment.center,
|
||||||
fontFamily: 'monospace',
|
|
||||||
fontWeight: FontWeight.bold,
|
|
||||||
letterSpacing: 8,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
const SizedBox(height: 32),
|
|
||||||
Row(
|
|
||||||
mainAxisSize: MainAxisSize.min,
|
|
||||||
children: [
|
children: [
|
||||||
Icon(
|
const Icon(Icons.monitor, size: 80, color: Colors.indigo),
|
||||||
_connected ? Icons.circle : Icons.circle_outlined,
|
const SizedBox(height: 24),
|
||||||
color: _connected ? Colors.green : Colors.orange,
|
Text('Room Code', style: Theme.of(context).textTheme.titleMedium),
|
||||||
size: 14,
|
const SizedBox(height: 8),
|
||||||
|
SelectableText(
|
||||||
|
_roomId,
|
||||||
|
style: Theme.of(context).textTheme.displaySmall?.copyWith(
|
||||||
|
fontFamily: 'monospace',
|
||||||
|
fontWeight: FontWeight.bold,
|
||||||
|
letterSpacing: 8,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
const SizedBox(height: 32),
|
||||||
|
Row(
|
||||||
|
mainAxisSize: MainAxisSize.min,
|
||||||
|
children: [
|
||||||
|
Icon(
|
||||||
|
_connected ? Icons.circle : Icons.circle_outlined,
|
||||||
|
color: _connected ? Colors.green : Colors.orange,
|
||||||
|
size: 14,
|
||||||
|
),
|
||||||
|
const SizedBox(width: 8),
|
||||||
|
Text(_status),
|
||||||
|
],
|
||||||
),
|
),
|
||||||
const SizedBox(width: 8),
|
|
||||||
Text(_status),
|
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
],
|
),
|
||||||
),
|
|
||||||
|
if (_showStats)
|
||||||
|
Positioned(
|
||||||
|
top: 12,
|
||||||
|
left: 12,
|
||||||
|
child: _StatsOverlay(lines: [
|
||||||
|
'Host:',
|
||||||
|
' FPS: $_statFps',
|
||||||
|
' Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
|
||||||
|
' Dropped: $_statDropped/s',
|
||||||
|
' Buffer: $_statBufferKb KB',
|
||||||
|
' Send: ${_statSendMs.toStringAsFixed(1)}ms',
|
||||||
|
'',
|
||||||
|
if (_viewerStats == null)
|
||||||
|
'Viewer: no data'
|
||||||
|
else ...[
|
||||||
|
'Viewer:',
|
||||||
|
' FPS: ${_viewerStats!['fps']}',
|
||||||
|
' Bitrate: ${(_viewerStats!['bitrateMbps'] as num).toStringAsFixed(2)} Mbps',
|
||||||
|
' Dropped: ${_viewerStats!['dropped']}/s',
|
||||||
|
' Res: ${_viewerStats!['resolution']}',
|
||||||
|
' Latency: ${_viewerStats!['latencyMs']}ms',
|
||||||
|
' Submit: ${_viewerStats!['decodeMs'] ?? '—'}ms',
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class _StatsOverlay extends StatelessWidget {
|
||||||
|
final List<String> lines;
|
||||||
|
const _StatsOverlay({required this.lines});
|
||||||
|
|
||||||
|
@override
|
||||||
|
Widget build(BuildContext context) {
|
||||||
|
return Container(
|
||||||
|
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
|
||||||
|
decoration: BoxDecoration(
|
||||||
|
color: Colors.black.withValues(alpha: 0.6),
|
||||||
|
borderRadius: BorderRadius.circular(6),
|
||||||
|
),
|
||||||
|
child: Column(
|
||||||
|
crossAxisAlignment: CrossAxisAlignment.start,
|
||||||
|
children: lines.map((l) => Text(
|
||||||
|
l,
|
||||||
|
style: const TextStyle(
|
||||||
|
color: Colors.white,
|
||||||
|
fontFamily: 'monospace',
|
||||||
|
fontSize: 12,
|
||||||
|
),
|
||||||
|
)).toList(),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import 'dart:async';
|
import 'dart:async';
|
||||||
import 'dart:convert';
|
import 'dart:convert';
|
||||||
|
import 'dart:typed_data';
|
||||||
|
|
||||||
import 'package:flutter/material.dart';
|
import 'package:flutter/material.dart';
|
||||||
import 'package:flutter/services.dart';
|
import 'package:flutter/services.dart';
|
||||||
|
|
@ -16,10 +17,11 @@ class ViewerScreen extends StatefulWidget {
|
||||||
}
|
}
|
||||||
|
|
||||||
class _ViewerScreenState extends State<ViewerScreen> {
|
class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
|
static const _videoCh = MethodChannel("com.pulsar/video");
|
||||||
|
|
||||||
final _sig = SignallingService();
|
final _sig = SignallingService();
|
||||||
RTCPeerConnection? _pc;
|
RTCPeerConnection? _pc;
|
||||||
RTCDataChannel? _dataChannel;
|
RTCDataChannel? _dataChannel;
|
||||||
final _remoteRenderer = RTCVideoRenderer();
|
|
||||||
|
|
||||||
final _roomCodeCtrl = TextEditingController();
|
final _roomCodeCtrl = TextEditingController();
|
||||||
String _status = 'Enter a room code to connect';
|
String _status = 'Enter a room code to connect';
|
||||||
|
|
@ -28,23 +30,46 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
|
|
||||||
StreamSubscription? _sigSub;
|
StreamSubscription? _sigSub;
|
||||||
|
|
||||||
// size of the rendered video widget, used to normalise pointer coords
|
int? _textureId;
|
||||||
Size _videoSize = Size.zero;
|
bool _decoderReady = false;
|
||||||
final _videoKey = GlobalKey();
|
bool _settingUpDecoder = false;
|
||||||
|
|
||||||
@override
|
// frames that arrive while createDecoder is in-flight are queued here and
|
||||||
void initState() {
|
// drained once the decoder is ready, so we don't miss the initial keyframe
|
||||||
super.initState();
|
final _pendingNals = <(Uint8List, int)>[];
|
||||||
_remoteRenderer.initialize();
|
|
||||||
}
|
// widget key for normalising input coords
|
||||||
|
final _videoKey = GlobalKey();
|
||||||
|
Size _videoSize = Size.zero;
|
||||||
|
|
||||||
|
// stats
|
||||||
|
bool _showStats = false;
|
||||||
|
int _statFps = 0;
|
||||||
|
double _statBitrateMbps = 0;
|
||||||
|
int _statDropped = 0;
|
||||||
|
String _statResolution = '—';
|
||||||
|
int _statLatencyMs = 0;
|
||||||
|
double _statDecodeMs = 0; // actually submit time for feedNal
|
||||||
|
|
||||||
|
int _frameCount = 0;
|
||||||
|
int _byteCount = 0;
|
||||||
|
int _droppedCount = 0;
|
||||||
|
double _decodeMsSum = 0;
|
||||||
|
int _decodeMsCount = 0;
|
||||||
|
Timer? _statsTimer;
|
||||||
|
|
||||||
@override
|
@override
|
||||||
void dispose() {
|
void dispose() {
|
||||||
|
_statsTimer?.cancel();
|
||||||
_sigSub?.cancel();
|
_sigSub?.cancel();
|
||||||
_sig.dispose();
|
_sig.dispose();
|
||||||
_pc?.close();
|
_pc?.close();
|
||||||
_remoteRenderer.dispose();
|
|
||||||
_roomCodeCtrl.dispose();
|
_roomCodeCtrl.dispose();
|
||||||
|
|
||||||
|
if (_textureId != null) {
|
||||||
|
_videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
|
||||||
|
}
|
||||||
|
|
||||||
super.dispose();
|
super.dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -92,6 +117,7 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case 'host_disconnected':
|
case 'host_disconnected':
|
||||||
|
_statsTimer?.cancel();
|
||||||
setState(() {
|
setState(() {
|
||||||
_status = 'Host disconnected';
|
_status = 'Host disconnected';
|
||||||
_streaming = false;
|
_streaming = false;
|
||||||
|
|
@ -109,9 +135,9 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
|
|
||||||
Future<void> _initPeerConnection(String roomId) async {
|
Future<void> _initPeerConnection(String roomId) async {
|
||||||
final config = {
|
final config = {
|
||||||
'iceServers': [
|
'iceServers': [{'urls': 'stun:stun.l.google.com:19302'}],
|
||||||
{'urls': 'stun:stun.l.google.com:19302'},
|
'sdpSemantics': 'unified-plan',
|
||||||
],
|
'encodedInsertableStreams': false,
|
||||||
};
|
};
|
||||||
|
|
||||||
_pc = await createPeerConnection(config);
|
_pc = await createPeerConnection(config);
|
||||||
|
|
@ -120,18 +146,17 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
_sig.send({
|
_sig.send({
|
||||||
'type': 'signal',
|
'type': 'signal',
|
||||||
'roomId': roomId,
|
'roomId': roomId,
|
||||||
'data': {
|
'data': {'type': 'candidate', 'candidate': candidate.toMap()},
|
||||||
'type': 'candidate',
|
|
||||||
'candidate': candidate.toMap(),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
_pc!.onConnectionState = (state) {
|
_pc!.onConnectionState = (state) {
|
||||||
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
||||||
setState(() => _status = 'Connected');
|
setState(() => _status = 'Connected');
|
||||||
|
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
|
||||||
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
||||||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
||||||
|
_statsTimer?.cancel();
|
||||||
setState(() {
|
setState(() {
|
||||||
_status = 'Connection lost';
|
_status = 'Connection lost';
|
||||||
_streaming = false;
|
_streaming = false;
|
||||||
|
|
@ -139,21 +164,133 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
_pc!.onTrack = (event) {
|
|
||||||
if (event.track.kind == 'video') {
|
|
||||||
setState(() {
|
|
||||||
_remoteRenderer.srcObject = event.streams[0];
|
|
||||||
_streaming = true;
|
|
||||||
_status = 'Streaming';
|
|
||||||
});
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
_pc!.onDataChannel = (channel) {
|
_pc!.onDataChannel = (channel) {
|
||||||
_dataChannel = channel;
|
_dataChannel = channel;
|
||||||
|
|
||||||
|
channel.onMessage = (msg) {
|
||||||
|
if (msg.isBinary) _onData(msg.binary);
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void _flushStats() {
|
||||||
|
setState(() {
|
||||||
|
_statFps = _frameCount;
|
||||||
|
_statBitrateMbps = _byteCount * 8 / 1e6;
|
||||||
|
_statDropped = _droppedCount;
|
||||||
|
_statDecodeMs = _decodeMsCount > 0 ? _decodeMsSum / _decodeMsCount : 0;
|
||||||
|
_frameCount = 0;
|
||||||
|
_byteCount = 0;
|
||||||
|
_droppedCount = 0;
|
||||||
|
_decodeMsSum = 0; _decodeMsCount = 0;
|
||||||
|
});
|
||||||
|
_sendInput({
|
||||||
|
'type': 'stats',
|
||||||
|
'fps': _statFps,
|
||||||
|
'bitrateMbps': _statBitrateMbps,
|
||||||
|
'dropped': _statDropped,
|
||||||
|
'resolution': _statResolution,
|
||||||
|
'latencyMs': _statLatencyMs,
|
||||||
|
'decodeMs': double.parse(_statDecodeMs.toStringAsFixed(1)),
|
||||||
|
'paintMs': 0.0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
void _onData(Uint8List bytes) {
|
||||||
|
if (bytes.isEmpty) return;
|
||||||
|
|
||||||
|
_byteCount += bytes.length;
|
||||||
|
|
||||||
|
final type = bytes[0];
|
||||||
|
|
||||||
|
if (type == 0x01) {
|
||||||
|
// config packet: SPS+PPS — create/reconfigure the decoder
|
||||||
|
final spsPps = Uint8List.sublistView(bytes, 1);
|
||||||
|
_setupDecoder(spsPps);
|
||||||
|
|
||||||
|
} else if (type == 0x02) {
|
||||||
|
// frame packet: [0x02][8-byte ts][Annex B NALU]
|
||||||
|
if (bytes.length < 10) return;
|
||||||
|
|
||||||
|
final bd = ByteData.sublistView(bytes, 1, 9);
|
||||||
|
final ts = bd.getInt64(0, Endian.big);
|
||||||
|
final latency = DateTime.now().millisecondsSinceEpoch - ts;
|
||||||
|
|
||||||
|
final nal = Uint8List.sublistView(bytes, 9);
|
||||||
|
|
||||||
|
// drop frames that are already stale — prevents burst playback after a lag spike
|
||||||
|
if (latency > 300) return;
|
||||||
|
|
||||||
|
if (!_decoderReady) {
|
||||||
|
_pendingNals.add((nal, latency));
|
||||||
|
} else {
|
||||||
|
_feedNal(nal, latency);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<void> _setupDecoder(Uint8List spsPps) async {
|
||||||
|
// a second 0x01 packet can arrive while createDecoder is still awaiting;
|
||||||
|
// ignore it — the first decoder will be ready soon enough
|
||||||
|
if (_settingUpDecoder) return;
|
||||||
|
_settingUpDecoder = true;
|
||||||
|
|
||||||
|
_decoderReady = false;
|
||||||
|
_pendingNals.clear();
|
||||||
|
|
||||||
|
if (_textureId != null) {
|
||||||
|
await _videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
|
||||||
|
_textureId = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
final res = await _videoCh.invokeMethod<Map>('createDecoder', {'spsPps': spsPps});
|
||||||
|
final id = (res?['textureId'] as num?)?.toInt();
|
||||||
|
|
||||||
|
if (!mounted) return;
|
||||||
|
setState(() {
|
||||||
|
_textureId = id;
|
||||||
|
if (!_streaming) _streaming = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
_decoderReady = true;
|
||||||
|
|
||||||
|
// drain frames that queued up while createDecoder was in-flight
|
||||||
|
final pending = List.of(_pendingNals);
|
||||||
|
_pendingNals.clear();
|
||||||
|
for (final (nal, latency) in pending) {
|
||||||
|
_feedNal(nal, latency);
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
_decoderReady = false;
|
||||||
|
} finally {
|
||||||
|
_settingUpDecoder = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<void> _feedNal(Uint8List nal, int latency) async {
|
||||||
|
if (_textureId == null) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
final t0 = DateTime.now();
|
||||||
|
await _videoCh.invokeMethod('feedNal', {
|
||||||
|
'textureId': _textureId,
|
||||||
|
'nal': nal,
|
||||||
|
});
|
||||||
|
final elapsed = DateTime.now().difference(t0).inMicroseconds / 1000.0;
|
||||||
|
|
||||||
|
if (!mounted) return;
|
||||||
|
|
||||||
|
_decodeMsSum += elapsed;
|
||||||
|
_decodeMsCount++;
|
||||||
|
_frameCount++;
|
||||||
|
|
||||||
|
setState(() {
|
||||||
|
_statLatencyMs = latency.clamp(0, 9999);
|
||||||
|
});
|
||||||
|
} catch (_) {}
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> _handleSignalData(Map<String, dynamic> data, String roomId) async {
|
Future<void> _handleSignalData(Map<String, dynamic> data, String roomId) async {
|
||||||
final sigType = data['type'] as String?;
|
final sigType = data['type'] as String?;
|
||||||
|
|
||||||
|
|
@ -169,7 +306,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
'roomId': roomId,
|
'roomId': roomId,
|
||||||
'data': {'type': 'answer', 'sdp': answer.sdp},
|
'data': {'type': 'answer', 'sdp': answer.sdp},
|
||||||
});
|
});
|
||||||
|
|
||||||
} else if (sigType == 'candidate') {
|
} else if (sigType == 'candidate') {
|
||||||
final raw = data['candidate'] as Map<String, dynamic>;
|
final raw = data['candidate'] as Map<String, dynamic>;
|
||||||
final candidate = RTCIceCandidate(
|
final candidate = RTCIceCandidate(
|
||||||
|
|
@ -188,7 +324,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
} catch (_) {}
|
} catch (_) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
// get video widget size via the global key
|
|
||||||
void _updateVideoSize() {
|
void _updateVideoSize() {
|
||||||
final ctx = _videoKey.currentContext;
|
final ctx = _videoKey.currentContext;
|
||||||
if (ctx != null) {
|
if (ctx != null) {
|
||||||
|
|
@ -206,8 +341,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- input capture
|
|
||||||
|
|
||||||
void _onPointerMove(PointerMoveEvent e) {
|
void _onPointerMove(PointerMoveEvent e) {
|
||||||
final n = _normalise(e.localPosition);
|
final n = _normalise(e.localPosition);
|
||||||
_sendInput({'type': 'move', 'x': n.dx, 'y': n.dy});
|
_sendInput({'type': 'move', 'x': n.dx, 'y': n.dy});
|
||||||
|
|
@ -215,7 +348,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
|
|
||||||
void _onPointerDown(PointerDownEvent e) {
|
void _onPointerDown(PointerDownEvent e) {
|
||||||
final n = _normalise(e.localPosition);
|
final n = _normalise(e.localPosition);
|
||||||
// button 0 = left, 1 = right (PointerDeviceKind bit not directly available)
|
|
||||||
_sendInput({'type': 'click', 'x': n.dx, 'y': n.dy, 'button': 0});
|
_sendInput({'type': 'click', 'x': n.dx, 'y': n.dy, 'button': 0});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -234,6 +366,14 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
title: const Text('Pulsar — Viewer'),
|
title: const Text('Pulsar — Viewer'),
|
||||||
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
||||||
),
|
),
|
||||||
|
floatingActionButton: _streaming
|
||||||
|
? FloatingActionButton(
|
||||||
|
mini: true,
|
||||||
|
tooltip: 'Stats',
|
||||||
|
onPressed: () => setState(() => _showStats = !_showStats),
|
||||||
|
child: const Icon(Icons.bar_chart),
|
||||||
|
)
|
||||||
|
: null,
|
||||||
body: _streaming ? _buildStream() : _buildJoinView(),
|
body: _streaming ? _buildStream() : _buildJoinView(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -296,20 +436,66 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
||||||
}
|
}
|
||||||
|
|
||||||
Widget _buildStream() {
|
Widget _buildStream() {
|
||||||
return Focus(
|
return Stack(
|
||||||
autofocus: true,
|
children: [
|
||||||
onKeyEvent: (_, e) {
|
Focus(
|
||||||
_onKeyEvent(e);
|
autofocus: true,
|
||||||
return KeyEventResult.handled;
|
onKeyEvent: (_, e) {
|
||||||
},
|
_onKeyEvent(e);
|
||||||
child: Listener(
|
return KeyEventResult.handled;
|
||||||
onPointerMove: _onPointerMove,
|
},
|
||||||
onPointerDown: _onPointerDown,
|
child: Listener(
|
||||||
child: RTCVideoView(
|
onPointerMove: _onPointerMove,
|
||||||
_remoteRenderer,
|
onPointerDown: _onPointerDown,
|
||||||
key: _videoKey,
|
child: _textureId != null
|
||||||
objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
|
? SizedBox.expand(
|
||||||
|
key: _videoKey,
|
||||||
|
child: Texture(textureId: _textureId!),
|
||||||
|
)
|
||||||
|
: const Center(child: CircularProgressIndicator()),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
|
|
||||||
|
if (_showStats)
|
||||||
|
Positioned(
|
||||||
|
top: 12,
|
||||||
|
left: 12,
|
||||||
|
child: _StatsOverlay(lines: [
|
||||||
|
'FPS: $_statFps',
|
||||||
|
'Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
|
||||||
|
'Dropped: $_statDropped/s',
|
||||||
|
'Res: $_statResolution',
|
||||||
|
'Latency: ${_statLatencyMs}ms',
|
||||||
|
'Submit: ${_statDecodeMs.toStringAsFixed(1)}ms',
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class _StatsOverlay extends StatelessWidget {
|
||||||
|
final List<String> lines;
|
||||||
|
const _StatsOverlay({required this.lines});
|
||||||
|
|
||||||
|
@override
|
||||||
|
Widget build(BuildContext context) {
|
||||||
|
return Container(
|
||||||
|
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
|
||||||
|
decoration: BoxDecoration(
|
||||||
|
color: Colors.black.withValues(alpha: 0.6),
|
||||||
|
borderRadius: BorderRadius.circular(6),
|
||||||
|
),
|
||||||
|
child: Column(
|
||||||
|
crossAxisAlignment: CrossAxisAlignment.start,
|
||||||
|
children: lines.map((l) => Text(
|
||||||
|
l,
|
||||||
|
style: const TextStyle(
|
||||||
|
color: Colors.white,
|
||||||
|
fontFamily: 'monospace',
|
||||||
|
fontSize: 12,
|
||||||
|
),
|
||||||
|
)).toList(),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -20,10 +20,10 @@ class SignallingService {
|
||||||
} catch (_) {}
|
} catch (_) {}
|
||||||
},
|
},
|
||||||
onError: (err) {
|
onError: (err) {
|
||||||
_controller.addError(err);
|
if (!_controller.isClosed) _controller.addError(err);
|
||||||
},
|
},
|
||||||
onDone: () {
|
onDone: () {
|
||||||
_controller.addError(Exception('WebSocket connection closed'));
|
if (!_controller.isClosed) _controller.addError(Exception('WebSocket connection closed'));
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
PODS:
|
PODS:
|
||||||
- flutter_webrtc (0.9.36):
|
- flutter_webrtc (1.4.0):
|
||||||
- FlutterMacOS
|
- FlutterMacOS
|
||||||
- WebRTC-SDK (= 114.5735.08)
|
- WebRTC-SDK (= 144.7559.01)
|
||||||
- FlutterMacOS (1.0.0)
|
- FlutterMacOS (1.0.0)
|
||||||
- WebRTC-SDK (114.5735.08)
|
- WebRTC-SDK (144.7559.01)
|
||||||
|
|
||||||
DEPENDENCIES:
|
DEPENDENCIES:
|
||||||
- flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`)
|
- flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`)
|
||||||
|
|
@ -20,9 +20,9 @@ EXTERNAL SOURCES:
|
||||||
:path: Flutter/ephemeral
|
:path: Flutter/ephemeral
|
||||||
|
|
||||||
SPEC CHECKSUMS:
|
SPEC CHECKSUMS:
|
||||||
flutter_webrtc: cf7dc44d26cbb5c5f1ae5f583dab545871f287f9
|
flutter_webrtc: dbacf72a9bd951ccfa5997198f0a6214bb848ede
|
||||||
FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1
|
FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1
|
||||||
WebRTC-SDK: c24d2a6c9f571f2ed42297cb8ffba9557093142b
|
WebRTC-SDK: ab9b5319e458c2bfebdc92b3600740da35d5630d
|
||||||
|
|
||||||
PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009
|
PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,20 +1,37 @@
|
||||||
import Cocoa
|
import Cocoa
|
||||||
import FlutterMacOS
|
import FlutterMacOS
|
||||||
import CoreGraphics
|
import CoreGraphics
|
||||||
|
import ScreenCaptureKit
|
||||||
|
import CoreMedia
|
||||||
|
import CoreVideo
|
||||||
|
import VideoToolbox
|
||||||
|
|
||||||
@main
|
@main
|
||||||
class AppDelegate: FlutterAppDelegate {
|
class AppDelegate: FlutterAppDelegate {
|
||||||
private var inputChannel: FlutterMethodChannel?
|
private var inputChannel: FlutterMethodChannel?
|
||||||
|
private var videoChannel: FlutterMethodChannel?
|
||||||
|
private var frameEventSink: FlutterEventSink?
|
||||||
|
|
||||||
|
// SCK objects (typed Any? to satisfy 10.15 deployment target)
|
||||||
|
private var scStream: Any?
|
||||||
|
private var screenDelegate: Any?
|
||||||
|
private var streamReady = false
|
||||||
|
|
||||||
|
// VTCompressionSession (Any? for same reason)
|
||||||
|
private var vtSession: Any?
|
||||||
|
private var encoderReady = false
|
||||||
|
|
||||||
|
// resolution last seen from SCK (to build config)
|
||||||
|
private var captureWidth = 1280
|
||||||
|
private var captureHeight = 720
|
||||||
|
|
||||||
override func applicationDidFinishLaunching(_ notification: Notification) {
|
override func applicationDidFinishLaunching(_ notification: Notification) {
|
||||||
guard let window = mainFlutterWindow,
|
guard let window = mainFlutterWindow,
|
||||||
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
||||||
|
|
||||||
inputChannel = FlutterMethodChannel(
|
let messenger = ctrl.engine.binaryMessenger
|
||||||
name: "com.pulsar/input",
|
|
||||||
binaryMessenger: ctrl.engine.binaryMessenger
|
|
||||||
)
|
|
||||||
|
|
||||||
|
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
||||||
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
||||||
if call.method == "injectInput",
|
if call.method == "injectInput",
|
||||||
let args = call.arguments as? [String: Any] {
|
let args = call.arguments as? [String: Any] {
|
||||||
|
|
@ -24,8 +41,302 @@ class AppDelegate: FlutterAppDelegate {
|
||||||
result(FlutterMethodNotImplemented)
|
result(FlutterMethodNotImplemented)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// event channel for pushing encoded NAL units to Flutter
|
||||||
|
let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
|
||||||
|
frameChannel.setStreamHandler(self)
|
||||||
|
|
||||||
|
videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
|
||||||
|
videoChannel?.setMethodCallHandler { [weak self] call, result in
|
||||||
|
guard let self = self else { return }
|
||||||
|
switch call.method {
|
||||||
|
case "start":
|
||||||
|
self.startEncoder(result: result)
|
||||||
|
case "stop":
|
||||||
|
self.stopEncoder()
|
||||||
|
result(nil)
|
||||||
|
case "forceKeyframe":
|
||||||
|
self.requestKeyframe()
|
||||||
|
result(nil)
|
||||||
|
default:
|
||||||
|
result(FlutterMethodNotImplemented)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if #available(macOS 12.3, *) {
|
||||||
|
Task { await self.ensureSCKStream() }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: - SCK stream setup
|
||||||
|
|
||||||
|
@available(macOS 12.3, *)
|
||||||
|
private func ensureSCKStream() async {
|
||||||
|
guard scStream == nil else { return }
|
||||||
|
|
||||||
|
guard let content = try? await SCShareableContent.current else { return }
|
||||||
|
guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
|
||||||
|
|
||||||
|
let filter = SCContentFilter(display: display, excludingWindows: [])
|
||||||
|
|
||||||
|
let srcW = Double(display.width)
|
||||||
|
let srcH = Double(display.height)
|
||||||
|
let scale = min(1.0, 1280.0 / srcW)
|
||||||
|
let dstW = Int(srcW * scale)
|
||||||
|
let dstH = Int(srcH * scale)
|
||||||
|
|
||||||
|
captureWidth = dstW
|
||||||
|
captureHeight = dstH
|
||||||
|
|
||||||
|
let cfg = SCStreamConfiguration()
|
||||||
|
cfg.width = dstW
|
||||||
|
cfg.height = dstH
|
||||||
|
cfg.pixelFormat = kCVPixelFormatType_32BGRA
|
||||||
|
cfg.showsCursor = true
|
||||||
|
cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
|
||||||
|
|
||||||
|
let del = ScreenDelegate { [weak self] sampleBuffer in
|
||||||
|
self?.encodeFrame(sampleBuffer)
|
||||||
|
}
|
||||||
|
screenDelegate = del
|
||||||
|
|
||||||
|
let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
|
||||||
|
try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
|
||||||
|
|
||||||
|
do {
|
||||||
|
try await stream.startCapture()
|
||||||
|
scStream = stream
|
||||||
|
streamReady = true
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - VTCompressionSession
|
||||||
|
|
||||||
|
private func startEncoder(result: @escaping FlutterResult) {
|
||||||
|
let w = captureWidth
|
||||||
|
let h = captureHeight
|
||||||
|
|
||||||
|
var session: VTCompressionSession?
|
||||||
|
let status = VTCompressionSessionCreate(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
|
width: Int32(w),
|
||||||
|
height: Int32(h),
|
||||||
|
codecType: kCMVideoCodecType_H264,
|
||||||
|
encoderSpecification: nil,
|
||||||
|
imageBufferAttributes: nil,
|
||||||
|
compressedDataAllocator: nil,
|
||||||
|
outputCallback: vtOutputCallback,
|
||||||
|
refcon: Unmanaged.passUnretained(self).toOpaque(),
|
||||||
|
compressionSessionOut: &session
|
||||||
|
)
|
||||||
|
|
||||||
|
guard status == noErr, let session = session else {
|
||||||
|
result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
|
||||||
|
value: kVTProfileLevel_H264_Baseline_AutoLevel)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
|
||||||
|
value: 6_000_000 as CFNumber)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
|
||||||
|
value: 60 as CFNumber)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
|
||||||
|
value: 120 as CFNumber)
|
||||||
|
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
|
||||||
|
value: 0 as CFNumber)
|
||||||
|
|
||||||
|
VTCompressionSessionPrepareToEncodeFrames(session)
|
||||||
|
vtSession = VTSessionBox(session)
|
||||||
|
encoderReady = true
|
||||||
|
|
||||||
|
// extract SPS+PPS from a forced keyframe by encoding a dummy frame
|
||||||
|
// instead, return empty bytes and let the real SPS+PPS come through the callback
|
||||||
|
// host_screen will call forceKeyframe right after start
|
||||||
|
result(FlutterStandardTypedData(bytes: Data()))
|
||||||
|
}
|
||||||
|
|
||||||
|
private func stopEncoder() {
|
||||||
|
guard let box = vtSession as? VTSessionBox else { return }
|
||||||
|
VTCompressionSessionInvalidate(box.session)
|
||||||
|
vtSession = nil
|
||||||
|
encoderReady = false
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestKeyframe() {
|
||||||
|
forceKeyframeNext = true
|
||||||
|
}
|
||||||
|
|
||||||
|
private var forceKeyframeNext = false
|
||||||
|
|
||||||
|
private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
|
||||||
|
guard encoderReady, let box = vtSession as? VTSessionBox else { return }
|
||||||
|
let s = box.session
|
||||||
|
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||||
|
|
||||||
|
var frameProps: CFDictionary? = nil
|
||||||
|
if forceKeyframeNext {
|
||||||
|
frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
|
||||||
|
forceKeyframeNext = false
|
||||||
|
}
|
||||||
|
|
||||||
|
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
||||||
|
VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
|
||||||
|
duration: .invalid, frameProperties: frameProps,
|
||||||
|
sourceFrameRefcon: nil, infoFlagsOut: nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - VT output callback (C function)
|
||||||
|
|
||||||
|
// called by VideoToolbox on an internal thread — collect Data packets here,
|
||||||
|
// then hop to main before touching the event sink
|
||||||
|
func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
|
||||||
|
var packets: [Data] = []
|
||||||
|
|
||||||
|
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
|
||||||
|
let isKeyframe = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
|
||||||
|
|
||||||
|
if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
|
||||||
|
if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
|
||||||
|
}
|
||||||
|
|
||||||
|
if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
|
||||||
|
var totalLen = 0
|
||||||
|
var rawPtr: UnsafeMutablePointer<CChar>? = nil
|
||||||
|
if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
|
||||||
|
totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
|
||||||
|
let ptr = rawPtr {
|
||||||
|
|
||||||
|
let ts = Int64(Date().timeIntervalSince1970 * 1000)
|
||||||
|
var offset = 0
|
||||||
|
|
||||||
|
while offset + 4 <= totalLen {
|
||||||
|
// read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
|
||||||
|
let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
|
||||||
|
let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
|
||||||
|
let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
|
||||||
|
let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
|
||||||
|
let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
|
||||||
|
offset += 4
|
||||||
|
|
||||||
|
guard naluLen > 0, offset + naluLen <= totalLen else { break }
|
||||||
|
|
||||||
|
var packet = Data(capacity: 1 + 8 + 4 + naluLen)
|
||||||
|
packet.append(0x02)
|
||||||
|
withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
|
||||||
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||||
|
packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
|
||||||
|
packets.append(packet)
|
||||||
|
|
||||||
|
offset += naluLen
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
guard !packets.isEmpty else { return }
|
||||||
|
DispatchQueue.main.async { [weak self] in
|
||||||
|
guard let sink = self?.frameEventSink else { return }
|
||||||
|
for packet in packets {
|
||||||
|
sink(FlutterStandardTypedData(bytes: packet))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
|
||||||
|
var spsCount = 0
|
||||||
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
||||||
|
parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
|
||||||
|
nalUnitHeaderLengthOut: nil)
|
||||||
|
var packet = Data()
|
||||||
|
packet.append(0x01)
|
||||||
|
|
||||||
|
for i in 0 ..< spsCount {
|
||||||
|
var ptr: UnsafePointer<UInt8>? = nil
|
||||||
|
var len = 0
|
||||||
|
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
|
||||||
|
parameterSetPointerOut: &ptr,
|
||||||
|
parameterSetSizeOut: &len,
|
||||||
|
parameterSetCountOut: nil,
|
||||||
|
nalUnitHeaderLengthOut: nil)
|
||||||
|
if let ptr = ptr {
|
||||||
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||||
|
packet.append(Data(bytes: ptr, count: len))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PPS sits at index spsCount
|
||||||
|
var ppsPtr: UnsafePointer<UInt8>? = nil
|
||||||
|
var ppsLen = 0
|
||||||
|
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
|
||||||
|
parameterSetPointerOut: &ppsPtr,
|
||||||
|
parameterSetSizeOut: &ppsLen,
|
||||||
|
parameterSetCountOut: nil,
|
||||||
|
nalUnitHeaderLengthOut: nil) == noErr,
|
||||||
|
let ppsPtr = ppsPtr {
|
||||||
|
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||||
|
packet.append(Data(bytes: ppsPtr, count: ppsLen))
|
||||||
|
}
|
||||||
|
|
||||||
|
return packet.count > 1 ? packet : nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths — unused in normal operation)
|
||||||
|
|
||||||
|
private func captureCG() -> Data? {
|
||||||
|
let displayID = CGMainDisplayID()
|
||||||
|
guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
|
||||||
|
let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
|
||||||
|
let srcW = CGFloat(composited.width)
|
||||||
|
let scale = min(1.0, 1280.0 / srcW)
|
||||||
|
let dstW = Int(srcW * scale)
|
||||||
|
let dstH = Int(CGFloat(composited.height) * scale)
|
||||||
|
let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
||||||
|
guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
|
||||||
|
bytesPerRow: 0, space: colorSpace,
|
||||||
|
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
|
||||||
|
ctx.interpolationQuality = .high
|
||||||
|
ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
|
||||||
|
guard let scaled = ctx.makeImage() else { return nil }
|
||||||
|
return jpegData(from: scaled, quality: 0.4)
|
||||||
|
}
|
||||||
|
|
||||||
|
private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
|
||||||
|
let nsImage = NSImage(cgImage: cgImage, size: .zero)
|
||||||
|
guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
|
||||||
|
return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
|
||||||
|
}
|
||||||
|
|
||||||
|
private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
|
||||||
|
let imgW = cgImage.width
|
||||||
|
let imgH = cgImage.height
|
||||||
|
let mousePt = NSEvent.mouseLocation
|
||||||
|
let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
|
||||||
|
let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
|
||||||
|
let cursorX = mousePt.x * scale
|
||||||
|
let cursorY = (screenH - mousePt.y) * scale
|
||||||
|
let cursor = NSCursor.current
|
||||||
|
let cursorImg = cursor.image
|
||||||
|
let hotspot = cursor.hotSpot
|
||||||
|
let cw = cursorImg.size.width * scale
|
||||||
|
let ch = cursorImg.size.height * scale
|
||||||
|
let drawX = cursorX - hotspot.x * scale
|
||||||
|
let drawY = cursorY - hotspot.y * scale
|
||||||
|
let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
||||||
|
guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
|
||||||
|
bytesPerRow: 0, space: colorSpace,
|
||||||
|
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
|
||||||
|
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
|
||||||
|
var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
|
||||||
|
if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
|
||||||
|
ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
|
||||||
|
}
|
||||||
|
return ctx.makeImage()
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - Input injection
|
||||||
|
|
||||||
private func injectInput(_ ev: [String: Any]) {
|
private func injectInput(_ ev: [String: Any]) {
|
||||||
guard let type = ev["type"] as? String else { return }
|
guard let type = ev["type"] as? String else { return }
|
||||||
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
|
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
|
||||||
|
|
@ -34,24 +345,17 @@ class AppDelegate: FlutterAppDelegate {
|
||||||
case "move":
|
case "move":
|
||||||
let nx = ev["x"] as? Double ?? 0
|
let nx = ev["x"] as? Double ?? 0
|
||||||
let ny = ev["y"] as? Double ?? 0
|
let ny = ev["y"] as? Double ?? 0
|
||||||
// CoreGraphics origin is top-left on screen, NSScreen origin is bottom-left
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
||||||
let px = CGFloat(nx) * screen.width
|
|
||||||
let py = CGFloat(ny) * screen.height
|
|
||||||
let pt = CGPoint(x: px, y: py)
|
|
||||||
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
|
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
|
||||||
.post(tap: .cghidEventTap)
|
.post(tap: .cghidEventTap)
|
||||||
|
|
||||||
case "click":
|
case "click":
|
||||||
let nx = ev["x"] as? Double ?? 0
|
let nx = ev["x"] as? Double ?? 0
|
||||||
let ny = ev["y"] as? Double ?? 0
|
let ny = ev["y"] as? Double ?? 0
|
||||||
let btn = ev["button"] as? Int ?? 0
|
let btn = ev["button"] as? Int ?? 0
|
||||||
let px = CGFloat(nx) * screen.width
|
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
||||||
let py = CGFloat(ny) * screen.height
|
|
||||||
let pt = CGPoint(x: px, y: py)
|
|
||||||
|
|
||||||
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
|
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
|
||||||
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
|
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
|
||||||
|
|
||||||
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
||||||
.post(tap: .cghidEventTap)
|
.post(tap: .cghidEventTap)
|
||||||
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
||||||
|
|
@ -69,16 +373,71 @@ class AppDelegate: FlutterAppDelegate {
|
||||||
.post(tap: .cghidEventTap)
|
.post(tap: .cghidEventTap)
|
||||||
}
|
}
|
||||||
|
|
||||||
default:
|
default: break
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
|
// MARK: - App lifecycle
|
||||||
return true
|
|
||||||
|
override func applicationWillTerminate(_ notification: Notification) {
|
||||||
|
stopEncoder()
|
||||||
|
if #available(macOS 12.3, *), let stream = scStream as? SCStream {
|
||||||
|
Task { try? await stream.stopCapture() }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool {
|
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||||
return true
|
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
|
||||||
|
|
||||||
|
private class VTSessionBox {
|
||||||
|
let session: VTCompressionSession
|
||||||
|
init(_ s: VTCompressionSession) { session = s }
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - VT output C callback
|
||||||
|
|
||||||
|
private func vtOutputCallback(
|
||||||
|
outputCallbackRefCon: UnsafeMutableRawPointer?,
|
||||||
|
sourceFrameRefCon: UnsafeMutableRawPointer?,
|
||||||
|
status: OSStatus,
|
||||||
|
infoFlags: VTEncodeInfoFlags,
|
||||||
|
sampleBuffer: CMSampleBuffer?
|
||||||
|
) {
|
||||||
|
guard status == noErr, let sampleBuffer = sampleBuffer,
|
||||||
|
let refcon = outputCallbackRefCon else { return }
|
||||||
|
let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
|
||||||
|
delegate.handleEncodedFrame(sampleBuffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - SCK delegate
|
||||||
|
|
||||||
|
@available(macOS 12.3, *)
|
||||||
|
class ScreenDelegate: NSObject, SCStreamOutput {
|
||||||
|
private let onFrame: (CMSampleBuffer) -> Void
|
||||||
|
|
||||||
|
init(onFrame: @escaping (CMSampleBuffer) -> Void) {
|
||||||
|
self.onFrame = onFrame
|
||||||
|
}
|
||||||
|
|
||||||
|
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||||
|
guard type == .screen else { return }
|
||||||
|
onFrame(sampleBuffer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
|
||||||
|
|
||||||
|
extension AppDelegate: FlutterStreamHandler {
|
||||||
|
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
|
||||||
|
frameEventSink = events
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func onCancel(withArguments arguments: Any?) -> FlutterError? {
|
||||||
|
frameEventSink = nil
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
32
pubspec.lock
32
pubspec.lock
|
|
@ -77,10 +77,10 @@ packages:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
name: dart_webrtc
|
name: dart_webrtc
|
||||||
sha256: ac7ef077084b3e54004716f1d736fcd839e1b60bc3f21f4122a35a9bb5ca2e47
|
sha256: f6d615bddea5e458ce180a914f3055c234ffb52fb7397a51b3491e76d6d7edb2
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "1.4.8"
|
version: "1.8.1"
|
||||||
fake_async:
|
fake_async:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
@ -127,10 +127,10 @@ packages:
|
||||||
dependency: "direct main"
|
dependency: "direct main"
|
||||||
description:
|
description:
|
||||||
name: flutter_webrtc
|
name: flutter_webrtc
|
||||||
sha256: "2f17fb96e0c9c6ff75f6b1c36d94755461fc7f36a5c28386f5ee5a18b98688c8"
|
sha256: c7b0a67ca2c878575fc5c146d801cd874f58f5f1ef5fa6e8eb0c93d413beb948
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "0.9.48+hotfix.1"
|
version: "1.4.1"
|
||||||
glob:
|
glob:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
@ -203,6 +203,14 @@ packages:
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "6.1.0"
|
version: "6.1.0"
|
||||||
|
logger:
|
||||||
|
dependency: transitive
|
||||||
|
description:
|
||||||
|
name: logger
|
||||||
|
sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c"
|
||||||
|
url: "https://pub.dev"
|
||||||
|
source: hosted
|
||||||
|
version: "2.7.0"
|
||||||
logging:
|
logging:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
@ -323,14 +331,6 @@ packages:
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "3.1.6"
|
version: "3.1.6"
|
||||||
platform_detect:
|
|
||||||
dependency: transitive
|
|
||||||
description:
|
|
||||||
name: platform_detect
|
|
||||||
sha256: "6e91e2158b9db7f6f4581a0f0b3a77171570b392108de196ba0c52e3c1976334"
|
|
||||||
url: "https://pub.dev"
|
|
||||||
source: hosted
|
|
||||||
version: "2.1.6"
|
|
||||||
plugin_platform_interface:
|
plugin_platform_interface:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
@ -444,18 +444,18 @@ packages:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
name: web
|
name: web
|
||||||
sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27"
|
sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "0.5.1"
|
version: "1.1.1"
|
||||||
web_socket_channel:
|
web_socket_channel:
|
||||||
dependency: "direct main"
|
dependency: "direct main"
|
||||||
description:
|
description:
|
||||||
name: web_socket_channel
|
name: web_socket_channel
|
||||||
sha256: "58c6666b342a38816b2e7e50ed0f1e261959630becd4c879c4f26bfa14aa5a42"
|
sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "2.4.5"
|
version: "2.4.0"
|
||||||
webrtc_interface:
|
webrtc_interface:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ dependencies:
|
||||||
# The following adds the Cupertino Icons font to your application.
|
# The following adds the Cupertino Icons font to your application.
|
||||||
# Use with the CupertinoIcons class for iOS style icons.
|
# Use with the CupertinoIcons class for iOS style icons.
|
||||||
cupertino_icons: ^1.0.8
|
cupertino_icons: ^1.0.8
|
||||||
flutter_webrtc: ^0.9.47
|
flutter_webrtc: ^1.4.1
|
||||||
web_socket_channel: ^2.4.0
|
web_socket_channel: ^2.4.0
|
||||||
|
|
||||||
dev_dependencies:
|
dev_dependencies:
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue