inital
This commit is contained in:
ImBenji
parent
0ed740ad19
commit
bfdaf4a801
9 changed files with 1081 additions and 149 deletions
|
|
@ -1,4 +1,13 @@
|
|||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<uses-permission android:name="android.permission.INTERNET"/>
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
|
||||
<uses-permission android:name="android.permission.CHANGE_NETWORK_STATE"/>
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO"/>
|
||||
<uses-permission android:name="android.permission.CAMERA"/>
|
||||
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS"/>
|
||||
<uses-permission android:name="android.permission.BLUETOOTH" android:maxSdkVersion="30"/>
|
||||
<uses-permission android:name="android.permission.BLUETOOTH_CONNECT"/>
|
||||
|
||||
<application
|
||||
android:label="pulsar"
|
||||
android:name="${applicationName}"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,255 @@
|
|||
package net.imbenji.pulsar;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo;
|
||||
import android.media.MediaFormat;
|
||||
import android.os.Handler;
|
||||
import android.os.HandlerThread;
|
||||
import android.view.Surface;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import io.flutter.embedding.android.FlutterActivity;
|
||||
import io.flutter.embedding.engine.FlutterEngine;
|
||||
import io.flutter.plugin.common.MethodChannel;
|
||||
import io.flutter.view.TextureRegistry;
|
||||
|
||||
public class MainActivity extends FlutterActivity {
|
||||
|
||||
private static final String CHANNEL = "com.pulsar/video";
|
||||
|
||||
// one decoder entry per textureId
|
||||
private static class DecoderEntry {
|
||||
TextureRegistry.SurfaceTextureEntry textureEntry;
|
||||
Surface surface;
|
||||
MediaCodec codec;
|
||||
HandlerThread outputThread;
|
||||
Handler outputHandler;
|
||||
HandlerThread inputThread;
|
||||
Handler inputHandler;
|
||||
AtomicBoolean running = new AtomicBoolean(true);
|
||||
}
|
||||
|
||||
private final Map<Long, DecoderEntry> decoders = new HashMap<>();
|
||||
|
||||
@Override
|
||||
public void configureFlutterEngine(FlutterEngine flutterEngine) {
|
||||
super.configureFlutterEngine(flutterEngine);
|
||||
|
||||
TextureRegistry registry = flutterEngine.getRenderer();
|
||||
|
||||
new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), CHANNEL)
|
||||
.setMethodCallHandler((call, result) -> {
|
||||
switch (call.method) {
|
||||
|
||||
case "createDecoder": {
|
||||
byte[] spsPps = call.argument("spsPps");
|
||||
if (spsPps == null) {
|
||||
result.error("BAD_ARGS", "spsPps required", null);
|
||||
break;
|
||||
}
|
||||
|
||||
TextureRegistry.SurfaceTextureEntry entry = registry.createSurfaceTexture();
|
||||
long texId = entry.id();
|
||||
|
||||
// we don't know the video size yet — configure with a resonable
|
||||
// placeholder; MediaCodec will adapt once it sees the stream
|
||||
int width = 1280;
|
||||
int height = 720;
|
||||
|
||||
MediaFormat fmt = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
|
||||
|
||||
// csd-0 must include Annex B start codes — pass the whole SPS+PPS blob
|
||||
// (MediaCodec H264 accepts combined SPS+PPS with start codes in csd-0)
|
||||
fmt.setByteBuffer("csd-0", ByteBuffer.wrap(spsPps));
|
||||
|
||||
entry.surfaceTexture().setDefaultBufferSize(width, height);
|
||||
|
||||
fmt.setInteger(MediaFormat.KEY_COLOR_FORMAT,
|
||||
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
|
||||
|
||||
Surface surface = new Surface(entry.surfaceTexture());
|
||||
|
||||
MediaCodec codec;
|
||||
try {
|
||||
codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
|
||||
codec.configure(fmt, surface, null, 0);
|
||||
codec.start();
|
||||
} catch (IOException e) {
|
||||
surface.release();
|
||||
entry.release();
|
||||
result.error("CODEC_FAIL", e.getMessage(), null);
|
||||
break;
|
||||
}
|
||||
|
||||
DecoderEntry dec = new DecoderEntry();
|
||||
dec.textureEntry = entry;
|
||||
dec.surface = surface;
|
||||
dec.codec = codec;
|
||||
|
||||
HandlerThread ot = new HandlerThread("decoder-out-" + texId);
|
||||
ot.start();
|
||||
dec.outputThread = ot;
|
||||
dec.outputHandler = new Handler(ot.getLooper());
|
||||
|
||||
HandlerThread it = new HandlerThread("decoder-in-" + texId);
|
||||
it.start();
|
||||
dec.inputThread = it;
|
||||
dec.inputHandler = new Handler(it.getLooper());
|
||||
|
||||
drainOutputLoop(dec);
|
||||
|
||||
decoders.put(texId, dec);
|
||||
|
||||
Map<String, Long> res = new HashMap<>();
|
||||
res.put("textureId", texId);
|
||||
result.success(res);
|
||||
break;
|
||||
}
|
||||
|
||||
case "feedNal": {
|
||||
long texId = ((Number) call.argument("textureId")).longValue();
|
||||
byte[] nal = call.argument("nal");
|
||||
|
||||
DecoderEntry dec = decoders.get(texId);
|
||||
if (dec == null || nal == null) {
|
||||
result.error("NO_DECODER", "decoder not found", null);
|
||||
break;
|
||||
}
|
||||
|
||||
dec.inputHandler.post(() -> {
|
||||
try {
|
||||
int idx = dec.codec.dequeueInputBuffer(0);
|
||||
if (idx >= 0) {
|
||||
ByteBuffer buf = dec.codec.getInputBuffer(idx);
|
||||
if (buf != null) {
|
||||
buf.clear();
|
||||
buf.put(nal);
|
||||
dec.codec.queueInputBuffer(idx, 0, nal.length,
|
||||
System.nanoTime() / 1000, 0);
|
||||
}
|
||||
}
|
||||
// if idx < 0 the decoder is backed up — drop this NAL silently
|
||||
} catch (Exception ignored) {}
|
||||
});
|
||||
|
||||
result.success(null);
|
||||
break;
|
||||
}
|
||||
|
||||
case "stopDecoder": {
|
||||
long texId = ((Number) call.argument("textureId")).longValue();
|
||||
DecoderEntry dec = decoders.remove(texId);
|
||||
if (dec != null) releaseDecoder(dec);
|
||||
result.success(null);
|
||||
break;
|
||||
}
|
||||
|
||||
case "start":
|
||||
case "stop":
|
||||
case "forceKeyframe":
|
||||
result.success(null);
|
||||
break;
|
||||
|
||||
default:
|
||||
result.notImplemented();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void drainOutputLoop(DecoderEntry dec) {
|
||||
dec.outputHandler.post(() -> {
|
||||
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
|
||||
while (dec.running.get()) {
|
||||
try {
|
||||
// block up to 10ms waiting for a decoded frame
|
||||
int idx = dec.codec.dequeueOutputBuffer(info, 10_000);
|
||||
if (idx >= 0) {
|
||||
dec.codec.releaseOutputBuffer(idx, true);
|
||||
}
|
||||
// idx == INFO_TRY_AGAIN_LATER / FORMAT_CHANGED / BUFFERS_CHANGED → loop
|
||||
} catch (Exception e) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private final Handler mainHandler = new Handler(android.os.Looper.getMainLooper());
|
||||
|
||||
private void releaseDecoder(DecoderEntry dec) {
|
||||
dec.running.set(false);
|
||||
// codec stop/release must happen on the decoder thread (after the drain loop exits)
|
||||
// but entry.release() calls FlutterJNI.unregisterTexture which requires the main thread
|
||||
dec.outputHandler.post(() -> {
|
||||
try {
|
||||
dec.codec.stop();
|
||||
dec.codec.release();
|
||||
} catch (Exception ignored) {}
|
||||
dec.surface.release();
|
||||
mainHandler.post(() -> dec.textureEntry.release());
|
||||
});
|
||||
dec.outputThread.quitSafely();
|
||||
dec.inputThread.quitSafely();
|
||||
}
|
||||
|
||||
// splits an Annex B blob into individual NAL unit ByteBuffers
|
||||
// returns null if fewer than 2 NALUs found
|
||||
private ByteBuffer[] splitAnnexB(byte[] data) {
|
||||
java.util.List<ByteBuffer> units = new java.util.ArrayList<>();
|
||||
int start = -1;
|
||||
|
||||
for (int i = 0; i < data.length - 3; i++) {
|
||||
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1) {
|
||||
if (start >= 0) {
|
||||
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
|
||||
units.add(bb);
|
||||
}
|
||||
start = i + 4;
|
||||
i += 3;
|
||||
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
|
||||
if (start >= 0) {
|
||||
ByteBuffer bb = ByteBuffer.wrap(data, start, i - start);
|
||||
units.add(bb);
|
||||
}
|
||||
start = i + 3;
|
||||
i += 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (start >= 0 && start < data.length) {
|
||||
units.add(ByteBuffer.wrap(data, start, data.length - start));
|
||||
}
|
||||
|
||||
return units.size() >= 2 ? units.toArray(new ByteBuffer[0]) : null;
|
||||
}
|
||||
|
||||
// crude SPS parser — reads width/height from the first SPS NALU
|
||||
// returns null if it cant parse
|
||||
private int[] dimsFromSps(ByteBuffer spsBuf) {
|
||||
try {
|
||||
byte[] sps = new byte[spsBuf.remaining()];
|
||||
spsBuf.duplicate().get(sps);
|
||||
|
||||
// skip NAL header byte (forbidden_zero_bit + nal_ref_idc + nal_unit_type)
|
||||
// a minimal parse isn't reliable without a full RBSP/Exp-Golomb parser,
|
||||
// so just return null and let MediaCodec figure it out
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onDestroy() {
|
||||
for (DecoderEntry dec : decoders.values()) {
|
||||
releaseDecoder(dec);
|
||||
}
|
||||
decoders.clear();
|
||||
super.onDestroy();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:math';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
|
|
@ -8,8 +9,9 @@ import 'package:flutter_webrtc/flutter_webrtc.dart';
|
|||
|
||||
import '../services/signalling.dart';
|
||||
|
||||
// platform channel for injecting input into the OS
|
||||
const _inputChannel = MethodChannel('com.pulsar/input');
|
||||
const _videoChannel = MethodChannel('com.pulsar/video');
|
||||
const _frameEvents = EventChannel('com.pulsar/video/frames');
|
||||
|
||||
class HostScreen extends StatefulWidget {
|
||||
final String serverUrl;
|
||||
|
|
@ -23,13 +25,31 @@ class _HostScreenState extends State<HostScreen> {
|
|||
final _sig = SignallingService();
|
||||
RTCPeerConnection? _pc;
|
||||
RTCDataChannel? _dataChannel;
|
||||
MediaStream? _localStream;
|
||||
|
||||
String _roomId = '';
|
||||
String _status = 'Initialising...';
|
||||
bool _connected = false;
|
||||
|
||||
StreamSubscription? _sigSub;
|
||||
StreamSubscription? _frameSub;
|
||||
|
||||
// stats
|
||||
bool _showStats = false;
|
||||
int _statFps = 0;
|
||||
double _statBitrateMbps = 0;
|
||||
int _statDropped = 0;
|
||||
int _statBufferKb = 0;
|
||||
double _statSendMs = 0;
|
||||
|
||||
int _frameCount = 0;
|
||||
int _byteCount = 0;
|
||||
int _droppedCount = 0;
|
||||
double _sendMsSum = 0;
|
||||
int _sendMsCount = 0;
|
||||
Timer? _statsTimer;
|
||||
|
||||
// viewer stats received over data channel
|
||||
Map<String, dynamic>? _viewerStats;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
|
|
@ -67,7 +87,6 @@ class _HostScreenState extends State<HostScreen> {
|
|||
switch (type) {
|
||||
case 'created':
|
||||
_setState('Waiting for viewer...');
|
||||
await _startCapture();
|
||||
break;
|
||||
|
||||
case 'peer_joined':
|
||||
|
|
@ -82,6 +101,7 @@ class _HostScreenState extends State<HostScreen> {
|
|||
|
||||
case 'peer_disconnected':
|
||||
_setState('Viewer disconnected');
|
||||
_stopCapture();
|
||||
setState(() => _connected = false);
|
||||
break;
|
||||
|
||||
|
|
@ -91,17 +111,6 @@ class _HostScreenState extends State<HostScreen> {
|
|||
}
|
||||
}
|
||||
|
||||
Future<void> _startCapture() async {
|
||||
try {
|
||||
_localStream = await navigator.mediaDevices.getDisplayMedia({
|
||||
'video': {'cursor': 'always'},
|
||||
'audio': false,
|
||||
});
|
||||
} catch (e) {
|
||||
_setState('Screen capture failed: $e');
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _createOffer() async {
|
||||
final config = {
|
||||
'iceServers': [
|
||||
|
|
@ -111,31 +120,34 @@ class _HostScreenState extends State<HostScreen> {
|
|||
|
||||
_pc = await createPeerConnection(config);
|
||||
|
||||
// data channel for input events (host creates it as offerer)
|
||||
_dataChannel = await _pc!.createDataChannel(
|
||||
'input',
|
||||
RTCDataChannelInit()..ordered = true,
|
||||
);
|
||||
// unordered + unreliable — drop stale frames, only the latest matters
|
||||
final dcInit = RTCDataChannelInit()
|
||||
..ordered = false
|
||||
..maxRetransmits = 0;
|
||||
|
||||
_dataChannel = await _pc!.createDataChannel('frames', dcInit);
|
||||
|
||||
_dataChannel!.onMessage = (msg) {
|
||||
_handleInputEvent(msg.text);
|
||||
};
|
||||
|
||||
// add screen tracks
|
||||
if (_localStream != null) {
|
||||
for (final track in _localStream!.getTracks()) {
|
||||
await _pc!.addTrack(track, _localStream!);
|
||||
if (msg.isBinary) return;
|
||||
try {
|
||||
final parsed = jsonDecode(msg.text) as Map<String, dynamic>;
|
||||
if (parsed['type'] == 'stats') {
|
||||
setState(() {
|
||||
_viewerStats = parsed;
|
||||
});
|
||||
} else {
|
||||
_handleInputEvent(msg.text);
|
||||
}
|
||||
} catch (_) {
|
||||
_handleInputEvent(msg.text);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
_pc!.onIceCandidate = (candidate) {
|
||||
_sig.send({
|
||||
'type': 'signal',
|
||||
'roomId': _roomId,
|
||||
'data': {
|
||||
'type': 'candidate',
|
||||
'candidate': candidate.toMap(),
|
||||
},
|
||||
'data': {'type': 'candidate', 'candidate': candidate.toMap()},
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -143,9 +155,11 @@ class _HostScreenState extends State<HostScreen> {
|
|||
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
||||
_setState('Connected');
|
||||
setState(() => _connected = true);
|
||||
_startCapture();
|
||||
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
||||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
||||
_setState('Connection lost');
|
||||
_stopCapture();
|
||||
setState(() => _connected = false);
|
||||
}
|
||||
};
|
||||
|
|
@ -177,6 +191,58 @@ class _HostScreenState extends State<HostScreen> {
|
|||
}
|
||||
}
|
||||
|
||||
Future<void> _startCapture() async {
|
||||
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
|
||||
|
||||
// subscribe BEFORE starting the encoder so the initial SPS+PPS+keyframe isn't dropped
|
||||
_frameSub = _frameEvents.receiveBroadcastStream().listen((dynamic raw) {
|
||||
final bytes = (raw as Uint8List);
|
||||
final dc = _dataChannel;
|
||||
if (dc == null) return;
|
||||
|
||||
final buffered = dc.bufferedAmount ?? 0;
|
||||
if (buffered > 262144) {
|
||||
_droppedCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
final sendStart = DateTime.now();
|
||||
dc.send(RTCDataChannelMessage.fromBinary(bytes));
|
||||
_sendMsSum += DateTime.now().difference(sendStart).inMicroseconds / 1000.0;
|
||||
_sendMsCount++;
|
||||
_frameCount++;
|
||||
_byteCount += bytes.length;
|
||||
} catch (_) {}
|
||||
});
|
||||
|
||||
await _videoChannel.invokeMethod('start');
|
||||
await _videoChannel.invokeMethod('forceKeyframe');
|
||||
}
|
||||
|
||||
void _stopCapture() {
|
||||
_frameSub?.cancel();
|
||||
_frameSub = null;
|
||||
_statsTimer?.cancel();
|
||||
_statsTimer = null;
|
||||
_videoChannel.invokeMethod('stop');
|
||||
}
|
||||
|
||||
void _flushStats() {
|
||||
final buf = _dataChannel?.bufferedAmount ?? 0;
|
||||
setState(() {
|
||||
_statFps = _frameCount;
|
||||
_statBitrateMbps = _byteCount * 8 / 1e6;
|
||||
_statDropped = _droppedCount;
|
||||
_statBufferKb = (buf / 1024).round();
|
||||
_statSendMs = _sendMsCount > 0 ? _sendMsSum / _sendMsCount : 0;
|
||||
_frameCount = 0;
|
||||
_byteCount = 0;
|
||||
_droppedCount = 0;
|
||||
_sendMsSum = 0; _sendMsCount = 0;
|
||||
});
|
||||
}
|
||||
|
||||
void _handleInputEvent(String json) async {
|
||||
try {
|
||||
final ev = jsonDecode(json) as Map<String, dynamic>;
|
||||
|
|
@ -190,10 +256,10 @@ class _HostScreenState extends State<HostScreen> {
|
|||
|
||||
@override
|
||||
void dispose() {
|
||||
_stopCapture();
|
||||
_sigSub?.cancel();
|
||||
_sig.dispose();
|
||||
_pc?.close();
|
||||
_localStream?.dispose();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
|
|
@ -204,37 +270,100 @@ class _HostScreenState extends State<HostScreen> {
|
|||
title: const Text('Pulsar — Host'),
|
||||
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
||||
),
|
||||
body: Center(
|
||||
child: Column(
|
||||
mainAxisAlignment: MainAxisAlignment.center,
|
||||
children: [
|
||||
const Icon(Icons.monitor, size: 80, color: Colors.indigo),
|
||||
const SizedBox(height: 24),
|
||||
Text('Room Code', style: Theme.of(context).textTheme.titleMedium),
|
||||
const SizedBox(height: 8),
|
||||
SelectableText(
|
||||
_roomId,
|
||||
style: Theme.of(context).textTheme.displaySmall?.copyWith(
|
||||
fontFamily: 'monospace',
|
||||
fontWeight: FontWeight.bold,
|
||||
letterSpacing: 8,
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 32),
|
||||
Row(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
floatingActionButton: FloatingActionButton(
|
||||
mini: true,
|
||||
tooltip: 'Stats',
|
||||
onPressed: () => setState(() => _showStats = !_showStats),
|
||||
child: const Icon(Icons.bar_chart),
|
||||
),
|
||||
body: Stack(
|
||||
children: [
|
||||
Center(
|
||||
child: Column(
|
||||
mainAxisAlignment: MainAxisAlignment.center,
|
||||
children: [
|
||||
Icon(
|
||||
_connected ? Icons.circle : Icons.circle_outlined,
|
||||
color: _connected ? Colors.green : Colors.orange,
|
||||
size: 14,
|
||||
const Icon(Icons.monitor, size: 80, color: Colors.indigo),
|
||||
const SizedBox(height: 24),
|
||||
Text('Room Code', style: Theme.of(context).textTheme.titleMedium),
|
||||
const SizedBox(height: 8),
|
||||
SelectableText(
|
||||
_roomId,
|
||||
style: Theme.of(context).textTheme.displaySmall?.copyWith(
|
||||
fontFamily: 'monospace',
|
||||
fontWeight: FontWeight.bold,
|
||||
letterSpacing: 8,
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 32),
|
||||
Row(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
Icon(
|
||||
_connected ? Icons.circle : Icons.circle_outlined,
|
||||
color: _connected ? Colors.green : Colors.orange,
|
||||
size: 14,
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
Text(_status),
|
||||
],
|
||||
),
|
||||
const SizedBox(width: 8),
|
||||
Text(_status),
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
|
||||
if (_showStats)
|
||||
Positioned(
|
||||
top: 12,
|
||||
left: 12,
|
||||
child: _StatsOverlay(lines: [
|
||||
'Host:',
|
||||
' FPS: $_statFps',
|
||||
' Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
|
||||
' Dropped: $_statDropped/s',
|
||||
' Buffer: $_statBufferKb KB',
|
||||
' Send: ${_statSendMs.toStringAsFixed(1)}ms',
|
||||
'',
|
||||
if (_viewerStats == null)
|
||||
'Viewer: no data'
|
||||
else ...[
|
||||
'Viewer:',
|
||||
' FPS: ${_viewerStats!['fps']}',
|
||||
' Bitrate: ${(_viewerStats!['bitrateMbps'] as num).toStringAsFixed(2)} Mbps',
|
||||
' Dropped: ${_viewerStats!['dropped']}/s',
|
||||
' Res: ${_viewerStats!['resolution']}',
|
||||
' Latency: ${_viewerStats!['latencyMs']}ms',
|
||||
' Submit: ${_viewerStats!['decodeMs'] ?? '—'}ms',
|
||||
],
|
||||
]),
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class _StatsOverlay extends StatelessWidget {
|
||||
final List<String> lines;
|
||||
const _StatsOverlay({required this.lines});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Container(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
|
||||
decoration: BoxDecoration(
|
||||
color: Colors.black.withValues(alpha: 0.6),
|
||||
borderRadius: BorderRadius.circular(6),
|
||||
),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: lines.map((l) => Text(
|
||||
l,
|
||||
style: const TextStyle(
|
||||
color: Colors.white,
|
||||
fontFamily: 'monospace',
|
||||
fontSize: 12,
|
||||
),
|
||||
)).toList(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
|
|
@ -16,10 +17,11 @@ class ViewerScreen extends StatefulWidget {
|
|||
}
|
||||
|
||||
class _ViewerScreenState extends State<ViewerScreen> {
|
||||
static const _videoCh = MethodChannel("com.pulsar/video");
|
||||
|
||||
final _sig = SignallingService();
|
||||
RTCPeerConnection? _pc;
|
||||
RTCDataChannel? _dataChannel;
|
||||
final _remoteRenderer = RTCVideoRenderer();
|
||||
|
||||
final _roomCodeCtrl = TextEditingController();
|
||||
String _status = 'Enter a room code to connect';
|
||||
|
|
@ -28,23 +30,46 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
|
||||
StreamSubscription? _sigSub;
|
||||
|
||||
// size of the rendered video widget, used to normalise pointer coords
|
||||
Size _videoSize = Size.zero;
|
||||
final _videoKey = GlobalKey();
|
||||
int? _textureId;
|
||||
bool _decoderReady = false;
|
||||
bool _settingUpDecoder = false;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
_remoteRenderer.initialize();
|
||||
}
|
||||
// frames that arrive while createDecoder is in-flight are queued here and
|
||||
// drained once the decoder is ready, so we don't miss the initial keyframe
|
||||
final _pendingNals = <(Uint8List, int)>[];
|
||||
|
||||
// widget key for normalising input coords
|
||||
final _videoKey = GlobalKey();
|
||||
Size _videoSize = Size.zero;
|
||||
|
||||
// stats
|
||||
bool _showStats = false;
|
||||
int _statFps = 0;
|
||||
double _statBitrateMbps = 0;
|
||||
int _statDropped = 0;
|
||||
String _statResolution = '—';
|
||||
int _statLatencyMs = 0;
|
||||
double _statDecodeMs = 0; // actually submit time for feedNal
|
||||
|
||||
int _frameCount = 0;
|
||||
int _byteCount = 0;
|
||||
int _droppedCount = 0;
|
||||
double _decodeMsSum = 0;
|
||||
int _decodeMsCount = 0;
|
||||
Timer? _statsTimer;
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_statsTimer?.cancel();
|
||||
_sigSub?.cancel();
|
||||
_sig.dispose();
|
||||
_pc?.close();
|
||||
_remoteRenderer.dispose();
|
||||
_roomCodeCtrl.dispose();
|
||||
|
||||
if (_textureId != null) {
|
||||
_videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
|
||||
}
|
||||
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
|
|
@ -92,6 +117,7 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
break;
|
||||
|
||||
case 'host_disconnected':
|
||||
_statsTimer?.cancel();
|
||||
setState(() {
|
||||
_status = 'Host disconnected';
|
||||
_streaming = false;
|
||||
|
|
@ -109,9 +135,9 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
|
||||
Future<void> _initPeerConnection(String roomId) async {
|
||||
final config = {
|
||||
'iceServers': [
|
||||
{'urls': 'stun:stun.l.google.com:19302'},
|
||||
],
|
||||
'iceServers': [{'urls': 'stun:stun.l.google.com:19302'}],
|
||||
'sdpSemantics': 'unified-plan',
|
||||
'encodedInsertableStreams': false,
|
||||
};
|
||||
|
||||
_pc = await createPeerConnection(config);
|
||||
|
|
@ -120,18 +146,17 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
_sig.send({
|
||||
'type': 'signal',
|
||||
'roomId': roomId,
|
||||
'data': {
|
||||
'type': 'candidate',
|
||||
'candidate': candidate.toMap(),
|
||||
},
|
||||
'data': {'type': 'candidate', 'candidate': candidate.toMap()},
|
||||
});
|
||||
};
|
||||
|
||||
_pc!.onConnectionState = (state) {
|
||||
if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) {
|
||||
setState(() => _status = 'Connected');
|
||||
_statsTimer = Timer.periodic(const Duration(seconds: 1), (_) => _flushStats());
|
||||
} else if (state == RTCPeerConnectionState.RTCPeerConnectionStateFailed ||
|
||||
state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected) {
|
||||
_statsTimer?.cancel();
|
||||
setState(() {
|
||||
_status = 'Connection lost';
|
||||
_streaming = false;
|
||||
|
|
@ -139,21 +164,133 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
}
|
||||
};
|
||||
|
||||
_pc!.onTrack = (event) {
|
||||
if (event.track.kind == 'video') {
|
||||
setState(() {
|
||||
_remoteRenderer.srcObject = event.streams[0];
|
||||
_streaming = true;
|
||||
_status = 'Streaming';
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
_pc!.onDataChannel = (channel) {
|
||||
_dataChannel = channel;
|
||||
|
||||
channel.onMessage = (msg) {
|
||||
if (msg.isBinary) _onData(msg.binary);
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
void _flushStats() {
|
||||
setState(() {
|
||||
_statFps = _frameCount;
|
||||
_statBitrateMbps = _byteCount * 8 / 1e6;
|
||||
_statDropped = _droppedCount;
|
||||
_statDecodeMs = _decodeMsCount > 0 ? _decodeMsSum / _decodeMsCount : 0;
|
||||
_frameCount = 0;
|
||||
_byteCount = 0;
|
||||
_droppedCount = 0;
|
||||
_decodeMsSum = 0; _decodeMsCount = 0;
|
||||
});
|
||||
_sendInput({
|
||||
'type': 'stats',
|
||||
'fps': _statFps,
|
||||
'bitrateMbps': _statBitrateMbps,
|
||||
'dropped': _statDropped,
|
||||
'resolution': _statResolution,
|
||||
'latencyMs': _statLatencyMs,
|
||||
'decodeMs': double.parse(_statDecodeMs.toStringAsFixed(1)),
|
||||
'paintMs': 0.0,
|
||||
});
|
||||
}
|
||||
|
||||
void _onData(Uint8List bytes) {
|
||||
if (bytes.isEmpty) return;
|
||||
|
||||
_byteCount += bytes.length;
|
||||
|
||||
final type = bytes[0];
|
||||
|
||||
if (type == 0x01) {
|
||||
// config packet: SPS+PPS — create/reconfigure the decoder
|
||||
final spsPps = Uint8List.sublistView(bytes, 1);
|
||||
_setupDecoder(spsPps);
|
||||
|
||||
} else if (type == 0x02) {
|
||||
// frame packet: [0x02][8-byte ts][Annex B NALU]
|
||||
if (bytes.length < 10) return;
|
||||
|
||||
final bd = ByteData.sublistView(bytes, 1, 9);
|
||||
final ts = bd.getInt64(0, Endian.big);
|
||||
final latency = DateTime.now().millisecondsSinceEpoch - ts;
|
||||
|
||||
final nal = Uint8List.sublistView(bytes, 9);
|
||||
|
||||
// drop frames that are already stale — prevents burst playback after a lag spike
|
||||
if (latency > 300) return;
|
||||
|
||||
if (!_decoderReady) {
|
||||
_pendingNals.add((nal, latency));
|
||||
} else {
|
||||
_feedNal(nal, latency);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _setupDecoder(Uint8List spsPps) async {
|
||||
// a second 0x01 packet can arrive while createDecoder is still awaiting;
|
||||
// ignore it — the first decoder will be ready soon enough
|
||||
if (_settingUpDecoder) return;
|
||||
_settingUpDecoder = true;
|
||||
|
||||
_decoderReady = false;
|
||||
_pendingNals.clear();
|
||||
|
||||
if (_textureId != null) {
|
||||
await _videoCh.invokeMethod('stopDecoder', {'textureId': _textureId});
|
||||
_textureId = null;
|
||||
}
|
||||
|
||||
try {
|
||||
final res = await _videoCh.invokeMethod<Map>('createDecoder', {'spsPps': spsPps});
|
||||
final id = (res?['textureId'] as num?)?.toInt();
|
||||
|
||||
if (!mounted) return;
|
||||
setState(() {
|
||||
_textureId = id;
|
||||
if (!_streaming) _streaming = true;
|
||||
});
|
||||
|
||||
_decoderReady = true;
|
||||
|
||||
// drain frames that queued up while createDecoder was in-flight
|
||||
final pending = List.of(_pendingNals);
|
||||
_pendingNals.clear();
|
||||
for (final (nal, latency) in pending) {
|
||||
_feedNal(nal, latency);
|
||||
}
|
||||
} catch (_) {
|
||||
_decoderReady = false;
|
||||
} finally {
|
||||
_settingUpDecoder = false;
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _feedNal(Uint8List nal, int latency) async {
|
||||
if (_textureId == null) return;
|
||||
|
||||
try {
|
||||
final t0 = DateTime.now();
|
||||
await _videoCh.invokeMethod('feedNal', {
|
||||
'textureId': _textureId,
|
||||
'nal': nal,
|
||||
});
|
||||
final elapsed = DateTime.now().difference(t0).inMicroseconds / 1000.0;
|
||||
|
||||
if (!mounted) return;
|
||||
|
||||
_decodeMsSum += elapsed;
|
||||
_decodeMsCount++;
|
||||
_frameCount++;
|
||||
|
||||
setState(() {
|
||||
_statLatencyMs = latency.clamp(0, 9999);
|
||||
});
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
Future<void> _handleSignalData(Map<String, dynamic> data, String roomId) async {
|
||||
final sigType = data['type'] as String?;
|
||||
|
||||
|
|
@ -169,7 +306,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
'roomId': roomId,
|
||||
'data': {'type': 'answer', 'sdp': answer.sdp},
|
||||
});
|
||||
|
||||
} else if (sigType == 'candidate') {
|
||||
final raw = data['candidate'] as Map<String, dynamic>;
|
||||
final candidate = RTCIceCandidate(
|
||||
|
|
@ -188,7 +324,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
} catch (_) {}
|
||||
}
|
||||
|
||||
// get video widget size via the global key
|
||||
void _updateVideoSize() {
|
||||
final ctx = _videoKey.currentContext;
|
||||
if (ctx != null) {
|
||||
|
|
@ -206,8 +341,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
);
|
||||
}
|
||||
|
||||
// ---- input capture
|
||||
|
||||
void _onPointerMove(PointerMoveEvent e) {
|
||||
final n = _normalise(e.localPosition);
|
||||
_sendInput({'type': 'move', 'x': n.dx, 'y': n.dy});
|
||||
|
|
@ -215,7 +348,6 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
|
||||
void _onPointerDown(PointerDownEvent e) {
|
||||
final n = _normalise(e.localPosition);
|
||||
// button 0 = left, 1 = right (PointerDeviceKind bit not directly available)
|
||||
_sendInput({'type': 'click', 'x': n.dx, 'y': n.dy, 'button': 0});
|
||||
}
|
||||
|
||||
|
|
@ -234,6 +366,14 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
title: const Text('Pulsar — Viewer'),
|
||||
backgroundColor: Theme.of(context).colorScheme.inversePrimary,
|
||||
),
|
||||
floatingActionButton: _streaming
|
||||
? FloatingActionButton(
|
||||
mini: true,
|
||||
tooltip: 'Stats',
|
||||
onPressed: () => setState(() => _showStats = !_showStats),
|
||||
child: const Icon(Icons.bar_chart),
|
||||
)
|
||||
: null,
|
||||
body: _streaming ? _buildStream() : _buildJoinView(),
|
||||
);
|
||||
}
|
||||
|
|
@ -296,20 +436,66 @@ class _ViewerScreenState extends State<ViewerScreen> {
|
|||
}
|
||||
|
||||
Widget _buildStream() {
|
||||
return Focus(
|
||||
autofocus: true,
|
||||
onKeyEvent: (_, e) {
|
||||
_onKeyEvent(e);
|
||||
return KeyEventResult.handled;
|
||||
},
|
||||
child: Listener(
|
||||
onPointerMove: _onPointerMove,
|
||||
onPointerDown: _onPointerDown,
|
||||
child: RTCVideoView(
|
||||
_remoteRenderer,
|
||||
key: _videoKey,
|
||||
objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
|
||||
return Stack(
|
||||
children: [
|
||||
Focus(
|
||||
autofocus: true,
|
||||
onKeyEvent: (_, e) {
|
||||
_onKeyEvent(e);
|
||||
return KeyEventResult.handled;
|
||||
},
|
||||
child: Listener(
|
||||
onPointerMove: _onPointerMove,
|
||||
onPointerDown: _onPointerDown,
|
||||
child: _textureId != null
|
||||
? SizedBox.expand(
|
||||
key: _videoKey,
|
||||
child: Texture(textureId: _textureId!),
|
||||
)
|
||||
: const Center(child: CircularProgressIndicator()),
|
||||
),
|
||||
),
|
||||
|
||||
if (_showStats)
|
||||
Positioned(
|
||||
top: 12,
|
||||
left: 12,
|
||||
child: _StatsOverlay(lines: [
|
||||
'FPS: $_statFps',
|
||||
'Bitrate: ${_statBitrateMbps.toStringAsFixed(2)} Mbps',
|
||||
'Dropped: $_statDropped/s',
|
||||
'Res: $_statResolution',
|
||||
'Latency: ${_statLatencyMs}ms',
|
||||
'Submit: ${_statDecodeMs.toStringAsFixed(1)}ms',
|
||||
]),
|
||||
),
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class _StatsOverlay extends StatelessWidget {
|
||||
final List<String> lines;
|
||||
const _StatsOverlay({required this.lines});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return Container(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 10, vertical: 8),
|
||||
decoration: BoxDecoration(
|
||||
color: Colors.black.withValues(alpha: 0.6),
|
||||
borderRadius: BorderRadius.circular(6),
|
||||
),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: lines.map((l) => Text(
|
||||
l,
|
||||
style: const TextStyle(
|
||||
color: Colors.white,
|
||||
fontFamily: 'monospace',
|
||||
fontSize: 12,
|
||||
),
|
||||
)).toList(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,10 @@ class SignallingService {
|
|||
} catch (_) {}
|
||||
},
|
||||
onError: (err) {
|
||||
_controller.addError(err);
|
||||
if (!_controller.isClosed) _controller.addError(err);
|
||||
},
|
||||
onDone: () {
|
||||
_controller.addError(Exception('WebSocket connection closed'));
|
||||
if (!_controller.isClosed) _controller.addError(Exception('WebSocket connection closed'));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
PODS:
|
||||
- flutter_webrtc (0.9.36):
|
||||
- flutter_webrtc (1.4.0):
|
||||
- FlutterMacOS
|
||||
- WebRTC-SDK (= 114.5735.08)
|
||||
- WebRTC-SDK (= 144.7559.01)
|
||||
- FlutterMacOS (1.0.0)
|
||||
- WebRTC-SDK (114.5735.08)
|
||||
- WebRTC-SDK (144.7559.01)
|
||||
|
||||
DEPENDENCIES:
|
||||
- flutter_webrtc (from `Flutter/ephemeral/.symlinks/plugins/flutter_webrtc/macos`)
|
||||
|
|
@ -20,9 +20,9 @@ EXTERNAL SOURCES:
|
|||
:path: Flutter/ephemeral
|
||||
|
||||
SPEC CHECKSUMS:
|
||||
flutter_webrtc: cf7dc44d26cbb5c5f1ae5f583dab545871f287f9
|
||||
flutter_webrtc: dbacf72a9bd951ccfa5997198f0a6214bb848ede
|
||||
FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1
|
||||
WebRTC-SDK: c24d2a6c9f571f2ed42297cb8ffba9557093142b
|
||||
WebRTC-SDK: ab9b5319e458c2bfebdc92b3600740da35d5630d
|
||||
|
||||
PODFILE CHECKSUM: 54d867c82ac51cbd61b565781b9fada492027009
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,37 @@
|
|||
import Cocoa
|
||||
import FlutterMacOS
|
||||
import CoreGraphics
|
||||
import ScreenCaptureKit
|
||||
import CoreMedia
|
||||
import CoreVideo
|
||||
import VideoToolbox
|
||||
|
||||
@main
|
||||
class AppDelegate: FlutterAppDelegate {
|
||||
private var inputChannel: FlutterMethodChannel?
|
||||
private var videoChannel: FlutterMethodChannel?
|
||||
private var frameEventSink: FlutterEventSink?
|
||||
|
||||
// SCK objects (typed Any? to satisfy 10.15 deployment target)
|
||||
private var scStream: Any?
|
||||
private var screenDelegate: Any?
|
||||
private var streamReady = false
|
||||
|
||||
// VTCompressionSession (Any? for same reason)
|
||||
private var vtSession: Any?
|
||||
private var encoderReady = false
|
||||
|
||||
// resolution last seen from SCK (to build config)
|
||||
private var captureWidth = 1280
|
||||
private var captureHeight = 720
|
||||
|
||||
override func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
guard let window = mainFlutterWindow,
|
||||
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
||||
|
||||
inputChannel = FlutterMethodChannel(
|
||||
name: "com.pulsar/input",
|
||||
binaryMessenger: ctrl.engine.binaryMessenger
|
||||
)
|
||||
let messenger = ctrl.engine.binaryMessenger
|
||||
|
||||
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
||||
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
||||
if call.method == "injectInput",
|
||||
let args = call.arguments as? [String: Any] {
|
||||
|
|
@ -24,8 +41,302 @@ class AppDelegate: FlutterAppDelegate {
|
|||
result(FlutterMethodNotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
// event channel for pushing encoded NAL units to Flutter
|
||||
let frameChannel = FlutterEventChannel(name: "com.pulsar/video/frames", binaryMessenger: messenger)
|
||||
frameChannel.setStreamHandler(self)
|
||||
|
||||
videoChannel = FlutterMethodChannel(name: "com.pulsar/video", binaryMessenger: messenger)
|
||||
videoChannel?.setMethodCallHandler { [weak self] call, result in
|
||||
guard let self = self else { return }
|
||||
switch call.method {
|
||||
case "start":
|
||||
self.startEncoder(result: result)
|
||||
case "stop":
|
||||
self.stopEncoder()
|
||||
result(nil)
|
||||
case "forceKeyframe":
|
||||
self.requestKeyframe()
|
||||
result(nil)
|
||||
default:
|
||||
result(FlutterMethodNotImplemented)
|
||||
}
|
||||
}
|
||||
|
||||
if #available(macOS 12.3, *) {
|
||||
Task { await self.ensureSCKStream() }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - SCK stream setup
|
||||
|
||||
@available(macOS 12.3, *)
|
||||
private func ensureSCKStream() async {
|
||||
guard scStream == nil else { return }
|
||||
|
||||
guard let content = try? await SCShareableContent.current else { return }
|
||||
guard let display = content.displays.first(where: { $0.displayID == CGMainDisplayID() }) else { return }
|
||||
|
||||
let filter = SCContentFilter(display: display, excludingWindows: [])
|
||||
|
||||
let srcW = Double(display.width)
|
||||
let srcH = Double(display.height)
|
||||
let scale = min(1.0, 1280.0 / srcW)
|
||||
let dstW = Int(srcW * scale)
|
||||
let dstH = Int(srcH * scale)
|
||||
|
||||
captureWidth = dstW
|
||||
captureHeight = dstH
|
||||
|
||||
let cfg = SCStreamConfiguration()
|
||||
cfg.width = dstW
|
||||
cfg.height = dstH
|
||||
cfg.pixelFormat = kCVPixelFormatType_32BGRA
|
||||
cfg.showsCursor = true
|
||||
cfg.minimumFrameInterval = CMTime(value: 1, timescale: 60)
|
||||
|
||||
let del = ScreenDelegate { [weak self] sampleBuffer in
|
||||
self?.encodeFrame(sampleBuffer)
|
||||
}
|
||||
screenDelegate = del
|
||||
|
||||
let stream = SCStream(filter: filter, configuration: cfg, delegate: nil)
|
||||
try? stream.addStreamOutput(del, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive))
|
||||
|
||||
do {
|
||||
try await stream.startCapture()
|
||||
scStream = stream
|
||||
streamReady = true
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// MARK: - VTCompressionSession
|
||||
|
||||
private func startEncoder(result: @escaping FlutterResult) {
|
||||
let w = captureWidth
|
||||
let h = captureHeight
|
||||
|
||||
var session: VTCompressionSession?
|
||||
let status = VTCompressionSessionCreate(
|
||||
allocator: kCFAllocatorDefault,
|
||||
width: Int32(w),
|
||||
height: Int32(h),
|
||||
codecType: kCMVideoCodecType_H264,
|
||||
encoderSpecification: nil,
|
||||
imageBufferAttributes: nil,
|
||||
compressedDataAllocator: nil,
|
||||
outputCallback: vtOutputCallback,
|
||||
refcon: Unmanaged.passUnretained(self).toOpaque(),
|
||||
compressionSessionOut: &session
|
||||
)
|
||||
|
||||
guard status == noErr, let session = session else {
|
||||
result(FlutterError(code: "VT_CREATE_FAILED", message: "VTCompressionSessionCreate status \(status)", details: nil))
|
||||
return
|
||||
}
|
||||
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel,
|
||||
value: kVTProfileLevel_H264_Baseline_AutoLevel)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AllowFrameReordering, value: kCFBooleanFalse)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate,
|
||||
value: 6_000_000 as CFNumber)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate,
|
||||
value: 60 as CFNumber)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval,
|
||||
value: 120 as CFNumber)
|
||||
VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
|
||||
value: 0 as CFNumber)
|
||||
|
||||
VTCompressionSessionPrepareToEncodeFrames(session)
|
||||
vtSession = VTSessionBox(session)
|
||||
encoderReady = true
|
||||
|
||||
// extract SPS+PPS from a forced keyframe by encoding a dummy frame
|
||||
// instead, return empty bytes and let the real SPS+PPS come through the callback
|
||||
// host_screen will call forceKeyframe right after start
|
||||
result(FlutterStandardTypedData(bytes: Data()))
|
||||
}
|
||||
|
||||
private func stopEncoder() {
|
||||
guard let box = vtSession as? VTSessionBox else { return }
|
||||
VTCompressionSessionInvalidate(box.session)
|
||||
vtSession = nil
|
||||
encoderReady = false
|
||||
}
|
||||
|
||||
func requestKeyframe() {
|
||||
forceKeyframeNext = true
|
||||
}
|
||||
|
||||
private var forceKeyframeNext = false
|
||||
|
||||
private func encodeFrame(_ sampleBuffer: CMSampleBuffer) {
|
||||
guard encoderReady, let box = vtSession as? VTSessionBox else { return }
|
||||
let s = box.session
|
||||
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
|
||||
|
||||
var frameProps: CFDictionary? = nil
|
||||
if forceKeyframeNext {
|
||||
frameProps = [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary
|
||||
forceKeyframeNext = false
|
||||
}
|
||||
|
||||
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
|
||||
VTCompressionSessionEncodeFrame(s, imageBuffer: pixelBuffer, presentationTimeStamp: pts,
|
||||
duration: .invalid, frameProperties: frameProps,
|
||||
sourceFrameRefcon: nil, infoFlagsOut: nil)
|
||||
}
|
||||
|
||||
// MARK: - VT output callback (C function)
|
||||
|
||||
// called by VideoToolbox on an internal thread — collect Data packets here,
|
||||
// then hop to main before touching the event sink
|
||||
func handleEncodedFrame(_ sampleBuffer: CMSampleBuffer) {
|
||||
var packets: [Data] = []
|
||||
|
||||
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: false) as? [[CFString: Any]]
|
||||
let isKeyframe = !(attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool ?? false)
|
||||
|
||||
if isKeyframe, let fmt = CMSampleBufferGetFormatDescription(sampleBuffer) {
|
||||
if let cfg = buildParameterSetsPacket(fmt) { packets.append(cfg) }
|
||||
}
|
||||
|
||||
if let dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) {
|
||||
var totalLen = 0
|
||||
var rawPtr: UnsafeMutablePointer<CChar>? = nil
|
||||
if CMBlockBufferGetDataPointer(dataBuffer, atOffset: 0, lengthAtOffsetOut: nil,
|
||||
totalLengthOut: &totalLen, dataPointerOut: &rawPtr) == noErr,
|
||||
let ptr = rawPtr {
|
||||
|
||||
let ts = Int64(Date().timeIntervalSince1970 * 1000)
|
||||
var offset = 0
|
||||
|
||||
while offset + 4 <= totalLen {
|
||||
// read AVCC 4-byte big-endian length byte-by-byte to avoid alignment issues
|
||||
let b0 = UInt32(UInt8(bitPattern: ptr[offset]))
|
||||
let b1 = UInt32(UInt8(bitPattern: ptr[offset + 1]))
|
||||
let b2 = UInt32(UInt8(bitPattern: ptr[offset + 2]))
|
||||
let b3 = UInt32(UInt8(bitPattern: ptr[offset + 3]))
|
||||
let naluLen = Int((b0 << 24) | (b1 << 16) | (b2 << 8) | b3)
|
||||
offset += 4
|
||||
|
||||
guard naluLen > 0, offset + naluLen <= totalLen else { break }
|
||||
|
||||
var packet = Data(capacity: 1 + 8 + 4 + naluLen)
|
||||
packet.append(0x02)
|
||||
withUnsafeBytes(of: ts.bigEndian) { packet.append(contentsOf: $0) }
|
||||
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||
packet.append(Data(bytes: ptr.advanced(by: offset), count: naluLen))
|
||||
packets.append(packet)
|
||||
|
||||
offset += naluLen
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
guard !packets.isEmpty else { return }
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
guard let sink = self?.frameEventSink else { return }
|
||||
for packet in packets {
|
||||
sink(FlutterStandardTypedData(bytes: packet))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func buildParameterSetsPacket(_ fmt: CMFormatDescription) -> Data? {
|
||||
var spsCount = 0
|
||||
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: 0, parameterSetPointerOut: nil,
|
||||
parameterSetSizeOut: nil, parameterSetCountOut: &spsCount,
|
||||
nalUnitHeaderLengthOut: nil)
|
||||
var packet = Data()
|
||||
packet.append(0x01)
|
||||
|
||||
for i in 0 ..< spsCount {
|
||||
var ptr: UnsafePointer<UInt8>? = nil
|
||||
var len = 0
|
||||
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: i,
|
||||
parameterSetPointerOut: &ptr,
|
||||
parameterSetSizeOut: &len,
|
||||
parameterSetCountOut: nil,
|
||||
nalUnitHeaderLengthOut: nil)
|
||||
if let ptr = ptr {
|
||||
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||
packet.append(Data(bytes: ptr, count: len))
|
||||
}
|
||||
}
|
||||
|
||||
// PPS sits at index spsCount
|
||||
var ppsPtr: UnsafePointer<UInt8>? = nil
|
||||
var ppsLen = 0
|
||||
if CMVideoFormatDescriptionGetH264ParameterSetAtIndex(fmt, parameterSetIndex: spsCount,
|
||||
parameterSetPointerOut: &ppsPtr,
|
||||
parameterSetSizeOut: &ppsLen,
|
||||
parameterSetCountOut: nil,
|
||||
nalUnitHeaderLengthOut: nil) == noErr,
|
||||
let ppsPtr = ppsPtr {
|
||||
packet.append(contentsOf: [0x00, 0x00, 0x00, 0x01])
|
||||
packet.append(Data(bytes: ppsPtr, count: ppsLen))
|
||||
}
|
||||
|
||||
return packet.count > 1 ? packet : nil
|
||||
}
|
||||
|
||||
// MARK: - CGWindowListCreateImage fallback (kept for non-SCK paths — unused in normal operation)
|
||||
|
||||
private func captureCG() -> Data? {
|
||||
let displayID = CGMainDisplayID()
|
||||
guard let cgImage = CGDisplayCreateImage(displayID) else { return nil }
|
||||
let composited = compositeCursor(onto: cgImage, displayID: displayID) ?? cgImage
|
||||
let srcW = CGFloat(composited.width)
|
||||
let scale = min(1.0, 1280.0 / srcW)
|
||||
let dstW = Int(srcW * scale)
|
||||
let dstH = Int(CGFloat(composited.height) * scale)
|
||||
let colorSpace = composited.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
||||
guard let ctx = CGContext(data: nil, width: dstW, height: dstH, bitsPerComponent: 8,
|
||||
bytesPerRow: 0, space: colorSpace,
|
||||
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return nil }
|
||||
ctx.interpolationQuality = .high
|
||||
ctx.draw(composited, in: CGRect(x: 0, y: 0, width: dstW, height: dstH))
|
||||
guard let scaled = ctx.makeImage() else { return nil }
|
||||
return jpegData(from: scaled, quality: 0.4)
|
||||
}
|
||||
|
||||
private func jpegData(from cgImage: CGImage, quality: Double) -> Data? {
|
||||
let nsImage = NSImage(cgImage: cgImage, size: .zero)
|
||||
guard let tiff = nsImage.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiff) else { return nil }
|
||||
return bitmap.representation(using: .jpeg, properties: [.compressionFactor: quality])
|
||||
}
|
||||
|
||||
private func compositeCursor(onto cgImage: CGImage, displayID: CGDirectDisplayID) -> CGImage? {
|
||||
let imgW = cgImage.width
|
||||
let imgH = cgImage.height
|
||||
let mousePt = NSEvent.mouseLocation
|
||||
let scale = CGFloat(imgW) / CGFloat(CGDisplayPixelsWide(displayID))
|
||||
let screenH = CGFloat(CGDisplayPixelsHigh(displayID)) / scale
|
||||
let cursorX = mousePt.x * scale
|
||||
let cursorY = (screenH - mousePt.y) * scale
|
||||
let cursor = NSCursor.current
|
||||
let cursorImg = cursor.image
|
||||
let hotspot = cursor.hotSpot
|
||||
let cw = cursorImg.size.width * scale
|
||||
let ch = cursorImg.size.height * scale
|
||||
let drawX = cursorX - hotspot.x * scale
|
||||
let drawY = cursorY - hotspot.y * scale
|
||||
let colorSpace = cgImage.colorSpace ?? CGColorSpaceCreateDeviceRGB()
|
||||
guard let ctx = CGContext(data: nil, width: imgW, height: imgH, bitsPerComponent: 8,
|
||||
bytesPerRow: 0, space: colorSpace,
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) else { return nil }
|
||||
ctx.draw(cgImage, in: CGRect(x: 0, y: 0, width: imgW, height: imgH))
|
||||
var cursorCGRect = NSRect(x: 0, y: 0, width: cursorImg.size.width, height: cursorImg.size.height)
|
||||
if let cursorCG = cursorImg.cgImage(forProposedRect: &cursorCGRect, context: nil, hints: nil) {
|
||||
ctx.draw(cursorCG, in: CGRect(x: drawX, y: CGFloat(imgH) - drawY - ch, width: cw, height: ch))
|
||||
}
|
||||
return ctx.makeImage()
|
||||
}
|
||||
|
||||
// MARK: - Input injection
|
||||
|
||||
private func injectInput(_ ev: [String: Any]) {
|
||||
guard let type = ev["type"] as? String else { return }
|
||||
let screen = NSScreen.main?.frame ?? CGRect(x: 0, y: 0, width: 1920, height: 1080)
|
||||
|
|
@ -34,24 +345,17 @@ class AppDelegate: FlutterAppDelegate {
|
|||
case "move":
|
||||
let nx = ev["x"] as? Double ?? 0
|
||||
let ny = ev["y"] as? Double ?? 0
|
||||
// CoreGraphics origin is top-left on screen, NSScreen origin is bottom-left
|
||||
let px = CGFloat(nx) * screen.width
|
||||
let py = CGFloat(ny) * screen.height
|
||||
let pt = CGPoint(x: px, y: py)
|
||||
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
||||
CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: pt, mouseButton: .left)?
|
||||
.post(tap: .cghidEventTap)
|
||||
|
||||
case "click":
|
||||
let nx = ev["x"] as? Double ?? 0
|
||||
let ny = ev["y"] as? Double ?? 0
|
||||
let nx = ev["x"] as? Double ?? 0
|
||||
let ny = ev["y"] as? Double ?? 0
|
||||
let btn = ev["button"] as? Int ?? 0
|
||||
let px = CGFloat(nx) * screen.width
|
||||
let py = CGFloat(ny) * screen.height
|
||||
let pt = CGPoint(x: px, y: py)
|
||||
|
||||
let pt = CGPoint(x: CGFloat(nx) * screen.width, y: CGFloat(ny) * screen.height)
|
||||
let (downType, upType, mouseBtn): (CGEventType, CGEventType, CGMouseButton) =
|
||||
btn == 1 ? (.rightMouseDown, .rightMouseUp, .right) : (.leftMouseDown, .leftMouseUp, .left)
|
||||
|
||||
CGEvent(mouseEventSource: nil, mouseType: downType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
||||
.post(tap: .cghidEventTap)
|
||||
CGEvent(mouseEventSource: nil, mouseType: upType, mouseCursorPosition: pt, mouseButton: mouseBtn)?
|
||||
|
|
@ -69,16 +373,71 @@ class AppDelegate: FlutterAppDelegate {
|
|||
.post(tap: .cghidEventTap)
|
||||
}
|
||||
|
||||
default:
|
||||
break
|
||||
default: break
|
||||
}
|
||||
}
|
||||
|
||||
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
|
||||
return true
|
||||
// MARK: - App lifecycle
|
||||
|
||||
override func applicationWillTerminate(_ notification: Notification) {
|
||||
stopEncoder()
|
||||
if #available(macOS 12.3, *), let stream = scStream as? SCStream {
|
||||
Task { try? await stream.stopCapture() }
|
||||
}
|
||||
}
|
||||
|
||||
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool {
|
||||
return true
|
||||
override func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||
override func applicationSupportsSecureRestorableState(_ app: NSApplication) -> Bool { true }
|
||||
}
|
||||
|
||||
// MARK: - VTCompressionSession wrapper (CF types can't be conditionally cast from Any?)
|
||||
|
||||
private class VTSessionBox {
|
||||
let session: VTCompressionSession
|
||||
init(_ s: VTCompressionSession) { session = s }
|
||||
}
|
||||
|
||||
// MARK: - VT output C callback
|
||||
|
||||
private func vtOutputCallback(
|
||||
outputCallbackRefCon: UnsafeMutableRawPointer?,
|
||||
sourceFrameRefCon: UnsafeMutableRawPointer?,
|
||||
status: OSStatus,
|
||||
infoFlags: VTEncodeInfoFlags,
|
||||
sampleBuffer: CMSampleBuffer?
|
||||
) {
|
||||
guard status == noErr, let sampleBuffer = sampleBuffer,
|
||||
let refcon = outputCallbackRefCon else { return }
|
||||
let delegate = Unmanaged<AppDelegate>.fromOpaque(refcon).takeUnretainedValue()
|
||||
delegate.handleEncodedFrame(sampleBuffer)
|
||||
}
|
||||
|
||||
// MARK: - SCK delegate
|
||||
|
||||
@available(macOS 12.3, *)
|
||||
class ScreenDelegate: NSObject, SCStreamOutput {
|
||||
private let onFrame: (CMSampleBuffer) -> Void
|
||||
|
||||
init(onFrame: @escaping (CMSampleBuffer) -> Void) {
|
||||
self.onFrame = onFrame
|
||||
}
|
||||
|
||||
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
|
||||
guard type == .screen else { return }
|
||||
onFrame(sampleBuffer)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - FlutterStreamHandler (AppDelegate handles frame events directly)
|
||||
|
||||
extension AppDelegate: FlutterStreamHandler {
|
||||
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? {
|
||||
frameEventSink = events
|
||||
return nil
|
||||
}
|
||||
|
||||
func onCancel(withArguments arguments: Any?) -> FlutterError? {
|
||||
frameEventSink = nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
|
|||
32
pubspec.lock
32
pubspec.lock
|
|
@ -77,10 +77,10 @@ packages:
|
|||
dependency: transitive
|
||||
description:
|
||||
name: dart_webrtc
|
||||
sha256: ac7ef077084b3e54004716f1d736fcd839e1b60bc3f21f4122a35a9bb5ca2e47
|
||||
sha256: f6d615bddea5e458ce180a914f3055c234ffb52fb7397a51b3491e76d6d7edb2
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.4.8"
|
||||
version: "1.8.1"
|
||||
fake_async:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
@ -127,10 +127,10 @@ packages:
|
|||
dependency: "direct main"
|
||||
description:
|
||||
name: flutter_webrtc
|
||||
sha256: "2f17fb96e0c9c6ff75f6b1c36d94755461fc7f36a5c28386f5ee5a18b98688c8"
|
||||
sha256: c7b0a67ca2c878575fc5c146d801cd874f58f5f1ef5fa6e8eb0c93d413beb948
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "0.9.48+hotfix.1"
|
||||
version: "1.4.1"
|
||||
glob:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
@ -203,6 +203,14 @@ packages:
|
|||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "6.1.0"
|
||||
logger:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: logger
|
||||
sha256: "25aee487596a6257655a1e091ec2ae66bc30e7af663592cc3a27e6591e05035c"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.7.0"
|
||||
logging:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
@ -323,14 +331,6 @@ packages:
|
|||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "3.1.6"
|
||||
platform_detect:
|
||||
dependency: transitive
|
||||
description:
|
||||
name: platform_detect
|
||||
sha256: "6e91e2158b9db7f6f4581a0f0b3a77171570b392108de196ba0c52e3c1976334"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.1.6"
|
||||
plugin_platform_interface:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
@ -444,18 +444,18 @@ packages:
|
|||
dependency: transitive
|
||||
description:
|
||||
name: web
|
||||
sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27"
|
||||
sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "0.5.1"
|
||||
version: "1.1.1"
|
||||
web_socket_channel:
|
||||
dependency: "direct main"
|
||||
description:
|
||||
name: web_socket_channel
|
||||
sha256: "58c6666b342a38816b2e7e50ed0f1e261959630becd4c879c4f26bfa14aa5a42"
|
||||
sha256: d88238e5eac9a42bb43ca4e721edba3c08c6354d4a53063afaa568516217621b
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "2.4.5"
|
||||
version: "2.4.0"
|
||||
webrtc_interface:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ dependencies:
|
|||
# The following adds the Cupertino Icons font to your application.
|
||||
# Use with the CupertinoIcons class for iOS style icons.
|
||||
cupertino_icons: ^1.0.8
|
||||
flutter_webrtc: ^0.9.47
|
||||
flutter_webrtc: ^1.4.1
|
||||
web_socket_channel: ^2.4.0
|
||||
|
||||
dev_dependencies:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue