implement video encoding and decoding functionality with screen capture support
This commit is contained in:
ImBenji
parent
bfdaf4a801
commit
37fef7de25
10 changed files with 1163 additions and 48 deletions
|
|
@ -25,11 +25,16 @@ class AppDelegate: FlutterAppDelegate {
|
|||
private var captureWidth = 1280
|
||||
private var captureHeight = 720
|
||||
|
||||
// viewer decoder sessions keyed by texture id
|
||||
private var decoders: [Int64: VideoDecoderSession] = [:]
|
||||
private var textureRegistry: FlutterTextureRegistry?
|
||||
|
||||
override func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
guard let window = mainFlutterWindow,
|
||||
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
||||
|
||||
let messenger = ctrl.engine.binaryMessenger
|
||||
textureRegistry = ctrl.engine.textureRegistry
|
||||
|
||||
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
||||
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
||||
|
|
@ -58,6 +63,40 @@ class AppDelegate: FlutterAppDelegate {
|
|||
case "forceKeyframe":
|
||||
self.requestKeyframe()
|
||||
result(nil)
|
||||
|
||||
case "createDecoder":
|
||||
guard let args = call.arguments as? [String: Any],
|
||||
let spsPpsTyped = args["spsPps"] as? FlutterStandardTypedData,
|
||||
let registry = self.textureRegistry else {
|
||||
result(FlutterError(code: "INVALID_ARGS", message: nil, details: nil))
|
||||
return
|
||||
}
|
||||
if let decoder = VideoDecoderSession(spsPps: spsPpsTyped.data, textureRegistry: registry) {
|
||||
self.decoders[decoder.textureId] = decoder
|
||||
result(["textureId": decoder.textureId])
|
||||
} else {
|
||||
result(FlutterError(code: "DECODER_FAILED", message: "Failed to create H264 decoder", details: nil))
|
||||
}
|
||||
|
||||
case "feedNal":
|
||||
guard let args = call.arguments as? [String: Any],
|
||||
let textureId = (args["textureId"] as? NSNumber)?.int64Value,
|
||||
let nalTyped = args["nal"] as? FlutterStandardTypedData,
|
||||
let decoder = self.decoders[textureId] else {
|
||||
result(nil)
|
||||
return
|
||||
}
|
||||
decoder.feedNal(nalTyped.data)
|
||||
result(nil)
|
||||
|
||||
case "stopDecoder":
|
||||
if let args = call.arguments as? [String: Any],
|
||||
let textureId = (args["textureId"] as? NSNumber)?.int64Value {
|
||||
self.decoders[textureId]?.invalidate()
|
||||
self.decoders.removeValue(forKey: textureId)
|
||||
}
|
||||
result(nil)
|
||||
|
||||
default:
|
||||
result(FlutterMethodNotImplemented)
|
||||
}
|
||||
|
|
@ -441,3 +480,206 @@ extension AppDelegate: FlutterStreamHandler {
|
|||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - VT decompression C callback
|
||||
|
||||
private func vtDecompressOutputCallback(
|
||||
refCon: UnsafeMutableRawPointer?,
|
||||
frameRefCon: UnsafeMutableRawPointer?,
|
||||
status: OSStatus,
|
||||
infoFlags: VTDecodeInfoFlags,
|
||||
imageBuffer: CVImageBuffer?,
|
||||
pts: CMTime,
|
||||
duration: CMTime
|
||||
) {
|
||||
guard let refCon = refCon, status == noErr, let imageBuffer = imageBuffer else { return }
|
||||
let session = Unmanaged<VideoDecoderSession>.fromOpaque(refCon).takeUnretainedValue()
|
||||
session.handleDecodedFrame(imageBuffer)
|
||||
}
|
||||
|
||||
// MARK: - VideoDecoderSession
|
||||
|
||||
class VideoDecoderSession: NSObject, FlutterTexture {
|
||||
private(set) var textureId: Int64 = -1
|
||||
|
||||
private var decompressionSession: VTDecompressionSession?
|
||||
private var formatDescription: CMVideoFormatDescription?
|
||||
private var latestPixelBuffer: CVPixelBuffer?
|
||||
private let lock = NSLock()
|
||||
private weak var textureRegistry: FlutterTextureRegistry?
|
||||
|
||||
init?(spsPps: Data, textureRegistry: FlutterTextureRegistry) {
|
||||
self.textureRegistry = textureRegistry
|
||||
super.init()
|
||||
guard setupSession(spsPps: spsPps) else { return nil }
|
||||
textureId = textureRegistry.register(self)
|
||||
}
|
||||
|
||||
// MARK: Parse Annex-B SPS+PPS and create VTDecompressionSession
|
||||
|
||||
private func setupSession(spsPps: Data) -> Bool {
|
||||
// Split on 4-byte Annex-B start codes to get raw NALU payloads
|
||||
let bytes = [UInt8](spsPps)
|
||||
var nalus: [Data] = []
|
||||
var scanStart = 0
|
||||
var i = 0
|
||||
while i + 4 <= bytes.count {
|
||||
if bytes[i] == 0 && bytes[i+1] == 0 && bytes[i+2] == 0 && bytes[i+3] == 1 {
|
||||
if i > scanStart { nalus.append(Data(bytes[scanStart..<i])) }
|
||||
scanStart = i + 4
|
||||
i += 4
|
||||
} else {
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
if scanStart < bytes.count { nalus.append(Data(bytes[scanStart...])) }
|
||||
guard nalus.count >= 2 else { return false }
|
||||
|
||||
let sps = nalus[0]
|
||||
let pps = nalus[1]
|
||||
|
||||
var formatDesc: CMVideoFormatDescription?
|
||||
let fmtStatus: OSStatus = sps.withUnsafeBytes { spsBuf in
|
||||
pps.withUnsafeBytes { ppsBuf in
|
||||
var ptrs: [UnsafePointer<UInt8>] = [
|
||||
spsBuf.bindMemory(to: UInt8.self).baseAddress!,
|
||||
ppsBuf.bindMemory(to: UInt8.self).baseAddress!
|
||||
]
|
||||
var sizes: [Int] = [sps.count, pps.count]
|
||||
return CMVideoFormatDescriptionCreateFromH264ParameterSets(
|
||||
allocator: kCFAllocatorDefault,
|
||||
parameterSetCount: 2,
|
||||
parameterSetPointers: &ptrs,
|
||||
parameterSetSizes: &sizes,
|
||||
nalUnitHeaderLength: 4,
|
||||
formatDescriptionOut: &formatDesc
|
||||
)
|
||||
}
|
||||
}
|
||||
guard fmtStatus == noErr, let formatDesc = formatDesc else { return false }
|
||||
self.formatDescription = formatDesc
|
||||
|
||||
let dims = CMVideoFormatDescriptionGetDimensions(formatDesc)
|
||||
let imageAttrs: [CFString: Any] = [
|
||||
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey: Int(dims.width),
|
||||
kCVPixelBufferHeightKey: Int(dims.height),
|
||||
kCVPixelBufferIOSurfacePropertiesKey: [:] as [String: Any]
|
||||
]
|
||||
|
||||
var callback = VTDecompressionOutputCallbackRecord(
|
||||
decompressionOutputCallback: vtDecompressOutputCallback,
|
||||
decompressionOutputRefCon: Unmanaged.passUnretained(self).toOpaque()
|
||||
)
|
||||
|
||||
var session: VTDecompressionSession?
|
||||
let sessionStatus = VTDecompressionSessionCreate(
|
||||
allocator: kCFAllocatorDefault,
|
||||
formatDescription: formatDesc,
|
||||
decoderSpecification: nil,
|
||||
imageBufferAttributes: imageAttrs as CFDictionary,
|
||||
outputCallback: &callback,
|
||||
decompressionSessionOut: &session
|
||||
)
|
||||
guard sessionStatus == noErr, let session = session else { return false }
|
||||
self.decompressionSession = session
|
||||
return true
|
||||
}
|
||||
|
||||
// MARK: Feed an Annex-B NAL unit for decoding
|
||||
|
||||
func feedNal(_ data: Data) {
|
||||
guard let session = decompressionSession,
|
||||
let formatDesc = formatDescription,
|
||||
data.count > 4 else { return }
|
||||
|
||||
// Convert Annex-B start code to AVCC 4-byte big-endian length prefix
|
||||
let naluLen = UInt32(data.count - 4).bigEndian
|
||||
var avcc = Data(capacity: data.count)
|
||||
withUnsafeBytes(of: naluLen) { avcc.append(contentsOf: $0) }
|
||||
avcc.append(data.advanced(by: 4))
|
||||
|
||||
var blockBuffer: CMBlockBuffer?
|
||||
guard CMBlockBufferCreateWithMemoryBlock(
|
||||
allocator: kCFAllocatorDefault,
|
||||
memoryBlock: nil,
|
||||
blockLength: avcc.count,
|
||||
blockAllocator: kCFAllocatorDefault,
|
||||
customBlockSource: nil,
|
||||
offsetToData: 0,
|
||||
dataLength: avcc.count,
|
||||
flags: 0,
|
||||
blockBufferOut: &blockBuffer
|
||||
) == kCMBlockBufferNoErr, let blockBuffer = blockBuffer else { return }
|
||||
|
||||
avcc.withUnsafeBytes { ptr in
|
||||
_ = CMBlockBufferReplaceDataBytes(
|
||||
with: ptr.baseAddress!,
|
||||
blockBuffer: blockBuffer,
|
||||
offsetIntoDestination: 0,
|
||||
dataLength: avcc.count
|
||||
)
|
||||
}
|
||||
|
||||
var timingInfo = CMSampleTimingInfo(
|
||||
duration: CMTime(value: 1, timescale: 60),
|
||||
presentationTimeStamp: CMTime(seconds: CACurrentMediaTime(), preferredTimescale: 600),
|
||||
decodeTimeStamp: .invalid
|
||||
)
|
||||
var sampleBuffer: CMSampleBuffer?
|
||||
guard CMSampleBufferCreate(
|
||||
allocator: kCFAllocatorDefault,
|
||||
dataBuffer: blockBuffer,
|
||||
dataReady: true,
|
||||
makeDataReadyCallback: nil,
|
||||
refcon: nil,
|
||||
formatDescription: formatDesc,
|
||||
sampleCount: 1,
|
||||
sampleTimingEntryCount: 1,
|
||||
sampleTimingArray: &timingInfo,
|
||||
sampleSizeEntryCount: 0,
|
||||
sampleSizeArray: nil,
|
||||
sampleBufferOut: &sampleBuffer
|
||||
) == noErr, let sampleBuffer = sampleBuffer else { return }
|
||||
|
||||
VTDecompressionSessionDecodeFrame(
|
||||
session,
|
||||
sampleBuffer: sampleBuffer,
|
||||
flags: [],
|
||||
frameRefcon: nil,
|
||||
infoFlagsOut: nil
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: Called from the VT C callback
|
||||
|
||||
func handleDecodedFrame(_ imageBuffer: CVImageBuffer) {
|
||||
lock.lock()
|
||||
latestPixelBuffer = imageBuffer as CVPixelBuffer
|
||||
lock.unlock()
|
||||
DispatchQueue.main.async { [weak self] in
|
||||
guard let self = self else { return }
|
||||
self.textureRegistry?.textureFrameAvailable(self.textureId)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: FlutterTexture
|
||||
|
||||
func copyPixelBuffer() -> Unmanaged<CVPixelBuffer>? {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
guard let pb = latestPixelBuffer else { return nil }
|
||||
return Unmanaged.passRetained(pb)
|
||||
}
|
||||
|
||||
// MARK: Teardown
|
||||
|
||||
func invalidate() {
|
||||
if let session = decompressionSession {
|
||||
VTDecompressionSessionInvalidate(session)
|
||||
}
|
||||
decompressionSession = nil
|
||||
latestPixelBuffer = nil
|
||||
textureRegistry?.unregisterTexture(textureId)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -239,10 +239,10 @@ packages:
|
|||
dependency: transitive
|
||||
description:
|
||||
name: meta
|
||||
sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349"
|
||||
sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
|
||||
url: "https://pub.dev"
|
||||
source: hosted
|
||||
version: "1.18.0"
|
||||
version: "1.17.0"
|
||||
native_toolchain_c:
|
||||
dependency: transitive
|
||||
description:
|
||||
|
|
@ -481,5 +481,5 @@ packages:
|
|||
source: hosted
|
||||
version: "3.1.3"
|
||||
sdks:
|
||||
dart: ">=3.12.0-210.1.beta <4.0.0"
|
||||
dart: ">=3.11.0 <4.0.0"
|
||||
flutter: ">=3.38.4"
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
|
|||
version: 1.0.0+1
|
||||
|
||||
environment:
|
||||
sdk: ^3.12.0-210.1.beta
|
||||
sdk: ^3.11.0
|
||||
|
||||
# Dependencies specify other packages that your package needs in order to work.
|
||||
# To automatically upgrade your package dependencies to the latest versions
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ add_executable(${BINARY_NAME} WIN32
|
|||
"main.cpp"
|
||||
"utils.cpp"
|
||||
"win32_window.cpp"
|
||||
"screen_encoder.cpp"
|
||||
"video_decoder.cpp"
|
||||
"${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc"
|
||||
"Runner.rc"
|
||||
"runner.exe.manifest"
|
||||
|
|
@ -34,6 +36,14 @@ target_compile_definitions(${BINARY_NAME} PRIVATE "NOMINMAX")
|
|||
# dependencies here.
|
||||
target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app)
|
||||
target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib")
|
||||
target_link_libraries(${BINARY_NAME} PRIVATE
|
||||
"d3d11.lib"
|
||||
"dxgi.lib"
|
||||
"mf.lib"
|
||||
"mfplat.lib"
|
||||
"mfuuid.lib"
|
||||
"propsys.lib"
|
||||
)
|
||||
target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}")
|
||||
|
||||
# Run the Flutter tool portions of the build. This must not be removed.
|
||||
|
|
|
|||
|
|
@ -3,10 +3,15 @@
|
|||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include <flutter/event_channel.h>
|
||||
#include <flutter/event_stream_handler_functions.h>
|
||||
#include <flutter/method_channel.h>
|
||||
#include <flutter/standard_method_codec.h>
|
||||
#include <flutter/encodable_value.h>
|
||||
#include <windows.h>
|
||||
#include <mfapi.h>
|
||||
#include <flutter_plugin_registrar.h>
|
||||
#include <flutter_texture_registrar.h>
|
||||
|
||||
#include "flutter/generated_plugin_registrant.h"
|
||||
|
||||
|
|
@ -15,15 +20,14 @@ FlutterWindow::FlutterWindow(const flutter::DartProject& project)
|
|||
|
||||
FlutterWindow::~FlutterWindow() {}
|
||||
|
||||
// ─── Input injection helper ──────────────────────────────────────────────────
|
||||
|
||||
static void HandleInjectInput(const flutter::EncodableMap& args) {
|
||||
auto typeIt = args.find(flutter::EncodableValue("type"));
|
||||
if (typeIt == args.end()) return;
|
||||
|
||||
std::string type = std::get<std::string>(typeIt->second);
|
||||
|
||||
int screenW = GetSystemMetrics(SM_CXSCREEN);
|
||||
int screenH = GetSystemMetrics(SM_CYSCREEN);
|
||||
|
||||
if (type == "move") {
|
||||
double nx = std::get<double>(args.at(flutter::EncodableValue("x")));
|
||||
double ny = std::get<double>(args.at(flutter::EncodableValue("y")));
|
||||
|
|
@ -42,7 +46,6 @@ static void HandleInjectInput(const flutter::EncodableMap& args) {
|
|||
auto btnIt = args.find(flutter::EncodableValue("button"));
|
||||
if (btnIt != args.end()) btn = std::get<int>(btnIt->second);
|
||||
|
||||
// move first
|
||||
INPUT moveInput = {};
|
||||
moveInput.type = INPUT_MOUSE;
|
||||
moveInput.mi.dx = (LONG)(nx * 65535);
|
||||
|
|
@ -53,7 +56,6 @@ static void HandleInjectInput(const flutter::EncodableMap& args) {
|
|||
INPUT clickInput[2] = {};
|
||||
clickInput[0].type = INPUT_MOUSE;
|
||||
clickInput[1].type = INPUT_MOUSE;
|
||||
|
||||
if (btn == 1) {
|
||||
clickInput[0].mi.dwFlags = MOUSEEVENTF_RIGHTDOWN;
|
||||
clickInput[1].mi.dwFlags = MOUSEEVENTF_RIGHTUP;
|
||||
|
|
@ -75,41 +77,149 @@ static void HandleInjectInput(const flutter::EncodableMap& args) {
|
|||
keyInput.type = INPUT_KEYBOARD;
|
||||
keyInput.ki.wVk = vk;
|
||||
if (type == "keyup") keyInput.ki.dwFlags = KEYEVENTF_KEYUP;
|
||||
|
||||
SendInput(1, &keyInput, sizeof(INPUT));
|
||||
}
|
||||
}
|
||||
|
||||
// ─── OnCreate ────────────────────────────────────────────────────────────────
|
||||
|
||||
bool FlutterWindow::OnCreate() {
|
||||
if (!Win32Window::OnCreate()) {
|
||||
return false;
|
||||
}
|
||||
if (!Win32Window::OnCreate()) return false;
|
||||
|
||||
MFStartup(MF_VERSION);
|
||||
|
||||
RECT frame = GetClientArea();
|
||||
|
||||
flutter_controller_ = std::make_unique<flutter::FlutterViewController>(
|
||||
frame.right - frame.left, frame.bottom - frame.top, project_);
|
||||
|
||||
if (!flutter_controller_->engine() || !flutter_controller_->view()) {
|
||||
if (!flutter_controller_->engine() || !flutter_controller_->view())
|
||||
return false;
|
||||
}
|
||||
|
||||
RegisterPlugins(flutter_controller_->engine());
|
||||
|
||||
// register input injection channel
|
||||
auto channel = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
||||
flutter_controller_->engine()->messenger(),
|
||||
"com.pulsar/input",
|
||||
&flutter::StandardMethodCodec::GetInstance()
|
||||
);
|
||||
auto* messenger = flutter_controller_->engine()->messenger();
|
||||
|
||||
channel->SetMethodCallHandler(
|
||||
// Get the texture registrar via the C API (avoids linking flutter_wrapper_plugin).
|
||||
auto reg_ref = flutter_controller_->engine()->GetRegistrarForPlugin("pulsar");
|
||||
tex_reg_ = FlutterDesktopRegistrarGetTextureRegistrar(reg_ref);
|
||||
|
||||
// ── com.pulsar/input MethodChannel ──────────────────────────────────────
|
||||
auto input_ch = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
||||
messenger, "com.pulsar/input",
|
||||
&flutter::StandardMethodCodec::GetInstance());
|
||||
|
||||
input_ch->SetMethodCallHandler(
|
||||
[](const flutter::MethodCall<flutter::EncodableValue>& call,
|
||||
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
||||
if (call.method_name() == "injectInput") {
|
||||
if (const auto* args = std::get_if<flutter::EncodableMap>(call.arguments())) {
|
||||
if (const auto* args =
|
||||
std::get_if<flutter::EncodableMap>(call.arguments()))
|
||||
HandleInjectInput(*args);
|
||||
result->Success();
|
||||
} else {
|
||||
result->NotImplemented();
|
||||
}
|
||||
});
|
||||
|
||||
// ── com.pulsar/video/frames EventChannel ────────────────────────────────
|
||||
screen_encoder_ = std::make_unique<ScreenEncoder>();
|
||||
auto* enc = screen_encoder_.get(); // raw ptr safe: outlives channels
|
||||
|
||||
auto frame_ch = std::make_shared<flutter::EventChannel<flutter::EncodableValue>>(
|
||||
messenger, "com.pulsar/video/frames",
|
||||
&flutter::StandardMethodCodec::GetInstance());
|
||||
|
||||
frame_ch->SetStreamHandler(
|
||||
std::make_unique<flutter::StreamHandlerFunctions<flutter::EncodableValue>>(
|
||||
[enc](const flutter::EncodableValue*,
|
||||
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& sink)
|
||||
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
||||
enc->Start(std::move(sink));
|
||||
return nullptr;
|
||||
},
|
||||
[enc](const flutter::EncodableValue*)
|
||||
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
||||
enc->Stop();
|
||||
return nullptr;
|
||||
}));
|
||||
|
||||
// ── com.pulsar/video MethodChannel ──────────────────────────────────────
|
||||
auto* decoders = &decoders_; // raw ptr safe: outlives channels
|
||||
auto video_ch = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
||||
messenger, "com.pulsar/video",
|
||||
&flutter::StandardMethodCodec::GetInstance());
|
||||
|
||||
video_ch->SetMethodCallHandler(
|
||||
[enc, decoders, this](
|
||||
const flutter::MethodCall<flutter::EncodableValue>& call,
|
||||
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
||||
const auto& m = call.method_name();
|
||||
|
||||
if (m == "start") {
|
||||
result->Success();
|
||||
|
||||
} else if (m == "stop") {
|
||||
enc->Stop();
|
||||
result->Success();
|
||||
|
||||
} else if (m == "forceKeyframe") {
|
||||
enc->ForceKeyframe();
|
||||
result->Success();
|
||||
|
||||
} else if (m == "createDecoder") {
|
||||
const auto* args =
|
||||
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||
if (!args) { result->Error("BAD_ARGS", ""); return; }
|
||||
|
||||
auto it = args->find(flutter::EncodableValue("spsPps"));
|
||||
if (it == args->end()) {
|
||||
result->Error("BAD_ARGS", "missing spsPps");
|
||||
return;
|
||||
}
|
||||
const auto& sps_pps =
|
||||
std::get<std::vector<uint8_t>>(it->second);
|
||||
|
||||
auto dec = std::make_unique<VideoDecoder>(
|
||||
tex_reg_, sps_pps.data(), sps_pps.size());
|
||||
if (!dec->is_valid()) {
|
||||
result->Error("INIT_FAILED", "decoder init failed");
|
||||
return;
|
||||
}
|
||||
int64_t id = dec->texture_id();
|
||||
(*decoders)[id] = std::move(dec);
|
||||
result->Success(flutter::EncodableValue(flutter::EncodableMap{
|
||||
{flutter::EncodableValue("textureId"),
|
||||
flutter::EncodableValue(id)}}));
|
||||
|
||||
} else if (m == "stopDecoder") {
|
||||
const auto* args =
|
||||
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||
if (args) {
|
||||
auto it = args->find(flutter::EncodableValue("textureId"));
|
||||
if (it != args->end())
|
||||
decoders->erase(std::get<int64_t>(it->second));
|
||||
}
|
||||
result->Success();
|
||||
|
||||
} else if (m == "feedNal") {
|
||||
const auto* args =
|
||||
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||
if (!args) { result->Error("BAD_ARGS", ""); return; }
|
||||
|
||||
auto id_it = args->find(flutter::EncodableValue("textureId"));
|
||||
auto nal_it = args->find(flutter::EncodableValue("nal"));
|
||||
if (id_it == args->end() || nal_it == args->end()) {
|
||||
result->Error("BAD_ARGS", "missing args");
|
||||
return;
|
||||
}
|
||||
int64_t id = std::get<int64_t>(id_it->second);
|
||||
const auto& nal = std::get<std::vector<uint8_t>>(nal_it->second);
|
||||
|
||||
auto dit = decoders->find(id);
|
||||
if (dit != decoders->end())
|
||||
dit->second->FeedNal(nal.data(), nal.size());
|
||||
result->Success();
|
||||
|
||||
} else {
|
||||
result->NotImplemented();
|
||||
}
|
||||
|
|
@ -117,27 +227,31 @@ bool FlutterWindow::OnCreate() {
|
|||
|
||||
SetChildContent(flutter_controller_->view()->GetNativeWindow());
|
||||
|
||||
flutter_controller_->engine()->SetNextFrameCallback([&]() {
|
||||
this->Show();
|
||||
});
|
||||
flutter_controller_->engine()->SetNextFrameCallback([&]() { this->Show(); });
|
||||
flutter_controller_->ForceRedraw();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── OnDestroy ───────────────────────────────────────────────────────────────
|
||||
|
||||
void FlutterWindow::OnDestroy() {
|
||||
if (flutter_controller_) {
|
||||
flutter_controller_ = nullptr;
|
||||
}
|
||||
decoders_.clear();
|
||||
screen_encoder_.reset();
|
||||
if (flutter_controller_) flutter_controller_ = nullptr;
|
||||
|
||||
MFShutdown();
|
||||
Win32Window::OnDestroy();
|
||||
}
|
||||
|
||||
// ─── MessageHandler ──────────────────────────────────────────────────────────
|
||||
|
||||
LRESULT FlutterWindow::MessageHandler(HWND hwnd, UINT const message,
|
||||
WPARAM const wparam,
|
||||
LPARAM const lparam) noexcept {
|
||||
if (flutter_controller_) {
|
||||
std::optional<LRESULT> result =
|
||||
flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam, lparam);
|
||||
flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam,
|
||||
lparam);
|
||||
if (result) return *result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,31 +3,34 @@
|
|||
|
||||
#include <flutter/dart_project.h>
|
||||
#include <flutter/flutter_view_controller.h>
|
||||
#include <flutter_plugin_registrar.h>
|
||||
#include <flutter_texture_registrar.h>
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
#include "screen_encoder.h"
|
||||
#include "video_decoder.h"
|
||||
#include "win32_window.h"
|
||||
|
||||
// A window that does nothing but host a Flutter view.
|
||||
class FlutterWindow : public Win32Window {
|
||||
public:
|
||||
// Creates a new FlutterWindow hosting a Flutter view running |project|.
|
||||
explicit FlutterWindow(const flutter::DartProject& project);
|
||||
virtual ~FlutterWindow();
|
||||
|
||||
protected:
|
||||
// Win32Window:
|
||||
bool OnCreate() override;
|
||||
void OnDestroy() override;
|
||||
LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam,
|
||||
LPARAM const lparam) noexcept override;
|
||||
|
||||
private:
|
||||
// The project to run.
|
||||
flutter::DartProject project_;
|
||||
|
||||
// The Flutter instance hosted by this window.
|
||||
std::unique_ptr<flutter::FlutterViewController> flutter_controller_;
|
||||
|
||||
FlutterDesktopTextureRegistrarRef tex_reg_ = nullptr;
|
||||
std::unique_ptr<ScreenEncoder> screen_encoder_;
|
||||
std::map<int64_t, std::unique_ptr<VideoDecoder>> decoders_;
|
||||
};
|
||||
|
||||
#endif // RUNNER_FLUTTER_WINDOW_H_
|
||||
|
|
|
|||
361
windows/runner/screen_encoder.cpp
Normal file
361
windows/runner/screen_encoder.cpp
Normal file
|
|
@ -0,0 +1,361 @@
|
|||
#include "screen_encoder.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <codecapi.h>
|
||||
#include <mfapi.h>
|
||||
#include <mferror.h>
|
||||
#include <mfidl.h>
|
||||
|
||||
using Microsoft::WRL::ComPtr;
|
||||
|
||||
static constexpr LONGLONG kFrameDuration = 333333; // 100ns units ≈ 30fps
|
||||
static constexpr UINT32 kBitrate = 6'000'000;
|
||||
|
||||
// Scan an Annex-B bitstream for the span covering the SPS (type 7) through
|
||||
// the end of the PPS (type 8). Returns false if either is missing.
|
||||
static bool ExtractSpsPps(const uint8_t* data, size_t size,
|
||||
size_t& sps_start, size_t& pps_end) {
|
||||
struct Nal { size_t start; int type; };
|
||||
std::vector<Nal> nals;
|
||||
|
||||
size_t i = 0;
|
||||
while (i + 4 <= size) {
|
||||
size_t hdr_len = 0;
|
||||
int nal_type = 0;
|
||||
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 &&
|
||||
i + 4 < size) {
|
||||
hdr_len = 4;
|
||||
nal_type = data[i+4] & 0x1F;
|
||||
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 &&
|
||||
i + 3 < size) {
|
||||
hdr_len = 3;
|
||||
nal_type = data[i+3] & 0x1F;
|
||||
}
|
||||
if (hdr_len) {
|
||||
nals.push_back({i, nal_type});
|
||||
i += hdr_len;
|
||||
} else {
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
size_t sps_idx = SIZE_MAX, pps_idx = SIZE_MAX;
|
||||
for (size_t j = 0; j < nals.size(); ++j) {
|
||||
if (nals[j].type == 7) sps_idx = j;
|
||||
if (nals[j].type == 8) pps_idx = j;
|
||||
}
|
||||
if (sps_idx == SIZE_MAX || pps_idx == SIZE_MAX) return false;
|
||||
|
||||
sps_start = nals[sps_idx].start;
|
||||
size_t after_pps = pps_idx + 1;
|
||||
pps_end = (after_pps < nals.size()) ? nals[after_pps].start : size;
|
||||
return sps_start < pps_end;
|
||||
}
|
||||
|
||||
// ─── ScreenEncoder ───────────────────────────────────────────────────────────
|
||||
|
||||
ScreenEncoder::ScreenEncoder() {}
|
||||
|
||||
ScreenEncoder::~ScreenEncoder() { Stop(); }
|
||||
|
||||
void ScreenEncoder::Start(
|
||||
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink) {
|
||||
Stop();
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||
sink_ = std::move(sink);
|
||||
}
|
||||
config_sent_ = false;
|
||||
sample_ts_ = 0;
|
||||
running_ = true;
|
||||
thread_ = std::thread(&ScreenEncoder::CaptureLoop, this);
|
||||
}
|
||||
|
||||
void ScreenEncoder::Stop() {
|
||||
running_ = false;
|
||||
if (thread_.joinable()) thread_.join();
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||
sink_.reset();
|
||||
}
|
||||
encoder_.Reset();
|
||||
dupl_.Reset();
|
||||
staging_.Reset();
|
||||
d3d_ctx_.Reset();
|
||||
d3d_dev_.Reset();
|
||||
enc_width_ = enc_height_ = 0;
|
||||
}
|
||||
|
||||
void ScreenEncoder::ForceKeyframe() { force_kf_ = true; }
|
||||
|
||||
// ─── D3D / DXGI init ─────────────────────────────────────────────────────────
|
||||
|
||||
bool ScreenEncoder::InitD3D() {
|
||||
D3D_FEATURE_LEVEL level;
|
||||
HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0,
|
||||
nullptr, 0, D3D11_SDK_VERSION,
|
||||
&d3d_dev_, &level, &d3d_ctx_);
|
||||
if (FAILED(hr)) return false;
|
||||
|
||||
ComPtr<IDXGIDevice> dxgi_dev;
|
||||
ComPtr<IDXGIAdapter> adapter;
|
||||
ComPtr<IDXGIOutput> output;
|
||||
ComPtr<IDXGIOutput1> output1;
|
||||
|
||||
d3d_dev_.As(&dxgi_dev);
|
||||
dxgi_dev->GetAdapter(&adapter);
|
||||
if (FAILED(adapter->EnumOutputs(0, &output))) return false;
|
||||
if (FAILED(output.As(&output1))) return false;
|
||||
|
||||
hr = output1->DuplicateOutput(d3d_dev_.Get(), &dupl_);
|
||||
return SUCCEEDED(hr);
|
||||
}
|
||||
|
||||
// ─── Encoder init ────────────────────────────────────────────────────────────
|
||||
|
||||
bool ScreenEncoder::InitEncoder(UINT width, UINT height) {
|
||||
MFT_REGISTER_TYPE_INFO out_info{MFMediaType_Video, MFVideoFormat_H264};
|
||||
|
||||
UINT32 count = 0;
|
||||
IMFActivate** activates = nullptr;
|
||||
HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER,
|
||||
MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER,
|
||||
nullptr, &out_info, &activates, &count);
|
||||
if (FAILED(hr) || count == 0) return false;
|
||||
|
||||
hr = activates[0]->ActivateObject(IID_PPV_ARGS(&encoder_));
|
||||
for (UINT32 i = 0; i < count; ++i) activates[i]->Release();
|
||||
CoTaskMemFree(activates);
|
||||
if (FAILED(hr)) return false;
|
||||
|
||||
// Output: H264
|
||||
ComPtr<IMFMediaType> out_type;
|
||||
MFCreateMediaType(&out_type);
|
||||
out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
|
||||
MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, width, height);
|
||||
MFSetAttributeRatio(out_type.Get(), MF_MT_FRAME_RATE, 30, 1);
|
||||
out_type->SetUINT32(MF_MT_AVG_BITRATE, kBitrate);
|
||||
out_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
|
||||
out_type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High);
|
||||
if (FAILED(encoder_->SetOutputType(0, out_type.Get(), 0))) return false;
|
||||
|
||||
// Input: NV12
|
||||
ComPtr<IMFMediaType> in_type;
|
||||
MFCreateMediaType(&in_type);
|
||||
in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
|
||||
MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, width, height);
|
||||
MFSetAttributeRatio(in_type.Get(), MF_MT_FRAME_RATE, 30, 1);
|
||||
in_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
|
||||
in_type->SetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32)width);
|
||||
if (FAILED(encoder_->SetInputType(0, in_type.Get(), 0))) return false;
|
||||
|
||||
// Keyframe every 30 frames via media type attribute
|
||||
out_type->SetUINT32(MF_MT_MAX_KEYFRAME_SPACING, 30);
|
||||
|
||||
encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0);
|
||||
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
|
||||
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
|
||||
|
||||
enc_width_ = width;
|
||||
enc_height_ = height;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Frame capture ───────────────────────────────────────────────────────────
|
||||
|
||||
bool ScreenEncoder::CaptureFrame(std::vector<uint8_t>& bgra,
|
||||
UINT& width, UINT& height) {
|
||||
DXGI_OUTDUPL_FRAME_INFO info{};
|
||||
ComPtr<IDXGIResource> res;
|
||||
|
||||
HRESULT hr = dupl_->AcquireNextFrame(16, &info, &res);
|
||||
if (hr == DXGI_ERROR_WAIT_TIMEOUT) return false;
|
||||
if (FAILED(hr)) {
|
||||
dupl_.Reset();
|
||||
staging_.Reset();
|
||||
InitD3D();
|
||||
return false;
|
||||
}
|
||||
|
||||
ComPtr<ID3D11Texture2D> tex;
|
||||
res.As(&tex);
|
||||
|
||||
D3D11_TEXTURE2D_DESC desc{};
|
||||
tex->GetDesc(&desc);
|
||||
width = desc.Width;
|
||||
height = desc.Height;
|
||||
|
||||
// Recreate staging texture on size change
|
||||
if (!staging_) {
|
||||
D3D11_TEXTURE2D_DESC sd{};
|
||||
sd.Width = width;
|
||||
sd.Height = height;
|
||||
sd.MipLevels = 1;
|
||||
sd.ArraySize = 1;
|
||||
sd.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||
sd.SampleDesc.Count = 1;
|
||||
sd.Usage = D3D11_USAGE_STAGING;
|
||||
sd.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
|
||||
d3d_dev_->CreateTexture2D(&sd, nullptr, &staging_);
|
||||
}
|
||||
|
||||
d3d_ctx_->CopyResource(staging_.Get(), tex.Get());
|
||||
dupl_->ReleaseFrame();
|
||||
|
||||
D3D11_MAPPED_SUBRESOURCE mapped{};
|
||||
hr = d3d_ctx_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped);
|
||||
if (FAILED(hr)) return false;
|
||||
|
||||
bgra.resize(width * height * 4);
|
||||
const uint8_t* src = static_cast<const uint8_t*>(mapped.pData);
|
||||
for (UINT row = 0; row < height; ++row)
|
||||
memcpy(&bgra[row * width * 4], src + row * mapped.RowPitch, width * 4);
|
||||
|
||||
d3d_ctx_->Unmap(staging_.Get(), 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Color conversion: BGRA → NV12 ──────────────────────────────────────────
|
||||
|
||||
void ScreenEncoder::BgraToNv12(const uint8_t* bgra, std::vector<uint8_t>& nv12,
|
||||
UINT w, UINT h) {
|
||||
nv12.resize(w * h * 3 / 2);
|
||||
uint8_t* Y = nv12.data();
|
||||
uint8_t* UV = Y + w * h;
|
||||
|
||||
for (UINT row = 0; row < h; ++row) {
|
||||
for (UINT col = 0; col < w; ++col) {
|
||||
const uint8_t* p = bgra + (row * w + col) * 4;
|
||||
const int b = p[0], g = p[1], r = p[2];
|
||||
Y[row * w + col] =
|
||||
(uint8_t)(((66*r + 129*g + 25*b + 128) >> 8) + 16);
|
||||
|
||||
if ((row & 1) == 0 && (col & 1) == 0) {
|
||||
const UINT off = (row / 2) * w + col;
|
||||
UV[off] = (uint8_t)(((-38*r - 74*g + 112*b + 128) >> 8) + 128);
|
||||
UV[off+1] = (uint8_t)(((112*r - 94*g - 18*b + 128) >> 8) + 128);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Encode one frame ────────────────────────────────────────────────────────
|
||||
|
||||
void ScreenEncoder::EncodeFrame(const std::vector<uint8_t>& bgra,
|
||||
UINT w, UINT h, bool keyframe) {
|
||||
if (enc_width_ != w || enc_height_ != h) {
|
||||
encoder_.Reset();
|
||||
config_sent_ = false;
|
||||
if (!InitEncoder(w, h)) return;
|
||||
}
|
||||
|
||||
// Build NV12 input sample
|
||||
std::vector<uint8_t> nv12;
|
||||
BgraToNv12(bgra.data(), nv12, w, h);
|
||||
|
||||
ComPtr<IMFSample> in_sample;
|
||||
ComPtr<IMFMediaBuffer> in_buf;
|
||||
MFCreateMemoryBuffer((DWORD)nv12.size(), &in_buf);
|
||||
{
|
||||
BYTE* ptr = nullptr;
|
||||
in_buf->Lock(&ptr, nullptr, nullptr);
|
||||
memcpy(ptr, nv12.data(), nv12.size());
|
||||
in_buf->Unlock();
|
||||
}
|
||||
in_buf->SetCurrentLength((DWORD)nv12.size());
|
||||
MFCreateSample(&in_sample);
|
||||
in_sample->SetSampleTime(sample_ts_);
|
||||
in_sample->SetSampleDuration(kFrameDuration);
|
||||
in_sample->AddBuffer(in_buf.Get());
|
||||
if (keyframe) in_sample->SetUINT32(MFSampleExtension_CleanPoint, 1);
|
||||
sample_ts_ += kFrameDuration;
|
||||
|
||||
if (FAILED(encoder_->ProcessInput(0, in_sample.Get(), 0))) return;
|
||||
|
||||
// Drain output samples
|
||||
MFT_OUTPUT_STREAM_INFO si{};
|
||||
encoder_->GetOutputStreamInfo(0, &si);
|
||||
|
||||
while (true) {
|
||||
ComPtr<IMFSample> out_sample;
|
||||
ComPtr<IMFMediaBuffer> out_buf;
|
||||
if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) {
|
||||
MFCreateMemoryBuffer(si.cbSize ? si.cbSize : w * h * 2, &out_buf);
|
||||
MFCreateSample(&out_sample);
|
||||
out_sample->AddBuffer(out_buf.Get());
|
||||
}
|
||||
|
||||
MFT_OUTPUT_DATA_BUFFER out_data{};
|
||||
out_data.pSample = out_sample.Get();
|
||||
DWORD status = 0;
|
||||
HRESULT hr = encoder_->ProcessOutput(0, 1, &out_data, &status);
|
||||
if (out_data.pEvents) out_data.pEvents->Release();
|
||||
if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break;
|
||||
if (FAILED(hr)) break;
|
||||
|
||||
ComPtr<IMFSample> result(out_data.pSample);
|
||||
if (!result) break;
|
||||
|
||||
ComPtr<IMFMediaBuffer> flat;
|
||||
result->ConvertToContiguousBuffer(&flat);
|
||||
BYTE* enc = nullptr; DWORD enc_len = 0;
|
||||
flat->Lock(&enc, nullptr, &enc_len);
|
||||
|
||||
if (!config_sent_) {
|
||||
size_t sps_start = 0, pps_end = 0;
|
||||
if (ExtractSpsPps(enc, enc_len, sps_start, pps_end)) {
|
||||
std::vector<uint8_t> cfg(1 + (pps_end - sps_start));
|
||||
cfg[0] = 0x01;
|
||||
memcpy(&cfg[1], enc + sps_start, pps_end - sps_start);
|
||||
SendEvent(std::move(cfg));
|
||||
config_sent_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t now_ms =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
std::vector<uint8_t> frame(1 + 8 + enc_len);
|
||||
frame[0] = 0x02;
|
||||
for (int i = 0; i < 8; ++i)
|
||||
frame[1 + i] = static_cast<uint8_t>(now_ms >> (56 - i * 8));
|
||||
memcpy(&frame[9], enc, enc_len);
|
||||
|
||||
flat->Unlock();
|
||||
SendEvent(std::move(frame));
|
||||
}
|
||||
}
|
||||
|
||||
void ScreenEncoder::SendEvent(std::vector<uint8_t> data) {
|
||||
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||
if (sink_) sink_->Success(flutter::EncodableValue(std::move(data)));
|
||||
}
|
||||
|
||||
// ─── Capture loop ────────────────────────────────────────────────────────────
|
||||
|
||||
void ScreenEncoder::CaptureLoop() {
|
||||
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
||||
|
||||
if (!InitD3D()) {
|
||||
CoUninitialize();
|
||||
return;
|
||||
}
|
||||
|
||||
while (running_) {
|
||||
std::vector<uint8_t> bgra;
|
||||
UINT w = 0, h = 0;
|
||||
if (CaptureFrame(bgra, w, h)) {
|
||||
bool kf = force_kf_.exchange(false);
|
||||
EncodeFrame(bgra, w, h, kf);
|
||||
}
|
||||
}
|
||||
|
||||
if (encoder_) {
|
||||
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0);
|
||||
encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0);
|
||||
}
|
||||
CoUninitialize();
|
||||
}
|
||||
57
windows/runner/screen_encoder.h
Normal file
57
windows/runner/screen_encoder.h
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <windows.h>
|
||||
#include <d3d11.h>
|
||||
#include <dxgi1_2.h>
|
||||
#include <mftransform.h>
|
||||
#include <wrl/client.h>
|
||||
|
||||
#include <flutter/event_sink.h>
|
||||
#include <flutter/encodable_value.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
class ScreenEncoder {
|
||||
public:
|
||||
ScreenEncoder();
|
||||
~ScreenEncoder();
|
||||
|
||||
void Start(std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink);
|
||||
void Stop();
|
||||
void ForceKeyframe();
|
||||
|
||||
private:
|
||||
void CaptureLoop();
|
||||
bool InitD3D();
|
||||
bool InitEncoder(UINT width, UINT height);
|
||||
bool CaptureFrame(std::vector<uint8_t>& bgra, UINT& width, UINT& height);
|
||||
void EncodeFrame(const std::vector<uint8_t>& bgra, UINT width, UINT height,
|
||||
bool keyframe);
|
||||
void BgraToNv12(const uint8_t* bgra, std::vector<uint8_t>& nv12, UINT width,
|
||||
UINT height);
|
||||
void SendEvent(std::vector<uint8_t> data);
|
||||
|
||||
Microsoft::WRL::ComPtr<ID3D11Device> d3d_dev_;
|
||||
Microsoft::WRL::ComPtr<ID3D11DeviceContext> d3d_ctx_;
|
||||
Microsoft::WRL::ComPtr<IDXGIOutputDuplication> dupl_;
|
||||
Microsoft::WRL::ComPtr<ID3D11Texture2D> staging_;
|
||||
Microsoft::WRL::ComPtr<IMFTransform> encoder_;
|
||||
|
||||
UINT enc_width_ = 0;
|
||||
UINT enc_height_ = 0;
|
||||
bool config_sent_ = false;
|
||||
LONGLONG sample_ts_ = 0;
|
||||
|
||||
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink_;
|
||||
std::mutex sink_mu_;
|
||||
|
||||
std::atomic<bool> running_ {false};
|
||||
std::atomic<bool> force_kf_ {false};
|
||||
std::thread thread_;
|
||||
};
|
||||
279
windows/runner/video_decoder.cpp
Normal file
279
windows/runner/video_decoder.cpp
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
#include "video_decoder.h"
|
||||
|
||||
#include <mfapi.h>
|
||||
#include <mferror.h>
|
||||
#include <mfidl.h>
|
||||
|
||||
using Microsoft::WRL::ComPtr;
|
||||
|
||||
// ─── Minimal SPS bit-reader ───────────────────────────────────────────────────
|
||||
|
||||
struct BitReader {
|
||||
const uint8_t* data;
|
||||
size_t pos = 0;
|
||||
|
||||
uint32_t Bit() {
|
||||
uint32_t v = (data[pos / 8] >> (7 - pos % 8)) & 1;
|
||||
++pos;
|
||||
return v;
|
||||
}
|
||||
uint32_t Bits(int n) {
|
||||
uint32_t v = 0;
|
||||
for (int i = 0; i < n; ++i) v = (v << 1) | Bit();
|
||||
return v;
|
||||
}
|
||||
uint32_t UEG() {
|
||||
int lz = 0;
|
||||
while (Bit() == 0 && lz < 32) ++lz;
|
||||
if (lz == 0) return 0;
|
||||
return (1u << lz) - 1 + Bits(lz);
|
||||
}
|
||||
int32_t SEG() {
|
||||
uint32_t code = UEG();
|
||||
return (code & 1) ? static_cast<int32_t>((code + 1) / 2)
|
||||
: -static_cast<int32_t>(code / 2);
|
||||
}
|
||||
};
|
||||
|
||||
// ─── SPS parser ───────────────────────────────────────────────────────────────
|
||||
|
||||
bool VideoDecoder::ParseSpsSize(const uint8_t* data, size_t len,
|
||||
int& out_w, int& out_h) {
|
||||
size_t nal_start = SIZE_MAX;
|
||||
for (size_t i = 0; i + 4 <= len; ++i) {
|
||||
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 &&
|
||||
i + 4 < len && (data[i+4] & 0x1F) == 7) {
|
||||
nal_start = i + 4; break;
|
||||
}
|
||||
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 &&
|
||||
i + 3 < len && (data[i+3] & 0x1F) == 7) {
|
||||
nal_start = i + 3; break;
|
||||
}
|
||||
}
|
||||
if (nal_start == SIZE_MAX) return false;
|
||||
|
||||
BitReader br{data + nal_start + 1};
|
||||
|
||||
uint8_t profile_idc = static_cast<uint8_t>(br.Bits(8));
|
||||
br.Bits(8); br.Bits(8); br.UEG();
|
||||
|
||||
static const uint8_t kHiProfiles[] = {100,110,122,244,44,83,86,118,128};
|
||||
bool hi = false;
|
||||
for (uint8_t p : kHiProfiles) if (profile_idc == p) { hi = true; break; }
|
||||
if (hi) {
|
||||
uint32_t chroma = br.UEG();
|
||||
if (chroma == 3) br.Bit();
|
||||
br.UEG(); br.UEG(); br.Bit();
|
||||
if (br.Bit()) {
|
||||
for (int idx = 0; idx < (chroma != 3 ? 8 : 12); ++idx) {
|
||||
if (br.Bit()) {
|
||||
int sz = idx < 6 ? 16 : 64;
|
||||
int last = 8, next = 8;
|
||||
for (int j = 0; j < sz; ++j) {
|
||||
if (next != 0) next = (last + br.SEG() + 256) % 256;
|
||||
last = next == 0 ? last : next;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
br.UEG();
|
||||
uint32_t poc = br.UEG();
|
||||
if (poc == 0) br.UEG();
|
||||
else if (poc == 1) { br.Bit(); br.SEG(); br.SEG(); uint32_t nc=br.UEG(); for(uint32_t j=0;j<nc;++j) br.SEG(); }
|
||||
|
||||
br.UEG(); br.Bit();
|
||||
|
||||
uint32_t pw = br.UEG();
|
||||
uint32_t ph = br.UEG();
|
||||
int w = (int)((pw + 1) * 16);
|
||||
int h = (int)((ph + 1) * 16);
|
||||
|
||||
bool frame_mbs_only = br.Bit() != 0;
|
||||
if (!frame_mbs_only) br.Bit();
|
||||
br.Bit();
|
||||
|
||||
if (br.Bit()) {
|
||||
uint32_t cl=br.UEG(), cr=br.UEG(), ct=br.UEG(), cb=br.UEG();
|
||||
int ux = 2, uy = frame_mbs_only ? 2 : 4;
|
||||
w -= ux * (int)(cl + cr);
|
||||
h -= uy * (int)(ct + cb);
|
||||
}
|
||||
|
||||
if (w <= 0 || h <= 0) return false;
|
||||
out_w = w; out_h = h;
|
||||
return true;
|
||||
}
|
||||
|
||||
// ─── Pixel buffer callback (called from render thread) ────────────────────────
|
||||
|
||||
const FlutterDesktopPixelBuffer* VideoDecoder::PixelBufferCallback(
|
||||
size_t, size_t, void* user_data) {
|
||||
return &static_cast<VideoDecoder*>(user_data)->pixel_buf_;
|
||||
}
|
||||
|
||||
// ─── VideoDecoder ─────────────────────────────────────────────────────────────
|
||||
|
||||
VideoDecoder::VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg,
|
||||
const uint8_t* sps_pps, size_t sps_pps_len)
|
||||
: tex_reg_(tex_reg) {
|
||||
Init(sps_pps, sps_pps_len);
|
||||
}
|
||||
|
||||
VideoDecoder::~VideoDecoder() {
|
||||
if (texture_id_ >= 0) {
|
||||
HANDLE done = CreateEvent(nullptr, TRUE, FALSE, nullptr);
|
||||
FlutterDesktopTextureRegistrarUnregisterExternalTexture(
|
||||
tex_reg_, texture_id_,
|
||||
[](void* ud) { SetEvent(static_cast<HANDLE>(ud)); },
|
||||
done);
|
||||
WaitForSingleObject(done, 3000);
|
||||
CloseHandle(done);
|
||||
}
|
||||
if (decoder_)
|
||||
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0);
|
||||
}
|
||||
|
||||
bool VideoDecoder::Init(const uint8_t* sps_pps, size_t sps_pps_len) {
|
||||
int w = 0, h = 0;
|
||||
if (!ParseSpsSize(sps_pps, sps_pps_len, w, h)) {
|
||||
w = GetSystemMetrics(SM_CXSCREEN);
|
||||
h = GetSystemMetrics(SM_CYSCREEN);
|
||||
}
|
||||
width_ = w; height_ = h;
|
||||
|
||||
MFT_REGISTER_TYPE_INFO in_info{MFMediaType_Video, MFVideoFormat_H264};
|
||||
UINT32 count = 0; IMFActivate** acts = nullptr;
|
||||
HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER,
|
||||
MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER,
|
||||
&in_info, nullptr, &acts, &count);
|
||||
if (FAILED(hr) || count == 0) return false;
|
||||
hr = acts[0]->ActivateObject(IID_PPV_ARGS(&decoder_));
|
||||
for (UINT32 i = 0; i < count; ++i) acts[i]->Release();
|
||||
CoTaskMemFree(acts);
|
||||
if (FAILED(hr)) return false;
|
||||
|
||||
ComPtr<IMFMediaType> in_type;
|
||||
MFCreateMediaType(&in_type);
|
||||
in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
|
||||
MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h);
|
||||
if (FAILED(decoder_->SetInputType(0, in_type.Get(), 0))) return false;
|
||||
|
||||
ComPtr<IMFMediaType> out_type;
|
||||
MFCreateMediaType(&out_type);
|
||||
out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||
out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
|
||||
MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h);
|
||||
if (FAILED(decoder_->SetOutputType(0, out_type.Get(), 0))) return false;
|
||||
|
||||
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
|
||||
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
|
||||
|
||||
// Register Flutter BGRA texture via C API
|
||||
frame_data_.assign(static_cast<size_t>(w * h * 4), 0);
|
||||
pixel_buf_.buffer = frame_data_.data();
|
||||
pixel_buf_.width = static_cast<size_t>(w);
|
||||
pixel_buf_.height = static_cast<size_t>(h);
|
||||
|
||||
FlutterDesktopTextureInfo info{};
|
||||
info.type = kFlutterDesktopPixelBufferTexture;
|
||||
info.pixel_buffer_config.callback = &VideoDecoder::PixelBufferCallback;
|
||||
info.pixel_buffer_config.user_data = this;
|
||||
|
||||
texture_id_ = FlutterDesktopTextureRegistrarRegisterExternalTexture(
|
||||
tex_reg_, &info);
|
||||
|
||||
// Prime decoder with SPS+PPS
|
||||
FeedNal(sps_pps, sps_pps_len);
|
||||
return texture_id_ >= 0;
|
||||
}
|
||||
|
||||
// ─── NV12 → BGRA ──────────────────────────────────────────────────────────────
|
||||
|
||||
void VideoDecoder::Nv12ToBgra(const uint8_t* nv12, int stride,
|
||||
uint8_t* bgra, int w, int h) {
|
||||
const uint8_t* Y = nv12;
|
||||
const uint8_t* UV = nv12 + stride * h;
|
||||
|
||||
for (int row = 0; row < h; ++row) {
|
||||
for (int col = 0; col < w; ++col) {
|
||||
int y = Y[row * stride + col];
|
||||
int uv = (row / 2) * stride + (col & ~1);
|
||||
int u = UV[uv];
|
||||
int v = UV[uv + 1];
|
||||
|
||||
int c = y - 16, d = u - 128, e = v - 128;
|
||||
int r = (298*c + 409*e + 128) >> 8;
|
||||
int g = (298*c - 100*d - 208*e + 128) >> 8;
|
||||
int b = (298*c + 516*d + 128) >> 8;
|
||||
|
||||
uint8_t* p = bgra + (row * w + col) * 4;
|
||||
p[0] = static_cast<uint8_t>(b < 0 ? 0 : b > 255 ? 255 : b);
|
||||
p[1] = static_cast<uint8_t>(g < 0 ? 0 : g > 255 ? 255 : g);
|
||||
p[2] = static_cast<uint8_t>(r < 0 ? 0 : r > 255 ? 255 : r);
|
||||
p[3] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── FeedNal ──────────────────────────────────────────────────────────────────
|
||||
|
||||
void VideoDecoder::FeedNal(const uint8_t* nal, size_t len) {
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
if (!decoder_ || !len) return;
|
||||
|
||||
ComPtr<IMFSample> in_sample;
|
||||
ComPtr<IMFMediaBuffer> in_buf;
|
||||
MFCreateMemoryBuffer((DWORD)len, &in_buf);
|
||||
{
|
||||
BYTE* ptr = nullptr;
|
||||
in_buf->Lock(&ptr, nullptr, nullptr);
|
||||
memcpy(ptr, nal, len);
|
||||
in_buf->Unlock();
|
||||
}
|
||||
in_buf->SetCurrentLength((DWORD)len);
|
||||
MFCreateSample(&in_sample);
|
||||
in_sample->SetSampleTime(input_ts_);
|
||||
in_sample->SetSampleDuration(333333);
|
||||
in_sample->AddBuffer(in_buf.Get());
|
||||
input_ts_ += 333333;
|
||||
|
||||
if (FAILED(decoder_->ProcessInput(0, in_sample.Get(), 0))) return;
|
||||
|
||||
MFT_OUTPUT_STREAM_INFO si{};
|
||||
decoder_->GetOutputStreamInfo(0, &si);
|
||||
|
||||
while (true) {
|
||||
ComPtr<IMFSample> out_sample;
|
||||
ComPtr<IMFMediaBuffer> out_buf;
|
||||
if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) {
|
||||
DWORD sz = si.cbSize ? si.cbSize : static_cast<DWORD>(width_ * height_ * 3 / 2);
|
||||
MFCreateMemoryBuffer(sz, &out_buf);
|
||||
MFCreateSample(&out_sample);
|
||||
out_sample->AddBuffer(out_buf.Get());
|
||||
}
|
||||
|
||||
MFT_OUTPUT_DATA_BUFFER out_data{};
|
||||
out_data.pSample = out_sample.Get();
|
||||
DWORD status = 0;
|
||||
HRESULT hr = decoder_->ProcessOutput(0, 1, &out_data, &status);
|
||||
if (out_data.pEvents) out_data.pEvents->Release();
|
||||
if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break;
|
||||
if (FAILED(hr)) break;
|
||||
|
||||
ComPtr<IMFSample> result(out_data.pSample);
|
||||
if (!result) break;
|
||||
|
||||
ComPtr<IMFMediaBuffer> flat;
|
||||
result->ConvertToContiguousBuffer(&flat);
|
||||
BYTE* nv12_ptr = nullptr; DWORD nv12_len = 0;
|
||||
flat->Lock(&nv12_ptr, nullptr, &nv12_len);
|
||||
Nv12ToBgra(nv12_ptr, width_, frame_data_.data(), width_, height_);
|
||||
flat->Unlock();
|
||||
|
||||
FlutterDesktopTextureRegistrarMarkExternalTextureFrameAvailable(
|
||||
tex_reg_, texture_id_);
|
||||
}
|
||||
}
|
||||
49
windows/runner/video_decoder.h
Normal file
49
windows/runner/video_decoder.h
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <windows.h>
|
||||
#include <flutter_texture_registrar.h>
|
||||
#include <flutter_plugin_registrar.h>
|
||||
#include <mftransform.h>
|
||||
#include <wrl/client.h>
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
class VideoDecoder {
|
||||
public:
|
||||
// spsPps: Annex-B SPS+PPS bytes (the payload from a 0x01 config packet).
|
||||
VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg,
|
||||
const uint8_t* sps_pps, size_t sps_pps_len);
|
||||
~VideoDecoder();
|
||||
|
||||
int64_t texture_id() const { return texture_id_; }
|
||||
bool is_valid() const { return texture_id_ >= 0; }
|
||||
|
||||
void FeedNal(const uint8_t* nal, size_t len);
|
||||
|
||||
private:
|
||||
bool Init(const uint8_t* sps_pps, size_t sps_pps_len);
|
||||
bool ParseSpsSize(const uint8_t* data, size_t len, int& w, int& h);
|
||||
void Nv12ToBgra(const uint8_t* nv12, int stride,
|
||||
uint8_t* bgra, int w, int h);
|
||||
|
||||
static const FlutterDesktopPixelBuffer* PixelBufferCallback(
|
||||
size_t width, size_t height, void* user_data);
|
||||
|
||||
FlutterDesktopTextureRegistrarRef tex_reg_;
|
||||
int64_t texture_id_ = -1;
|
||||
|
||||
Microsoft::WRL::ComPtr<IMFTransform> decoder_;
|
||||
|
||||
FlutterDesktopPixelBuffer pixel_buf_{};
|
||||
std::vector<uint8_t> frame_data_;
|
||||
|
||||
int width_ = 0;
|
||||
int height_ = 0;
|
||||
LONGLONG input_ts_ = 0;
|
||||
|
||||
std::mutex mu_;
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue