From 37fef7de2599c1ac352d8e1541e7c62634018d7d Mon Sep 17 00:00:00 2001 From: ImBenji Date: Tue, 5 May 2026 04:39:18 +0100 Subject: [PATCH] implement video encoding and decoding functionality with screen capture support --- macos/Runner/AppDelegate.swift | 242 ++++++++++++++++++++ pubspec.lock | 6 +- pubspec.yaml | 2 +- windows/runner/CMakeLists.txt | 10 + windows/runner/flutter_window.cpp | 190 ++++++++++++---- windows/runner/flutter_window.h | 15 +- windows/runner/screen_encoder.cpp | 361 ++++++++++++++++++++++++++++++ windows/runner/screen_encoder.h | 57 +++++ windows/runner/video_decoder.cpp | 279 +++++++++++++++++++++++ windows/runner/video_decoder.h | 49 ++++ 10 files changed, 1163 insertions(+), 48 deletions(-) create mode 100644 windows/runner/screen_encoder.cpp create mode 100644 windows/runner/screen_encoder.h create mode 100644 windows/runner/video_decoder.cpp create mode 100644 windows/runner/video_decoder.h diff --git a/macos/Runner/AppDelegate.swift b/macos/Runner/AppDelegate.swift index d4f860e..1dde45d 100644 --- a/macos/Runner/AppDelegate.swift +++ b/macos/Runner/AppDelegate.swift @@ -25,11 +25,16 @@ class AppDelegate: FlutterAppDelegate { private var captureWidth = 1280 private var captureHeight = 720 + // viewer decoder sessions keyed by texture id + private var decoders: [Int64: VideoDecoderSession] = [:] + private var textureRegistry: FlutterTextureRegistry? + override func applicationDidFinishLaunching(_ notification: Notification) { guard let window = mainFlutterWindow, let ctrl = window.contentViewController as? FlutterViewController else { return } let messenger = ctrl.engine.binaryMessenger + textureRegistry = ctrl.engine.textureRegistry inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger) inputChannel?.setMethodCallHandler { [weak self] call, result in @@ -58,6 +63,40 @@ class AppDelegate: FlutterAppDelegate { case "forceKeyframe": self.requestKeyframe() result(nil) + + case "createDecoder": + guard let args = call.arguments as? [String: Any], + let spsPpsTyped = args["spsPps"] as? FlutterStandardTypedData, + let registry = self.textureRegistry else { + result(FlutterError(code: "INVALID_ARGS", message: nil, details: nil)) + return + } + if let decoder = VideoDecoderSession(spsPps: spsPpsTyped.data, textureRegistry: registry) { + self.decoders[decoder.textureId] = decoder + result(["textureId": decoder.textureId]) + } else { + result(FlutterError(code: "DECODER_FAILED", message: "Failed to create H264 decoder", details: nil)) + } + + case "feedNal": + guard let args = call.arguments as? [String: Any], + let textureId = (args["textureId"] as? NSNumber)?.int64Value, + let nalTyped = args["nal"] as? FlutterStandardTypedData, + let decoder = self.decoders[textureId] else { + result(nil) + return + } + decoder.feedNal(nalTyped.data) + result(nil) + + case "stopDecoder": + if let args = call.arguments as? [String: Any], + let textureId = (args["textureId"] as? NSNumber)?.int64Value { + self.decoders[textureId]?.invalidate() + self.decoders.removeValue(forKey: textureId) + } + result(nil) + default: result(FlutterMethodNotImplemented) } @@ -441,3 +480,206 @@ extension AppDelegate: FlutterStreamHandler { return nil } } + +// MARK: - VT decompression C callback + +private func vtDecompressOutputCallback( + refCon: UnsafeMutableRawPointer?, + frameRefCon: UnsafeMutableRawPointer?, + status: OSStatus, + infoFlags: VTDecodeInfoFlags, + imageBuffer: CVImageBuffer?, + pts: CMTime, + duration: CMTime +) { + guard let refCon = refCon, status == noErr, let imageBuffer = imageBuffer else { return } + let session = Unmanaged.fromOpaque(refCon).takeUnretainedValue() + session.handleDecodedFrame(imageBuffer) +} + +// MARK: - VideoDecoderSession + +class VideoDecoderSession: NSObject, FlutterTexture { + private(set) var textureId: Int64 = -1 + + private var decompressionSession: VTDecompressionSession? + private var formatDescription: CMVideoFormatDescription? + private var latestPixelBuffer: CVPixelBuffer? + private let lock = NSLock() + private weak var textureRegistry: FlutterTextureRegistry? + + init?(spsPps: Data, textureRegistry: FlutterTextureRegistry) { + self.textureRegistry = textureRegistry + super.init() + guard setupSession(spsPps: spsPps) else { return nil } + textureId = textureRegistry.register(self) + } + + // MARK: Parse Annex-B SPS+PPS and create VTDecompressionSession + + private func setupSession(spsPps: Data) -> Bool { + // Split on 4-byte Annex-B start codes to get raw NALU payloads + let bytes = [UInt8](spsPps) + var nalus: [Data] = [] + var scanStart = 0 + var i = 0 + while i + 4 <= bytes.count { + if bytes[i] == 0 && bytes[i+1] == 0 && bytes[i+2] == 0 && bytes[i+3] == 1 { + if i > scanStart { nalus.append(Data(bytes[scanStart..= 2 else { return false } + + let sps = nalus[0] + let pps = nalus[1] + + var formatDesc: CMVideoFormatDescription? + let fmtStatus: OSStatus = sps.withUnsafeBytes { spsBuf in + pps.withUnsafeBytes { ppsBuf in + var ptrs: [UnsafePointer] = [ + spsBuf.bindMemory(to: UInt8.self).baseAddress!, + ppsBuf.bindMemory(to: UInt8.self).baseAddress! + ] + var sizes: [Int] = [sps.count, pps.count] + return CMVideoFormatDescriptionCreateFromH264ParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: 2, + parameterSetPointers: &ptrs, + parameterSetSizes: &sizes, + nalUnitHeaderLength: 4, + formatDescriptionOut: &formatDesc + ) + } + } + guard fmtStatus == noErr, let formatDesc = formatDesc else { return false } + self.formatDescription = formatDesc + + let dims = CMVideoFormatDescriptionGetDimensions(formatDesc) + let imageAttrs: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey: Int(dims.width), + kCVPixelBufferHeightKey: Int(dims.height), + kCVPixelBufferIOSurfacePropertiesKey: [:] as [String: Any] + ] + + var callback = VTDecompressionOutputCallbackRecord( + decompressionOutputCallback: vtDecompressOutputCallback, + decompressionOutputRefCon: Unmanaged.passUnretained(self).toOpaque() + ) + + var session: VTDecompressionSession? + let sessionStatus = VTDecompressionSessionCreate( + allocator: kCFAllocatorDefault, + formatDescription: formatDesc, + decoderSpecification: nil, + imageBufferAttributes: imageAttrs as CFDictionary, + outputCallback: &callback, + decompressionSessionOut: &session + ) + guard sessionStatus == noErr, let session = session else { return false } + self.decompressionSession = session + return true + } + + // MARK: Feed an Annex-B NAL unit for decoding + + func feedNal(_ data: Data) { + guard let session = decompressionSession, + let formatDesc = formatDescription, + data.count > 4 else { return } + + // Convert Annex-B start code to AVCC 4-byte big-endian length prefix + let naluLen = UInt32(data.count - 4).bigEndian + var avcc = Data(capacity: data.count) + withUnsafeBytes(of: naluLen) { avcc.append(contentsOf: $0) } + avcc.append(data.advanced(by: 4)) + + var blockBuffer: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, + memoryBlock: nil, + blockLength: avcc.count, + blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, + offsetToData: 0, + dataLength: avcc.count, + flags: 0, + blockBufferOut: &blockBuffer + ) == kCMBlockBufferNoErr, let blockBuffer = blockBuffer else { return } + + avcc.withUnsafeBytes { ptr in + _ = CMBlockBufferReplaceDataBytes( + with: ptr.baseAddress!, + blockBuffer: blockBuffer, + offsetIntoDestination: 0, + dataLength: avcc.count + ) + } + + var timingInfo = CMSampleTimingInfo( + duration: CMTime(value: 1, timescale: 60), + presentationTimeStamp: CMTime(seconds: CACurrentMediaTime(), preferredTimescale: 600), + decodeTimeStamp: .invalid + ) + var sampleBuffer: CMSampleBuffer? + guard CMSampleBufferCreate( + allocator: kCFAllocatorDefault, + dataBuffer: blockBuffer, + dataReady: true, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDesc, + sampleCount: 1, + sampleTimingEntryCount: 1, + sampleTimingArray: &timingInfo, + sampleSizeEntryCount: 0, + sampleSizeArray: nil, + sampleBufferOut: &sampleBuffer + ) == noErr, let sampleBuffer = sampleBuffer else { return } + + VTDecompressionSessionDecodeFrame( + session, + sampleBuffer: sampleBuffer, + flags: [], + frameRefcon: nil, + infoFlagsOut: nil + ) + } + + // MARK: Called from the VT C callback + + func handleDecodedFrame(_ imageBuffer: CVImageBuffer) { + lock.lock() + latestPixelBuffer = imageBuffer as CVPixelBuffer + lock.unlock() + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + self.textureRegistry?.textureFrameAvailable(self.textureId) + } + } + + // MARK: FlutterTexture + + func copyPixelBuffer() -> Unmanaged? { + lock.lock() + defer { lock.unlock() } + guard let pb = latestPixelBuffer else { return nil } + return Unmanaged.passRetained(pb) + } + + // MARK: Teardown + + func invalidate() { + if let session = decompressionSession { + VTDecompressionSessionInvalidate(session) + } + decompressionSession = nil + latestPixelBuffer = nil + textureRegistry?.unregisterTexture(textureId) + } +} diff --git a/pubspec.lock b/pubspec.lock index 6d9384d..545b998 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -239,10 +239,10 @@ packages: dependency: transitive description: name: meta - sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349" + sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394" url: "https://pub.dev" source: hosted - version: "1.18.0" + version: "1.17.0" native_toolchain_c: dependency: transitive description: @@ -481,5 +481,5 @@ packages: source: hosted version: "3.1.3" sdks: - dart: ">=3.12.0-210.1.beta <4.0.0" + dart: ">=3.11.0 <4.0.0" flutter: ">=3.38.4" diff --git a/pubspec.yaml b/pubspec.yaml index 8a311e0..c691f34 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -19,7 +19,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev version: 1.0.0+1 environment: - sdk: ^3.12.0-210.1.beta + sdk: ^3.11.0 # Dependencies specify other packages that your package needs in order to work. # To automatically upgrade your package dependencies to the latest versions diff --git a/windows/runner/CMakeLists.txt b/windows/runner/CMakeLists.txt index 394917c..6e5b2bc 100644 --- a/windows/runner/CMakeLists.txt +++ b/windows/runner/CMakeLists.txt @@ -11,6 +11,8 @@ add_executable(${BINARY_NAME} WIN32 "main.cpp" "utils.cpp" "win32_window.cpp" + "screen_encoder.cpp" + "video_decoder.cpp" "${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc" "Runner.rc" "runner.exe.manifest" @@ -34,6 +36,14 @@ target_compile_definitions(${BINARY_NAME} PRIVATE "NOMINMAX") # dependencies here. target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app) target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib") +target_link_libraries(${BINARY_NAME} PRIVATE + "d3d11.lib" + "dxgi.lib" + "mf.lib" + "mfplat.lib" + "mfuuid.lib" + "propsys.lib" +) target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}") # Run the Flutter tool portions of the build. This must not be removed. diff --git a/windows/runner/flutter_window.cpp b/windows/runner/flutter_window.cpp index b7e1cbd..c823f34 100644 --- a/windows/runner/flutter_window.cpp +++ b/windows/runner/flutter_window.cpp @@ -3,10 +3,15 @@ #include #include +#include +#include #include #include #include #include +#include +#include +#include #include "flutter/generated_plugin_registrant.h" @@ -15,23 +20,22 @@ FlutterWindow::FlutterWindow(const flutter::DartProject& project) FlutterWindow::~FlutterWindow() {} +// ─── Input injection helper ────────────────────────────────────────────────── + static void HandleInjectInput(const flutter::EncodableMap& args) { auto typeIt = args.find(flutter::EncodableValue("type")); if (typeIt == args.end()) return; std::string type = std::get(typeIt->second); - int screenW = GetSystemMetrics(SM_CXSCREEN); - int screenH = GetSystemMetrics(SM_CYSCREEN); - if (type == "move") { double nx = std::get(args.at(flutter::EncodableValue("x"))); double ny = std::get(args.at(flutter::EncodableValue("y"))); INPUT input = {}; - input.type = INPUT_MOUSE; - input.mi.dx = (LONG)(nx * 65535); - input.mi.dy = (LONG)(ny * 65535); + input.type = INPUT_MOUSE; + input.mi.dx = (LONG)(nx * 65535); + input.mi.dy = (LONG)(ny * 65535); input.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE; SendInput(1, &input, sizeof(INPUT)); @@ -42,18 +46,16 @@ static void HandleInjectInput(const flutter::EncodableMap& args) { auto btnIt = args.find(flutter::EncodableValue("button")); if (btnIt != args.end()) btn = std::get(btnIt->second); - // move first INPUT moveInput = {}; - moveInput.type = INPUT_MOUSE; - moveInput.mi.dx = (LONG)(nx * 65535); - moveInput.mi.dy = (LONG)(ny * 65535); + moveInput.type = INPUT_MOUSE; + moveInput.mi.dx = (LONG)(nx * 65535); + moveInput.mi.dy = (LONG)(ny * 65535); moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE; SendInput(1, &moveInput, sizeof(INPUT)); INPUT clickInput[2] = {}; clickInput[0].type = INPUT_MOUSE; clickInput[1].type = INPUT_MOUSE; - if (btn == 1) { clickInput[0].mi.dwFlags = MOUSEEVENTF_RIGHTDOWN; clickInput[1].mi.dwFlags = MOUSEEVENTF_RIGHTUP; @@ -68,48 +70,156 @@ static void HandleInjectInput(const flutter::EncodableMap& args) { if (keyIt == args.end()) return; // Flutter logical key IDs dont map 1:1 to VKs — use low byte as rough VK - int logicalKey = std::get(keyIt->second); - WORD vk = (WORD)(logicalKey & 0xFF); + int logicalKey = std::get(keyIt->second); + WORD vk = (WORD)(logicalKey & 0xFF); INPUT keyInput = {}; - keyInput.type = INPUT_KEYBOARD; - keyInput.ki.wVk = vk; + keyInput.type = INPUT_KEYBOARD; + keyInput.ki.wVk = vk; if (type == "keyup") keyInput.ki.dwFlags = KEYEVENTF_KEYUP; - SendInput(1, &keyInput, sizeof(INPUT)); } } +// ─── OnCreate ──────────────────────────────────────────────────────────────── + bool FlutterWindow::OnCreate() { - if (!Win32Window::OnCreate()) { - return false; - } + if (!Win32Window::OnCreate()) return false; + + MFStartup(MF_VERSION); RECT frame = GetClientArea(); - flutter_controller_ = std::make_unique( frame.right - frame.left, frame.bottom - frame.top, project_); - if (!flutter_controller_->engine() || !flutter_controller_->view()) { + if (!flutter_controller_->engine() || !flutter_controller_->view()) return false; - } + RegisterPlugins(flutter_controller_->engine()); - // register input injection channel - auto channel = std::make_shared>( - flutter_controller_->engine()->messenger(), - "com.pulsar/input", - &flutter::StandardMethodCodec::GetInstance() - ); + auto* messenger = flutter_controller_->engine()->messenger(); - channel->SetMethodCallHandler( + // Get the texture registrar via the C API (avoids linking flutter_wrapper_plugin). + auto reg_ref = flutter_controller_->engine()->GetRegistrarForPlugin("pulsar"); + tex_reg_ = FlutterDesktopRegistrarGetTextureRegistrar(reg_ref); + + // ── com.pulsar/input MethodChannel ────────────────────────────────────── + auto input_ch = std::make_shared>( + messenger, "com.pulsar/input", + &flutter::StandardMethodCodec::GetInstance()); + + input_ch->SetMethodCallHandler( [](const flutter::MethodCall& call, std::unique_ptr> result) { if (call.method_name() == "injectInput") { - if (const auto* args = std::get_if(call.arguments())) { + if (const auto* args = + std::get_if(call.arguments())) HandleInjectInput(*args); + result->Success(); + } else { + result->NotImplemented(); + } + }); + + // ── com.pulsar/video/frames EventChannel ──────────────────────────────── + screen_encoder_ = std::make_unique(); + auto* enc = screen_encoder_.get(); // raw ptr safe: outlives channels + + auto frame_ch = std::make_shared>( + messenger, "com.pulsar/video/frames", + &flutter::StandardMethodCodec::GetInstance()); + + frame_ch->SetStreamHandler( + std::make_unique>( + [enc](const flutter::EncodableValue*, + std::unique_ptr>&& sink) + -> std::unique_ptr> { + enc->Start(std::move(sink)); + return nullptr; + }, + [enc](const flutter::EncodableValue*) + -> std::unique_ptr> { + enc->Stop(); + return nullptr; + })); + + // ── com.pulsar/video MethodChannel ────────────────────────────────────── + auto* decoders = &decoders_; // raw ptr safe: outlives channels + auto video_ch = std::make_shared>( + messenger, "com.pulsar/video", + &flutter::StandardMethodCodec::GetInstance()); + + video_ch->SetMethodCallHandler( + [enc, decoders, this]( + const flutter::MethodCall& call, + std::unique_ptr> result) { + const auto& m = call.method_name(); + + if (m == "start") { + result->Success(); + + } else if (m == "stop") { + enc->Stop(); + result->Success(); + + } else if (m == "forceKeyframe") { + enc->ForceKeyframe(); + result->Success(); + + } else if (m == "createDecoder") { + const auto* args = + std::get_if(call.arguments()); + if (!args) { result->Error("BAD_ARGS", ""); return; } + + auto it = args->find(flutter::EncodableValue("spsPps")); + if (it == args->end()) { + result->Error("BAD_ARGS", "missing spsPps"); + return; + } + const auto& sps_pps = + std::get>(it->second); + + auto dec = std::make_unique( + tex_reg_, sps_pps.data(), sps_pps.size()); + if (!dec->is_valid()) { + result->Error("INIT_FAILED", "decoder init failed"); + return; + } + int64_t id = dec->texture_id(); + (*decoders)[id] = std::move(dec); + result->Success(flutter::EncodableValue(flutter::EncodableMap{ + {flutter::EncodableValue("textureId"), + flutter::EncodableValue(id)}})); + + } else if (m == "stopDecoder") { + const auto* args = + std::get_if(call.arguments()); + if (args) { + auto it = args->find(flutter::EncodableValue("textureId")); + if (it != args->end()) + decoders->erase(std::get(it->second)); } result->Success(); + + } else if (m == "feedNal") { + const auto* args = + std::get_if(call.arguments()); + if (!args) { result->Error("BAD_ARGS", ""); return; } + + auto id_it = args->find(flutter::EncodableValue("textureId")); + auto nal_it = args->find(flutter::EncodableValue("nal")); + if (id_it == args->end() || nal_it == args->end()) { + result->Error("BAD_ARGS", "missing args"); + return; + } + int64_t id = std::get(id_it->second); + const auto& nal = std::get>(nal_it->second); + + auto dit = decoders->find(id); + if (dit != decoders->end()) + dit->second->FeedNal(nal.data(), nal.size()); + result->Success(); + } else { result->NotImplemented(); } @@ -117,27 +227,31 @@ bool FlutterWindow::OnCreate() { SetChildContent(flutter_controller_->view()->GetNativeWindow()); - flutter_controller_->engine()->SetNextFrameCallback([&]() { - this->Show(); - }); + flutter_controller_->engine()->SetNextFrameCallback([&]() { this->Show(); }); flutter_controller_->ForceRedraw(); - return true; } +// ─── OnDestroy ─────────────────────────────────────────────────────────────── + void FlutterWindow::OnDestroy() { - if (flutter_controller_) { - flutter_controller_ = nullptr; - } + decoders_.clear(); + screen_encoder_.reset(); + if (flutter_controller_) flutter_controller_ = nullptr; + + MFShutdown(); Win32Window::OnDestroy(); } +// ─── MessageHandler ────────────────────────────────────────────────────────── + LRESULT FlutterWindow::MessageHandler(HWND hwnd, UINT const message, WPARAM const wparam, LPARAM const lparam) noexcept { if (flutter_controller_) { std::optional result = - flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam, lparam); + flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam, + lparam); if (result) return *result; } diff --git a/windows/runner/flutter_window.h b/windows/runner/flutter_window.h index 6da0652..098461b 100644 --- a/windows/runner/flutter_window.h +++ b/windows/runner/flutter_window.h @@ -3,31 +3,34 @@ #include #include +#include +#include +#include #include +#include "screen_encoder.h" +#include "video_decoder.h" #include "win32_window.h" -// A window that does nothing but host a Flutter view. class FlutterWindow : public Win32Window { public: - // Creates a new FlutterWindow hosting a Flutter view running |project|. explicit FlutterWindow(const flutter::DartProject& project); virtual ~FlutterWindow(); protected: - // Win32Window: bool OnCreate() override; void OnDestroy() override; LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam, LPARAM const lparam) noexcept override; private: - // The project to run. flutter::DartProject project_; - - // The Flutter instance hosted by this window. std::unique_ptr flutter_controller_; + + FlutterDesktopTextureRegistrarRef tex_reg_ = nullptr; + std::unique_ptr screen_encoder_; + std::map> decoders_; }; #endif // RUNNER_FLUTTER_WINDOW_H_ diff --git a/windows/runner/screen_encoder.cpp b/windows/runner/screen_encoder.cpp new file mode 100644 index 0000000..25bb617 --- /dev/null +++ b/windows/runner/screen_encoder.cpp @@ -0,0 +1,361 @@ +#include "screen_encoder.h" + +#include +#include +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +static constexpr LONGLONG kFrameDuration = 333333; // 100ns units ≈ 30fps +static constexpr UINT32 kBitrate = 6'000'000; + +// Scan an Annex-B bitstream for the span covering the SPS (type 7) through +// the end of the PPS (type 8). Returns false if either is missing. +static bool ExtractSpsPps(const uint8_t* data, size_t size, + size_t& sps_start, size_t& pps_end) { + struct Nal { size_t start; int type; }; + std::vector nals; + + size_t i = 0; + while (i + 4 <= size) { + size_t hdr_len = 0; + int nal_type = 0; + if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 && + i + 4 < size) { + hdr_len = 4; + nal_type = data[i+4] & 0x1F; + } else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 && + i + 3 < size) { + hdr_len = 3; + nal_type = data[i+3] & 0x1F; + } + if (hdr_len) { + nals.push_back({i, nal_type}); + i += hdr_len; + } else { + ++i; + } + } + + size_t sps_idx = SIZE_MAX, pps_idx = SIZE_MAX; + for (size_t j = 0; j < nals.size(); ++j) { + if (nals[j].type == 7) sps_idx = j; + if (nals[j].type == 8) pps_idx = j; + } + if (sps_idx == SIZE_MAX || pps_idx == SIZE_MAX) return false; + + sps_start = nals[sps_idx].start; + size_t after_pps = pps_idx + 1; + pps_end = (after_pps < nals.size()) ? nals[after_pps].start : size; + return sps_start < pps_end; +} + +// ─── ScreenEncoder ─────────────────────────────────────────────────────────── + +ScreenEncoder::ScreenEncoder() {} + +ScreenEncoder::~ScreenEncoder() { Stop(); } + +void ScreenEncoder::Start( + std::unique_ptr> sink) { + Stop(); + { + std::lock_guard lk(sink_mu_); + sink_ = std::move(sink); + } + config_sent_ = false; + sample_ts_ = 0; + running_ = true; + thread_ = std::thread(&ScreenEncoder::CaptureLoop, this); +} + +void ScreenEncoder::Stop() { + running_ = false; + if (thread_.joinable()) thread_.join(); + { + std::lock_guard lk(sink_mu_); + sink_.reset(); + } + encoder_.Reset(); + dupl_.Reset(); + staging_.Reset(); + d3d_ctx_.Reset(); + d3d_dev_.Reset(); + enc_width_ = enc_height_ = 0; +} + +void ScreenEncoder::ForceKeyframe() { force_kf_ = true; } + +// ─── D3D / DXGI init ───────────────────────────────────────────────────────── + +bool ScreenEncoder::InitD3D() { + D3D_FEATURE_LEVEL level; + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, + nullptr, 0, D3D11_SDK_VERSION, + &d3d_dev_, &level, &d3d_ctx_); + if (FAILED(hr)) return false; + + ComPtr dxgi_dev; + ComPtr adapter; + ComPtr output; + ComPtr output1; + + d3d_dev_.As(&dxgi_dev); + dxgi_dev->GetAdapter(&adapter); + if (FAILED(adapter->EnumOutputs(0, &output))) return false; + if (FAILED(output.As(&output1))) return false; + + hr = output1->DuplicateOutput(d3d_dev_.Get(), &dupl_); + return SUCCEEDED(hr); +} + +// ─── Encoder init ──────────────────────────────────────────────────────────── + +bool ScreenEncoder::InitEncoder(UINT width, UINT height) { + MFT_REGISTER_TYPE_INFO out_info{MFMediaType_Video, MFVideoFormat_H264}; + + UINT32 count = 0; + IMFActivate** activates = nullptr; + HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER, + nullptr, &out_info, &activates, &count); + if (FAILED(hr) || count == 0) return false; + + hr = activates[0]->ActivateObject(IID_PPV_ARGS(&encoder_)); + for (UINT32 i = 0; i < count; ++i) activates[i]->Release(); + CoTaskMemFree(activates); + if (FAILED(hr)) return false; + + // Output: H264 + ComPtr out_type; + MFCreateMediaType(&out_type); + out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, width, height); + MFSetAttributeRatio(out_type.Get(), MF_MT_FRAME_RATE, 30, 1); + out_type->SetUINT32(MF_MT_AVG_BITRATE, kBitrate); + out_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + out_type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High); + if (FAILED(encoder_->SetOutputType(0, out_type.Get(), 0))) return false; + + // Input: NV12 + ComPtr in_type; + MFCreateMediaType(&in_type); + in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, width, height); + MFSetAttributeRatio(in_type.Get(), MF_MT_FRAME_RATE, 30, 1); + in_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive); + in_type->SetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32)width); + if (FAILED(encoder_->SetInputType(0, in_type.Get(), 0))) return false; + + // Keyframe every 30 frames via media type attribute + out_type->SetUINT32(MF_MT_MAX_KEYFRAME_SPACING, 30); + + encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + + enc_width_ = width; + enc_height_ = height; + return true; +} + +// ─── Frame capture ─────────────────────────────────────────────────────────── + +bool ScreenEncoder::CaptureFrame(std::vector& bgra, + UINT& width, UINT& height) { + DXGI_OUTDUPL_FRAME_INFO info{}; + ComPtr res; + + HRESULT hr = dupl_->AcquireNextFrame(16, &info, &res); + if (hr == DXGI_ERROR_WAIT_TIMEOUT) return false; + if (FAILED(hr)) { + dupl_.Reset(); + staging_.Reset(); + InitD3D(); + return false; + } + + ComPtr tex; + res.As(&tex); + + D3D11_TEXTURE2D_DESC desc{}; + tex->GetDesc(&desc); + width = desc.Width; + height = desc.Height; + + // Recreate staging texture on size change + if (!staging_) { + D3D11_TEXTURE2D_DESC sd{}; + sd.Width = width; + sd.Height = height; + sd.MipLevels = 1; + sd.ArraySize = 1; + sd.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + sd.SampleDesc.Count = 1; + sd.Usage = D3D11_USAGE_STAGING; + sd.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + d3d_dev_->CreateTexture2D(&sd, nullptr, &staging_); + } + + d3d_ctx_->CopyResource(staging_.Get(), tex.Get()); + dupl_->ReleaseFrame(); + + D3D11_MAPPED_SUBRESOURCE mapped{}; + hr = d3d_ctx_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped); + if (FAILED(hr)) return false; + + bgra.resize(width * height * 4); + const uint8_t* src = static_cast(mapped.pData); + for (UINT row = 0; row < height; ++row) + memcpy(&bgra[row * width * 4], src + row * mapped.RowPitch, width * 4); + + d3d_ctx_->Unmap(staging_.Get(), 0); + return true; +} + +// ─── Color conversion: BGRA → NV12 ────────────────────────────────────────── + +void ScreenEncoder::BgraToNv12(const uint8_t* bgra, std::vector& nv12, + UINT w, UINT h) { + nv12.resize(w * h * 3 / 2); + uint8_t* Y = nv12.data(); + uint8_t* UV = Y + w * h; + + for (UINT row = 0; row < h; ++row) { + for (UINT col = 0; col < w; ++col) { + const uint8_t* p = bgra + (row * w + col) * 4; + const int b = p[0], g = p[1], r = p[2]; + Y[row * w + col] = + (uint8_t)(((66*r + 129*g + 25*b + 128) >> 8) + 16); + + if ((row & 1) == 0 && (col & 1) == 0) { + const UINT off = (row / 2) * w + col; + UV[off] = (uint8_t)(((-38*r - 74*g + 112*b + 128) >> 8) + 128); + UV[off+1] = (uint8_t)(((112*r - 94*g - 18*b + 128) >> 8) + 128); + } + } + } +} + +// ─── Encode one frame ──────────────────────────────────────────────────────── + +void ScreenEncoder::EncodeFrame(const std::vector& bgra, + UINT w, UINT h, bool keyframe) { + if (enc_width_ != w || enc_height_ != h) { + encoder_.Reset(); + config_sent_ = false; + if (!InitEncoder(w, h)) return; + } + + // Build NV12 input sample + std::vector nv12; + BgraToNv12(bgra.data(), nv12, w, h); + + ComPtr in_sample; + ComPtr in_buf; + MFCreateMemoryBuffer((DWORD)nv12.size(), &in_buf); + { + BYTE* ptr = nullptr; + in_buf->Lock(&ptr, nullptr, nullptr); + memcpy(ptr, nv12.data(), nv12.size()); + in_buf->Unlock(); + } + in_buf->SetCurrentLength((DWORD)nv12.size()); + MFCreateSample(&in_sample); + in_sample->SetSampleTime(sample_ts_); + in_sample->SetSampleDuration(kFrameDuration); + in_sample->AddBuffer(in_buf.Get()); + if (keyframe) in_sample->SetUINT32(MFSampleExtension_CleanPoint, 1); + sample_ts_ += kFrameDuration; + + if (FAILED(encoder_->ProcessInput(0, in_sample.Get(), 0))) return; + + // Drain output samples + MFT_OUTPUT_STREAM_INFO si{}; + encoder_->GetOutputStreamInfo(0, &si); + + while (true) { + ComPtr out_sample; + ComPtr out_buf; + if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) { + MFCreateMemoryBuffer(si.cbSize ? si.cbSize : w * h * 2, &out_buf); + MFCreateSample(&out_sample); + out_sample->AddBuffer(out_buf.Get()); + } + + MFT_OUTPUT_DATA_BUFFER out_data{}; + out_data.pSample = out_sample.Get(); + DWORD status = 0; + HRESULT hr = encoder_->ProcessOutput(0, 1, &out_data, &status); + if (out_data.pEvents) out_data.pEvents->Release(); + if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break; + if (FAILED(hr)) break; + + ComPtr result(out_data.pSample); + if (!result) break; + + ComPtr flat; + result->ConvertToContiguousBuffer(&flat); + BYTE* enc = nullptr; DWORD enc_len = 0; + flat->Lock(&enc, nullptr, &enc_len); + + if (!config_sent_) { + size_t sps_start = 0, pps_end = 0; + if (ExtractSpsPps(enc, enc_len, sps_start, pps_end)) { + std::vector cfg(1 + (pps_end - sps_start)); + cfg[0] = 0x01; + memcpy(&cfg[1], enc + sps_start, pps_end - sps_start); + SendEvent(std::move(cfg)); + config_sent_ = true; + } + } + + const int64_t now_ms = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + std::vector frame(1 + 8 + enc_len); + frame[0] = 0x02; + for (int i = 0; i < 8; ++i) + frame[1 + i] = static_cast(now_ms >> (56 - i * 8)); + memcpy(&frame[9], enc, enc_len); + + flat->Unlock(); + SendEvent(std::move(frame)); + } +} + +void ScreenEncoder::SendEvent(std::vector data) { + std::lock_guard lk(sink_mu_); + if (sink_) sink_->Success(flutter::EncodableValue(std::move(data))); +} + +// ─── Capture loop ──────────────────────────────────────────────────────────── + +void ScreenEncoder::CaptureLoop() { + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + + if (!InitD3D()) { + CoUninitialize(); + return; + } + + while (running_) { + std::vector bgra; + UINT w = 0, h = 0; + if (CaptureFrame(bgra, w, h)) { + bool kf = force_kf_.exchange(false); + EncodeFrame(bgra, w, h, kf); + } + } + + if (encoder_) { + encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0); + encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0); + } + CoUninitialize(); +} diff --git a/windows/runner/screen_encoder.h b/windows/runner/screen_encoder.h new file mode 100644 index 0000000..9f90253 --- /dev/null +++ b/windows/runner/screen_encoder.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +class ScreenEncoder { + public: + ScreenEncoder(); + ~ScreenEncoder(); + + void Start(std::unique_ptr> sink); + void Stop(); + void ForceKeyframe(); + + private: + void CaptureLoop(); + bool InitD3D(); + bool InitEncoder(UINT width, UINT height); + bool CaptureFrame(std::vector& bgra, UINT& width, UINT& height); + void EncodeFrame(const std::vector& bgra, UINT width, UINT height, + bool keyframe); + void BgraToNv12(const uint8_t* bgra, std::vector& nv12, UINT width, + UINT height); + void SendEvent(std::vector data); + + Microsoft::WRL::ComPtr d3d_dev_; + Microsoft::WRL::ComPtr d3d_ctx_; + Microsoft::WRL::ComPtr dupl_; + Microsoft::WRL::ComPtr staging_; + Microsoft::WRL::ComPtr encoder_; + + UINT enc_width_ = 0; + UINT enc_height_ = 0; + bool config_sent_ = false; + LONGLONG sample_ts_ = 0; + + std::unique_ptr> sink_; + std::mutex sink_mu_; + + std::atomic running_ {false}; + std::atomic force_kf_ {false}; + std::thread thread_; +}; diff --git a/windows/runner/video_decoder.cpp b/windows/runner/video_decoder.cpp new file mode 100644 index 0000000..0663e49 --- /dev/null +++ b/windows/runner/video_decoder.cpp @@ -0,0 +1,279 @@ +#include "video_decoder.h" + +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +// ─── Minimal SPS bit-reader ─────────────────────────────────────────────────── + +struct BitReader { + const uint8_t* data; + size_t pos = 0; + + uint32_t Bit() { + uint32_t v = (data[pos / 8] >> (7 - pos % 8)) & 1; + ++pos; + return v; + } + uint32_t Bits(int n) { + uint32_t v = 0; + for (int i = 0; i < n; ++i) v = (v << 1) | Bit(); + return v; + } + uint32_t UEG() { + int lz = 0; + while (Bit() == 0 && lz < 32) ++lz; + if (lz == 0) return 0; + return (1u << lz) - 1 + Bits(lz); + } + int32_t SEG() { + uint32_t code = UEG(); + return (code & 1) ? static_cast((code + 1) / 2) + : -static_cast(code / 2); + } +}; + +// ─── SPS parser ─────────────────────────────────────────────────────────────── + +bool VideoDecoder::ParseSpsSize(const uint8_t* data, size_t len, + int& out_w, int& out_h) { + size_t nal_start = SIZE_MAX; + for (size_t i = 0; i + 4 <= len; ++i) { + if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 && + i + 4 < len && (data[i+4] & 0x1F) == 7) { + nal_start = i + 4; break; + } + if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 && + i + 3 < len && (data[i+3] & 0x1F) == 7) { + nal_start = i + 3; break; + } + } + if (nal_start == SIZE_MAX) return false; + + BitReader br{data + nal_start + 1}; + + uint8_t profile_idc = static_cast(br.Bits(8)); + br.Bits(8); br.Bits(8); br.UEG(); + + static const uint8_t kHiProfiles[] = {100,110,122,244,44,83,86,118,128}; + bool hi = false; + for (uint8_t p : kHiProfiles) if (profile_idc == p) { hi = true; break; } + if (hi) { + uint32_t chroma = br.UEG(); + if (chroma == 3) br.Bit(); + br.UEG(); br.UEG(); br.Bit(); + if (br.Bit()) { + for (int idx = 0; idx < (chroma != 3 ? 8 : 12); ++idx) { + if (br.Bit()) { + int sz = idx < 6 ? 16 : 64; + int last = 8, next = 8; + for (int j = 0; j < sz; ++j) { + if (next != 0) next = (last + br.SEG() + 256) % 256; + last = next == 0 ? last : next; + } + } + } + } + } + + br.UEG(); + uint32_t poc = br.UEG(); + if (poc == 0) br.UEG(); + else if (poc == 1) { br.Bit(); br.SEG(); br.SEG(); uint32_t nc=br.UEG(); for(uint32_t j=0;j(user_data)->pixel_buf_; +} + +// ─── VideoDecoder ───────────────────────────────────────────────────────────── + +VideoDecoder::VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg, + const uint8_t* sps_pps, size_t sps_pps_len) + : tex_reg_(tex_reg) { + Init(sps_pps, sps_pps_len); +} + +VideoDecoder::~VideoDecoder() { + if (texture_id_ >= 0) { + HANDLE done = CreateEvent(nullptr, TRUE, FALSE, nullptr); + FlutterDesktopTextureRegistrarUnregisterExternalTexture( + tex_reg_, texture_id_, + [](void* ud) { SetEvent(static_cast(ud)); }, + done); + WaitForSingleObject(done, 3000); + CloseHandle(done); + } + if (decoder_) + decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0); +} + +bool VideoDecoder::Init(const uint8_t* sps_pps, size_t sps_pps_len) { + int w = 0, h = 0; + if (!ParseSpsSize(sps_pps, sps_pps_len, w, h)) { + w = GetSystemMetrics(SM_CXSCREEN); + h = GetSystemMetrics(SM_CYSCREEN); + } + width_ = w; height_ = h; + + MFT_REGISTER_TYPE_INFO in_info{MFMediaType_Video, MFVideoFormat_H264}; + UINT32 count = 0; IMFActivate** acts = nullptr; + HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER, + MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER, + &in_info, nullptr, &acts, &count); + if (FAILED(hr) || count == 0) return false; + hr = acts[0]->ActivateObject(IID_PPV_ARGS(&decoder_)); + for (UINT32 i = 0; i < count; ++i) acts[i]->Release(); + CoTaskMemFree(acts); + if (FAILED(hr)) return false; + + ComPtr in_type; + MFCreateMediaType(&in_type); + in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); + MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h); + if (FAILED(decoder_->SetInputType(0, in_type.Get(), 0))) return false; + + ComPtr out_type; + MFCreateMediaType(&out_type); + out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h); + if (FAILED(decoder_->SetOutputType(0, out_type.Get(), 0))) return false; + + decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + + // Register Flutter BGRA texture via C API + frame_data_.assign(static_cast(w * h * 4), 0); + pixel_buf_.buffer = frame_data_.data(); + pixel_buf_.width = static_cast(w); + pixel_buf_.height = static_cast(h); + + FlutterDesktopTextureInfo info{}; + info.type = kFlutterDesktopPixelBufferTexture; + info.pixel_buffer_config.callback = &VideoDecoder::PixelBufferCallback; + info.pixel_buffer_config.user_data = this; + + texture_id_ = FlutterDesktopTextureRegistrarRegisterExternalTexture( + tex_reg_, &info); + + // Prime decoder with SPS+PPS + FeedNal(sps_pps, sps_pps_len); + return texture_id_ >= 0; +} + +// ─── NV12 → BGRA ────────────────────────────────────────────────────────────── + +void VideoDecoder::Nv12ToBgra(const uint8_t* nv12, int stride, + uint8_t* bgra, int w, int h) { + const uint8_t* Y = nv12; + const uint8_t* UV = nv12 + stride * h; + + for (int row = 0; row < h; ++row) { + for (int col = 0; col < w; ++col) { + int y = Y[row * stride + col]; + int uv = (row / 2) * stride + (col & ~1); + int u = UV[uv]; + int v = UV[uv + 1]; + + int c = y - 16, d = u - 128, e = v - 128; + int r = (298*c + 409*e + 128) >> 8; + int g = (298*c - 100*d - 208*e + 128) >> 8; + int b = (298*c + 516*d + 128) >> 8; + + uint8_t* p = bgra + (row * w + col) * 4; + p[0] = static_cast(b < 0 ? 0 : b > 255 ? 255 : b); + p[1] = static_cast(g < 0 ? 0 : g > 255 ? 255 : g); + p[2] = static_cast(r < 0 ? 0 : r > 255 ? 255 : r); + p[3] = 255; + } + } +} + +// ─── FeedNal ────────────────────────────────────────────────────────────────── + +void VideoDecoder::FeedNal(const uint8_t* nal, size_t len) { + std::lock_guard lk(mu_); + if (!decoder_ || !len) return; + + ComPtr in_sample; + ComPtr in_buf; + MFCreateMemoryBuffer((DWORD)len, &in_buf); + { + BYTE* ptr = nullptr; + in_buf->Lock(&ptr, nullptr, nullptr); + memcpy(ptr, nal, len); + in_buf->Unlock(); + } + in_buf->SetCurrentLength((DWORD)len); + MFCreateSample(&in_sample); + in_sample->SetSampleTime(input_ts_); + in_sample->SetSampleDuration(333333); + in_sample->AddBuffer(in_buf.Get()); + input_ts_ += 333333; + + if (FAILED(decoder_->ProcessInput(0, in_sample.Get(), 0))) return; + + MFT_OUTPUT_STREAM_INFO si{}; + decoder_->GetOutputStreamInfo(0, &si); + + while (true) { + ComPtr out_sample; + ComPtr out_buf; + if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) { + DWORD sz = si.cbSize ? si.cbSize : static_cast(width_ * height_ * 3 / 2); + MFCreateMemoryBuffer(sz, &out_buf); + MFCreateSample(&out_sample); + out_sample->AddBuffer(out_buf.Get()); + } + + MFT_OUTPUT_DATA_BUFFER out_data{}; + out_data.pSample = out_sample.Get(); + DWORD status = 0; + HRESULT hr = decoder_->ProcessOutput(0, 1, &out_data, &status); + if (out_data.pEvents) out_data.pEvents->Release(); + if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break; + if (FAILED(hr)) break; + + ComPtr result(out_data.pSample); + if (!result) break; + + ComPtr flat; + result->ConvertToContiguousBuffer(&flat); + BYTE* nv12_ptr = nullptr; DWORD nv12_len = 0; + flat->Lock(&nv12_ptr, nullptr, &nv12_len); + Nv12ToBgra(nv12_ptr, width_, frame_data_.data(), width_, height_); + flat->Unlock(); + + FlutterDesktopTextureRegistrarMarkExternalTextureFrameAvailable( + tex_reg_, texture_id_); + } +} diff --git a/windows/runner/video_decoder.h b/windows/runner/video_decoder.h new file mode 100644 index 0000000..4343887 --- /dev/null +++ b/windows/runner/video_decoder.h @@ -0,0 +1,49 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +class VideoDecoder { + public: + // spsPps: Annex-B SPS+PPS bytes (the payload from a 0x01 config packet). + VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg, + const uint8_t* sps_pps, size_t sps_pps_len); + ~VideoDecoder(); + + int64_t texture_id() const { return texture_id_; } + bool is_valid() const { return texture_id_ >= 0; } + + void FeedNal(const uint8_t* nal, size_t len); + + private: + bool Init(const uint8_t* sps_pps, size_t sps_pps_len); + bool ParseSpsSize(const uint8_t* data, size_t len, int& w, int& h); + void Nv12ToBgra(const uint8_t* nv12, int stride, + uint8_t* bgra, int w, int h); + + static const FlutterDesktopPixelBuffer* PixelBufferCallback( + size_t width, size_t height, void* user_data); + + FlutterDesktopTextureRegistrarRef tex_reg_; + int64_t texture_id_ = -1; + + Microsoft::WRL::ComPtr decoder_; + + FlutterDesktopPixelBuffer pixel_buf_{}; + std::vector frame_data_; + + int width_ = 0; + int height_ = 0; + LONGLONG input_ts_ = 0; + + std::mutex mu_; +};