implement video encoding and decoding functionality with screen capture support
This commit is contained in:
parent
bfdaf4a801
commit
37fef7de25
10 changed files with 1163 additions and 48 deletions
|
|
@ -25,11 +25,16 @@ class AppDelegate: FlutterAppDelegate {
|
||||||
private var captureWidth = 1280
|
private var captureWidth = 1280
|
||||||
private var captureHeight = 720
|
private var captureHeight = 720
|
||||||
|
|
||||||
|
// viewer decoder sessions keyed by texture id
|
||||||
|
private var decoders: [Int64: VideoDecoderSession] = [:]
|
||||||
|
private var textureRegistry: FlutterTextureRegistry?
|
||||||
|
|
||||||
override func applicationDidFinishLaunching(_ notification: Notification) {
|
override func applicationDidFinishLaunching(_ notification: Notification) {
|
||||||
guard let window = mainFlutterWindow,
|
guard let window = mainFlutterWindow,
|
||||||
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
let ctrl = window.contentViewController as? FlutterViewController else { return }
|
||||||
|
|
||||||
let messenger = ctrl.engine.binaryMessenger
|
let messenger = ctrl.engine.binaryMessenger
|
||||||
|
textureRegistry = ctrl.engine.textureRegistry
|
||||||
|
|
||||||
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
inputChannel = FlutterMethodChannel(name: "com.pulsar/input", binaryMessenger: messenger)
|
||||||
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
inputChannel?.setMethodCallHandler { [weak self] call, result in
|
||||||
|
|
@ -58,6 +63,40 @@ class AppDelegate: FlutterAppDelegate {
|
||||||
case "forceKeyframe":
|
case "forceKeyframe":
|
||||||
self.requestKeyframe()
|
self.requestKeyframe()
|
||||||
result(nil)
|
result(nil)
|
||||||
|
|
||||||
|
case "createDecoder":
|
||||||
|
guard let args = call.arguments as? [String: Any],
|
||||||
|
let spsPpsTyped = args["spsPps"] as? FlutterStandardTypedData,
|
||||||
|
let registry = self.textureRegistry else {
|
||||||
|
result(FlutterError(code: "INVALID_ARGS", message: nil, details: nil))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if let decoder = VideoDecoderSession(spsPps: spsPpsTyped.data, textureRegistry: registry) {
|
||||||
|
self.decoders[decoder.textureId] = decoder
|
||||||
|
result(["textureId": decoder.textureId])
|
||||||
|
} else {
|
||||||
|
result(FlutterError(code: "DECODER_FAILED", message: "Failed to create H264 decoder", details: nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
case "feedNal":
|
||||||
|
guard let args = call.arguments as? [String: Any],
|
||||||
|
let textureId = (args["textureId"] as? NSNumber)?.int64Value,
|
||||||
|
let nalTyped = args["nal"] as? FlutterStandardTypedData,
|
||||||
|
let decoder = self.decoders[textureId] else {
|
||||||
|
result(nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
decoder.feedNal(nalTyped.data)
|
||||||
|
result(nil)
|
||||||
|
|
||||||
|
case "stopDecoder":
|
||||||
|
if let args = call.arguments as? [String: Any],
|
||||||
|
let textureId = (args["textureId"] as? NSNumber)?.int64Value {
|
||||||
|
self.decoders[textureId]?.invalidate()
|
||||||
|
self.decoders.removeValue(forKey: textureId)
|
||||||
|
}
|
||||||
|
result(nil)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
result(FlutterMethodNotImplemented)
|
result(FlutterMethodNotImplemented)
|
||||||
}
|
}
|
||||||
|
|
@ -441,3 +480,206 @@ extension AppDelegate: FlutterStreamHandler {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MARK: - VT decompression C callback
|
||||||
|
|
||||||
|
private func vtDecompressOutputCallback(
|
||||||
|
refCon: UnsafeMutableRawPointer?,
|
||||||
|
frameRefCon: UnsafeMutableRawPointer?,
|
||||||
|
status: OSStatus,
|
||||||
|
infoFlags: VTDecodeInfoFlags,
|
||||||
|
imageBuffer: CVImageBuffer?,
|
||||||
|
pts: CMTime,
|
||||||
|
duration: CMTime
|
||||||
|
) {
|
||||||
|
guard let refCon = refCon, status == noErr, let imageBuffer = imageBuffer else { return }
|
||||||
|
let session = Unmanaged<VideoDecoderSession>.fromOpaque(refCon).takeUnretainedValue()
|
||||||
|
session.handleDecodedFrame(imageBuffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: - VideoDecoderSession
|
||||||
|
|
||||||
|
class VideoDecoderSession: NSObject, FlutterTexture {
|
||||||
|
private(set) var textureId: Int64 = -1
|
||||||
|
|
||||||
|
private var decompressionSession: VTDecompressionSession?
|
||||||
|
private var formatDescription: CMVideoFormatDescription?
|
||||||
|
private var latestPixelBuffer: CVPixelBuffer?
|
||||||
|
private let lock = NSLock()
|
||||||
|
private weak var textureRegistry: FlutterTextureRegistry?
|
||||||
|
|
||||||
|
init?(spsPps: Data, textureRegistry: FlutterTextureRegistry) {
|
||||||
|
self.textureRegistry = textureRegistry
|
||||||
|
super.init()
|
||||||
|
guard setupSession(spsPps: spsPps) else { return nil }
|
||||||
|
textureId = textureRegistry.register(self)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: Parse Annex-B SPS+PPS and create VTDecompressionSession
|
||||||
|
|
||||||
|
private func setupSession(spsPps: Data) -> Bool {
|
||||||
|
// Split on 4-byte Annex-B start codes to get raw NALU payloads
|
||||||
|
let bytes = [UInt8](spsPps)
|
||||||
|
var nalus: [Data] = []
|
||||||
|
var scanStart = 0
|
||||||
|
var i = 0
|
||||||
|
while i + 4 <= bytes.count {
|
||||||
|
if bytes[i] == 0 && bytes[i+1] == 0 && bytes[i+2] == 0 && bytes[i+3] == 1 {
|
||||||
|
if i > scanStart { nalus.append(Data(bytes[scanStart..<i])) }
|
||||||
|
scanStart = i + 4
|
||||||
|
i += 4
|
||||||
|
} else {
|
||||||
|
i += 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if scanStart < bytes.count { nalus.append(Data(bytes[scanStart...])) }
|
||||||
|
guard nalus.count >= 2 else { return false }
|
||||||
|
|
||||||
|
let sps = nalus[0]
|
||||||
|
let pps = nalus[1]
|
||||||
|
|
||||||
|
var formatDesc: CMVideoFormatDescription?
|
||||||
|
let fmtStatus: OSStatus = sps.withUnsafeBytes { spsBuf in
|
||||||
|
pps.withUnsafeBytes { ppsBuf in
|
||||||
|
var ptrs: [UnsafePointer<UInt8>] = [
|
||||||
|
spsBuf.bindMemory(to: UInt8.self).baseAddress!,
|
||||||
|
ppsBuf.bindMemory(to: UInt8.self).baseAddress!
|
||||||
|
]
|
||||||
|
var sizes: [Int] = [sps.count, pps.count]
|
||||||
|
return CMVideoFormatDescriptionCreateFromH264ParameterSets(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
|
parameterSetCount: 2,
|
||||||
|
parameterSetPointers: &ptrs,
|
||||||
|
parameterSetSizes: &sizes,
|
||||||
|
nalUnitHeaderLength: 4,
|
||||||
|
formatDescriptionOut: &formatDesc
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
guard fmtStatus == noErr, let formatDesc = formatDesc else { return false }
|
||||||
|
self.formatDescription = formatDesc
|
||||||
|
|
||||||
|
let dims = CMVideoFormatDescriptionGetDimensions(formatDesc)
|
||||||
|
let imageAttrs: [CFString: Any] = [
|
||||||
|
kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA,
|
||||||
|
kCVPixelBufferWidthKey: Int(dims.width),
|
||||||
|
kCVPixelBufferHeightKey: Int(dims.height),
|
||||||
|
kCVPixelBufferIOSurfacePropertiesKey: [:] as [String: Any]
|
||||||
|
]
|
||||||
|
|
||||||
|
var callback = VTDecompressionOutputCallbackRecord(
|
||||||
|
decompressionOutputCallback: vtDecompressOutputCallback,
|
||||||
|
decompressionOutputRefCon: Unmanaged.passUnretained(self).toOpaque()
|
||||||
|
)
|
||||||
|
|
||||||
|
var session: VTDecompressionSession?
|
||||||
|
let sessionStatus = VTDecompressionSessionCreate(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
|
formatDescription: formatDesc,
|
||||||
|
decoderSpecification: nil,
|
||||||
|
imageBufferAttributes: imageAttrs as CFDictionary,
|
||||||
|
outputCallback: &callback,
|
||||||
|
decompressionSessionOut: &session
|
||||||
|
)
|
||||||
|
guard sessionStatus == noErr, let session = session else { return false }
|
||||||
|
self.decompressionSession = session
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: Feed an Annex-B NAL unit for decoding
|
||||||
|
|
||||||
|
func feedNal(_ data: Data) {
|
||||||
|
guard let session = decompressionSession,
|
||||||
|
let formatDesc = formatDescription,
|
||||||
|
data.count > 4 else { return }
|
||||||
|
|
||||||
|
// Convert Annex-B start code to AVCC 4-byte big-endian length prefix
|
||||||
|
let naluLen = UInt32(data.count - 4).bigEndian
|
||||||
|
var avcc = Data(capacity: data.count)
|
||||||
|
withUnsafeBytes(of: naluLen) { avcc.append(contentsOf: $0) }
|
||||||
|
avcc.append(data.advanced(by: 4))
|
||||||
|
|
||||||
|
var blockBuffer: CMBlockBuffer?
|
||||||
|
guard CMBlockBufferCreateWithMemoryBlock(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
|
memoryBlock: nil,
|
||||||
|
blockLength: avcc.count,
|
||||||
|
blockAllocator: kCFAllocatorDefault,
|
||||||
|
customBlockSource: nil,
|
||||||
|
offsetToData: 0,
|
||||||
|
dataLength: avcc.count,
|
||||||
|
flags: 0,
|
||||||
|
blockBufferOut: &blockBuffer
|
||||||
|
) == kCMBlockBufferNoErr, let blockBuffer = blockBuffer else { return }
|
||||||
|
|
||||||
|
avcc.withUnsafeBytes { ptr in
|
||||||
|
_ = CMBlockBufferReplaceDataBytes(
|
||||||
|
with: ptr.baseAddress!,
|
||||||
|
blockBuffer: blockBuffer,
|
||||||
|
offsetIntoDestination: 0,
|
||||||
|
dataLength: avcc.count
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
var timingInfo = CMSampleTimingInfo(
|
||||||
|
duration: CMTime(value: 1, timescale: 60),
|
||||||
|
presentationTimeStamp: CMTime(seconds: CACurrentMediaTime(), preferredTimescale: 600),
|
||||||
|
decodeTimeStamp: .invalid
|
||||||
|
)
|
||||||
|
var sampleBuffer: CMSampleBuffer?
|
||||||
|
guard CMSampleBufferCreate(
|
||||||
|
allocator: kCFAllocatorDefault,
|
||||||
|
dataBuffer: blockBuffer,
|
||||||
|
dataReady: true,
|
||||||
|
makeDataReadyCallback: nil,
|
||||||
|
refcon: nil,
|
||||||
|
formatDescription: formatDesc,
|
||||||
|
sampleCount: 1,
|
||||||
|
sampleTimingEntryCount: 1,
|
||||||
|
sampleTimingArray: &timingInfo,
|
||||||
|
sampleSizeEntryCount: 0,
|
||||||
|
sampleSizeArray: nil,
|
||||||
|
sampleBufferOut: &sampleBuffer
|
||||||
|
) == noErr, let sampleBuffer = sampleBuffer else { return }
|
||||||
|
|
||||||
|
VTDecompressionSessionDecodeFrame(
|
||||||
|
session,
|
||||||
|
sampleBuffer: sampleBuffer,
|
||||||
|
flags: [],
|
||||||
|
frameRefcon: nil,
|
||||||
|
infoFlagsOut: nil
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: Called from the VT C callback
|
||||||
|
|
||||||
|
func handleDecodedFrame(_ imageBuffer: CVImageBuffer) {
|
||||||
|
lock.lock()
|
||||||
|
latestPixelBuffer = imageBuffer as CVPixelBuffer
|
||||||
|
lock.unlock()
|
||||||
|
DispatchQueue.main.async { [weak self] in
|
||||||
|
guard let self = self else { return }
|
||||||
|
self.textureRegistry?.textureFrameAvailable(self.textureId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: FlutterTexture
|
||||||
|
|
||||||
|
func copyPixelBuffer() -> Unmanaged<CVPixelBuffer>? {
|
||||||
|
lock.lock()
|
||||||
|
defer { lock.unlock() }
|
||||||
|
guard let pb = latestPixelBuffer else { return nil }
|
||||||
|
return Unmanaged.passRetained(pb)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MARK: Teardown
|
||||||
|
|
||||||
|
func invalidate() {
|
||||||
|
if let session = decompressionSession {
|
||||||
|
VTDecompressionSessionInvalidate(session)
|
||||||
|
}
|
||||||
|
decompressionSession = nil
|
||||||
|
latestPixelBuffer = nil
|
||||||
|
textureRegistry?.unregisterTexture(textureId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -239,10 +239,10 @@ packages:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
name: meta
|
name: meta
|
||||||
sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349"
|
sha256: "23f08335362185a5ea2ad3a4e597f1375e78bce8a040df5c600c8d3552ef2394"
|
||||||
url: "https://pub.dev"
|
url: "https://pub.dev"
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "1.18.0"
|
version: "1.17.0"
|
||||||
native_toolchain_c:
|
native_toolchain_c:
|
||||||
dependency: transitive
|
dependency: transitive
|
||||||
description:
|
description:
|
||||||
|
|
@ -481,5 +481,5 @@ packages:
|
||||||
source: hosted
|
source: hosted
|
||||||
version: "3.1.3"
|
version: "3.1.3"
|
||||||
sdks:
|
sdks:
|
||||||
dart: ">=3.12.0-210.1.beta <4.0.0"
|
dart: ">=3.11.0 <4.0.0"
|
||||||
flutter: ">=3.38.4"
|
flutter: ">=3.38.4"
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev
|
||||||
version: 1.0.0+1
|
version: 1.0.0+1
|
||||||
|
|
||||||
environment:
|
environment:
|
||||||
sdk: ^3.12.0-210.1.beta
|
sdk: ^3.11.0
|
||||||
|
|
||||||
# Dependencies specify other packages that your package needs in order to work.
|
# Dependencies specify other packages that your package needs in order to work.
|
||||||
# To automatically upgrade your package dependencies to the latest versions
|
# To automatically upgrade your package dependencies to the latest versions
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,8 @@ add_executable(${BINARY_NAME} WIN32
|
||||||
"main.cpp"
|
"main.cpp"
|
||||||
"utils.cpp"
|
"utils.cpp"
|
||||||
"win32_window.cpp"
|
"win32_window.cpp"
|
||||||
|
"screen_encoder.cpp"
|
||||||
|
"video_decoder.cpp"
|
||||||
"${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc"
|
"${FLUTTER_MANAGED_DIR}/generated_plugin_registrant.cc"
|
||||||
"Runner.rc"
|
"Runner.rc"
|
||||||
"runner.exe.manifest"
|
"runner.exe.manifest"
|
||||||
|
|
@ -34,6 +36,14 @@ target_compile_definitions(${BINARY_NAME} PRIVATE "NOMINMAX")
|
||||||
# dependencies here.
|
# dependencies here.
|
||||||
target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app)
|
target_link_libraries(${BINARY_NAME} PRIVATE flutter flutter_wrapper_app)
|
||||||
target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib")
|
target_link_libraries(${BINARY_NAME} PRIVATE "dwmapi.lib")
|
||||||
|
target_link_libraries(${BINARY_NAME} PRIVATE
|
||||||
|
"d3d11.lib"
|
||||||
|
"dxgi.lib"
|
||||||
|
"mf.lib"
|
||||||
|
"mfplat.lib"
|
||||||
|
"mfuuid.lib"
|
||||||
|
"propsys.lib"
|
||||||
|
)
|
||||||
target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}")
|
target_include_directories(${BINARY_NAME} PRIVATE "${CMAKE_SOURCE_DIR}")
|
||||||
|
|
||||||
# Run the Flutter tool portions of the build. This must not be removed.
|
# Run the Flutter tool portions of the build. This must not be removed.
|
||||||
|
|
|
||||||
|
|
@ -3,10 +3,15 @@
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include <flutter/event_channel.h>
|
||||||
|
#include <flutter/event_stream_handler_functions.h>
|
||||||
#include <flutter/method_channel.h>
|
#include <flutter/method_channel.h>
|
||||||
#include <flutter/standard_method_codec.h>
|
#include <flutter/standard_method_codec.h>
|
||||||
#include <flutter/encodable_value.h>
|
#include <flutter/encodable_value.h>
|
||||||
#include <windows.h>
|
#include <windows.h>
|
||||||
|
#include <mfapi.h>
|
||||||
|
#include <flutter_plugin_registrar.h>
|
||||||
|
#include <flutter_texture_registrar.h>
|
||||||
|
|
||||||
#include "flutter/generated_plugin_registrant.h"
|
#include "flutter/generated_plugin_registrant.h"
|
||||||
|
|
||||||
|
|
@ -15,23 +20,22 @@ FlutterWindow::FlutterWindow(const flutter::DartProject& project)
|
||||||
|
|
||||||
FlutterWindow::~FlutterWindow() {}
|
FlutterWindow::~FlutterWindow() {}
|
||||||
|
|
||||||
|
// ─── Input injection helper ──────────────────────────────────────────────────
|
||||||
|
|
||||||
static void HandleInjectInput(const flutter::EncodableMap& args) {
|
static void HandleInjectInput(const flutter::EncodableMap& args) {
|
||||||
auto typeIt = args.find(flutter::EncodableValue("type"));
|
auto typeIt = args.find(flutter::EncodableValue("type"));
|
||||||
if (typeIt == args.end()) return;
|
if (typeIt == args.end()) return;
|
||||||
|
|
||||||
std::string type = std::get<std::string>(typeIt->second);
|
std::string type = std::get<std::string>(typeIt->second);
|
||||||
|
|
||||||
int screenW = GetSystemMetrics(SM_CXSCREEN);
|
|
||||||
int screenH = GetSystemMetrics(SM_CYSCREEN);
|
|
||||||
|
|
||||||
if (type == "move") {
|
if (type == "move") {
|
||||||
double nx = std::get<double>(args.at(flutter::EncodableValue("x")));
|
double nx = std::get<double>(args.at(flutter::EncodableValue("x")));
|
||||||
double ny = std::get<double>(args.at(flutter::EncodableValue("y")));
|
double ny = std::get<double>(args.at(flutter::EncodableValue("y")));
|
||||||
|
|
||||||
INPUT input = {};
|
INPUT input = {};
|
||||||
input.type = INPUT_MOUSE;
|
input.type = INPUT_MOUSE;
|
||||||
input.mi.dx = (LONG)(nx * 65535);
|
input.mi.dx = (LONG)(nx * 65535);
|
||||||
input.mi.dy = (LONG)(ny * 65535);
|
input.mi.dy = (LONG)(ny * 65535);
|
||||||
input.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
input.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
||||||
SendInput(1, &input, sizeof(INPUT));
|
SendInput(1, &input, sizeof(INPUT));
|
||||||
|
|
||||||
|
|
@ -42,18 +46,16 @@ static void HandleInjectInput(const flutter::EncodableMap& args) {
|
||||||
auto btnIt = args.find(flutter::EncodableValue("button"));
|
auto btnIt = args.find(flutter::EncodableValue("button"));
|
||||||
if (btnIt != args.end()) btn = std::get<int>(btnIt->second);
|
if (btnIt != args.end()) btn = std::get<int>(btnIt->second);
|
||||||
|
|
||||||
// move first
|
|
||||||
INPUT moveInput = {};
|
INPUT moveInput = {};
|
||||||
moveInput.type = INPUT_MOUSE;
|
moveInput.type = INPUT_MOUSE;
|
||||||
moveInput.mi.dx = (LONG)(nx * 65535);
|
moveInput.mi.dx = (LONG)(nx * 65535);
|
||||||
moveInput.mi.dy = (LONG)(ny * 65535);
|
moveInput.mi.dy = (LONG)(ny * 65535);
|
||||||
moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
moveInput.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE;
|
||||||
SendInput(1, &moveInput, sizeof(INPUT));
|
SendInput(1, &moveInput, sizeof(INPUT));
|
||||||
|
|
||||||
INPUT clickInput[2] = {};
|
INPUT clickInput[2] = {};
|
||||||
clickInput[0].type = INPUT_MOUSE;
|
clickInput[0].type = INPUT_MOUSE;
|
||||||
clickInput[1].type = INPUT_MOUSE;
|
clickInput[1].type = INPUT_MOUSE;
|
||||||
|
|
||||||
if (btn == 1) {
|
if (btn == 1) {
|
||||||
clickInput[0].mi.dwFlags = MOUSEEVENTF_RIGHTDOWN;
|
clickInput[0].mi.dwFlags = MOUSEEVENTF_RIGHTDOWN;
|
||||||
clickInput[1].mi.dwFlags = MOUSEEVENTF_RIGHTUP;
|
clickInput[1].mi.dwFlags = MOUSEEVENTF_RIGHTUP;
|
||||||
|
|
@ -68,48 +70,156 @@ static void HandleInjectInput(const flutter::EncodableMap& args) {
|
||||||
if (keyIt == args.end()) return;
|
if (keyIt == args.end()) return;
|
||||||
|
|
||||||
// Flutter logical key IDs dont map 1:1 to VKs — use low byte as rough VK
|
// Flutter logical key IDs dont map 1:1 to VKs — use low byte as rough VK
|
||||||
int logicalKey = std::get<int>(keyIt->second);
|
int logicalKey = std::get<int>(keyIt->second);
|
||||||
WORD vk = (WORD)(logicalKey & 0xFF);
|
WORD vk = (WORD)(logicalKey & 0xFF);
|
||||||
|
|
||||||
INPUT keyInput = {};
|
INPUT keyInput = {};
|
||||||
keyInput.type = INPUT_KEYBOARD;
|
keyInput.type = INPUT_KEYBOARD;
|
||||||
keyInput.ki.wVk = vk;
|
keyInput.ki.wVk = vk;
|
||||||
if (type == "keyup") keyInput.ki.dwFlags = KEYEVENTF_KEYUP;
|
if (type == "keyup") keyInput.ki.dwFlags = KEYEVENTF_KEYUP;
|
||||||
|
|
||||||
SendInput(1, &keyInput, sizeof(INPUT));
|
SendInput(1, &keyInput, sizeof(INPUT));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── OnCreate ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
bool FlutterWindow::OnCreate() {
|
bool FlutterWindow::OnCreate() {
|
||||||
if (!Win32Window::OnCreate()) {
|
if (!Win32Window::OnCreate()) return false;
|
||||||
return false;
|
|
||||||
}
|
MFStartup(MF_VERSION);
|
||||||
|
|
||||||
RECT frame = GetClientArea();
|
RECT frame = GetClientArea();
|
||||||
|
|
||||||
flutter_controller_ = std::make_unique<flutter::FlutterViewController>(
|
flutter_controller_ = std::make_unique<flutter::FlutterViewController>(
|
||||||
frame.right - frame.left, frame.bottom - frame.top, project_);
|
frame.right - frame.left, frame.bottom - frame.top, project_);
|
||||||
|
|
||||||
if (!flutter_controller_->engine() || !flutter_controller_->view()) {
|
if (!flutter_controller_->engine() || !flutter_controller_->view())
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
RegisterPlugins(flutter_controller_->engine());
|
RegisterPlugins(flutter_controller_->engine());
|
||||||
|
|
||||||
// register input injection channel
|
auto* messenger = flutter_controller_->engine()->messenger();
|
||||||
auto channel = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
|
||||||
flutter_controller_->engine()->messenger(),
|
|
||||||
"com.pulsar/input",
|
|
||||||
&flutter::StandardMethodCodec::GetInstance()
|
|
||||||
);
|
|
||||||
|
|
||||||
channel->SetMethodCallHandler(
|
// Get the texture registrar via the C API (avoids linking flutter_wrapper_plugin).
|
||||||
|
auto reg_ref = flutter_controller_->engine()->GetRegistrarForPlugin("pulsar");
|
||||||
|
tex_reg_ = FlutterDesktopRegistrarGetTextureRegistrar(reg_ref);
|
||||||
|
|
||||||
|
// ── com.pulsar/input MethodChannel ──────────────────────────────────────
|
||||||
|
auto input_ch = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
||||||
|
messenger, "com.pulsar/input",
|
||||||
|
&flutter::StandardMethodCodec::GetInstance());
|
||||||
|
|
||||||
|
input_ch->SetMethodCallHandler(
|
||||||
[](const flutter::MethodCall<flutter::EncodableValue>& call,
|
[](const flutter::MethodCall<flutter::EncodableValue>& call,
|
||||||
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
||||||
if (call.method_name() == "injectInput") {
|
if (call.method_name() == "injectInput") {
|
||||||
if (const auto* args = std::get_if<flutter::EncodableMap>(call.arguments())) {
|
if (const auto* args =
|
||||||
|
std::get_if<flutter::EncodableMap>(call.arguments()))
|
||||||
HandleInjectInput(*args);
|
HandleInjectInput(*args);
|
||||||
|
result->Success();
|
||||||
|
} else {
|
||||||
|
result->NotImplemented();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── com.pulsar/video/frames EventChannel ────────────────────────────────
|
||||||
|
screen_encoder_ = std::make_unique<ScreenEncoder>();
|
||||||
|
auto* enc = screen_encoder_.get(); // raw ptr safe: outlives channels
|
||||||
|
|
||||||
|
auto frame_ch = std::make_shared<flutter::EventChannel<flutter::EncodableValue>>(
|
||||||
|
messenger, "com.pulsar/video/frames",
|
||||||
|
&flutter::StandardMethodCodec::GetInstance());
|
||||||
|
|
||||||
|
frame_ch->SetStreamHandler(
|
||||||
|
std::make_unique<flutter::StreamHandlerFunctions<flutter::EncodableValue>>(
|
||||||
|
[enc](const flutter::EncodableValue*,
|
||||||
|
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>>&& sink)
|
||||||
|
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
||||||
|
enc->Start(std::move(sink));
|
||||||
|
return nullptr;
|
||||||
|
},
|
||||||
|
[enc](const flutter::EncodableValue*)
|
||||||
|
-> std::unique_ptr<flutter::StreamHandlerError<flutter::EncodableValue>> {
|
||||||
|
enc->Stop();
|
||||||
|
return nullptr;
|
||||||
|
}));
|
||||||
|
|
||||||
|
// ── com.pulsar/video MethodChannel ──────────────────────────────────────
|
||||||
|
auto* decoders = &decoders_; // raw ptr safe: outlives channels
|
||||||
|
auto video_ch = std::make_shared<flutter::MethodChannel<flutter::EncodableValue>>(
|
||||||
|
messenger, "com.pulsar/video",
|
||||||
|
&flutter::StandardMethodCodec::GetInstance());
|
||||||
|
|
||||||
|
video_ch->SetMethodCallHandler(
|
||||||
|
[enc, decoders, this](
|
||||||
|
const flutter::MethodCall<flutter::EncodableValue>& call,
|
||||||
|
std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result) {
|
||||||
|
const auto& m = call.method_name();
|
||||||
|
|
||||||
|
if (m == "start") {
|
||||||
|
result->Success();
|
||||||
|
|
||||||
|
} else if (m == "stop") {
|
||||||
|
enc->Stop();
|
||||||
|
result->Success();
|
||||||
|
|
||||||
|
} else if (m == "forceKeyframe") {
|
||||||
|
enc->ForceKeyframe();
|
||||||
|
result->Success();
|
||||||
|
|
||||||
|
} else if (m == "createDecoder") {
|
||||||
|
const auto* args =
|
||||||
|
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||||
|
if (!args) { result->Error("BAD_ARGS", ""); return; }
|
||||||
|
|
||||||
|
auto it = args->find(flutter::EncodableValue("spsPps"));
|
||||||
|
if (it == args->end()) {
|
||||||
|
result->Error("BAD_ARGS", "missing spsPps");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const auto& sps_pps =
|
||||||
|
std::get<std::vector<uint8_t>>(it->second);
|
||||||
|
|
||||||
|
auto dec = std::make_unique<VideoDecoder>(
|
||||||
|
tex_reg_, sps_pps.data(), sps_pps.size());
|
||||||
|
if (!dec->is_valid()) {
|
||||||
|
result->Error("INIT_FAILED", "decoder init failed");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int64_t id = dec->texture_id();
|
||||||
|
(*decoders)[id] = std::move(dec);
|
||||||
|
result->Success(flutter::EncodableValue(flutter::EncodableMap{
|
||||||
|
{flutter::EncodableValue("textureId"),
|
||||||
|
flutter::EncodableValue(id)}}));
|
||||||
|
|
||||||
|
} else if (m == "stopDecoder") {
|
||||||
|
const auto* args =
|
||||||
|
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||||
|
if (args) {
|
||||||
|
auto it = args->find(flutter::EncodableValue("textureId"));
|
||||||
|
if (it != args->end())
|
||||||
|
decoders->erase(std::get<int64_t>(it->second));
|
||||||
}
|
}
|
||||||
result->Success();
|
result->Success();
|
||||||
|
|
||||||
|
} else if (m == "feedNal") {
|
||||||
|
const auto* args =
|
||||||
|
std::get_if<flutter::EncodableMap>(call.arguments());
|
||||||
|
if (!args) { result->Error("BAD_ARGS", ""); return; }
|
||||||
|
|
||||||
|
auto id_it = args->find(flutter::EncodableValue("textureId"));
|
||||||
|
auto nal_it = args->find(flutter::EncodableValue("nal"));
|
||||||
|
if (id_it == args->end() || nal_it == args->end()) {
|
||||||
|
result->Error("BAD_ARGS", "missing args");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int64_t id = std::get<int64_t>(id_it->second);
|
||||||
|
const auto& nal = std::get<std::vector<uint8_t>>(nal_it->second);
|
||||||
|
|
||||||
|
auto dit = decoders->find(id);
|
||||||
|
if (dit != decoders->end())
|
||||||
|
dit->second->FeedNal(nal.data(), nal.size());
|
||||||
|
result->Success();
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
result->NotImplemented();
|
result->NotImplemented();
|
||||||
}
|
}
|
||||||
|
|
@ -117,27 +227,31 @@ bool FlutterWindow::OnCreate() {
|
||||||
|
|
||||||
SetChildContent(flutter_controller_->view()->GetNativeWindow());
|
SetChildContent(flutter_controller_->view()->GetNativeWindow());
|
||||||
|
|
||||||
flutter_controller_->engine()->SetNextFrameCallback([&]() {
|
flutter_controller_->engine()->SetNextFrameCallback([&]() { this->Show(); });
|
||||||
this->Show();
|
|
||||||
});
|
|
||||||
flutter_controller_->ForceRedraw();
|
flutter_controller_->ForceRedraw();
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── OnDestroy ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
void FlutterWindow::OnDestroy() {
|
void FlutterWindow::OnDestroy() {
|
||||||
if (flutter_controller_) {
|
decoders_.clear();
|
||||||
flutter_controller_ = nullptr;
|
screen_encoder_.reset();
|
||||||
}
|
if (flutter_controller_) flutter_controller_ = nullptr;
|
||||||
|
|
||||||
|
MFShutdown();
|
||||||
Win32Window::OnDestroy();
|
Win32Window::OnDestroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── MessageHandler ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
LRESULT FlutterWindow::MessageHandler(HWND hwnd, UINT const message,
|
LRESULT FlutterWindow::MessageHandler(HWND hwnd, UINT const message,
|
||||||
WPARAM const wparam,
|
WPARAM const wparam,
|
||||||
LPARAM const lparam) noexcept {
|
LPARAM const lparam) noexcept {
|
||||||
if (flutter_controller_) {
|
if (flutter_controller_) {
|
||||||
std::optional<LRESULT> result =
|
std::optional<LRESULT> result =
|
||||||
flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam, lparam);
|
flutter_controller_->HandleTopLevelWindowProc(hwnd, message, wparam,
|
||||||
|
lparam);
|
||||||
if (result) return *result;
|
if (result) return *result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,31 +3,34 @@
|
||||||
|
|
||||||
#include <flutter/dart_project.h>
|
#include <flutter/dart_project.h>
|
||||||
#include <flutter/flutter_view_controller.h>
|
#include <flutter/flutter_view_controller.h>
|
||||||
|
#include <flutter_plugin_registrar.h>
|
||||||
|
#include <flutter_texture_registrar.h>
|
||||||
|
|
||||||
|
#include <map>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
|
||||||
|
#include "screen_encoder.h"
|
||||||
|
#include "video_decoder.h"
|
||||||
#include "win32_window.h"
|
#include "win32_window.h"
|
||||||
|
|
||||||
// A window that does nothing but host a Flutter view.
|
|
||||||
class FlutterWindow : public Win32Window {
|
class FlutterWindow : public Win32Window {
|
||||||
public:
|
public:
|
||||||
// Creates a new FlutterWindow hosting a Flutter view running |project|.
|
|
||||||
explicit FlutterWindow(const flutter::DartProject& project);
|
explicit FlutterWindow(const flutter::DartProject& project);
|
||||||
virtual ~FlutterWindow();
|
virtual ~FlutterWindow();
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
// Win32Window:
|
|
||||||
bool OnCreate() override;
|
bool OnCreate() override;
|
||||||
void OnDestroy() override;
|
void OnDestroy() override;
|
||||||
LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam,
|
LRESULT MessageHandler(HWND window, UINT const message, WPARAM const wparam,
|
||||||
LPARAM const lparam) noexcept override;
|
LPARAM const lparam) noexcept override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// The project to run.
|
|
||||||
flutter::DartProject project_;
|
flutter::DartProject project_;
|
||||||
|
|
||||||
// The Flutter instance hosted by this window.
|
|
||||||
std::unique_ptr<flutter::FlutterViewController> flutter_controller_;
|
std::unique_ptr<flutter::FlutterViewController> flutter_controller_;
|
||||||
|
|
||||||
|
FlutterDesktopTextureRegistrarRef tex_reg_ = nullptr;
|
||||||
|
std::unique_ptr<ScreenEncoder> screen_encoder_;
|
||||||
|
std::map<int64_t, std::unique_ptr<VideoDecoder>> decoders_;
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif // RUNNER_FLUTTER_WINDOW_H_
|
#endif // RUNNER_FLUTTER_WINDOW_H_
|
||||||
|
|
|
||||||
361
windows/runner/screen_encoder.cpp
Normal file
361
windows/runner/screen_encoder.cpp
Normal file
|
|
@ -0,0 +1,361 @@
|
||||||
|
#include "screen_encoder.h"
|
||||||
|
|
||||||
|
#include <chrono>
|
||||||
|
#include <codecapi.h>
|
||||||
|
#include <mfapi.h>
|
||||||
|
#include <mferror.h>
|
||||||
|
#include <mfidl.h>
|
||||||
|
|
||||||
|
using Microsoft::WRL::ComPtr;
|
||||||
|
|
||||||
|
static constexpr LONGLONG kFrameDuration = 333333; // 100ns units ≈ 30fps
|
||||||
|
static constexpr UINT32 kBitrate = 6'000'000;
|
||||||
|
|
||||||
|
// Scan an Annex-B bitstream for the span covering the SPS (type 7) through
|
||||||
|
// the end of the PPS (type 8). Returns false if either is missing.
|
||||||
|
static bool ExtractSpsPps(const uint8_t* data, size_t size,
|
||||||
|
size_t& sps_start, size_t& pps_end) {
|
||||||
|
struct Nal { size_t start; int type; };
|
||||||
|
std::vector<Nal> nals;
|
||||||
|
|
||||||
|
size_t i = 0;
|
||||||
|
while (i + 4 <= size) {
|
||||||
|
size_t hdr_len = 0;
|
||||||
|
int nal_type = 0;
|
||||||
|
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 &&
|
||||||
|
i + 4 < size) {
|
||||||
|
hdr_len = 4;
|
||||||
|
nal_type = data[i+4] & 0x1F;
|
||||||
|
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 &&
|
||||||
|
i + 3 < size) {
|
||||||
|
hdr_len = 3;
|
||||||
|
nal_type = data[i+3] & 0x1F;
|
||||||
|
}
|
||||||
|
if (hdr_len) {
|
||||||
|
nals.push_back({i, nal_type});
|
||||||
|
i += hdr_len;
|
||||||
|
} else {
|
||||||
|
++i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t sps_idx = SIZE_MAX, pps_idx = SIZE_MAX;
|
||||||
|
for (size_t j = 0; j < nals.size(); ++j) {
|
||||||
|
if (nals[j].type == 7) sps_idx = j;
|
||||||
|
if (nals[j].type == 8) pps_idx = j;
|
||||||
|
}
|
||||||
|
if (sps_idx == SIZE_MAX || pps_idx == SIZE_MAX) return false;
|
||||||
|
|
||||||
|
sps_start = nals[sps_idx].start;
|
||||||
|
size_t after_pps = pps_idx + 1;
|
||||||
|
pps_end = (after_pps < nals.size()) ? nals[after_pps].start : size;
|
||||||
|
return sps_start < pps_end;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── ScreenEncoder ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
ScreenEncoder::ScreenEncoder() {}
|
||||||
|
|
||||||
|
ScreenEncoder::~ScreenEncoder() { Stop(); }
|
||||||
|
|
||||||
|
void ScreenEncoder::Start(
|
||||||
|
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink) {
|
||||||
|
Stop();
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||||
|
sink_ = std::move(sink);
|
||||||
|
}
|
||||||
|
config_sent_ = false;
|
||||||
|
sample_ts_ = 0;
|
||||||
|
running_ = true;
|
||||||
|
thread_ = std::thread(&ScreenEncoder::CaptureLoop, this);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScreenEncoder::Stop() {
|
||||||
|
running_ = false;
|
||||||
|
if (thread_.joinable()) thread_.join();
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||||
|
sink_.reset();
|
||||||
|
}
|
||||||
|
encoder_.Reset();
|
||||||
|
dupl_.Reset();
|
||||||
|
staging_.Reset();
|
||||||
|
d3d_ctx_.Reset();
|
||||||
|
d3d_dev_.Reset();
|
||||||
|
enc_width_ = enc_height_ = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScreenEncoder::ForceKeyframe() { force_kf_ = true; }
|
||||||
|
|
||||||
|
// ─── D3D / DXGI init ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
bool ScreenEncoder::InitD3D() {
|
||||||
|
D3D_FEATURE_LEVEL level;
|
||||||
|
HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, 0,
|
||||||
|
nullptr, 0, D3D11_SDK_VERSION,
|
||||||
|
&d3d_dev_, &level, &d3d_ctx_);
|
||||||
|
if (FAILED(hr)) return false;
|
||||||
|
|
||||||
|
ComPtr<IDXGIDevice> dxgi_dev;
|
||||||
|
ComPtr<IDXGIAdapter> adapter;
|
||||||
|
ComPtr<IDXGIOutput> output;
|
||||||
|
ComPtr<IDXGIOutput1> output1;
|
||||||
|
|
||||||
|
d3d_dev_.As(&dxgi_dev);
|
||||||
|
dxgi_dev->GetAdapter(&adapter);
|
||||||
|
if (FAILED(adapter->EnumOutputs(0, &output))) return false;
|
||||||
|
if (FAILED(output.As(&output1))) return false;
|
||||||
|
|
||||||
|
hr = output1->DuplicateOutput(d3d_dev_.Get(), &dupl_);
|
||||||
|
return SUCCEEDED(hr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Encoder init ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
bool ScreenEncoder::InitEncoder(UINT width, UINT height) {
|
||||||
|
MFT_REGISTER_TYPE_INFO out_info{MFMediaType_Video, MFVideoFormat_H264};
|
||||||
|
|
||||||
|
UINT32 count = 0;
|
||||||
|
IMFActivate** activates = nullptr;
|
||||||
|
HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER,
|
||||||
|
MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER,
|
||||||
|
nullptr, &out_info, &activates, &count);
|
||||||
|
if (FAILED(hr) || count == 0) return false;
|
||||||
|
|
||||||
|
hr = activates[0]->ActivateObject(IID_PPV_ARGS(&encoder_));
|
||||||
|
for (UINT32 i = 0; i < count; ++i) activates[i]->Release();
|
||||||
|
CoTaskMemFree(activates);
|
||||||
|
if (FAILED(hr)) return false;
|
||||||
|
|
||||||
|
// Output: H264
|
||||||
|
ComPtr<IMFMediaType> out_type;
|
||||||
|
MFCreateMediaType(&out_type);
|
||||||
|
out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||||
|
out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
|
||||||
|
MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, width, height);
|
||||||
|
MFSetAttributeRatio(out_type.Get(), MF_MT_FRAME_RATE, 30, 1);
|
||||||
|
out_type->SetUINT32(MF_MT_AVG_BITRATE, kBitrate);
|
||||||
|
out_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
|
||||||
|
out_type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_High);
|
||||||
|
if (FAILED(encoder_->SetOutputType(0, out_type.Get(), 0))) return false;
|
||||||
|
|
||||||
|
// Input: NV12
|
||||||
|
ComPtr<IMFMediaType> in_type;
|
||||||
|
MFCreateMediaType(&in_type);
|
||||||
|
in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||||
|
in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
|
||||||
|
MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, width, height);
|
||||||
|
MFSetAttributeRatio(in_type.Get(), MF_MT_FRAME_RATE, 30, 1);
|
||||||
|
in_type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
|
||||||
|
in_type->SetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32)width);
|
||||||
|
if (FAILED(encoder_->SetInputType(0, in_type.Get(), 0))) return false;
|
||||||
|
|
||||||
|
// Keyframe every 30 frames via media type attribute
|
||||||
|
out_type->SetUINT32(MF_MT_MAX_KEYFRAME_SPACING, 30);
|
||||||
|
|
||||||
|
encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0);
|
||||||
|
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
|
||||||
|
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
|
||||||
|
|
||||||
|
enc_width_ = width;
|
||||||
|
enc_height_ = height;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Frame capture ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
bool ScreenEncoder::CaptureFrame(std::vector<uint8_t>& bgra,
|
||||||
|
UINT& width, UINT& height) {
|
||||||
|
DXGI_OUTDUPL_FRAME_INFO info{};
|
||||||
|
ComPtr<IDXGIResource> res;
|
||||||
|
|
||||||
|
HRESULT hr = dupl_->AcquireNextFrame(16, &info, &res);
|
||||||
|
if (hr == DXGI_ERROR_WAIT_TIMEOUT) return false;
|
||||||
|
if (FAILED(hr)) {
|
||||||
|
dupl_.Reset();
|
||||||
|
staging_.Reset();
|
||||||
|
InitD3D();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
ComPtr<ID3D11Texture2D> tex;
|
||||||
|
res.As(&tex);
|
||||||
|
|
||||||
|
D3D11_TEXTURE2D_DESC desc{};
|
||||||
|
tex->GetDesc(&desc);
|
||||||
|
width = desc.Width;
|
||||||
|
height = desc.Height;
|
||||||
|
|
||||||
|
// Recreate staging texture on size change
|
||||||
|
if (!staging_) {
|
||||||
|
D3D11_TEXTURE2D_DESC sd{};
|
||||||
|
sd.Width = width;
|
||||||
|
sd.Height = height;
|
||||||
|
sd.MipLevels = 1;
|
||||||
|
sd.ArraySize = 1;
|
||||||
|
sd.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
|
||||||
|
sd.SampleDesc.Count = 1;
|
||||||
|
sd.Usage = D3D11_USAGE_STAGING;
|
||||||
|
sd.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
|
||||||
|
d3d_dev_->CreateTexture2D(&sd, nullptr, &staging_);
|
||||||
|
}
|
||||||
|
|
||||||
|
d3d_ctx_->CopyResource(staging_.Get(), tex.Get());
|
||||||
|
dupl_->ReleaseFrame();
|
||||||
|
|
||||||
|
D3D11_MAPPED_SUBRESOURCE mapped{};
|
||||||
|
hr = d3d_ctx_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped);
|
||||||
|
if (FAILED(hr)) return false;
|
||||||
|
|
||||||
|
bgra.resize(width * height * 4);
|
||||||
|
const uint8_t* src = static_cast<const uint8_t*>(mapped.pData);
|
||||||
|
for (UINT row = 0; row < height; ++row)
|
||||||
|
memcpy(&bgra[row * width * 4], src + row * mapped.RowPitch, width * 4);
|
||||||
|
|
||||||
|
d3d_ctx_->Unmap(staging_.Get(), 0);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Color conversion: BGRA → NV12 ──────────────────────────────────────────
|
||||||
|
|
||||||
|
void ScreenEncoder::BgraToNv12(const uint8_t* bgra, std::vector<uint8_t>& nv12,
|
||||||
|
UINT w, UINT h) {
|
||||||
|
nv12.resize(w * h * 3 / 2);
|
||||||
|
uint8_t* Y = nv12.data();
|
||||||
|
uint8_t* UV = Y + w * h;
|
||||||
|
|
||||||
|
for (UINT row = 0; row < h; ++row) {
|
||||||
|
for (UINT col = 0; col < w; ++col) {
|
||||||
|
const uint8_t* p = bgra + (row * w + col) * 4;
|
||||||
|
const int b = p[0], g = p[1], r = p[2];
|
||||||
|
Y[row * w + col] =
|
||||||
|
(uint8_t)(((66*r + 129*g + 25*b + 128) >> 8) + 16);
|
||||||
|
|
||||||
|
if ((row & 1) == 0 && (col & 1) == 0) {
|
||||||
|
const UINT off = (row / 2) * w + col;
|
||||||
|
UV[off] = (uint8_t)(((-38*r - 74*g + 112*b + 128) >> 8) + 128);
|
||||||
|
UV[off+1] = (uint8_t)(((112*r - 94*g - 18*b + 128) >> 8) + 128);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Encode one frame ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void ScreenEncoder::EncodeFrame(const std::vector<uint8_t>& bgra,
|
||||||
|
UINT w, UINT h, bool keyframe) {
|
||||||
|
if (enc_width_ != w || enc_height_ != h) {
|
||||||
|
encoder_.Reset();
|
||||||
|
config_sent_ = false;
|
||||||
|
if (!InitEncoder(w, h)) return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build NV12 input sample
|
||||||
|
std::vector<uint8_t> nv12;
|
||||||
|
BgraToNv12(bgra.data(), nv12, w, h);
|
||||||
|
|
||||||
|
ComPtr<IMFSample> in_sample;
|
||||||
|
ComPtr<IMFMediaBuffer> in_buf;
|
||||||
|
MFCreateMemoryBuffer((DWORD)nv12.size(), &in_buf);
|
||||||
|
{
|
||||||
|
BYTE* ptr = nullptr;
|
||||||
|
in_buf->Lock(&ptr, nullptr, nullptr);
|
||||||
|
memcpy(ptr, nv12.data(), nv12.size());
|
||||||
|
in_buf->Unlock();
|
||||||
|
}
|
||||||
|
in_buf->SetCurrentLength((DWORD)nv12.size());
|
||||||
|
MFCreateSample(&in_sample);
|
||||||
|
in_sample->SetSampleTime(sample_ts_);
|
||||||
|
in_sample->SetSampleDuration(kFrameDuration);
|
||||||
|
in_sample->AddBuffer(in_buf.Get());
|
||||||
|
if (keyframe) in_sample->SetUINT32(MFSampleExtension_CleanPoint, 1);
|
||||||
|
sample_ts_ += kFrameDuration;
|
||||||
|
|
||||||
|
if (FAILED(encoder_->ProcessInput(0, in_sample.Get(), 0))) return;
|
||||||
|
|
||||||
|
// Drain output samples
|
||||||
|
MFT_OUTPUT_STREAM_INFO si{};
|
||||||
|
encoder_->GetOutputStreamInfo(0, &si);
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
ComPtr<IMFSample> out_sample;
|
||||||
|
ComPtr<IMFMediaBuffer> out_buf;
|
||||||
|
if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) {
|
||||||
|
MFCreateMemoryBuffer(si.cbSize ? si.cbSize : w * h * 2, &out_buf);
|
||||||
|
MFCreateSample(&out_sample);
|
||||||
|
out_sample->AddBuffer(out_buf.Get());
|
||||||
|
}
|
||||||
|
|
||||||
|
MFT_OUTPUT_DATA_BUFFER out_data{};
|
||||||
|
out_data.pSample = out_sample.Get();
|
||||||
|
DWORD status = 0;
|
||||||
|
HRESULT hr = encoder_->ProcessOutput(0, 1, &out_data, &status);
|
||||||
|
if (out_data.pEvents) out_data.pEvents->Release();
|
||||||
|
if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break;
|
||||||
|
if (FAILED(hr)) break;
|
||||||
|
|
||||||
|
ComPtr<IMFSample> result(out_data.pSample);
|
||||||
|
if (!result) break;
|
||||||
|
|
||||||
|
ComPtr<IMFMediaBuffer> flat;
|
||||||
|
result->ConvertToContiguousBuffer(&flat);
|
||||||
|
BYTE* enc = nullptr; DWORD enc_len = 0;
|
||||||
|
flat->Lock(&enc, nullptr, &enc_len);
|
||||||
|
|
||||||
|
if (!config_sent_) {
|
||||||
|
size_t sps_start = 0, pps_end = 0;
|
||||||
|
if (ExtractSpsPps(enc, enc_len, sps_start, pps_end)) {
|
||||||
|
std::vector<uint8_t> cfg(1 + (pps_end - sps_start));
|
||||||
|
cfg[0] = 0x01;
|
||||||
|
memcpy(&cfg[1], enc + sps_start, pps_end - sps_start);
|
||||||
|
SendEvent(std::move(cfg));
|
||||||
|
config_sent_ = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const int64_t now_ms =
|
||||||
|
std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||||
|
std::chrono::system_clock::now().time_since_epoch())
|
||||||
|
.count();
|
||||||
|
std::vector<uint8_t> frame(1 + 8 + enc_len);
|
||||||
|
frame[0] = 0x02;
|
||||||
|
for (int i = 0; i < 8; ++i)
|
||||||
|
frame[1 + i] = static_cast<uint8_t>(now_ms >> (56 - i * 8));
|
||||||
|
memcpy(&frame[9], enc, enc_len);
|
||||||
|
|
||||||
|
flat->Unlock();
|
||||||
|
SendEvent(std::move(frame));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScreenEncoder::SendEvent(std::vector<uint8_t> data) {
|
||||||
|
std::lock_guard<std::mutex> lk(sink_mu_);
|
||||||
|
if (sink_) sink_->Success(flutter::EncodableValue(std::move(data)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Capture loop ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void ScreenEncoder::CaptureLoop() {
|
||||||
|
CoInitializeEx(nullptr, COINIT_MULTITHREADED);
|
||||||
|
|
||||||
|
if (!InitD3D()) {
|
||||||
|
CoUninitialize();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (running_) {
|
||||||
|
std::vector<uint8_t> bgra;
|
||||||
|
UINT w = 0, h = 0;
|
||||||
|
if (CaptureFrame(bgra, w, h)) {
|
||||||
|
bool kf = force_kf_.exchange(false);
|
||||||
|
EncodeFrame(bgra, w, h, kf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (encoder_) {
|
||||||
|
encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0);
|
||||||
|
encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0);
|
||||||
|
}
|
||||||
|
CoUninitialize();
|
||||||
|
}
|
||||||
57
windows/runner/screen_encoder.h
Normal file
57
windows/runner/screen_encoder.h
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <windows.h>
|
||||||
|
#include <d3d11.h>
|
||||||
|
#include <dxgi1_2.h>
|
||||||
|
#include <mftransform.h>
|
||||||
|
#include <wrl/client.h>
|
||||||
|
|
||||||
|
#include <flutter/event_sink.h>
|
||||||
|
#include <flutter/encodable_value.h>
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
|
#include <memory>
|
||||||
|
#include <mutex>
|
||||||
|
#include <thread>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
class ScreenEncoder {
|
||||||
|
public:
|
||||||
|
ScreenEncoder();
|
||||||
|
~ScreenEncoder();
|
||||||
|
|
||||||
|
void Start(std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink);
|
||||||
|
void Stop();
|
||||||
|
void ForceKeyframe();
|
||||||
|
|
||||||
|
private:
|
||||||
|
void CaptureLoop();
|
||||||
|
bool InitD3D();
|
||||||
|
bool InitEncoder(UINT width, UINT height);
|
||||||
|
bool CaptureFrame(std::vector<uint8_t>& bgra, UINT& width, UINT& height);
|
||||||
|
void EncodeFrame(const std::vector<uint8_t>& bgra, UINT width, UINT height,
|
||||||
|
bool keyframe);
|
||||||
|
void BgraToNv12(const uint8_t* bgra, std::vector<uint8_t>& nv12, UINT width,
|
||||||
|
UINT height);
|
||||||
|
void SendEvent(std::vector<uint8_t> data);
|
||||||
|
|
||||||
|
Microsoft::WRL::ComPtr<ID3D11Device> d3d_dev_;
|
||||||
|
Microsoft::WRL::ComPtr<ID3D11DeviceContext> d3d_ctx_;
|
||||||
|
Microsoft::WRL::ComPtr<IDXGIOutputDuplication> dupl_;
|
||||||
|
Microsoft::WRL::ComPtr<ID3D11Texture2D> staging_;
|
||||||
|
Microsoft::WRL::ComPtr<IMFTransform> encoder_;
|
||||||
|
|
||||||
|
UINT enc_width_ = 0;
|
||||||
|
UINT enc_height_ = 0;
|
||||||
|
bool config_sent_ = false;
|
||||||
|
LONGLONG sample_ts_ = 0;
|
||||||
|
|
||||||
|
std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> sink_;
|
||||||
|
std::mutex sink_mu_;
|
||||||
|
|
||||||
|
std::atomic<bool> running_ {false};
|
||||||
|
std::atomic<bool> force_kf_ {false};
|
||||||
|
std::thread thread_;
|
||||||
|
};
|
||||||
279
windows/runner/video_decoder.cpp
Normal file
279
windows/runner/video_decoder.cpp
Normal file
|
|
@ -0,0 +1,279 @@
|
||||||
|
#include "video_decoder.h"
|
||||||
|
|
||||||
|
#include <mfapi.h>
|
||||||
|
#include <mferror.h>
|
||||||
|
#include <mfidl.h>
|
||||||
|
|
||||||
|
using Microsoft::WRL::ComPtr;
|
||||||
|
|
||||||
|
// ─── Minimal SPS bit-reader ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
struct BitReader {
|
||||||
|
const uint8_t* data;
|
||||||
|
size_t pos = 0;
|
||||||
|
|
||||||
|
uint32_t Bit() {
|
||||||
|
uint32_t v = (data[pos / 8] >> (7 - pos % 8)) & 1;
|
||||||
|
++pos;
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
uint32_t Bits(int n) {
|
||||||
|
uint32_t v = 0;
|
||||||
|
for (int i = 0; i < n; ++i) v = (v << 1) | Bit();
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
uint32_t UEG() {
|
||||||
|
int lz = 0;
|
||||||
|
while (Bit() == 0 && lz < 32) ++lz;
|
||||||
|
if (lz == 0) return 0;
|
||||||
|
return (1u << lz) - 1 + Bits(lz);
|
||||||
|
}
|
||||||
|
int32_t SEG() {
|
||||||
|
uint32_t code = UEG();
|
||||||
|
return (code & 1) ? static_cast<int32_t>((code + 1) / 2)
|
||||||
|
: -static_cast<int32_t>(code / 2);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─── SPS parser ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
bool VideoDecoder::ParseSpsSize(const uint8_t* data, size_t len,
|
||||||
|
int& out_w, int& out_h) {
|
||||||
|
size_t nal_start = SIZE_MAX;
|
||||||
|
for (size_t i = 0; i + 4 <= len; ++i) {
|
||||||
|
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1 &&
|
||||||
|
i + 4 < len && (data[i+4] & 0x1F) == 7) {
|
||||||
|
nal_start = i + 4; break;
|
||||||
|
}
|
||||||
|
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1 &&
|
||||||
|
i + 3 < len && (data[i+3] & 0x1F) == 7) {
|
||||||
|
nal_start = i + 3; break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (nal_start == SIZE_MAX) return false;
|
||||||
|
|
||||||
|
BitReader br{data + nal_start + 1};
|
||||||
|
|
||||||
|
uint8_t profile_idc = static_cast<uint8_t>(br.Bits(8));
|
||||||
|
br.Bits(8); br.Bits(8); br.UEG();
|
||||||
|
|
||||||
|
static const uint8_t kHiProfiles[] = {100,110,122,244,44,83,86,118,128};
|
||||||
|
bool hi = false;
|
||||||
|
for (uint8_t p : kHiProfiles) if (profile_idc == p) { hi = true; break; }
|
||||||
|
if (hi) {
|
||||||
|
uint32_t chroma = br.UEG();
|
||||||
|
if (chroma == 3) br.Bit();
|
||||||
|
br.UEG(); br.UEG(); br.Bit();
|
||||||
|
if (br.Bit()) {
|
||||||
|
for (int idx = 0; idx < (chroma != 3 ? 8 : 12); ++idx) {
|
||||||
|
if (br.Bit()) {
|
||||||
|
int sz = idx < 6 ? 16 : 64;
|
||||||
|
int last = 8, next = 8;
|
||||||
|
for (int j = 0; j < sz; ++j) {
|
||||||
|
if (next != 0) next = (last + br.SEG() + 256) % 256;
|
||||||
|
last = next == 0 ? last : next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
br.UEG();
|
||||||
|
uint32_t poc = br.UEG();
|
||||||
|
if (poc == 0) br.UEG();
|
||||||
|
else if (poc == 1) { br.Bit(); br.SEG(); br.SEG(); uint32_t nc=br.UEG(); for(uint32_t j=0;j<nc;++j) br.SEG(); }
|
||||||
|
|
||||||
|
br.UEG(); br.Bit();
|
||||||
|
|
||||||
|
uint32_t pw = br.UEG();
|
||||||
|
uint32_t ph = br.UEG();
|
||||||
|
int w = (int)((pw + 1) * 16);
|
||||||
|
int h = (int)((ph + 1) * 16);
|
||||||
|
|
||||||
|
bool frame_mbs_only = br.Bit() != 0;
|
||||||
|
if (!frame_mbs_only) br.Bit();
|
||||||
|
br.Bit();
|
||||||
|
|
||||||
|
if (br.Bit()) {
|
||||||
|
uint32_t cl=br.UEG(), cr=br.UEG(), ct=br.UEG(), cb=br.UEG();
|
||||||
|
int ux = 2, uy = frame_mbs_only ? 2 : 4;
|
||||||
|
w -= ux * (int)(cl + cr);
|
||||||
|
h -= uy * (int)(ct + cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (w <= 0 || h <= 0) return false;
|
||||||
|
out_w = w; out_h = h;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Pixel buffer callback (called from render thread) ────────────────────────
|
||||||
|
|
||||||
|
const FlutterDesktopPixelBuffer* VideoDecoder::PixelBufferCallback(
|
||||||
|
size_t, size_t, void* user_data) {
|
||||||
|
return &static_cast<VideoDecoder*>(user_data)->pixel_buf_;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── VideoDecoder ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
VideoDecoder::VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg,
|
||||||
|
const uint8_t* sps_pps, size_t sps_pps_len)
|
||||||
|
: tex_reg_(tex_reg) {
|
||||||
|
Init(sps_pps, sps_pps_len);
|
||||||
|
}
|
||||||
|
|
||||||
|
VideoDecoder::~VideoDecoder() {
|
||||||
|
if (texture_id_ >= 0) {
|
||||||
|
HANDLE done = CreateEvent(nullptr, TRUE, FALSE, nullptr);
|
||||||
|
FlutterDesktopTextureRegistrarUnregisterExternalTexture(
|
||||||
|
tex_reg_, texture_id_,
|
||||||
|
[](void* ud) { SetEvent(static_cast<HANDLE>(ud)); },
|
||||||
|
done);
|
||||||
|
WaitForSingleObject(done, 3000);
|
||||||
|
CloseHandle(done);
|
||||||
|
}
|
||||||
|
if (decoder_)
|
||||||
|
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool VideoDecoder::Init(const uint8_t* sps_pps, size_t sps_pps_len) {
|
||||||
|
int w = 0, h = 0;
|
||||||
|
if (!ParseSpsSize(sps_pps, sps_pps_len, w, h)) {
|
||||||
|
w = GetSystemMetrics(SM_CXSCREEN);
|
||||||
|
h = GetSystemMetrics(SM_CYSCREEN);
|
||||||
|
}
|
||||||
|
width_ = w; height_ = h;
|
||||||
|
|
||||||
|
MFT_REGISTER_TYPE_INFO in_info{MFMediaType_Video, MFVideoFormat_H264};
|
||||||
|
UINT32 count = 0; IMFActivate** acts = nullptr;
|
||||||
|
HRESULT hr = MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER,
|
||||||
|
MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER,
|
||||||
|
&in_info, nullptr, &acts, &count);
|
||||||
|
if (FAILED(hr) || count == 0) return false;
|
||||||
|
hr = acts[0]->ActivateObject(IID_PPV_ARGS(&decoder_));
|
||||||
|
for (UINT32 i = 0; i < count; ++i) acts[i]->Release();
|
||||||
|
CoTaskMemFree(acts);
|
||||||
|
if (FAILED(hr)) return false;
|
||||||
|
|
||||||
|
ComPtr<IMFMediaType> in_type;
|
||||||
|
MFCreateMediaType(&in_type);
|
||||||
|
in_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||||
|
in_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264);
|
||||||
|
MFSetAttributeSize(in_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h);
|
||||||
|
if (FAILED(decoder_->SetInputType(0, in_type.Get(), 0))) return false;
|
||||||
|
|
||||||
|
ComPtr<IMFMediaType> out_type;
|
||||||
|
MFCreateMediaType(&out_type);
|
||||||
|
out_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
|
||||||
|
out_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
|
||||||
|
MFSetAttributeSize(out_type.Get(), MF_MT_FRAME_SIZE, (UINT32)w, (UINT32)h);
|
||||||
|
if (FAILED(decoder_->SetOutputType(0, out_type.Get(), 0))) return false;
|
||||||
|
|
||||||
|
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
|
||||||
|
decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
|
||||||
|
|
||||||
|
// Register Flutter BGRA texture via C API
|
||||||
|
frame_data_.assign(static_cast<size_t>(w * h * 4), 0);
|
||||||
|
pixel_buf_.buffer = frame_data_.data();
|
||||||
|
pixel_buf_.width = static_cast<size_t>(w);
|
||||||
|
pixel_buf_.height = static_cast<size_t>(h);
|
||||||
|
|
||||||
|
FlutterDesktopTextureInfo info{};
|
||||||
|
info.type = kFlutterDesktopPixelBufferTexture;
|
||||||
|
info.pixel_buffer_config.callback = &VideoDecoder::PixelBufferCallback;
|
||||||
|
info.pixel_buffer_config.user_data = this;
|
||||||
|
|
||||||
|
texture_id_ = FlutterDesktopTextureRegistrarRegisterExternalTexture(
|
||||||
|
tex_reg_, &info);
|
||||||
|
|
||||||
|
// Prime decoder with SPS+PPS
|
||||||
|
FeedNal(sps_pps, sps_pps_len);
|
||||||
|
return texture_id_ >= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── NV12 → BGRA ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void VideoDecoder::Nv12ToBgra(const uint8_t* nv12, int stride,
|
||||||
|
uint8_t* bgra, int w, int h) {
|
||||||
|
const uint8_t* Y = nv12;
|
||||||
|
const uint8_t* UV = nv12 + stride * h;
|
||||||
|
|
||||||
|
for (int row = 0; row < h; ++row) {
|
||||||
|
for (int col = 0; col < w; ++col) {
|
||||||
|
int y = Y[row * stride + col];
|
||||||
|
int uv = (row / 2) * stride + (col & ~1);
|
||||||
|
int u = UV[uv];
|
||||||
|
int v = UV[uv + 1];
|
||||||
|
|
||||||
|
int c = y - 16, d = u - 128, e = v - 128;
|
||||||
|
int r = (298*c + 409*e + 128) >> 8;
|
||||||
|
int g = (298*c - 100*d - 208*e + 128) >> 8;
|
||||||
|
int b = (298*c + 516*d + 128) >> 8;
|
||||||
|
|
||||||
|
uint8_t* p = bgra + (row * w + col) * 4;
|
||||||
|
p[0] = static_cast<uint8_t>(b < 0 ? 0 : b > 255 ? 255 : b);
|
||||||
|
p[1] = static_cast<uint8_t>(g < 0 ? 0 : g > 255 ? 255 : g);
|
||||||
|
p[2] = static_cast<uint8_t>(r < 0 ? 0 : r > 255 ? 255 : r);
|
||||||
|
p[3] = 255;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── FeedNal ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
void VideoDecoder::FeedNal(const uint8_t* nal, size_t len) {
|
||||||
|
std::lock_guard<std::mutex> lk(mu_);
|
||||||
|
if (!decoder_ || !len) return;
|
||||||
|
|
||||||
|
ComPtr<IMFSample> in_sample;
|
||||||
|
ComPtr<IMFMediaBuffer> in_buf;
|
||||||
|
MFCreateMemoryBuffer((DWORD)len, &in_buf);
|
||||||
|
{
|
||||||
|
BYTE* ptr = nullptr;
|
||||||
|
in_buf->Lock(&ptr, nullptr, nullptr);
|
||||||
|
memcpy(ptr, nal, len);
|
||||||
|
in_buf->Unlock();
|
||||||
|
}
|
||||||
|
in_buf->SetCurrentLength((DWORD)len);
|
||||||
|
MFCreateSample(&in_sample);
|
||||||
|
in_sample->SetSampleTime(input_ts_);
|
||||||
|
in_sample->SetSampleDuration(333333);
|
||||||
|
in_sample->AddBuffer(in_buf.Get());
|
||||||
|
input_ts_ += 333333;
|
||||||
|
|
||||||
|
if (FAILED(decoder_->ProcessInput(0, in_sample.Get(), 0))) return;
|
||||||
|
|
||||||
|
MFT_OUTPUT_STREAM_INFO si{};
|
||||||
|
decoder_->GetOutputStreamInfo(0, &si);
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
ComPtr<IMFSample> out_sample;
|
||||||
|
ComPtr<IMFMediaBuffer> out_buf;
|
||||||
|
if (!(si.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES)) {
|
||||||
|
DWORD sz = si.cbSize ? si.cbSize : static_cast<DWORD>(width_ * height_ * 3 / 2);
|
||||||
|
MFCreateMemoryBuffer(sz, &out_buf);
|
||||||
|
MFCreateSample(&out_sample);
|
||||||
|
out_sample->AddBuffer(out_buf.Get());
|
||||||
|
}
|
||||||
|
|
||||||
|
MFT_OUTPUT_DATA_BUFFER out_data{};
|
||||||
|
out_data.pSample = out_sample.Get();
|
||||||
|
DWORD status = 0;
|
||||||
|
HRESULT hr = decoder_->ProcessOutput(0, 1, &out_data, &status);
|
||||||
|
if (out_data.pEvents) out_data.pEvents->Release();
|
||||||
|
if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break;
|
||||||
|
if (FAILED(hr)) break;
|
||||||
|
|
||||||
|
ComPtr<IMFSample> result(out_data.pSample);
|
||||||
|
if (!result) break;
|
||||||
|
|
||||||
|
ComPtr<IMFMediaBuffer> flat;
|
||||||
|
result->ConvertToContiguousBuffer(&flat);
|
||||||
|
BYTE* nv12_ptr = nullptr; DWORD nv12_len = 0;
|
||||||
|
flat->Lock(&nv12_ptr, nullptr, &nv12_len);
|
||||||
|
Nv12ToBgra(nv12_ptr, width_, frame_data_.data(), width_, height_);
|
||||||
|
flat->Unlock();
|
||||||
|
|
||||||
|
FlutterDesktopTextureRegistrarMarkExternalTextureFrameAvailable(
|
||||||
|
tex_reg_, texture_id_);
|
||||||
|
}
|
||||||
|
}
|
||||||
49
windows/runner/video_decoder.h
Normal file
49
windows/runner/video_decoder.h
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include <windows.h>
|
||||||
|
#include <flutter_texture_registrar.h>
|
||||||
|
#include <flutter_plugin_registrar.h>
|
||||||
|
#include <mftransform.h>
|
||||||
|
#include <wrl/client.h>
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <mutex>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
class VideoDecoder {
|
||||||
|
public:
|
||||||
|
// spsPps: Annex-B SPS+PPS bytes (the payload from a 0x01 config packet).
|
||||||
|
VideoDecoder(FlutterDesktopTextureRegistrarRef tex_reg,
|
||||||
|
const uint8_t* sps_pps, size_t sps_pps_len);
|
||||||
|
~VideoDecoder();
|
||||||
|
|
||||||
|
int64_t texture_id() const { return texture_id_; }
|
||||||
|
bool is_valid() const { return texture_id_ >= 0; }
|
||||||
|
|
||||||
|
void FeedNal(const uint8_t* nal, size_t len);
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool Init(const uint8_t* sps_pps, size_t sps_pps_len);
|
||||||
|
bool ParseSpsSize(const uint8_t* data, size_t len, int& w, int& h);
|
||||||
|
void Nv12ToBgra(const uint8_t* nv12, int stride,
|
||||||
|
uint8_t* bgra, int w, int h);
|
||||||
|
|
||||||
|
static const FlutterDesktopPixelBuffer* PixelBufferCallback(
|
||||||
|
size_t width, size_t height, void* user_data);
|
||||||
|
|
||||||
|
FlutterDesktopTextureRegistrarRef tex_reg_;
|
||||||
|
int64_t texture_id_ = -1;
|
||||||
|
|
||||||
|
Microsoft::WRL::ComPtr<IMFTransform> decoder_;
|
||||||
|
|
||||||
|
FlutterDesktopPixelBuffer pixel_buf_{};
|
||||||
|
std::vector<uint8_t> frame_data_;
|
||||||
|
|
||||||
|
int width_ = 0;
|
||||||
|
int height_ = 0;
|
||||||
|
LONGLONG input_ts_ = 0;
|
||||||
|
|
||||||
|
std::mutex mu_;
|
||||||
|
};
|
||||||
Loading…
Add table
Reference in a new issue