diff --git a/kernel/src/kernel/frame.cpp b/kernel/src/kernel/frame.cpp index 85baae5..9017924 100644 --- a/kernel/src/kernel/frame.cpp +++ b/kernel/src/kernel/frame.cpp @@ -19,10 +19,17 @@ struct Render_Frame::Private { std::array measurements{}; /* 每种原始耗时首次记录值的加一编码。 */ }; Render_Frame::Render_Frame(Frame_Identity identity) : d(std::make_unique()) { + begin(identity); +} +void Render_Frame::begin(Frame_Identity identity) noexcept { d->identity = identity; d->created_at = std::chrono::steady_clock::now(); const auto system_now = std::chrono::system_clock::now().time_since_epoch(); d->created_time_unix_ns = static_cast(std::chrono::duration_cast(system_now).count()); + for (auto& marker : d->markers) + marker.store(0, std::memory_order_relaxed); + for (auto& measurement : d->measurements) + measurement.store(0, std::memory_order_relaxed); d->markers[static_cast(Frame_Trace_Marker::created)].store(encode_present_value(0), std::memory_order_relaxed); } Render_Frame::~Render_Frame() = default; diff --git a/kernel/src/kernel/frame.hpp b/kernel/src/kernel/frame.hpp index 099bd93..54075cb 100644 --- a/kernel/src/kernel/frame.hpp +++ b/kernel/src/kernel/frame.hpp @@ -26,11 +26,7 @@ enum class Frame_Trace_Marker : std::uint8_t { scene_render_finished, callback_started, callback_finished, - video_encode_queued, - video_encode_started, - video_encode_finished, - stream_publish_started, - stream_publish_finished, + frame_ready, count }; enum class Frame_Trace_Measurement : std::uint8_t { @@ -44,7 +40,6 @@ enum class Frame_Trace_Measurement : std::uint8_t { gpu_copy_ns, gpu_total_ns, readback_ns, - stream_publish_tail_ns, count }; struct Frame_Identity { @@ -69,6 +64,9 @@ public: void record(Frame_Trace_Measurement measurement, std::uint64_t value_ns) noexcept; [[nodiscard]] std::vector trace_points() const; [[nodiscard]] std::vector trace_values() const; +protected: + /* 物理帧槽再次承载新逻辑帧时,重建其唯一身份和诊断时间原点。 */ + void begin(Frame_Identity identity) noexcept; private: struct Private; std::unique_ptr d; /* 帧身份、时钟原点与原子诊断槽位的唯一所有权。 */ diff --git a/render_2D/render_2D/base/Frame_2D.cpp b/render_2D/render_2D/base/Frame_2D.cpp index b347beb..457b42d 100644 --- a/render_2D/render_2D/base/Frame_2D.cpp +++ b/render_2D/render_2D/base/Frame_2D.cpp @@ -30,9 +30,13 @@ struct Frame_2D::Private { }; Frame_2D::Frame_2D(Frame_Identity identity, Pixel_Format output_format) : Render_Frame(identity), d(std::make_unique()) { + begin(identity, output_format); +} +void Frame_2D::begin(Frame_Identity identity, Pixel_Format output_format) { if (std::ranges::find(supported_pixel_formats, output_format) == supported_pixel_formats.end()) throw std::invalid_argument("Frame_2D output pixel format is unsupported"); + Render_Frame::begin(identity); d->output_format = output_format; } Frame_2D::~Frame_2D() = default; diff --git a/render_2D/render_2D/base/Frame_2D.hpp b/render_2D/render_2D/base/Frame_2D.hpp index afe65de..5b7a1b7 100644 --- a/render_2D/render_2D/base/Frame_2D.hpp +++ b/render_2D/render_2D/base/Frame_2D.hpp @@ -19,6 +19,9 @@ public: explicit Frame_2D(Frame_Identity identity, Pixel_Format output_format = native_pixel_format); ~Frame_2D() override; + /* 保留 Blend2D 图像分配,只重启物理槽承载的新逻辑帧。 */ + void begin(Frame_Identity identity, + Pixel_Format output_format = native_pixel_format); [[nodiscard]] Pixel_Format output_format() const noexcept; [[nodiscard]] Image_View image() const; /* 按本帧声明的输出格式生成连续像素;原生格式只做必要的行打包。 */ diff --git a/render_3D/render_3D/base/Frame_3D.cpp b/render_3D/render_3D/base/Frame_3D.cpp index a8e5c59..77cd87a 100644 --- a/render_3D/render_3D/base/Frame_3D.cpp +++ b/render_3D/render_3D/base/Frame_3D.cpp @@ -6,12 +6,23 @@ struct Frame_3D::Private { Extent extent{}; /* 本帧 GPU 读回图像尺寸。 */ std::shared_ptr> pixels{}; /* 合并提交共享的不可变 RGBA8 像素。 */ Frame_3D_Output output{Frame_3D_Output::pixels}; /* 调用方要求的最终帧输出。 */ + Frame_Identity rendered_identity{}; /* 产生本像素的真实 GPU 提交帧身份。 */ }; Frame_3D::Frame_3D(Frame_Identity identity, Frame_3D_Output output, Pixel_Format output_format) : Render_Frame(identity), d(std::make_unique()) { + begin(identity, output, output_format); +} +void Frame_3D::begin(Frame_Identity identity, Frame_3D_Output output, + Pixel_Format output_format) { + if (output_format != native_pixel_format) + throw std::invalid_argument("Frame_3D output pixel format is unsupported"); + Render_Frame::begin(identity); d->output = output; d->output_format = output_format; + d->extent = {}; + d->pixels.reset(); + d->rendered_identity = {}; } Frame_3D::~Frame_3D() = default; Extent Frame_3D::extent() const noexcept { return d->extent; } @@ -19,16 +30,23 @@ std::span Frame_3D::pixels() const noexcept { if (!d->pixels) return {}; return *d->pixels; } +std::shared_ptr> Frame_3D::share_pixels() const noexcept { + return d->pixels; +} +Frame_Identity Frame_3D::rendered_identity() const noexcept { + return d->rendered_identity; +} Frame_3D_Output Frame_3D::output() const noexcept { return d->output; } Pixel_Format Frame_3D::output_format() const noexcept { return d->output_format; } void detail::Frame_3D_Access::assign_pixels( std::span frames, Extent extent, - std::vector pixels) { + std::vector pixels, Frame_Identity rendered_identity) { auto shared_pixels = std::make_shared>(std::move(pixels)); for (Frame_3D* frame : frames) { if (!frame) throw std::invalid_argument("Render_Scene_3D requires non-null external frames"); frame->d->extent = extent; + frame->d->rendered_identity = rendered_identity; frame->d->pixels = frame->d->output == Frame_3D_Output::pixels ? shared_pixels : nullptr; @@ -38,6 +56,7 @@ void detail::Frame_3D_Access::share_pixels(Frame_3D* source, Frame_3D* target) { if (!source || !target) throw std::invalid_argument("Render_Scene_3D requires non-null external frames"); target->d->extent = source->d->extent; + target->d->rendered_identity = source->d->rendered_identity; target->d->pixels = target->d->output == Frame_3D_Output::pixels ? source->d->pixels : nullptr; diff --git a/render_3D/render_3D/base/Frame_3D.hpp b/render_3D/render_3D/base/Frame_3D.hpp index 8f4373f..c9c0139 100644 --- a/render_3D/render_3D/base/Frame_3D.hpp +++ b/render_3D/render_3D/base/Frame_3D.hpp @@ -14,7 +14,8 @@ class Frame_3D; namespace detail { struct Frame_3D_Access { static void assign_pixels(std::span frames, Extent extent, - std::vector pixels); + std::vector pixels, + Frame_Identity rendered_identity); static void share_pixels(Frame_3D* source, Frame_3D* target); }; } @@ -26,8 +27,14 @@ public: Frame_3D_Output output = Frame_3D_Output::pixels, Pixel_Format output_format = native_pixel_format); ~Frame_3D() override; + /* 释放上一逻辑帧的共享读回引用,并重启同一物理帧槽。 */ + void begin(Frame_Identity identity, + Frame_3D_Output output = Frame_3D_Output::pixels, + Pixel_Format output_format = native_pixel_format); [[nodiscard]] Extent extent() const noexcept; [[nodiscard]] std::span pixels() const noexcept; + [[nodiscard]] std::shared_ptr> share_pixels() const noexcept; + [[nodiscard]] Frame_Identity rendered_identity() const noexcept; [[nodiscard]] Frame_3D_Output output() const noexcept; [[nodiscard]] Pixel_Format output_format() const noexcept; private: diff --git a/render_3D/render_3D/detail/Async_Render_Backend.cpp b/render_3D/render_3D/detail/Async_Render_Backend.cpp index 7aa8a69..878dffa 100644 --- a/render_3D/render_3D/detail/Async_Render_Backend.cpp +++ b/render_3D/render_3D/detail/Async_Render_Backend.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include namespace aethera::render_3d::detail { namespace { +constexpr std::uint64_t backend_observation_period{100}; + float wheel_step(double pixel, double angle) { if (angle != 0.0) return static_cast(angle / 120.0); return static_cast(pixel / 100.0); @@ -66,7 +69,7 @@ struct Async_Render_Backend::Implementation Frame_Callback callback{}; /* render 入队时捕获的完成出口。 */ }; struct Submission { - Prepared_Visual_Batch visuals{}; /* Scene CPU Prepare 发布的大批量不可变快照。 */ + Shared_Prepared_Visual_Batch visuals{}; /* Scene CPU Prepare 发布的大批量不可变快照。 */ Scene_3D_Parameters parameters{}; /* 本帧 Scene、Camera 与轴参数快照。 */ std::vector completions{}; /* 合并到同一实际画面的逻辑帧。 */ }; @@ -74,6 +77,7 @@ struct Async_Render_Backend::Implementation Datoviz_Visual_Backend::Pending_Frame backend_frame{}; /* 三缓冲目标对应的已录制提交。 */ std::vector completions{}; /* GPU 完成时一起结清的逻辑帧。 */ Extent extent{}; /* 失败时生成空结果所需的尺寸。 */ + std::atomic_bool submitted{}; /* 区分提交前取消与 GPU 已拥有资源后的终止。 */ }; struct Gpu_Completion { std::shared_ptr pending{}; /* 完成前保持帧目标和回调集合存活。 */ @@ -84,9 +88,7 @@ struct Async_Render_Backend::Implementation std::shared_ptr event{}; /* Scene 转移给准备域的输入事件。 */ Extent viewport{}; /* 事件产生时的物理像素视口。 */ }; - struct Stop_Command {}; - using Command = std::variant; + using Command = std::variant; static constexpr std::size_t command_capacity = 64; std::shared_ptr render_domain; /* 同 GPU 唯一的轻量 Queue Submit 域。 */ @@ -101,6 +103,7 @@ struct Async_Render_Backend::Implementation Frame_Callback frame_callback{}; /* 后续 render 捕获的完成出口。 */ std::mutex failure_mutex{}; /* 保护跨线程传播的最后 Unknown Failure。 */ std::exception_ptr failure{}; /* 后端不可用的根因。 */ + std::mutex admission_mutex{}; /* render/event 与 stop 的原子准入边界。 */ std::atomic_bool available{true}; /* 对 Scene 发布的后端可接收状态。 */ std::atomic_bool stop_requested{}; /* 拒绝新命令并等待在途槽结清。 */ @@ -110,17 +113,18 @@ struct Async_Render_Backend::Implementation : render_domain(Render_Domain::acquire(gpu_index_value)), gpu_index(gpu_index_value), validation_enabled(validation_enabled_value), - visual_registrations(std::move(visuals)), - preparation_thread([this] { run(); }) {} - ~Implementation() { stop(); } + visual_registrations(std::move(visuals)) {} + ~Implementation(); + void start(); void stop() noexcept; void run() noexcept; void fail(std::exception_ptr value) noexcept; void enqueue(Command command); void initialize(const Scene_3D_Parameters& parameters); - void render(Prepared_Visual_Batch visuals, - Scene_3D_Parameters parameters, Frame_3D* frame); + [[nodiscard]] Async_Render_Backend::Submit_Result render( + Shared_Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, + Frame_3D* frame); void accept(Submission submission); void merge_deferred(Submission submission); void prepare(Submission submission); @@ -136,32 +140,49 @@ Async_Render_Backend::Async_Render_Backend( std::uint32_t gpu_index, bool validation_enabled, std::vector visuals) : implementation_(std::make_shared( - gpu_index, validation_enabled, std::move(visuals))) {} -Async_Render_Backend::~Async_Render_Backend() = default; + gpu_index, validation_enabled, std::move(visuals))) { + implementation_->start(); +} +Async_Render_Backend::~Async_Render_Backend() { + implementation_->stop(); +} + +Async_Render_Backend::Implementation::~Implementation() { + stop(); + if (preparation_thread.joinable() && + preparation_thread.get_id() == std::this_thread::get_id()) + preparation_thread.detach(); +} +void Async_Render_Backend::Implementation::start() { + auto self = shared_from_this(); + preparation_thread = std::thread([self = std::move(self)] { self->run(); }); +} void Async_Render_Backend::Implementation::stop() noexcept { - available.store(false, std::memory_order_release); - if (!stop_requested.exchange(true, std::memory_order_acq_rel)) { - try { - enqueue(Stop_Command{}); - } - catch (...) { - std::terminate(); - } + { + std::lock_guard lock(admission_mutex); + available.store(false, std::memory_order_release); + stop_requested.store(true, std::memory_order_release); } - if (preparation_thread.joinable()) preparation_thread.join(); + if (preparation_thread.joinable() && + preparation_thread.get_id() != std::this_thread::get_id()) + preparation_thread.join(); } void Async_Render_Backend::Implementation::fail( std::exception_ptr value) noexcept { - const auto description = failure_description(value); - std::fprintf(stderr, "Aethera 3D backend unavailable: %s\n", - description.c_str()); - std::fflush(stderr); - { + available.store(false, std::memory_order_release); + try { + const auto description = failure_description(value); + std::fprintf(stderr, "Aethera 3D backend unavailable: %s\n", + description.c_str()); + std::fflush(stderr); + } + catch (...) {} + try { std::lock_guard lock(failure_mutex); if (!failure) failure = std::move(value); } - available.store(false, std::memory_order_release); + catch (...) {} } void Async_Render_Backend::Implementation::enqueue(Command command) { if (!commands.enqueue(std::move(command))) throw std::bad_alloc{}; @@ -174,20 +195,33 @@ void Async_Render_Backend::Implementation::initialize( visual_registrations.clear(); visual_registrations.shrink_to_fit(); } -void Async_Render_Backend::Implementation::render( - Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, +Async_Render_Backend::Submit_Result +Async_Render_Backend::Implementation::render( + Shared_Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, Frame_3D* frame) { if (!frame) throw std::invalid_argument("3D backend requires a non-null external frame"); - if (!available.load(std::memory_order_acquire)) return; + if (!visuals || visuals->empty()) + throw std::invalid_argument("3D backend requires prepared Visual snapshots"); + std::lock_guard admission_lock(admission_mutex); + if (!available.load(std::memory_order_acquire) || + stop_requested.load(std::memory_order_acquire)) + return Async_Render_Backend::Submit_Result::backend_unavailable; Frame_Callback callback; { std::lock_guard lock(callback_mutex); callback = frame_callback; } frame->mark(Frame_Trace_Marker::backend_queue_entered); - enqueue(Submission{std::move(visuals), parameters, - {Completion{frame, std::move(callback)}}}); + try { + enqueue(Submission{std::move(visuals), std::move(parameters), + {Completion{frame, std::move(callback)}}}); + } + catch (...) { + fail(std::current_exception()); + throw; + } + return Async_Render_Backend::Submit_Result::queued; } void Async_Render_Backend::Implementation::merge_deferred( Submission submission) { @@ -209,7 +243,7 @@ void Async_Render_Backend::Implementation::accept(Submission submission) { return; } if (backend && - !backend->can_prepare(submission.parameters, submission.visuals)) { + !backend->can_prepare(submission.parameters, *submission.visuals)) { merge_deferred(std::move(submission)); return; } @@ -227,8 +261,10 @@ void Async_Render_Backend::Implementation::prepare(Submission submission) { submission.completions, [](const Completion& completion) { return completion.output->output() == Frame_3D_Output::pixels; }); + const bool observe = sequence == 1 || + sequence % backend_observation_period == 0; prepared = backend->prepare( - submission.parameters, submission.visuals, sequence, true, readback); + submission.parameters, *submission.visuals, sequence, observe, readback); } catch (...) { fail(std::current_exception()); @@ -284,7 +320,7 @@ void Async_Render_Backend::Implementation::submit( catch (...) { self->fail(std::current_exception()); } - }, true); + }, pending->backend_frame.trace.observed); if (!reservation) { self->enqueue(Gpu_Completion{ pending, std::nullopt, @@ -293,6 +329,7 @@ void Async_Render_Backend::Implementation::submit( return; } self->backend->submit(pending->backend_frame); + pending->submitted.store(true, std::memory_order_release); for (const auto& completion : pending->completions) completion.output->mark(Frame_Trace_Marker::gpu_submitted); reservation.reservation.watch(pending->backend_frame.device, @@ -321,19 +358,23 @@ void Async_Render_Backend::Implementation::resolve( outputs.reserve(pending->completions.size()); for (const auto& completion : pending->completions) outputs.push_back(completion.output); + const Frame_Identity rendered_identity = pending->completions.empty() + ? Frame_Identity{} + : pending->completions.back().output->identity(); if (completed) { for (Frame_3D* output : outputs) { output->mark(Frame_Trace_Marker::readback_finished); record_datoviz_trace(output, completed->trace); } Frame_3D_Access::assign_pixels(outputs, completed->extent, - std::move(completed->pixels)); + std::move(completed->pixels), + rendered_identity); } else { const auto pixel_count = static_cast(pending->extent.width) * pending->extent.height * 4U; Frame_3D_Access::assign_pixels( - outputs, pending->extent, std::vector(pixel_count)); + outputs, pending->extent, std::vector(pixel_count), {}); } complete(std::move(pending->completions)); } @@ -346,8 +387,7 @@ void Async_Render_Backend::Implementation::complete( item.callback(item.output); } catch (...) { - std::lock_guard lock(failure_mutex); - failure = std::current_exception(); + fail(std::current_exception()); } item.output->mark(Frame_Trace_Marker::callback_finished); } @@ -363,12 +403,27 @@ void Async_Render_Backend::Implementation::finish(Gpu_Completion completion) { std::optional completed; if (completion.failure) { fail(std::move(completion.failure)); - backend->discard(std::move(completion.pending->backend_frame)); + if (completion.pending->submitted.load(std::memory_order_acquire)) + backend->quarantine(std::move(completion.pending->backend_frame)); + else + backend->discard(std::move(completion.pending->backend_frame)); } else if (!completion.result || completion.result->error != Gpu_Completion_Service::Completion_Error::none) { - backend->discard(std::move(completion.pending->backend_frame)); + const auto code = completion.result + ? static_cast(completion.result->error) + : std::numeric_limits::max(); + const auto vulkan = completion.result + ? static_cast(completion.result->vulkan_result) + : static_cast(VK_ERROR_UNKNOWN); + fail(std::make_exception_ptr(std::runtime_error( + "Datoviz GPU completion failed: error=" + + std::to_string(code) + ", VkResult=" + std::to_string(vulkan)))); + if (completion.pending->submitted.load(std::memory_order_acquire)) + backend->quarantine(std::move(completion.pending->backend_frame)); + else + backend->discard(std::move(completion.pending->backend_frame)); } else { completed = backend->collect( @@ -422,34 +477,39 @@ void Async_Render_Backend::Implementation::dispatch(Event_Command command) { command.event->accept(); } void Async_Render_Backend::Implementation::run() noexcept { - bool stopping{}; for (;;) { Command command; - commands.wait_dequeue(command); - try { - if (auto* submission = std::get_if(&command)) - accept(std::move(*submission)); - else if (auto* completion = std::get_if(&command)) - finish(std::move(*completion)); - else if (auto* event = std::get_if(&command)) - dispatch(std::move(*event)); - else stopping = true; - } - catch (...) { - fail(std::current_exception()); - if (auto* submission = std::get_if(&command)) { - auto pending = std::make_shared(); - pending->extent = submission->parameters.viewport; - pending->completions = std::move(submission->completions); - try { resolve(std::move(pending), std::nullopt); } - catch (...) { fail(std::current_exception()); } + const bool received = commands.wait_dequeue_timed( + command, std::chrono::milliseconds(10)); + if (received) { + try { + if (auto* submission = std::get_if(&command)) + accept(std::move(*submission)); + else if (auto* completion = std::get_if(&command)) + finish(std::move(*completion)); + else if (auto* event = std::get_if(&command)) + dispatch(std::move(*event)); } - else if (auto* completion = std::get_if(&command)) { - try { resolve(std::move(completion->pending), std::nullopt); } - catch (...) { fail(std::current_exception()); } + catch (...) { + fail(std::current_exception()); + if (auto* submission = std::get_if(&command)) { + auto pending = std::make_shared(); + pending->extent = submission->parameters.viewport; + pending->completions = std::move(submission->completions); + try { resolve(std::move(pending), std::nullopt); } + catch (...) { fail(std::current_exception()); } + } + else if (auto* completion = std::get_if(&command)) { + try { resolve(std::move(completion->pending), std::nullopt); } + catch (...) { fail(std::current_exception()); } + } } } - if (stopping && (!backend || backend->idle())) { + // render/event 与 stop 共享 admission_mutex,因此停止后只有已经 + // 提交到 GPU 的完成通知还能到达。只有一次真实的 timed dequeue + // 为空且 backend 已无在途目标,准备域才完成排空并退出。 + if (!received && stop_requested.load(std::memory_order_acquire) && + (!backend || backend->idle())) { if (deferred_submission) { auto pending = std::make_shared(); pending->extent = deferred_submission->parameters.viewport; @@ -465,10 +525,10 @@ void Async_Render_Backend::Implementation::run() noexcept { } } -void Async_Render_Backend::render( - Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, +Async_Render_Backend::Submit_Result Async_Render_Backend::render( + Shared_Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, Frame_3D* frame) { - implementation_->render(std::move(visuals), parameters, frame); + return implementation_->render(std::move(visuals), parameters, frame); } void Async_Render_Backend::set_frame_callback(Frame_Callback callback) { std::lock_guard lock(implementation_->callback_mutex); @@ -481,8 +541,6 @@ Async_Render_Backend::Dispatch_Event_Result Async_Render_Backend::dispatch_event(std::shared_ptr event, Extent viewport) { auto self = implementation_; - if (!self->available.load(std::memory_order_acquire)) - return Dispatch_Event_Result::backend_unavailable; const auto* pointer = dynamic_cast(event.get()); const auto* wheel = dynamic_cast(event.get()); const auto* key = dynamic_cast(event.get()); @@ -501,6 +559,10 @@ Async_Render_Backend::dispatch_event(std::shared_ptr event, return Dispatch_Event_Result::invalid_event; } else if (!key) return Dispatch_Event_Result::ignored; + std::lock_guard admission_lock(self->admission_mutex); + if (!self->available.load(std::memory_order_acquire) || + self->stop_requested.load(std::memory_order_acquire)) + return Dispatch_Event_Result::backend_unavailable; try { self->enqueue(Implementation::Event_Command{std::move(event), viewport}); } diff --git a/render_3D/render_3D/detail/Async_Render_Backend.hpp b/render_3D/render_3D/detail/Async_Render_Backend.hpp index 0ecda87..f86c183 100644 --- a/render_3D/render_3D/detail/Async_Render_Backend.hpp +++ b/render_3D/render_3D/detail/Async_Render_Backend.hpp @@ -7,14 +7,19 @@ namespace aethera::render_3d::detail { class Async_Render_Backend final { public: using Frame_Callback = std::function; + enum class Submit_Result : std::uint8_t { + queued, + backend_unavailable + }; Async_Render_Backend(std::uint32_t gpu_index, bool validation_enabled, std::vector visuals); ~Async_Render_Backend(); Async_Render_Backend(const Async_Render_Backend&) = delete; Async_Render_Backend& operator=(const Async_Render_Backend&) = delete; /* 将 Visual 快照所有权无等待转移给所属 GPU Render Domain。 */ - void render(Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, - Frame_3D* frame); + [[nodiscard]] Submit_Result render( + Shared_Prepared_Visual_Batch visuals, Scene_3D_Parameters parameters, + Frame_3D* frame); /* 设置后续提交捕获的逻辑帧完成出口。 */ void set_frame_callback(Frame_Callback callback); [[nodiscard]] bool available() const noexcept; diff --git a/render_3D/render_3D/detail/Backend_Types.hpp b/render_3D/render_3D/detail/Backend_Types.hpp index 56de091..daa460f 100644 --- a/render_3D/render_3D/detail/Backend_Types.hpp +++ b/render_3D/render_3D/detail/Backend_Types.hpp @@ -4,6 +4,7 @@ #include #include "../camera/Camera.hpp" #include +#include #include namespace aethera::render_3d::detail { using Visual_Identity = std::uintptr_t; @@ -17,6 +18,8 @@ struct Prepared_Visual_Instance { Prepared_Visual visual{}; /* Prepare 发布的不可变字段快照。 */ }; using Prepared_Visual_Batch = std::vector; +using Shared_Prepared_Visual_Batch = + std::shared_ptr; struct Scene_3D_Parameters { Extent viewport{560, 320}; /* 离屏渲染目标尺寸,单位为像素。 */ Linear_Color clear_color{}; /* 每帧开始时写入的线性背景色。 */ diff --git a/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp b/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp index 0f36d11..78efe0a 100644 --- a/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp +++ b/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -245,6 +247,7 @@ public: [[nodiscard]] DvzGpuCtx* gpu_context() const noexcept { return gpu_context_; } [[nodiscard]] std::mutex& api_mutex() noexcept { return api_mutex_; } + [[nodiscard]] std::mutex& queue_mutex() noexcept { return queue_mutex_; } private: Datoviz_Render_Context(std::uint32_t gpu_index, bool validation_enabled) { @@ -258,9 +261,39 @@ private: } DvzGpuCtx* gpu_context_{}; /* 共享 GPU Device 与分配器的唯一所有权。 */ - std::mutex api_mutex_{}; /* 同 GPU Datoviz/Vulkan API 与 Frame Target 状态的唯一串行化域。 */ + std::mutex api_mutex_{}; /* 同 GPU Datoviz 对象结构修改的串行化域。 */ + std::mutex queue_mutex_{}; /* VkQueue 及会内部使用主队列的 Datoviz 调用的外部同步域。 */ }; +void retain_quarantined_context( + std::shared_ptr& context) noexcept { + if (!context) return; + struct Registry { + std::atomic_flag lock = ATOMIC_FLAG_INIT; + std::array, 64> contexts{}; + }; + static Registry registry; + while (registry.lock.test_and_set(std::memory_order_acquire)) + std::this_thread::yield(); + for (auto& retained : registry.contexts) { + if (retained.get() == context.get()) { + context.reset(); + registry.lock.clear(std::memory_order_release); + return; + } + } + for (auto& retained : registry.contexts) { + if (!retained) { + retained = std::move(context); + registry.lock.clear(std::memory_order_release); + return; + } + } + registry.lock.clear(std::memory_order_release); + static_cast(new (std::nothrow) + std::shared_ptr(std::move(context))); +} + class Datoviz_Visual_Backend::Frame_Target final { public: struct Collection { @@ -319,8 +352,14 @@ public: raise_context("creating Datoviz frame target", std::current_exception()); } } - ~Frame_Target() noexcept(false) { - destroy(); + ~Frame_Target() noexcept { + try { + if (quarantined_) release_without_destroy(); + else destroy(); + } + catch (...) { + release_without_destroy(); + } } [[nodiscard]] bool can_reuse( std::uint64_t command_revision, bool observe, @@ -348,56 +387,74 @@ public: /* Reset invalidates the previous recording immediately. Only * finish_recording() may publish the new cache identity. */ recorded_ = false; - if (observing_ && !timestamps_initialized_) { - initialize_timestamps( - dvz_gpu_ctx_device(gpu_context_), - dvz_gpu_ctx_queue(gpu_context_, DVZ_QUEUE_MAIN)); + try { + if (observing_ && !timestamps_initialized_) { + initialize_timestamps( + dvz_gpu_ctx_device(gpu_context_), + dvz_gpu_ctx_queue(gpu_context_, DVZ_QUEUE_MAIN)); + } + dvz_cmd_reset(commands_); + if (dvz_cmd_begin_result(commands_) != 0) + throw std::runtime_error( + "failed to begin Datoviz command buffer"); + recording_ = true; + DvzBarriers barriers{}; + dvz_barriers(&barriers); + auto* image_barrier = dvz_barriers_image( + &barriers, dvz_image_handle(image_, 0)); + if (image_barrier == nullptr) + throw std::runtime_error( + "failed to allocate Datoviz image barrier"); + if (completed_layout_ == VK_IMAGE_LAYOUT_UNDEFINED) { + dvz_barrier_image_stage( + image_barrier, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); + dvz_barrier_image_access( + image_barrier, 0, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); + } + else if (completed_layout_ == + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { + dvz_barrier_image_stage( + image_barrier, VK_PIPELINE_STAGE_2_TRANSFER_BIT, + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); + dvz_barrier_image_access( + image_barrier, VK_ACCESS_2_TRANSFER_READ_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); + } + else { + dvz_barrier_image_stage( + image_barrier, + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); + dvz_barrier_image_access( + image_barrier, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); + } + dvz_barrier_image_layout( + image_barrier, completed_layout_, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + dvz_barrier_image_aspect(image_barrier, VK_IMAGE_ASPECT_COLOR_BIT); + dvz_barrier_image_mip(image_barrier, 0, 1); + dvz_barrier_image_layers(image_barrier, 0, 1); + dvz_cmd_barriers(commands_, &barriers); + if (observing_ && timestamps_supported_) { + const VkCommandBuffer command_buffer = + dvz_commands_handle(commands_); + vkCmdResetQueryPool(command_buffer, query_pool_, 0, 4); + vkCmdWriteTimestamp( + command_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + query_pool_, 0); + } } - dvz_cmd_reset(commands_); - if (dvz_cmd_begin_result(commands_) != 0) throw std::runtime_error("failed to begin Datoviz command buffer"); - DvzBarriers barriers{}; - dvz_barriers(&barriers); - auto* image_barrier = dvz_barriers_image(&barriers, dvz_image_handle(image_, 0)); - if (completed_layout_ == VK_IMAGE_LAYOUT_UNDEFINED) { - dvz_barrier_image_stage(image_barrier, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); - dvz_barrier_image_access( - image_barrier, 0, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); + catch (...) { + abort(); + throw; } - else if (completed_layout_ == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL) { - dvz_barrier_image_stage(image_barrier, VK_PIPELINE_STAGE_2_TRANSFER_BIT, - VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); - dvz_barrier_image_access( - image_barrier, VK_ACCESS_2_TRANSFER_READ_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); - } - else { - dvz_barrier_image_stage(image_barrier, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, - VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT); - dvz_barrier_image_access( - image_barrier, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, - VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); - } - dvz_barrier_image_layout(image_barrier, completed_layout_, - VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); - dvz_barrier_image_aspect(image_barrier, VK_IMAGE_ASPECT_COLOR_BIT); - dvz_barrier_image_mip(image_barrier, 0, 1); - dvz_barrier_image_layers(image_barrier, 0, 1); - dvz_cmd_barriers(commands_, &barriers); - if (observing_ && timestamps_supported_) { - const VkCommandBuffer command_buffer = - dvz_commands_handle(commands_); - vkCmdResetQueryPool(command_buffer, query_pool_, 0, 4); - vkCmdWriteTimestamp( - command_buffer, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - query_pool_, 0); - } - recording_ = true; } [[nodiscard]] DvzStreamFrame stream_frame() const { DvzStreamFrame frame{}; @@ -541,8 +598,20 @@ public: [[nodiscard]] std::uint64_t generation() const noexcept { return generation_; } + void quarantine_after_submission() noexcept { + recording_ = false; + prepared_ = false; + in_flight_ = false; + observing_ = false; + readback_requested_ = false; + recorded_ = false; + quarantined_ = true; + } [[nodiscard]] Extent extent() const noexcept { return extent_; } [[nodiscard]] bool available() const noexcept { + return !quarantined_ && !recording_ && !prepared_ && !in_flight_; + } + [[nodiscard]] bool quiescent() const noexcept { return !recording_ && !prepared_ && !in_flight_; } void abort() noexcept { @@ -553,6 +622,16 @@ public: readback_requested_ = false; } private: + void release_without_destroy() noexcept { + query_pool_ = VK_NULL_HANDLE; + readback_ = nullptr; + fence_ = nullptr; + submit_ = nullptr; + commands_ = nullptr; + view_ = nullptr; + image_ = nullptr; + gpu_context_ = nullptr; + } void initialize_timestamps(DvzDevice* device, DvzQueue* queue) noexcept { timestamps_initialized_ = true; if (device == nullptr || queue == nullptr || @@ -698,6 +777,7 @@ private: bool recorded_{}; bool recorded_observing_{}; bool recorded_readback_{}; + bool quarantined_{}; }; struct Datoviz_Visual_Backend::Frame_Targets { static constexpr std::size_t count = 3; @@ -707,11 +787,12 @@ Datoviz_Visual_Backend::Datoviz_Visual_Backend( std::uint32_t gpu_index, bool validation_enabled, const std::vector& visuals, const Scene_3D_Parameters& initial_scene) - : domain_thread_(std::this_thread::get_id()), + : preparation_thread_(std::this_thread::get_id()), targets_(std::make_unique()) { try { render_context_ = Datoviz_Render_Context::acquire(gpu_index, validation_enabled); - std::lock_guard api_lock(render_context_->api_mutex()); + std::scoped_lock context_lock(render_context_->api_mutex(), + render_context_->queue_mutex()); auto* gpu_context = render_context_->gpu_context(); DvzDrp2RuntimeConfig runtime_configuration = dvz_drp2_runtime_vklite_config( dvz_gpu_ctx_device(gpu_context), dvz_gpu_ctx_alloc(gpu_context)); @@ -724,15 +805,20 @@ Datoviz_Visual_Backend::Datoviz_Visual_Backend( create_scene(visuals, initial_scene); } catch (...) { - destroy(); - raise_context("creating Datoviz backend", std::current_exception()); + const auto failure = std::current_exception(); + try { destroy(); } + catch (...) {} + raise_context("creating Datoviz backend", failure); } } -Datoviz_Visual_Backend::~Datoviz_Visual_Backend() noexcept(false) { - destroy(); +Datoviz_Visual_Backend::~Datoviz_Visual_Backend() noexcept { + try { destroy(); } + catch (...) { abandon_resources(); } } void Datoviz_Visual_Backend::require_domain() const { - if (std::this_thread::get_id() != domain_thread_) throw std::logic_error("Datoviz objects may only be used on the render domain"); + if (std::this_thread::get_id() != preparation_thread_) + throw std::logic_error( + "Datoviz Scene preparation may only run on its preparation thread"); } void Datoviz_Visual_Backend::create_scene( const std::vector& registrations, @@ -1732,12 +1818,74 @@ Datoviz_Visual_Backend::Frame_Target& Datoviz_Visual_Backend::target( throw std::logic_error("Datoviz pending frame target no longer exists"); return *result; } +std::optional +Datoviz_Visual_Backend::reuse_recorded_target( + const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& prepared, + std::uint64_t frame_sequence, bool observe, bool readback) { + Datoviz_Frame_Trace trace; + trace.render_sequence = frame_sequence; + trace.observed = observe; + const std::uint64_t started = observe ? trace_now_ns() : 0; + std::lock_guard target_lock(target_mutex_); + if (!matches_command_structure(scene, prepared)) return std::nullopt; + + for (std::uint8_t target_index = 0; + target_index < targets_->values.size(); ++target_index) { + auto& candidate = targets_->values[target_index]; + if (!candidate || candidate->extent() != scene.viewport || + !candidate->can_reuse(command_revision_, observe, readback)) + continue; + + /* The command buffer already binds this target's independent region. + * Only immutable frame payload bytes may change on this path. No + * Scene graph, emitter, runtime, allocator, camera or axis object is + * touched, so different Scene preparation workers can write their own + * mapped allocations in parallel without weakening the global API + * serialization used by structural Datoviz operations. */ + for (const auto& source : prepared) { + auto visual = std::ranges::find_if( + visuals_, [&](const Visual_Instance& value) { + return value.identity == source.identity; + }); + if (visual == visuals_.end()) + throw std::logic_error( + "recorded Datoviz frame contains an unknown Visual identity"); + if (!uses_external_attributes(visual->family)) continue; + if (!source.visual.data) + throw std::logic_error( + "recorded Datoviz Visual has no immutable payload"); + if (!source.visual.data->positions.empty()) + upload_external_attributes( + *visual, *source.visual.data, target_index); + visual->applied_revision = source.visual.revision; + visual->applied = source.visual; + } + candidate->reuse(command_revision_, observe, readback); + if (observe) trace.apply_ns = trace_now_ns() - started; + return Pending_Frame{ + candidate->device(), candidate->fence(), scene.viewport, + frame_sequence, target_index, candidate->generation(), + std::move(trace) + }; + } + return std::nullopt; +} std::optional Datoviz_Visual_Backend::prepare( const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& visuals, std::uint64_t frame_sequence, bool observe, bool readback) { require_domain(); - std::lock_guard api_lock(render_context_->api_mutex()); if (scene.viewport.empty()) return std::nullopt; + if (auto reused = reuse_recorded_target( + scene, visuals, frame_sequence, observe, readback)) + return reused; + /* + * DRP2 runtime 的结构执行会创建/上传资源,并可能在内部使用共享主 + * VkQueue。它必须和 Render Domain 的显式 submit 使用同一外部同步域。 + * 热路径的已录制目标复用在此锁之前返回,仍可在各 Scene 准备线程并行。 + */ + std::scoped_lock context_lock(render_context_->api_mutex(), + render_context_->queue_mutex()); + std::lock_guard target_lock(target_mutex_); const auto target_index = acquire_target(scene.viewport); if (!target_index) return std::nullopt; Datoviz_Frame_Trace trace; @@ -1804,10 +1952,12 @@ std::optional Datoviz_Visual_Backend::pre bool released{}; /* finish_recording 成功后禁止回滚。 */ ~Recording_Scope() { if (!released) target.abort(); } } recording_scope{frame_target}; - DvzSceneFrameArtifact* artifact{}; + std::unique_ptr + artifact{nullptr, &dvz_scene_frame_artifact_destroy}; try { if (observe) phase_started = trace_now_ns(); - artifact = emit(scene, *target_index); + artifact.reset(emit(scene, *target_index)); if (observe) trace.emit_ns = trace_now_ns() - phase_started; } catch (...) { @@ -1816,13 +1966,14 @@ std::optional Datoviz_Visual_Backend::pre } if (observe) { trace.artifact_status = static_cast( - dvz_scene_frame_artifact_status(artifact)); + dvz_scene_frame_artifact_status(artifact.get())); trace.artifact_resource_version = - dvz_scene_frame_artifact_resource_version(artifact); + dvz_scene_frame_artifact_resource_version(artifact.get()); trace.artifact_frame_index = - dvz_scene_frame_artifact_frame_index(artifact); + dvz_scene_frame_artifact_frame_index(artifact.get()); } - const DvzDrp2CommandStream* stream = dvz_scene_frame_artifact_stream(artifact); + const DvzDrp2CommandStream* stream = + dvz_scene_frame_artifact_stream(artifact.get()); register_external_attributes(stream, *target_index); const DvzStreamFrame target_frame = frame_target.stream_frame(); if (observe) phase_started = trace_now_ns(); @@ -1839,12 +1990,12 @@ std::optional Datoviz_Visual_Backend::pre trace.validation_command_index = result.command_index; if (!trace.validation_ok) { if (char* json = dvz_scene_frame_artifact_json( - artifact, "renderive_frame")) { + artifact.get(), "renderive_frame")) { trace.artifact_json = json; dvz_drp2_stream_json_destroy(json); } } - dvz_scene_frame_artifact_destroy(artifact); + artifact.reset(); if (!attached) { frame_target.abort(); throw std::runtime_error("failed to attach the Datoviz point frame target"); @@ -1870,7 +2021,12 @@ std::optional Datoviz_Visual_Backend::pre }; } void Datoviz_Visual_Backend::submit(Pending_Frame& pending) { - std::lock_guard api_lock(render_context_->api_mutex()); + /* + * 所有显式 GPU 提交仍只由 Render Domain 执行;queue_mutex 只满足 Vulkan + * 对同一 VkQueue 的外部同步要求,不把 CPU apply、读回或回调串行化。 + */ + std::lock_guard queue_lock(render_context_->queue_mutex()); + std::lock_guard target_lock(target_mutex_); const std::uint64_t started = pending.trace.observed ? trace_now_ns() : 0; target(pending).submit(); if (pending.trace.observed) @@ -1879,7 +2035,9 @@ void Datoviz_Visual_Backend::submit(Pending_Frame& pending) { Datoviz_Visual_Backend::Completed_Frame Datoviz_Visual_Backend::collect( Pending_Frame pending) { require_domain(); - std::lock_guard api_lock(render_context_->api_mutex()); + /* Fence 已完成,读回缓冲与查询池只属于本目标槽;不同 Scene 的映射内存 + * 下载可以并行,且不能反向阻塞 Render_Domain 的下一批提交。 */ + std::lock_guard target_lock(target_mutex_); const std::uint64_t readback_started = pending.trace.observed ? trace_now_ns() : 0; @@ -1890,14 +2048,27 @@ Datoviz_Visual_Backend::Completed_Frame Datoviz_Visual_Backend::collect( } void Datoviz_Visual_Backend::discard(Pending_Frame pending) { require_domain(); - std::lock_guard api_lock(render_context_->api_mutex()); + std::lock_guard target_lock(target_mutex_); target(pending).discard_after_completion(); } +void Datoviz_Visual_Backend::quarantine(Pending_Frame pending) noexcept { + quarantined_ = true; + try { + std::lock_guard target_lock(target_mutex_); + if (pending.target_index < targets_->values.size()) { + auto& frame_target = targets_->values[pending.target_index]; + if (frame_target && + frame_target->generation() == pending.target_generation) + frame_target->quarantine_after_submission(); + } + } + catch (...) {} +} bool Datoviz_Visual_Backend::can_prepare( const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& prepared) const { - if (!render_context_) return false; - std::lock_guard api_lock(render_context_->api_mutex()); + if (!render_context_ || quarantined_) return false; + std::lock_guard target_lock(target_mutex_); const bool target_available = std::ranges::any_of( targets_->values, [](const auto& target) { return !target || target->available(); @@ -1912,18 +2083,33 @@ bool Datoviz_Visual_Backend::can_prepare( } bool Datoviz_Visual_Backend::idle() const { if (!render_context_) return true; - std::lock_guard api_lock(render_context_->api_mutex()); + std::lock_guard target_lock(target_mutex_); return std::ranges::all_of(targets_->values, [](const auto& target) { - return !target || target->available(); + return !target || target->quiescent(); }); } +void Datoviz_Visual_Backend::abandon_resources() noexcept { + quarantined_ = true; + static_cast(targets_.release()); + retain_quarantined_context(render_context_); +} void Datoviz_Visual_Backend::destroy() { - if (std::this_thread::get_id() != domain_thread_) + if (std::this_thread::get_id() != preparation_thread_) throw std::logic_error( - "Datoviz backend may only be destroyed on the render domain"); + "Datoviz backend may only be destroyed on its preparation thread"); + if (quarantined_) { + abandon_resources(); + return; + } auto context = render_context_; std::unique_lock api_lock; - if (context) api_lock = std::unique_lock(context->api_mutex()); + std::unique_lock queue_lock; + if (context) { + api_lock = std::unique_lock(context->api_mutex(), std::defer_lock); + queue_lock = std::unique_lock(context->queue_mutex(), std::defer_lock); + std::lock(api_lock, queue_lock); + } + std::lock_guard target_lock(target_mutex_); for (auto& slot : runtime_slots_) { if (slot.runtime != nullptr) { dvz_drp2_runtime_destroy(slot.runtime); diff --git a/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp b/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp index bd05f76..00eacc4 100644 --- a/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp +++ b/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ public: Datoviz_Visual_Backend(std::uint32_t gpu_index, bool validation_enabled, const std::vector& visuals, const Scene_3D_Parameters& initial_scene); - ~Datoviz_Visual_Backend() noexcept(false); + ~Datoviz_Visual_Backend() noexcept; Datoviz_Visual_Backend(const Datoviz_Visual_Backend&) = delete; Datoviz_Visual_Backend& operator=(const Datoviz_Visual_Backend&) = delete; [[nodiscard]] std::optional prepare( @@ -45,6 +46,7 @@ public: void submit(Pending_Frame& pending); [[nodiscard]] Completed_Frame collect(Pending_Frame pending); void discard(Pending_Frame pending); + void quarantine(Pending_Frame pending) noexcept; [[nodiscard]] bool can_prepare( const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& visuals) const; @@ -100,13 +102,19 @@ private: std::uint32_t item_count); void register_external_attributes(const DvzDrp2CommandStream* stream, std::uint8_t target_index); + [[nodiscard]] std::optional reuse_recorded_target( + const Scene_3D_Parameters& scene, + const Prepared_Visual_Batch& visuals, + std::uint64_t frame_sequence, bool observe, bool readback); [[nodiscard]] DvzSceneFrameArtifact* emit( const Scene_3D_Parameters& scene, std::uint8_t target_index); [[nodiscard]] std::optional acquire_target(Extent extent); [[nodiscard]] Frame_Target& target(const Pending_Frame& pending); + void abandon_resources() noexcept; void destroy(); - std::thread::id domain_thread_; /* 唯一允许访问 Datoviz 对象的线程。 */ + std::thread::id preparation_thread_; /* Scene API、资源准备与回收的唯一线程。 */ std::shared_ptr render_context_; /* 同一 GPU 上所有后端共享的 Device 与分配器。 */ + mutable std::mutex target_mutex_{}; /* 当前 Scene 三个目标槽的唯一生命周期同步源。 */ struct Runtime_Slot { DvzDrp2Runtime* runtime{}; /* Executes only the matching frame target's stream. */ DvzFramePlanEmitter* emitter{}; /* Retained semantic state paired one-to-one with runtime. */ @@ -131,6 +139,7 @@ private: std::optional applied_camera_{}; /* 已应用到 Panel 的 Camera 配置。 */ std::optional> applied_axes_{}; /* 已生成 Visual 的轴描述快照。 */ bool input_changed_{}; /* 输入控制器是否产生尚未录入完成帧的资源更新。 */ + bool quarantined_{}; /* GPU 已提交后失效时,整套资源停止销毁与复用。 */ }; } // namespace aethera::render_3d::detail diff --git a/render_3D/render_3D/detail/Gpu_Completion_Service.cpp b/render_3D/render_3D/detail/Gpu_Completion_Service.cpp index efba1bc..f10d1d5 100644 --- a/render_3D/render_3D/detail/Gpu_Completion_Service.cpp +++ b/render_3D/render_3D/detail/Gpu_Completion_Service.cpp @@ -21,10 +21,14 @@ Gpu_Completion_Service::~Gpu_Completion_Service() { } Gpu_Completion_Service::Reservation::Reservation( std::shared_ptr pending) noexcept : pending_(std::move(pending)) {} -Gpu_Completion_Service::Reservation::~Reservation() noexcept(false) { - cancel(); +Gpu_Completion_Service::Reservation::~Reservation() noexcept { + try { cancel(); } + catch (...) {} } Gpu_Completion_Service::Reservation::Reservation(Reservation&& other) noexcept : pending_(std::exchange(other.pending_, {})) {} +Gpu_Completion_Service::Prepare_Result::operator bool() const noexcept { + return result == Admission_Result::none; +} void Gpu_Completion_Service::Reservation::watch(VkDevice device, VkFence fence) { if (!pending_ || device == VK_NULL_HANDLE || fence == VK_NULL_HANDLE) throw std::logic_error("GPU completion reservation or fence is invalid"); @@ -100,7 +104,14 @@ Gpu_Completion_Service::Prepare_Result Gpu_Completion_Service::prepare( release_slot(); return {{}, Admission_Result::stopping}; } - { std::lock_guard lock(pending_mutex_); pending_.push_back(pending); } + try { + std::lock_guard lock(pending_mutex_); + pending_.push_back(pending); + } + catch (...) { + release_slot(); + throw; + } wake(); return {Reservation(std::move(pending)), Admission_Result::none}; } @@ -121,12 +132,13 @@ void Gpu_Completion_Service::wake() noexcept { wake_generation_.fetch_add(1, std::memory_order_release); wake_condition_.notify_one(); } -void Gpu_Completion_Service::run() { +void Gpu_Completion_Service::run() noexcept { struct Device_Fences { VkDevice device{VK_NULL_HANDLE}; std::vector fences; }; std::vector> active; + try { active.reserve(static_cast(default_capacity)); std::size_t wait_group_index{}; const auto wait_age_ns = [](std::chrono::steady_clock::time_point started) { @@ -137,7 +149,13 @@ void Gpu_Completion_Service::run() { }; const auto finish = [this, &wait_age_ns]( const std::shared_ptr& pending, - VkResult result, Completion_Error error) { + VkResult result, Completion_Error error) noexcept { + struct Slot_Release { + Gpu_Completion_Service* service; + ~Slot_Release() { service->release_slot(); } + } slot_release{this}; + watched_.fetch_sub(1, std::memory_order_relaxed); + try { Completion completion; Exception_Handler on_exception; std::chrono::steady_clock::time_point watched_at{}; @@ -150,7 +168,6 @@ void Gpu_Completion_Service::run() { observe = pending->observe; pending->status = Pending_Fence::Status::canceled; } - watched_.fetch_sub(1, std::memory_order_relaxed); Result completion_result; completion_result.error = error; completion_result.vulkan_result = result; @@ -159,13 +176,24 @@ void Gpu_Completion_Service::run() { fault_count_.fetch_add(1, std::memory_order_relaxed); abandoned_count_.fetch_add(1, std::memory_order_relaxed); } - try { - completion(std::move(completion_result)); + try { completion(std::move(completion_result)); } + catch (...) { + try { + on_exception(contextual_exception( + "delivering GPU completion", std::current_exception())); + } + catch (...) {} + } } catch (...) { - on_exception(contextual_exception("delivering GPU completion", std::current_exception())); + try { + std::lock_guard lock(pending->mutex); + if (pending->on_exception) + pending->on_exception(std::current_exception()); + pending->status = Pending_Fence::Status::canceled; + } + catch (...) {} } - release_slot(); }; for (;;) { const std::uint64_t wake_generation = @@ -299,6 +327,44 @@ void Gpu_Completion_Service::run() { : Completion_Error::vulkan_failure); } } + } + catch (...) { + stopping_.store(true, std::memory_order_release); + const auto service_failure = std::current_exception(); + const auto fail_pending = [this, &service_failure]( + const std::shared_ptr& pending) noexcept { + if (!pending) return; + Exception_Handler on_exception; + bool watched{}; + try { + std::lock_guard lock(pending->mutex); + watched = pending->status == Pending_Fence::Status::watched; + on_exception = std::move(pending->on_exception); + pending->completion = {}; + pending->status = Pending_Fence::Status::canceled; + } + catch (...) {} + if (watched) + watched_.fetch_sub(1, std::memory_order_relaxed); + if (on_exception) { + try { on_exception(service_failure); } + catch (...) {} + } + release_slot(); + }; + for (const auto& pending : active) fail_pending(pending); + for (;;) { + std::shared_ptr pending; + try { + std::lock_guard lock(pending_mutex_); + if (pending_.empty()) break; + pending = std::move(pending_.front()); + pending_.pop_front(); + } + catch (...) { break; } + fail_pending(pending); + } + } } } diff --git a/render_3D/render_3D/detail/Gpu_Completion_Service.hpp b/render_3D/render_3D/detail/Gpu_Completion_Service.hpp index 4e81db4..5f3a507 100644 --- a/render_3D/render_3D/detail/Gpu_Completion_Service.hpp +++ b/render_3D/render_3D/detail/Gpu_Completion_Service.hpp @@ -46,7 +46,7 @@ public: class Reservation final { public: Reservation() = default; - ~Reservation() noexcept(false); + ~Reservation() noexcept; Reservation(const Reservation&) = delete; Reservation& operator=(const Reservation&) = delete; Reservation(Reservation&& other) noexcept; @@ -61,9 +61,7 @@ public: struct Prepare_Result { Reservation reservation; /* 成功时返回的 fence reservation。 */ Admission_Result result{}; /* 准入结果。 */ - [[nodiscard]] explicit operator bool() const noexcept { - return result == Admission_Result::none; - } + [[nodiscard]] explicit operator bool() const noexcept; }; static Gpu_Completion_Service& instance(); Gpu_Completion_Service(const Gpu_Completion_Service&) = delete; @@ -96,7 +94,7 @@ private: void acquire_slot(); void release_slot() noexcept; void wake() noexcept; - void run(); + void run() noexcept; static constexpr std::ptrdiff_t default_capacity = 1024; static constexpr std::uint64_t fence_wait_timeout_ns = 1'000'000; static constexpr std::uint64_t maximum_fence_age_ns = 30'000'000'000ULL; diff --git a/render_3D/render_3D/detail/Render_Domain.cpp b/render_3D/render_3D/detail/Render_Domain.cpp index 30838ed..abc4d78 100644 --- a/render_3D/render_3D/detail/Render_Domain.cpp +++ b/render_3D/render_3D/detail/Render_Domain.cpp @@ -1,4 +1,5 @@ #include "Render_Domain.hpp" +#include #include #include #include @@ -50,8 +51,6 @@ void Render_Domain::request_stop() noexcept { expected, true, std::memory_order_acq_rel)) return; while (active_posters_.load(std::memory_order_acquire) != 0) std::this_thread::yield(); - std::unique_ptr stop; - if (!tasks_.enqueue(std::move(stop))) std::terminate(); } Render_Domain::Post_Result Render_Domain::post( std::function function, Exception_Handler on_exception) { @@ -72,28 +71,37 @@ Render_Domain::Post_Result Render_Domain::post( active_posters_.fetch_sub(1, std::memory_order_release); return Post_Result::queued; } -void Render_Domain::run() { +void Render_Domain::run() noexcept { current_domain_ = this; for (;;) { std::unique_ptr task; - tasks_.wait_dequeue(task); - if (!task) { + const bool received = tasks_.wait_dequeue_timed( + task, std::chrono::milliseconds(1)); + if (received && task) { + try { + task->function(); + } + catch (...) { + try { + task->on_exception( + contextual_exception("executing render domain task", + std::current_exception())); + } + catch (...) {} + } + task.reset(); + } + // 停止后不再接受新生产者;一次真实的 timed dequeue 为空,才说明 + // 已经消费完所有先于停止边界进入的任务。size_approx() 只能诊断, + // 不能参与线程退出正确性。 + if (!received && stopping_.load(std::memory_order_acquire) && + active_posters_.load(std::memory_order_acquire) == 0 && + !task) { current_domain_ = nullptr; - return; - } - try { - task->function(); - } - catch (...) { - task->on_exception( - contextual_exception("executing render domain task", - std::current_exception())); - } - task.reset(); - if (destroy_on_exit_.load(std::memory_order_acquire)) { - current_domain_ = nullptr; - if (thread_.joinable()) thread_.detach(); - delete this; + if (destroy_on_exit_.load(std::memory_order_acquire)) { + if (thread_.joinable()) thread_.detach(); + delete this; + } return; } } diff --git a/render_3D/render_3D/detail/Render_Domain.hpp b/render_3D/render_3D/detail/Render_Domain.hpp index 20a0f69..91bacef 100644 --- a/render_3D/render_3D/detail/Render_Domain.hpp +++ b/render_3D/render_3D/detail/Render_Domain.hpp @@ -43,11 +43,11 @@ private: Render_Domain(); static void destroy(Render_Domain* domain) noexcept; void request_stop() noexcept; - void run(); + void run() noexcept; static constexpr std::size_t initial_capacity = 64; inline static thread_local Render_Domain* current_domain_{}; /* 当前线程正在执行的 GPU 域。 */ moodycamel::BlockingConcurrentQueue> tasks_{initial_capacity}; /* 多生产者闭包入口与唯一消费队列。 */ - std::atomic_size_t active_posters_{}; /* 停止哨兵前仍可能入队的生产者数量。 */ + std::atomic_size_t active_posters_{}; /* 停止准入边界内仍可能完成入队的生产者数量。 */ std::atomic_bool stopping_{}; /* 拒绝新闭包并准备退出。 */ std::atomic_bool destroy_on_exit_{}; /* 最后一个引用在域线程释放时由该线程自销毁。 */ std::thread thread_; /* 唯一允许访问 Datoviz/Vulkan 对象的线程。 */ diff --git a/render_3D/render_3D/scene/Render_Scene_3D.ipp b/render_3D/render_3D/scene/Render_Scene_3D.ipp index 9d7bb76..7d45a19 100644 --- a/render_3D/render_3D/scene/Render_Scene_3D.ipp +++ b/render_3D/render_3D/scene/Render_Scene_3D.ipp @@ -10,7 +10,8 @@ struct Scene_Paint_Context { Object* scene{}; /* 仅在 Scene 拥有本上下文期间读取当前 Prop。 */ Frame_3D* frame{}; /* 当前 Submit 图对应的外部帧;process 返回后清空。 */ Scene_3D_Parameters parameters{}; /* 本帧从 Scene 组件读取的一致快照。 */ - Prepared_Visual_Batch visuals{}; /* Submit 图发布的本帧 Visual 快照集合。 */ + std::shared_ptr visuals{ /* Plot 准备域拥有的 Visual 快照;提交只共享不可变版本。 */ + std::make_shared()}; }; } struct Render_Scene_3D::Private : Prev_Private { @@ -67,7 +68,15 @@ Render_Scene_3D::Builder::add_renderable(Visual_Object* visual_value) { .bind_paint_target(std::move(context), [](void* raw_context, const Root* identity, const detail::Prepared_Visual& prepared) { auto& submission = *static_cast*>(raw_context); - submission.visuals.push_back({reinterpret_cast(identity), prepared}); + const auto visual_identity = reinterpret_cast(identity); + auto& visuals = *submission.visuals; + const auto found = std::ranges::find( + visuals, visual_identity, + &detail::Prepared_Visual_Instance::identity); + if (found == visuals.end()) + visuals.push_back({visual_identity, prepared}); + else + found->visual = prepared; }); }; visuals.push_back(binding); @@ -151,7 +160,9 @@ void Render_Scene_3D::Private::initialize_backend( backend = std::make_shared(gpu_index, validation_enabled, std::move(visuals)); paint_context = std::make_shared>( - detail::Scene_Paint_Context{backend, object, nullptr, initial, {}}); + detail::Scene_Paint_Context{ + backend, object, nullptr, initial, + std::make_shared()}); } inline void Render_Scene_3D::Private::dispatch_events(Extent viewport) { this->consume_events([&](const std::shared_ptr& event) { @@ -163,8 +174,6 @@ template void Render_Scene_3D::Private::process(Object* object, Callback&& callback) requires std::invocable { Frame_3D* frame = active_frame; if (!frame) throw std::logic_error("3D Scene process has no external frame"); - camera_component->advance_object(); - axes_component->advance_object(); const auto& prop = object->template read_prop(); if (!prop.view_active || prop.viewport.empty() || !backend || !backend->available()) return; frame->mark(Frame_Trace_Marker::scene_render_started); @@ -173,19 +182,52 @@ void Render_Scene_3D::Private::process(Object* object, Callback&& callback) requ frame->mark(Frame_Trace_Marker::event_dispatch_finished); auto context = std::static_pointer_cast>(paint_context); context->parameters = parameters(object); - context->visuals.clear(); + /* + * Visual 的 Prepared 快照只由数据变化更新,并持久保存在 Scene_Paint_Context。 + * 静态帧不推进/复制 Renderable 双缓冲,也不启动 Kernel Taskflow,只提交已经准备好的 + * 完整快照。动态 Visual 变脏时才进入完整 Prepare/Submit 路径并原位更新这份缓存。 + */ + bool prepare_dirty = context->visuals->empty(); + const auto current_prepare = object->template current_dependency_graph(); + current_prepare.for_each_bound([&](Renderable* renderable, Renderable::Private&) { + prepare_dirty = prepare_dirty || renderable->template dirty(); + }); + bool submit_dirty{}; + const auto current_submit = object->template current_dependency_graph(); + current_submit.for_each_bound([&](Renderable* renderable, Renderable::Private&) { + submit_dirty = submit_dirty || renderable->template dirty(); + }); + if (!prepare_dirty && !submit_dirty) { + frame->mark(Frame_Trace_Marker::prepare_started); + frame->mark(Frame_Trace_Marker::prepare_finished); + frame->mark(Frame_Trace_Marker::paint_started); + const auto result = context->backend->render( + context->visuals, context->parameters, frame); + frame->mark(Frame_Trace_Marker::paint_finished); + if (result == detail::Async_Render_Backend::Submit_Result::queued) + std::invoke(std::forward(callback)); + return; + } + + /* 后端仍可能持有上一版快照时只在真正变脏的帧执行一次 COW。 + * 静态帧仅复制 shared_ptr,禁止把完整 Prepared Visual 集合复制进提交队列。 */ + if (context->visuals.use_count() != 1) + context->visuals = + std::make_shared(*context->visuals); + camera_component->advance_object(); + axes_component->advance_object(); struct Frame_Context_Scope { Frame_3D*& target; /* Visual Submit 回调读取的当前外部帧槽位。 */ Frame_3D* previous{}; /* 嵌套调用前的帧;析构时恢复。 */ ~Frame_Context_Scope() { target = previous; } } frame_context_scope{context->frame, context->frame}; context->frame = frame; + bool backend_submitted{}; Prev_Private::process(object, frame, [&](const Scene::Private::Result&) { frame->mark(Frame_Trace_Marker::paint_started); const auto prepare = object->template current_dependency_graph(); prepare.for_each_bound([](Renderable* renderable, Renderable::Private& data) { if (data.dispatch->state.get(renderable)->prepare_executed) renderable->template mark_dirty(); }); const auto submit = object->template current_dependency_graph(); - submit.for_each_bound([](Renderable* renderable, Renderable::Private&) { renderable->template mark_dirty(); }); const auto result = submit.for_each_topological_view([&](const auto& view, const Dependency_Graph::Node& node) { auto* data = view.private_data(node); if (!data) return; @@ -209,12 +251,14 @@ void Render_Scene_3D::Private::process(Object* object, Callback&& callback) requ dispatch->state.notify(root); }); if (!result) throw std::logic_error("render scene submit graph became invalid during submission"); - if (context->visuals.empty()) + if (context->visuals->empty()) throw std::logic_error("render scene submit graph published no Visual snapshots"); - context->backend->render(context->visuals, context->parameters, context->frame); + backend_submitted = context->backend->render( + context->visuals, context->parameters, context->frame) == + detail::Async_Render_Backend::Submit_Result::queued; frame->mark(Frame_Trace_Marker::paint_finished); }); - std::invoke(std::forward(callback)); + if (backend_submitted) std::invoke(std::forward(callback)); } template Render_Scene_3D::Render_Result Render_Scene_3D::Private::render(Object* object, Frame_3D* frame) { diff --git a/render_3D/render_3D/visual/Basic_Visual.ipp b/render_3D/render_3D/visual/Basic_Visual.ipp index a85b0d2..02cb781 100644 --- a/render_3D/render_3D/visual/Basic_Visual.ipp +++ b/render_3D/render_3D/visual/Basic_Visual.ipp @@ -43,7 +43,7 @@ template bool Basic_Visual::Private::valid_items(const std template void Basic_Visual::Private::bind_paint_target(std::shared_ptr context, Enqueue_Paint enqueue) { paint_target = {std::move(context), enqueue}; } template template void Basic_Visual::Private::prepare_data(Object* object) { using Visual_Tag = typename Basic_Visual::Base_Tag; const auto& prop = object->template read_prop(); if (!valid_items(prop.items) || !detail::finite(prop.transform)) throw std::invalid_argument("3D visual property contains invalid data"); auto data = std::make_shared(); Spec::prepare(prop.items, *data); auto& output = object->template pending_buffer(); output = {}; output.family = Spec::family; output.transform = prop.transform; output.visible = prop.visible; output.depth_test = prop.depth_test; output.revision = next_revision++; output.data = std::move(data); if constexpr (requires { prop.style; }) output.point_style = prop.style; object->template update_state<&State::prepared_item_count, &State::prepared_revision>([&](State_Access states) { auto& state = states.template get(); state.prepared_item_count = prop.items.size(); state.prepared_revision = output.revision; }); } template template tf::Taskflow Basic_Visual::Private::build_paint_graph(Object* object, const Prop&) { tf::Taskflow graph; graph.emplace([this, object] { if (auto context = paint_target.context.lock(); context && paint_target.enqueue) { const auto& pending = object->template pending_buffer(); const auto& current = object->template current_buffer(); paint_target.enqueue(context.get(), object, pending.revision >= current.revision ? pending : current); } }).name("render_3d.paint.publish"); return graph; } -template template bool Basic_Visual::Private::should_paint(Object*, const State&, bool) { return paint_target.enqueue != nullptr && !paint_target.context.expired(); } +template template bool Basic_Visual::Private::should_paint(Object*, const State&, bool dirty) { return dirty && paint_target.enqueue != nullptr && !paint_target.context.expired(); } template template void Basic_Visual::Private::after_prop_set(Object* object, Member Owner::*, Prop_Access) { if constexpr (std::same_as) object->template mark_dirty(); } template template void Basic_Visual::Private::before_advance(Object*, Prop_Type*, State_Access pending_states, const Prop_Type* current_prop, State_Access) { using Visual_Tag = typename Basic_Visual::Base_Tag; pending_states.template get().item_count = static_cast(*current_prop).items.size(); } template template const typename Basic_Visual::Private::Dispatch& Basic_Visual::Private::dispatch_for() { using Visual_Tag = typename Basic_Visual::Base_Tag; static const Dispatch value{[](Root* root, std::vector items) { if (!Private::valid_items(items)) return Update_Items_Result::invalid_data; static_cast(root)->template set<&Prop::items>(std::move(items)); return Update_Items_Result::updated; }, [](Root* root, const std::function&)>& edit) { if (!edit) return Edit_Items_Result::empty_edit; auto* object = static_cast(root); auto items = object->template read_prop().items; edit(items); if (!Private::valid_items(items)) return Edit_Items_Result::invalid_data; object->template set<&Prop::items>(std::move(items)); return Edit_Items_Result::updated; }, [](const Root* root) { return static_cast(root)->template read_prop().items.size(); }}; return value; } diff --git a/web_server/cmake/rely.cmake b/web_server/cmake/rely.cmake index 0bc421a..238b216 100644 --- a/web_server/cmake/rely.cmake +++ b/web_server/cmake/rely.cmake @@ -1,4 +1,38 @@ include_guard(GLOBAL) +block() + rcl_init(render_kernel) + set(oneTBB_option ${base_options}) + list(APPEND oneTBB_option + -DBUILD_SHARED_LIBS=ON + -DTBB_BUILD=ON + -DTBB_INSTALL=ON + -DTBB_TEST=OFF + -DTBB_EXAMPLES=OFF + -DTBB_STRICT=OFF + -DTBB4PY_BUILD=OFF + -DTBBMALLOC_BUILD=ON + -DTBBMALLOC_PROXY_BUILD=ON + ) + _register_git_cmake_library(render_kernel::oneTBB + "https://github.com/uxlfoundation/oneTBB.git" + "v2023.1.0" + ) + rcl_get_effective_install_dir(render_kernel::oneTBB ONETBB_ROOT) + set(TBB_CONFIG_DIR "${ONETBB_ROOT}/lib/cmake/TBB") + rcl_cmake_library_set_cmake_options(render_kernel::oneTBB ${oneTBB_option}) + rcl_cmake_library_set_other_use_opt(render_kernel::oneTBB + "-DTBB_DIR=\"${TBB_CONFIG_DIR}\"" + ) + rcl_cmake_library_set_init_script(render_kernel::oneTBB + "set(ONETBB_ROOT \"${ONETBB_ROOT}\")" + "set(TBB_DIR \"${TBB_CONFIG_DIR}\")" + ) + rcl_cmake_library_set_clear_script(render_kernel::oneTBB + "unset(ONETBB_ROOT)" + "unset(TBB_DIR)" + ) + rcl_export_configuration(render_kernel) +endblock() block() set(libdatachannel_option ${base_options}) list(APPEND libdatachannel_option @@ -42,4 +76,4 @@ block() "unset(libdatachannel_DIR)" ) rcl_export_configuration(aethera_kernel) -endblock() \ No newline at end of file +endblock() diff --git a/web_server/main.cmake b/web_server/main.cmake index ee298e0..bbfa3e6 100644 --- a/web_server/main.cmake +++ b/web_server/main.cmake @@ -1,7 +1,12 @@ include("${CMAKE_CURRENT_LIST_DIR}/third_party/asio/main.cmake") include("${CMAKE_CURRENT_LIST_DIR}/cmake/rely.cmake") include("${CMAKE_CURRENT_LIST_DIR}/third_party/ffmpeg.cmake") -set(Aethera_Web_dependencies aethera_kernel::libdatachannel global::drogon global::spdlog global::pfr) +set(Aethera_Web_dependencies + aethera_kernel::libdatachannel + render_kernel::oneTBB + global::drogon + global::spdlog + global::pfr) rcl_add_dependency_action_targets(Aethera_Web_env ${Aethera_Web_dependencies}) set_target_properties(Aethera_Web_env PROPERTIES FOLDER Aethera_Web) function(renderive_configure_frontend) @@ -51,6 +56,10 @@ function(renderive_configure_frontend) COMMAND "${CMAKE_COMMAND}" -E env "PATH=${node_path}" "${npm_executable}" ci + COMMAND "${node_executable}" + "${frontend_dir}/node_modules/typescript/bin/tsc" --version + COMMAND "${node_executable}" + "${frontend_dir}/node_modules/vite/bin/vite.js" --version COMMAND "${CMAKE_COMMAND}" -E touch "${install_stamp}" DEPENDS "${frontend_dir}/package.json" @@ -92,6 +101,7 @@ rcl_load_dependency_environment(${Aethera_Web_dependencies}) find_package(Drogon CONFIG REQUIRED) find_package(spdlog CONFIG REQUIRED) find_package(libdatachannel REQUIRED) +find_package(TBB CONFIG REQUIRED) if (NOT TARGET Aethera_render_2D) rcl_log_append(${CMAKE_CURRENT_LIST_LINE} "[FATAL_ERROR] Aethera_Web requires Aethera_render_2D") return() @@ -115,15 +125,20 @@ if (NOT TARGET structive::property_core) endif () set(Aethera_Web_source_dir "${CMAKE_CURRENT_LIST_DIR}/src") append_glob_source(Aethera_Web_sources "${Aethera_Web_source_dir}") -add_executable(Aethera_Web_Server ${Aethera_Web_sources}) -target_include_directories(Aethera_Web_Server PRIVATE +set(Aethera_Web_entry_sources + "${Aethera_Web_source_dir}/main.cpp" + "${Aethera_Web_source_dir}/Web_Server.cpp") +set(Aethera_Web_core_sources ${Aethera_Web_sources}) +list(REMOVE_ITEM Aethera_Web_core_sources ${Aethera_Web_entry_sources}) +add_library(Aethera_Web_Core STATIC ${Aethera_Web_core_sources}) +target_include_directories(Aethera_Web_Core PUBLIC "$" "$" "$" ) -target_compile_features(Aethera_Web_Server PRIVATE cxx_std_20) -target_compile_definitions(Aethera_Web_Server PRIVATE NOMINMAX) -target_link_libraries(Aethera_Web_Server PRIVATE +target_compile_features(Aethera_Web_Core PUBLIC cxx_std_20) +target_compile_definitions(Aethera_Web_Core PUBLIC NOMINMAX) +target_link_libraries(Aethera_Web_Core PUBLIC Aethera_render_2D Aethera_render_3D Drogon::Drogon @@ -133,6 +148,10 @@ target_link_libraries(Aethera_Web_Server PRIVATE LibDataChannel::LibDataChannel Aethera::FFmpeg ) +add_executable(Aethera_Web_Server ${Aethera_Web_entry_sources}) +target_link_libraries(Aethera_Web_Server PRIVATE + Aethera_Web_Core + TBB::tbbmalloc_proxy) add_dependencies(Aethera_Web_Server Aethera_Web_Assets) add_custom_command(TARGET Aethera_Web_Server POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy_if_different @@ -141,5 +160,56 @@ add_custom_command(TARGET Aethera_Web_Server POST_BUILD COMMENT "Deploying complete FFmpeg shared runtime for Aethera WebRTC video" VERBATIM) if (MSVC) + target_compile_options(Aethera_Web_Core PRIVATE /utf-8 /bigobj) target_compile_options(Aethera_Web_Server PRIVATE /utf-8 /bigobj) + # The Debug CRT serializes every allocation behind one process-wide lock. + # A gallery frame touches 2D paint, 3D prepare and RTP packetization in + # parallel, so that diagnostic allocator would otherwise become the + # renderer's dominant synchronization primitive. Force-load oneTBB's + # standard-allocation proxy for the server process only. + target_link_options(Aethera_Web_Server PRIVATE + "/INCLUDE:__TBB_malloc_proxy") + add_custom_command(TARGET Aethera_Web_Server POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + "$" + "$" + "$" + COMMENT "Deploying the scalable allocator runtime for Aethera Web" + VERBATIM) +endif () + +if (Aethera_BUILD_TESTS) + set(Aethera_Web_test_dir "${CMAKE_CURRENT_LIST_DIR}/tests") + append_glob_source(Aethera_Web_test_sources "${Aethera_Web_test_dir}") + set(Aethera_Web_test_targets) + foreach (Aethera_Web_test_source IN LISTS Aethera_Web_test_sources) + if (NOT Aethera_Web_test_source MATCHES "\\.(c|cc|cpp|cxx)$") + continue() + endif () + get_filename_component(Aethera_Web_test_name + "${Aethera_Web_test_source}" NAME_WE) + set(Aethera_Web_test_target "Aethera_Web_${Aethera_Web_test_name}") + add_executable("${Aethera_Web_test_target}" "${Aethera_Web_test_source}") + target_link_libraries("${Aethera_Web_test_target}" PRIVATE + Aethera_Web_Core GTest::gtest_main) + if (MSVC) + target_compile_options("${Aethera_Web_test_target}" PRIVATE /utf-8 /bigobj) + endif () + add_custom_command(TARGET "${Aethera_Web_test_target}" POST_BUILD + COMMAND "${CMAKE_COMMAND}" -E copy_if_different + ${Aethera_FFmpeg_runtime_libraries} + "$" + COMMENT "Deploying FFmpeg runtime for ${Aethera_Web_test_target}" + VERBATIM) + add_test(NAME "${Aethera_Web_test_target}" + COMMAND "${Aethera_Web_test_target}") + set_tests_properties("${Aethera_Web_test_target}" PROPERTIES + LABELS "Aethera_Web") + list(APPEND Aethera_Web_test_targets "${Aethera_Web_test_target}") + endforeach () + add_custom_target(Aethera_Web_check + COMMAND "${CMAKE_CTEST_COMMAND}" --test-dir "${CMAKE_BINARY_DIR}" + -C "$" -L "^Aethera_Web$" --output-on-failure + DEPENDS ${Aethera_Web_test_targets} + USES_TERMINAL) endif () diff --git a/web_server/src/Exclusive_Page.cpp b/web_server/src/Exclusive_Page.cpp new file mode 100644 index 0000000..cf293bc --- /dev/null +++ b/web_server/src/Exclusive_Page.cpp @@ -0,0 +1,39 @@ +#include "Exclusive_Page.hpp" +#include +#include +#include +#include + +namespace aethera::web { +namespace { +struct Exclusive_Page_State { + std::mutex mutex; + std::string page; /* 当前唯一浏览器页面实例标识。 */ + std::size_t connection_count{}; /* 该页面的 Gallery 与各 Plot 连接总数。 */ +}; + +Exclusive_Page_State state; +} + +bool valid_page_id(std::string_view page) { + return page.size() >= 16 && page.size() <= 64 && + std::ranges::all_of(page, [](unsigned char value) { + return std::isalnum(value) || value == '-' || value == '_'; + }); +} + +bool acquire_exclusive_page(std::string_view page) { + std::lock_guard lock(state.mutex); + if (state.connection_count != 0 && state.page != page) return false; + if (state.connection_count == 0) state.page = page; + ++state.connection_count; + return true; +} + +void release_exclusive_page(std::string_view page) { + std::lock_guard lock(state.mutex); + if (state.page != page || state.connection_count == 0) return; + --state.connection_count; + if (state.connection_count == 0) state.page.clear(); +} +} diff --git a/web_server/src/Exclusive_Page.hpp b/web_server/src/Exclusive_Page.hpp new file mode 100644 index 0000000..d6d376e --- /dev/null +++ b/web_server/src/Exclusive_Page.hpp @@ -0,0 +1,8 @@ +#pragma once +#include + +namespace aethera::web { +[[nodiscard]] bool valid_page_id(std::string_view page); +[[nodiscard]] bool acquire_exclusive_page(std::string_view page); +void release_exclusive_page(std::string_view page); +} diff --git a/web_server/src/Gallery_Video_Stream.cpp b/web_server/src/Gallery_Video_Stream.cpp new file mode 100644 index 0000000..62c9be0 --- /dev/null +++ b/web_server/src/Gallery_Video_Stream.cpp @@ -0,0 +1,530 @@ +#include "Gallery_Video_Stream.hpp" +#include "detail/Gallery_Frame_Atlas.hpp" +#include "detail/Gallery_Frame_Clock.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web { +namespace { +// Scene 的业务视口是 720x420;媒体图集必须保持同一像素分辨率,禁止通过 +// 缩小每图画布换取帧率。每个媒体组四列、八图,编码面为 2880x840。 +constexpr std::uint32_t tile_width{720}; +constexpr std::uint32_t tile_height{420}; +constexpr std::uint32_t atlas_columns{4}; +constexpr double gallery_frame_rate{100.0}; +constexpr auto metric_interval{std::chrono::seconds(1)}; + +std::string exception_description(const std::exception_ptr& failure) { + try { + if (failure) std::rethrow_exception(failure); + } + catch (const std::exception& error) { + return error.what(); + } + catch (...) { + return "non-standard gallery video failure"; + } + return "empty gallery video failure"; +} + +double percentile(std::vector values, double quantile) { + if (values.empty()) return 0.0; + std::ranges::sort(values); + const auto index = static_cast(std::llround( + quantile * static_cast(values.size() - 1U))); + return values[std::min(index, values.size() - 1U)]; +} + +double average(const std::vector& values) { + return values.empty() + ? 0.0 + : std::accumulate(values.begin(), values.end(), 0.0) / + static_cast(values.size()); +} +} + +struct Gallery_Video_Stream::Private { + struct Source { + Plot_Entry entry; /* 布局槽位关联的实际 Plot。 */ + Plot::Stream_Id stream{}; /* Plot 完成帧的唯一订阅标识。 */ + }; + + asio::any_io_executor clock_executor; /* bind_plots 时转交给页面级绝对帧时钟。 */ + asio::strand encoding_strand; + std::vector sources{}; /* 已按业务标识排序的稳定图集来源。 */ + std::unique_ptr atlas{}; /* 最近完成帧与 RGBA 图集的唯一状态源。 */ + std::unique_ptr clock{}; /* 页面所有 Plot 共用的绝对 100 Hz 时钟。 */ + H264_Encoder encoder{gallery_frame_rate}; /* 页面唯一的实际可用硬件 H.264 编码器。 */ + struct Consumer { + Stream_Id id{}; /* 当前独占网页媒体连接的订阅身份。 */ + std::shared_ptr handler{}; /* 发布期间保持回调存活,不复制 std::function。 */ + std::shared_ptr video_readiness{}; /* 编码前查询 WebRTC 三槽发送窗口。 */ + }; + mutable std::mutex consumers_mutex; + Stream_Id next_consumer_id{1}; /* 在互斥量内区分重连前后的独占消费者。 */ + std::optional consumer{}; /* 独占网页对应的唯一媒体消费者。 */ + std::atomic_bool stopping{}; /* 拒绝关闭开始后的新 tick、帧与订阅。 */ + std::mutex pending_mutex; + std::optional pending_encode_tick{}; /* 编码拥塞时保留的最新页面时钟点。 */ + bool encode_scheduled{}; /* encoding_strand 是否已有唯一消费任务。 */ + std::atomic_uint64_t skipped_encode_ticks{}; /* 被最新帧策略合并的页面编码 tick 总数。 */ + std::atomic_uint64_t delivered_clock_ticks{}; /* 实际分发给全部 Plot 的页面 tick 总数。 */ + std::atomic_bool failed{}; /* 首次 Unknown Failure 后停止全部热路径。 */ + std::optional terminal_failure{}; /* 编码/时钟任务首次 Unknown Failure。 */ + std::uint64_t encoded_frame_count{}; /* 成功产出 H.264 access unit 的累计数量。 */ + std::chrono::steady_clock::time_point metric_started{}; /* 当前一秒指标窗口的起点。 */ + std::uint64_t metric_encoded_start{}; /* 指标窗口起点的累计编码帧数。 */ + std::uint64_t metric_clock_start{}; /* 指标窗口起点的累计已分发 tick 数。 */ + std::vector metric_completion_starts{}; /* 指标窗口起点各 Plot 逻辑完成回调数。 */ + std::vector metric_rendered_starts{}; /* 指标窗口起点各 Plot 真实画面数。 */ + std::vector metric_compose_samples{}; /* 当前窗口图集快照合成耗时,单位毫秒。 */ + std::vector metric_encode_samples{}; /* 当前窗口实际硬件编码耗时,单位毫秒。 */ + std::vector metric_publish_samples{}; /* 当前窗口 WebRTC 发送调用耗时,单位毫秒。 */ + + Private(asio::any_io_executor value_clock_executor, + asio::any_io_executor encoding_executor, + std::vector plots) + : clock_executor(std::move(value_clock_executor)), + encoding_strand(asio::make_strand(std::move(encoding_executor))) { + std::ranges::sort(plots, {}, &Plot_Entry::id); + std::vector source_ids; + source_ids.reserve(plots.size()); + sources.reserve(plots.size()); + for (auto& plot : plots) { + if (!plot.plot) + throw std::invalid_argument("gallery video source has no Plot"); + source_ids.push_back(plot.id); + sources.push_back(Source{std::move(plot)}); + } + atlas = std::make_unique( + tile_width, tile_height, atlas_columns, std::move(source_ids)); + metric_completion_starts.resize(sources.size()); + metric_rendered_starts.resize(sources.size()); + } + + [[nodiscard]] std::shared_ptr + consumer_handler() const { + std::lock_guard lock(consumers_mutex); + return consumer ? consumer->handler : nullptr; + } + + [[nodiscard]] bool consumer_accepts_video() const noexcept { + std::shared_ptr readiness; + { + std::lock_guard lock(consumers_mutex); + if (consumer) readiness = consumer->video_readiness; + } + if (!readiness) return false; + try { return (*readiness)(); } + catch (...) { return false; } + } + + void publish(std::optional video, + std::string metadata) noexcept { + if (!video && metadata.empty()) return; + try { + const auto handler = consumer_handler(); + if (!handler) return; + try { + (*handler)(Gallery_Stream_Frame{ + std::move(video), std::move(metadata)}); + return; + } + catch (...) {} + { + std::lock_guard lock(consumers_mutex); + if (consumer && consumer->handler == handler) + consumer.reset(); + } + if (clock) clock->stop(); + } + catch (...) {} + } + + void fail(std::exception_ptr failure) noexcept { + if (failed.exchange(true, std::memory_order_acq_rel)) return; + if (clock) clock->stop(); + try { + terminal_failure = exception_description(failure); + publish(std::nullopt, nlohmann::json{ + {"kind", "gallery_error"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"message", *terminal_failure} + }.dump()); + } + catch (...) {} + } + + void accept_frame(std::size_t slot, + std::shared_ptr frame) { + if (stopping.load(std::memory_order_acquire) || + failed.load(std::memory_order_acquire) || + !frame) return; + static_cast(atlas->accept_frame(slot, frame->pixels)); + } + + void dispatch_tick(Plot_Render_Tick tick, + std::weak_ptr lifetime) { + if (stopping.load(std::memory_order_acquire) || + failed.load(std::memory_order_acquire)) return; + tick.width = tile_width; + tick.height = tile_height; + delivered_clock_ticks.fetch_add(1, std::memory_order_relaxed); + /* + * 时钟只调用 render:每个 Plot 在自己的并行 strand 完成数据生成和 + * Scene Prepare。编码使用上一个已经完成的图集快照,不等待本 tick 的 + * GPU 结果,因此 CPU Prepare、GPU 三槽和页面视频编码能够持续重叠。 + */ + for (const auto& source : sources) + source.entry.plot->schedule_render(tick); + queue_encoding(tick, std::move(lifetime)); + } + + void queue_encoding(Plot_Render_Tick tick, + std::weak_ptr lifetime) { + bool schedule{}; + { + std::lock_guard lock(pending_mutex); + if (pending_encode_tick) + skipped_encode_ticks.fetch_add(1, std::memory_order_relaxed); + pending_encode_tick = tick; + if (!encode_scheduled) { + encode_scheduled = true; + schedule = true; + } + } + if (!schedule) return; + asio::post(encoding_strand, [lifetime] { + if (const auto owner = lifetime.lock()) + owner->d->encode_latest(lifetime); + }); + } + + [[nodiscard]] std::string make_metrics( + const Plot_Render_Tick& tick, + const detail::Gallery_Atlas_Composition& composition, + std::size_t encoded_bytes, + Video_Encoder_Backend encoder_backend) { + const auto now = std::chrono::steady_clock::now(); + if (metric_started.time_since_epoch().count() == 0) { + metric_started = now; + metric_encoded_start = encoded_frame_count; + metric_clock_start = delivered_clock_ticks.load(std::memory_order_relaxed); + for (std::size_t slot = 0; slot < composition.sources.size(); ++slot) { + metric_completion_starts[slot] = + composition.sources[slot].completion_count; + metric_rendered_starts[slot] = + composition.sources[slot].rendered_frame_count; + } + metric_compose_samples.clear(); + metric_encode_samples.clear(); + metric_publish_samples.clear(); + return {}; + } + const auto elapsed = now - metric_started; + if (elapsed < metric_interval) return {}; + const double seconds = std::chrono::duration(elapsed).count(); + const auto clock_total = + delivered_clock_ticks.load(std::memory_order_relaxed); + nlohmann::json source_metrics = nlohmann::json::object(); + const auto description = atlas->describe(); + for (std::size_t slot = 0; slot < composition.sources.size(); ++slot) { + const auto& progress = composition.sources[slot]; + const auto completed = progress.completion_count - + metric_completion_starts[slot]; + const auto rendered = progress.rendered_frame_count - + metric_rendered_starts[slot]; + source_metrics[description.sources[slot].id] = { + {"has_rendered_frame", progress.rendered_sequence != 0}, + {"logical_completion_rate_fps", + static_cast(completed) / seconds}, + {"rendered_frame_rate_fps", + static_cast(rendered) / seconds}, + {"logical_completion_count", progress.completion_count}, + {"rendered_frame_count", progress.rendered_frame_count}, + {"latest_completion_sequence", progress.completion_sequence}, + {"latest_rendered_sequence", progress.rendered_sequence}, + {"latest_rendered_clock_sequence", + progress.rendered_correlation_id}, + {"clock_lag_ticks", + progress.rendered_correlation_id != 0 && + tick.sequence >= progress.rendered_correlation_id + ? tick.sequence - progress.rendered_correlation_id : 0} + }; + metric_completion_starts[slot] = progress.completion_count; + metric_rendered_starts[slot] = progress.rendered_frame_count; + } + auto output = nlohmann::json{ + {"kind", "gallery_metrics"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"clock_sequence", tick.sequence}, + {"encoded_frame_count", encoded_frame_count}, + {"encoder_backend", video_encoder_backend_name(encoder_backend)}, + {"profile_level_id", h264_profile_level_id()}, + {"target_frame_rate_fps", gallery_frame_rate}, + {"clock_delivery_rate_fps", + static_cast(clock_total - metric_clock_start) / seconds}, + {"encoded_frame_rate_fps", + static_cast(encoded_frame_count - metric_encoded_start) / seconds}, + {"compose_average_ms", average(metric_compose_samples)}, + {"compose_p95_ms", percentile(metric_compose_samples, 0.95)}, + {"encode_average_ms", average(metric_encode_samples)}, + {"encode_p95_ms", percentile(metric_encode_samples, 0.95)}, + {"publish_average_ms", average(metric_publish_samples)}, + {"publish_p95_ms", percentile(metric_publish_samples, 0.95)}, + {"encoded_bytes", encoded_bytes}, + {"fresh_tiles", composition.fresh_tile_count}, + {"missing_tiles", composition.missing_tile_count}, + {"rejected_frames", composition.rejected_frame_count}, + {"skipped_encode_ticks", + skipped_encode_ticks.load(std::memory_order_relaxed)}, + {"sources", std::move(source_metrics)} + }.dump(); + metric_started = now; + metric_encoded_start = encoded_frame_count; + metric_clock_start = clock_total; + metric_compose_samples.clear(); + metric_encode_samples.clear(); + metric_publish_samples.clear(); + return output; + } + + void encode_latest(std::weak_ptr lifetime) { + Plot_Render_Tick tick; + { + std::lock_guard lock(pending_mutex); + if (!pending_encode_tick) { + encode_scheduled = false; + return; + } + tick = *pending_encode_tick; + pending_encode_tick.reset(); + } + if (!stopping.load(std::memory_order_acquire) && + !failed.load(std::memory_order_acquire) && + consumer_accepts_video()) { + try { + const auto compose_started = std::chrono::steady_clock::now(); + auto composition = atlas->compose(); + metric_compose_samples.push_back( + std::chrono::duration( + std::chrono::steady_clock::now() - compose_started) + .count()); + const auto started = std::chrono::steady_clock::now(); + auto video = encoder.encode( + composition.rgba, composition.width, composition.height, + Video_Pixel_Layout::rgba, tick.sequence, + std::chrono::microseconds{static_cast( + std::llround(tick.time_milliseconds * 1'000.0))}); + const double encode_time = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + metric_encode_samples.push_back(encode_time); + if (video) { + ++encoded_frame_count; + const auto encoded_bytes = video->annex_b.size(); + const auto encoder_backend = video->backend; + auto metadata = make_metrics( + tick, composition, encoded_bytes, encoder_backend); + const auto publish_started = + std::chrono::steady_clock::now(); + publish(std::move(*video), std::move(metadata)); + metric_publish_samples.push_back( + std::chrono::duration( + std::chrono::steady_clock::now() - publish_started) + .count()); + } + } + catch (...) { + fail(std::current_exception()); + } + } else if (!stopping.load(std::memory_order_acquire) && + !failed.load(std::memory_order_acquire)) { + skipped_encode_ticks.fetch_add(1, std::memory_order_relaxed); + } + bool again{}; + { + std::lock_guard lock(pending_mutex); + if (pending_encode_tick && + !failed.load(std::memory_order_acquire)) again = true; + else encode_scheduled = false; + } + if (again) { + try { + asio::post(encoding_strand, [lifetime] { + if (const auto owner = lifetime.lock()) + owner->d->encode_latest(lifetime); + }); + } + catch (...) { fail(std::current_exception()); } + } + } +}; + +std::shared_ptr Gallery_Video_Stream::create( + asio::any_io_executor clock_executor, + asio::any_io_executor encoding_executor, + std::vector plots) { + auto result = std::shared_ptr(new Gallery_Video_Stream( + std::move(clock_executor), std::move(encoding_executor), std::move(plots))); + result->bind_plots(); + return result; +} + +Gallery_Video_Stream::Gallery_Video_Stream( + asio::any_io_executor clock_executor, + asio::any_io_executor encoding_executor, + std::vector plots) + : d(std::make_unique(std::move(clock_executor), + std::move(encoding_executor), + std::move(plots))) {} + +Gallery_Video_Stream::~Gallery_Video_Stream() { + shutdown(); +} + +void Gallery_Video_Stream::bind_plots() { + const auto weak = weak_from_this(); + d->clock = std::make_unique( + std::move(d->clock_executor), gallery_frame_rate, + [weak](Plot_Render_Tick tick) { + if (const auto owner = weak.lock()) + owner->d->dispatch_tick(tick, weak); + }, + [weak](std::exception_ptr failure) { + if (const auto owner = weak.lock()) { + asio::post(owner->d->encoding_strand, + [weak, failure = std::move(failure)]() mutable { + if (const auto stream = weak.lock()) + stream->d->fail(std::move(failure)); + }); + } + }); + for (std::size_t slot = 0; slot < d->sources.size(); ++slot) { + auto& source = d->sources[slot]; + source.stream = source.entry.plot->subscribe( + [weak, slot](std::shared_ptr frame) { + const auto owner = weak.lock(); + if (!owner || owner->d->stopping.load(std::memory_order_acquire)) + return; + /* + * 每个 Plot 只替换自己槽位的不可变最近帧快照。这里不能把 + * 每个源帧重新投递到单一编码 strand,否则 24 个高帧率图 + * 会先制造任务洪泛,再让图集编码永远追赶旧任务。 + */ + owner->d->accept_frame(slot, std::move(frame)); + }); + source.entry.plot->configure_stream(source.stream, tile_width, tile_height); + } +} + +Gallery_Video_Stream::Stream_Id Gallery_Video_Stream::subscribe( + Stream_Handler handler, Video_Readiness video_readiness) { + if (d->stopping.load(std::memory_order_acquire)) + throw std::logic_error("gallery video stream is shutting down"); + if (d->failed.load(std::memory_order_acquire)) + throw std::logic_error("gallery video stream is unavailable"); + if (!handler) + throw std::invalid_argument("gallery video subscription requires a handler"); + if (!video_readiness) + throw std::invalid_argument( + "gallery video subscription requires WebRTC readiness"); + Stream_Id id{}; + { + std::lock_guard lock(d->consumers_mutex); + id = d->next_consumer_id++; + // Exclusive_Page 已保证调用者来自唯一网页。重载时新旧 WebSocket 会短暂 + // 重叠,最新连接原子替换旧回调;旧连接的 unsubscribe(id) 不会误删新连接。 + d->consumer = Private::Consumer{ + id, std::make_shared(std::move(handler)), + std::make_shared( + std::move(video_readiness))}; + } + try { + d->clock->start(); + } + catch (...) { + std::lock_guard lock(d->consumers_mutex); + if (d->consumer && d->consumer->id == id) d->consumer.reset(); + throw; + } + return id; +} + +void Gallery_Video_Stream::unsubscribe(Stream_Id stream) { + bool removed{}; + { + std::lock_guard lock(d->consumers_mutex); + if (d->consumer && d->consumer->id == stream) { + d->consumer.reset(); + removed = true; + } + } + if (removed) d->clock->stop(); +} + +void Gallery_Video_Stream::request_video_key_frame() { + const auto weak = weak_from_this(); + asio::post(d->encoding_strand, [weak] { + if (const auto owner = weak.lock(); + owner && !owner->d->stopping.load(std::memory_order_acquire) && + !owner->d->failed.load(std::memory_order_acquire)) + owner->d->encoder.request_key_frame(); + }); +} + +void Gallery_Video_Stream::shutdown() noexcept { + if (d->stopping.exchange(true, std::memory_order_acq_rel)) return; + if (d->clock) d->clock->stop(); + for (auto& source : d->sources) { + if (source.stream == 0) continue; + try { source.entry.plot->unsubscribe(source.stream); } + catch (...) {} + source.stream = 0; + } + try { + std::lock_guard lock(d->consumers_mutex); + d->consumer.reset(); + } + catch (...) {} + try { + std::lock_guard lock(d->pending_mutex); + d->pending_encode_tick.reset(); + } + catch (...) {} +} + +std::string Gallery_Video_Stream::layout_description() const { + const auto description = d->atlas->describe(); + nlohmann::json plots = nlohmann::json::object(); + for (const auto& source : description.sources) + plots[source.id] = {{"column", source.column}, {"row", source.row}}; + return nlohmann::json{ + {"kind", "gallery_layout"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"columns", description.columns}, + {"rows", description.rows}, + {"tile_width", description.tile_width}, + {"tile_height", description.tile_height}, + {"width", description.width}, + {"height", description.height}, + {"video", {{"codec", "H264"}, + {"profile_level_id", h264_profile_level_id()}, + {"transport", "WebRTC"}}}, + {"plots", std::move(plots)} + }.dump(); +} +} diff --git a/web_server/src/Gallery_Video_Stream.hpp b/web_server/src/Gallery_Video_Stream.hpp new file mode 100644 index 0000000..56413ae --- /dev/null +++ b/web_server/src/Gallery_Video_Stream.hpp @@ -0,0 +1,53 @@ +#pragma once +#include "H264_Encoder.hpp" +#include "Plot.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web { +struct Gallery_Stream_Frame { + std::optional video{}; /* 页面唯一 H.264 图集 access unit;交付时转移所有权。 */ + std::string metadata{}; /* 低频图集编码和来源完整性指标。 */ +}; + +class Gallery_Video_Stream final + : public std::enable_shared_from_this { +public: + struct Plot_Entry { + std::string id; /* 图集布局和浏览器裁剪使用的业务图标识。 */ + std::shared_ptr plot; /* 当前图实际 Scene 与界面之间的连接对象。 */ + }; + using Stream_Id = std::uint64_t; + using Stream_Handler = + std::function; + using Video_Readiness = std::function; + + [[nodiscard]] static std::shared_ptr create( + asio::any_io_executor clock_executor, + asio::any_io_executor encoding_executor, + std::vector plots); + ~Gallery_Video_Stream(); + Gallery_Video_Stream(const Gallery_Video_Stream&) = delete; + Gallery_Video_Stream& operator=(const Gallery_Video_Stream&) = delete; + + [[nodiscard]] Stream_Id subscribe(Stream_Handler handler, + Video_Readiness video_readiness); + void unsubscribe(Stream_Id stream); + void request_video_key_frame(); + void shutdown() noexcept; + [[nodiscard]] std::string layout_description() const; + +private: + struct Private; + Gallery_Video_Stream(asio::any_io_executor clock_executor, + asio::any_io_executor encoding_executor, + std::vector plots); + void bind_plots(); + std::unique_ptr d; +}; +} diff --git a/web_server/src/Gallery_WebSocket.cpp b/web_server/src/Gallery_WebSocket.cpp new file mode 100644 index 0000000..3db38cd --- /dev/null +++ b/web_server/src/Gallery_WebSocket.cpp @@ -0,0 +1,265 @@ +#include "Gallery_WebSocket.hpp" +#include "Exclusive_Page.hpp" +#include "WebRtc_Video_Session.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web { +namespace { +std::string media_failure_description(const std::exception_ptr& failure) { + try { + if (failure) std::rethrow_exception(failure); + } + catch (const std::exception& error) { + return error.what(); + } + catch (...) { + return "non-standard WebRTC failure"; + } + return "empty WebRTC failure"; +} +} + +struct Gallery_WebSocket::Private { + std::weak_ptr connection; + std::shared_ptr stream; + std::unique_ptr video; + Gallery_Video_Stream::Stream_Id subscription{}; + std::atomic_bool attached{}; + std::atomic_bool media_failed{}; /* 首次 WebRTC 发送 Unknown Failure 后停止重复报告。 */ + std::string page_id; /* 与各 Plot WebSocket 共用的独占页面租约。 */ +}; + +Gallery_WebSocket::Gallery_WebSocket( + drogon::WebSocketConnectionPtr connection, + std::shared_ptr stream, + std::string page_id) + : d(std::make_unique()) { + d->connection = std::move(connection); + d->stream = std::move(stream); + d->page_id = std::move(page_id); +} + +Gallery_WebSocket::~Gallery_WebSocket() { close(); } + +void Gallery_WebSocket::start() { + if (d->attached.exchange(true, std::memory_order_acq_rel)) return; + const auto weak = weak_from_this(); + d->video = std::make_unique( + [weak](std::string signal) { + const auto socket = weak.lock(); + if (!socket) return; + const auto connection = socket->d->connection.lock(); + if (connection && connection->connected()) + connection->send(std::move(signal), drogon::WebSocketMessageType::Text); + }, + [weak] { + if (const auto socket = weak.lock()) + socket->d->stream->request_video_key_frame(); + }, + [weak](std::exception_ptr failure) { + if (const auto socket = weak.lock()) + socket->fail_media(std::move(failure)); + }); + d->subscription = d->stream->subscribe( + [weak](Gallery_Stream_Frame frame) { + if (const auto socket = weak.lock()) socket->deliver(std::move(frame)); + }, + [weak] { + const auto socket = weak.lock(); + return socket && socket->d->attached.load(std::memory_order_acquire) && + socket->d->video && socket->d->video->can_accept_video(); + }); + if (const auto connection = d->connection.lock(); + connection && connection->connected()) + connection->send(d->stream->layout_description(), + drogon::WebSocketMessageType::Text); + d->video->start(); +} + +void Gallery_WebSocket::deliver( + Gallery_Stream_Frame frame) { + if (!d->attached.load(std::memory_order_acquire)) return; + const auto connection = d->connection.lock(); + if (!connection || !connection->connected()) return; + if (frame.video && !d->media_failed.load(std::memory_order_acquire)) { + try { + static_cast(d->video->send(std::move(*frame.video))); + } catch (...) { + fail_media(std::current_exception()); + } + } + if (!frame.metadata.empty()) { + try { + connection->send(std::move(frame.metadata), + drogon::WebSocketMessageType::Text); + } + catch (...) {} + } +} + +void Gallery_WebSocket::fail_media(std::exception_ptr failure) noexcept { + if (d->media_failed.exchange(true, std::memory_order_acq_rel)) return; + try { + const auto connection = d->connection.lock(); + if (connection && connection->connected()) + connection->send(nlohmann::json{ + {"kind", "gallery_error"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"message", media_failure_description(failure)}}.dump(), + drogon::WebSocketMessageType::Text); + } + catch (...) {} +} + +void Gallery_WebSocket::receive(std::string_view message) { + const auto json = nlohmann::json::parse(message, nullptr, false); + if (json.is_discarded() || !json.is_object()) return; + const auto kind = json.value("kind", std::string{}); + try { + if (kind == "webrtc_answer") { + const auto sdp = json.value("sdp", std::string{}); + if (!sdp.empty()) d->video->accept_answer(sdp); + } else if (kind == "webrtc_candidate") { + const auto candidate = json.value("candidate", std::string{}); + if (!candidate.empty()) + d->video->add_remote_candidate( + candidate, json.value("mid", std::string{})); + } else if (kind == "request_key_frame") { + d->stream->request_video_key_frame(); + } + } + catch (const nlohmann::json::exception&) {} + catch (...) { fail_media(std::current_exception()); } +} + +void Gallery_WebSocket::close() noexcept { + if (!d->attached.exchange(false, std::memory_order_acq_rel)) return; + try { + if (d->subscription != 0) d->stream->unsubscribe(d->subscription); + } + catch (...) {} + d->subscription = 0; + if (d->video) d->video->close(); + try { release_exclusive_page(d->page_id); } + catch (...) {} +} + +Gallery_WebSocket_Controller::Gallery_WebSocket_Controller( + Stream_Resolver value_resolver) + : resolve_stream(std::move(value_resolver)) { + if (!resolve_stream) + throw std::invalid_argument( + "gallery WebSocket requires a media stream resolver"); +} + +void Gallery_WebSocket_Controller::initPathRouting() { + drogon::app().registerWebSocketController( + "/ws/gallery", classTypeName()); +} + +void Gallery_WebSocket_Controller::handleNewConnection( + const drogon::HttpRequestPtr& request, + const drogon::WebSocketConnectionPtr& connection) { + try { + const auto stream = resolve_stream(request->getParameter("group")); + if (!stream) { + try { + connection->send(nlohmann::json{ + {"kind", "gallery_error"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"message", "unknown gallery media group"}}.dump(), + drogon::WebSocketMessageType::Text); + } + catch (...) {} + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Unknown gallery media group"); + } + catch (...) {} + return; + } + const auto page_id = request->getParameter("page"); + if (!valid_page_id(page_id) || !acquire_exclusive_page(page_id)) { + try { + connection->send(nlohmann::json{{"kind", "exclusive_page_rejected"}, + {"message", "Aethera Gallery is already owned by another page instance"}}.dump(), + drogon::WebSocketMessageType::Text); + } + catch (...) {} + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera Gallery allows one page instance"); + } + catch (...) {} + return; + } + try { + auto socket = std::make_shared( + connection, stream, page_id); + connection->setContext(socket); + connection->setPingMessage( + "aethera-gallery-video", std::chrono::seconds(20)); + socket->start(); + } + catch (...) { + if (const auto socket = connection->getContext()) + socket->close(); + else + release_exclusive_page(page_id); + try { + connection->send(nlohmann::json{ + {"kind", "gallery_error"}, + {"protocol", "aethera.gallery.video"}, + {"version", 2}, + {"message", media_failure_description( + std::current_exception())}}.dump(), + drogon::WebSocketMessageType::Text); + } + catch (...) {} + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera Gallery media setup failed"); + } + catch (...) {} + } + } + catch (...) { + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera Gallery controller failed"); + } + catch (...) {} + } +} + +void Gallery_WebSocket_Controller::handleNewMessage( + const drogon::WebSocketConnectionPtr& connection, + std::string&& message, + const drogon::WebSocketMessageType& type) { + try { + if (type != drogon::WebSocketMessageType::Text || + message.size() > 256 * 1024) return; + if (const auto socket = connection->getContext()) + socket->receive(message); + } + catch (...) {} +} + +void Gallery_WebSocket_Controller::handleConnectionClosed( + const drogon::WebSocketConnectionPtr& connection) { + try { + if (const auto socket = connection->getContext()) + socket->close(); + connection->clearContext(); + } + catch (...) {} +} +} diff --git a/web_server/src/Gallery_WebSocket.hpp b/web_server/src/Gallery_WebSocket.hpp new file mode 100644 index 0000000..802a83e --- /dev/null +++ b/web_server/src/Gallery_WebSocket.hpp @@ -0,0 +1,46 @@ +#pragma once +#include "Gallery_Video_Stream.hpp" +#include +#include +#include +#include +#include + +namespace aethera::web { +class Gallery_WebSocket final + : public std::enable_shared_from_this { +public: + Gallery_WebSocket(drogon::WebSocketConnectionPtr connection, + std::shared_ptr stream, + std::string page_id); + ~Gallery_WebSocket(); + Gallery_WebSocket(const Gallery_WebSocket&) = delete; + Gallery_WebSocket& operator=(const Gallery_WebSocket&) = delete; + void start(); + void receive(std::string_view message); + void close() noexcept; +private: + void deliver(Gallery_Stream_Frame frame); + void fail_media(std::exception_ptr failure) noexcept; + struct Private; + std::unique_ptr d; +}; + +class Gallery_WebSocket_Controller final + : public drogon::WebSocketController { +public: + using Stream_Resolver = + std::function(std::string_view)>; + explicit Gallery_WebSocket_Controller( + Stream_Resolver resolve_stream); + void handleNewMessage(const drogon::WebSocketConnectionPtr& connection, + std::string&& message, + const drogon::WebSocketMessageType& type) override; + void handleNewConnection(const drogon::HttpRequestPtr& request, + const drogon::WebSocketConnectionPtr& connection) override; + void handleConnectionClosed(const drogon::WebSocketConnectionPtr& connection) override; + static void initPathRouting(); +private: + Stream_Resolver resolve_stream; +}; +} diff --git a/web_server/src/Graph_WebSocket.cpp b/web_server/src/Graph_WebSocket.cpp index f86f943..10334c8 100644 --- a/web_server/src/Graph_WebSocket.cpp +++ b/web_server/src/Graph_WebSocket.cpp @@ -1,9 +1,5 @@ #include "Graph_WebSocket.hpp" -#include "WebRtc_Video_Session.hpp" -#include -#include -#include -#include +#include "Exclusive_Page.hpp" #include #include #include @@ -11,157 +7,66 @@ #include #include #include -#include -#include namespace aethera::web { namespace { std::string graph_id_from_path(std::string_view path) { const auto split = path.find_last_of('/'); - return split == std::string_view::npos ? std::string{} : std::string(path.substr(split + 1)); + return split == std::string_view::npos + ? std::string{} + : std::string(path.substr(split + 1)); } -struct Exclusive_Page_State { - std::mutex mutex; - std::string page; - std::size_t connection_count{}; -}; - -Exclusive_Page_State exclusive_page; - -asio::any_io_executor media_delivery_executor() { - /* rtc::Track 与 Drogon 发送可能产生媒体背压,禁止占用 Plot 编码域。 - * 每个 Graph 仍通过自己的 strand 保持 H.264/RTP 顺序。 */ - static asio::thread_pool pool([] { - const auto hardware = std::max(2U, std::thread::hardware_concurrency()); - return std::min(8U, std::max(2U, hardware / 2U)); - }()); - return pool.get_executor(); -} - -bool valid_page_id(std::string_view page) { - return page.size() >= 16 && page.size() <= 64 && - std::ranges::all_of(page, [](unsigned char value) { - return std::isalnum(value) || value == '-' || value == '_'; - }); -} - -std::uint32_t video_dimension(std::uint32_t value, std::uint32_t minimum, +std::uint32_t input_dimension(std::uint32_t value, std::uint32_t minimum, std::uint32_t maximum) { - const auto clamped = std::clamp(value, minimum, maximum); - return std::clamp(((clamped + 16U) / 32U) * 32U, minimum, maximum) & ~1U; -} - -bool acquire_page(std::string_view page) { - std::lock_guard lock(exclusive_page.mutex); - if (exclusive_page.connection_count != 0 && exclusive_page.page != page) return false; - if (exclusive_page.connection_count == 0) exclusive_page.page = page; - ++exclusive_page.connection_count; - return true; -} - -void release_page(std::string_view page) { - std::lock_guard lock(exclusive_page.mutex); - if (exclusive_page.page != page || exclusive_page.connection_count == 0) return; - --exclusive_page.connection_count; - if (exclusive_page.connection_count == 0) exclusive_page.page.clear(); + return std::clamp(value, minimum, maximum); } } struct Graph_WebSocket::Private { - Private() : delivery_strand(asio::make_strand(media_delivery_executor())) {} std::weak_ptr connection; std::shared_ptr plot; - std::unique_ptr video; Plot::Stream_Id stream{}; std::atomic_bool attached{}; std::mutex viewport_mutex; - std::uint32_t width{720}; - std::uint32_t height{420}; - std::string page_id; /* 同一网页的多图连接共享一个独占租约。 */ - asio::strand delivery_strand; /* 当前 Graph 唯一的媒体发送顺序域。 */ - std::mutex pending_mutex; - std::shared_ptr pending_frame; /* 媒体背压时只保留尚未发送的最新完成帧。 */ - bool delivery_scheduled{}; /* pending_frame 是否已有唯一消费任务。 */ + std::uint32_t width{320}; /* 当前图集槽位对应的输入坐标宽度。 */ + std::uint32_t height{192}; /* 当前图集槽位对应的输入坐标高度。 */ + std::string page_id; /* 与页面级媒体连接共用的独占租约。 */ }; Graph_WebSocket::Graph_WebSocket(drogon::WebSocketConnectionPtr connection, - std::shared_ptr plot, std::string page_id) + std::shared_ptr plot, + std::string page_id) : d(std::make_unique()) { - d->connection = connection; + d->connection = std::move(connection); d->plot = std::move(plot); d->page_id = std::move(page_id); } + Graph_WebSocket::~Graph_WebSocket() { close(); } void Graph_WebSocket::start() { if (d->attached.exchange(true, std::memory_order_acq_rel)) return; const auto weak = weak_from_this(); - d->video = std::make_unique( - [weak](std::string signal) { - const auto socket = weak.lock(); - if (!socket) return; - const auto connection = socket->d->connection.lock(); - if (connection && connection->connected()) - connection->send(std::move(signal), drogon::WebSocketMessageType::Text); - }, - [weak] { - if (const auto socket = weak.lock()) - socket->d->plot->request_video_key_frame(); - }); d->stream = d->plot->subscribe( [weak](std::shared_ptr frame) { if (const auto socket = weak.lock()) - socket->enqueue_frame(std::move(frame)); + socket->deliver_frame(std::move(frame)); }); - d->video->start(); } -void Graph_WebSocket::enqueue_frame( +void Graph_WebSocket::deliver_frame( std::shared_ptr frame) { - if (!frame || !d->attached.load(std::memory_order_acquire)) return; - bool schedule{}; - { - std::lock_guard lock(d->pending_mutex); - d->pending_frame = std::move(frame); - if (!d->delivery_scheduled) { - d->delivery_scheduled = true; - schedule = true; - } - } - if (!schedule) return; - const auto weak = weak_from_this(); - asio::post(d->delivery_strand, [weak] { - if (const auto socket = weak.lock()) socket->deliver_frame(); - }); -} - -void Graph_WebSocket::deliver_frame() { - std::shared_ptr frame; - { - std::lock_guard lock(d->pending_mutex); - frame = std::move(d->pending_frame); - } - if (frame && d->attached.load(std::memory_order_acquire)) { - const auto connection = d->connection.lock(); - if (connection && connection->connected()) { - if (frame->video) - static_cast(d->video->send(*frame->video)); + if (!frame || frame->metadata.empty() || + !d->attached.load(std::memory_order_acquire)) return; + const auto connection = d->connection.lock(); + if (connection && connection->connected()) { + try { connection->send(frame->metadata, drogon::WebSocketMessageType::Text); } + catch (...) {} } - bool schedule{}; - { - std::lock_guard lock(d->pending_mutex); - if (d->pending_frame) schedule = true; - else d->delivery_scheduled = false; - } - if (!schedule) return; - const auto weak = weak_from_this(); - asio::post(d->delivery_strand, [weak] { - if (const auto socket = weak.lock()) socket->deliver_frame(); - }); } void Graph_WebSocket::receive(std::string_view message) { @@ -169,31 +74,17 @@ void Graph_WebSocket::receive(std::string_view message) { if (json.is_discarded() || !json.is_object()) return; const auto kind = json.value("kind", std::string{}); try { - if (kind == "webrtc_answer") { - const auto sdp = json.value("sdp", std::string{}); - if (!sdp.empty()) d->video->accept_answer(sdp); - return; - } - if (kind == "webrtc_candidate") { - const auto candidate = json.value("candidate", std::string{}); - if (!candidate.empty()) - d->video->add_remote_candidate(candidate, json.value("mid", std::string{})); - return; - } if (kind == "stream") { - std::uint32_t width{720}; - std::uint32_t height{420}; + std::uint32_t width{320}; + std::uint32_t height{192}; if (const auto viewport = json.find("viewport"); viewport != json.end() && viewport->is_object()) { - width = video_dimension(viewport->value("width", 720U), 160U, 1920U); - height = video_dimension(viewport->value("height", 420U), 128U, 1080U); + width = input_dimension(viewport->value("width", 320U), 160U, 1920U); + height = input_dimension(viewport->value("height", 192U), 120U, 1080U); } - { - std::lock_guard lock(d->viewport_mutex); - d->width = width; - d->height = height; - } - d->plot->configure_stream(d->stream, width, height); + std::lock_guard lock(d->viewport_mutex); + d->width = width; + d->height = height; return; } if (kind == "manual_render") { @@ -216,14 +107,19 @@ void Graph_WebSocket::receive(std::string_view message) { } Plot_Input_Event decoded; decoded.type = *type; - const auto read_point = [&](std::string_view key, render_2d::Point_F& point, + const auto read_point = [&](std::string_view key, + render_2d::Point_F& point, bool viewport_relative) { const auto value = input->find(key); if (value == input->end() || !value->is_object()) return; const auto x = value->value("x", 0.0); const auto y = value->value("y", 0.0); - point.x = viewport_relative ? std::clamp(x, 0.0, static_cast(width)) : x; - point.y = viewport_relative ? std::clamp(y, 0.0, static_cast(height)) : y; + point.x = viewport_relative + ? std::clamp(x, 0.0, static_cast(width)) + : x; + point.y = viewport_relative + ? std::clamp(y, 0.0, static_cast(height)) + : y; }; read_point("position", decoded.position, true); read_point("global_position", decoded.global_position, false); @@ -233,65 +129,117 @@ void Graph_WebSocket::receive(std::string_view message) { std::clamp(input->value("buttons", 0), 0, 255)); decoded.modifiers = static_cast( std::clamp(input->value("modifiers", 0), 0, 15)); - decoded.pixel_delta_x = std::clamp(input->value("pixel_delta_x", 0.0), -4096.0, 4096.0); - decoded.pixel_delta_y = std::clamp(input->value("pixel_delta_y", 0.0), -4096.0, 4096.0); - decoded.angle_delta_x = std::clamp(input->value("angle_delta_x", 0.0), -120.0, 120.0); - decoded.angle_delta_y = std::clamp(input->value("angle_delta_y", 0.0), -120.0, 120.0); + decoded.pixel_delta_x = std::clamp( + input->value("pixel_delta_x", 0.0), -4096.0, 4096.0); + decoded.pixel_delta_y = std::clamp( + input->value("pixel_delta_y", 0.0), -4096.0, 4096.0); + decoded.angle_delta_x = std::clamp( + input->value("angle_delta_x", 0.0), -120.0, 120.0); + decoded.angle_delta_y = std::clamp( + input->value("angle_delta_y", 0.0), -120.0, 120.0); decoded.key = magic_enum::enum_cast( input->value("key", std::string{"unknown"})).value_or(Key::unknown); decoded.native_key = input->value("native_key", 0U); decoded.auto_repeat = input->value("auto_repeat", false); d->plot->submit_input(std::move(decoded)); - } catch (const nlohmann::json::exception&) { - return; } + catch (const nlohmann::json::exception&) {} + catch (...) {} } -void Graph_WebSocket::close() { +void Graph_WebSocket::close() noexcept { if (!d->attached.exchange(false, std::memory_order_acq_rel)) return; - d->plot->unsubscribe(d->stream); - { - std::lock_guard lock(d->pending_mutex); - d->pending_frame.reset(); - } - if (d->video) d->video->close(); - release_page(d->page_id); + try { d->plot->unsubscribe(d->stream); } + catch (...) {} + try { release_exclusive_page(d->page_id); } + catch (...) {} } Graph_WebSocket_Controller::Graph_WebSocket_Controller(Plot_Resolver resolver) : resolve_plot(std::move(resolver)) {} + +void Graph_WebSocket_Controller::initPathRouting() { + drogon::app().registerWebSocketControllerRegex( + "^/ws/plot/[^/]+$", classTypeName()); +} + void Graph_WebSocket_Controller::handleNewConnection( const drogon::HttpRequestPtr& request, const drogon::WebSocketConnectionPtr& connection) { + try { auto plot = resolve_plot(graph_id_from_path(request->path())); if (!plot) { - connection->shutdown(drogon::CloseCode::kViolation, "Unknown Aethera plot"); + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Unknown Aethera plot"); + } + catch (...) {} return; } const auto page_id = request->getParameter("page"); - if (!valid_page_id(page_id) || !acquire_page(page_id)) { - connection->send(nlohmann::json{{"kind", "exclusive_page_rejected"}, - {"message", "Aethera Gallery is already owned by another page instance"}}.dump(), - drogon::WebSocketMessageType::Text); - connection->shutdown(drogon::CloseCode::kViolation, - "Aethera Gallery allows one page instance"); + if (!valid_page_id(page_id) || !acquire_exclusive_page(page_id)) { + try { + connection->send(nlohmann::json{{"kind", "exclusive_page_rejected"}, + {"message", "Aethera Gallery is already owned by another page instance"}}.dump(), + drogon::WebSocketMessageType::Text); + } + catch (...) {} + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera Gallery allows one page instance"); + } + catch (...) {} return; } - auto socket = std::make_shared( - connection, std::move(plot), page_id); - connection->setContext(socket); - connection->setPingMessage("aethera-gallery", std::chrono::seconds(20)); - socket->start(); + try { + auto socket = std::make_shared( + connection, std::move(plot), page_id); + connection->setContext(socket); + connection->setPingMessage( + "aethera-gallery-plot", std::chrono::seconds(20)); + socket->start(); + } + catch (...) { + if (const auto socket = connection->getContext()) + socket->close(); + else + release_exclusive_page(page_id); + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera plot setup failed"); + } + catch (...) {} + } + } + catch (...) { + try { + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera plot controller failed"); + } + catch (...) {} + } } + void Graph_WebSocket_Controller::handleNewMessage( - const drogon::WebSocketConnectionPtr& connection, std::string&& message, + const drogon::WebSocketConnectionPtr& connection, + std::string&& message, const drogon::WebSocketMessageType& type) { - if (type != drogon::WebSocketMessageType::Text || message.size() > 64 * 1024) return; - if (const auto socket = connection->getContext()) socket->receive(message); + try { + if (type != drogon::WebSocketMessageType::Text || + message.size() > 64 * 1024) return; + if (const auto socket = connection->getContext()) + socket->receive(message); + } + catch (...) {} } + void Graph_WebSocket_Controller::handleConnectionClosed( const drogon::WebSocketConnectionPtr& connection) { - if (const auto socket = connection->getContext()) socket->close(); - connection->clearContext(); + try { + if (const auto socket = connection->getContext()) + socket->close(); + connection->clearContext(); + } + catch (...) {} } } diff --git a/web_server/src/Graph_WebSocket.hpp b/web_server/src/Graph_WebSocket.hpp index 12f7edf..c5ee33e 100644 --- a/web_server/src/Graph_WebSocket.hpp +++ b/web_server/src/Graph_WebSocket.hpp @@ -16,10 +16,9 @@ public: Graph_WebSocket& operator=(const Graph_WebSocket&) = delete; void start(); void receive(std::string_view message); - void close(); + void close() noexcept; private: - void enqueue_frame(std::shared_ptr frame); - void deliver_frame(); + void deliver_frame(std::shared_ptr frame); struct Private; std::unique_ptr d; }; @@ -35,9 +34,7 @@ public: void handleNewConnection(const drogon::HttpRequestPtr& request, const drogon::WebSocketConnectionPtr& connection) override; void handleConnectionClosed(const drogon::WebSocketConnectionPtr& connection) override; - WS_PATH_LIST_BEGIN - WS_ADD_PATH_VIA_REGEX("^/ws/plot/[^/]+$"); - WS_PATH_LIST_END + static void initPathRouting(); private: Plot_Resolver resolve_plot; }; diff --git a/web_server/src/H264_Encoder.cpp b/web_server/src/H264_Encoder.cpp index d16484d..131b358 100644 --- a/web_server/src/H264_Encoder.cpp +++ b/web_server/src/H264_Encoder.cpp @@ -2,100 +2,416 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include +#include extern "C" { #include +#include #include +#include #include #include } namespace aethera::web { namespace { +constexpr std::string_view high_profile_level_5_1{"640033"}; + std::runtime_error ffmpeg_failure(std::string operation, int code) { std::array description{}; av_strerror(code, description.data(), description.size()); return std::runtime_error(std::move(operation) + ": " + description.data()); } + void require_ffmpeg(int result, std::string operation) { if (result < 0) throw ffmpeg_failure(std::move(operation), result); } + +std::string failure_description(const std::exception_ptr& failure) { + try { + if (failure) std::rethrow_exception(failure); + } + catch (const std::exception& error) { + return error.what(); + } + catch (...) { + return "non-standard encoder failure"; + } + return "empty encoder failure"; +} + +AVPixelFormat input_pixel_format(Video_Pixel_Layout layout) noexcept { + return layout == Video_Pixel_Layout::bgra ? AV_PIX_FMT_BGRA + : AV_PIX_FMT_RGBA; +} + +struct Codec_Context_Deleter { + void operator()(AVCodecContext* value) const noexcept { + avcodec_free_context(&value); + } +}; + +struct Frame_Deleter { + void operator()(AVFrame* value) const noexcept { + av_frame_free(&value); + } +}; + +struct Packet_Deleter { + void operator()(AVPacket* value) const noexcept { + av_packet_free(&value); + } +}; + +struct Buffer_Deleter { + void operator()(AVBufferRef* value) const noexcept { + av_buffer_unref(&value); + } +}; + +struct Sws_Context_Deleter { + void operator()(SwsContext* value) const noexcept { + sws_freeContext(value); + } +}; + +using Codec_Context_Owner = + std::unique_ptr; +using Frame_Owner = std::unique_ptr; +using Packet_Owner = std::unique_ptr; +using Buffer_Owner = std::unique_ptr; +using Sws_Context_Owner = std::unique_ptr; +} + +std::string_view video_encoder_backend_name( + Video_Encoder_Backend backend) noexcept { + switch (backend) { + case Video_Encoder_Backend::nvenc: return "nvenc"; + case Video_Encoder_Backend::vulkan_video: return "vulkan_video"; + } + return "unknown"; +} + +std::string_view h264_profile_level_id() noexcept { + return high_profile_level_5_1; } struct H264_Encoder::Private { - double frame_rate{}; /* 编码时间基与 GOP 计算使用的服务端帧时钟频率。 */ - AVCodecContext* codec_context{}; /* 当前尺寸对应的 libx264 低延迟编码上下文。 */ - AVFrame* yuv_frame{}; /* swscale 写入并提交给编码器的复用 YUV420P 帧。 */ + struct Submitted_Frame { + std::uint64_t sequence{}; /* 编码器延迟期间保留的输入业务身份。 */ + std::chrono::microseconds presentation_time{}; /* 与 AVFrame::pts 一一对应的媒体时间。 */ + }; + + struct Configuration { + Codec_Context_Owner codec_context{}; /* 尚未发布的完整编码上下文。 */ + Frame_Owner software_frame{}; /* NVENC 原生 RGB 或 Vulkan 的 NV12 暂存帧。 */ + Packet_Owner packet{}; /* 尚未发布的复用输出包。 */ + Buffer_Owner hardware_device{}; /* Vulkan Video 独占的 FFmpeg Device。 */ + Buffer_Owner hardware_frames{}; /* Vulkan NV12 帧池。 */ + Sws_Context_Owner converter{}; /* 仅 Vulkan 路径的 RGB 到 NV12 转换器。 */ + Video_Encoder_Backend backend{Video_Encoder_Backend::nvenc}; + }; + + double frame_rate{}; /* 页面媒体时钟频率,也是 GOP 的计算基准。 */ + AVCodecContext* codec_context{}; /* 当前图集尺寸对应的唯一硬件编码上下文。 */ + AVFrame* software_frame{}; /* 当前硬件路径需要的复用 CPU 输入帧。 */ AVPacket* packet{}; /* 接收单个 H.264 access unit 的复用包。 */ - SwsContext* scaler{}; /* RGBA/BGRA 到 YUV420P 的像素转换上下文。 */ - std::uint32_t width{}; /* 当前编码尺寸;尺寸变化时整体重建编码器。 */ - std::uint32_t height{}; /* 当前编码尺寸;H.264 4:2:0 要求偶数。 */ - Video_Pixel_Layout layout{Video_Pixel_Layout::bgra}; /* scaler 当前输入格式。 */ - bool key_frame_requested{true}; /* Track 新建或恢复时,下一 access unit 必须可独立解码。 */ + AVBufferRef* hardware_device{}; /* Vulkan Video 后端的 FFmpeg Device。 */ + AVBufferRef* hardware_frames{}; /* Vulkan Video NV12 帧池。 */ + SwsContext* converter{}; /* Vulkan Video 的 RGBA/BGRA 到 NV12 转换器。 */ + std::optional backend{}; /* 当前成功配置的唯一实际编码后端。 */ + std::deque submitted_frames{}; /* 延迟输出 access unit 的身份队列。 */ + std::uint32_t width{}; /* 当前硬件编码图集宽度。 */ + std::uint32_t height{}; /* 当前硬件编码图集高度。 */ + Video_Pixel_Layout layout{Video_Pixel_Layout::bgra}; /* 当前调用方提交的原生像素排列。 */ + bool key_frame_requested{true}; /* 新 Track 建立后强制下一次提交为 IDR。 */ explicit Private(double value_frame_rate) : frame_rate(value_frame_rate) { if (!std::isfinite(frame_rate) || frame_rate <= 0.0) throw std::invalid_argument("H.264 frame rate must be positive"); } + ~Private() { reset(); } + void reset() noexcept { - sws_freeContext(scaler); - scaler = nullptr; + submitted_frames.clear(); av_packet_free(&packet); - av_frame_free(&yuv_frame); + av_frame_free(&software_frame); + sws_freeContext(converter); + converter = nullptr; avcodec_free_context(&codec_context); + av_buffer_unref(&hardware_frames); + av_buffer_unref(&hardware_device); + backend.reset(); width = 0; height = 0; key_frame_requested = true; } - void configure(std::uint32_t next_width, std::uint32_t next_height, - Video_Pixel_Layout next_layout) { - if ((next_width & 1U) != 0 || (next_height & 1U) != 0) - throw std::invalid_argument("H.264 YUV420P dimensions must be even"); - if (codec_context && width == next_width && height == next_height && - layout == next_layout) return; + + void configure_common(AVCodecContext& context, + std::uint32_t next_width, + std::uint32_t next_height, + AVPixelFormat pixel_format) const { + context.width = static_cast(next_width); + context.height = static_cast(next_height); + context.pix_fmt = pixel_format; + context.time_base = AVRational{1, 1'000'000}; + context.framerate = av_d2q(frame_rate, 1000); + context.bit_rate = std::clamp( + static_cast(next_width) * next_height * 15, + 4'000'000, 50'000'000); + context.rc_max_rate = context.bit_rate; + context.rc_buffer_size = std::max( + context.bit_rate / + static_cast(std::lround(frame_rate)), + 64'000); + context.gop_size = std::max( + 1, static_cast(std::lround(frame_rate * 2.0))); + context.max_b_frames = 0; + context.profile = AV_PROFILE_H264_HIGH; + context.level = 51; + context.color_range = AVCOL_RANGE_MPEG; + context.color_primaries = AVCOL_PRI_BT709; + context.color_trc = AVCOL_TRC_BT709; + context.colorspace = AVCOL_SPC_BT709; + context.flags |= AV_CODEC_FLAG_LOW_DELAY; + } + + Frame_Owner make_software_frame(AVPixelFormat format, + std::uint32_t next_width, + std::uint32_t next_height, + std::string_view operation) const { + Frame_Owner frame{av_frame_alloc()}; + if (!frame) throw std::bad_alloc{}; + frame->format = format; + frame->width = static_cast(next_width); + frame->height = static_cast(next_height); + frame->duration = static_cast( + std::llround(1'000'000.0 / frame_rate)); + require_ffmpeg(av_frame_get_buffer(frame.get(), 32), + std::string(operation)); + return frame; + } + + Configuration configure_nvenc(std::uint32_t next_width, + std::uint32_t next_height, + Video_Pixel_Layout next_layout) const { + const AVCodec* codec = avcodec_find_encoder_by_name("h264_nvenc"); + if (!codec) + throw std::runtime_error("FFmpeg h264_nvenc encoder is unavailable"); + Configuration result; + result.backend = Video_Encoder_Backend::nvenc; + result.codec_context.reset(avcodec_alloc_context3(codec)); + if (!result.codec_context) throw std::bad_alloc{}; + configure_common(*result.codec_context, next_width, next_height, + input_pixel_format(next_layout)); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "preset", "p1", 0), + "setting NVENC preset"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "tune", "ull", 0), + "setting NVENC ultra-low-latency tune"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "profile", "high", 0), + "setting NVENC H.264 profile"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "level", "5.1", 0), + "setting NVENC H.264 level"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "rc", "cbr", 0), + "setting NVENC rate control"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "delay", 0, 0), + "disabling NVENC output delay"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "rc-lookahead", 0, 0), + "disabling NVENC lookahead"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "zerolatency", 1, 0), + "enabling NVENC zero latency"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "forced-idr", 1, 0), + "enabling forced NVENC IDR frames"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "no-scenecut", 1, 0), + "disabling NVENC scene-cut frames"); + require_ffmpeg(avcodec_open2(result.codec_context.get(), codec, nullptr), + "opening h264_nvenc encoder"); + result.software_frame = make_software_frame( + result.codec_context->pix_fmt, next_width, next_height, + "allocating the NVENC source frame"); + result.packet.reset(av_packet_alloc()); + if (!result.packet) throw std::bad_alloc{}; + return result; + } + + Configuration configure_vulkan(std::uint32_t next_width, + std::uint32_t next_height, + Video_Pixel_Layout next_layout) const { + const AVCodec* codec = avcodec_find_encoder_by_name("h264_vulkan"); + if (!codec) + throw std::runtime_error("FFmpeg h264_vulkan encoder is unavailable"); + Configuration result; + result.backend = Video_Encoder_Backend::vulkan_video; + AVBufferRef* device{}; + require_ffmpeg(av_hwdevice_ctx_create( + &device, AV_HWDEVICE_TYPE_VULKAN, + nullptr, nullptr, 0), + "creating the FFmpeg Vulkan Video device"); + result.hardware_device.reset(device); + result.hardware_frames.reset( + av_hwframe_ctx_alloc(result.hardware_device.get())); + if (!result.hardware_frames) throw std::bad_alloc{}; + auto* frames = reinterpret_cast( + result.hardware_frames->data); + frames->format = AV_PIX_FMT_VULKAN; + frames->sw_format = AV_PIX_FMT_NV12; + frames->width = static_cast(next_width); + frames->height = static_cast(next_height); + frames->initial_pool_size = 8; + require_ffmpeg(av_hwframe_ctx_init(result.hardware_frames.get()), + "creating the Vulkan Video NV12 frame pool"); + + result.codec_context.reset(avcodec_alloc_context3(codec)); + if (!result.codec_context) throw std::bad_alloc{}; + configure_common(*result.codec_context, next_width, next_height, + AV_PIX_FMT_VULKAN); + result.codec_context->hw_frames_ctx = + av_buffer_ref(result.hardware_frames.get()); + if (!result.codec_context->hw_frames_ctx) throw std::bad_alloc{}; + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "profile", AV_PROFILE_H264_HIGH, 0), + "setting Vulkan Video H.264 profile"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "level", 51, 0), + "setting Vulkan Video H.264 level"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "usage", "stream", 0), + "setting Vulkan Video streaming usage"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "content", "rendered", 0), + "setting Vulkan Video rendered content"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "tune", "ull", 0), + "setting Vulkan Video ultra-low latency"); + require_ffmpeg(av_opt_set(result.codec_context->priv_data, + "rc_mode", "cbr", 0), + "setting Vulkan Video rate control"); + require_ffmpeg(av_opt_set_int(result.codec_context->priv_data, + "async_depth", 3, 0), + "setting Vulkan Video pipeline depth"); + require_ffmpeg(avcodec_open2(result.codec_context.get(), codec, nullptr), + "opening h264_vulkan encoder"); + + result.software_frame = make_software_frame( + AV_PIX_FMT_NV12, next_width, next_height, + "allocating the Vulkan Video NV12 staging frame"); + result.converter.reset(sws_getContext( + static_cast(next_width), static_cast(next_height), + input_pixel_format(next_layout), + static_cast(next_width), static_cast(next_height), + AV_PIX_FMT_NV12, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr)); + if (!result.converter) + throw std::runtime_error( + "failed to create the Vulkan Video RGB to NV12 converter"); + result.packet.reset(av_packet_alloc()); + if (!result.packet) throw std::bad_alloc{}; + return result; + } + + void adopt(Configuration configuration, std::uint32_t next_width, + std::uint32_t next_height, + Video_Pixel_Layout next_layout) { + /* 配置是一次事务:只有一种硬件路径的所有资源都可用后,才替换 + * 当前编码器。失败的候选配置完全由局部 RAII 对象回收。 */ reset(); - const AVCodec* codec = avcodec_find_encoder_by_name("libx264"); - if (!codec) throw std::runtime_error("FFmpeg libx264 encoder is unavailable"); - codec_context = avcodec_alloc_context3(codec); - if (!codec_context) throw std::bad_alloc{}; + codec_context = configuration.codec_context.release(); + software_frame = configuration.software_frame.release(); + packet = configuration.packet.release(); + hardware_device = configuration.hardware_device.release(); + hardware_frames = configuration.hardware_frames.release(); + converter = configuration.converter.release(); + backend = configuration.backend; width = next_width; height = next_height; layout = next_layout; - codec_context->width = static_cast(width); - codec_context->height = static_cast(height); - codec_context->pix_fmt = AV_PIX_FMT_YUV420P; - codec_context->time_base = AVRational{1, 1'000'000}; - codec_context->framerate = av_d2q(frame_rate, 1000); - codec_context->bit_rate = std::clamp( - static_cast(width) * height * 8, 1'000'000, 24'000'000); - codec_context->gop_size = std::max(1, static_cast(std::lround(frame_rate * 2.0))); - codec_context->max_b_frames = 0; - codec_context->thread_count = 1; - codec_context->flags |= AV_CODEC_FLAG_LOW_DELAY; - require_ffmpeg(av_opt_set(codec_context->priv_data, "preset", "ultrafast", 0), - "setting x264 preset"); - require_ffmpeg(av_opt_set(codec_context->priv_data, "tune", "zerolatency", 0), - "setting x264 tune"); - require_ffmpeg(av_opt_set(codec_context->priv_data, "profile", "baseline", 0), - "setting H.264 profile"); - require_ffmpeg(av_opt_set(codec_context->priv_data, "x264-params", - "annexb=1:repeat-headers=1:scenecut=0", 0), "setting x264 RTP parameters"); - require_ffmpeg(avcodec_open2(codec_context, codec, nullptr), "opening libx264 encoder"); - yuv_frame = av_frame_alloc(); - packet = av_packet_alloc(); - if (!yuv_frame || !packet) throw std::bad_alloc{}; - yuv_frame->format = codec_context->pix_fmt; - yuv_frame->width = codec_context->width; - yuv_frame->height = codec_context->height; - require_ffmpeg(av_frame_get_buffer(yuv_frame, 32), "allocating H.264 YUV frame"); - const AVPixelFormat source_format = layout == Video_Pixel_Layout::bgra - ? AV_PIX_FMT_BGRA : AV_PIX_FMT_RGBA; - scaler = sws_getContext(codec_context->width, codec_context->height, - source_format, codec_context->width, codec_context->height, - codec_context->pix_fmt, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); - if (!scaler) throw std::runtime_error("failed to create FFmpeg pixel converter"); + } + + void configure(std::uint32_t next_width, std::uint32_t next_height, + Video_Pixel_Layout next_layout) { + if (next_width == 0 || next_height == 0) + throw std::invalid_argument("H.264 dimensions must be non-zero"); + if ((next_width & 1U) != 0 || (next_height & 1U) != 0) + throw std::invalid_argument("H.264 dimensions must be even"); + if (codec_context && software_frame && packet && backend && + width == next_width && height == next_height && + layout == next_layout) return; + + std::exception_ptr nvenc_failure; + try { + adopt(configure_nvenc(next_width, next_height, next_layout), + next_width, next_height, next_layout); + return; + } + catch (...) { + nvenc_failure = std::current_exception(); + } + try { + adopt(configure_vulkan(next_width, next_height, next_layout), + next_width, next_height, next_layout); + } + catch (...) { + throw std::runtime_error( + "no compatible hardware H.264 encoder: NVENC: " + + failure_description(nvenc_failure) + "; Vulkan Video: " + + failure_description(std::current_exception())); + } + } + + void fill_software_frame(std::span pixels) { + require_ffmpeg(av_frame_make_writable(software_frame), + "making the hardware encoder source frame writable"); + const auto row_bytes = static_cast(width) * 4U; + if (backend == Video_Encoder_Backend::nvenc) { + for (std::uint32_t row = 0; row < height; ++row) + std::memcpy( + software_frame->data[0] + + static_cast(row) * + software_frame->linesize[0], + pixels.data() + static_cast(row) * row_bytes, + row_bytes); + return; + } + const std::array source_data{ + reinterpret_cast(pixels.data()), + nullptr, nullptr, nullptr}; + const std::array source_stride{ + static_cast(row_bytes), 0, 0, 0}; + const int converted = sws_scale( + converter, source_data.data(), source_stride.data(), 0, + static_cast(height), software_frame->data, + software_frame->linesize); + if (converted != static_cast(height)) + throw std::runtime_error( + "Vulkan Video RGB to NV12 conversion produced an incomplete frame"); + } + + Frame_Owner make_vulkan_frame() const { + Frame_Owner frame{av_frame_alloc()}; + if (!frame) throw std::bad_alloc{}; + require_ffmpeg(av_hwframe_get_buffer(hardware_frames, frame.get(), 0), + "acquiring a Vulkan Video frame"); + require_ffmpeg(av_hwframe_transfer_data( + frame.get(), software_frame, 0), + "uploading the NV12 frame to Vulkan Video"); + return frame; } }; @@ -107,37 +423,72 @@ void H264_Encoder::request_key_frame() { d->key_frame_requested = true; } -std::shared_ptr H264_Encoder::encode( +std::optional H264_Encoder::encode( std::span pixels, std::uint32_t width, std::uint32_t height, Video_Pixel_Layout layout, std::uint64_t sequence, std::chrono::microseconds presentation_time) { + if (width == 0 || height == 0 || (width & 1U) != 0 || + (height & 1U) != 0 || + static_cast(width) > + std::numeric_limits::max() / + static_cast(height) / 4U) + throw std::invalid_argument("H.264 source dimensions are invalid"); const std::size_t required = static_cast(width) * height * 4U; if (pixels.size() != required) - throw std::invalid_argument("H.264 source frame byte size does not match its dimensions"); + throw std::invalid_argument( + "H.264 source frame byte size does not match its dimensions"); d->configure(width, height, layout); - require_ffmpeg(av_frame_make_writable(d->yuv_frame), "making H.264 frame writable"); - const std::uint8_t* source[]{reinterpret_cast(pixels.data())}; - const int source_stride[]{static_cast(width * 4U)}; - if (sws_scale(d->scaler, source, source_stride, 0, static_cast(height), - d->yuv_frame->data, d->yuv_frame->linesize) != static_cast(height)) - throw std::runtime_error("FFmpeg did not convert the complete video frame"); - d->yuv_frame->pts = presentation_time.count(); - d->yuv_frame->pict_type = d->key_frame_requested - ? AV_PICTURE_TYPE_I - : AV_PICTURE_TYPE_NONE; - require_ffmpeg(avcodec_send_frame(d->codec_context, d->yuv_frame), - "submitting frame to H.264 encoder"); - const int received = avcodec_receive_packet(d->codec_context, d->packet); - if (received == AVERROR(EAGAIN)) return {}; - require_ffmpeg(received, "receiving H.264 access unit"); - auto output = std::make_shared(); - output->annex_b.assign(reinterpret_cast(d->packet->data), - reinterpret_cast(d->packet->data + d->packet->size)); - output->presentation_time = presentation_time; - output->sequence = sequence; - output->key_frame = (d->packet->flags & AV_PKT_FLAG_KEY) != 0; + d->fill_software_frame(pixels); + + Frame_Owner hardware_frame; + AVFrame* input = d->software_frame; + if (d->backend == Video_Encoder_Backend::vulkan_video) { + hardware_frame = d->make_vulkan_frame(); + input = hardware_frame.get(); + } + input->pts = presentation_time.count(); + input->duration = static_cast( + std::llround(1'000'000.0 / d->frame_rate)); + input->pict_type = d->key_frame_requested + ? AV_PICTURE_TYPE_I + : AV_PICTURE_TYPE_NONE; + + d->submitted_frames.push_back({sequence, presentation_time}); + const int submitted = avcodec_send_frame(d->codec_context, input); + if (submitted < 0) { + d->submitted_frames.pop_back(); + throw ffmpeg_failure("submitting a frame to the hardware encoder", + submitted); + } d->key_frame_requested = false; av_packet_unref(d->packet); + const int received = avcodec_receive_packet(d->codec_context, d->packet); + if (received == AVERROR(EAGAIN)) return {}; + require_ffmpeg(received, + "receiving a H.264 access unit from the hardware encoder"); + + struct Packet_Unref_Scope { + AVPacket* packet{}; /* 输出复制完成或抛出后统一归还复用包。 */ + ~Packet_Unref_Scope() { av_packet_unref(packet); } + } packet_scope{d->packet}; + + if (d->submitted_frames.empty()) + throw std::logic_error("hardware encoder produced an uncorrelated frame"); + const auto submitted_frame = d->submitted_frames.front(); + if (d->packet->pts != AV_NOPTS_VALUE && + d->packet->pts != submitted_frame.presentation_time.count()) + throw std::logic_error( + "hardware encoder reordered a frame despite zero-reorder policy"); + d->submitted_frames.pop_front(); + + Encoded_Video_Frame output; + output.annex_b.assign( + reinterpret_cast(d->packet->data), + reinterpret_cast(d->packet->data + d->packet->size)); + output.presentation_time = submitted_frame.presentation_time; + output.sequence = submitted_frame.sequence; + output.backend = *d->backend; + output.key_frame = (d->packet->flags & AV_PKT_FLAG_KEY) != 0; return output; } } diff --git a/web_server/src/H264_Encoder.hpp b/web_server/src/H264_Encoder.hpp index 9f5b7a7..b45041c 100644 --- a/web_server/src/H264_Encoder.hpp +++ b/web_server/src/H264_Encoder.hpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include namespace aethera::web { @@ -12,10 +14,20 @@ enum class Video_Pixel_Layout : std::uint8_t { rgba }; +enum class Video_Encoder_Backend : std::uint8_t { + nvenc, + vulkan_video +}; + +[[nodiscard]] std::string_view video_encoder_backend_name( + Video_Encoder_Backend backend) noexcept; +[[nodiscard]] std::string_view h264_profile_level_id() noexcept; + struct Encoded_Video_Frame { std::vector annex_b{}; /* 带起始码的完整 H.264 access unit,可直接交给 RTP packetizer。 */ std::chrono::microseconds presentation_time{}; /* 从 Plot 帧时钟起点累计的媒体时间戳。 */ std::uint64_t sequence{}; /* 对应 Render_Frame 的单调序号。 */ + Video_Encoder_Backend backend{Video_Encoder_Backend::nvenc}; /* 实际产生本 access unit 的硬件编码后端。 */ bool key_frame{}; /* 本 access unit 是否可独立解码。 */ }; @@ -26,7 +38,7 @@ public: H264_Encoder(const H264_Encoder&) = delete; H264_Encoder& operator=(const H264_Encoder&) = delete; void request_key_frame(); - [[nodiscard]] std::shared_ptr encode( + [[nodiscard]] std::optional encode( std::span pixels, std::uint32_t width, std::uint32_t height, Video_Pixel_Layout layout, std::uint64_t sequence, std::chrono::microseconds presentation_time); diff --git a/web_server/src/Plot.cpp b/web_server/src/Plot.cpp index 8672b54..26dc1d0 100644 --- a/web_server/src/Plot.cpp +++ b/web_server/src/Plot.cpp @@ -1,26 +1,23 @@ #include "Plot.hpp" #include "Renderable_Adapter.hpp" -#include -#include -#include #include -#include -#include #include -#include -#include #include +#include #include #include #include +#include #include #include #include +#include +#include +#include #include #include #include #include -#include #include #include #include @@ -32,17 +29,20 @@ using namespace render_2d; using namespace render_3d; using Scene_2D = Impl; using Scene_3D = Impl; -constexpr std::uint16_t frame_protocol_version{7}; +constexpr std::uint16_t frame_protocol_version{9}; +constexpr std::uint64_t diagnostic_sample_period{100}; -asio::any_io_executor video_encoding_executor() { - /* 编码与 Plot 帧时钟/2D 绘制是不同的 CPU 资源域。每个 Plot 的 strand - * 仍保证单个 FFmpeg 上下文串行,独立池只让不同 Plot 并行编码,避免 - * 多路视频把图调度和属性命令一起堵在 Web graph pool 中。 */ - static asio::thread_pool pool([] { - const auto hardware = std::max(2U, std::thread::hardware_concurrency()); - return std::min(8U, std::max(2U, hardware / 2U)); - }()); - return pool.get_executor(); +std::string exception_description(const std::exception_ptr& failure) { + try { + if (failure) std::rethrow_exception(failure); + } + catch (const std::exception& error) { + return error.what(); + } + catch (...) { + return "non-standard Plot failure"; + } + return "empty Plot failure"; } enum class Frame_Pacing_Mode : std::uint8_t { @@ -54,8 +54,8 @@ enum class Frame_Pacing_Mode : std::uint8_t { struct Frame_Pacing_Properties { bool render_enabled{true}; bool video_enabled{true}; - Frame_Pacing_Mode mode{Frame_Pacing_Mode::fixed_rate}; /* 唯一职责是决定何时调用 Scene::render。 */ - double fixed_rate_fps{100.0}; /* 不参与帧完成、编码或网络回调。 */ + Frame_Pacing_Mode mode{Frame_Pacing_Mode::fixed_rate}; /* 只决定共享时钟到达时是否调用 Scene::render。 */ + double fixed_rate_fps{100.0}; /* 页面级时钟当前上限为每秒一百次。 */ }; class Frame_Policy final { @@ -88,31 +88,50 @@ std::optional parse_pacing_mode(std::string_view value) { return std::nullopt; } +std::string_view pixel_format_name(render_2d::Pixel_Format format) { + switch (format) { + case render_2d::Pixel_Format::bgra8_premultiplied: + return "bgra8_premultiplied"; + case render_2d::Pixel_Format::rgba8: + return "rgba8"; + } + throw std::logic_error("unknown 2D pixel format"); +} + +std::string_view pixel_format_name(render_3d::Pixel_Format format) { + switch (format) { + case render_3d::Pixel_Format::rgba8_unorm: + return "rgba8"; + } + throw std::logic_error("unknown 3D pixel format"); +} + nlohmann::json Frame_Policy::schema() const { std::lock_guard lock(mutex); return { - {"id", "frame-analysis"}, {"label", "渲染与传输分析"}, {"kind", "analysis"}, + {"id", "frame-analysis"}, {"label", "渲染与媒体流水线"}, {"kind", "analysis"}, {"fields", nlohmann::json::array({ {{"key", "render_enabled"}, {"label", "持续渲染与采样"}, {"editor", "boolean"}, - {"editable", true}, {"description", "控制后端是否继续调用 Scene::render;浏览器画面隐藏不会修改此项。"}, + {"editable", true}, {"description", "控制共享帧时钟是否继续调用当前 Scene;画面隐藏不会修改此项。"}, {"technical_description", "Authoritative server-side render and sampling switch."}, {"value", pacing.render_enabled}}, - {{"key", "video_enabled"}, {"label", "WebRTC 视频传输"}, {"editor", "boolean"}, - {"editable", true}, {"description", "控制完成帧是否编码为 H.264 并通过 WebRTC 发送;默认开启以进行完整链路压测。"}, - {"technical_description", "Authoritative server-side video encoding and delivery switch."}, + {{"key", "video_enabled"}, {"label", "图集视频传输"}, {"editor", "boolean"}, + {"editable", true}, {"description", "控制完成帧是否进入页面级 RGBA 图集;默认开启,用于完整链路压测。"}, + {"technical_description", "Authoritative tile publication switch for the shared gallery video."}, {"value", pacing.video_enabled}}, {{"key", "pacing_mode"}, {"label", "服务端帧策略"}, {"editor", "select"}, - {"editable", true}, {"description", "只控制服务端调用 Scene::render 的节奏;完成回调始终直接发布。"}, - {"technical_description", "Server render clock policy; independent from completion and WebRTC delivery."}, + {"editable", true}, {"description", "只控制 Scene::render 的调用节奏;完成回调只负责归还帧并发布结果。"}, + {"technical_description", "Render policy driven by the common 100 Hz gallery clock."}, {"value", pacing_mode_name(pacing.mode)}, {"options", nlohmann::json::array({ {{"value", "manual"}, {"label", "手动渲染"}}, {{"value", "fixed_rate"}, {"label", "固定频率"}}, - {{"value", "maximum_rate"}, {"label", "容量允许即渲染"}} + {{"value", "maximum_rate"}, {"label", "共享时钟满速"}} })}}, {{"key", "fixed_rate_fps"}, {"label", "目标帧率"}, {"editor", "number"}, - {"editable", true}, {"description", "固定频率模式下服务端每秒触发 render 的次数。"}, - {"technical_description", "Server-side Scene::render calls per second in fixed_rate mode."}, + {"editable", true}, {"minimum", 0.1}, {"maximum", 100.0}, {"step", 0.1}, + {"description", "固定频率模式下从页面级 100 Hz 时钟采样当前 Scene 的次数。"}, + {"technical_description", "Per-plot rate selected from the common gallery render timeline."}, {"value", pacing.fixed_rate_fps}} })} }; @@ -130,18 +149,21 @@ nlohmann::json Frame_Policy::write_prop(std::string_view key, {"value", target}}; } if (key == "pacing_mode") { - if (!value.is_string()) return {{"success", false}, {"error", "pacing_mode requires a string"}}; + if (!value.is_string()) + return {{"success", false}, {"error", "pacing_mode requires a string"}}; const auto parsed = parse_pacing_mode(value.get_ref()); - if (!parsed) return {{"success", false}, {"error", "unknown frame pacing mode"}}; + if (!parsed) + return {{"success", false}, {"error", "unknown frame pacing mode"}}; pacing.mode = *parsed; return {{"success", true}, {"component", "frame-analysis"}, {"key", key}, {"value", pacing_mode_name(pacing.mode)}}; } if (key == "fixed_rate_fps") { - if (!value.is_number()) return {{"success", false}, {"error", "fixed_rate_fps requires a number"}}; + if (!value.is_number()) + return {{"success", false}, {"error", "fixed_rate_fps requires a number"}}; const double next = value.get(); - if (!std::isfinite(next) || next < 0.1 || next > 240.0) - return {{"success", false}, {"error", "fixed_rate_fps must be between 0.1 and 240"}}; + if (!std::isfinite(next) || next < 0.1 || next > 100.0) + return {{"success", false}, {"error", "fixed_rate_fps must be between 0.1 and 100"}}; pacing.fixed_rate_fps = next; return {{"success", true}, {"component", "frame-analysis"}, {"key", key}, {"value", pacing.fixed_rate_fps}}; @@ -150,7 +172,11 @@ nlohmann::json Frame_Policy::write_prop(std::string_view key, } nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, - std::uint32_t height, std::size_t encoded_bytes, + std::uint32_t height, std::size_t pixel_bytes, + Frame_Identity rendered_identity, + std::string_view output_format, + std::string_view native_format, + nlohmann::json supported_formats, const Frame_Pacing_Properties& pacing) { nlohmann::json markers = nlohmann::json::object(); for (const auto& point : frame.trace_points()) @@ -163,13 +189,16 @@ nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, {"kind", "frame_metadata"}, {"protocol", "aethera.video.frame"}, {"version", frame_protocol_version}, {"sequence", identity.sequence}, {"correlation_id", identity.correlation_id}, + {"rendered_sequence", rendered_identity.sequence}, + {"rendered_correlation_id", rendered_identity.correlation_id}, {"created_time_unix_ms", static_cast(frame.created_time_unix_ns()) / 1'000'000.0}, - {"delivery", encoded_bytes == 0 ? "diagnostics" : "webrtc-video"}, - {"pixel", {{"width", width}, {"height", height}, {"format", "yuv420p"}, - {"native_format", "h264-annex-b"}, {"supported_formats", {"h264"}}, - {"byte_length", encoded_bytes}}}, - {"video", {{"codec", "H264"}, {"transport", "WebRTC"}, - {"encoded_bytes", encoded_bytes}}}, + {"delivery", pixel_bytes == 0 ? "diagnostics" : "gallery-video"}, + {"pixel", {{"width", width}, {"height", height}, {"format", output_format}, + {"native_format", native_format}, + {"supported_formats", std::move(supported_formats)}, + {"byte_length", pixel_bytes}}}, + {"video", {{"codec", "H264"}, + {"transport", "shared WebRTC atlas"}}}, {"pacing", {{"mode", pacing_mode_name(pacing.mode)}, {"fixed_rate_fps", pacing.fixed_rate_fps}, {"render_enabled", pacing.render_enabled}, @@ -179,16 +208,6 @@ nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, }; } -struct Schema_Query { Plot::Json_Handler handler; }; -struct Prop_Write { - std::string component; - std::string key; - nlohmann::json value; - Plot::Json_Handler handler; -}; -struct Data_Generation { nlohmann::json input; Plot::Json_Handler handler; }; -using Plot_Command = std::variant; - template void dispatch_plot_input(Scene_Object& scene, const Plot_Input_Event& input) { const auto dispatch = [&](auto event) { scene.dispatch_event(std::move(event)); }; @@ -238,65 +257,94 @@ void dispatch_plot_input(Scene_Object& scene, const Plot_Input_Event& input) { struct Plot::Private { using Scene = std::variant, std::unique_ptr>; using Frame = std::variant, std::unique_ptr>; - struct Managed_Frame { - std::chrono::microseconds presentation_time{}; /* Plot 帧时钟产生的媒体时间戳。 */ - Frame frame{}; /* Scene 借用,Plot 保存到完成回调返回。 */ + + enum class Frame_State : std::uint8_t { + available, + in_flight, + callback_retired }; + + struct Managed_Frame { + std::chrono::microseconds presentation_time{}; /* 共享页面时钟产生的媒体时间戳。 */ + Frame frame{}; /* 三缓冲物理槽拥有且反复承载逻辑帧。 */ + Frame_State state{Frame_State::available}; /* 本槽唯一生命周期状态。 */ + }; + struct Consumer { Stream_Handler handler; std::uint32_t width{}; std::uint32_t height{}; }; + struct Stream_Snapshot { - std::vector consumers; + std::vector> consumers; std::uint32_t width{}; std::uint32_t height{}; }; asio::strand strand; - asio::strand encode_strand; /* 每个 Plot 串行编码,但不占用帧时钟执行域。 */ - asio::experimental::concurrent_channel commands; - asio::steady_timer render_clock; std::unique_ptr view; - Scene scene; std::once_flag start_once; mutable std::mutex consumers_mutex; - std::unordered_map consumers; /* 订阅配置的唯一权威来源。 */ + std::unordered_map consumers; /* 媒体图集和诊断连接的唯一订阅表。 */ std::atomic_uint64_t next_stream_id{1}; + std::atomic> terminal_failure{}; /* 首次 Plot Unknown Failure 的唯一终止状态。 */ std::uint64_t next_frame_sequence{1}; Frame_Policy frame_policy{}; - H264_Encoder encoder{100.0}; - std::atomic_uint64_t last_stream_publish_tail_ns{}; /* 最近一帧编码结束到消费者投递返回的诊断值。 */ - mutable std::mutex frame_mutex; /* 三缓冲帧所有权从 Plot 时钟移交给 Scene 回调与编码域。 */ - std::unordered_map active_frames{}; /* Scene 当前借用、尚未完成回调的帧。 */ - std::shared_ptr pending_encoding_frame{}; /* 编码器忙时唯一保留的最新完成帧。 */ - bool encoder_running{}; /* 当前是否有且仅有一个编码任务占用本 Plot 编码器。 */ - static constexpr std::size_t scene_frame_capacity = 3; /* Scene/GPU 独立三缓冲;媒体域另有正在处理与最新待处理两个有界槽位。 */ + mutable std::mutex frame_mutex; + static constexpr std::size_t scene_frame_capacity{3}; + std::array frame_slots{}; /* Scene 借用的稳定三缓冲物理帧。 */ + std::optional callback_retired_slot{}; /* 上一个回调返回后才可重新开始的槽。 */ + Scene scene; /* 析构顺序保证 Scene 先停止,再释放物理帧。 */ + mutable std::mutex tick_mutex; + std::optional pending_tick{}; /* 时钟拥塞时只保留尚未处理的最新时间点。 */ + bool tick_task_scheduled{}; /* Plot strand 上是否已有唯一消费任务。 */ + double last_clock_render_time_ms{-std::numeric_limits::infinity()}; std::chrono::steady_clock::time_point clock_origin{std::chrono::steady_clock::now()}; template Private(asio::any_io_executor executor, std::unique_ptr value_scene, std::unique_ptr value_view) - : strand(asio::make_strand(executor)), - encode_strand(asio::make_strand(video_encoding_executor())), - commands(strand, 32), - render_clock(strand), view(std::move(value_view)), scene(std::move(value_scene)) {} + : strand(asio::make_strand(executor)), view(std::move(value_view)), + scene(std::move(value_scene)) { + for (auto& slot : frame_slots) { + if constexpr (std::same_as) + slot.frame = std::make_unique(Frame_Identity{}); + else + slot.frame = std::make_unique(Frame_Identity{}); + } + } [[nodiscard]] nlohmann::json schema() const; [[nodiscard]] Stream_Snapshot stream_snapshot() const; - [[nodiscard]] std::chrono::steady_clock::duration clock_interval() const; - void start_clock(); - void schedule_clock(); - void clock_tick(); - void render_frame(); - void queue_completed_frame(Render_Frame* frame, std::weak_ptr lifetime); - void schedule_encoding(std::shared_ptr managed, - std::weak_ptr lifetime); - void encode_and_publish(std::shared_ptr managed, - std::weak_ptr lifetime); - void finish_encoding(std::weak_ptr lifetime); + void publish(std::shared_ptr frame) noexcept; + void consume_tick(); + void clock_tick(const Plot_Render_Tick& tick); + void render_frame(Plot_Render_Tick tick); + void queue_completed_frame(Render_Frame* frame); + void fail(std::exception_ptr failure) noexcept; }; +void Plot::Private::fail(std::exception_ptr failure) noexcept { + try { + auto description = std::make_shared( + exception_description(failure)); + std::shared_ptr empty; + if (!terminal_failure.compare_exchange_strong( + empty, description, std::memory_order_acq_rel, + std::memory_order_acquire)) return; + const auto output = std::make_shared( + Plot_Stream_Frame{nlohmann::json{ + {"kind", "plot_error"}, + {"protocol", "aethera.video.frame"}, + {"version", frame_protocol_version}, + {"message", *description} + }.dump(), {}}); + publish(std::move(output)); + } + catch (...) {} +} + nlohmann::json Plot::Private::schema() const { auto result = view->schema(); auto analysis = frame_policy.schema(); @@ -311,233 +359,256 @@ Plot::Private::Stream_Snapshot Plot::Private::stream_snapshot() const { std::lock_guard lock(consumers_mutex); result.consumers.reserve(consumers.size()); for (const auto& [id, consumer] : consumers) { - static_cast(id); + result.consumers.emplace_back(id, consumer); if (consumer.width == 0 || consumer.height == 0) continue; - result.consumers.push_back(consumer); result.width = std::max(result.width, consumer.width); result.height = std::max(result.height, consumer.height); } - result.width = std::clamp(result.width == 0 ? 720U : result.width, 160U, 1920U) & ~1U; - result.height = std::clamp(result.height == 0 ? 420U : result.height, 120U, 1080U) & ~1U; + result.width = std::clamp(result.width == 0 ? 320U : result.width, 160U, 1920U) & ~1U; + result.height = std::clamp(result.height == 0 ? 192U : result.height, 120U, 1080U) & ~1U; return result; } -std::chrono::steady_clock::duration Plot::Private::clock_interval() const { - const auto pacing = frame_policy.snapshot(); - if (pacing.mode == Frame_Pacing_Mode::manual) return std::chrono::milliseconds(100); - if (pacing.mode == Frame_Pacing_Mode::maximum_rate) return std::chrono::milliseconds(1); - return std::chrono::duration_cast( - std::chrono::duration(1.0 / pacing.fixed_rate_fps)); -} - -void Plot::Private::start_clock() { - render_clock.expires_after(clock_interval()); - schedule_clock(); -} - -void Plot::Private::schedule_clock() { - render_clock.async_wait([this](const asio::error_code& error) { - if (error) return; - /* - * The frame rate is a timeline frequency, not a sleep duration after render work. Advance - * from the previous absolute deadline so CPU preparation cannot accumulate clock drift. - * When a deadline was missed, skip the elapsed periods instead of issuing a catch-up burst. - */ - const auto interval = clock_interval(); - auto next_deadline = render_clock.expiry() + interval; - const auto now = std::chrono::steady_clock::now(); - if (next_deadline <= now) { - const auto missed_periods = (now - next_deadline) / interval + 1; - next_deadline += interval * missed_periods; +void Plot::Private::publish( + std::shared_ptr frame) noexcept { + if (!frame) return; + try { + const auto snapshot = stream_snapshot(); + std::vector failed_consumers; + for (const auto& [id, consumer] : snapshot.consumers) { + if (!consumer.handler) continue; + try { + consumer.handler(frame); + } + catch (...) { + failed_consumers.push_back(id); + } } - render_clock.expires_at(next_deadline); - schedule_clock(); - clock_tick(); - }); + if (failed_consumers.empty()) return; + std::lock_guard lock(consumers_mutex); + for (const auto id : failed_consumers) consumers.erase(id); + } + catch (...) {} } -void Plot::Private::clock_tick() { +void Plot::Private::consume_tick() { + std::optional tick; + { + std::lock_guard lock(tick_mutex); + tick = std::exchange(pending_tick, {}); + tick_task_scheduled = false; + } + if (tick) clock_tick(*tick); +} + +void Plot::Private::clock_tick(const Plot_Render_Tick& tick) { + if (terminal_failure.load(std::memory_order_acquire)) return; const auto pacing = frame_policy.snapshot(); if (!pacing.render_enabled || pacing.mode == Frame_Pacing_Mode::manual) return; - { - std::lock_guard lock(frame_mutex); - /* 已完成帧属于媒体域,不能反向占用 Scene/GPU 三缓冲槽位。 */ - if (active_frames.size() >= scene_frame_capacity) return; + if (pacing.mode == Frame_Pacing_Mode::fixed_rate) { + if (tick.time_milliseconds < last_clock_render_time_ms) + last_clock_render_time_ms = -std::numeric_limits::infinity(); + const double interval = 1'000.0 / pacing.fixed_rate_fps; + if (tick.time_milliseconds - last_clock_render_time_ms + 0.01 < interval) return; } - if (!stream_snapshot().consumers.empty()) render_frame(); + last_clock_render_time_ms = tick.time_milliseconds; + render_frame(tick); } -void Plot::Private::render_frame() { +void Plot::Private::render_frame(Plot_Render_Tick tick) { + if (terminal_failure.load(std::memory_order_acquire)) return; const auto streams = stream_snapshot(); const auto pacing = frame_policy.snapshot(); if (!pacing.render_enabled || streams.consumers.empty()) return; { std::lock_guard lock(frame_mutex); - if (active_frames.size() >= scene_frame_capacity) return; + if (std::ranges::none_of(frame_slots, [](const Managed_Frame& slot) { + return slot.state == Frame_State::available; + })) return; } - const auto elapsed = std::chrono::steady_clock::now() - clock_origin; - Plot_Render_Tick tick{next_frame_sequence, - std::chrono::duration(elapsed).count(), streams.width, streams.height}; - const Frame_Identity identity{next_frame_sequence, next_frame_sequence}; - ++next_frame_sequence; - view->update(tick); /* CPU 数据准备发生在 render 提交之前。 */ + tick.width = streams.width; + tick.height = streams.height; + /* + * 各图的采样、网格构造和属性快照都在 Plot 自己的准备域完成。 + * 进入 Scene::render 后只剩已经准备好的 Visual 批次与轻量提交; + * 共享 Render Domain 不承担业务数据生成。 + */ + view->update(tick); - Managed_Frame managed; - managed.presentation_time = std::chrono::duration_cast( - std::chrono::duration(tick.time_milliseconds)); - if (std::holds_alternative>(scene)) - managed.frame = std::make_unique(identity, Frame_2D::native_pixel_format); - else - managed.frame = std::make_unique(identity, - pacing.video_enabled - ? Frame_3D_Output::pixels - : Frame_3D_Output::diagnostics, - Frame_3D::native_pixel_format); - Render_Frame* address = std::visit([](const auto& value) -> Render_Frame* { return value.get(); }, - managed.frame); + const std::uint64_t sequence = next_frame_sequence++; + const Frame_Identity identity{sequence, tick.sequence == 0 ? sequence : tick.sequence}; + std::size_t slot_index{}; + Managed_Frame* managed{}; { std::lock_guard lock(frame_mutex); - const auto [active, inserted] = - active_frames.emplace(address, std::move(managed)); - if (!inserted) - throw std::logic_error("Plot received a duplicate frame address"); + const auto available = std::ranges::find_if( + frame_slots, [](const Managed_Frame& slot) { + return slot.state == Frame_State::available; + }); + if (available == frame_slots.end()) return; + slot_index = static_cast( + std::distance(frame_slots.begin(), available)); + managed = &*available; + managed->state = Frame_State::in_flight; + managed->presentation_time = + std::chrono::duration_cast( + std::chrono::duration( + tick.time_milliseconds)); } - - if (auto* scene_2d = std::get_if>(&scene)) { - (*scene_2d)->set<&Render_Scene_2D::Prop::viewport>( - Size{static_cast(tick.width), static_cast(tick.height)}); - const auto result = (*scene_2d)->render( - static_cast(address)); - if (result != Render_Scene_2D::Render_Result::completed) { - std::lock_guard lock(frame_mutex); - active_frames.erase(address); - } - return; - } - auto& scene_3d = std::get>(scene); - scene_3d->set<&Render_Scene_3D::Prop::viewport>(Extent{tick.width, tick.height}); - const auto result = scene_3d->render( - static_cast(address)); - if (result != Render_Scene_3D::Render_Result::submitted) { + const auto rollback_unsubmitted = [this, slot_index] { std::lock_guard lock(frame_mutex); - active_frames.erase(address); + auto& slot = frame_slots[slot_index]; + if (slot.state == Frame_State::in_flight) + slot.state = Frame_State::available; + }; + + try { + if (auto* scene_2d = std::get_if>(&scene)) { + auto& output = *std::get>(managed->frame); + output.begin(identity, pacing.video_enabled + ? render_2d::Pixel_Format::rgba8 + : Frame_2D::native_pixel_format); + (*scene_2d)->set<&Render_Scene_2D::Prop::viewport>( + Size{static_cast(tick.width), static_cast(tick.height)}); + const auto result = (*scene_2d)->render(&output); + if (result != Render_Scene_2D::Render_Result::completed) + rollback_unsubmitted(); + return; + } + auto& output = *std::get>(managed->frame); + output.begin(identity, pacing.video_enabled ? Frame_3D_Output::pixels + : Frame_3D_Output::diagnostics, + Frame_3D::native_pixel_format); + auto& scene_3d = std::get>(scene); + scene_3d->set<&Render_Scene_3D::Prop::viewport>(Extent{tick.width, tick.height}); + const auto result = scene_3d->render(&output); + if (result == Render_Scene_3D::Render_Result::submitted) return; + rollback_unsubmitted(); + if (result == Render_Scene_3D::Render_Result::backend_unavailable) + throw std::runtime_error("3D render backend became unavailable before submission"); + } + catch (...) { + rollback_unsubmitted(); + throw; } } -void Plot::Private::queue_completed_frame(Render_Frame* frame, - std::weak_ptr lifetime) { - std::shared_ptr managed; - std::shared_ptr discarded; - std::shared_ptr scheduled; +void Plot::Private::queue_completed_frame(Render_Frame* frame) { + if (!frame) + throw std::invalid_argument("Plot received a null completed frame"); + std::size_t slot_index{scene_frame_capacity}; + Managed_Frame* managed{}; { std::lock_guard lock(frame_mutex); - const auto active = active_frames.find(frame); - if (active == active_frames.end()) - throw std::logic_error( - "frame callback has no externally owned active frame"); - managed = std::make_shared(std::move(active->second)); - active_frames.erase(active); /* - * Scene 的每个完成回调都在这里闭环。视频只需要仍可能被观看的最新完成帧: - * 编码器忙时保留一张最新待编码帧,更新替代的旧帧直接释放,避免过时画面 - * 占满共享编码线程池。正在编码和最新待编码分别构成三缓冲的后两级。 + * Scene 在用户回调返回之后才标记 callback_finished,因此当前回调 + * 不能立即重启自己的物理帧。下一个串行完成回调开始时,上一个 + * callback_retired 槽已经确定离开 Scene,可安全归还。三槽由此在 + * 不改变 Scene 回调契约的前提下形成稳定的循环生命周期。 */ - discarded = std::exchange(pending_encoding_frame, std::move(managed)); - if (!encoder_running) { - encoder_running = true; - scheduled = std::exchange(pending_encoding_frame, {}); + if (callback_retired_slot) { + auto& retired = frame_slots[*callback_retired_slot]; + if (retired.state != Frame_State::callback_retired) + throw std::logic_error("Plot retired frame state is inconsistent"); + retired.state = Frame_State::available; + callback_retired_slot.reset(); } - } - frame->mark(Frame_Trace_Marker::video_encode_queued); - if (scheduled) schedule_encoding(std::move(scheduled), std::move(lifetime)); -} - -void Plot::Private::schedule_encoding(std::shared_ptr managed, - std::weak_ptr lifetime) { - asio::post(encode_strand, [lifetime, managed = std::move(managed)] { - if (const auto owner = lifetime.lock()) - owner->d->encode_and_publish(managed, lifetime); - }); -} - -void Plot::Private::encode_and_publish(std::shared_ptr managed, - std::weak_ptr lifetime) { - struct Encoding_Completion final { - Private& plot; /* 需要归还编码调度权的 Plot 实现。 */ - std::weak_ptr lifetime; /* 下一待编码帧只有在 Plot 存活时才继续调度。 */ - ~Encoding_Completion() { plot.finish_encoding(std::move(lifetime)); } - } completion{*this, lifetime}; - const auto streams = stream_snapshot(); - const auto pacing = frame_policy.snapshot(); - std::shared_ptr video; - std::uint32_t output_width{}; - std::uint32_t output_height{}; - Render_Frame* frame = std::visit( - [](const auto& value) -> Render_Frame* { return value.get(); }, managed->frame); - frame->record( - Frame_Trace_Measurement::stream_publish_tail_ns, - last_stream_publish_tail_ns.load(std::memory_order_acquire)); - frame->mark(Frame_Trace_Marker::video_encode_started); - if (pacing.video_enabled) { - const auto sequence = frame->identity().sequence; - if (auto* frame_2d = std::get_if>(&managed->frame)) { - auto pixels = (*frame_2d)->output_pixels(); - output_width = static_cast(pixels.width); - output_height = static_cast(pixels.height); - video = encoder.encode(pixels.bytes, output_width, output_height, - Video_Pixel_Layout::bgra, sequence, - managed->presentation_time); - } else { - auto& frame_3d = std::get>(managed->frame); - const auto extent = frame_3d->extent(); - output_width = extent.width; - output_height = extent.height; - /* 三维后端满载时会把多次提交合并为最新画面,并完成全部历史回调。 - * 因此完成 Frame 的 extent 才是像素尺寸的唯一权威,创建时的 tick - * 只表示当时请求的视口,禁止用它解释合并后的共享像素。 */ - if (frame_3d->output() == Frame_3D_Output::pixels) - video = encoder.encode(frame_3d->pixels(), output_width, - output_height, Video_Pixel_Layout::rgba, - sequence, managed->presentation_time); + for (std::size_t index = 0; index < frame_slots.size(); ++index) { + Render_Frame* address = std::visit( + [](const auto& value) -> Render_Frame* { return value.get(); }, + frame_slots[index].frame); + if (address != frame) continue; + slot_index = index; + managed = &frame_slots[index]; + break; } - } else if (auto* frame_2d = std::get_if>(&managed->frame)) { - const auto image = (*frame_2d)->image(); - output_width = static_cast(image.width); - output_height = static_cast(image.height); - } else { - const auto extent = std::get>(managed->frame)->extent(); - output_width = extent.width; - output_height = extent.height; + if (!managed || managed->state != Frame_State::in_flight) + throw std::logic_error("frame callback has no externally owned active frame"); } - frame->mark(Frame_Trace_Marker::video_encode_finished); - const auto publish_tail_started = std::chrono::steady_clock::now(); - frame->mark(Frame_Trace_Marker::stream_publish_started); - const auto bytes = video ? video->annex_b.size() : 0U; - frame->mark(Frame_Trace_Marker::stream_publish_finished); - const auto metadata = frame_metadata(*frame, output_width, output_height, - bytes, pacing).dump(); - const auto published = std::make_shared( - Plot_Stream_Frame{metadata, pacing.video_enabled ? video : nullptr}); - for (const auto& consumer : streams.consumers) { - if (!consumer.handler) continue; - consumer.handler(published); - } - last_stream_publish_tail_ns.store( - static_cast(std::chrono::duration_cast( - std::chrono::steady_clock::now() - publish_tail_started).count()), - std::memory_order_release); -} -void Plot::Private::finish_encoding(std::weak_ptr lifetime) { - std::shared_ptr scheduled; - { + const auto retire = [this, slot_index] { std::lock_guard lock(frame_mutex); - if (!encoder_running) - throw std::logic_error("Plot encoding scheduler completed without an active encoder"); - scheduled = std::exchange(pending_encoding_frame, {}); - if (!scheduled) encoder_running = false; + auto& completed = frame_slots[slot_index]; + if (completed.state != Frame_State::in_flight) return; + if (callback_retired_slot) + throw std::logic_error("Plot has more than one callback-retired frame"); + completed.state = Frame_State::callback_retired; + callback_retired_slot = slot_index; + }; + + try { + const auto pacing = frame_policy.snapshot(); + const auto identity = frame->identity(); + Frame_Identity rendered_identity = identity; + std::shared_ptr> pixel_storage; + std::uint32_t width{}; + std::uint32_t height{}; + std::string_view output_format{"rgba8"}; + std::string_view native_format{"rgba8"}; + if (auto* frame_2d = + std::get_if>(&managed->frame)) { + const auto image = (*frame_2d)->image(); + width = static_cast(image.width); + height = static_cast(image.height); + output_format = pixel_format_name((*frame_2d)->output_format()); + native_format = pixel_format_name(Frame_2D::native_pixel_format); + if (pacing.video_enabled) { + auto output = (*frame_2d)->output_pixels(); + pixel_storage = std::make_shared>( + std::move(output.bytes)); + width = static_cast(output.width); + height = static_cast(output.height); + } + } + else { + auto& frame_3d = + std::get>(managed->frame); + output_format = pixel_format_name(frame_3d->output_format()); + native_format = pixel_format_name(Frame_3D::native_pixel_format); + rendered_identity = frame_3d->rendered_identity(); + const auto extent = frame_3d->extent(); + width = extent.width; + height = extent.height; + if (pacing.video_enabled && + frame_3d->output() == Frame_3D_Output::pixels) + pixel_storage = frame_3d->share_pixels(); + } + + auto pixels = std::make_shared(Plot_Pixel_Frame{ + std::move(pixel_storage), managed->presentation_time, + identity.sequence, identity.correlation_id, + rendered_identity.sequence, rendered_identity.correlation_id, + width, height}); + + frame->mark(Frame_Trace_Marker::frame_ready); + std::string metadata; + if (identity.sequence == 1 || + identity.sequence % diagnostic_sample_period == 0) { + nlohmann::json supported_formats = nlohmann::json::array(); + if (std::holds_alternative>( + managed->frame)) { + for (const auto format : Frame_2D::supported_pixel_formats) + supported_formats.push_back(pixel_format_name(format)); + } + else { + for (const auto format : Frame_3D::supported_pixel_formats) + supported_formats.push_back(pixel_format_name(format)); + } + metadata = frame_metadata( + *frame, width, height, + pixels->rgba ? pixels->rgba->size() : 0U, + rendered_identity, output_format, native_format, + std::move(supported_formats), pacing).dump(); + } + const auto published = std::make_shared( + Plot_Stream_Frame{std::move(metadata), std::move(pixels)}); + publish(std::move(published)); + retire(); + } + catch (...) { + retire(); + throw; } - if (scheduled) schedule_encoding(std::move(scheduled), std::move(lifetime)); } Plot::Plot(asio::any_io_executor executor, std::unique_ptr scene, @@ -546,57 +617,62 @@ Plot::Plot(asio::any_io_executor executor, std::unique_ptr scene, Plot::Plot(asio::any_io_executor executor, std::unique_ptr scene, std::unique_ptr view) : d(std::make_unique(std::move(executor), std::move(scene), std::move(view))) {} -Plot::~Plot() { - d->render_clock.cancel(); - d->commands.close(); -} +Plot::~Plot() = default; void Plot::ensure_started() { std::call_once(d->start_once, [this] { - auto self = shared_from_this(); /* - * 设计边界:帧策略时钟只触发 render;Scene 完成回调只消费完成帧并立即发布。 - * 两者不存在 request/ack 或“回调安排下一帧”的依赖,因此 GPU、编码和发送可流水重叠。 + * 页面级帧时钟只触发 render;Scene 完成回调直接归还帧并发布结果。 + * 回调不请求下一帧,也不重新投递到 Plot strand;这使准备、GPU + * 提交和页面级媒体图集能够按各自的资源域自然流水。 */ if (auto* scene = std::get_if>(&d->scene)) { - (*scene)->set_frame_callback([weak = weak_from_this()](Frame_2D* frame) { - if (auto owner = weak.lock()) - owner->d->queue_completed_frame(frame, owner); - }); + (*scene)->set_frame_callback( + [weak = weak_from_this()](Frame_2D* frame) { + if (auto owner = weak.lock()) { + try { owner->d->queue_completed_frame(frame); } + catch (...) { owner->d->fail(std::current_exception()); } + } + }); } else { std::get>(d->scene)->set_frame_callback( [weak = weak_from_this()](Frame_3D* frame) { - if (auto owner = weak.lock()) - owner->d->queue_completed_frame(frame, owner); + if (auto owner = weak.lock()) { + try { owner->d->queue_completed_frame(frame); } + catch (...) { owner->d->fail(std::current_exception()); } + } }); } - d->start_clock(); - asio::co_spawn(d->strand, [self]() -> asio::awaitable { - for (;;) { - asio::error_code error; - auto command = co_await self->d->commands.async_receive( - asio::redirect_error(asio::use_awaitable, error)); - if (error) co_return; - if (auto* query = std::get_if(&command)) { - query->handler(self->d->schema()); - } else if (auto* write = std::get_if(&command)) { - write->handler(write->component == "frame-analysis" - ? self->d->frame_policy.write_prop(write->key, write->value) - : self->d->view->write_prop(write->component, write->key, write->value)); - } else { - auto& generation = std::get(command); - generation.handler(self->d->view->generate_data(generation.input)); - } - } - }, [](std::exception_ptr exception) { if (exception) std::rethrow_exception(exception); }); }); } Plot::Stream_Id Plot::subscribe(Stream_Handler handler) { + if (!handler) + throw std::invalid_argument("Plot subscription requires a handler"); ensure_started(); const auto id = d->next_stream_id.fetch_add(1, std::memory_order_relaxed); - std::lock_guard lock(d->consumers_mutex); - d->consumers.emplace(id, Private::Consumer{std::move(handler)}); + Stream_Handler notification; + { + std::lock_guard lock(d->consumers_mutex); + d->consumers.emplace(id, Private::Consumer{std::move(handler)}); + if (d->terminal_failure.load(std::memory_order_acquire)) + notification = d->consumers.at(id).handler; + } + if (notification) { + const auto failure = d->terminal_failure.load(std::memory_order_acquire); + try { + notification(std::make_shared( + Plot_Stream_Frame{nlohmann::json{ + {"kind", "plot_error"}, + {"protocol", "aethera.video.frame"}, + {"version", frame_protocol_version}, + {"message", failure ? *failure : "Plot unavailable"} + }.dump(), {}})); + } + catch (...) { + unsubscribe(id); + } + } return id; } @@ -605,7 +681,8 @@ void Plot::unsubscribe(Stream_Id stream) { d->consumers.erase(stream); } -void Plot::configure_stream(Stream_Id stream, std::uint32_t width, std::uint32_t height) { +void Plot::configure_stream(Stream_Id stream, std::uint32_t width, + std::uint32_t height) { std::lock_guard lock(d->consumers_mutex); const auto found = d->consumers.find(stream); if (found == d->consumers.end()) return; @@ -613,11 +690,29 @@ void Plot::configure_stream(Stream_Id stream, std::uint32_t width, std::uint32_t found->second.height = height; } -void Plot::request_video_key_frame() { +void Plot::schedule_render(Plot_Render_Tick tick) { ensure_started(); + if (d->terminal_failure.load(std::memory_order_acquire)) return; + bool schedule{}; + { + std::lock_guard lock(d->tick_mutex); + d->pending_tick = tick; + if (!d->tick_task_scheduled) { + d->tick_task_scheduled = true; + schedule = true; + } + } + if (!schedule) return; const auto weak = weak_from_this(); - asio::post(d->encode_strand, [weak] { - if (const auto owner = weak.lock()) owner->d->encoder.request_key_frame(); + asio::post(d->strand, [weak] { + const auto owner = weak.lock(); + if (!owner) return; + try { + owner->d->consume_tick(); + } + catch (...) { + owner->d->fail(std::current_exception()); + } }); } @@ -625,36 +720,105 @@ void Plot::render_once() { ensure_started(); const auto weak = weak_from_this(); asio::post(d->strand, [weak] { - if (const auto owner = weak.lock()) owner->d->render_frame(); + const auto owner = weak.lock(); + if (!owner) return; + try { + const auto elapsed = std::chrono::steady_clock::now() - + owner->d->clock_origin; + owner->d->render_frame(Plot_Render_Tick{ + 0, std::chrono::duration(elapsed).count()}); + } + catch (...) { + owner->d->fail(std::current_exception()); + } }); } void Plot::submit_input(Plot_Input_Event event) { ensure_started(); - if (auto* scene = std::get_if>(&d->scene)) - dispatch_plot_input(**scene, event); - else - dispatch_plot_input(*std::get>(d->scene), event); + if (d->terminal_failure.load(std::memory_order_acquire)) return; + const auto weak = weak_from_this(); + asio::post(d->strand, [weak, event = std::move(event)] { + const auto owner = weak.lock(); + if (!owner || owner->d->terminal_failure.load(std::memory_order_acquire)) + return; + try { + if (auto* scene = std::get_if>( + &owner->d->scene)) + dispatch_plot_input(**scene, event); + else + dispatch_plot_input( + *std::get>(owner->d->scene), event); + } + catch (...) { + owner->d->fail(std::current_exception()); + } + }); } void Plot::async_schema(Json_Handler handler) { + if (!handler) throw std::invalid_argument("Plot schema handler is empty"); ensure_started(); - if (!d->commands.try_send(asio::error_code{}, Plot_Command{Schema_Query{std::move(handler)}})) - throw std::runtime_error("plot command queue is unavailable"); + auto self = shared_from_this(); + asio::post(d->strand, [self, handler = std::move(handler)]() mutable { + nlohmann::json result; + try { + result = self->d->schema(); + } + catch (...) { + const auto failure = std::current_exception(); + self->d->fail(failure); + result = {{"success", false}, + {"error", exception_description(failure)}}; + } + try { handler(std::move(result)); } + catch (...) {} + }); } void Plot::async_write_prop(std::string component, std::string key, nlohmann::json value, Json_Handler handler) { + if (!handler) throw std::invalid_argument("Plot property handler is empty"); ensure_started(); - if (!d->commands.try_send(asio::error_code{}, Plot_Command{Prop_Write{ - std::move(component), std::move(key), std::move(value), std::move(handler)}})) - throw std::runtime_error("plot command queue is unavailable"); + auto self = shared_from_this(); + asio::post(d->strand, + [self, component = std::move(component), key = std::move(key), + value = std::move(value), handler = std::move(handler)]() mutable { + nlohmann::json result; + try { + result = component == "frame-analysis" + ? self->d->frame_policy.write_prop(key, value) + : self->d->view->write_prop(component, key, value); + } + catch (...) { + const auto failure = std::current_exception(); + self->d->fail(failure); + result = {{"success", false}, + {"error", exception_description(failure)}}; + } + try { handler(std::move(result)); } + catch (...) {} + }); } void Plot::async_generate_data(nlohmann::json input, Json_Handler handler) { + if (!handler) throw std::invalid_argument("Plot data handler is empty"); ensure_started(); - if (!d->commands.try_send(asio::error_code{}, Plot_Command{Data_Generation{ - std::move(input), std::move(handler)}})) - throw std::runtime_error("plot command queue is unavailable"); + auto self = shared_from_this(); + asio::post(d->strand, + [self, input = std::move(input), handler = std::move(handler)]() mutable { + nlohmann::json result; + try { + result = self->d->view->generate_data(input); + } + catch (...) { + const auto failure = std::current_exception(); + self->d->fail(failure); + result = {{"success", false}, + {"error", exception_description(failure)}}; + } + try { handler(std::move(result)); } + catch (...) {} + }); } } diff --git a/web_server/src/Plot.hpp b/web_server/src/Plot.hpp index 5803963..a09a1d7 100644 --- a/web_server/src/Plot.hpp +++ b/web_server/src/Plot.hpp @@ -1,14 +1,16 @@ #pragma once -#include "H264_Encoder.hpp" #include #include #include #include +#include +#include #include #include #include #include #include +#include namespace aethera::web { struct Plot_Input_Event { @@ -28,15 +30,26 @@ struct Plot_Input_Event { }; struct Plot_Render_Tick { - std::uint64_t sequence{}; /* 服务端帧时钟分配的序号,不来源于浏览器请求。 */ - double time_milliseconds{}; /* Plot 启动后累计的单调时间,供动画生成器推进。 */ - std::uint32_t width{720}; - std::uint32_t height{420}; + std::uint64_t sequence{}; /* 页面级帧时钟分配的关联序号。 */ + double time_milliseconds{}; /* 页面级单调时间线,所有图共享同一个动画时刻。 */ + std::uint32_t width{320}; /* 当前图在媒体图集中的固定像素宽度。 */ + std::uint32_t height{192}; /* 当前图在媒体图集中的固定像素高度。 */ +}; + +struct Plot_Pixel_Frame { + std::shared_ptr> rgba{}; /* 可选的不可变 RGBA8;停传输时为空但完成身份仍有效。 */ + std::chrono::microseconds presentation_time{}; /* 页面媒体时钟上的显示时间戳。 */ + std::uint64_t sequence{}; /* 对应 Plot 完成帧的局部单调序号。 */ + std::uint64_t correlation_id{}; /* 触发本帧的页面级时钟序号;手动帧等于局部序号。 */ + std::uint64_t rendered_sequence{}; /* 实际产生当前像素的 Scene/GPU 提交帧序号。 */ + std::uint64_t rendered_correlation_id{}; /* 实际画面对应的页面级时钟序号。 */ + std::uint32_t width{}; /* RGBA 像素宽度。 */ + std::uint32_t height{}; /* RGBA 像素高度。 */ }; struct Plot_Stream_Frame { - std::string metadata{}; /* 低频诊断仍走 WebSocket 文本。 */ - std::shared_ptr video{}; /* 同一编码帧由全部 WebRTC 订阅者共享。 */ + std::string metadata{}; /* 抽样诊断文本;空值表示本帧不产生诊断流量。 */ + std::shared_ptr pixels{}; /* 每帧完成进度及其可选图集像素。 */ }; class Plot final : public std::enable_shared_from_this { @@ -70,7 +83,7 @@ public: [[nodiscard]] Stream_Id subscribe(Stream_Handler handler); void unsubscribe(Stream_Id stream); void configure_stream(Stream_Id stream, std::uint32_t width, std::uint32_t height); - void request_video_key_frame(); + void schedule_render(Plot_Render_Tick tick); void render_once(); void submit_input(Plot_Input_Event event); void async_schema(Json_Handler handler); diff --git a/web_server/src/WebRtc_Video_Session.cpp b/web_server/src/WebRtc_Video_Session.cpp index ccf2ce3..aa111b3 100644 --- a/web_server/src/WebRtc_Video_Session.cpp +++ b/web_server/src/WebRtc_Video_Session.cpp @@ -1,17 +1,22 @@ #include "WebRtc_Video_Session.hpp" +#include #include #include #include +#include #include #include +#include #include #include -#include +#include #include namespace aethera::web { namespace { constexpr std::uint8_t h264_payload_type{102}; +constexpr std::size_t maximum_outstanding_video_frames{3}; +constexpr std::size_t maximum_transport_buffered_bytes{4U * 1024U * 1024U}; std::uint32_t make_ssrc() { std::random_device source; @@ -21,115 +26,272 @@ std::uint32_t make_ssrc() { struct WebRtc_Video_Session::Private { struct Callback_State { - Signal_Handler signal_handler; /* WebSocket 只负责 SDP/ICE 信令,不承载帧像素。 */ - Ready_Handler ready_handler; /* Track 打开后请求首个可独立解码的视频帧。 */ + Signal_Handler signal_handler; /* SDP/ICE 信令的唯一交付出口。 */ + Ready_Handler ready_handler; /* Track 打开后请求首个 IDR。 */ + Failure_Handler failure_handler; /* 内部线程异常的唯一隔离出口。 */ std::atomic_bool track_open{}; std::atomic_bool closed{}; + std::atomic_bool failure_reported{}; + + void report(std::exception_ptr failure) noexcept { + if (failure_reported.exchange(true, std::memory_order_acq_rel) || + !failure_handler) + return; + try { failure_handler(std::move(failure)); } + catch (...) {} + } }; + std::shared_ptr callbacks; std::shared_ptr peer; std::shared_ptr video_track; std::shared_ptr rtp_config; - std::mutex media_mutex; /* 关闭与发送之间不得跨越 Track 生命周期。 */ + std::mutex media_mutex; /* 发送、信令修改与关闭的生命周期边界。 */ + moodycamel::BlockingConcurrentQueue + pending_frames{4}; /* 尚未发送的最新编码帧入口。 */ + std::thread sender_thread{}; /* 唯一的 RTP/SRTP 发送资源域。 */ + std::atomic_size_t outstanding_video_frames{}; /* sendFrame 中和队列中的完整 access unit 数。 */ - Private(Signal_Handler value_signal_handler, - Ready_Handler value_ready_handler) + Private(Signal_Handler signal_handler, Ready_Handler ready_handler, + Failure_Handler failure_handler) : callbacks(std::make_shared()) { - callbacks->signal_handler = std::move(value_signal_handler); - callbacks->ready_handler = std::move(value_ready_handler); + if (!signal_handler || !failure_handler) + throw std::invalid_argument( + "WebRTC session requires signal and failure handlers"); + callbacks->signal_handler = std::move(signal_handler); + callbacks->ready_handler = std::move(ready_handler); + callbacks->failure_handler = std::move(failure_handler); + } + + void run_sender() noexcept { + for (;;) { + Encoded_Video_Frame frame; + pending_frames.wait_dequeue(frame); + if (frame.annex_b.empty() || + callbacks->closed.load(std::memory_order_acquire)) + return; + try { + std::lock_guard lock(media_mutex); + if (!video_track || + !callbacks->track_open.load(std::memory_order_acquire) || + callbacks->closed.load(std::memory_order_acquire)) { + outstanding_video_frames.fetch_sub( + 1, std::memory_order_release); + continue; + } + /* rtc::binary 与编码器 access unit 使用相同的 byte vector。 + * 这里把所有权直接交给 packetizer,禁止再次复制完整 H.264 帧。 */ + video_track->sendFrame( + std::move(frame.annex_b), + rtc::FrameInfo(std::chrono::duration( + frame.presentation_time))); + outstanding_video_frames.fetch_sub( + 1, std::memory_order_release); + } + catch (...) { + outstanding_video_frames.fetch_sub( + 1, std::memory_order_release); + if (!callbacks->closed.load(std::memory_order_acquire) && + callbacks->track_open.exchange( + false, std::memory_order_acq_rel)) + callbacks->report(std::current_exception()); + return; + } + } } }; WebRtc_Video_Session::WebRtc_Video_Session( - Signal_Handler signal_handler, Ready_Handler ready_handler) - : d(std::make_unique(std::move(signal_handler), - std::move(ready_handler))) {} + Signal_Handler signal_handler, Ready_Handler ready_handler, + Failure_Handler failure_handler) + : d(std::make_unique( + std::move(signal_handler), std::move(ready_handler), + std::move(failure_handler))) {} + WebRtc_Video_Session::~WebRtc_Video_Session() { close(); } void WebRtc_Video_Session::start() { std::lock_guard lock(d->media_mutex); if (d->peer) return; - d->peer = std::make_shared(rtc::Configuration{}); + if (d->callbacks->closed.load(std::memory_order_acquire)) + throw std::logic_error("WebRTC session is closed"); + + /* Peer、Track 与 RTP handler 先在局部完整构造;任何一步失败都不会把 + * 半初始化会话发布给发送路径。 */ + auto peer = std::make_shared(rtc::Configuration{}); const std::weak_ptr callbacks{d->callbacks}; - d->peer->onLocalDescription([callbacks](rtc::Description description) { + peer->onLocalDescription([callbacks](rtc::Description description) { const auto state = callbacks.lock(); if (!state || state->closed.load(std::memory_order_acquire)) return; - state->signal_handler(nlohmann::json{{"kind", "webrtc_offer"}, - {"type", std::string(description.typeString())}, - {"sdp", std::string(description)}}.dump()); + try { + state->signal_handler(nlohmann::json{ + {"kind", "webrtc_offer"}, + {"type", std::string(description.typeString())}, + {"sdp", std::string(description)}}.dump()); + } + catch (...) { state->report(std::current_exception()); } }); - d->peer->onLocalCandidate([callbacks](rtc::Candidate candidate) { + peer->onLocalCandidate([callbacks](rtc::Candidate candidate) { const auto state = callbacks.lock(); if (!state || state->closed.load(std::memory_order_acquire)) return; - state->signal_handler(nlohmann::json{{"kind", "webrtc_candidate"}, - {"candidate", std::string(candidate)}, - {"mid", candidate.mid()}}.dump()); + try { + state->signal_handler(nlohmann::json{ + {"kind", "webrtc_candidate"}, + {"candidate", std::string(candidate)}, + {"mid", candidate.mid()}}.dump()); + } + catch (...) { state->report(std::current_exception()); } }); const auto ssrc = make_ssrc(); - rtc::Description::Video video("aethera-video", rtc::Description::Direction::SendOnly); - video.addH264Codec(h264_payload_type); - video.addSSRC(ssrc, "aethera-video", "aethera-plot", "aethera-video"); - d->video_track = d->peer->addTrack(video); - d->rtp_config = std::make_shared( - ssrc, "aethera-video", h264_payload_type, rtc::H264RtpPacketizer::ClockRate); + rtc::Description::Video video( + "aethera-video", rtc::Description::Direction::SendOnly); + video.addH264Codec( + h264_payload_type, + "profile-level-id=" + std::string{h264_profile_level_id()} + + ";packetization-mode=1;level-asymmetry-allowed=1"); + video.addSSRC(ssrc, "aethera-video", "aethera-gallery", + "aethera-video"); + auto video_track = peer->addTrack(video); + auto rtp_config = std::make_shared( + ssrc, "aethera-video", h264_payload_type, + rtc::H264RtpPacketizer::ClockRate); auto packetizer = std::make_shared( - rtc::NalUnit::Separator::StartSequence, d->rtp_config); - packetizer->addToChain(std::make_shared(d->rtp_config)); + rtc::NalUnit::Separator::StartSequence, rtp_config); + packetizer->addToChain( + std::make_shared(rtp_config)); packetizer->addToChain(std::make_shared()); - d->video_track->setMediaHandler(packetizer); - d->video_track->onOpen([callbacks] { + /* + * PLI/FIR 是接收端发现 H.264 参考链不可继续解码后的恢复协议,不是前端 + * 控制消息。必须在同一 Track 的 RTCP handler 链中直接转成编码器 IDR + * 请求;否则 Chrome 会持续报告 PLI,而编码器仍输出依赖已丢失参考帧的 + * P 帧,表现为 RTP 收帧正常、浏览器却每秒只能解出一两帧。 + */ + packetizer->addToChain(std::make_shared([callbacks] { + if (const auto state = callbacks.lock()) { + try { + if (!state->closed.load(std::memory_order_acquire) && + state->ready_handler) + state->ready_handler(); + } + catch (...) { state->report(std::current_exception()); } + } + })); + /* + * 不在这里接 PacingHandler。它把 packetizer 产出的 RTP 分片转存到内部 + * 不可观测、无容量上限的队列,sendFrame() 随即返回会让上层误以为帧已 + * 排出;多路原生分辨率运行时该隐藏队列可增长到数 GiB。直接交给 Track + * 的发送链,并由 can_accept_video() 查询 Channel::bufferedAmount(),使 + * 编码入口的背压对应真正的 DTLS/UDP 传输积压。 + */ + video_track->setMediaHandler(packetizer); + video_track->onOpen([callbacks] { if (const auto state = callbacks.lock()) { state->track_open.store(true, std::memory_order_release); - if (!state->closed.load(std::memory_order_acquire) && - state->ready_handler) - state->ready_handler(); + try { + if (!state->closed.load(std::memory_order_acquire) && + state->ready_handler) + state->ready_handler(); + } + catch (...) { state->report(std::current_exception()); } } }); - d->video_track->onClosed([callbacks] { + video_track->onClosed([callbacks] { if (const auto state = callbacks.lock()) state->track_open.store(false, std::memory_order_release); }); - d->peer->setLocalDescription(); + peer->setLocalDescription(); + + d->peer = std::move(peer); + d->video_track = std::move(video_track); + d->rtp_config = std::move(rtp_config); + d->sender_thread = std::thread([data = d.get()] { data->run_sender(); }); } void WebRtc_Video_Session::accept_answer(std::string_view sdp) { std::lock_guard lock(d->media_mutex); - if (!d->peer) throw std::logic_error("WebRTC session has not started"); - d->peer->setRemoteDescription(rtc::Description(std::string(sdp), "answer")); + if (!d->peer || d->callbacks->closed.load(std::memory_order_acquire)) + throw std::logic_error("WebRTC session is not active"); + d->peer->setRemoteDescription( + rtc::Description(std::string(sdp), "answer")); } -void WebRtc_Video_Session::add_remote_candidate(std::string_view candidate, - std::string_view mid) { +void WebRtc_Video_Session::add_remote_candidate( + std::string_view candidate, std::string_view mid) { std::lock_guard lock(d->media_mutex); - if (!d->peer) throw std::logic_error("WebRTC session has not started"); - d->peer->addRemoteCandidate(rtc::Candidate(std::string(candidate), std::string(mid))); + if (!d->peer || d->callbacks->closed.load(std::memory_order_acquire)) + throw std::logic_error("WebRTC session is not active"); + d->peer->addRemoteCandidate( + rtc::Candidate(std::string(candidate), std::string(mid))); } WebRtc_Video_Session::Send_Result WebRtc_Video_Session::send( - const Encoded_Video_Frame& frame) { - std::lock_guard lock(d->media_mutex); - if (!d->video_track || !d->callbacks->track_open.load(std::memory_order_acquire) || - frame.annex_b.empty()) return Send_Result::not_open; - try { - d->video_track->sendFrame( - reinterpret_cast(frame.annex_b.data()), frame.annex_b.size(), - rtc::FrameInfo(std::chrono::duration(frame.presentation_time))); - return Send_Result::sent; - } catch (const std::runtime_error&) { - /* LibDataChannel 可在 isOpen() 后由其内部线程关闭 Track。 */ - if (!d->video_track->isOpen()) return Send_Result::not_open; - throw; + Encoded_Video_Frame frame) { + if (frame.annex_b.empty() || + !d->callbacks->track_open.load(std::memory_order_acquire) || + d->callbacks->closed.load(std::memory_order_acquire) || + !d->sender_thread.joinable()) + return Send_Result::not_open; + + const auto previous = d->outstanding_video_frames.fetch_add( + 1, std::memory_order_acq_rel); + if (previous >= maximum_outstanding_video_frames) { + d->outstanding_video_frames.fetch_sub( + 1, std::memory_order_release); + return Send_Result::not_open; } + + /* + * 最新帧合并只允许发生在 Gallery_Video_Stream 的编码入口。H.264 P 帧 + * 引用此前 access unit;编码后删除任意一帧会破坏浏览器参考链,表现为 + * readyState 正常但画面永久冻结。这里保持编码器产出顺序完整,发送线程 + * 只负责把已编码所有权依次交给 RTP/SRTP。 + */ + // 构造参数只预分配首批槽位;try_enqueue() 在这批槽位瞬时占满时不会 + // 扩容,会把正常背压误判成内存失败。enqueue() 允许 concurrentqueue + // 按需增加块,实际持续突发由 RTP pacer 消化。 + if (!d->pending_frames.enqueue(std::move(frame))) { + d->outstanding_video_frames.fetch_sub( + 1, std::memory_order_release); + throw std::bad_alloc{}; + } + return Send_Result::queued; } -void WebRtc_Video_Session::close() { +bool WebRtc_Video_Session::can_accept_video() const noexcept { std::lock_guard lock(d->media_mutex); - if (d->callbacks->closed.exchange(true, std::memory_order_acq_rel)) return; - d->callbacks->track_open.store(false, std::memory_order_release); - if (d->video_track) d->video_track->close(); - if (d->peer) d->peer->close(); - d->video_track.reset(); - d->peer.reset(); + return d->callbacks->track_open.load(std::memory_order_acquire) && + !d->callbacks->closed.load(std::memory_order_acquire) && + d->sender_thread.joinable() && + d->video_track && + d->video_track->bufferedAmount() < maximum_transport_buffered_bytes && + d->outstanding_video_frames.load(std::memory_order_acquire) < + maximum_outstanding_video_frames; +} + +void WebRtc_Video_Session::close() noexcept { + try { + if (d->callbacks->closed.exchange(true, std::memory_order_acq_rel)) + return; + d->callbacks->track_open.store(false, std::memory_order_release); + Encoded_Video_Frame discarded; + while (d->pending_frames.try_dequeue(discarded)) {} + while (!d->pending_frames.try_enqueue( + Encoded_Video_Frame{})) + std::this_thread::yield(); + if (d->sender_thread.joinable()) d->sender_thread.join(); + while (d->pending_frames.try_dequeue(discarded)) {} + d->outstanding_video_frames.store(0, std::memory_order_release); + + std::lock_guard lock(d->media_mutex); + try { if (d->video_track) d->video_track->close(); } + catch (...) {} + try { if (d->peer) d->peer->close(); } + catch (...) {} + d->video_track.reset(); + d->rtp_config.reset(); + d->peer.reset(); + } + catch (...) {} } } diff --git a/web_server/src/WebRtc_Video_Session.hpp b/web_server/src/WebRtc_Video_Session.hpp index 0b4b760..1540cf2 100644 --- a/web_server/src/WebRtc_Video_Session.hpp +++ b/web_server/src/WebRtc_Video_Session.hpp @@ -1,5 +1,7 @@ #pragma once #include "H264_Encoder.hpp" +#include +#include #include #include #include @@ -9,14 +11,16 @@ namespace aethera::web { class WebRtc_Video_Session final { public: enum class Send_Result : std::uint8_t { - sent, + queued, not_open }; using Signal_Handler = std::function; using Ready_Handler = std::function; + using Failure_Handler = std::function; WebRtc_Video_Session(Signal_Handler signal_handler, - Ready_Handler ready_handler); + Ready_Handler ready_handler, + Failure_Handler failure_handler); ~WebRtc_Video_Session(); WebRtc_Video_Session(const WebRtc_Video_Session&) = delete; WebRtc_Video_Session& operator=(const WebRtc_Video_Session&) = delete; @@ -24,8 +28,10 @@ public: void start(); void accept_answer(std::string_view sdp); void add_remote_candidate(std::string_view candidate, std::string_view mid); - [[nodiscard]] Send_Result send(const Encoded_Video_Frame& frame); - void close(); + [[nodiscard]] Send_Result send( + Encoded_Video_Frame frame); + [[nodiscard]] bool can_accept_video() const noexcept; + void close() noexcept; private: struct Private; diff --git a/web_server/src/Web_Server.cpp b/web_server/src/Web_Server.cpp index 6254f0d..b479b77 100644 --- a/web_server/src/Web_Server.cpp +++ b/web_server/src/Web_Server.cpp @@ -1,19 +1,26 @@ #include "Web_Server.hpp" +#include "Gallery_Video_Stream.hpp" +#include "Gallery_WebSocket.hpp" #include "Graph_WebSocket.hpp" #include "Gallery_Plots.hpp" #include #include #include #include +#include #include +#include #include #include #include #include +#include namespace aethera::web { namespace { using Plot_Map = std::unordered_map>; +using Gallery_Stream_Map = + std::unordered_map>; drogon::HttpResponsePtr json_response(nlohmann::json value) { auto response = drogon::HttpResponse::newHttpResponse(); @@ -41,6 +48,12 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) * 两类 Plot 使用同一套 Plot/帧策略实现,但执行资源按渲染架构隔离。 */ auto plot_2d_pool = std::make_shared(plot_workers); auto plot_3d_pool = std::make_shared(plot_workers); + auto gallery_clock_pool = std::make_shared(1); + // 原生 720x420 画质按四图一个媒体组并行编码。每路图集仍是完整像素 + // 视频,但较小的 access unit 缩短 libdatachannel packetize/DTLS 临界段; + // 时钟只负责轻量 tick,六路编码各有独立 strand 和执行线程。 + auto gallery_encoding_pool = std::make_shared( + std::min(6U, hardware_threads)); const auto executor_2d = plot_2d_pool->get_executor(); const auto executor_3d = plot_3d_pool->get_executor(); @@ -70,11 +83,65 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) plots->emplace("datoviz_text", make_datoviz_text_plot(executor_3d)); plots->emplace("datoviz_volume", make_datoviz_volume_plot(executor_3d)); + std::vector gallery_2d; + std::vector gallery_3d; + gallery_2d.reserve(8); + gallery_3d.reserve(16); + for (const auto& [id, plot] : *plots) + (id.starts_with("datoviz_") ? gallery_3d : gallery_2d) + .push_back({id, plot}); + const auto entry_order = [](const auto& left, const auto& right) { + return left.id < right.id; + }; + std::ranges::sort(gallery_2d, entry_order); + std::ranges::sort(gallery_3d, entry_order); + + auto gallery_streams = std::make_shared(); + auto plot_media = std::make_shared< + std::unordered_map>(); + const auto add_media_group = [&gallery_streams, &plot_media, + &gallery_clock_pool, + &gallery_encoding_pool]( + std::string id, + std::vector entries) { + const auto path = "/ws/gallery?group=" + id; + for (const auto& entry : entries) + plot_media->emplace(entry.id, path); + gallery_streams->emplace( + std::move(id), Gallery_Video_Stream::create( + gallery_clock_pool->get_executor(), + gallery_encoding_pool->get_executor(), std::move(entries))); + }; + constexpr std::size_t plots_per_media_group{4}; + const auto add_media_groups = [&add_media_group]( + std::string_view prefix, + std::vector entries) { + std::size_t group_index{}; + for (std::size_t offset = 0; offset < entries.size(); + offset += plots_per_media_group) { + const auto end = std::min( + entries.size(), offset + plots_per_media_group); + std::vector group( + std::make_move_iterator(entries.begin() + + static_cast(offset)), + std::make_move_iterator(entries.begin() + + static_cast(end))); + add_media_group(std::string{prefix} + "-" + + std::to_string(++group_index), std::move(group)); + } + }; + add_media_groups("2d", std::move(gallery_2d)); + add_media_groups("3d", std::move(gallery_3d)); auto resolve_plot = [plots](std::string_view id) { return find_plot(*plots, id); }; auto websocket = std::make_shared(resolve_plot); + auto gallery_websocket = std::make_shared( + [gallery_streams](std::string_view id) { + const auto found = gallery_streams->find(std::string{id}); + return found == gallery_streams->end() ? nullptr : found->second; + }); auto& app = drogon::app(); - app.registerHandler("/plot", [plots](const drogon::HttpRequestPtr&, + app.registerHandler("/plot", [plots, plot_media](const drogon::HttpRequestPtr&, std::function&& callback) { nlohmann::json result = nlohmann::json::array(); for (const auto& [id, plot] : *plots) { @@ -82,6 +149,7 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) result.push_back({ {"id", id}, {"title", id}, {"category", "Plots"}, {"description", ""}, {"dimension", id.starts_with("datoviz_") ? "3D" : "2D"}, {"websocket", "/ws/plot/" + id}, + {"media", plot_media->at(id)}, {"schema", "/plot/" + id + "/schema"}}); } callback(json_response(std::move(result))); @@ -156,6 +224,7 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) }, {drogon::Post}); app.registerController(websocket) + .registerController(gallery_websocket) .setDocumentRoot(asset_root.string()) .setHomePage("index.html") .setStaticFileHeaders({{"Cache-Control", "no-store"}}) @@ -163,10 +232,20 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) .setThreadNum(std::min(8U, hardware_threads)) .setIdleConnectionTimeout(90) .run(); + /* 先解除 Plot 完成帧订阅并停止页面时钟,再停止其执行资源。 + * 这样销毁期不存在回调访问已释放图集、计时器跨线程 cancel 或半途编码。 */ + for (const auto& [id, stream] : *gallery_streams) { + static_cast(id); + stream->shutdown(); + } plot_2d_pool->stop(); plot_3d_pool->stop(); + gallery_clock_pool->stop(); + gallery_encoding_pool->stop(); plot_2d_pool->join(); plot_3d_pool->join(); + gallery_clock_pool->join(); + gallery_encoding_pool->join(); return 0; } } diff --git a/web_server/src/detail/Gallery_Frame_Atlas.cpp b/web_server/src/detail/Gallery_Frame_Atlas.cpp new file mode 100644 index 0000000..a282df2 --- /dev/null +++ b/web_server/src/detail/Gallery_Frame_Atlas.cpp @@ -0,0 +1,167 @@ +#include "Gallery_Frame_Atlas.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web::detail { +struct Gallery_Frame_Atlas::Private { + struct Source { + mutable std::mutex mutex; /* 仅保护当前 Plot 的最近完成快照。 */ + std::shared_ptr latest_completion; /* 最近逻辑完成身份;像素可为空。 */ + std::shared_ptr latest_pixels; /* 最近可合成的实际 RGBA 画面。 */ + std::uint64_t composited_rendered_sequence{}; /* 图集像素当前包含的真实画面序号。 */ + std::uint64_t completion_count{}; /* 当前槽位接受的逻辑完成回调总数。 */ + std::uint64_t rendered_frame_count{}; /* 当前槽位接受的不同真实画面总数。 */ + }; + + Gallery_Atlas_Description description{}; /* 尺寸与槽位布局的唯一权威描述。 */ + std::vector> sources{}; /* 各 Plot 独立锁定的最近完成快照。 */ + std::vector rgba{}; /* 编码器直接读取的持久 RGBA8 图集。 */ + std::atomic_uint64_t rejected_frame_count{}; /* 两次 compose 之间拒绝的帧数。 */ + + Private(std::uint32_t tile_width, std::uint32_t tile_height, + std::uint32_t columns, std::vector source_ids) + : description{tile_width, tile_height, columns} { + if (tile_width == 0 || tile_height == 0 || columns == 0 || + source_ids.empty()) + throw std::invalid_argument( + "gallery atlas dimensions and source list must be non-empty"); + if ((tile_width & 1U) != 0 || (tile_height & 1U) != 0) + throw std::invalid_argument( + "gallery atlas tile dimensions must be even for H.264"); + description.rows = static_cast( + (source_ids.size() + columns - 1U) / columns); + if (tile_width > std::numeric_limits::max() / columns || + tile_height > std::numeric_limits::max() / + description.rows) + throw std::overflow_error("gallery atlas dimensions overflow"); + description.width = tile_width * columns; + description.height = tile_height * description.rows; + description.sources.reserve(source_ids.size()); + sources.reserve(source_ids.size()); + for (std::size_t slot = 0; slot < source_ids.size(); ++slot) { + Gallery_Atlas_Source_Description source{ + std::move(source_ids[slot]), slot, + static_cast(slot % columns), + static_cast(slot / columns)}; + description.sources.push_back(std::move(source)); + sources.push_back(std::make_unique()); + } + const auto pixel_count = static_cast(description.width) * + description.height; + if (pixel_count > std::numeric_limits::max() / 4U) + throw std::overflow_error("gallery atlas byte size overflow"); + rgba.resize(pixel_count * 4U); + for (std::size_t offset = 3; offset < rgba.size(); offset += 4U) + rgba[offset] = std::byte{255}; + } +}; + +Gallery_Frame_Atlas::Gallery_Frame_Atlas( + std::uint32_t tile_width, std::uint32_t tile_height, + std::uint32_t columns, std::vector source_ids) + : d(std::make_unique(tile_width, tile_height, columns, + std::move(source_ids))) {} + +Gallery_Frame_Atlas::~Gallery_Frame_Atlas() = default; + +Gallery_Frame_Atlas::Accept_Frame_Result Gallery_Frame_Atlas::accept_frame( + std::size_t slot, std::shared_ptr frame) { + if (slot >= d->sources.size()) + throw std::out_of_range("gallery atlas source slot is invalid"); + const auto expected = static_cast(d->description.tile_width) * + d->description.tile_height * 4U; + if (!frame || frame->width != d->description.tile_width || + frame->height != d->description.tile_height || + (frame->rgba && frame->rgba->size() != expected)) { + d->rejected_frame_count.fetch_add(1, std::memory_order_relaxed); + return Accept_Frame_Result::invalid_frame; + } + auto& source = *d->sources[slot]; + std::lock_guard lock(source.mutex); + if (source.latest_completion && + frame->sequence <= source.latest_completion->sequence) { + d->rejected_frame_count.fetch_add(1, std::memory_order_relaxed); + return Accept_Frame_Result::stale_frame; + } + if (frame->rendered_sequence != 0 && + (!source.latest_completion || + frame->rendered_sequence != + source.latest_completion->rendered_sequence)) + ++source.rendered_frame_count; + source.latest_completion = frame; + if (frame->rgba && frame->rendered_sequence != 0) + source.latest_pixels = std::move(frame); + ++source.completion_count; + return Accept_Frame_Result::accepted; +} + +Gallery_Atlas_Composition Gallery_Frame_Atlas::compose() { + Gallery_Atlas_Composition result; + result.width = d->description.width; + result.height = d->description.height; + result.rejected_frame_count = + d->rejected_frame_count.exchange(0, std::memory_order_acq_rel); + result.sources.reserve(d->sources.size()); + const auto source_row_bytes = + static_cast(d->description.tile_width) * 4U; + for (std::size_t slot = 0; slot < d->sources.size(); ++slot) { + auto& source = *d->sources[slot]; + std::shared_ptr latest_completion; + std::shared_ptr latest_pixels; + std::uint64_t completion_count{}; + std::uint64_t rendered_frame_count{}; + { + std::lock_guard lock(source.mutex); + latest_completion = source.latest_completion; + latest_pixels = source.latest_pixels; + completion_count = source.completion_count; + rendered_frame_count = source.rendered_frame_count; + } + if (!latest_completion) { + ++result.missing_tile_count; + result.sources.push_back({}); + continue; + } + if (!latest_pixels) { + ++result.missing_tile_count; + } + else if (latest_pixels->rendered_sequence != + source.composited_rendered_sequence) { + const auto& layout = d->description.sources[slot]; + for (std::uint32_t y = 0; y < d->description.tile_height; ++y) { + const auto source_offset = + static_cast(y) * source_row_bytes; + const auto destination_offset = + (static_cast( + layout.row * d->description.tile_height + y) * + d->description.width + + layout.column * d->description.tile_width) * 4U; + std::memcpy(d->rgba.data() + destination_offset, + latest_pixels->rgba->data() + source_offset, + source_row_bytes); + } + source.composited_rendered_sequence = + latest_pixels->rendered_sequence; + ++result.fresh_tile_count; + } + result.sources.push_back({ + latest_completion->sequence, + latest_completion->correlation_id, + latest_completion->rendered_sequence, + latest_completion->rendered_correlation_id, + completion_count, + rendered_frame_count}); + } + result.rgba = d->rgba; + return result; +} + +Gallery_Atlas_Description Gallery_Frame_Atlas::describe() const { + return d->description; +} +} diff --git a/web_server/src/detail/Gallery_Frame_Atlas.hpp b/web_server/src/detail/Gallery_Frame_Atlas.hpp new file mode 100644 index 0000000..b577929 --- /dev/null +++ b/web_server/src/detail/Gallery_Frame_Atlas.hpp @@ -0,0 +1,72 @@ +#pragma once +#include "../Plot.hpp" +#include +#include +#include +#include +#include +#include + +namespace aethera::web::detail { +struct Gallery_Atlas_Source_Description { + std::string id; /* 图集槽位对应的 Plot 业务标识。 */ + std::size_t slot{}; /* 从左到右、从上到下的稳定槽位序号。 */ + std::uint32_t column{}; /* 浏览器裁剪使用的零起始列号。 */ + std::uint32_t row{}; /* 浏览器裁剪使用的零起始行号。 */ +}; + +struct Gallery_Atlas_Description { + std::uint32_t tile_width{}; /* 单个 Plot 完成帧的固定像素宽度。 */ + std::uint32_t tile_height{}; /* 单个 Plot 完成帧的固定像素高度。 */ + std::uint32_t columns{}; /* 图集固定列数。 */ + std::uint32_t rows{}; /* 容纳全部 Plot 后计算得到的行数。 */ + std::uint32_t width{}; /* H.264 输入图集的像素宽度。 */ + std::uint32_t height{}; /* H.264 输入图集的像素高度。 */ + std::vector sources; /* 每个 Plot 在图集中的唯一布局描述。 */ +}; + +struct Gallery_Atlas_Source_Progress { + std::uint64_t completion_sequence{}; /* 当前槽位最近一次逻辑完成回调的局部序号。 */ + std::uint64_t completion_correlation_id{}; /* 最近逻辑回调对应的页面时钟序号。 */ + std::uint64_t rendered_sequence{}; /* 当前像素对应的真实 Scene/GPU 画面序号。 */ + std::uint64_t rendered_correlation_id{}; /* 当前真实画面对应的页面时钟序号。 */ + std::uint64_t completion_count{}; /* 本图集生命周期内接受的逻辑完成回调数。 */ + std::uint64_t rendered_frame_count{}; /* 本图集生命周期内接受的不同真实画面数。 */ +}; + +struct Gallery_Atlas_Composition { + std::span rgba{}; /* 在下一次 compose 前有效的连续 RGBA8 图集。 */ + std::uint32_t width{}; /* 本次图集像素宽度。 */ + std::uint32_t height{}; /* 本次图集像素高度。 */ + std::uint32_t fresh_tile_count{}; /* 本次实际复制了新完成帧的槽位数。 */ + std::uint32_t missing_tile_count{}; /* 尚未收到首帧的槽位数。 */ + std::uint64_t rejected_frame_count{}; /* 上次 compose 后拒绝的无效或过期帧数。 */ + std::vector sources; /* 合成时刻各槽位的完成进度快照。 */ +}; + +class Gallery_Frame_Atlas final { +public: + enum class Accept_Frame_Result : std::uint8_t { + accepted, + invalid_frame, + stale_frame + }; + + Gallery_Frame_Atlas(std::uint32_t tile_width, + std::uint32_t tile_height, + std::uint32_t columns, + std::vector source_ids); + ~Gallery_Frame_Atlas(); + Gallery_Frame_Atlas(const Gallery_Frame_Atlas&) = delete; + Gallery_Frame_Atlas& operator=(const Gallery_Frame_Atlas&) = delete; + + [[nodiscard]] Accept_Frame_Result accept_frame( + std::size_t slot, std::shared_ptr frame); + [[nodiscard]] Gallery_Atlas_Composition compose(); + [[nodiscard]] Gallery_Atlas_Description describe() const; + +private: + struct Private; + std::unique_ptr d; +}; +} diff --git a/web_server/src/detail/Gallery_Frame_Clock.cpp b/web_server/src/detail/Gallery_Frame_Clock.cpp new file mode 100644 index 0000000..091527e --- /dev/null +++ b/web_server/src/detail/Gallery_Frame_Clock.cpp @@ -0,0 +1,146 @@ +#include "Gallery_Frame_Clock.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web::detail { +struct Gallery_Frame_Clock::Private + : std::enable_shared_from_this { + asio::strand strand; + asio::steady_timer timer; + std::chrono::nanoseconds interval{}; /* 相邻页面级 tick 的固定单调时间间隔。 */ + Tick_Handler tick_handler{}; /* 每个绝对时钟点唯一的业务消费出口。 */ + Failure_Handler failure_handler{}; /* 计时任务 Unknown Failure 的隔离出口。 */ + std::atomic_bool requested{}; /* 跨线程 start/stop 的唯一请求状态。 */ + bool running{}; /* 仅在 strand 上读写的计时器运行状态。 */ + std::chrono::steady_clock::time_point origin{}; /* 当前运行周期媒体时间戳的绝对原点。 */ + + Private(asio::any_io_executor executor, double frame_rate, + Tick_Handler value_tick_handler, + Failure_Handler value_failure_handler) + : strand(asio::make_strand(std::move(executor))), timer(strand), + tick_handler(std::move(value_tick_handler)), + failure_handler(std::move(value_failure_handler)) { + if (!std::isfinite(frame_rate) || frame_rate <= 0.0) + throw std::invalid_argument("gallery frame rate must be positive"); + interval = std::chrono::nanoseconds{ + static_cast(std::llround(1'000'000'000.0 / frame_rate))}; + if (interval.count() <= 0) + throw std::invalid_argument("gallery frame rate exceeds clock resolution"); + if (!tick_handler) + throw std::invalid_argument("gallery frame clock requires a tick handler"); + } + + void begin() { + if (!requested.load(std::memory_order_acquire) || running) return; + running = true; + origin = std::chrono::steady_clock::now(); + timer.expires_at(origin + interval); + arm(); + } + + void cancel() { + running = false; + timer.cancel(); + } + + void report(std::exception_ptr failure) noexcept { + running = false; + requested.store(false, std::memory_order_release); + try { timer.cancel(); } + catch (...) {} + if (!failure_handler) return; + try { failure_handler(std::move(failure)); } + catch (...) {} + } + + void arm() { + const std::weak_ptr weak = shared_from_this(); + timer.async_wait([weak](const asio::error_code& error) { + const auto self = weak.lock(); + if (!self) return; + if (error) { + if (error != asio::error::operation_aborted) + self->report(std::make_exception_ptr(std::runtime_error( + "gallery frame timer failed: " + error.message()))); + return; + } + if (!self->running || + !self->requested.load(std::memory_order_acquire)) return; + self->fire(); + }); + } + + void fire() { + try { + const auto deadline = timer.expiry(); + const auto elapsed = deadline - origin; + const auto sequence = static_cast(elapsed / interval); + auto next = deadline + interval; + const auto now = std::chrono::steady_clock::now(); + if (next <= now) { + const auto missed = (now - next) / interval + 1; + next += interval * missed; + } + timer.expires_at(next); + arm(); + tick_handler(Plot_Render_Tick{ + sequence, + std::chrono::duration(elapsed).count()}); + } + catch (...) { + report(std::current_exception()); + } + } +}; + +Gallery_Frame_Clock::Gallery_Frame_Clock( + asio::any_io_executor executor, double frame_rate, + Tick_Handler tick_handler, Failure_Handler failure_handler) + : d(std::make_shared(std::move(executor), frame_rate, + std::move(tick_handler), + std::move(failure_handler))) {} + +Gallery_Frame_Clock::~Gallery_Frame_Clock() { + stop(); + d.reset(); +} + +void Gallery_Frame_Clock::start() { + if (d->requested.exchange(true, std::memory_order_acq_rel)) return; + const std::weak_ptr weak = d; + try { + asio::post(d->strand, [weak] { + if (const auto self = weak.lock()) { + try { self->begin(); } + catch (...) { self->report(std::current_exception()); } + } + }); + } + catch (...) { + d->requested.store(false, std::memory_order_release); + throw; + } +} + +void Gallery_Frame_Clock::stop() noexcept { + if (!d || !d->requested.exchange(false, std::memory_order_acq_rel)) return; + const std::weak_ptr weak = d; + try { + asio::post(d->strand, [weak] { + if (const auto self = weak.lock()) { + try { self->cancel(); } + catch (...) { self->report(std::current_exception()); } + } + }); + } + catch (...) {} +} +} diff --git a/web_server/src/detail/Gallery_Frame_Clock.hpp b/web_server/src/detail/Gallery_Frame_Clock.hpp new file mode 100644 index 0000000..92b09c3 --- /dev/null +++ b/web_server/src/detail/Gallery_Frame_Clock.hpp @@ -0,0 +1,28 @@ +#pragma once +#include "../Plot.hpp" +#include +#include +#include +#include + +namespace aethera::web::detail { +class Gallery_Frame_Clock final { +public: + using Tick_Handler = std::function; + using Failure_Handler = std::function; + + Gallery_Frame_Clock(asio::any_io_executor executor, double frame_rate, + Tick_Handler tick_handler, + Failure_Handler failure_handler); + ~Gallery_Frame_Clock(); + Gallery_Frame_Clock(const Gallery_Frame_Clock&) = delete; + Gallery_Frame_Clock& operator=(const Gallery_Frame_Clock&) = delete; + + void start(); + void stop() noexcept; + +private: + struct Private; + std::shared_ptr d; /* 异步计时操作完成前保持时钟状态存活。 */ +}; +} diff --git a/web_server/tests/Gallery_Frame_Atlas_Tests.cpp b/web_server/tests/Gallery_Frame_Atlas_Tests.cpp new file mode 100644 index 0000000..5d5d686 --- /dev/null +++ b/web_server/tests/Gallery_Frame_Atlas_Tests.cpp @@ -0,0 +1,187 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web::detail { +namespace { +std::shared_ptr solid_frame( + std::uint64_t sequence, std::uint64_t correlation_id, + std::uint32_t width, std::uint32_t height, + std::byte red, std::byte green, std::byte blue, + std::uint64_t rendered_sequence = 0, + std::uint64_t rendered_correlation_id = 0) { + auto pixels = std::make_shared>( + static_cast(width) * height * 4U); + for (std::size_t offset = 0; offset < pixels->size(); offset += 4U) { + (*pixels)[offset] = red; + (*pixels)[offset + 1U] = green; + (*pixels)[offset + 2U] = blue; + (*pixels)[offset + 3U] = std::byte{255}; + } + return std::make_shared(Plot_Pixel_Frame{ + std::move(pixels), {}, sequence, correlation_id, + rendered_sequence == 0 ? sequence : rendered_sequence, + rendered_correlation_id == 0 ? correlation_id + : rendered_correlation_id, + width, height}); +} + +std::size_t pixel_offset(const Gallery_Atlas_Description& description, + std::uint32_t x, std::uint32_t y) { + return (static_cast(y) * description.width + x) * 4U; +} +} + +TEST(Gallery_Frame_Atlas, Layout_And_Changed_Tiles_Have_One_Authority) { + Gallery_Frame_Atlas atlas(2, 2, 2, {"alpha", "beta", "gamma"}); + const auto description = atlas.describe(); + ASSERT_EQ(description.columns, 2U); + ASSERT_EQ(description.rows, 2U); + ASSERT_EQ(description.width, 4U); + ASSERT_EQ(description.height, 4U); + ASSERT_EQ(description.sources.size(), 3U); + EXPECT_EQ(description.sources[2].column, 0U); + EXPECT_EQ(description.sources[2].row, 1U); + + EXPECT_EQ(atlas.accept_frame( + 0, solid_frame(1, 11, 2, 2, std::byte{10}, + std::byte{20}, std::byte{30})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + EXPECT_EQ(atlas.accept_frame( + 2, solid_frame(7, 17, 2, 2, std::byte{70}, + std::byte{80}, std::byte{90})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + + const auto first = atlas.compose(); + ASSERT_EQ(first.rgba.size(), 4U * 4U * 4U); + EXPECT_EQ(first.fresh_tile_count, 2U); + EXPECT_EQ(first.missing_tile_count, 1U); + EXPECT_EQ(first.sources[0].rendered_correlation_id, 11U); + EXPECT_EQ(first.sources[2].completion_count, 1U); + EXPECT_EQ(first.sources[2].rendered_frame_count, 1U); + const auto alpha = pixel_offset(description, 0, 0); + const auto gamma = pixel_offset(description, 0, 2); + EXPECT_EQ(first.rgba[alpha], std::byte{10}); + EXPECT_EQ(first.rgba[alpha + 3U], std::byte{255}); + EXPECT_EQ(first.rgba[gamma], std::byte{70}); + EXPECT_EQ(first.rgba[gamma + 2U], std::byte{90}); + + const auto unchanged = atlas.compose(); + EXPECT_EQ(unchanged.fresh_tile_count, 0U); + EXPECT_EQ(unchanged.missing_tile_count, 1U); +} + +TEST(Gallery_Frame_Atlas, Invalid_And_Stale_Frames_Do_Not_Replace_Latest) { + Gallery_Frame_Atlas atlas(2, 2, 1, {"only"}); + EXPECT_EQ(atlas.accept_frame( + 0, solid_frame(4, 14, 2, 2, std::byte{4}, + std::byte{5}, std::byte{6})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + EXPECT_EQ(atlas.accept_frame( + 0, solid_frame(3, 13, 2, 2, std::byte{30}, + std::byte{31}, std::byte{32})), + Gallery_Frame_Atlas::Accept_Frame_Result::stale_frame); + EXPECT_EQ(atlas.accept_frame( + 0, solid_frame(5, 15, 4, 2, std::byte{50}, + std::byte{51}, std::byte{52})), + Gallery_Frame_Atlas::Accept_Frame_Result::invalid_frame); + const auto composition = atlas.compose(); + EXPECT_EQ(composition.rejected_frame_count, 2U); + EXPECT_EQ(composition.sources[0].completion_sequence, 4U); + EXPECT_EQ(composition.rgba[0], std::byte{4}); + EXPECT_EQ(atlas.compose().rejected_frame_count, 0U); +} + +TEST(Gallery_Frame_Atlas, Logical_History_Callback_Does_Not_Fake_A_New_Image) { + Gallery_Frame_Atlas atlas(2, 2, 1, {"only"}); + ASSERT_EQ(atlas.accept_frame( + 0, solid_frame(4, 14, 2, 2, std::byte{4}, + std::byte{5}, std::byte{6})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + ASSERT_EQ(atlas.compose().fresh_tile_count, 1U); + ASSERT_EQ(atlas.accept_frame( + 0, solid_frame(5, 15, 2, 2, std::byte{50}, + std::byte{51}, std::byte{52}, 4, 14)), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + const auto composition = atlas.compose(); + EXPECT_EQ(composition.fresh_tile_count, 0U); + EXPECT_EQ(composition.sources[0].completion_count, 2U); + EXPECT_EQ(composition.sources[0].rendered_frame_count, 1U); + EXPECT_EQ(composition.sources[0].completion_sequence, 5U); + EXPECT_EQ(composition.sources[0].rendered_sequence, 4U); + EXPECT_EQ(composition.rgba[0], std::byte{4}); +} + +TEST(Gallery_Frame_Atlas, Diagnostics_Only_Completion_Preserves_Progress_And_Image) { + Gallery_Frame_Atlas atlas(2, 2, 1, {"only"}); + ASSERT_EQ(atlas.accept_frame( + 0, solid_frame(1, 11, 2, 2, std::byte{7}, + std::byte{8}, std::byte{9})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + ASSERT_EQ(atlas.compose().fresh_tile_count, 1U); + + auto diagnostics = std::make_shared( + Plot_Pixel_Frame{{}, {}, 2, 12, 2, 12, 2, 2}); + ASSERT_EQ(atlas.accept_frame(0, std::move(diagnostics)), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + const auto composition = atlas.compose(); + EXPECT_EQ(composition.fresh_tile_count, 0U); + EXPECT_EQ(composition.missing_tile_count, 0U); + EXPECT_EQ(composition.sources[0].completion_sequence, 2U); + EXPECT_EQ(composition.sources[0].rendered_sequence, 2U); + EXPECT_EQ(composition.sources[0].completion_count, 2U); + EXPECT_EQ(composition.sources[0].rendered_frame_count, 2U); + EXPECT_EQ(composition.rgba[0], std::byte{7}); +} + +TEST(Gallery_Frame_Atlas, Plot_Producers_Accept_In_Parallel_While_Encoding_Composes) { + constexpr std::size_t source_count{8}; + constexpr std::uint64_t frame_count{500}; + std::vector ids; + ids.reserve(source_count); + for (std::size_t slot = 0; slot < source_count; ++slot) + ids.push_back("source-" + std::to_string(slot)); + Gallery_Frame_Atlas atlas(2, 2, 4, std::move(ids)); + + std::atomic_size_t completed_producers{}; + std::vector producers; + producers.reserve(source_count); + for (std::size_t slot = 0; slot < source_count; ++slot) { + producers.emplace_back([&, slot] { + for (std::uint64_t sequence = 1; sequence <= frame_count; + ++sequence) { + const auto color = std::byte{ + static_cast((slot + sequence) % 251U)}; + EXPECT_EQ(atlas.accept_frame( + slot, solid_frame( + sequence, sequence, 2, 2, color, + std::byte{static_cast(slot)}, + std::byte{0})), + Gallery_Frame_Atlas::Accept_Frame_Result::accepted); + } + completed_producers.fetch_add(1, std::memory_order_release); + }); + } + + while (completed_producers.load(std::memory_order_acquire) < source_count) + static_cast(atlas.compose()); + for (auto& producer : producers) producer.join(); + + const auto final = atlas.compose(); + ASSERT_EQ(final.sources.size(), source_count); + EXPECT_EQ(final.missing_tile_count, 0U); + EXPECT_EQ(final.rejected_frame_count, 0U); + for (const auto& source : final.sources) { + EXPECT_EQ(source.completion_sequence, frame_count); + EXPECT_EQ(source.rendered_sequence, frame_count); + EXPECT_EQ(source.completion_count, frame_count); + EXPECT_EQ(source.rendered_frame_count, frame_count); + } +} +} diff --git a/web_server/tests/Gallery_Frame_Clock_Tests.cpp b/web_server/tests/Gallery_Frame_Clock_Tests.cpp new file mode 100644 index 0000000..b990e2c --- /dev/null +++ b/web_server/tests/Gallery_Frame_Clock_Tests.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web::detail { +TEST(Gallery_Frame_Clock, Produces_One_Monotonic_Absolute_Timeline) { + asio::thread_pool pool(1); + std::mutex mutex; + std::condition_variable condition; + std::vector ticks; + std::exception_ptr failure; + { + Gallery_Frame_Clock clock( + pool.get_executor(), 100.0, + [&](Plot_Render_Tick tick) { + std::lock_guard lock(mutex); + ticks.push_back(tick); + condition.notify_all(); + }, + [&](std::exception_ptr value) { + std::lock_guard lock(mutex); + failure = std::move(value); + condition.notify_all(); + }); + clock.start(); + clock.start(); + { + std::unique_lock lock(mutex); + ASSERT_TRUE(condition.wait_for(lock, std::chrono::seconds(2), [&] { + return ticks.size() >= 8 || failure; + })); + ASSERT_FALSE(failure); + ASSERT_GE(ticks.size(), 8U); + for (std::size_t index = 1; index < ticks.size(); ++index) { + EXPECT_GT(ticks[index].sequence, ticks[index - 1].sequence); + EXPECT_GT(ticks[index].time_milliseconds, + ticks[index - 1].time_milliseconds); + } + EXPECT_NEAR(ticks.front().time_milliseconds, + static_cast(ticks.front().sequence) * 10.0, + 0.001); + } + clock.stop(); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + std::size_t stopped_count{}; + { + std::lock_guard lock(mutex); + stopped_count = ticks.size(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + { + std::lock_guard lock(mutex); + EXPECT_EQ(ticks.size(), stopped_count); + } + } + pool.stop(); + pool.join(); +} +} diff --git a/web_server/tests/H264_Encoder_Tests.cpp b/web_server/tests/H264_Encoder_Tests.cpp new file mode 100644 index 0000000..5764d37 --- /dev/null +++ b/web_server/tests/H264_Encoder_Tests.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace aethera::web { +namespace { +std::vector test_pattern(std::uint32_t width, + std::uint32_t height) { + std::vector result( + static_cast(width) * height * 4U); + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) { + const auto offset = + (static_cast(y) * width + x) * 4U; + result[offset] = std::byte{static_cast(x)}; + result[offset + 1U] = std::byte{static_cast(y)}; + result[offset + 2U] = + std::byte{static_cast(x ^ y)}; + result[offset + 3U] = std::byte{255}; + } + } + return result; +} +} + +TEST(H264_Encoder, Rejects_Invalid_Input_Before_FFmpeg) { + H264_Encoder encoder(100.0); + const auto pixels = test_pattern(320, 192); + EXPECT_THROW(static_cast(encoder.encode( + pixels, 319, 192, Video_Pixel_Layout::rgba, + 1, std::chrono::microseconds{10'000})), + std::invalid_argument); + EXPECT_THROW(static_cast(encoder.encode( + std::span{pixels}.first(pixels.size() - 1U), + 320, 192, Video_Pixel_Layout::rgba, 1, + std::chrono::microseconds{10'000})), + std::invalid_argument); +} + +TEST(H264_Encoder, Hardware_Backend_Produces_Annex_B_At_One_Hundred_Fps) { + constexpr std::uint32_t width{1'920}; + constexpr std::uint32_t height{768}; + H264_Encoder encoder(100.0); + auto pixels = test_pattern(width, height); + std::optional first; + for (std::uint64_t sequence = 1; sequence <= 4 && !first; ++sequence) { + first = encoder.encode( + pixels, width, height, Video_Pixel_Layout::rgba, sequence, + std::chrono::microseconds{static_cast(sequence * 10'000)}); + } + ASSERT_TRUE(first); + ASSERT_FALSE(first->annex_b.empty()); + EXPECT_TRUE(first->key_frame); + ASSERT_GE(first->annex_b.size(), 4U); + EXPECT_EQ(first->annex_b[0], std::byte{0}); + EXPECT_EQ(first->annex_b[1], std::byte{0}); + EXPECT_TRUE((first->annex_b[2] == std::byte{1}) || + (first->annex_b[2] == std::byte{0} && + first->annex_b[3] == std::byte{1})); + EXPECT_NE(video_encoder_backend_name(first->backend), "unknown"); + EXPECT_EQ(h264_profile_level_id(), "640033"); +} + +TEST(H264_Encoder, Hardware_Pipeline_Drains_A_Continuous_Frame_Sequence) { + constexpr std::uint32_t width{1'920}; + constexpr std::uint32_t height{768}; + H264_Encoder encoder(100.0); + auto pixels = test_pattern(width, height); + std::uint64_t produced{}; + std::uint64_t latest_sequence{}; + std::optional backend; + const auto started = std::chrono::steady_clock::now(); + for (std::uint64_t sequence = 1; sequence <= 120; ++sequence) { + pixels[0] = std::byte{static_cast(sequence)}; + const auto frame = encoder.encode( + pixels, width, height, Video_Pixel_Layout::rgba, sequence, + std::chrono::microseconds{ + static_cast(sequence * 10'000)}); + if (!frame) continue; + ++produced; + EXPECT_GT(frame->sequence, latest_sequence); + latest_sequence = frame->sequence; + EXPECT_FALSE(frame->annex_b.empty()); + if (!backend) backend = frame->backend; + EXPECT_EQ(frame->backend, *backend); + } + const auto elapsed = std::chrono::steady_clock::now() - started; + EXPECT_GE(produced, 115U); + EXPECT_GE(latest_sequence, 115U); + EXPECT_LE(elapsed, std::chrono::milliseconds{1'200}); +} +} diff --git a/webapp_gallery/src/app.tsx b/webapp_gallery/src/app.tsx index 5bdcce8..ccc3007 100644 --- a/webapp_gallery/src/app.tsx +++ b/webapp_gallery/src/app.tsx @@ -13,7 +13,7 @@ import "react-resizable/css/styles.css"; echarts.use([LineChart, DataZoomComponent, GridComponent, LegendComponent, TooltipComponent, CanvasRenderer]); -type Plot = {id: string; title: string; category: string; description: string; dimension: "2D" | "3D"; websocket: string; schema: string}; +type Plot = {id: string; title: string; category: string; description: string; dimension: "2D" | "3D"; websocket: string; media: string; schema: string}; type Option = {value: string; label: string}; type Editor = "boolean" | "integer" | "number" | "text" | "select" | "color" | "point2" | "size" | "rect" | "range" | "pen" | "brush" | "font" | "color-map" | "vector3" | "matrix4" | "marker-list" | "surface-marker-list" | "json"; type Color_Channel_Scale = "normalized" | "byte"; @@ -27,40 +27,58 @@ type Schema = {protocol: "aethera.plot.inspector"; version: 2; components: Compo type State_Histories = Record; type Stream_Status = "IDLE" | "CONNECTING" | "LIVE" | "OFFLINE"; type Frame_Pacing_Mode = "manual" | "fixed_rate" | "maximum_rate"; -type Frame_Delivery = "webrtc-video" | "diagnostics"; -type Pixel_Format = "yuv420p" | "h264-annex-b" | "h264"; -type Frame_Metadata = {kind: "frame_metadata"; protocol: "aethera.video.frame"; version: 7; sequence: number; correlation_id: number; +type Frame_Delivery = "gallery-video" | "diagnostics"; +type Pixel_Format = "rgba8" | "bgra8_premultiplied"; +type Frame_Metadata = {kind: "frame_metadata"; protocol: "aethera.video.frame"; version: 9; sequence: number; correlation_id: number; + rendered_sequence: number; rendered_correlation_id: number; delivery: Frame_Delivery; created_time_unix_ms: number; pixel: {width: number; height: number; format: Pixel_Format; native_format: Pixel_Format; supported_formats: Pixel_Format[]; byte_length: number}; pacing: {mode: Frame_Pacing_Mode; fixed_rate_fps: number; render_enabled: boolean; video_enabled: boolean}; trace: {clock: "steady_elapsed_ns"; markers: Record; measurements: Record}}; type Frame_Stage_Values = Record; -type Frame_Sample = {sequence: number; correlation_id: number; delivery: Frame_Delivery; generated_at_ms: number; received_at_ms: number; values: Frame_Stage_Values}; +type Frame_Sample = {sequence: number; correlation_id: number; rendered_sequence: number; rendered_correlation_id: number; + delivery: Frame_Delivery; generated_at_ms: number; received_at_ms: number; values: Frame_Stage_Values}; type Video_Playback_Metrics = {frame_rate_fps: number; presented_frames: number; dropped_frames: number; current_time_seconds: number; ready_state: number}; +type Gallery_Tile = {column: number; row: number}; +type Gallery_Layout = {kind: "gallery_layout"; protocol: "aethera.gallery.video"; version: 2; columns: number; rows: number; + tile_width: number; tile_height: number; width: number; height: number; plots: Record}; +type Gallery_Source_Metrics = {has_rendered_frame: boolean; logical_completion_rate_fps: number; rendered_frame_rate_fps: number; + logical_completion_count: number; rendered_frame_count: number; latest_completion_sequence: number; + latest_rendered_sequence: number; latest_rendered_clock_sequence: number; clock_lag_ticks: number}; +type Gallery_Transport_Metrics = {kind: "gallery_metrics"; protocol: "aethera.gallery.video"; version: 2; clock_sequence: number; + encoded_frame_count: number; encoder_backend: "nvenc" | "vulkan_video"; target_frame_rate_fps: number; clock_delivery_rate_fps: number; encoded_frame_rate_fps: number; + compose_average_ms: number; compose_p95_ms: number; encode_average_ms: number; encode_p95_ms: number; + publish_average_ms: number; publish_p95_ms: number; encoded_bytes: number; fresh_tiles: number; missing_tiles: number; + rejected_frames: number; skipped_encode_ticks: number; sources: Record}; +type Gallery_Video_State = {status: Stream_Status; stream: MediaStream | null; layout: Gallery_Layout | null; + playback: Video_Playback_Metrics; transport: Gallery_Transport_Metrics | null; error: string | null}; +type Gallery_Video_States = Record; type Frame_Diagnostics = {metadata: Frame_Metadata; samples: Frame_Sample[]; frame_rate_fps: number; delivery_frame_rate_fps: number; interval_jitter_p95_ms: number; - request_to_pixels_average_ms: number; request_to_pixels_p50_ms: number; request_to_pixels_p95_ms: number; request_to_pixels_p99_ms: number; + server_completion_average_ms: number; server_completion_p50_ms: number; server_completion_p95_ms: number; server_completion_p99_ms: number; frame_interval_average_ms: number; frame_interval_p95_ms: number; dropped_sequence_count: number; latest: Frame_Stage_Values; video_playback: Video_Playback_Metrics}; -type Frame_Metrics = {sequence: number; generated_time_unix_ms: number; request_to_pixels_ms: number; average_request_to_pixels_ms: number; - p95_request_to_pixels_ms: number; p99_request_to_pixels_ms: number; frame_rate_fps: number; p95_frame_interval_jitter_ms: number; +type Frame_Metrics = {sequence: number; generated_time_unix_ms: number; server_completion_ms: number; average_server_completion_ms: number; + p95_server_completion_ms: number; p99_server_completion_ms: number; frame_rate_fps: number; p95_frame_interval_jitter_ms: number; pacing_mode: Frame_Pacing_Mode; fixed_rate_fps: number; delivery: Frame_Delivery; video_playback: Video_Playback_Metrics}; type Stage_Statistic = "average" | "variability" | "p95" | "p99"; type Stage_Unit = "value" | "percentage"; const default_plot_execution_policy = (): Plot_Execution_Policy => ({visible: true}); -const page_instance_id = crypto.randomUUID(); +const page_instance_storage_key = "aethera.gallery.page-instance"; -function video_viewport(video: HTMLVideoElement) { - const bounds = video.getBoundingClientRect(); - const quantize = (value: number, minimum: number, maximum: number) => - Math.max(minimum, Math.min(maximum, Math.round(value / 32) * 32)); - return { - width: quantize(bounds.width * devicePixelRatio, 160, 1920), - height: quantize(bounds.height * devicePixelRatio, 128, 1080) - }; +function page_instance_identity() { + const existing = sessionStorage.getItem(page_instance_storage_key); + if (existing) return existing; + const created = crypto.randomUUID(); + sessionStorage.setItem(page_instance_storage_key, created); + return created; } +// 一个浏览器标签页是媒体和诊断连接的所有者。刷新仍属于同一标签页, +// 因而必须复用身份,让新连接可在旧 WebSocket 异步关闭期间直接接管。 +const page_instance_id = page_instance_identity(); + function socket_url(path: string) { const url = new URL(path, location.href); url.protocol = location.protocol === "https:" ? "wss:" : "ws:"; @@ -71,15 +89,37 @@ function socket_url(path: string) { function valid_frame_metadata(value: unknown): value is Frame_Metadata { if (!value || typeof value !== "object") return false; const frame = value as Partial; - return frame.kind === "frame_metadata" && frame.protocol === "aethera.video.frame" && frame.version === 7 + return frame.kind === "frame_metadata" && frame.protocol === "aethera.video.frame" && frame.version === 9 && typeof frame.sequence === "number" && typeof frame.correlation_id === "number" - && (frame.delivery === "webrtc-video" || frame.delivery === "diagnostics") - && Boolean(frame.pixel) && frame.pixel?.format === "yuv420p" - && frame.pixel?.native_format === "h264-annex-b" + && typeof frame.rendered_sequence === "number" && typeof frame.rendered_correlation_id === "number" + && (frame.delivery === "gallery-video" || frame.delivery === "diagnostics") + && Boolean(frame.pixel) + && (frame.pixel?.format === "rgba8" || frame.pixel?.format === "bgra8_premultiplied") + && (frame.pixel?.native_format === "rgba8" || frame.pixel?.native_format === "bgra8_premultiplied") && Array.isArray(frame.pixel?.supported_formats) && Boolean(frame.trace); } +function valid_gallery_layout(value: unknown): value is Gallery_Layout { + if (!value || typeof value !== "object") return false; + const layout = value as Partial; + return layout.kind === "gallery_layout" && + layout.protocol === "aethera.gallery.video" && layout.version === 2 && + typeof layout.tile_width === "number" && typeof layout.tile_height === "number" && + typeof layout.width === "number" && typeof layout.height === "number" && + Boolean(layout.plots); +} + +function valid_gallery_metrics(value: unknown): value is Gallery_Transport_Metrics { + if (!value || typeof value !== "object") return false; + const metrics = value as Partial; + return metrics.kind === "gallery_metrics" && + metrics.protocol === "aethera.gallery.video" && metrics.version === 2 && + typeof metrics.encoded_frame_rate_fps === "number" && + typeof metrics.clock_delivery_rate_fps === "number" && + Boolean(metrics.sources); +} + function percentile(values: number[], ratio: number) { if (values.length === 0) return 0; const sorted = [...values].sort((left, right) => left - right); @@ -112,13 +152,11 @@ function marker_interval_ms(metadata: Frame_Metadata, start: string, finish: str return start_ms === null || finish_ms === null ? null : Math.max(0, finish_ms - start_ms); } -function frame_stage_values(metadata: Frame_Metadata, request_started_at: number, metadata_received_at: number, - pixels_received_at: number, canvas_upload_ms: number): Frame_Stage_Values { +function frame_stage_values(metadata: Frame_Metadata): Frame_Stage_Values { + const server_completion = marker_ms(metadata, "frame_ready") ?? 0; const values: Frame_Stage_Values = { - request_to_metadata_ms: Math.max(0, metadata_received_at - request_started_at), - request_to_pixels_ms: Math.max(0, pixels_received_at - request_started_at), - metadata_to_pixels_ms: Math.max(0, pixels_received_at - metadata_received_at), - canvas_upload_ms, + server_completion_ms: server_completion, + diagnostic_delivery_age_ms: Math.max(0, Date.now() - metadata.created_time_unix_ms), payload_megabytes: metadata.pixel.byte_length / (1024 * 1024) }; const marker_intervals: Array<[string, string, string]> = [ @@ -129,17 +167,12 @@ function frame_stage_values(metadata: Frame_Metadata, request_started_at: number ["backend_queue_ms", "backend_queue_entered", "backend_queue_left"], ["gpu_submission_ms", "gpu_submitted", "gpu_completed"], ["readback_stage_ms", "readback_started", "readback_finished"], - ["callback_ms", "callback_started", "callback_finished"], - ["video_encode_queue_ms", "video_encode_queued", "video_encode_started"], - ["video_encode_ms", "video_encode_started", "video_encode_finished"], - ["stream_publish_ms", "stream_publish_started", "stream_publish_finished"] + ["callback_ms", "callback_started", "frame_ready"] ]; for (const [key, start, finish] of marker_intervals) { const value = marker_interval_ms(metadata, start, finish); if (value !== null) values[key] = value; } - const server_completion = marker_ms(metadata, "stream_publish_started"); - if (server_completion !== null) values.server_to_stream_ready_ms = server_completion; for (const [key, value] of Object.entries(metadata.trace.measurements)) if (Number.isFinite(value)) values[key.endsWith("_ns") ? `${key.slice(0, -3)}_ms` : `${key}_ms`] = value / 1_000_000; for (const [key, value] of Object.entries(metadata.trace.markers)) if (Number.isFinite(value)) @@ -148,25 +181,15 @@ function frame_stage_values(metadata: Frame_Metadata, request_started_at: number } function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dimension"]): Frame_Stage_Values { - const total = Math.max(0, values.request_to_presentation_opportunity_ms ?? values.request_to_pixels_ms ?? 0); - const server = Math.min(total, Math.max(0, values.server_to_stream_ready_ms ?? 0)); - let remaining = server; + const total = Math.max(0, values.server_completion_ms ?? 0); + let remaining = total; const take = (requested: number) => { const value = Math.min(remaining, Math.max(0, requested)); remaining -= value; return value; }; const interval = (start: string, finish: string) => Math.max(0, (values[`marker_${finish}_ms`] ?? 0) - (values[`marker_${start}_ms`] ?? 0)); - const request_to_metadata = Math.min(total, Math.max(0, values.request_to_metadata_ms ?? 0)); - const metadata_to_pixels = Math.min(Math.max(0, total - request_to_metadata), Math.max(0, values.metadata_to_pixels_ms ?? 0)); - const canvas_upload = Math.min(Math.max(0, total - request_to_metadata - metadata_to_pixels), Math.max(0, values.canvas_upload_ms ?? 0)); - const publish_tail = Math.min( - Math.max(0, request_to_metadata - server), - Math.max(0, values.stream_publish_tail_ms ?? 0)); - const stages: Frame_Stage_Values = { - pipeline_request_transport_ms: Math.max( - 0, request_to_metadata - server - publish_tail) - }; + const stages: Frame_Stage_Values = {}; if (dimension === "2D") { const scene = Math.max(0, values.scene_render_ms ?? 0); const event = Math.min(scene, Math.max(0, values.event_dispatch_ms ?? 0)); @@ -177,10 +200,7 @@ function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dime stages.pipeline_2d_paint_ms = take(paint); stages.pipeline_2d_scene_coordination_ms = take(Math.max(0, scene - event - prepare - paint)); stages.pipeline_2d_callback_ms = take(values.callback_ms ?? 0); - stages.pipeline_2d_encode_queue_ms = take(values.video_encode_queue_ms ?? 0); - stages.pipeline_2d_encode_ms = take(values.video_encode_ms ?? 0); stages.pipeline_2d_frame_handoff_ms = remaining; - stages.pipeline_2d_publish_tail_ms = publish_tail; remaining = 0; } else { const scene = Math.max(0, values.scene_render_ms ?? 0); @@ -219,40 +239,37 @@ function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dime stages.pipeline_3d_gpu_sync_ms = take(gpu_window); stages.pipeline_3d_readback_ms = take(values.readback_stage_ms ?? values.readback_ms ?? 0); stages.pipeline_3d_callback_ms = take(values.callback_ms ?? 0); - stages.pipeline_3d_encode_queue_ms = take(values.video_encode_queue_ms ?? 0); - stages.pipeline_3d_encode_ms = take(values.video_encode_ms ?? 0); stages.pipeline_3d_completion_handoff_ms = remaining; - stages.pipeline_3d_publish_tail_ms = publish_tail; remaining = 0; } - return {...values, ...stages, - pipeline_payload_transport_ms: metadata_to_pixels, - pipeline_canvas_upload_ms: canvas_upload, - pipeline_presentation_wait_ms: Math.max(0, total - request_to_metadata - metadata_to_pixels - canvas_upload) - }; + return {...values, ...stages}; } function build_frame_diagnostics(metadata: Frame_Metadata, samples: Frame_Sample[]): Frame_Diagnostics { - const request_latencies = samples.map(sample => sample.values.request_to_pixels_ms).filter(Number.isFinite); + const completion_latencies = samples.map(sample => sample.values.server_completion_ms).filter(Number.isFinite); const intervals = samples.slice(1).map((sample, index) => sample.received_at_ms - samples[index].received_at_ms).filter(value => value >= 0); const median_interval = percentile(intervals, .5); const interval_jitter = intervals.map(value => Math.abs(value - median_interval)); + const sequence_steps = samples.slice(1).map((sample, index) => + Math.max(0, sample.sequence - samples[index].sequence)); + const expected_sequence_step = percentile(sequence_steps.filter(value => value > 0), .5); let dropped_sequence_count = 0; - for (let index = 1; index < samples.length; ++index) - dropped_sequence_count += Math.max(0, samples[index].correlation_id - samples[index - 1].correlation_id - 1); + for (const step of sequence_steps) + dropped_sequence_count += Math.max(0, step - expected_sequence_step); const delivery_elapsed = samples.length > 1 ? samples.at(-1)!.received_at_ms - samples[0].received_at_ms : 0; const source_elapsed = samples.length > 1 ? samples.at(-1)!.generated_at_ms - samples[0].generated_at_ms : 0; - const source_frames = samples.length > 1 ? samples.at(-1)!.correlation_id - samples[0].correlation_id : 0; + const source_frames = samples.length > 1 + ? Math.max(0, samples.at(-1)!.sequence - samples[0].sequence) : 0; return { metadata, samples, frame_rate_fps: source_elapsed > 0 ? source_frames * 1000 / source_elapsed : 0, delivery_frame_rate_fps: delivery_elapsed > 0 ? (samples.length - 1) * 1000 / delivery_elapsed : 0, interval_jitter_p95_ms: percentile(interval_jitter, .95), - request_to_pixels_average_ms: average(request_latencies), - request_to_pixels_p50_ms: percentile(request_latencies, .5), - request_to_pixels_p95_ms: percentile(request_latencies, .95), - request_to_pixels_p99_ms: percentile(request_latencies, .99), + server_completion_average_ms: average(completion_latencies), + server_completion_p50_ms: percentile(completion_latencies, .5), + server_completion_p95_ms: percentile(completion_latencies, .95), + server_completion_p99_ms: percentile(completion_latencies, .99), frame_interval_average_ms: average(intervals), frame_interval_p95_ms: percentile(intervals, .95), dropped_sequence_count, @@ -261,113 +278,259 @@ function build_frame_diagnostics(metadata: Frame_Metadata, samples: Frame_Sample }; } -function use_plot_stream(plot: Plot, video_ref: React.RefObject) { +const empty_video_playback = (): Video_Playback_Metrics => ({ + frame_rate_fps: 0, presented_frames: 0, dropped_frames: 0, + current_time_seconds: 0, ready_state: 0 +}); + +const connecting_gallery_video = (): Gallery_Video_State => ({ + status: "CONNECTING", stream: null, layout: null, + playback: empty_video_playback(), transport: null, error: null +}); + +function use_gallery_videos(plots: Plot[]): Gallery_Video_States { + const media = useMemo(() => [...new Set(plots.map(plot => plot.media))].sort(), [plots]); + const media_key = media.join("\n"); + const [states, set_states] = useState({}); + useEffect(() => { + if (media.length === 0) { + set_states({}); + return; + } + set_states(current => Object.fromEntries(media.map(endpoint => [ + endpoint, current[endpoint] ?? connecting_gallery_video() + ]))); + const cleanups = media.map(endpoint => { + let stopped = false; + let peer: RTCPeerConnection | null = null; + let signal_chain = Promise.resolve(); + let previous_time = performance.now(); + let previous_decoded = 0; + const update = (change: (current: Gallery_Video_State) => Gallery_Video_State) => + set_states(current => ({...current, + [endpoint]: change(current[endpoint] ?? connecting_gallery_video())})); + const socket = new ReconnectingWebSocket(socket_url(endpoint), [], { + minReconnectionDelay: 300, maxReconnectionDelay: 5000, + reconnectionDelayGrowFactor: 1.6, maxRetries: Number.POSITIVE_INFINITY + }); + const sample_stats = async () => { + if (!peer) return; + try { + const reports = await peer.getStats(); + let decoded = 0; + let dropped = 0; + let found = false; + reports.forEach(report => { + if (report.type !== "inbound-rtp" || report.kind !== "video") return; + const inbound = report as RTCInboundRtpStreamStats; + decoded = inbound.framesDecoded ?? 0; + dropped = inbound.framesDropped ?? 0; + found = true; + }); + if (!found) return; + const now = performance.now(); + const elapsed = now - previous_time; + const frame_rate_fps = elapsed > 0 + ? Math.max(0, decoded - previous_decoded) * 1000 / elapsed : 0; + previous_time = now; + previous_decoded = decoded; + update(current => ({...current, playback: { + frame_rate_fps, + presented_frames: decoded, + dropped_frames: dropped, + current_time_seconds: 0, + ready_state: current.stream ? 4 : 0 + }})); + } catch (error) { + if (!stopped) update(current => ({...current, status: "OFFLINE", + error: error instanceof Error ? error.message : "WebRTC statistics failed"})); + } + }; + const create_peer = () => { + peer?.close(); + peer = new RTCPeerConnection(); + previous_time = performance.now(); + previous_decoded = 0; + peer.onicecandidate = event => { + if (!event.candidate || socket.readyState !== WebSocket.OPEN) return; + socket.send(JSON.stringify({kind: "webrtc_candidate", + candidate: event.candidate.candidate, + mid: event.candidate.sdpMid ?? "0"})); + }; + peer.onconnectionstatechange = () => { + if (!peer || stopped) return; + if (peer.connectionState === "failed" || peer.connectionState === "closed") + update(current => ({...current, status: "OFFLINE", stream: null, + error: `WebRTC ${peer?.connectionState ?? "closed"}`})); + }; + peer.ontrack = event => { + const stream = event.streams[0] ?? new MediaStream([event.track]); + update(current => ({...current, stream, status: "LIVE", error: null})); + }; + }; + socket.onopen = () => { + create_peer(); + update(current => ({...current, status: "CONNECTING", error: null})); + }; + socket.onclose = () => { + peer?.close(); + peer = null; + if (!stopped) update(current => ({...current, status: "CONNECTING", stream: null})); + }; + socket.onmessage = event => { + if (typeof event.data !== "string") return; + let decoded: unknown; + try { decoded = JSON.parse(event.data); } catch { return; } + if (!decoded || typeof decoded !== "object") return; + const message = decoded as Record; + if (valid_gallery_layout(decoded)) { + update(current => ({...current, layout: decoded})); + window.dispatchEvent(new CustomEvent("aethera:gallery-layout", + {detail: {...decoded, media: endpoint}})); + return; + } + if (valid_gallery_metrics(decoded)) { + update(current => ({...current, transport: decoded})); + window.dispatchEvent(new CustomEvent("aethera:gallery-metrics", + {detail: {...decoded, media: endpoint}})); + return; + } + if (message.kind === "gallery_error") { + update(current => ({...current, status: "OFFLINE", + error: typeof message.message === "string" ? message.message : "Gallery video failed"})); + return; + } + if (message.kind === "exclusive_page_rejected") { + stopped = true; + update(current => ({...current, status: "OFFLINE", stream: null, + error: "Gallery is already owned by another page"})); + socket.close(1000, "Gallery is owned by another page"); + return; + } + const signal = decoded as {kind?: string; sdp?: string; candidate?: string; mid?: string}; + signal_chain = signal_chain.then(async () => { + if (!peer) return; + if (signal.kind === "webrtc_offer" && signal.sdp) { + await peer.setRemoteDescription({type: "offer", sdp: signal.sdp}); + const answer = await peer.createAnswer(); + await peer.setLocalDescription(answer); + if (socket.readyState === WebSocket.OPEN) + socket.send(JSON.stringify({kind: "webrtc_answer", sdp: answer.sdp})); + } else if (signal.kind === "webrtc_candidate" && signal.candidate) { + await peer.addIceCandidate({candidate: signal.candidate, + sdpMid: signal.mid ?? "0"}); + } + }).catch(error => update(current => ({...current, status: "OFFLINE", + error: error instanceof Error ? error.message : "WebRTC negotiation failed"}))); + }; + const stats_timer = window.setInterval(() => void sample_stats(), 500); + return () => { + stopped = true; + window.clearInterval(stats_timer); + peer?.close(); + socket.close(); + }; + }); + return () => cleanups.forEach(cleanup => cleanup()); + }, [media_key]); + return states; +} + +function use_plot_stream(plot: Plot, + surface_ref: React.RefObject, + shared_playback: Video_Playback_Metrics, + tile_width = 720, tile_height = 420) { const [status, set_status] = useState("CONNECTING"); const [metrics, set_metrics] = useState(null); + const [error, set_error] = useState(null); const socket_ref = useRef(null); const pending_pointer_move = useRef | null>(null); const pointer_frame = useRef(0); - const viewport_ref = useRef({width: 720, height: 420}); - const video_playback_ref = useRef({frame_rate_fps: 0, presented_frames: 0, dropped_frames: 0, current_time_seconds: 0, ready_state: 0}); - - const envelope = useCallback((kind: "input", event?: Record) => { - const video = video_ref.current; - if (!video) return null; - const bounds = video.getBoundingClientRect(); - if (bounds.width > 0 && bounds.height > 0) - viewport_ref.current = video_viewport(video); - return {kind, viewport: { - width: viewport_ref.current.width, height: viewport_ref.current.height - }, ...(event ? {event} : {})}; - }, [video_ref]); + const viewport_ref = useRef({width: tile_width, height: tile_height}); + const playback_ref = useRef(shared_playback); + useEffect(() => { playback_ref.current = shared_playback; }, [shared_playback]); + useEffect(() => { viewport_ref.current = {width: tile_width, height: tile_height}; }, [tile_width, tile_height]); const transmit = useCallback((kind: "input", event?: Record) => { const socket = socket_ref.current; - const message = envelope(kind, event); - if (socket?.readyState === WebSocket.OPEN && message) socket.send(JSON.stringify(message)); - }, [envelope]); + if (socket?.readyState !== WebSocket.OPEN) return; + socket.send(JSON.stringify({kind, viewport: viewport_ref.current, + ...(event ? {event} : {})})); + }, []); useEffect(() => { let stopped = false; let diagnostics_timer = 0; - let resize_timer = 0; let previous_diagnostics_publish_time = 0; let latest_metadata: Frame_Metadata | null = null; let samples: Frame_Sample[] = []; - let peer: RTCPeerConnection | null = null; - let signal_chain = Promise.resolve(); - let sent_viewport = {width: 0, height: 0}; const socket = new ReconnectingWebSocket(socket_url(plot.websocket), [], { - minReconnectionDelay: 300, - maxReconnectionDelay: 5000, - reconnectionDelayGrowFactor: 1.6, - maxRetries: Number.POSITIVE_INFINITY + minReconnectionDelay: 300, maxReconnectionDelay: 5000, + reconnectionDelayGrowFactor: 1.6, maxRetries: Number.POSITIVE_INFINITY }); socket_ref.current = socket; set_status("CONNECTING"); - const clear_diagnostics_timer = () => { if (diagnostics_timer) window.clearTimeout(diagnostics_timer); diagnostics_timer = 0; }; - const send_stream_control = () => { - if (socket.readyState !== WebSocket.OPEN) return; - const video = video_ref.current; - if (video) { - const bounds = video.getBoundingClientRect(); - if (bounds.width > 0 && bounds.height > 0) - viewport_ref.current = video_viewport(video); - } - if (sent_viewport.width === viewport_ref.current.width && - sent_viewport.height === viewport_ref.current.height) return; - sent_viewport = {...viewport_ref.current}; - socket.send(JSON.stringify({kind: "stream", viewport: viewport_ref.current})); + const clear_diagnostics_timer = () => { + if (diagnostics_timer) window.clearTimeout(diagnostics_timer); + diagnostics_timer = 0; }; const publish_diagnostics = (force: boolean) => { if (!latest_metadata || samples.length === 0) return; const now = performance.now(); const remaining = 250 - (now - previous_diagnostics_publish_time); if (!force && remaining > 0) { - if (!diagnostics_timer) diagnostics_timer = window.setTimeout(() => { diagnostics_timer = 0; publish_diagnostics(true); }, remaining); + if (!diagnostics_timer) + diagnostics_timer = window.setTimeout(() => { + diagnostics_timer = 0; + publish_diagnostics(true); + }, remaining); return; } clear_diagnostics_timer(); previous_diagnostics_publish_time = now; - const delivery_samples = samples.filter(sample => sample.delivery === latest_metadata!.delivery); + const delivery_samples = samples.filter( + sample => sample.delivery === latest_metadata!.delivery); if (delivery_samples.length === 0) return; - const diagnostics = {...build_frame_diagnostics(latest_metadata, delivery_samples), - video_playback: {...video_playback_ref.current}}; + const diagnostics = { + ...build_frame_diagnostics(latest_metadata, delivery_samples), + video_playback: {...playback_ref.current} + }; const latest = diagnostics.latest; - const latest_metrics: Frame_Metrics = { + set_metrics({ sequence: latest_metadata.sequence, generated_time_unix_ms: latest_metadata.created_time_unix_ms, - request_to_pixels_ms: latest.request_to_pixels_ms ?? 0, - average_request_to_pixels_ms: diagnostics.request_to_pixels_average_ms, - p95_request_to_pixels_ms: diagnostics.request_to_pixels_p95_ms, - p99_request_to_pixels_ms: diagnostics.request_to_pixels_p99_ms, + server_completion_ms: latest.server_completion_ms ?? 0, + average_server_completion_ms: diagnostics.server_completion_average_ms, + p95_server_completion_ms: diagnostics.server_completion_p95_ms, + p99_server_completion_ms: diagnostics.server_completion_p99_ms, frame_rate_fps: diagnostics.frame_rate_fps, p95_frame_interval_jitter_ms: diagnostics.interval_jitter_p95_ms, pacing_mode: latest_metadata.pacing.mode, fixed_rate_fps: latest_metadata.pacing.fixed_rate_fps, delivery: latest_metadata.delivery, video_playback: diagnostics.video_playback - }; - set_metrics(latest_metrics); - window.dispatchEvent(new CustomEvent("aethera-frame-diagnostics", {detail: {plot_id: plot.id, diagnostics}})); + }); + window.dispatchEvent(new CustomEvent("aethera-frame-diagnostics", { + detail: {plot_id: plot.id, diagnostics} + })); }; const complete_metadata = (metadata: Frame_Metadata) => { const received_at = performance.now(); - const server_age = Math.max(0, Date.now() - metadata.created_time_unix_ms); - const started_at = received_at - server_age; latest_metadata = metadata; const sample_capacity = plot.dimension === "2D" ? 900 : 600; samples = [...samples, { sequence: metadata.sequence, correlation_id: metadata.correlation_id, + rendered_sequence: metadata.rendered_sequence, + rendered_correlation_id: metadata.rendered_correlation_id, delivery: metadata.delivery, generated_at_ms: metadata.created_time_unix_ms, received_at_ms: received_at, - values: pipeline_stage_values(frame_stage_values(metadata, started_at, received_at, - received_at, 0), plot.dimension) + values: pipeline_stage_values(frame_stage_values(metadata), plot.dimension) }].slice(-sample_capacity); - const delivery_sample_count = samples.filter(sample => sample.delivery === metadata.delivery).length; - publish_diagnostics(delivery_sample_count === 1); + const delivery_count = samples.filter( + sample => sample.delivery === metadata.delivery).length; + publish_diagnostics(delivery_count === 1); }; const on_manual_frame = (event: Event) => { const detail = (event as CustomEvent<{plot_id: string}>).detail; @@ -376,11 +539,13 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { const detail = (event as CustomEvent<{plot_id: string}>).detail; - const video = video_ref.current; - if (detail?.plot_id !== plot.id || !video) return; - video.focus({preventScroll: true}); - transmit("input", {type: "key_press", key: "home", native_key: 36, modifiers: 0, auto_repeat: false}); - transmit("input", {type: "key_release", key: "home", native_key: 36, modifiers: 0, auto_repeat: false}); + const surface = surface_ref.current; + if (detail?.plot_id !== plot.id || !surface) return; + surface.focus({preventScroll: true}); + transmit("input", {type: "key_press", key: "home", native_key: 36, + modifiers: 0, auto_repeat: false}); + transmit("input", {type: "key_release", key: "home", native_key: 36, + modifiers: 0, auto_repeat: false}); }; const on_diagnostics_reset = (event: Event) => { const detail = (event as CustomEvent<{plot_id: string}>).detail; @@ -389,195 +554,187 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { - if (resize_timer) window.clearTimeout(resize_timer); - resize_timer = window.setTimeout(send_stream_control, 120); - }); - if (video_ref.current) resize_observer.observe(video_ref.current); - const create_peer = () => { - peer?.close(); - peer = new RTCPeerConnection(); - peer.onicecandidate = event => { - if (!event.candidate || socket.readyState !== WebSocket.OPEN) return; - socket.send(JSON.stringify({kind: "webrtc_candidate", candidate: event.candidate.candidate, - mid: event.candidate.sdpMid ?? "0"})); - }; - peer.ontrack = event => { - const video = video_ref.current; - if (!video) return; - video.srcObject = event.streams[0] ?? new MediaStream([event.track]); - void video.play(); - set_status("LIVE"); - }; - }; socket.onopen = () => { - create_peer(); - set_status("CONNECTING"); - sent_viewport = {width: 0, height: 0}; - send_stream_control(); + set_status("LIVE"); + set_error(null); + socket.send(JSON.stringify({kind: "stream", viewport: viewport_ref.current})); }; - socket.onclose = () => { peer?.close(); peer = null; if (!stopped) set_status("CONNECTING"); }; + socket.onclose = () => { if (!stopped) set_status("CONNECTING"); }; socket.onmessage = event => { if (typeof event.data !== "string") return; let decoded: unknown; try { decoded = JSON.parse(event.data); } catch { return; } - if (valid_frame_metadata(decoded)) { complete_metadata(decoded); return; } - if (!decoded || typeof decoded !== "object") return; - const signal = decoded as {kind?: string; sdp?: string; candidate?: string; mid?: string}; - if (signal.kind === "exclusive_page_rejected") { + if (valid_frame_metadata(decoded)) { + complete_metadata(decoded); + return; + } + const message = decoded as {kind?: string; message?: string}; + if (message?.kind === "plot_error") { + stopped = true; + set_status("OFFLINE"); + set_error(message.message ?? "Plot failed"); + socket.close(1011, "Plot failed"); + return; + } + if (message?.kind === "exclusive_page_rejected") { stopped = true; set_status("OFFLINE"); socket.close(1000, "Gallery is owned by another page"); - return; } - signal_chain = signal_chain.then(async () => { - if (!peer) return; - if (signal.kind === "webrtc_offer" && signal.sdp) { - await peer.setRemoteDescription({type: "offer", sdp: signal.sdp}); - const answer = await peer.createAnswer(); - await peer.setLocalDescription(answer); - if (socket.readyState === WebSocket.OPEN) - socket.send(JSON.stringify({kind: "webrtc_answer", sdp: answer.sdp})); - } else if (signal.kind === "webrtc_candidate" && signal.candidate) { - await peer.addIceCandidate({candidate: signal.candidate, sdpMid: signal.mid ?? "0"}); - } - }).catch(() => set_status("OFFLINE")); }; return () => { stopped = true; clear_diagnostics_timer(); - if (resize_timer) window.clearTimeout(resize_timer); - resize_observer.disconnect(); window.removeEventListener("aethera-manual-frame", on_manual_frame); window.removeEventListener("aethera-reset-camera", on_camera_reset); window.removeEventListener("aethera-reset-frame-diagnostics", on_diagnostics_reset); if (pointer_frame.current) cancelAnimationFrame(pointer_frame.current); pointer_frame.current = 0; - peer?.close(); socket_ref.current = null; socket.close(); }; - }, [plot.id, plot.websocket, plot.dimension, video_ref, transmit]); + }, [plot.id, plot.websocket, plot.dimension, surface_ref, transmit]); useEffect(() => { - const video = video_ref.current; - if (!video || typeof video.requestVideoFrameCallback !== "function") return; - let callback_id = 0; - let sample_started = performance.now(); - let sample_frames = 0; - const observe = (now: number, frame: VideoFrameCallbackMetadata) => { - const quality = video.getVideoPlaybackQuality(); - const presented_frames = frame.presentedFrames; - const elapsed = now - sample_started; - if (elapsed >= 500) { - video_playback_ref.current.frame_rate_fps = Math.max(0, presented_frames - sample_frames) * 1000 / elapsed; - sample_started = now; - sample_frames = presented_frames; - } - video_playback_ref.current.presented_frames = presented_frames; - video_playback_ref.current.dropped_frames = quality.droppedVideoFrames; - video_playback_ref.current.current_time_seconds = video.currentTime; - video_playback_ref.current.ready_state = video.readyState; - callback_id = video.requestVideoFrameCallback(observe); - }; - callback_id = video.requestVideoFrameCallback(observe); - return () => video.cancelVideoFrameCallback(callback_id); - }, [plot.id, video_ref]); - - useEffect(() => { - const video = video_ref.current; - if (!video) return; + const surface = surface_ref.current; + if (!surface) return; const point = (event: PointerEvent | WheelEvent) => { - const bounds = video.getBoundingClientRect(); + const bounds = surface.getBoundingClientRect(); return { x: (event.clientX - bounds.left) * viewport_ref.current.width / bounds.width, y: (event.clientY - bounds.top) * viewport_ref.current.height / bounds.height }; }; - const global_point = (event: PointerEvent | WheelEvent) => ({x: event.screenX * devicePixelRatio, y: event.screenY * devicePixelRatio}); + const global_point = (event: PointerEvent | WheelEvent) => ({ + x: event.screenX * devicePixelRatio, y: event.screenY * devicePixelRatio + }); const modifiers = (event: MouseEvent | KeyboardEvent) => - (event.ctrlKey ? 1 : 0) | (event.shiftKey ? 2 : 0) | (event.altKey ? 4 : 0) | (event.metaKey ? 8 : 0); - const button = (value: number) => value === 0 ? "left" : value === 1 ? "middle" : value === 2 ? "right" : "none"; - const active_button = (event: PointerEvent) => event.buttons & 1 ? "left" : event.buttons & 4 ? "middle" : event.buttons & 2 ? "right" : button(event.button); - const pointer_payload = (type: string, event: PointerEvent) => ({type, position: point(event), global_position: global_point(event), - button: type === "pointer_move" ? active_button(event) : button(event.button), buttons: event.buttons, modifiers: modifiers(event)}); + (event.ctrlKey ? 1 : 0) | (event.shiftKey ? 2 : 0) | + (event.altKey ? 4 : 0) | (event.metaKey ? 8 : 0); + const button = (value: number) => value === 0 ? "left" + : value === 1 ? "middle" : value === 2 ? "right" : "none"; + const active_button = (event: PointerEvent) => event.buttons & 1 ? "left" + : event.buttons & 4 ? "middle" : event.buttons & 2 ? "right" : button(event.button); + const pointer_payload = (type: string, event: PointerEvent) => ({ + type, position: point(event), global_position: global_point(event), + button: type === "pointer_move" ? active_button(event) : button(event.button), + buttons: event.buttons, modifiers: modifiers(event) + }); let drag_move_started = false; const on_pointer_move = (event: PointerEvent) => { event.stopPropagation(); const payload = pointer_payload("pointer_move", event); - if (event.buttons !== 0 && !drag_move_started) { drag_move_started = true; transmit("input", payload); return; } + if (event.buttons !== 0 && !drag_move_started) { + drag_move_started = true; + transmit("input", payload); + return; + } pending_pointer_move.current = payload; - if (!pointer_frame.current) pointer_frame.current = requestAnimationFrame(() => { - pointer_frame.current = 0; - if (pending_pointer_move.current) transmit("input", pending_pointer_move.current); - pending_pointer_move.current = null; - }); + if (!pointer_frame.current) + pointer_frame.current = requestAnimationFrame(() => { + pointer_frame.current = 0; + if (pending_pointer_move.current) + transmit("input", pending_pointer_move.current); + pending_pointer_move.current = null; + }); }; const on_pointer_down = (event: PointerEvent) => { - event.preventDefault(); event.stopPropagation(); video.focus({preventScroll: true}); video.setPointerCapture(event.pointerId); + event.preventDefault(); + event.stopPropagation(); + surface.focus({preventScroll: true}); + surface.setPointerCapture(event.pointerId); drag_move_started = false; pending_pointer_move.current = null; transmit("input", pointer_payload("pointer_press", event)); }; const on_pointer_up = (event: PointerEvent) => { - event.preventDefault(); event.stopPropagation(); - if (video.hasPointerCapture(event.pointerId)) video.releasePointerCapture(event.pointerId); - if (pending_pointer_move.current) transmit("input", pending_pointer_move.current); + event.preventDefault(); + event.stopPropagation(); + if (surface.hasPointerCapture(event.pointerId)) + surface.releasePointerCapture(event.pointerId); + if (pending_pointer_move.current) + transmit("input", pending_pointer_move.current); pending_pointer_move.current = null; drag_move_started = false; transmit("input", pointer_payload("pointer_release", event)); }; const on_pointer_cancel = (event: PointerEvent) => { - if (pending_pointer_move.current) transmit("input", pending_pointer_move.current); + if (pending_pointer_move.current) + transmit("input", pending_pointer_move.current); pending_pointer_move.current = null; drag_move_started = false; transmit("input", pointer_payload("pointer_release", event)); }; - const on_pointer_leave = (event: PointerEvent) => { event.stopPropagation(); pending_pointer_move.current = null; transmit("input", {type: "leave"}); }; - const on_wheel = (event: WheelEvent) => { - event.preventDefault(); event.stopPropagation(); - const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 16 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? video.clientHeight : 1; - transmit("input", {type: "wheel", position: point(event), global_position: global_point(event), button: "none", buttons: 0, - modifiers: modifiers(event), pixel_delta_x: -event.deltaX * scale, pixel_delta_y: -event.deltaY * scale, - angle_delta_x: Math.max(-120, Math.min(120, -event.deltaX * scale)), angle_delta_y: Math.max(-120, Math.min(120, -event.deltaY * scale))}); - }; - const key_name = (key: string) => ({Escape: "escape", Enter: "enter", " ": "space", Delete: "delete_key", Backspace: "backspace", - ArrowLeft: "left", ArrowRight: "right", ArrowUp: "up", ArrowDown: "down", Home: "home"}[key] ?? "unknown"); - const on_key = (type: "key_press" | "key_release") => (event: KeyboardEvent) => { + const on_pointer_leave = (event: PointerEvent) => { event.stopPropagation(); - transmit("input", {type, key: key_name(event.key), native_key: event.keyCode, modifiers: modifiers(event), auto_repeat: event.repeat}); + pending_pointer_move.current = null; + transmit("input", {type: "leave"}); }; + const on_wheel = (event: WheelEvent) => { + event.preventDefault(); + event.stopPropagation(); + const scale = event.deltaMode === WheelEvent.DOM_DELTA_LINE ? 16 + : event.deltaMode === WheelEvent.DOM_DELTA_PAGE ? surface.clientHeight : 1; + transmit("input", {type: "wheel", position: point(event), + global_position: global_point(event), button: "none", buttons: 0, + modifiers: modifiers(event), pixel_delta_x: -event.deltaX * scale, + pixel_delta_y: -event.deltaY * scale, + angle_delta_x: Math.max(-120, Math.min(120, -event.deltaX * scale)), + angle_delta_y: Math.max(-120, Math.min(120, -event.deltaY * scale))}); + }; + const key_name = (key: string) => ({Escape: "escape", Enter: "enter", + " ": "space", Delete: "delete_key", Backspace: "backspace", + ArrowLeft: "left", ArrowRight: "right", ArrowUp: "up", + ArrowDown: "down", Home: "home"}[key] ?? "unknown"); + const on_key = (type: "key_press" | "key_release") => + (event: KeyboardEvent) => { + event.stopPropagation(); + transmit("input", {type, key: key_name(event.key), + native_key: event.keyCode, modifiers: modifiers(event), + auto_repeat: event.repeat}); + }; const on_key_down = on_key("key_press"); const on_key_up = on_key("key_release"); const on_focus = () => transmit("input", {type: "show"}); const on_blur = () => transmit("input", {type: "hide"}); - const on_context = (event: MouseEvent) => { event.preventDefault(); event.stopPropagation(); }; - video.addEventListener("pointermove", on_pointer_move); - video.addEventListener("pointerdown", on_pointer_down); - video.addEventListener("pointerup", on_pointer_up); - video.addEventListener("pointercancel", on_pointer_cancel); - video.addEventListener("pointerleave", on_pointer_leave); - video.addEventListener("wheel", on_wheel, {passive: false}); - video.addEventListener("keydown", on_key_down); - video.addEventListener("keyup", on_key_up); - video.addEventListener("focus", on_focus); - video.addEventListener("blur", on_blur); - video.addEventListener("contextmenu", on_context); + const on_context = (event: MouseEvent) => { + event.preventDefault(); + event.stopPropagation(); + }; + surface.addEventListener("pointermove", on_pointer_move); + surface.addEventListener("pointerdown", on_pointer_down); + surface.addEventListener("pointerup", on_pointer_up); + surface.addEventListener("pointercancel", on_pointer_cancel); + surface.addEventListener("pointerleave", on_pointer_leave); + surface.addEventListener("wheel", on_wheel, {passive: false}); + surface.addEventListener("keydown", on_key_down); + surface.addEventListener("keyup", on_key_up); + surface.addEventListener("focus", on_focus); + surface.addEventListener("blur", on_blur); + surface.addEventListener("contextmenu", on_context); return () => { pending_pointer_move.current = null; - video.removeEventListener("pointermove", on_pointer_move); video.removeEventListener("pointerdown", on_pointer_down); - video.removeEventListener("pointerup", on_pointer_up); video.removeEventListener("pointercancel", on_pointer_cancel); - video.removeEventListener("pointerleave", on_pointer_leave); video.removeEventListener("wheel", on_wheel); - video.removeEventListener("keydown", on_key_down); video.removeEventListener("keyup", on_key_up); - video.removeEventListener("focus", on_focus); video.removeEventListener("blur", on_blur); video.removeEventListener("contextmenu", on_context); + surface.removeEventListener("pointermove", on_pointer_move); + surface.removeEventListener("pointerdown", on_pointer_down); + surface.removeEventListener("pointerup", on_pointer_up); + surface.removeEventListener("pointercancel", on_pointer_cancel); + surface.removeEventListener("pointerleave", on_pointer_leave); + surface.removeEventListener("wheel", on_wheel); + surface.removeEventListener("keydown", on_key_down); + surface.removeEventListener("keyup", on_key_up); + surface.removeEventListener("focus", on_focus); + surface.removeEventListener("blur", on_blur); + surface.removeEventListener("contextmenu", on_context); }; - }, [video_ref, transmit]); - return {status, metrics}; + }, [surface_ref, transmit]); + return {status, metrics, error}; } const member_labels: Record = {x: "横坐标", y: "纵坐标", z: "深度", width: "宽度", height: "高度", origin: "起点", target: "终点", @@ -758,26 +915,15 @@ function State_Field_View({component, field, histories}: {component: Component; } type Pipeline_Stage_Definition = [string, string, string]; -const pipeline_common_start: Pipeline_Stage_Definition = ["pipeline_request_transport_ms", "帧时钟与元数据传递", "服务端创建帧前后的调度,以及完成诊断元数据送达浏览器的合计衔接耗时;当前流程不存在浏览器逐帧请求。"]; -const pipeline_common_finish: Pipeline_Stage_Definition[] = [ - ["pipeline_payload_transport_ms", "像素传输", "浏览器收到元数据后,直到完整像素载荷到达的耗时。"], - ["pipeline_canvas_upload_ms", "视频解码呈现", "浏览器 WebRTC 视频管线的解码与呈现尾部耗时。"], - ["pipeline_presentation_wait_ms", "视频呈现等待", "编码帧进入媒体管线后等待浏览器呈现的耗时。"] -]; const pipeline_2d_definitions: Pipeline_Stage_Definition[] = [ - pipeline_common_start, ["pipeline_2d_event_ms", "2D 事件分发", "Scene 将当前输入事件分发给二维 Renderable 的耗时。"], ["pipeline_2d_prepare_ms", "2D 数据准备", "二维 Prepare 依赖图更新缓存、坐标映射和绘制数据的耗时。"], ["pipeline_2d_paint_ms", "Blend2D 绘制", "二维 Paint 任务图清屏并写入 Blend2D 帧缓存的耗时。"], ["pipeline_2d_scene_coordination_ms", "2D Scene 编排", "Scene 渲染区间内除事件、Prepare、Paint 外的依赖图编排耗时。"], - ["pipeline_2d_callback_ms", "2D 完成回调", "同步二维帧完成后回调到 Plot 发布线程的耗时。"], - ["pipeline_2d_encode_queue_ms", "2D 编码排队", "完成帧等待当前 Plot 编码串行域的耗时;多图满负载时可直接观察编码线程池竞争。"], - ["pipeline_2d_encode_ms", "2D 像素封装", "原生 BGRA 帧只进行连续行打包;仅在浏览器不支持 WebGL2 时由 Blend2D 转换为 RGBA。"], - ["pipeline_2d_frame_handoff_ms", "2D 帧建立与发布调度", "Render_Frame 建立、进入 Scene 以及编码完成后生成元数据的调度间隙。"], - ["pipeline_2d_publish_tail_ms", "2D 媒体发布尾部", "上一完成帧从编码结束到元数据生成并移交消费者返回的耗时;用于识别媒体背压。"] + ["pipeline_2d_callback_ms", "2D 完成帧提取", "同步二维帧完成回调中取得连续 RGBA8 像素并建立不可变完成帧的耗时。"], + ["pipeline_2d_frame_handoff_ms", "2D 帧建立与完成发布", "Render_Frame 建立、Scene 入口以及完成帧交给页面图集之间尚未由独立 marker 覆盖的衔接耗时。"] ]; const pipeline_3d_definitions: Pipeline_Stage_Definition[] = [ - pipeline_common_start, ["pipeline_3d_event_ms", "3D 事件入队", "Scene 将输入事件提交到本 Scene 独立 Datoviz 准备域的耗时。"], ["pipeline_3d_prepare_ms", "3D Visual Prepare", "将 Visual items 转换为不可变 Prepared_Visual GPU 字段的耗时;数据变化时执行。"], ["pipeline_3d_submit_graph_ms", "3D Submit 图", "Submit 依赖图把 Prepared_Visual 入队到异步后端的耗时。"], @@ -795,23 +941,18 @@ const pipeline_3d_definitions: Pipeline_Stage_Definition[] = [ ["pipeline_3d_gpu_copy_ms", "GPU 回读复制", "GPU 将渲染结果复制到可回读资源的设备时间。"], ["pipeline_3d_gpu_sync_ms", "GPU 同步等待", "GPU 提交到完成区间扣除已测设备阶段后的 fence/调度时间。"], ["pipeline_3d_readback_ms", "3D CPU 回读", "GPU 完成后由 Datoviz 收集并复制 RGBA 像素的耗时。"], - ["pipeline_3d_callback_ms", "3D 完成回调", "异步后端完成后回调到 Plot 发布线程的耗时。"], - ["pipeline_3d_encode_queue_ms", "3D 编码排队", "RGBA 完成帧等待当前 Plot 编码串行域的耗时;可识别多 Plot 共享工作线程导致的排队。"], - ["pipeline_3d_encode_ms", "3D H.264 编码", "把连续 RGBA 像素转换为 YUV420P 并编码成 H.264 access unit 的耗时。"], - ["pipeline_3d_completion_handoff_ms", "3D 完成调度衔接", "GPU 回读、回调与发布边界之间尚未由 trace marker 单独覆盖的调度衔接时间。"], - ["pipeline_3d_publish_tail_ms", "3D 媒体发布尾部", "上一完成帧从编码结束到元数据生成并移交消费者返回的耗时;用于识别媒体背压。"] + ["pipeline_3d_callback_ms", "3D 完成帧提取", "异步后端完成回调中共享原生 RGBA8 回读所有权并建立不可变完成帧的耗时。"], + ["pipeline_3d_completion_handoff_ms", "3D 完成发布衔接", "GPU 回读、回调与完成帧交给页面图集之间尚未由 trace marker 单独覆盖的调度耗时。"] ]; const pipeline_definitions = (dimension: Plot["dimension"], delivery: Frame_Delivery) => { const core = (dimension === "2D" ? pipeline_2d_definitions : pipeline_3d_definitions).map(definition => { - if (delivery === "webrtc-video") return definition; + if (delivery === "gallery-video") return definition; const [key] = definition; - if (key === "pipeline_2d_encode_ms") return [key, "2D 诊断编码", "跳过 BGRA 像素转换,仅生成二维诊断元数据的耗时。"] as Pipeline_Stage_Definition; if (key === "pipeline_3d_gpu_copy_ms") return [key, "GPU 回读复制(跳过)", "诊断帧不复制像素到回读资源;该项应接近零。"] as Pipeline_Stage_Definition; if (key === "pipeline_3d_readback_ms") return [key, "3D 完成收集", "收集 GPU 完成状态和时间戳、但不下载 RGBA 像素的耗时。"] as Pipeline_Stage_Definition; - if (key === "pipeline_3d_encode_ms") return [key, "3D 诊断编码", "跳过像素消息,仅生成三维诊断元数据的耗时。"] as Pipeline_Stage_Definition; return definition; }); - return [...core, ...(delivery === "webrtc-video" ? pipeline_common_finish : [])]; + return core; }; function diagnostic_value(value: number, key: string) { @@ -834,16 +975,15 @@ function Frame_Timeline_Chart({diagnostics, dimension, paused, on_context_menu}: const chart = chart_ref.current; if (!chart) return; const visible_samples = diagnostics.samples; - const total_series: [string, string, string] = diagnostics.metadata.delivery === "webrtc-video" - ? ["request_to_pixels_ms", "H.264 视频流水线", "#5ce4c2"] - : ["request_to_pixels_ms", "诊断往返", "#5ce4c2"]; - const pixel_delivery = diagnostics.metadata.delivery === "webrtc-video"; + const total_series: [string, string, string] = + ["server_completion_ms", "服务端完成流水线", "#5ce4c2"]; + const pixel_delivery = diagnostics.metadata.delivery === "gallery-video"; const series_keys: Array<[string, string, string]> = dimension === "2D" ? [ total_series, ["pipeline_2d_prepare_ms", "2D Prepare", "#62a8ff"], ["pipeline_2d_paint_ms", "Blend2D 绘制", "#f4bd63"], - pixel_delivery ? ["pipeline_payload_transport_ms", "像素传输", "#ff7d9c"] : ["pipeline_2d_encode_ms", "诊断编码", "#ff7d9c"], - pixel_delivery ? ["pipeline_presentation_wait_ms", "呈现等待", "#b998ff"] : ["pipeline_2d_frame_handoff_ms", "发布衔接", "#b998ff"] + ["pipeline_2d_scene_coordination_ms", "Scene 编排", "#ff7d9c"], + ["pipeline_2d_frame_handoff_ms", "完成发布衔接", "#b998ff"] ] : [ total_series, ["pipeline_3d_prepare_ms", "Visual Prepare", "#62a8ff"], @@ -894,21 +1034,21 @@ function Frame_Diagnostics_View({diagnostics, dimension, on_reset}: {diagnostics }); const total_value = stage_values.reduce((sum, [, , , value]) => sum + (Number.isFinite(value) ? value : 0), 0); const statistic_labels: Record = {average: "滑动平均", variability: "波动", p95: "P95", p99: "P99"}; - const pixel_delivery = displayed.metadata.delivery === "webrtc-video"; - const completion = "诊断元数据到达浏览器"; + const pixel_delivery = displayed.metadata.delivery === "gallery-video"; + const completion = "完成帧发布给页面图集"; const summaries: Array<[string, string, string]> = [ - ["Scene 完成帧率", `${displayed.frame_rate_fps.toFixed(1)} FPS`, "根据服务端帧序号跨度和创建时间计算;编码或媒体背压合并旧帧时仍反映真实 Scene 完成速率。"], - [pixel_delivery ? "视频投递帧率" : "诊断投递频率", `${displayed.delivery_frame_rate_fps.toFixed(1)} FPS`, `浏览器实际收到的${pixel_delivery ? "编码帧元数据" : "后台诊断帧"}频率;与 Scene 帧率的差值表示实时背压合并。`], - ["浏览器呈现帧率", `${displayed.video_playback.frame_rate_fps.toFixed(1)} FPS`, "由 requestVideoFrameCallback 统计的 video 元素真实呈现频率;服务端有帧而此项接近零时,问题位于关键帧、RTP、解码或呈现链路。"], - ["浏览器累计呈现", displayed.video_playback.presented_frames.toLocaleString("zh-CN"), `video readyState=${displayed.video_playback.ready_state},currentTime=${displayed.video_playback.current_time_seconds.toFixed(3)} s。`], - ["浏览器累计丢帧", displayed.video_playback.dropped_frames.toLocaleString("zh-CN"), "浏览器 VideoPlaybackQuality 报告的累计丢弃视频帧。"], - ["服务端完成平均", `${displayed.request_to_pixels_average_ms.toFixed(2)} ms`, `服务端创建帧到${completion}的平均耗时;不包含浏览器视频解码与真正呈现。`], - ["服务端完成 P50", `${displayed.request_to_pixels_p50_ms.toFixed(2)} ms`, "一半样本不超过该服务端完成耗时。"], - ["服务端完成 P95", `${displayed.request_to_pixels_p95_ms.toFixed(2)} ms`, "95% 样本不超过该服务端完成耗时,用于观察长尾。"], - ["服务端完成 P99", `${displayed.request_to_pixels_p99_ms.toFixed(2)} ms`, "99% 样本不超过该服务端完成耗时,用于观察极端长尾。"], + ["逻辑完成闭环率", `${displayed.frame_rate_fps.toFixed(1)} FPS`, "根据 Plot 逻辑帧序号跨度和创建时间计算;历史回调共享同一 GPU 画面时仍会分别结清。真实画面率以卡片 Gallery 指标为准。"], + ["诊断采样频率", `${displayed.delivery_frame_rate_fps.toFixed(1)} FPS`, "Plot WebSocket 实际收到抽样诊断 JSON 的频率;视频帧率由页面级 Gallery 指标单独统计。"], + ["浏览器解码帧率", `${displayed.video_playback.frame_rate_fps.toFixed(1)} FPS`, "由共享 WebRTC receiver 的 framesDecoded 增量统计;它不受显示器刷新率限制,可直接判断图集视频是否稳定收到一百帧。"], + ["浏览器累计解码", displayed.video_playback.presented_frames.toLocaleString("zh-CN"), `共享 WebRTC receiver 已解码帧数;readyState=${displayed.video_playback.ready_state}。`], + ["浏览器累计丢帧", displayed.video_playback.dropped_frames.toLocaleString("zh-CN"), "浏览器 inbound-rtp 统计报告的累计丢弃视频帧。"], + ["服务端完成平均", `${displayed.server_completion_average_ms.toFixed(2)} ms`, `服务端创建帧到${completion}的平均耗时;不包含独立的图集编码、WebRTC 和浏览器解码。`], + ["服务端完成 P50", `${displayed.server_completion_p50_ms.toFixed(2)} ms`, "一半样本不超过该服务端完成耗时。"], + ["服务端完成 P95", `${displayed.server_completion_p95_ms.toFixed(2)} ms`, "95% 样本不超过该服务端完成耗时,用于观察长尾。"], + ["服务端完成 P99", `${displayed.server_completion_p99_ms.toFixed(2)} ms`, "99% 样本不超过该服务端完成耗时,用于观察极端长尾。"], ["帧间隔平均", `${displayed.frame_interval_average_ms.toFixed(2)} ms`, `相邻两次${completion}的平均间隔。`], ["帧间隔抖动 P95", `${displayed.interval_jitter_p95_ms.toFixed(2)} ms`, "帧间隔相对中位数偏差的第 95 百分位。"], - ["投递帧序号缺口", displayed.dropped_sequence_count.toLocaleString("zh-CN"), "相邻已投递帧之间被实时编码或媒体背压合并的历史帧数量;Scene 完成回调本身仍逐帧闭环。"] + ["诊断样本额外缺口", displayed.dropped_sequence_count.toLocaleString("zh-CN"), "扣除正常抽样步长后,诊断序列仍缺少的页面时钟 tick 数;不等同于视频丢帧。"] ]; const open_context_menu = (event: React.MouseEvent) => { event.preventDefault(); @@ -924,14 +1064,14 @@ function Frame_Diagnostics_View({diagnostics, dimension, on_reset}: {diagnostics
当前为{pixel_delivery ? "WebRTC H.264 视频" : "后台诊断"}模式,区间内 {displayed.samples.length} 帧;图表右键可暂停并缩放查看。{pixel_delivery ? "视频轨独立于 WebSocket 信令与诊断元数据。" : "后台诊断仍执行真实渲染,但跳过 H.264 编码与视频发送。"}
{summaries.map(([label, value, description]) =>
{label}
{value}
)}
-
流水线阶段统计#{displayed.metadata.sequence} / 请求 {displayed.metadata.correlation_id}
+
流水线阶段统计#{displayed.metadata.sequence} / 时钟 {displayed.metadata.correlation_id}
{(["average", "variability", "p95", "p99"] as Stage_Statistic[]).map(mode => )}
{(["value", "percentage"] as Stage_Unit[]).map(unit => )}
-
服务端帧时钟建帧{definitions.map(([key, label, description]) => {label})}浏览器收到诊断
+
服务端帧时钟建帧{definitions.map(([key, label, description]) => {label})}完成帧进入页面图集
{stage_values.map(([key, label, description, value]) =>
{label}
{!Number.isFinite(value) ? "--" : stage_unit === "percentage" ? `${(total_value > 0 ? value / total_value * 100 : 0).toFixed(1)}%` : diagnostic_value(value, key)}
)}
@@ -1050,22 +1190,48 @@ function Frame_Statistics_Pane({plot, diagnostics, busy, on_refresh, on_reset}:
; } -const Plot_Card = memo(function Plot_Card({plot, selected, policy, on_policy, on_select}: { - plot: Plot; selected: boolean; policy: Plot_Execution_Policy; +const Plot_Card = memo(function Plot_Card({plot, selected, policy, gallery, on_policy, on_select}: { + plot: Plot; selected: boolean; policy: Plot_Execution_Policy; gallery: Gallery_Video_State; on_policy: (plot_id: string, patch: Partial) => void; on_select: (plot: Plot) => void; }) { const video_ref = useRef(null); + const surface_ref = useRef(null); const card_ref = useRef(null); - const {status, metrics} = use_plot_stream(plot, video_ref); + const tile = gallery.layout?.plots[plot.id] ?? null; + const tile_width = gallery.layout?.tile_width ?? 720; + const tile_height = gallery.layout?.tile_height ?? 420; + const graph = use_plot_stream(plot, surface_ref, gallery.playback, tile_width, tile_height); + const status: Stream_Status = graph.status === "OFFLINE" || gallery.status === "OFFLINE" ? "OFFLINE" + : graph.status === "LIVE" && gallery.status === "LIVE" ? "LIVE" : "CONNECTING"; + const metrics = graph.metrics; + const source_transport = gallery.transport?.sources[plot.id] ?? null; + const encoder_label = gallery.transport?.encoder_backend === "nvenc" + ? "NVENC" : gallery.transport?.encoder_backend === "vulkan_video" + ? "Vulkan Video" : "硬件编码"; + useEffect(() => { + const video = video_ref.current; + if (!video) return; + video.srcObject = gallery.stream; + if (gallery.stream) void video.play(); + }, [gallery.stream]); + const atlas_style: React.CSSProperties = gallery.layout && tile ? { + width: `${gallery.layout.columns * 100}%`, + height: `${gallery.layout.rows * 100}%`, + left: `${-tile.column * 100}%`, + top: `${-tile.row * 100}%` + } : {}; const generated_time = metrics ? new Date(metrics.generated_time_unix_ms).toLocaleTimeString("zh-CN", {hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: false, fractionalSecondDigits: 3}) : "尚未生成帧"; return
on_select(plot)} onPointerUpCapture={event => { - if (event.target instanceof HTMLVideoElement) on_select(plot); - }}>
绘图组件 · {plot.dimension}

{plot_labels[plot.id] ?? plot.title}

{{IDLE: "已停止", CONNECTING: "重连中", LIVE: "实时", OFFLINE: "已离线"}[status]}{metrics?.delivery === "diagnostics" ? "无像素传输" : metrics ? enum_label(metrics.pacing_mode) : "等待策略"}
服务端 {metrics ? metrics.frame_rate_fps.toFixed(1) : "--.-"} FPS呈现 {metrics ? metrics.video_playback.frame_rate_fps.toFixed(1) : "--.-"} FPS耗时 {metrics ? metrics.request_to_pixels_ms.toFixed(1) : "--.-"} msP95 {metrics ? metrics.p95_request_to_pixels_ms.toFixed(1) : "--.-"} ms
+ onClick={() => on_select(plot)}>
绘图组件 · {plot.dimension}

{plot_labels[plot.id] ?? plot.title}

{{IDLE: "已停止", CONNECTING: "重连中", LIVE: "实时", OFFLINE: "已离线"}[status]}{metrics?.delivery === "diagnostics" ? "无像素传输" : metrics ? enum_label(metrics.pacing_mode) : "等待策略"}
真实画面 {source_transport ? source_transport.rendered_frame_rate_fps.toFixed(1) : metrics ? metrics.frame_rate_fps.toFixed(1) : "--.-"} FPS逻辑闭环 {source_transport ? source_transport.logical_completion_rate_fps.toFixed(1) : "--.-"} FPS图集 {gallery.transport ? gallery.transport.encoded_frame_rate_fps.toFixed(1) : "--.-"} FPS解码 {gallery.playback.frame_rate_fps.toFixed(1)} FPS完成 {metrics ? metrics.server_completion_ms.toFixed(1) : "--.-"} ms{encoder_label} {gallery.transport ? gallery.transport.encode_average_ms.toFixed(1) : "--.-"} ms落后 {source_transport?.has_rendered_frame ? source_transport.clock_lag_ticks : "--"} tick
event.stopPropagation()}>
- {plot.description ?

{plot.description}

: null}
; + {plot.description ?

{plot.description}

: null}
+
; }); type Gallery_Breakpoint = "lg" | "md" | "sm" | "xs"; @@ -1123,7 +1289,7 @@ function save_gallery_layouts(scope: string, layouts: ResponsiveLayouts) => void; on_select: (plot: Plot) => void}) { +function Gallery_Grid({plots, selected, policies, layout_scope, galleries, on_policy, on_select}: {plots: Plot[]; selected: Plot | null; policies: Plot_Execution_Policies; layout_scope: string; galleries: Gallery_Video_States; on_policy: (plot_id: string, patch: Partial) => void; on_select: (plot: Plot) => void}) { const {width, containerRef, mounted} = useContainerWidth({measureBeforeMount: true}); const [stored_layouts, set_stored_layouts] = useState>(() => load_gallery_layouts(layout_scope)); const layouts = useMemo(() => Object.fromEntries((Object.keys(gallery_breakpoints) as Gallery_Breakpoint[]).map(breakpoint => { @@ -1149,7 +1315,7 @@ function Gallery_Grid({plots, selected, policies, layout_scope, on_policy, on_se margin={[16, 16]} containerPadding={[0, 0]} onLayoutChange={save_layouts} dragConfig={{handle: ".cardDragHandle", cancel: "canvas,button,input,select,textarea,a", threshold: 4}} resizeConfig={{handles: ["s", "e", "se", "sw", "w"]}}> - {plots.map(plot =>
)} + {plots.map(plot =>
)} : null}; } @@ -1213,6 +1379,7 @@ function load_workspace_model() { export function App() { const [plots, set_plots] = useState([]); const [category, set_category] = useState("全部"); const [selected, set_selected] = useState(null); + const gallery_videos = use_gallery_videos(plots); const [execution_policies, set_execution_policies] = useState({}); const [schema, set_schema] = useState(null); const [state_histories, set_state_histories] = useState({}); @@ -1290,7 +1457,7 @@ export function App() { - ; + ; const factory = (node: TabNode) => { if (node.getComponent() === "gallery") return gallery; if (!selected) return
请选择一个图形组件。
; diff --git a/webapp_gallery/src/styles.css b/webapp_gallery/src/styles.css index fb35529..8baec0f 100644 --- a/webapp_gallery/src/styles.css +++ b/webapp_gallery/src/styles.css @@ -71,7 +71,7 @@ nav { display: flex; flex-wrap: wrap; gap: 8px; padding: 18px 0; } .framePolicy { color: #8296b2; font-size: 9px; white-space: nowrap; } .frameMetrics { display: grid; grid-template-columns: auto auto; gap: 4px 9px; color: #8296b2; font: 9px/1.15 ui-monospace, monospace; font-variant-numeric: tabular-nums; white-space: nowrap; } .frameMetrics span:nth-child(2n) { text-align: right; } -.plotViewport { flex: 1; width: 100%; min-height: 160px; overflow: hidden; background: #070d18; } +.plotViewport { position: relative; flex: 1; width: 100%; min-height: 160px; overflow: hidden; outline: none; background: #070d18; overscroll-behavior: contain; touch-action: none; } .plotExecutionPolicy { display:flex; flex-wrap:wrap; gap:6px 12px; padding:8px 12px; border-block:1px solid rgba(126,155,194,.12); background:rgba(6,13,24,.58); font-size:11px; color:#9dafc7; } .plotExecutionPolicy label { display:flex; align-items:center; gap:5px; cursor:pointer; user-select:none; } .plotExecutionPolicy input { accent-color:#52dbc1; } @@ -80,8 +80,9 @@ nav { display: flex; flex-wrap: wrap; gap: 8px; padding: 18px 0; } .plotHiddenState { position:absolute; inset:0; display:grid; place-content:center; gap:5px; text-align:center; color:#91a5bf; background:repeating-linear-gradient(135deg,rgba(15,27,43,.96),rgba(15,27,43,.96) 12px,rgba(18,33,52,.96) 12px,rgba(18,33,52,.96) 24px); } .plotHiddenState strong { color:#d9e8f6; } canvas { display: block; width: 100%; height: 100%; background: #070d18; } -.plotViewport video { display: block; width: 100%; height: 100%; object-fit: fill; background: #070d18; overscroll-behavior: contain; touch-action: none; } -.plotViewport video:focus { outline: 1px solid #5ce4c2; outline-offset: -1px; } +.plotViewport video { position: absolute; display: block; max-width: none; max-height: none; object-fit: fill; pointer-events: none; background: #070d18; } +.plotViewport:focus { box-shadow: inset 0 0 0 1px #5ce4c2; } +.plotMediaError { position: absolute; inset: 0; z-index: 3; display: grid; place-content: center; padding: 18px; color: #ff9bae; text-align: center; background: #190b12e8; } .inspector { width: 100%; height: 100%; overflow: hidden; background: #080f1b; } .workspacePane { display: flex; flex-direction: column; width: 100%; height: 100%; min-height: 0; }