diff --git a/AGENTS.md b/AGENTS.md index ef88a2d..2187f94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,9 @@ D:\ae\tools 可能会有有用的工具 任何 fallback 都必须先规划，禁止直接实现 -你每次出bug 很多时候是编译缓存的原因 你清掉缓存重新编译 还不行 再去找bug 你写完清理你自己创建的所有进程 我自己启动服务 +你每次出bug 很多时候是编译缓存的原因 你清掉缓存重新编译 还不行 再去找bug + +你写完清理你自己创建的所有进程 我自己启动服务 写函数 和模块使用下面的约定 [错误处理规范](./Error_handling_specification.md) diff --git a/Project_detail_specification.md b/Project_detail_specification.md index 8bd3e8b..3a755c3 100644 --- a/Project_detail_specification.md +++ b/Project_detail_specification.md @@ -19,6 +19,7 @@ * `Scene::Private` 是输入事件流的唯一所有者：外部转移事件对象所有权，无锁提交到双缓冲队列，事件不得独立触发帧，只在下一次正常渲染的 Prepare 入口交换并按 FIFO 消费。2D 按 Renderable 区域与 Paint 顺序形成接受链，区域默认整个 viewport；3D 无等待提交渲染域，满载时保留当前事件供下一次 Prepare 重试。 * 帧由 Scene 外部创建和持有；每次 `render(frame*)` 只借用该帧并在完成回调返回同一地址。Scene、异步后端和 Web 层只向帧写入固定语义的单调时间点与原始耗时，不保存平均值、分位数或波动等衍生统计。 * Plot 使用服务端帧时钟调用 `Scene::render(frame*)`；帧策略只决定调用频率，完成回调不得安排下一帧，也不存在浏览器逐帧请求或 request/ack。Scene 回调返回完成帧后，Web 层在 Render Domain 之外把 2D 原生 BGRA 或 3D 原生 RGBA 编码为 H.264，经 LibDataChannel WebRTC 视频轨直接发布；WebSocket 只承载 SDP/ICE、输入事件和诊断 JSON。多个订阅者共享同一编码结果，关闭视频时 3D 必须使用 diagnostics 输出并跳过 GPU 像素回读。 +* 相机是 3D Scene 组件，只允许定义在 `render_3D/camera`；Kernel 和 2D 不得依赖相机类型。Web 层只为已有 `Camera_3D` 增加协议描述，不复制相机配置。Gallery 服务同一时刻只允许一个页面实例持有；该页面的所有 Plot 连接共享页面令牌，其他标签页或浏览器实例必须被拒绝。 * `Time_Axis` 是时间与 tick 的唯一权威来源；使用层先推进时间轴，再把同一 tick 分发给所有相关数据图元。时间窗口从第一条数据起始终锚定最新 tick，未产生数据的槽位保持背景。 * 图表选区保存两根轴上的数据范围，绘制时才映射为像素；选区作为独立 Renderable 在使用层与图元组合，禁止在各图元内复制选区状态。 * `Root` 只保存一个最终 `Private` 指针；`Builder::build()` 校验成功后创建并挂接完整 Private，`Root` 通过公共 Private 基类的虚析构统一释放。禁止直接公开该指针。 diff --git a/kernel/src/kernel/frame.hpp b/kernel/src/kernel/frame.hpp index 0973d1b..099bd93 100644 --- a/kernel/src/kernel/frame.hpp +++ b/kernel/src/kernel/frame.hpp @@ -26,6 +26,7 @@ enum class Frame_Trace_Marker : std::uint8_t { scene_render_finished, callback_started, callback_finished, + video_encode_queued, video_encode_started, video_encode_finished, stream_publish_started, @@ -43,6 +44,7 @@ enum class Frame_Trace_Measurement : std::uint8_t { gpu_copy_ns, gpu_total_ns, readback_ns, + stream_publish_tail_ns, count }; struct Frame_Identity { diff --git a/kernel/src/kernel/plot/Camera.hpp b/render_3D/render_3D/camera/Camera.hpp similarity index 97% rename from kernel/src/kernel/plot/Camera.hpp rename to render_3D/render_3D/camera/Camera.hpp index 7ba32b3..ce0acf3 100644 --- a/kernel/src/kernel/plot/Camera.hpp +++ b/render_3D/render_3D/camera/Camera.hpp @@ -1,9 +1,7 @@ #pragma once - #include -namespace aethera::plot { - +namespace aethera::render_3d { struct Spatial_Point { double x{}; double y{}; @@ -90,5 +88,4 @@ struct Camera_Descriptor { double far_plane{100.0}; bool operator==(const Camera_Descriptor&) const = default; }; - -} // namespace aethera::plot +} diff --git a/render_3D/render_3D/camera/Camera_3D.hpp b/render_3D/render_3D/camera/Camera_3D.hpp index 60f94b2..a7a2d83 100644 --- a/render_3D/render_3D/camera/Camera_3D.hpp +++ b/render_3D/render_3D/camera/Camera_3D.hpp @@ -1,19 +1,19 @@ #pragma once -#include +#include "Camera.hpp" #include namespace aethera::render_3d { struct Camera_3D : Def { struct Prop : Prev_Prop { - plot::Camera_View initial_view{}; - plot::Camera_Projection projection{plot::Camera_Projection::perspective}; - plot::Camera_Controller controller{plot::Camera_Controller::turntable}; - plot::Camera_Turntable_Control turntable_control{}; - plot::Camera_Arcball_Control arcball_control{}; - plot::Camera_Fly_Control fly_control{}; - plot::Camera_Panzoom_Control panzoom_control{}; + Camera_View initial_view{}; + Camera_Projection projection{Camera_Projection::perspective}; + Camera_Controller controller{Camera_Controller::turntable}; + Camera_Turntable_Control turntable_control{}; + Camera_Arcball_Control arcball_control{}; + Camera_Fly_Control fly_control{}; + Camera_Panzoom_Control panzoom_control{}; double vertical_field_of_view_degrees{45.0}; double near_plane{0.01}; double far_plane{100.0}; diff --git a/render_3D/render_3D/detail/Async_Render_Backend.cpp b/render_3D/render_3D/detail/Async_Render_Backend.cpp index 61067b4..7aa8a69 100644 --- a/render_3D/render_3D/detail/Async_Render_Backend.cpp +++ b/render_3D/render_3D/detail/Async_Render_Backend.cpp @@ -6,10 +6,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -44,6 +46,18 @@ void record_datoviz_trace(Frame_3D* frame, const Datoviz_Frame_Trace& trace) { frame->record(Frame_Trace_Measurement::gpu_copy_ns, trace.gpu->copy_ns); frame->record(Frame_Trace_Measurement::gpu_total_ns, trace.gpu->total_ns); } +std::string failure_description(const std::exception_ptr& failure) { + try { + if (failure) std::rethrow_exception(failure); + } + catch (const std::exception& error) { + return error.what(); + } + catch (...) { + return "non-standard exception"; + } + return "empty exception"; +} } struct Async_Render_Backend::Implementation : std::enable_shared_from_this { @@ -139,9 +153,13 @@ void Async_Render_Backend::Implementation::stop() noexcept { } void Async_Render_Backend::Implementation::fail( std::exception_ptr value) noexcept { + const auto description = failure_description(value); + std::fprintf(stderr, "Aethera 3D backend unavailable: %s\n", + description.c_str()); + std::fflush(stderr); { std::lock_guard lock(failure_mutex); - failure = std::move(value); + if (!failure) failure = std::move(value); } available.store(false, std::memory_order_release); } diff --git a/render_3D/render_3D/detail/Backend_Types.hpp b/render_3D/render_3D/detail/Backend_Types.hpp index dd135d6..56de091 100644 --- a/render_3D/render_3D/detail/Backend_Types.hpp +++ b/render_3D/render_3D/detail/Backend_Types.hpp @@ -2,7 +2,7 @@ #include "../visual/Prepared_Visual.hpp" #include #include -#include +#include "../camera/Camera.hpp" #include #include namespace aethera::render_3d::detail { @@ -20,7 +20,7 @@ using Prepared_Visual_Batch = std::vector; struct Scene_3D_Parameters { Extent viewport{560, 320}; /* 离屏渲染目标尺寸，单位为像素。 */ Linear_Color clear_color{}; /* 每帧开始时写入的线性背景色。 */ - plot::Camera_Descriptor camera{}; /* Camera 组件发布的本帧配置快照。 */ + Camera_Descriptor camera{}; /* Camera 组件发布的本帧配置快照。 */ plot::Axis_Descriptor x_axis{}; /* 三维数据 X 轴的业务范围与标签策略。 */ plot::Axis_Descriptor y_axis{}; /* 三维数据 Y 轴的业务范围与标签策略。 */ plot::Axis_Descriptor z_axis{}; /* 三维数据 Z 轴的业务范围与标签策略。 */ diff --git a/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp b/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp index f3400b9..0f36d11 100644 --- a/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp +++ b/render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp @@ -176,6 +176,49 @@ bool supports_item_interaction(Visual_Family family) noexcept { return family == Visual_Family::point || family == Visual_Family::pixel || family == Visual_Family::marker; } + +bool uses_external_attributes(Visual_Family family) noexcept { + return family == Visual_Family::mesh || family == Visual_Family::marker; +} + +bool has_coordinate_labels(const Prepared_Visual_Data& data) noexcept { + return std::ranges::any_of( + data.coordinate_label_visibility, + [](std::uint8_t visible) { return visible != 0; }); +} + +bool same_visual_structure(const Prepared_Visual& applied, + const Prepared_Visual& incoming) noexcept { + if (applied.family != incoming.family || + applied.transform != incoming.transform || + applied.visible != incoming.visible || + applied.depth_test != incoming.depth_test || + !applied.data || !incoming.data || + applied.data->positions.size() != incoming.data->positions.size()) + return false; + if (incoming.family == Visual_Family::marker && + (has_coordinate_labels(*applied.data) || + has_coordinate_labels(*incoming.data))) + return applied.revision == incoming.revision; + return true; +} + +std::string artifact_command_excerpt(const std::string& json, + std::uint32_t command_index) { + constexpr std::string_view marker{"{ \"cmd\":"}; + const auto first_index = command_index > 12 ? command_index - 12 : 0; + std::size_t position{}; + std::size_t first_position{}; + for (std::uint32_t index = 0; index <= command_index; ++index) { + position = json.find(marker, position); + if (position == std::string::npos) return {}; + if (index == first_index) first_position = position; + if (index != command_index) position += marker.size(); + } + const auto end = json.find('\n', position); + return json.substr(first_position, + end == std::string::npos ? 2048 : end - first_position); +} } // namespace class Datoviz_Render_Context final { public: @@ -279,11 +322,32 @@ public: ~Frame_Target() noexcept(false) { destroy(); } - void begin(bool observe, bool readback) { + [[nodiscard]] bool can_reuse( + std::uint64_t command_revision, bool observe, + bool readback) const noexcept { + return available() && recorded_ && + recorded_command_revision_ == command_revision && + recorded_observing_ == observe && + recorded_readback_ == readback; + } + void reuse(std::uint64_t command_revision, bool observe, bool readback) { + if (!can_reuse(command_revision, observe, readback)) + throw std::logic_error( + "Datoviz frame target command recording cannot be reused"); + observing_ = observe; + readback_requested_ = readback; + dvz_fence_reset(fence_); + prepared_ = true; + } + void begin(bool observe, bool readback, std::uint64_t command_revision) { if (recording_ || prepared_ || in_flight_) throw std::logic_error("Datoviz frame target is not available"); observing_ = observe; readback_requested_ = readback; + recording_command_revision_ = command_revision; + /* Reset invalidates the previous recording immediately. Only + * finish_recording() may publish the new cache identity. */ + recorded_ = false; if (observing_ && !timestamps_initialized_) { initialize_timestamps( dvz_gpu_ctx_device(gpu_context_), @@ -417,6 +481,10 @@ public: dvz_fence_reset(fence_); dvz_submit(submit_); dvz_submit_command(submit_, dvz_commands_handle(commands_)); + recorded_command_revision_ = recording_command_revision_; + recorded_observing_ = observing_; + recorded_readback_ = readback_requested_; + recorded_ = true; prepared_ = true; } void submit() { @@ -625,6 +693,11 @@ private: bool readback_requested_{}; bool timestamps_initialized_{}; bool timestamps_supported_{}; + std::uint64_t recording_command_revision_{}; + std::uint64_t recorded_command_revision_{}; + bool recorded_{}; + bool recorded_observing_{}; + bool recorded_readback_{}; }; struct Datoviz_Visual_Backend::Frame_Targets { static constexpr std::size_t count = 3; @@ -642,8 +715,12 @@ Datoviz_Visual_Backend::Datoviz_Visual_Backend( auto* gpu_context = render_context_->gpu_context(); DvzDrp2RuntimeConfig runtime_configuration = dvz_drp2_runtime_vklite_config( dvz_gpu_ctx_device(gpu_context), dvz_gpu_ctx_alloc(gpu_context)); - runtime_ = dvz_drp2_runtime_vklite(&runtime_configuration); - if (runtime_ == nullptr) throw std::runtime_error("failed to create Datoviz DRP2 runtime"); + for (auto& slot : runtime_slots_) { + slot.runtime = dvz_drp2_runtime_vklite(&runtime_configuration); + slot.emitter = dvz_frame_plan_emitter(); + if (slot.runtime == nullptr || slot.emitter == nullptr) + throw std::runtime_error("failed to create a Datoviz frame-target runtime"); + } create_scene(visuals, initial_scene); } catch (...) { @@ -1007,14 +1084,14 @@ void Datoviz_Visual_Backend::apply_axes(const Scene_3D_Parameters& scene) { applied_axes_ = descriptors; } -void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) { +void Datoviz_Visual_Backend::apply_camera(const Camera_Descriptor& source) { if (applied_camera_ && *applied_camera_ == source) return; if (camera_controller_ != nullptr) { dvz_controller_destroy(camera_controller_); camera_controller_ = nullptr; } DvzCameraDesc camera = dvz_camera_desc(); - const auto assign = [](vec3 target, plot::Spatial_Point value) { + const auto assign = [](vec3 target, Spatial_Point value) { target[0] = static_cast(value.x); target[1] = static_cast(value.y); target[2] = static_cast(value.z); @@ -1022,7 +1099,7 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) assign(camera.view.eye, source.initial_view.eye); assign(camera.view.target, source.initial_view.target); assign(camera.view.up, source.initial_view.up); - camera.projection.type = source.projection == plot::Camera_Projection::orthographic + camera.projection.type = source.projection == Camera_Projection::orthographic ? DVZ_CAMERA_ORTHOGRAPHIC : DVZ_CAMERA_PERSPECTIVE; camera.projection.fov_y = static_cast( @@ -1040,7 +1117,7 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) throw std::runtime_error("failed to apply Datoviz Camera component"); DvzDimMask dimensions = DVZ_DIM_MASK_XYZ; switch (source.controller) { - case plot::Camera_Controller::turntable: { + case Camera_Controller::turntable: { DvzTurntableDesc descriptor = dvz_turntable_desc(); descriptor.initial_view = camera.view; descriptor.yaw_speed = static_cast(source.turntable_control.yaw_speed); @@ -1060,7 +1137,7 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) camera_controller_ = dvz_turntable(scene_, &descriptor); break; } - case plot::Camera_Controller::arcball: { + case Camera_Controller::arcball: { DvzArcballDesc descriptor = dvz_arcball_desc(); if (source.arcball_control.constrain_rotation) descriptor.controller_flags |= DVZ_ARCBALL_FLAGS_CONSTRAIN; @@ -1076,10 +1153,10 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) } break; } - case plot::Camera_Controller::fly: { + case Camera_Controller::fly: { DvzFlyDesc descriptor = dvz_fly_desc(); descriptor.initial_view = camera.view; - descriptor.mode = source.fly_control.mode == plot::Camera_Fly_Mode::plane + descriptor.mode = source.fly_control.mode == Camera_Fly_Mode::plane ? DVZ_FLY_MODE_PLANE : DVZ_FLY_MODE_FREE; descriptor.speed = static_cast(source.fly_control.movement_speed); @@ -1093,7 +1170,7 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) camera_controller_ = dvz_fly(scene_, &descriptor); break; } - case plot::Camera_Controller::panzoom: { + case Camera_Controller::panzoom: { DvzPanzoomDesc descriptor = dvz_panzoom_desc(); if (source.panzoom_control.fixed_x) descriptor.controller_flags |= DVZ_PANZOOM_FLAGS_FIXED_X; if (source.panzoom_control.fixed_y) descriptor.controller_flags |= DVZ_PANZOOM_FLAGS_FIXED_Y; @@ -1108,8 +1185,35 @@ void Datoviz_Visual_Backend::apply_camera(const plot::Camera_Descriptor& source) throw std::runtime_error("failed to bind Datoviz camera controller"); applied_camera_ = source; } +bool Datoviz_Visual_Backend::matches_command_structure( + const Scene_3D_Parameters& scene, + const Prepared_Visual_Batch& prepared) const { + if (input_changed_ || figure_extent_ != scene.viewport || + !applied_camera_ || *applied_camera_ != scene.camera || + !applied_axes_ || *applied_axes_ != + std::array{scene.x_axis, scene.y_axis, scene.z_axis} || + prepared.size() != visuals_.size()) + return false; + for (const auto& source : prepared) { + const auto target = std::ranges::find_if( + visuals_, [&](const Visual_Instance& value) { + return value.identity == source.identity; + }); + if (target == visuals_.end()) + return false; + if (uses_external_attributes(target->family)) { + if (!target->applied || + !same_visual_structure(*target->applied, source.visual)) + return false; + } + else if (target->applied_revision != source.visual.revision) + return false; + } + return true; +} void Datoviz_Visual_Backend::apply( - const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& prepared) { + const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& prepared, + std::uint8_t target_index, bool bind_target) { require_domain(); if (figure_extent_ != scene.viewport) { if (dvz_figure_resize(figure_, scene.viewport.width, @@ -1132,18 +1236,163 @@ void Datoviz_Visual_Backend::apply( }); if (found == prepared.end()) throw std::logic_error("3D frame contains an unknown or missing Visual identity"); - apply_visual(target, found->visual); + apply_visual(target, found->visual, target_index, bind_target); } } + +void Datoviz_Visual_Backend::ensure_external_attributes( + Visual_Instance& target, const Prepared_Visual_Data& data) { + const auto item_count = data.positions.size(); + if (item_count == 0) return; + if (item_count > std::numeric_limits::max()) + throw std::length_error("Datoviz external attribute item count exceeds uint32"); + + std::size_t expected_count{}; + if (target.family == Visual_Family::mesh) expected_count = 3; + else if (target.family == Visual_Family::marker) expected_count = 5; + else throw std::logic_error("external attributes requested for an unsupported Visual family"); + if (target.attributes.size() == expected_count && + std::ranges::all_of(target.attributes, [&](const External_Attribute& value) { + return value.capacity >= item_count; + })) + return; + + std::uint64_t capacity = 1; + while (capacity < item_count) capacity *= 2; + if (capacity > std::numeric_limits::max()) + throw std::length_error("Datoviz external attribute capacity exceeds uint32"); + + std::vector attributes; + attributes.reserve(expected_count); + const auto create = [&](const char* name, std::uint32_t stride) { + const std::uint64_t byte_size = capacity * stride * Frame_Targets::count; + if (byte_size == 0 || byte_size > std::numeric_limits::max()) + throw std::length_error("Datoviz external attribute buffer is too large"); + auto descriptor = dvz_scene_buffer_desc(); + descriptor.usage = DVZ_SCENE_BUFFER_USAGE_VERTEX; + descriptor.stride = stride; + descriptor.byte_size = byte_size; + auto* scene_buffer = dvz_scene_buffer(scene_, &descriptor); + if (scene_buffer == nullptr) + throw std::runtime_error("failed to create Datoviz external scene buffer"); + + auto* gpu_buffer = allocate_wrapper( + dvz_buffer_create_wrapper, + "failed to allocate Datoviz external GPU buffer wrapper"); + auto* gpu_context = render_context_->gpu_context(); + dvz_buffer(dvz_gpu_ctx_device(gpu_context), dvz_gpu_ctx_alloc(gpu_context), + gpu_buffer); + dvz_buffer_size(gpu_buffer, static_cast(byte_size)); + dvz_buffer_usage(gpu_buffer, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); + dvz_buffer_flags(gpu_buffer, DVZ_ALLOC_HOST_ACCESS_SEQUENTIAL_WRITE); + if (dvz_buffer_create(gpu_buffer) != DVZ_OK) { + dvz_buffer_free(gpu_buffer); + throw std::runtime_error("failed to create Datoviz external GPU buffer"); + } + external_buffers_.push_back(gpu_buffer); + attributes.push_back({name, stride, static_cast(capacity), + scene_buffer, gpu_buffer, false}); + }; + + create("position", sizeof(std::array)); + create("color", sizeof(std::array)); + if (target.family == Visual_Family::mesh) + create("normal", sizeof(std::array)); + else { + create("diameter_px", sizeof(float)); + create("angle", sizeof(float)); + create("shape", sizeof(std::uint32_t)); + } + target.attributes = std::move(attributes); +} + +void Datoviz_Visual_Backend::upload_external_attributes( + Visual_Instance& target, const Prepared_Visual_Data& data, + std::uint8_t target_index) { + if (data.positions.empty()) return; + const auto upload = [&](External_Attribute& attribute, const void* source, + std::size_t count) { + if (source == nullptr || count != data.positions.size()) + throw std::logic_error("Datoviz external attribute fields have different item counts"); + const auto byte_count = static_cast(count * attribute.stride); + const auto byte_offset = static_cast(target_index) * + attribute.capacity * attribute.stride; + dvz_buffer_upload(attribute.gpu_buffer, byte_offset, byte_count, source); + }; + auto attribute = target.attributes.begin(); + upload(*attribute++, data.positions.data(), data.positions.size()); + upload(*attribute++, data.colors.data(), data.colors.size()); + if (target.family == Visual_Family::mesh) + upload(*attribute++, data.normals.data(), data.normals.size()); + else { + upload(*attribute++, data.sizes.data(), data.sizes.size()); + upload(*attribute++, data.angles.data(), data.angles.size()); + upload(*attribute++, data.shapes.data(), data.shapes.size()); + } +} + +void Datoviz_Visual_Backend::bind_external_attributes( + Visual_Instance& target, std::uint8_t target_index, + std::uint32_t item_count) { + for (auto& attribute : target.attributes) { + const std::uint64_t byte_offset = static_cast(target_index) * + attribute.capacity * attribute.stride; + if (dvz_visual_set_attr_buffer(target.visual, attribute.name.c_str(), + attribute.scene_buffer, byte_offset, + item_count) != DVZ_OK) + throw std::runtime_error("failed to bind Datoviz external visual attribute"); + } +} + +void Datoviz_Visual_Backend::register_external_attributes( + const DvzDrp2CommandStream* stream, std::uint8_t target_index) { + if (stream == nullptr) return; + if (target_index >= runtime_slots_.size()) + throw std::logic_error("Datoviz external attribute target is invalid"); + const auto target_bit = static_cast(1U << target_index); + for (auto& visual : visuals_) { + for (auto& attribute : visual.attributes) { + if ((attribute.registered_targets & target_bit) != 0) continue; + char key[DVZ_SCENE_LABEL_SIZE]{}; + if (!dvz_scene_buffer_resource_key(attribute.scene_buffer, key, + sizeof(key))) + throw std::runtime_error("failed to resolve Datoviz external buffer key"); + const auto id = dvz_drp2_stream_label_id(stream, key); + if (id == 0) + throw std::runtime_error("Datoviz frame omitted an external buffer label"); + DvzSceneBufferDesc scene_descriptor{}; + if (!dvz_scene_buffer_info(attribute.scene_buffer, + &scene_descriptor)) + throw std::runtime_error("failed to inspect Datoviz external buffer"); + auto descriptor = dvz_drp2_external_buffer_desc(); + descriptor.buffer = attribute.gpu_buffer; + descriptor.size = scene_descriptor.byte_size; + descriptor.usage = DVZ_DRP2_BUFFER_USAGE_VERTEX; + if (!dvz_drp2_runtime_register_external_buffer( + runtime_slots_[target_index].runtime, id, + &descriptor)) + throw std::runtime_error("failed to register Datoviz external GPU buffer"); + attribute.registered_targets |= target_bit; + } + } +} + void Datoviz_Visual_Backend::apply_visual( - Visual_Instance& target, const Prepared_Visual& point) { + Visual_Instance& target, const Prepared_Visual& point, + std::uint8_t target_index, bool bind_target) { if (point.family != target.family) throw std::logic_error("3D Visual family changed after Scene construction"); if (!point.data) throw std::logic_error("3D prepared visual has no immutable payload"); - if (point.revision == target.applied_revision) return; + if (!uses_external_attributes(target.family) && + point.revision == target.applied_revision) return; const auto& data = *point.data; auto* visual = target.visual; - if (target.coordinate_text != nullptr) { + const bool structure_changed = !target.applied || + !same_visual_structure(*target.applied, point); + const bool coordinate_text_changed = target.coordinate_text != nullptr && + (!target.applied || has_coordinate_labels(*target.applied->data) || + has_coordinate_labels(data)); + if (coordinate_text_changed) { std::vector strings; std::vector items; if (point.visible) { @@ -1178,7 +1427,7 @@ void Datoviz_Visual_Backend::apply_visual( static_cast(items.size())) != DVZ_OK) throw std::runtime_error("failed to upload Datoviz marker coordinate labels"); } - { + if (structure_changed) { mat4 transform{}; for (std::size_t row = 0; row < 4; ++row) for (std::size_t column = 0; column < 4; ++column) transform[row][column] = point.transform.values[row * 4 + column]; @@ -1201,9 +1450,22 @@ void Datoviz_Visual_Backend::apply_visual( if (data.positions.empty()) { if (dvz_visual_set_visible(visual, false) != DVZ_OK) throw std::runtime_error("failed to hide empty Datoviz point visual"); target.applied_revision = point.revision; + target.applied = point; return; } const auto count = static_cast(data.positions.size()); + if (uses_external_attributes(target.family)) { + ensure_external_attributes(target, data); + if (bind_target) + bind_external_attributes(target, target_index, count); + upload_external_attributes(target, data, target_index); + if (structure_changed && + dvz_visual_set_visible(visual, point.visible) != DVZ_OK) + throw std::runtime_error("failed to apply Datoviz external visual visibility"); + target.applied_revision = point.revision; + target.applied = point; + return; + } DvzResult result = DVZ_OK; switch (point.family) { case Visual_Family::point: { @@ -1330,6 +1592,7 @@ void Datoviz_Visual_Backend::apply_visual( dvz_visual_set_visible(visual, point.visible) != DVZ_OK) throw std::runtime_error("failed to upload Datoviz visual payload"); target.applied_revision = point.revision; + target.applied = point; } void Datoviz_Visual_Backend::dispatch_pointer( ::aethera::Event_Type event, float x, float y, @@ -1338,7 +1601,7 @@ void Datoviz_Visual_Backend::dispatch_pointer( require_domain(); std::lock_guard api_lock(render_context_->api_mutex()); input_changed_ = true; - if (applied_camera_ && applied_camera_->controller == plot::Camera_Controller::turntable) { + if (applied_camera_ && applied_camera_->controller == Camera_Controller::turntable) { if (mouse_button == ::aethera::Mouse_Button::left && !applied_camera_->turntable_control.rotate_enabled) return; if ((mouse_button == ::aethera::Mouse_Button::middle || @@ -1365,7 +1628,7 @@ void Datoviz_Visual_Backend::dispatch_wheel( std::lock_guard api_lock(render_context_->api_mutex()); input_changed_ = true; if (applied_camera_ && - applied_camera_->controller == plot::Camera_Controller::turntable && + applied_camera_->controller == Camera_Controller::turntable && !applied_camera_->turntable_control.zoom_enabled) return; dvz_pointer_emit_wheel( input_router_, x, y, static_cast(viewport.width), @@ -1409,7 +1672,10 @@ void Datoviz_Visual_Backend::dispatch_key( modifiers(event.modifiers), nullptr); } DvzSceneFrameArtifact* Datoviz_Visual_Backend::emit( - const Scene_3D_Parameters& scene) { + const Scene_3D_Parameters& scene, std::uint8_t target_index) { + if (target_index >= runtime_slots_.size() || + runtime_slots_[target_index].emitter == nullptr) + throw std::logic_error("Datoviz frame target emitter is unavailable"); DvzFramePlanEmitConfig configuration = dvz_frame_plan_emit_config(); configuration.shader_format = DVZ_SCENE_SHADER_FORMAT_GLSL; configuration.external_color_target = true; @@ -1424,8 +1690,14 @@ DvzSceneFrameArtifact* Datoviz_Visual_Backend::emit( const auto capabilities = offscreen_capabilities(); DvzDiagnosticReport report{}; dvz_diagnostic_report_init(&report); - auto* artifact = - dvz_figure_emit_frame(figure_, &capabilities, &report, &configuration); + /* Every frame target owns a DRP2 runtime and its matching retained emitter. + * A stream is incremental relative to exactly one runtime; sharing an emitter + * across the three targets makes target N receive commands that assume target + * N-1's resource state. Replaying retained payloads also compensates for the + * Scene-wide dirty flags committed by another target's preceding emission. */ + auto* artifact = dvz_figure_emit_frame_with_emitter( + figure_, runtime_slots_[target_index].emitter, true, + &capabilities, &report, &configuration); if (artifact == nullptr) { std::string message = "failed to emit Datoviz visual frame"; const auto count = dvz_diagnostic_report_count(&report); @@ -1472,9 +1744,15 @@ std::optional Datoviz_Visual_Backend::pre trace.render_sequence = frame_sequence; trace.observed = observe; std::uint64_t phase_started = observe ? trace_now_ns() : 0; - apply(scene, visuals); + const bool content_changed = !matches_command_structure(scene, visuals); + if (content_changed) ++command_revision_; + auto& frame_target = *targets_->values[*target_index]; + const bool bind_target = + !frame_target.can_reuse(command_revision_, observe, readback); + apply(scene, visuals, *target_index, bind_target); if (item_interaction_ != nullptr) { - static_cast(dvz_figure_process_queries(figure_, runtime_, nullptr)); + static_cast(dvz_figure_process_queries( + figure_, runtime_slots_[*target_index].runtime, nullptr)); DvzQueryResult query{}; bool resolved{}; while (dvz_scene_poll_query(scene_, &query)) resolved = true; @@ -1505,8 +1783,22 @@ std::optional Datoviz_Visual_Backend::pre } } if (observe) trace.apply_ns = trace_now_ns() - phase_started; - auto& frame_target = *targets_->values[*target_index]; - frame_target.begin(observe, readback); + /* A stable Scene does not need another Figure artifact or another DRP2 + * execution. Each of the three targets owns its recorded command buffer; + * once that target has seen the current content revision, the render-domain + * handoff is reduced to fence reset plus vkQueueSubmit. Data, camera, + * viewport and input revisions remain authoritative in the backend and + * invalidate all older target recordings through command_revision_. */ + if (frame_target.can_reuse(command_revision_, observe, readback)) { + frame_target.reuse(command_revision_, observe, readback); + input_changed_ = false; + return Pending_Frame{ + frame_target.device(), frame_target.fence(), scene.viewport, + frame_sequence, *target_index, frame_target.generation(), + std::move(trace) + }; + } + frame_target.begin(observe, readback, command_revision_); struct Recording_Scope { Frame_Target& target; /* 异常退出时回收尚未发布的录制槽。 */ bool released{}; /* finish_recording 成功后禁止回滚。 */ @@ -1515,7 +1807,7 @@ std::optional Datoviz_Visual_Backend::pre DvzSceneFrameArtifact* artifact{}; try { if (observe) phase_started = trace_now_ns(); - artifact = emit(scene); + artifact = emit(scene, *target_index); if (observe) trace.emit_ns = trace_now_ns() - phase_started; } catch (...) { @@ -1531,14 +1823,15 @@ std::optional Datoviz_Visual_Backend::pre dvz_scene_frame_artifact_frame_index(artifact); } const DvzDrp2CommandStream* stream = dvz_scene_frame_artifact_stream(artifact); + register_external_attributes(stream, *target_index); const DvzStreamFrame target_frame = frame_target.stream_frame(); if (observe) phase_started = trace_now_ns(); const bool attached = stream != nullptr && dvz_drp2_runtime_attach_frame_target( - runtime_, color_target_id, &target_frame); + runtime_slots_[*target_index].runtime, color_target_id, &target_frame); const DvzDrp2ValidationResult result = attached - ? dvz_drp2_runtime_execute(runtime_, stream) + ? dvz_drp2_runtime_execute(runtime_slots_[*target_index].runtime, stream) : DvzDrp2ValidationResult{}; if (observe) trace.execute_ns = trace_now_ns() - phase_started; trace.validation_ok = attached && result.ok; @@ -1562,11 +1855,9 @@ std::optional Datoviz_Visual_Backend::pre "failed to execute Datoviz point frame: validation code " + std::to_string(static_cast(result.code)) + ", command " + std::to_string(result.command_index); - if (!trace.artifact_json.empty()) { - message += ", artifact " + - trace.artifact_json.substr( - 0, std::min(trace.artifact_json.size(), 2048)); - } + if (!trace.artifact_json.empty()) + message += ", failing command " + artifact_command_excerpt( + trace.artifact_json, result.command_index); throw std::runtime_error(std::move(message)); } frame_target.finish_recording(); @@ -1615,23 +1906,9 @@ bool Datoviz_Visual_Backend::can_prepare( const bool all_targets_available = std::ranges::all_of( targets_->values, [](const auto& target) { return !target || target->available(); - }); + }); if (all_targets_available) return true; - if (input_changed_ || figure_extent_ != scene.viewport || - !applied_camera_ || *applied_camera_ != scene.camera || - !applied_axes_ || *applied_axes_ != - std::array{scene.x_axis, scene.y_axis, scene.z_axis}) - return false; - for (const auto& source : prepared) { - const auto target = std::ranges::find_if( - visuals_, [&](const Visual_Instance& value) { - return value.identity == source.identity; - }); - if (target == visuals_.end() || - target->applied_revision != source.visual.revision) - return false; - } - return prepared.size() == visuals_.size(); + return matches_command_structure(scene, prepared); } bool Datoviz_Visual_Backend::idle() const { if (!render_context_) return true; @@ -1647,11 +1924,23 @@ void Datoviz_Visual_Backend::destroy() { auto context = render_context_; std::unique_lock api_lock; if (context) api_lock = std::unique_lock(context->api_mutex()); - if (runtime_ != nullptr) { - dvz_drp2_runtime_destroy(runtime_); - runtime_ = nullptr; + for (auto& slot : runtime_slots_) { + if (slot.runtime != nullptr) { + dvz_drp2_runtime_destroy(slot.runtime); + slot.runtime = nullptr; + } + if (slot.emitter != nullptr) { + dvz_frame_plan_emitter_destroy(slot.emitter); + slot.emitter = nullptr; + } } for (auto& target : targets_->values) target.reset(); + for (auto* buffer : external_buffers_) { + if (buffer == nullptr) continue; + dvz_buffer_destroy(buffer); + dvz_buffer_free(buffer); + } + external_buffers_.clear(); if (camera_controller_ != nullptr) { dvz_controller_destroy(camera_controller_); camera_controller_ = nullptr; diff --git a/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp b/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp index cb149d4..bd05f76 100644 --- a/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp +++ b/render_3D/render_3D/detail/Datoviz_Visual_Backend.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include namespace aethera::render_3d::detail { @@ -59,32 +60,63 @@ public: private: class Frame_Target; struct Frame_Targets; + struct External_Attribute { + std::string name{}; /* Datoviz attribute semantic owned by this binding. */ + std::uint32_t stride{}; /* One planar attribute item in bytes. */ + std::uint32_t capacity{}; /* Items per independently writable frame region. */ + DvzSceneBuffer* scene_buffer{}; /* Scene-side stable resource label; Scene owns it. */ + DvzBuffer* gpu_buffer{}; /* Three-region runtime buffer owned by this backend. */ + std::uint8_t registered_targets{}; /* Runtime slots that have borrowed gpu_buffer. */ + }; struct Visual_Instance { Visual_Identity identity{}; /* Scene 注册的稳定身份。 */ Visual_Family family{Visual_Family::point}; /* 原生 Visual 的确定 family。 */ DvzVisual* visual{}; /* 由 Datoviz Scene 拥有。 */ DvzText* coordinate_text{}; /* Marker 的数据坐标 XYZ 标注。 */ std::uint64_t applied_revision{}; /* 此 Visual 已上传的 Prepare 版本。 */ + std::optional applied{}; /* Last applied metadata and immutable payload snapshot. */ + std::vector attributes{}; /* Dynamic payload buffers, one triple region per field. */ }; void require_domain() const; void create_scene(const std::vector& visuals, const Scene_3D_Parameters& initial_scene); [[nodiscard]] DvzVisual* create_visual(Visual_Family family); - void apply_camera(const plot::Camera_Descriptor& camera); + void apply_camera(const Camera_Descriptor& camera); void apply_axes(const Scene_3D_Parameters& scene); - void apply(const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& visuals); - void apply_visual(Visual_Instance& target, const Prepared_Visual& visual); - [[nodiscard]] DvzSceneFrameArtifact* emit(const Scene_3D_Parameters& scene); + [[nodiscard]] bool matches_command_structure( + const Scene_3D_Parameters& scene, + const Prepared_Visual_Batch& visuals) const; + void apply(const Scene_3D_Parameters& scene, const Prepared_Visual_Batch& visuals, + std::uint8_t target_index, bool bind_target); + void apply_visual(Visual_Instance& target, const Prepared_Visual& visual, + std::uint8_t target_index, bool bind_target); + void ensure_external_attributes(Visual_Instance& target, + const Prepared_Visual_Data& data); + void upload_external_attributes(Visual_Instance& target, + const Prepared_Visual_Data& data, + std::uint8_t target_index); + void bind_external_attributes(Visual_Instance& target, + std::uint8_t target_index, + std::uint32_t item_count); + void register_external_attributes(const DvzDrp2CommandStream* stream, + std::uint8_t target_index); + [[nodiscard]] DvzSceneFrameArtifact* emit( + const Scene_3D_Parameters& scene, std::uint8_t target_index); [[nodiscard]] std::optional acquire_target(Extent extent); [[nodiscard]] Frame_Target& target(const Pending_Frame& pending); void destroy(); std::thread::id domain_thread_; /* 唯一允许访问 Datoviz 对象的线程。 */ std::shared_ptr render_context_; /* 同一 GPU 上所有后端共享的 Device 与分配器。 */ - DvzDrp2Runtime* runtime_{}; /* DRP2 Vulkan 运行时；由本类拥有。 */ + struct Runtime_Slot { + DvzDrp2Runtime* runtime{}; /* Executes only the matching frame target's stream. */ + DvzFramePlanEmitter* emitter{}; /* Retained semantic state paired one-to-one with runtime. */ + }; + std::array runtime_slots_{}; /* Three independent runtime/emitter state machines. */ DvzScene* scene_{}; /* Datoviz Scene；由本类拥有。 */ DvzFigure* figure_{}; /* 当前离屏 Figure。 */ DvzPanel* panel_{}; /* 承载全部业务 Visual 的全屏 Panel。 */ std::vector visuals_{}; /* 按 Scene 稳定身份管理的原生 Visual。 */ + std::vector external_buffers_{}; /* Runtime-borrowed buffers retained through runtime destruction. */ DvzVisual* axes_visual_{}; /* 三维主轴、刻度和网格 Segment Visual。 */ DvzText* axes_text_{}; /* 随相机变换的三维刻度与轴标题。 */ DvzItemInteraction* item_interaction_{}; /* Datoviz 原生图元悬停与选择控制器。 */ @@ -95,7 +127,8 @@ private: std::unique_ptr targets_; /* 三个可并行处于准备、GPU 和读回阶段的目标。 */ Extent figure_extent_{}; /* Datoviz Figure 当前应用的像素尺寸。 */ std::uint64_t target_generation_{}; /* 每次重建 target_ 时递增的资源代次。 */ - std::optional applied_camera_{}; /* 已应用到 Panel 的 Camera 配置。 */ + std::uint64_t command_revision_{1}; /* Scene/Visual/Input 任一变化时递增，决定录制命令是否可复用。 */ + std::optional applied_camera_{}; /* 已应用到 Panel 的 Camera 配置。 */ std::optional> applied_axes_{}; /* 已生成 Visual 的轴描述快照。 */ bool input_changed_{}; /* 输入控制器是否产生尚未录入完成帧的资源更新。 */ }; diff --git a/render_3D/render_3D/scene/Render_Scene_3D.hpp b/render_3D/render_3D/scene/Render_Scene_3D.hpp index 6adfd3c..5b6a1e1 100644 --- a/render_3D/render_3D/scene/Render_Scene_3D.hpp +++ b/render_3D/render_3D/scene/Render_Scene_3D.hpp @@ -49,7 +49,7 @@ struct Render_Scene_3D : Def visuals{}; /* 本 Scene 的全部 Visual，注册顺序保持稳定。 */ Root* camera{}; /* 不拥有的 Camera 组件；生命周期必须覆盖 Scene。 */ Root* axes{}; /* 不拥有的三轴组件；生命周期必须覆盖 Scene。 */ - using Camera_Read = plot::Camera_Descriptor (*)(const Root*); + using Camera_Read = Camera_Descriptor (*)(const Root*); using Axes_Read = std::array (*)(const Root*); Camera_Read read_camera{}; Axes_Read read_axes{}; diff --git a/render_3D/render_3D/scene/Render_Scene_3D.ipp b/render_3D/render_3D/scene/Render_Scene_3D.ipp index d5a64d6..9d7bb76 100644 --- a/render_3D/render_3D/scene/Render_Scene_3D.ipp +++ b/render_3D/render_3D/scene/Render_Scene_3D.ipp @@ -28,7 +28,7 @@ struct Render_Scene_3D::Private : Prev_Private { std::shared_ptr paint_context{}; /* Paint 节点读取 Scene 当前 Prop 的生命周期门闩。 */ Root* camera_component{}; /* Builder 绑定的 Camera 组件。 */ Root* axes_component{}; /* Builder 绑定的三轴组件。 */ - plot::Camera_Descriptor (*read_camera)(const Root*){}; /* 读取 Camera 当前配置。 */ + Camera_Descriptor (*read_camera)(const Root*){}; /* 读取 Camera 当前配置。 */ std::array (*read_axes)(const Root*){}; /* 读取三轴当前配置。 */ Frame_3D* active_frame{}; /* 当前同步 process 借用的外部帧；提交完成后清空。 */ const Dispatch* dispatch{}; /* 最终 Scene 类型对应的静态公开分派表。 */ @@ -36,7 +36,7 @@ struct Render_Scene_3D::Private : Prev_Private { template void initialize_backend(Object* object, std::uint32_t gpu_index, bool validation_enabled, std::vector visuals, Root* camera, Root* axes, - plot::Camera_Descriptor (*camera_reader)(const Root*), + Camera_Descriptor (*camera_reader)(const Root*), std::array (*axes_reader)(const Root*)); template [[nodiscard]] detail::Scene_3D_Parameters parameters(Object* object) const; /* CRTP 覆盖：绑定 Scene 机制和 Render_Scene_3D 公开薄壳。 */ @@ -84,7 +84,7 @@ Render_Scene_3D::Builder::add_camera(Camera_Object* camera_value) { camera = camera_value; read_camera = [](const Root* root) { const auto& prop = static_cast(root)->template read_prop(); - return plot::Camera_Descriptor{prop.initial_view, prop.projection, prop.controller, + return Camera_Descriptor{prop.initial_view, prop.projection, prop.controller, prop.turntable_control, prop.arcball_control, prop.fly_control, prop.panzoom_control, prop.vertical_field_of_view_degrees, @@ -141,7 +141,7 @@ template void Render_Scene_3D::Private::initialize_backend( Object* object, std::uint32_t gpu_index, bool validation_enabled, std::vector visuals, - Root* camera, Root* axes, plot::Camera_Descriptor (*camera_reader)(const Root*), + Root* camera, Root* axes, Camera_Descriptor (*camera_reader)(const Root*), std::array (*axes_reader)(const Root*)) { camera_component = camera; axes_component = axes; diff --git a/render_3D/third_party/datoviz/include/datoviz/scene.h b/render_3D/third_party/datoviz/include/datoviz/scene.h index 8c1122b..b24323b 100644 --- a/render_3D/third_party/datoviz/include/datoviz/scene.h +++ b/render_3D/third_party/datoviz/include/datoviz/scene.h @@ -495,6 +495,29 @@ DVZ_EXPORT DvzSceneFrameArtifact* dvz_figure_emit_frame( const DvzFramePlanEmitConfig* cfg); +/** + * Emit an immutable frame artifact for one explicit retained emitter/runtime pair. + * + * Multiple independent DRP2 runtimes must never consume successive incremental streams from one + * emitter. Give every runtime its own emitter and set `replay_payloads` when another emitter may + * have committed the Scene's global dirty flags since this emitter last emitted. + * + * The caller owns `emitter`; this function neither stores nor destroys it. + * + * @param figure the figure + * @param emitter the persistent emitter paired with the destination runtime + * @param replay_payloads whether to replay current retained CPU payloads into this runtime + * @param caps the capability snapshot (nullable) + * @param report output diagnostic report (nullable) + * @param cfg the emission configuration (nullable) + * @return an owned frame artifact, or NULL on failure + */ +DVZ_EXPORT DvzSceneFrameArtifact* dvz_figure_emit_frame_with_emitter( + DvzFigure* figure, DvzFramePlanEmitter* emitter, bool replay_payloads, + const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, + const DvzFramePlanEmitConfig* cfg); + + /** * Destroy a frame artifact. * diff --git a/render_3D/third_party/datoviz/src/scene/core/_scene.h b/render_3D/third_party/datoviz/src/scene/core/_scene.h index e2a7e35..61ed135 100644 --- a/render_3D/third_party/datoviz/src/scene/core/_scene.h +++ b/render_3D/third_party/datoviz/src/scene/core/_scene.h @@ -384,6 +384,8 @@ DvzId _scene_next_id(DvzScene* scene); bool _scene_runtime_emitter_reset(DvzScene* scene); +void _scene_mark_runtime_payloads_dirty(DvzScene* scene); + struct DvzPanelView2DResolved { float view_extent[4]; diff --git a/render_3D/third_party/datoviz/src/scene/core/figure_emit.c b/render_3D/third_party/datoviz/src/scene/core/figure_emit.c index 7949e5f..299eb83 100644 --- a/render_3D/third_party/datoviz/src/scene/core/figure_emit.c +++ b/render_3D/third_party/datoviz/src/scene/core/figure_emit.c @@ -679,14 +679,13 @@ bool _scene_visual_mutation_allowed(const DvzScene* scene, const char* action) } -DvzDrp2CommandStream* _scene_figure_emit_stream_ex( - DvzFigure* figure, const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, - const DvzFramePlanEmitConfig* cfg) +DvzDrp2CommandStream* _scene_figure_emit_stream_with_emitter_ex( + DvzFigure* figure, DvzFramePlanEmitter* emitter, const DvzCapabilitySnapshot* caps, + DvzDiagnosticReport* report, const DvzFramePlanEmitConfig* cfg) { ANN(figure); ANN(figure->scene); - ANN(figure->scene->emitter); - DvzFramePlanEmitter* emitter = figure->scene->emitter; + ANN(emitter); DvzCapabilitySnapshot default_caps; DvzDiagnosticReport local_report; @@ -794,6 +793,19 @@ DvzDrp2CommandStream* _scene_figure_emit_stream_ex( +DvzDrp2CommandStream* _scene_figure_emit_stream_ex( + DvzFigure* figure, const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, + const DvzFramePlanEmitConfig* cfg) +{ + ANN(figure); + ANN(figure->scene); + ANN(figure->scene->emitter); + return _scene_figure_emit_stream_with_emitter_ex( + figure, figure->scene->emitter, caps, report, cfg); +} + + + DvzSceneFrameArtifact* dvz_figure_emit_frame( DvzFigure* figure, const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, const DvzFramePlanEmitConfig* cfg) @@ -813,6 +825,35 @@ DvzSceneFrameArtifact* dvz_figure_emit_frame( +DvzSceneFrameArtifact* dvz_figure_emit_frame_with_emitter( + DvzFigure* figure, DvzFramePlanEmitter* emitter, bool replay_payloads, + const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, + const DvzFramePlanEmitConfig* cfg) +{ + ANN(figure); + ANN(figure->scene); + ANN(emitter); + if (replay_payloads) + _scene_mark_runtime_payloads_dirty(figure->scene); + + const uint64_t resource_version = figure->artifact_resource_version + 1; + const uint64_t frame_index = figure->artifact_frame_index + 1; + DvzDrp2CommandStream* stream = _scene_figure_emit_stream_with_emitter_ex( + figure, emitter, caps, report, cfg); + if (stream == NULL) + return NULL; + DvzSceneFrameArtifact* artifact = + _scene_frame_artifact(stream, resource_version, frame_index); + if (artifact != NULL) + { + figure->artifact_resource_version = resource_version; + figure->artifact_frame_index = frame_index; + } + return artifact; +} + + + void dvz_scene_frame_artifact_destroy(DvzSceneFrameArtifact* artifact) { _scene_frame_artifact_destroy(artifact); diff --git a/render_3D/third_party/datoviz/src/scene/core/figure_emit_internal.h b/render_3D/third_party/datoviz/src/scene/core/figure_emit_internal.h index 5bb9443..2be2f99 100644 --- a/render_3D/third_party/datoviz/src/scene/core/figure_emit_internal.h +++ b/render_3D/third_party/datoviz/src/scene/core/figure_emit_internal.h @@ -19,3 +19,7 @@ bool _scene_figure_has_pending_render_work(const DvzFigure* figure); DvzDrp2CommandStream* _scene_figure_emit_stream_ex( DvzFigure* figure, const DvzCapabilitySnapshot* caps, DvzDiagnosticReport* report, const DvzFramePlanEmitConfig* cfg); + +DvzDrp2CommandStream* _scene_figure_emit_stream_with_emitter_ex( + DvzFigure* figure, DvzFramePlanEmitter* emitter, const DvzCapabilitySnapshot* caps, + DvzDiagnosticReport* report, const DvzFramePlanEmitConfig* cfg); diff --git a/render_3D/third_party/datoviz/src/scene/core/scene.c b/render_3D/third_party/datoviz/src/scene/core/scene.c index ed30864..7b14890 100644 --- a/render_3D/third_party/datoviz/src/scene/core/scene.c +++ b/render_3D/third_party/datoviz/src/scene/core/scene.c @@ -218,7 +218,7 @@ static void _scene_mark_visual_runtime_dirty(DvzVisual* visual) * * @param scene the scene */ -static void _scene_mark_runtime_payloads_dirty(DvzScene* scene) +void _scene_mark_runtime_payloads_dirty(DvzScene* scene) { ANN(scene); for (uint32_t i = 0; i < scene->visual_count; i++) diff --git a/render_3D/third_party/datoviz/src/scene/visuals/_visual_pipeline_internal.h b/render_3D/third_party/datoviz/src/scene/visuals/_visual_pipeline_internal.h index 526b1dd..bb1015d 100644 --- a/render_3D/third_party/datoviz/src/scene/visuals/_visual_pipeline_internal.h +++ b/render_3D/third_party/datoviz/src/scene/visuals/_visual_pipeline_internal.h @@ -59,6 +59,8 @@ bool _scene_visual_desc_finish_index( bool _scene_visual_has_dense_attr(const DvzVisual* visual, const char* name); +bool _scene_visual_has_bound_attr(const DvzVisual* visual, const char* name); + bool _scene_visual_desc_is_primitive(DvzSceneVisualDescKind kind); bool _scene_visual_desc_is_textured_mesh(DvzSceneVisualDescKind kind); diff --git a/render_3D/third_party/datoviz/src/scene/visuals/mesh/lowering.c b/render_3D/third_party/datoviz/src/scene/visuals/mesh/lowering.c index 7ac3391..4e30f32 100644 --- a/render_3D/third_party/datoviz/src/scene/visuals/mesh/lowering.c +++ b/render_3D/third_party/datoviz/src/scene/visuals/mesh/lowering.c @@ -45,8 +45,8 @@ bool _scene_mesh_visual_lowering(const DvzVisual* visual, DvzVisualLowering* out ? _scene_visual_family_desc_kind(DVZ_VISUAL_TYPE_MESH) : _scene_visual_family_desc_kind(DVZ_VISUAL_TYPE_PRIMITIVE); out->needs_material_params = - _scene_visual_has_dense_attr(visual, "normal") || - _scene_visual_has_dense_attr(visual, "item_state"); + _scene_visual_has_bound_attr(visual, "normal") || + _scene_visual_has_bound_attr(visual, "item_state"); return true; } diff --git a/render_3D/third_party/datoviz/src/scene/visuals/primitive/lowering.c b/render_3D/third_party/datoviz/src/scene/visuals/primitive/lowering.c index 2db5737..d3e3720 100644 --- a/render_3D/third_party/datoviz/src/scene/visuals/primitive/lowering.c +++ b/render_3D/third_party/datoviz/src/scene/visuals/primitive/lowering.c @@ -41,7 +41,7 @@ bool _scene_primitive_visual_lowering(const DvzVisual* visual, DvzVisualLowering out->draw_position_attr = "position"; out->renderable_kind = DVZ_RENDERABLE_INDEXED_MESH; out->desc_kind = DVZ_SCENE_VISUAL_DESC_PRIMITIVE; - out->needs_material_params = _scene_visual_has_dense_attr(visual, "normal"); + out->needs_material_params = _scene_visual_has_bound_attr(visual, "normal"); return true; } diff --git a/render_3D/third_party/datoviz/src/scene/visuals/registry/desc_kind.c b/render_3D/third_party/datoviz/src/scene/visuals/registry/desc_kind.c index 537c527..5ad6963 100644 --- a/render_3D/third_party/datoviz/src/scene/visuals/registry/desc_kind.c +++ b/render_3D/third_party/datoviz/src/scene/visuals/registry/desc_kind.c @@ -201,6 +201,26 @@ bool _scene_visual_has_dense_attr(const DvzVisual* visual, const char* name) } +/** + * Return whether one retained visual has an attribute payload available to the render pipeline. + * + * Unlike dense-data consumers, pipeline capability resolution must also recognize Scene buffers: + * their payload is deliberately absent from host memory when the application owns the GPU buffer. + * + * @param visual the retained visual + * @param name the attribute name + * @return whether dense data or an external Scene buffer is bound + */ +bool _scene_visual_has_bound_attr(const DvzVisual* visual, const char* name) +{ + ANN(visual); + ANN(name); + int attr_idx = _attr_index(visual, name); + return attr_idx >= 0 && visual->attrs[attr_idx].item_count > 0 && + (visual->attrs[attr_idx].data != NULL || visual->attrs[attr_idx].buffer != NULL); +} + + /** * Return whether a visual descriptor uses the primitive pipeline family. * diff --git a/render_3D/third_party/datoviz/src/scene/visuals/registry/pass_caps.c b/render_3D/third_party/datoviz/src/scene/visuals/registry/pass_caps.c index 2cc0e5f..b43fc63 100644 --- a/render_3D/third_party/datoviz/src/scene/visuals/registry/pass_caps.c +++ b/render_3D/third_party/datoviz/src/scene/visuals/registry/pass_caps.c @@ -131,7 +131,7 @@ bool _scene_visual_default_pass_caps( DvzSceneVisualDescKind kind = lowering->desc_kind; bool has_normals = (_scene_visual_desc_is_primitive(kind) || kind == DVZ_SCENE_VISUAL_DESC_TEXTURED_MESH) && - _scene_visual_has_dense_attr(visual, "normal"); + _scene_visual_has_bound_attr(visual, "normal"); bool point_like = kind == DVZ_SCENE_VISUAL_DESC_POINT || kind == DVZ_SCENE_VISUAL_DESC_PIXEL || kind == DVZ_SCENE_VISUAL_DESC_MARKER; diff --git a/web_server/main.cmake b/web_server/main.cmake index f7de5f5..ee298e0 100644 --- a/web_server/main.cmake +++ b/web_server/main.cmake @@ -136,11 +136,9 @@ target_link_libraries(Aethera_Web_Server PRIVATE add_dependencies(Aethera_Web_Server Aethera_Web_Assets) add_custom_command(TARGET Aethera_Web_Server POST_BUILD COMMAND "${CMAKE_COMMAND}" -E copy_if_different - "$" - "$" - "$" + ${Aethera_FFmpeg_runtime_libraries} "$" - COMMENT "Deploying FFmpeg runtime for Aethera WebRTC video" + COMMENT "Deploying complete FFmpeg shared runtime for Aethera WebRTC video" VERBATIM) if (MSVC) target_compile_options(Aethera_Web_Server PRIVATE /utf-8 /bigobj) diff --git a/web_server/src/Gallery_Plots_3D.cpp b/web_server/src/Gallery_Plots_3D.cpp index e18966c..29f1028 100644 --- a/web_server/src/Gallery_Plots_3D.cpp +++ b/web_server/src/Gallery_Plots_3D.cpp @@ -367,7 +367,7 @@ private: }; struct Scene_Components_3D { - plot::Camera_Descriptor camera{}; + Camera_Descriptor camera{}; std::array axes{ plot::Axis_Descriptor{{-1.0, 1.0}, plot::Axis_Scale::linear, "X", "", 5, 2, true, true, true}, plot::Axis_Descriptor{{-1.0, 1.0}, plot::Axis_Scale::linear, "Y", "", 5, 2, true, true, true}, @@ -713,9 +713,9 @@ std::shared_ptr make_datoviz_spectrogram_plot(asio::any_io_executor execut auto marker_result = Impl::Builder{} .set(&Marker_Visual::Prop::items, std::vector{ {{-0.35F, -0.18F, 0.72F}, color(255, 244, 170), 23.0F, 0.0F, - Marker_Shape::diamond, true}, + Marker_Shape::diamond, false}, {{0.28F, 0.34F, 0.86F}, color(88, 236, 211), 21.0F, 0.0F, - Marker_Shape::cross, true}}) + Marker_Shape::cross, false}}) .set(&Marker_Visual::Prop::depth_test, false) .build(); if (!marker_result) diff --git a/web_server/src/Graph_WebSocket.cpp b/web_server/src/Graph_WebSocket.cpp index 612749c..f86f943 100644 --- a/web_server/src/Graph_WebSocket.cpp +++ b/web_server/src/Graph_WebSocket.cpp @@ -1,5 +1,9 @@ #include "Graph_WebSocket.hpp" #include "WebRtc_Video_Session.hpp" +#include +#include +#include +#include #include #include #include @@ -7,6 +11,8 @@ #include #include #include +#include +#include namespace aethera::web { namespace { @@ -14,9 +20,56 @@ std::string graph_id_from_path(std::string_view path) { const auto split = path.find_last_of('/'); return split == std::string_view::npos ? std::string{} : std::string(path.substr(split + 1)); } + +struct Exclusive_Page_State { + std::mutex mutex; + std::string page; + std::size_t connection_count{}; +}; + +Exclusive_Page_State exclusive_page; + +asio::any_io_executor media_delivery_executor() { + /* rtc::Track 与 Drogon 发送可能产生媒体背压，禁止占用 Plot 编码域。 + * 每个 Graph 仍通过自己的 strand 保持 H.264/RTP 顺序。 */ + static asio::thread_pool pool([] { + const auto hardware = std::max(2U, std::thread::hardware_concurrency()); + return std::min(8U, std::max(2U, hardware / 2U)); + }()); + return pool.get_executor(); +} + +bool valid_page_id(std::string_view page) { + return page.size() >= 16 && page.size() <= 64 && + std::ranges::all_of(page, [](unsigned char value) { + return std::isalnum(value) || value == '-' || value == '_'; + }); +} + +std::uint32_t video_dimension(std::uint32_t value, std::uint32_t minimum, + std::uint32_t maximum) { + const auto clamped = std::clamp(value, minimum, maximum); + return std::clamp(((clamped + 16U) / 32U) * 32U, minimum, maximum) & ~1U; +} + +bool acquire_page(std::string_view page) { + std::lock_guard lock(exclusive_page.mutex); + if (exclusive_page.connection_count != 0 && exclusive_page.page != page) return false; + if (exclusive_page.connection_count == 0) exclusive_page.page = page; + ++exclusive_page.connection_count; + return true; +} + +void release_page(std::string_view page) { + std::lock_guard lock(exclusive_page.mutex); + if (exclusive_page.page != page || exclusive_page.connection_count == 0) return; + --exclusive_page.connection_count; + if (exclusive_page.connection_count == 0) exclusive_page.page.clear(); +} } struct Graph_WebSocket::Private { + Private() : delivery_strand(asio::make_strand(media_delivery_executor())) {} std::weak_ptr connection; std::shared_ptr plot; std::unique_ptr video; @@ -25,37 +78,92 @@ struct Graph_WebSocket::Private { std::mutex viewport_mutex; std::uint32_t width{720}; std::uint32_t height{420}; + std::string page_id; /* 同一网页的多图连接共享一个独占租约。 */ + asio::strand delivery_strand; /* 当前 Graph 唯一的媒体发送顺序域。 */ + std::mutex pending_mutex; + std::shared_ptr pending_frame; /* 媒体背压时只保留尚未发送的最新完成帧。 */ + bool delivery_scheduled{}; /* pending_frame 是否已有唯一消费任务。 */ }; Graph_WebSocket::Graph_WebSocket(drogon::WebSocketConnectionPtr connection, - std::shared_ptr plot) + std::shared_ptr plot, std::string page_id) : d(std::make_unique()) { d->connection = connection; d->plot = std::move(plot); + d->page_id = std::move(page_id); } Graph_WebSocket::~Graph_WebSocket() { close(); } void Graph_WebSocket::start() { if (d->attached.exchange(true, std::memory_order_acq_rel)) return; const auto weak = weak_from_this(); - d->video = std::make_unique([weak](std::string signal) { - const auto socket = weak.lock(); - if (!socket) return; - const auto connection = socket->d->connection.lock(); - if (connection && connection->connected()) - connection->send(std::move(signal), drogon::WebSocketMessageType::Text); - }); - d->stream = d->plot->subscribe([weak](std::shared_ptr frame) { - const auto socket = weak.lock(); - if (!socket || !socket->d->attached.load(std::memory_order_acquire)) return; - const auto connection = socket->d->connection.lock(); - if (!connection || !connection->connected()) return; - if (frame->video) socket->d->video->send(*frame->video); - connection->send(frame->metadata, drogon::WebSocketMessageType::Text); - }); + d->video = std::make_unique( + [weak](std::string signal) { + const auto socket = weak.lock(); + if (!socket) return; + const auto connection = socket->d->connection.lock(); + if (connection && connection->connected()) + connection->send(std::move(signal), drogon::WebSocketMessageType::Text); + }, + [weak] { + if (const auto socket = weak.lock()) + socket->d->plot->request_video_key_frame(); + }); + d->stream = d->plot->subscribe( + [weak](std::shared_ptr frame) { + if (const auto socket = weak.lock()) + socket->enqueue_frame(std::move(frame)); + }); d->video->start(); } +void Graph_WebSocket::enqueue_frame( + std::shared_ptr frame) { + if (!frame || !d->attached.load(std::memory_order_acquire)) return; + bool schedule{}; + { + std::lock_guard lock(d->pending_mutex); + d->pending_frame = std::move(frame); + if (!d->delivery_scheduled) { + d->delivery_scheduled = true; + schedule = true; + } + } + if (!schedule) return; + const auto weak = weak_from_this(); + asio::post(d->delivery_strand, [weak] { + if (const auto socket = weak.lock()) socket->deliver_frame(); + }); +} + +void Graph_WebSocket::deliver_frame() { + std::shared_ptr frame; + { + std::lock_guard lock(d->pending_mutex); + frame = std::move(d->pending_frame); + } + if (frame && d->attached.load(std::memory_order_acquire)) { + const auto connection = d->connection.lock(); + if (connection && connection->connected()) { + if (frame->video) + static_cast(d->video->send(*frame->video)); + connection->send(frame->metadata, + drogon::WebSocketMessageType::Text); + } + } + bool schedule{}; + { + std::lock_guard lock(d->pending_mutex); + if (d->pending_frame) schedule = true; + else d->delivery_scheduled = false; + } + if (!schedule) return; + const auto weak = weak_from_this(); + asio::post(d->delivery_strand, [weak] { + if (const auto socket = weak.lock()) socket->deliver_frame(); + }); +} + void Graph_WebSocket::receive(std::string_view message) { const auto json = nlohmann::json::parse(message, nullptr, false); if (json.is_discarded() || !json.is_object()) return; @@ -73,21 +181,19 @@ void Graph_WebSocket::receive(std::string_view message) { return; } if (kind == "stream") { - const bool active = json.value("active", false); - const bool video = json.value("video", true); std::uint32_t width{720}; std::uint32_t height{420}; if (const auto viewport = json.find("viewport"); viewport != json.end() && viewport->is_object()) { - width = std::clamp(viewport->value("width", 720U), 160U, 1920U); - height = std::clamp(viewport->value("height", 420U), 120U, 1080U); + width = video_dimension(viewport->value("width", 720U), 160U, 1920U); + height = video_dimension(viewport->value("height", 420U), 128U, 1080U); } { std::lock_guard lock(d->viewport_mutex); d->width = width; d->height = height; } - d->plot->configure_stream(d->stream, active, video, width, height); + d->plot->configure_stream(d->stream, width, height); return; } if (kind == "manual_render") { @@ -144,7 +250,12 @@ void Graph_WebSocket::receive(std::string_view message) { void Graph_WebSocket::close() { if (!d->attached.exchange(false, std::memory_order_acq_rel)) return; d->plot->unsubscribe(d->stream); + { + std::lock_guard lock(d->pending_mutex); + d->pending_frame.reset(); + } if (d->video) d->video->close(); + release_page(d->page_id); } Graph_WebSocket_Controller::Graph_WebSocket_Controller(Plot_Resolver resolver) @@ -157,7 +268,17 @@ void Graph_WebSocket_Controller::handleNewConnection( connection->shutdown(drogon::CloseCode::kViolation, "Unknown Aethera plot"); return; } - auto socket = std::make_shared(connection, std::move(plot)); + const auto page_id = request->getParameter("page"); + if (!valid_page_id(page_id) || !acquire_page(page_id)) { + connection->send(nlohmann::json{{"kind", "exclusive_page_rejected"}, + {"message", "Aethera Gallery is already owned by another page instance"}}.dump(), + drogon::WebSocketMessageType::Text); + connection->shutdown(drogon::CloseCode::kViolation, + "Aethera Gallery allows one page instance"); + return; + } + auto socket = std::make_shared( + connection, std::move(plot), page_id); connection->setContext(socket); connection->setPingMessage("aethera-gallery", std::chrono::seconds(20)); socket->start(); diff --git a/web_server/src/Graph_WebSocket.hpp b/web_server/src/Graph_WebSocket.hpp index 7b523b3..12f7edf 100644 --- a/web_server/src/Graph_WebSocket.hpp +++ b/web_server/src/Graph_WebSocket.hpp @@ -9,7 +9,8 @@ namespace aethera::web { class Graph_WebSocket final : public std::enable_shared_from_this { public: - Graph_WebSocket(drogon::WebSocketConnectionPtr connection, std::shared_ptr plot); + Graph_WebSocket(drogon::WebSocketConnectionPtr connection, + std::shared_ptr plot, std::string page_id); ~Graph_WebSocket(); Graph_WebSocket(const Graph_WebSocket&) = delete; Graph_WebSocket& operator=(const Graph_WebSocket&) = delete; @@ -17,6 +18,8 @@ public: void receive(std::string_view message); void close(); private: + void enqueue_frame(std::shared_ptr frame); + void deliver_frame(); struct Private; std::unique_ptr d; }; diff --git a/web_server/src/H264_Encoder.cpp b/web_server/src/H264_Encoder.cpp index 3c32f0f..d16484d 100644 --- a/web_server/src/H264_Encoder.cpp +++ b/web_server/src/H264_Encoder.cpp @@ -32,7 +32,7 @@ struct H264_Encoder::Private { std::uint32_t width{}; /* 当前编码尺寸；尺寸变化时整体重建编码器。 */ std::uint32_t height{}; /* 当前编码尺寸；H.264 4:2:0 要求偶数。 */ Video_Pixel_Layout layout{Video_Pixel_Layout::bgra}; /* scaler 当前输入格式。 */ - bool first_frame{true}; /* 编码器重建后必须立即产生可独立解码的关键帧。 */ + bool key_frame_requested{true}; /* Track 新建或恢复时，下一 access unit 必须可独立解码。 */ explicit Private(double value_frame_rate) : frame_rate(value_frame_rate) { if (!std::isfinite(frame_rate) || frame_rate <= 0.0) @@ -47,7 +47,7 @@ struct H264_Encoder::Private { avcodec_free_context(&codec_context); width = 0; height = 0; - first_frame = true; + key_frame_requested = true; } void configure(std::uint32_t next_width, std::uint32_t next_height, Video_Pixel_Layout next_layout) { @@ -103,6 +103,10 @@ H264_Encoder::H264_Encoder(double frame_rate) : d(std::make_unique(frame_rate)) {} H264_Encoder::~H264_Encoder() = default; +void H264_Encoder::request_key_frame() { + d->key_frame_requested = true; +} + std::shared_ptr H264_Encoder::encode( std::span pixels, std::uint32_t width, std::uint32_t height, Video_Pixel_Layout layout, @@ -118,7 +122,9 @@ std::shared_ptr H264_Encoder::encode( d->yuv_frame->data, d->yuv_frame->linesize) != static_cast(height)) throw std::runtime_error("FFmpeg did not convert the complete video frame"); d->yuv_frame->pts = presentation_time.count(); - d->yuv_frame->pict_type = d->first_frame ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; + d->yuv_frame->pict_type = d->key_frame_requested + ? AV_PICTURE_TYPE_I + : AV_PICTURE_TYPE_NONE; require_ffmpeg(avcodec_send_frame(d->codec_context, d->yuv_frame), "submitting frame to H.264 encoder"); const int received = avcodec_receive_packet(d->codec_context, d->packet); @@ -130,7 +136,7 @@ std::shared_ptr H264_Encoder::encode( output->presentation_time = presentation_time; output->sequence = sequence; output->key_frame = (d->packet->flags & AV_PKT_FLAG_KEY) != 0; - d->first_frame = false; + d->key_frame_requested = false; av_packet_unref(d->packet); return output; } diff --git a/web_server/src/H264_Encoder.hpp b/web_server/src/H264_Encoder.hpp index da7a520..9f5b7a7 100644 --- a/web_server/src/H264_Encoder.hpp +++ b/web_server/src/H264_Encoder.hpp @@ -25,6 +25,7 @@ public: ~H264_Encoder(); H264_Encoder(const H264_Encoder&) = delete; H264_Encoder& operator=(const H264_Encoder&) = delete; + void request_key_frame(); [[nodiscard]] std::shared_ptr encode( std::span pixels, std::uint32_t width, std::uint32_t height, Video_Pixel_Layout layout, diff --git a/web_server/src/Plot.cpp b/web_server/src/Plot.cpp index a734cf2..8672b54 100644 --- a/web_server/src/Plot.cpp +++ b/web_server/src/Plot.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +34,17 @@ using Scene_2D = Impl; using Scene_3D = Impl; constexpr std::uint16_t frame_protocol_version{7}; +asio::any_io_executor video_encoding_executor() { + /* 编码与 Plot 帧时钟/2D 绘制是不同的 CPU 资源域。每个 Plot 的 strand + * 仍保证单个 FFmpeg 上下文串行，独立池只让不同 Plot 并行编码，避免 + * 多路视频把图调度和属性命令一起堵在 Web graph pool 中。 */ + static asio::thread_pool pool([] { + const auto hardware = std::max(2U, std::thread::hardware_concurrency()); + return std::min(8U, std::max(2U, hardware / 2U)); + }()); + return pool.get_executor(); +} + enum class Frame_Pacing_Mode : std::uint8_t { manual, fixed_rate, @@ -39,8 +52,10 @@ enum class Frame_Pacing_Mode : std::uint8_t { }; struct Frame_Pacing_Properties { + bool render_enabled{true}; + bool video_enabled{true}; Frame_Pacing_Mode mode{Frame_Pacing_Mode::fixed_rate}; /* 唯一职责是决定何时调用 Scene::render。 */ - double fixed_rate_fps{30.0}; /* 不参与帧完成、编码或网络回调。 */ + double fixed_rate_fps{100.0}; /* 不参与帧完成、编码或网络回调。 */ }; class Frame_Policy final { @@ -78,6 +93,14 @@ nlohmann::json Frame_Policy::schema() const { return { {"id", "frame-analysis"}, {"label", "渲染与传输分析"}, {"kind", "analysis"}, {"fields", nlohmann::json::array({ + {{"key", "render_enabled"}, {"label", "持续渲染与采样"}, {"editor", "boolean"}, + {"editable", true}, {"description", "控制后端是否继续调用 Scene::render；浏览器画面隐藏不会修改此项。"}, + {"technical_description", "Authoritative server-side render and sampling switch."}, + {"value", pacing.render_enabled}}, + {{"key", "video_enabled"}, {"label", "WebRTC 视频传输"}, {"editor", "boolean"}, + {"editable", true}, {"description", "控制完成帧是否编码为 H.264 并通过 WebRTC 发送；默认开启以进行完整链路压测。"}, + {"technical_description", "Authoritative server-side video encoding and delivery switch."}, + {"value", pacing.video_enabled}}, {{"key", "pacing_mode"}, {"label", "服务端帧策略"}, {"editor", "select"}, {"editable", true}, {"description", "只控制服务端调用 Scene::render 的节奏；完成回调始终直接发布。"}, {"technical_description", "Server render clock policy; independent from completion and WebRTC delivery."}, @@ -98,6 +121,14 @@ nlohmann::json Frame_Policy::schema() const { nlohmann::json Frame_Policy::write_prop(std::string_view key, const nlohmann::json& value) { std::lock_guard lock(mutex); + if (key == "render_enabled" || key == "video_enabled") { + if (!value.is_boolean()) + return {{"success", false}, {"error", "frame policy switch requires a boolean"}}; + bool& target = key == "render_enabled" ? pacing.render_enabled : pacing.video_enabled; + target = value.get(); + return {{"success", true}, {"component", "frame-analysis"}, {"key", key}, + {"value", target}}; + } if (key == "pacing_mode") { if (!value.is_string()) return {{"success", false}, {"error", "pacing_mode requires a string"}}; const auto parsed = parse_pacing_mode(value.get_ref()); @@ -140,7 +171,9 @@ nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, {"video", {{"codec", "H264"}, {"transport", "WebRTC"}, {"encoded_bytes", encoded_bytes}}}, {"pacing", {{"mode", pacing_mode_name(pacing.mode)}, - {"fixed_rate_fps", pacing.fixed_rate_fps}}}, + {"fixed_rate_fps", pacing.fixed_rate_fps}, + {"render_enabled", pacing.render_enabled}, + {"video_enabled", pacing.video_enabled}}}, {"trace", {{"clock", "steady_elapsed_ns"}, {"markers", std::move(markers)}, {"measurements", std::move(measurements)}}} }; @@ -206,22 +239,18 @@ struct Plot::Private { using Scene = std::variant, std::unique_ptr>; using Frame = std::variant, std::unique_ptr>; struct Managed_Frame { - Plot_Render_Tick tick{}; + std::chrono::microseconds presentation_time{}; /* Plot 帧时钟产生的媒体时间戳。 */ Frame frame{}; /* Scene 借用，Plot 保存到完成回调返回。 */ }; struct Consumer { Stream_Handler handler; - bool render_enabled{}; - bool video_enabled{}; std::uint32_t width{}; std::uint32_t height{}; }; struct Stream_Snapshot { std::vector consumers; - bool render_enabled{}; - bool video_enabled{}; - std::uint32_t width{720}; - std::uint32_t height{420}; + std::uint32_t width{}; + std::uint32_t height{}; }; asio::strand strand; @@ -236,28 +265,36 @@ struct Plot::Private { std::atomic_uint64_t next_stream_id{1}; std::uint64_t next_frame_sequence{1}; Frame_Policy frame_policy{}; - H264_Encoder encoder{60.0}; - std::unordered_map active_frames{}; - std::size_t encoding_frames{}; /* 已离开 Scene、仍占用三缓冲槽的帧数。 */ + H264_Encoder encoder{100.0}; + std::atomic_uint64_t last_stream_publish_tail_ns{}; /* 最近一帧编码结束到消费者投递返回的诊断值。 */ + mutable std::mutex frame_mutex; /* 三缓冲帧所有权从 Plot 时钟移交给 Scene 回调与编码域。 */ + std::unordered_map active_frames{}; /* Scene 当前借用、尚未完成回调的帧。 */ + std::shared_ptr pending_encoding_frame{}; /* 编码器忙时唯一保留的最新完成帧。 */ + bool encoder_running{}; /* 当前是否有且仅有一个编码任务占用本 Plot 编码器。 */ + static constexpr std::size_t scene_frame_capacity = 3; /* Scene/GPU 独立三缓冲；媒体域另有正在处理与最新待处理两个有界槽位。 */ std::chrono::steady_clock::time_point clock_origin{std::chrono::steady_clock::now()}; template Private(asio::any_io_executor executor, std::unique_ptr value_scene, std::unique_ptr value_view) - : strand(asio::make_strand(executor)), encode_strand(asio::make_strand(std::move(executor))), + : strand(asio::make_strand(executor)), + encode_strand(asio::make_strand(video_encoding_executor())), commands(strand, 32), render_clock(strand), view(std::move(value_view)), scene(std::move(value_scene)) {} [[nodiscard]] nlohmann::json schema() const; [[nodiscard]] Stream_Snapshot stream_snapshot() const; - [[nodiscard]] std::size_t frame_capacity() const noexcept; [[nodiscard]] std::chrono::steady_clock::duration clock_interval() const; + void start_clock(); void schedule_clock(); void clock_tick(); void render_frame(); void queue_completed_frame(Render_Frame* frame, std::weak_ptr lifetime); + void schedule_encoding(std::shared_ptr managed, + std::weak_ptr lifetime); void encode_and_publish(std::shared_ptr managed, std::weak_ptr lifetime); + void finish_encoding(std::weak_ptr lifetime); }; nlohmann::json Plot::Private::schema() const { @@ -275,10 +312,8 @@ Plot::Private::Stream_Snapshot Plot::Private::stream_snapshot() const { result.consumers.reserve(consumers.size()); for (const auto& [id, consumer] : consumers) { static_cast(id); + if (consumer.width == 0 || consumer.height == 0) continue; result.consumers.push_back(consumer); - if (!consumer.render_enabled) continue; - result.render_enabled = true; - result.video_enabled = result.video_enabled || consumer.video_enabled; result.width = std::max(result.width, consumer.width); result.height = std::max(result.height, consumer.height); } @@ -287,10 +322,6 @@ Plot::Private::Stream_Snapshot Plot::Private::stream_snapshot() const { return result; } -std::size_t Plot::Private::frame_capacity() const noexcept { - return 3U; -} - std::chrono::steady_clock::duration Plot::Private::clock_interval() const { const auto pacing = frame_policy.snapshot(); if (pacing.mode == Frame_Pacing_Mode::manual) return std::chrono::milliseconds(100); @@ -299,26 +330,51 @@ std::chrono::steady_clock::duration Plot::Private::clock_interval() const { std::chrono::duration(1.0 / pacing.fixed_rate_fps)); } -void Plot::Private::schedule_clock() { +void Plot::Private::start_clock() { render_clock.expires_after(clock_interval()); + schedule_clock(); +} + +void Plot::Private::schedule_clock() { render_clock.async_wait([this](const asio::error_code& error) { if (error) return; - clock_tick(); + /* + * The frame rate is a timeline frequency, not a sleep duration after render work. Advance + * from the previous absolute deadline so CPU preparation cannot accumulate clock drift. + * When a deadline was missed, skip the elapsed periods instead of issuing a catch-up burst. + */ + const auto interval = clock_interval(); + auto next_deadline = render_clock.expiry() + interval; + const auto now = std::chrono::steady_clock::now(); + if (next_deadline <= now) { + const auto missed_periods = (now - next_deadline) / interval + 1; + next_deadline += interval * missed_periods; + } + render_clock.expires_at(next_deadline); schedule_clock(); + clock_tick(); }); } void Plot::Private::clock_tick() { const auto pacing = frame_policy.snapshot(); - if (pacing.mode == Frame_Pacing_Mode::manual) return; - if (active_frames.size() + encoding_frames < frame_capacity() && - stream_snapshot().render_enabled) - render_frame(); + if (!pacing.render_enabled || pacing.mode == Frame_Pacing_Mode::manual) return; + { + std::lock_guard lock(frame_mutex); + /* 已完成帧属于媒体域，不能反向占用 Scene/GPU 三缓冲槽位。 */ + if (active_frames.size() >= scene_frame_capacity) return; + } + if (!stream_snapshot().consumers.empty()) render_frame(); } void Plot::Private::render_frame() { const auto streams = stream_snapshot(); - if (!streams.render_enabled || active_frames.size() + encoding_frames >= frame_capacity()) return; + const auto pacing = frame_policy.snapshot(); + if (!pacing.render_enabled || streams.consumers.empty()) return; + { + std::lock_guard lock(frame_mutex); + if (active_frames.size() >= scene_frame_capacity) return; + } const auto elapsed = std::chrono::steady_clock::now() - clock_origin; Plot_Render_Tick tick{next_frame_sequence, std::chrono::duration(elapsed).count(), streams.width, streams.height}; @@ -327,43 +383,77 @@ void Plot::Private::render_frame() { view->update(tick); /* CPU 数据准备发生在 render 提交之前。 */ Managed_Frame managed; - managed.tick = tick; + managed.presentation_time = std::chrono::duration_cast( + std::chrono::duration(tick.time_milliseconds)); if (std::holds_alternative>(scene)) managed.frame = std::make_unique(identity, Frame_2D::native_pixel_format); else managed.frame = std::make_unique(identity, - streams.video_enabled + pacing.video_enabled ? Frame_3D_Output::pixels : Frame_3D_Output::diagnostics, Frame_3D::native_pixel_format); Render_Frame* address = std::visit([](const auto& value) -> Render_Frame* { return value.get(); }, managed.frame); - auto [active, inserted] = active_frames.emplace(address, std::move(managed)); - if (!inserted) throw std::logic_error("Plot received a duplicate frame address"); + { + std::lock_guard lock(frame_mutex); + const auto [active, inserted] = + active_frames.emplace(address, std::move(managed)); + if (!inserted) + throw std::logic_error("Plot received a duplicate frame address"); + } if (auto* scene_2d = std::get_if>(&scene)) { (*scene_2d)->set<&Render_Scene_2D::Prop::viewport>( Size{static_cast(tick.width), static_cast(tick.height)}); const auto result = (*scene_2d)->render( - std::get>(active->second.frame).get()); - if (result != Render_Scene_2D::Render_Result::completed) active_frames.erase(address); + static_cast(address)); + if (result != Render_Scene_2D::Render_Result::completed) { + std::lock_guard lock(frame_mutex); + active_frames.erase(address); + } return; } auto& scene_3d = std::get>(scene); scene_3d->set<&Render_Scene_3D::Prop::viewport>(Extent{tick.width, tick.height}); const auto result = scene_3d->render( - std::get>(active->second.frame).get()); - if (result != Render_Scene_3D::Render_Result::submitted) active_frames.erase(address); + static_cast(address)); + if (result != Render_Scene_3D::Render_Result::submitted) { + std::lock_guard lock(frame_mutex); + active_frames.erase(address); + } } void Plot::Private::queue_completed_frame(Render_Frame* frame, std::weak_ptr lifetime) { - const auto active = active_frames.find(frame); - if (active == active_frames.end()) - throw std::logic_error("frame callback has no externally owned active frame"); - auto managed = std::make_shared(std::move(active->second)); - active_frames.erase(active); - ++encoding_frames; + std::shared_ptr managed; + std::shared_ptr discarded; + std::shared_ptr scheduled; + { + std::lock_guard lock(frame_mutex); + const auto active = active_frames.find(frame); + if (active == active_frames.end()) + throw std::logic_error( + "frame callback has no externally owned active frame"); + managed = std::make_shared(std::move(active->second)); + active_frames.erase(active); + /* + * Scene 的每个完成回调都在这里闭环。视频只需要仍可能被观看的最新完成帧： + * 编码器忙时保留一张最新待编码帧，更新替代的旧帧直接释放，避免过时画面 + * 占满共享编码线程池。正在编码和最新待编码分别构成三缓冲的后两级。 + */ + discarded = std::exchange(pending_encoding_frame, std::move(managed)); + if (!encoder_running) { + encoder_running = true; + scheduled = std::exchange(pending_encoding_frame, {}); + } + } + frame->mark(Frame_Trace_Marker::video_encode_queued); + if (scheduled) schedule_encoding(std::move(scheduled), std::move(lifetime)); +} + +void Plot::Private::schedule_encoding(std::shared_ptr managed, + std::weak_ptr lifetime) { asio::post(encode_strand, [lifetime, managed = std::move(managed)] { if (const auto owner = lifetime.lock()) owner->d->encode_and_publish(managed, lifetime); @@ -372,47 +462,82 @@ void Plot::Private::queue_completed_frame(Render_Frame* frame, void Plot::Private::encode_and_publish(std::shared_ptr managed, std::weak_ptr lifetime) { + struct Encoding_Completion final { + Private& plot; /* 需要归还编码调度权的 Plot 实现。 */ + std::weak_ptr lifetime; /* 下一待编码帧只有在 Plot 存活时才继续调度。 */ + ~Encoding_Completion() { plot.finish_encoding(std::move(lifetime)); } + } completion{*this, lifetime}; const auto streams = stream_snapshot(); + const auto pacing = frame_policy.snapshot(); std::shared_ptr video; + std::uint32_t output_width{}; + std::uint32_t output_height{}; Render_Frame* frame = std::visit( [](const auto& value) -> Render_Frame* { return value.get(); }, managed->frame); + frame->record( + Frame_Trace_Measurement::stream_publish_tail_ns, + last_stream_publish_tail_ns.load(std::memory_order_acquire)); frame->mark(Frame_Trace_Marker::video_encode_started); - if (streams.video_enabled) { - const auto presentation_time = std::chrono::duration_cast( - std::chrono::duration(managed->tick.time_milliseconds)); + if (pacing.video_enabled) { + const auto sequence = frame->identity().sequence; if (auto* frame_2d = std::get_if>(&managed->frame)) { auto pixels = (*frame_2d)->output_pixels(); - video = encoder.encode(pixels.bytes, managed->tick.width, managed->tick.height, - Video_Pixel_Layout::bgra, managed->tick.sequence, - presentation_time); + output_width = static_cast(pixels.width); + output_height = static_cast(pixels.height); + video = encoder.encode(pixels.bytes, output_width, output_height, + Video_Pixel_Layout::bgra, sequence, + managed->presentation_time); } else { auto& frame_3d = std::get>(managed->frame); - video = encoder.encode(frame_3d->pixels(), managed->tick.width, managed->tick.height, - Video_Pixel_Layout::rgba, managed->tick.sequence, - presentation_time); + const auto extent = frame_3d->extent(); + output_width = extent.width; + output_height = extent.height; + /* 三维后端满载时会把多次提交合并为最新画面，并完成全部历史回调。 + * 因此完成 Frame 的 extent 才是像素尺寸的唯一权威，创建时的 tick + * 只表示当时请求的视口，禁止用它解释合并后的共享像素。 */ + if (frame_3d->output() == Frame_3D_Output::pixels) + video = encoder.encode(frame_3d->pixels(), output_width, + output_height, Video_Pixel_Layout::rgba, + sequence, managed->presentation_time); } + } else if (auto* frame_2d = std::get_if>(&managed->frame)) { + const auto image = (*frame_2d)->image(); + output_width = static_cast(image.width); + output_height = static_cast(image.height); + } else { + const auto extent = std::get>(managed->frame)->extent(); + output_width = extent.width; + output_height = extent.height; } frame->mark(Frame_Trace_Marker::video_encode_finished); + const auto publish_tail_started = std::chrono::steady_clock::now(); frame->mark(Frame_Trace_Marker::stream_publish_started); const auto bytes = video ? video->annex_b.size() : 0U; frame->mark(Frame_Trace_Marker::stream_publish_finished); - const auto metadata = frame_metadata(*frame, managed->tick.width, managed->tick.height, - bytes, frame_policy.snapshot()).dump(); - const auto with_video = std::make_shared( - Plot_Stream_Frame{metadata, video}); - const auto diagnostics = std::make_shared( - Plot_Stream_Frame{metadata, {}}); + const auto metadata = frame_metadata(*frame, output_width, output_height, + bytes, pacing).dump(); + const auto published = std::make_shared( + Plot_Stream_Frame{metadata, pacing.video_enabled ? video : nullptr}); for (const auto& consumer : streams.consumers) { - if (!consumer.render_enabled || !consumer.handler) continue; - consumer.handler(consumer.video_enabled ? with_video : diagnostics); + if (!consumer.handler) continue; + consumer.handler(published); } - asio::post(strand, [lifetime] { - if (const auto owner = lifetime.lock()) { - if (owner->d->encoding_frames == 0) - throw std::logic_error("Plot encoding frame accounting underflow"); - --owner->d->encoding_frames; - } - }); + last_stream_publish_tail_ns.store( + static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now() - publish_tail_started).count()), + std::memory_order_release); +} + +void Plot::Private::finish_encoding(std::weak_ptr lifetime) { + std::shared_ptr scheduled; + { + std::lock_guard lock(frame_mutex); + if (!encoder_running) + throw std::logic_error("Plot encoding scheduler completed without an active encoder"); + scheduled = std::exchange(pending_encoding_frame, {}); + if (!scheduled) encoder_running = false; + } + if (scheduled) schedule_encoding(std::move(scheduled), std::move(lifetime)); } Plot::Plot(asio::any_io_executor executor, std::unique_ptr scene, @@ -435,25 +560,17 @@ void Plot::ensure_started() { */ if (auto* scene = std::get_if>(&d->scene)) { (*scene)->set_frame_callback([weak = weak_from_this()](Frame_2D* frame) { - if (auto owner = weak.lock()) { - frame->mark(Frame_Trace_Marker::callback_finished); - asio::post(owner->d->strand, [weak, frame] { - if (auto next = weak.lock()) next->d->queue_completed_frame(frame, next); - }); - } + if (auto owner = weak.lock()) + owner->d->queue_completed_frame(frame, owner); }); } else { std::get>(d->scene)->set_frame_callback( [weak = weak_from_this()](Frame_3D* frame) { - if (auto owner = weak.lock()) { - frame->mark(Frame_Trace_Marker::callback_finished); - asio::post(owner->d->strand, [weak, frame] { - if (auto next = weak.lock()) next->d->queue_completed_frame(frame, next); - }); - } + if (auto owner = weak.lock()) + owner->d->queue_completed_frame(frame, owner); }); } - d->schedule_clock(); + d->start_clock(); asio::co_spawn(d->strand, [self]() -> asio::awaitable { for (;;) { asio::error_code error; @@ -488,17 +605,22 @@ void Plot::unsubscribe(Stream_Id stream) { d->consumers.erase(stream); } -void Plot::configure_stream(Stream_Id stream, bool render_enabled, bool video_enabled, - std::uint32_t width, std::uint32_t height) { +void Plot::configure_stream(Stream_Id stream, std::uint32_t width, std::uint32_t height) { std::lock_guard lock(d->consumers_mutex); const auto found = d->consumers.find(stream); if (found == d->consumers.end()) return; - found->second.render_enabled = render_enabled; - found->second.video_enabled = video_enabled; found->second.width = width; found->second.height = height; } +void Plot::request_video_key_frame() { + ensure_started(); + const auto weak = weak_from_this(); + asio::post(d->encode_strand, [weak] { + if (const auto owner = weak.lock()) owner->d->encoder.request_key_frame(); + }); +} + void Plot::render_once() { ensure_started(); const auto weak = weak_from_this(); diff --git a/web_server/src/Plot.hpp b/web_server/src/Plot.hpp index c305545..5803963 100644 --- a/web_server/src/Plot.hpp +++ b/web_server/src/Plot.hpp @@ -69,8 +69,8 @@ public: [[nodiscard]] Stream_Id subscribe(Stream_Handler handler); void unsubscribe(Stream_Id stream); - void configure_stream(Stream_Id stream, bool render_enabled, bool video_enabled, - std::uint32_t width, std::uint32_t height); + void configure_stream(Stream_Id stream, std::uint32_t width, std::uint32_t height); + void request_video_key_frame(); void render_once(); void submit_input(Plot_Input_Event event); void async_schema(Json_Handler handler); diff --git a/web_server/src/WebRtc_Video_Session.cpp b/web_server/src/WebRtc_Video_Session.cpp index a61e831..ccf2ce3 100644 --- a/web_server/src/WebRtc_Video_Session.cpp +++ b/web_server/src/WebRtc_Video_Session.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace aethera::web { @@ -21,6 +22,7 @@ std::uint32_t make_ssrc() { struct WebRtc_Video_Session::Private { struct Callback_State { Signal_Handler signal_handler; /* WebSocket 只负责 SDP/ICE 信令，不承载帧像素。 */ + Ready_Handler ready_handler; /* Track 打开后请求首个可独立解码的视频帧。 */ std::atomic_bool track_open{}; std::atomic_bool closed{}; }; @@ -28,18 +30,24 @@ struct WebRtc_Video_Session::Private { std::shared_ptr peer; std::shared_ptr video_track; std::shared_ptr rtp_config; + std::mutex media_mutex; /* 关闭与发送之间不得跨越 Track 生命周期。 */ - explicit Private(Signal_Handler value_handler) + Private(Signal_Handler value_signal_handler, + Ready_Handler value_ready_handler) : callbacks(std::make_shared()) { - callbacks->signal_handler = std::move(value_handler); + callbacks->signal_handler = std::move(value_signal_handler); + callbacks->ready_handler = std::move(value_ready_handler); } }; -WebRtc_Video_Session::WebRtc_Video_Session(Signal_Handler signal_handler) - : d(std::make_unique(std::move(signal_handler))) {} +WebRtc_Video_Session::WebRtc_Video_Session( + Signal_Handler signal_handler, Ready_Handler ready_handler) + : d(std::make_unique(std::move(signal_handler), + std::move(ready_handler))) {} WebRtc_Video_Session::~WebRtc_Video_Session() { close(); } void WebRtc_Video_Session::start() { + std::lock_guard lock(d->media_mutex); if (d->peer) return; d->peer = std::make_shared(rtc::Configuration{}); const std::weak_ptr callbacks{d->callbacks}; @@ -71,8 +79,12 @@ void WebRtc_Video_Session::start() { packetizer->addToChain(std::make_shared()); d->video_track->setMediaHandler(packetizer); d->video_track->onOpen([callbacks] { - if (const auto state = callbacks.lock()) + if (const auto state = callbacks.lock()) { state->track_open.store(true, std::memory_order_release); + if (!state->closed.load(std::memory_order_acquire) && + state->ready_handler) + state->ready_handler(); + } }); d->video_track->onClosed([callbacks] { if (const auto state = callbacks.lock()) @@ -82,25 +94,37 @@ void WebRtc_Video_Session::start() { } void WebRtc_Video_Session::accept_answer(std::string_view sdp) { + std::lock_guard lock(d->media_mutex); if (!d->peer) throw std::logic_error("WebRTC session has not started"); d->peer->setRemoteDescription(rtc::Description(std::string(sdp), "answer")); } void WebRtc_Video_Session::add_remote_candidate(std::string_view candidate, std::string_view mid) { + std::lock_guard lock(d->media_mutex); if (!d->peer) throw std::logic_error("WebRTC session has not started"); d->peer->addRemoteCandidate(rtc::Candidate(std::string(candidate), std::string(mid))); } -void WebRtc_Video_Session::send(const Encoded_Video_Frame& frame) { +WebRtc_Video_Session::Send_Result WebRtc_Video_Session::send( + const Encoded_Video_Frame& frame) { + std::lock_guard lock(d->media_mutex); if (!d->video_track || !d->callbacks->track_open.load(std::memory_order_acquire) || - frame.annex_b.empty()) return; - d->video_track->sendFrame( - reinterpret_cast(frame.annex_b.data()), frame.annex_b.size(), - rtc::FrameInfo(std::chrono::duration(frame.presentation_time))); + frame.annex_b.empty()) return Send_Result::not_open; + try { + d->video_track->sendFrame( + reinterpret_cast(frame.annex_b.data()), frame.annex_b.size(), + rtc::FrameInfo(std::chrono::duration(frame.presentation_time))); + return Send_Result::sent; + } catch (const std::runtime_error&) { + /* LibDataChannel 可在 isOpen() 后由其内部线程关闭 Track。 */ + if (!d->video_track->isOpen()) return Send_Result::not_open; + throw; + } } void WebRtc_Video_Session::close() { + std::lock_guard lock(d->media_mutex); if (d->callbacks->closed.exchange(true, std::memory_order_acq_rel)) return; d->callbacks->track_open.store(false, std::memory_order_release); if (d->video_track) d->video_track->close(); diff --git a/web_server/src/WebRtc_Video_Session.hpp b/web_server/src/WebRtc_Video_Session.hpp index d09a819..0b4b760 100644 --- a/web_server/src/WebRtc_Video_Session.hpp +++ b/web_server/src/WebRtc_Video_Session.hpp @@ -8,9 +8,15 @@ namespace aethera::web { class WebRtc_Video_Session final { public: + enum class Send_Result : std::uint8_t { + sent, + not_open + }; using Signal_Handler = std::function; + using Ready_Handler = std::function; - explicit WebRtc_Video_Session(Signal_Handler signal_handler); + WebRtc_Video_Session(Signal_Handler signal_handler, + Ready_Handler ready_handler); ~WebRtc_Video_Session(); WebRtc_Video_Session(const WebRtc_Video_Session&) = delete; WebRtc_Video_Session& operator=(const WebRtc_Video_Session&) = delete; @@ -18,7 +24,7 @@ public: void start(); void accept_answer(std::string_view sdp); void add_remote_candidate(std::string_view candidate, std::string_view mid); - void send(const Encoded_Video_Frame& frame); + [[nodiscard]] Send_Result send(const Encoded_Video_Frame& frame); void close(); private: diff --git a/web_server/src/Web_Server.cpp b/web_server/src/Web_Server.cpp index e049215..6254f0d 100644 --- a/web_server/src/Web_Server.cpp +++ b/web_server/src/Web_Server.cpp @@ -36,34 +36,39 @@ std::shared_ptr find_plot(const Plot_Map& plots, std::string_view id) { int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) { const auto hardware_threads = std::max(2U, std::thread::hardware_concurrency()); - auto graph_pool = std::make_shared(std::min(8U, hardware_threads)); - const auto executor = graph_pool->get_executor(); + const auto plot_workers = std::min(8U, hardware_threads); + /* 2D 的同步 Blend2D 绘制不能占满 3D 的帧时钟与 Visual Prepare 域。 + * 两类 Plot 使用同一套 Plot/帧策略实现，但执行资源按渲染架构隔离。 */ + auto plot_2d_pool = std::make_shared(plot_workers); + auto plot_3d_pool = std::make_shared(plot_workers); + const auto executor_2d = plot_2d_pool->get_executor(); + const auto executor_3d = plot_3d_pool->get_executor(); auto plots = std::make_shared(); - plots->emplace("axes", make_axes_plot(executor)); - plots->emplace("spectrum", make_spectrum_plot(executor)); - plots->emplace("frequency_trace", make_frequency_trace_plot(executor)); - plots->emplace("sweep_spectrum", make_sweep_spectrum_plot(executor)); - plots->emplace("afterglow", make_afterglow_plot(executor)); - plots->emplace("waterfall", make_waterfall_plot(executor)); - plots->emplace("constellation", make_constellation_plot(executor)); - plots->emplace("selection_overlay", make_selection_overlay_plot(executor)); - plots->emplace("datoviz_point", make_datoviz_point_plot(executor)); - plots->emplace("datoviz_splat", make_datoviz_splat_plot(executor)); - plots->emplace("datoviz_pixel", make_datoviz_pixel_plot(executor)); - plots->emplace("datoviz_marker", make_datoviz_marker_plot(executor)); - plots->emplace("datoviz_sphere", make_datoviz_sphere_plot(executor)); - plots->emplace("datoviz_segment", make_datoviz_segment_plot(executor)); - plots->emplace("datoviz_vector", make_datoviz_vector_plot(executor)); - plots->emplace("datoviz_primitive", make_datoviz_primitive_plot(executor)); - plots->emplace("datoviz_mesh", make_datoviz_mesh_plot(executor)); - plots->emplace("datoviz_spectrogram", make_datoviz_spectrogram_plot(executor)); - plots->emplace("datoviz_path", make_datoviz_path_plot(executor)); - plots->emplace("datoviz_image", make_datoviz_image_plot(executor)); - plots->emplace("datoviz_labels", make_datoviz_labels_plot(executor)); - plots->emplace("datoviz_glyph", make_datoviz_glyph_plot(executor)); - plots->emplace("datoviz_text", make_datoviz_text_plot(executor)); - plots->emplace("datoviz_volume", make_datoviz_volume_plot(executor)); + plots->emplace("axes", make_axes_plot(executor_2d)); + plots->emplace("spectrum", make_spectrum_plot(executor_2d)); + plots->emplace("frequency_trace", make_frequency_trace_plot(executor_2d)); + plots->emplace("sweep_spectrum", make_sweep_spectrum_plot(executor_2d)); + plots->emplace("afterglow", make_afterglow_plot(executor_2d)); + plots->emplace("waterfall", make_waterfall_plot(executor_2d)); + plots->emplace("constellation", make_constellation_plot(executor_2d)); + plots->emplace("selection_overlay", make_selection_overlay_plot(executor_2d)); + plots->emplace("datoviz_point", make_datoviz_point_plot(executor_3d)); + plots->emplace("datoviz_splat", make_datoviz_splat_plot(executor_3d)); + plots->emplace("datoviz_pixel", make_datoviz_pixel_plot(executor_3d)); + plots->emplace("datoviz_marker", make_datoviz_marker_plot(executor_3d)); + plots->emplace("datoviz_sphere", make_datoviz_sphere_plot(executor_3d)); + plots->emplace("datoviz_segment", make_datoviz_segment_plot(executor_3d)); + plots->emplace("datoviz_vector", make_datoviz_vector_plot(executor_3d)); + plots->emplace("datoviz_primitive", make_datoviz_primitive_plot(executor_3d)); + plots->emplace("datoviz_mesh", make_datoviz_mesh_plot(executor_3d)); + plots->emplace("datoviz_spectrogram", make_datoviz_spectrogram_plot(executor_3d)); + plots->emplace("datoviz_path", make_datoviz_path_plot(executor_3d)); + plots->emplace("datoviz_image", make_datoviz_image_plot(executor_3d)); + plots->emplace("datoviz_labels", make_datoviz_labels_plot(executor_3d)); + plots->emplace("datoviz_glyph", make_datoviz_glyph_plot(executor_3d)); + plots->emplace("datoviz_text", make_datoviz_text_plot(executor_3d)); + plots->emplace("datoviz_volume", make_datoviz_volume_plot(executor_3d)); auto resolve_plot = [plots](std::string_view id) { return find_plot(*plots, id); }; auto websocket = std::make_shared(resolve_plot); @@ -158,8 +163,10 @@ int run_web_server(std::uint16_t port, const std::filesystem::path& asset_root) .setThreadNum(std::min(8U, hardware_threads)) .setIdleConnectionTimeout(90) .run(); - graph_pool->stop(); - graph_pool->join(); + plot_2d_pool->stop(); + plot_3d_pool->stop(); + plot_2d_pool->join(); + plot_3d_pool->join(); return 0; } } diff --git a/web_server/third_party/ffmpeg.cmake b/web_server/third_party/ffmpeg.cmake index c9d0e7a..7d0e403 100644 --- a/web_server/third_party/ffmpeg.cmake +++ b/web_server/third_party/ffmpeg.cmake @@ -2,6 +2,11 @@ include_guard(GLOBAL) set(Aethera_FFmpeg_root "${CMAKE_CURRENT_LIST_DIR}/ffmpeg-9.0.1-full_build-shared") +file(GLOB Aethera_FFmpeg_runtime_libraries CONFIGURE_DEPENDS + "${Aethera_FFmpeg_root}/bin/*.dll") +if (NOT Aethera_FFmpeg_runtime_libraries) + message(FATAL_ERROR "Aethera FFmpeg runtime libraries were not found") +endif () foreach(component IN ITEMS avutil avcodec swscale) add_library(Aethera_FFmpeg_${component} SHARED IMPORTED GLOBAL) diff --git a/webapp_gallery/src/app.tsx b/webapp_gallery/src/app.tsx index ccd4db9..5bdcce8 100644 --- a/webapp_gallery/src/app.tsx +++ b/webapp_gallery/src/app.tsx @@ -20,7 +20,7 @@ type Color_Channel_Scale = "normalized" | "byte"; type Field = {key: string; label: string; description: string; technical_description?: string; editor: Editor; editable: boolean; value: unknown; options?: Option[]; color_channel_scale?: Color_Channel_Scale; minimum?: number; maximum?: number; step?: number}; type Component = {id: string; label: string; kind: string; fields: Field[]; state: Record}; type Data_Generator = {label: string; description: string; fields: Field[]}; -type Plot_Execution_Policy = {visible: boolean; refresh_hidden: boolean; transfer_pixels: boolean}; +type Plot_Execution_Policy = {visible: boolean}; type Plot_Execution_Policies = Record; type Frame_Analysis = Omit & {data_generator?: Data_Generator}; type Schema = {protocol: "aethera.plot.inspector"; version: 2; components: Component[]; frame_analysis: Frame_Analysis}; @@ -32,23 +32,41 @@ type Pixel_Format = "yuv420p" | "h264-annex-b" | "h264"; type Frame_Metadata = {kind: "frame_metadata"; protocol: "aethera.video.frame"; version: 7; sequence: number; correlation_id: number; delivery: Frame_Delivery; created_time_unix_ms: number; pixel: {width: number; height: number; format: Pixel_Format; native_format: Pixel_Format; supported_formats: Pixel_Format[]; byte_length: number}; - pacing: {mode: Frame_Pacing_Mode; fixed_rate_fps: number}; + pacing: {mode: Frame_Pacing_Mode; fixed_rate_fps: number; render_enabled: boolean; video_enabled: boolean}; trace: {clock: "steady_elapsed_ns"; markers: Record; measurements: Record}}; type Frame_Stage_Values = Record; -type Frame_Sample = {sequence: number; correlation_id: number; delivery: Frame_Delivery; received_at_ms: number; values: Frame_Stage_Values}; -type Frame_Diagnostics = {metadata: Frame_Metadata; samples: Frame_Sample[]; frame_rate_fps: number; interval_jitter_p95_ms: number; +type Frame_Sample = {sequence: number; correlation_id: number; delivery: Frame_Delivery; generated_at_ms: number; received_at_ms: number; values: Frame_Stage_Values}; +type Video_Playback_Metrics = {frame_rate_fps: number; presented_frames: number; dropped_frames: number; current_time_seconds: number; ready_state: number}; +type Frame_Diagnostics = {metadata: Frame_Metadata; samples: Frame_Sample[]; frame_rate_fps: number; delivery_frame_rate_fps: number; interval_jitter_p95_ms: number; request_to_pixels_average_ms: number; request_to_pixels_p50_ms: number; request_to_pixels_p95_ms: number; request_to_pixels_p99_ms: number; - frame_interval_average_ms: number; frame_interval_p95_ms: number; dropped_sequence_count: number; latest: Frame_Stage_Values}; + frame_interval_average_ms: number; frame_interval_p95_ms: number; dropped_sequence_count: number; latest: Frame_Stage_Values; + video_playback: Video_Playback_Metrics}; type Frame_Metrics = {sequence: number; generated_time_unix_ms: number; request_to_pixels_ms: number; average_request_to_pixels_ms: number; p95_request_to_pixels_ms: number; p99_request_to_pixels_ms: number; frame_rate_fps: number; p95_frame_interval_jitter_ms: number; - pacing_mode: Frame_Pacing_Mode; fixed_rate_fps: number; delivery: Frame_Delivery}; -type Frame_Policy_Event = {plot_id: string; key: "pacing_mode" | "fixed_rate_fps"; value: unknown}; + pacing_mode: Frame_Pacing_Mode; fixed_rate_fps: number; delivery: Frame_Delivery; video_playback: Video_Playback_Metrics}; type Stage_Statistic = "average" | "variability" | "p95" | "p99"; type Stage_Unit = "value" | "percentage"; -const default_plot_execution_policy = (): Plot_Execution_Policy => ({visible: true, refresh_hidden: false, transfer_pixels: true}); +const default_plot_execution_policy = (): Plot_Execution_Policy => ({visible: true}); -function socket_url(path: string) { return `${location.protocol === "https:" ? "wss" : "ws"}://${location.host}${path}`; } +const page_instance_id = crypto.randomUUID(); + +function video_viewport(video: HTMLVideoElement) { + const bounds = video.getBoundingClientRect(); + const quantize = (value: number, minimum: number, maximum: number) => + Math.max(minimum, Math.min(maximum, Math.round(value / 32) * 32)); + return { + width: quantize(bounds.width * devicePixelRatio, 160, 1920), + height: quantize(bounds.height * devicePixelRatio, 128, 1080) + }; +} + +function socket_url(path: string) { + const url = new URL(path, location.href); + url.protocol = location.protocol === "https:" ? "wss:" : "ws:"; + url.searchParams.set("page", page_instance_id); + return url.toString(); +} function valid_frame_metadata(value: unknown): value is Frame_Metadata { if (!value || typeof value !== "object") return false; @@ -112,6 +130,7 @@ function frame_stage_values(metadata: Frame_Metadata, request_started_at: number ["gpu_submission_ms", "gpu_submitted", "gpu_completed"], ["readback_stage_ms", "readback_started", "readback_finished"], ["callback_ms", "callback_started", "callback_finished"], + ["video_encode_queue_ms", "video_encode_queued", "video_encode_started"], ["video_encode_ms", "video_encode_started", "video_encode_finished"], ["stream_publish_ms", "stream_publish_started", "stream_publish_finished"] ]; @@ -141,7 +160,13 @@ function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dime const request_to_metadata = Math.min(total, Math.max(0, values.request_to_metadata_ms ?? 0)); const metadata_to_pixels = Math.min(Math.max(0, total - request_to_metadata), Math.max(0, values.metadata_to_pixels_ms ?? 0)); const canvas_upload = Math.min(Math.max(0, total - request_to_metadata - metadata_to_pixels), Math.max(0, values.canvas_upload_ms ?? 0)); - const stages: Frame_Stage_Values = {pipeline_request_transport_ms: Math.max(0, request_to_metadata - server)}; + const publish_tail = Math.min( + Math.max(0, request_to_metadata - server), + Math.max(0, values.stream_publish_tail_ms ?? 0)); + const stages: Frame_Stage_Values = { + pipeline_request_transport_ms: Math.max( + 0, request_to_metadata - server - publish_tail) + }; if (dimension === "2D") { const scene = Math.max(0, values.scene_render_ms ?? 0); const event = Math.min(scene, Math.max(0, values.event_dispatch_ms ?? 0)); @@ -152,8 +177,10 @@ function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dime stages.pipeline_2d_paint_ms = take(paint); stages.pipeline_2d_scene_coordination_ms = take(Math.max(0, scene - event - prepare - paint)); stages.pipeline_2d_callback_ms = take(values.callback_ms ?? 0); + stages.pipeline_2d_encode_queue_ms = take(values.video_encode_queue_ms ?? 0); stages.pipeline_2d_encode_ms = take(values.video_encode_ms ?? 0); stages.pipeline_2d_frame_handoff_ms = remaining; + stages.pipeline_2d_publish_tail_ms = publish_tail; remaining = 0; } else { const scene = Math.max(0, values.scene_render_ms ?? 0); @@ -192,8 +219,10 @@ function pipeline_stage_values(values: Frame_Stage_Values, dimension: Plot["dime stages.pipeline_3d_gpu_sync_ms = take(gpu_window); stages.pipeline_3d_readback_ms = take(values.readback_stage_ms ?? values.readback_ms ?? 0); stages.pipeline_3d_callback_ms = take(values.callback_ms ?? 0); + stages.pipeline_3d_encode_queue_ms = take(values.video_encode_queue_ms ?? 0); stages.pipeline_3d_encode_ms = take(values.video_encode_ms ?? 0); stages.pipeline_3d_completion_handoff_ms = remaining; + stages.pipeline_3d_publish_tail_ms = publish_tail; remaining = 0; } return {...values, ...stages, @@ -211,11 +240,14 @@ function build_frame_diagnostics(metadata: Frame_Metadata, samples: Frame_Sample let dropped_sequence_count = 0; for (let index = 1; index < samples.length; ++index) dropped_sequence_count += Math.max(0, samples[index].correlation_id - samples[index - 1].correlation_id - 1); - const elapsed = samples.length > 1 ? samples.at(-1)!.received_at_ms - samples[0].received_at_ms : 0; + const delivery_elapsed = samples.length > 1 ? samples.at(-1)!.received_at_ms - samples[0].received_at_ms : 0; + const source_elapsed = samples.length > 1 ? samples.at(-1)!.generated_at_ms - samples[0].generated_at_ms : 0; + const source_frames = samples.length > 1 ? samples.at(-1)!.correlation_id - samples[0].correlation_id : 0; return { metadata, samples, - frame_rate_fps: elapsed > 0 ? (samples.length - 1) * 1000 / elapsed : 0, + frame_rate_fps: source_elapsed > 0 ? source_frames * 1000 / source_elapsed : 0, + delivery_frame_rate_fps: delivery_elapsed > 0 ? (samples.length - 1) * 1000 / delivery_elapsed : 0, interval_jitter_p95_ms: percentile(interval_jitter, .95), request_to_pixels_average_ms: average(request_latencies), request_to_pixels_p50_ms: percentile(request_latencies, .5), @@ -224,33 +256,26 @@ function build_frame_diagnostics(metadata: Frame_Metadata, samples: Frame_Sample frame_interval_average_ms: average(intervals), frame_interval_p95_ms: percentile(intervals, .95), dropped_sequence_count, - latest: samples.at(-1)?.values ?? {} + latest: samples.at(-1)?.values ?? {}, + video_playback: {frame_rate_fps: 0, presented_frames: 0, dropped_frames: 0, current_time_seconds: 0, ready_state: 0} }; } -function use_plot_stream(plot: Plot, video_ref: React.RefObject, stream_video: boolean, - enabled: boolean) { - const [status, set_status] = useState(enabled ? "CONNECTING" : "IDLE"); +function use_plot_stream(plot: Plot, video_ref: React.RefObject) { + const [status, set_status] = useState("CONNECTING"); const [metrics, set_metrics] = useState(null); const socket_ref = useRef(null); const pending_pointer_move = useRef | null>(null); const pointer_frame = useRef(0); const viewport_ref = useRef({width: 720, height: 420}); - const stream_video_ref = useRef(stream_video); - - useEffect(() => { - stream_video_ref.current = stream_video; - window.dispatchEvent(new CustomEvent("aethera-frame-delivery", {detail: {plot_id: plot.id}})); - }, [plot.id, stream_video]); + const video_playback_ref = useRef({frame_rate_fps: 0, presented_frames: 0, dropped_frames: 0, current_time_seconds: 0, ready_state: 0}); const envelope = useCallback((kind: "input", event?: Record) => { const video = video_ref.current; if (!video) return null; const bounds = video.getBoundingClientRect(); - if (bounds.width > 0 && bounds.height > 0) viewport_ref.current = { - width: Math.max(1, Math.round(bounds.width * devicePixelRatio)), - height: Math.max(1, Math.round(bounds.height * devicePixelRatio)) - }; + if (bounds.width > 0 && bounds.height > 0) + viewport_ref.current = video_viewport(video); return {kind, viewport: { width: viewport_ref.current.width, height: viewport_ref.current.height }, ...(event ? {event} : {})}; @@ -263,18 +288,15 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { - if (!enabled) { - socket_ref.current = null; - set_status("IDLE"); - return; - } let stopped = false; let diagnostics_timer = 0; + let resize_timer = 0; let previous_diagnostics_publish_time = 0; let latest_metadata: Frame_Metadata | null = null; let samples: Frame_Sample[] = []; let peer: RTCPeerConnection | null = null; let signal_chain = Promise.resolve(); + let sent_viewport = {width: 0, height: 0}; const socket = new ReconnectingWebSocket(socket_url(plot.websocket), [], { minReconnectionDelay: 300, maxReconnectionDelay: 5000, @@ -289,13 +311,13 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject 0 && bounds.height > 0) viewport_ref.current = { - width: Math.max(160, Math.round(bounds.width * devicePixelRatio)), - height: Math.max(120, Math.round(bounds.height * devicePixelRatio)) - }; + if (bounds.width > 0 && bounds.height > 0) + viewport_ref.current = video_viewport(video); } - socket.send(JSON.stringify({kind: "stream", active: true, video: stream_video_ref.current, - viewport: viewport_ref.current})); + if (sent_viewport.width === viewport_ref.current.width && + sent_viewport.height === viewport_ref.current.height) return; + sent_viewport = {...viewport_ref.current}; + socket.send(JSON.stringify({kind: "stream", viewport: viewport_ref.current})); }; const publish_diagnostics = (force: boolean) => { if (!latest_metadata || samples.length === 0) return; @@ -309,7 +331,8 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject sample.delivery === latest_metadata!.delivery); if (delivery_samples.length === 0) return; - const diagnostics = build_frame_diagnostics(latest_metadata, delivery_samples); + const diagnostics = {...build_frame_diagnostics(latest_metadata, delivery_samples), + video_playback: {...video_playback_ref.current}}; const latest = diagnostics.latest; const latest_metrics: Frame_Metrics = { sequence: latest_metadata.sequence, @@ -322,7 +345,8 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { - const detail = (event as CustomEvent<{plot_id: string}>).detail; - if (detail?.plot_id === plot.id) send_stream_control(); - }; window.addEventListener("aethera-manual-frame", on_manual_frame); window.addEventListener("aethera-reset-camera", on_camera_reset); window.addEventListener("aethera-reset-frame-diagnostics", on_diagnostics_reset); - window.addEventListener("aethera-frame-delivery", on_delivery_change); - const resize_observer = new ResizeObserver(send_stream_control); + const resize_observer = new ResizeObserver(() => { + if (resize_timer) window.clearTimeout(resize_timer); + resize_timer = window.setTimeout(send_stream_control, 120); + }); if (video_ref.current) resize_observer.observe(video_ref.current); const create_peer = () => { peer?.close(); @@ -392,7 +415,12 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { create_peer(); set_status("CONNECTING"); send_stream_control(); }; + socket.onopen = () => { + create_peer(); + set_status("CONNECTING"); + sent_viewport = {width: 0, height: 0}; + send_stream_control(); + }; socket.onclose = () => { peer?.close(); peer = null; if (!stopped) set_status("CONNECTING"); }; socket.onmessage = event => { if (typeof event.data !== "string") return; @@ -401,6 +429,12 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { if (!peer) return; if (signal.kind === "webrtc_offer" && signal.sdp) { @@ -417,25 +451,53 @@ function use_plot_stream(plot: Plot, video_ref: React.RefObject { stopped = true; clear_diagnostics_timer(); + if (resize_timer) window.clearTimeout(resize_timer); resize_observer.disconnect(); window.removeEventListener("aethera-manual-frame", on_manual_frame); window.removeEventListener("aethera-reset-camera", on_camera_reset); window.removeEventListener("aethera-reset-frame-diagnostics", on_diagnostics_reset); - window.removeEventListener("aethera-frame-delivery", on_delivery_change); if (pointer_frame.current) cancelAnimationFrame(pointer_frame.current); pointer_frame.current = 0; peer?.close(); socket_ref.current = null; socket.close(); }; - }, [plot.id, plot.websocket, plot.dimension, video_ref, transmit, enabled]); + }, [plot.id, plot.websocket, plot.dimension, video_ref, transmit]); + + useEffect(() => { + const video = video_ref.current; + if (!video || typeof video.requestVideoFrameCallback !== "function") return; + let callback_id = 0; + let sample_started = performance.now(); + let sample_frames = 0; + const observe = (now: number, frame: VideoFrameCallbackMetadata) => { + const quality = video.getVideoPlaybackQuality(); + const presented_frames = frame.presentedFrames; + const elapsed = now - sample_started; + if (elapsed >= 500) { + video_playback_ref.current.frame_rate_fps = Math.max(0, presented_frames - sample_frames) * 1000 / elapsed; + sample_started = now; + sample_frames = presented_frames; + } + video_playback_ref.current.presented_frames = presented_frames; + video_playback_ref.current.dropped_frames = quality.droppedVideoFrames; + video_playback_ref.current.current_time_seconds = video.currentTime; + video_playback_ref.current.ready_state = video.readyState; + callback_id = video.requestVideoFrameCallback(observe); + }; + callback_id = video.requestVideoFrameCallback(observe); + return () => video.cancelVideoFrameCallback(callback_id); + }, [plot.id, video_ref]); useEffect(() => { const video = video_ref.current; if (!video) return; const point = (event: PointerEvent | WheelEvent) => { const bounds = video.getBoundingClientRect(); - return {x: (event.clientX - bounds.left) * devicePixelRatio, y: (event.clientY - bounds.top) * devicePixelRatio}; + return { + x: (event.clientX - bounds.left) * viewport_ref.current.width / bounds.width, + y: (event.clientY - bounds.top) * viewport_ref.current.height / bounds.height + }; }; const global_point = (event: PointerEvent | WheelEvent) => ({x: event.screenX * devicePixelRatio, y: event.screenY * devicePixelRatio}); const modifiers = (event: MouseEvent | KeyboardEvent) => @@ -696,7 +758,7 @@ function State_Field_View({component, field, histories}: {component: Component; } type Pipeline_Stage_Definition = [string, string, string]; -const pipeline_common_start: Pipeline_Stage_Definition = ["pipeline_request_transport_ms", "请求与元数据往返", "浏览器请求、服务端建帧前调度以及完成元数据返回浏览器的合计衔接耗时。"]; +const pipeline_common_start: Pipeline_Stage_Definition = ["pipeline_request_transport_ms", "帧时钟与元数据传递", "服务端创建帧前后的调度，以及完成诊断元数据送达浏览器的合计衔接耗时；当前流程不存在浏览器逐帧请求。"]; const pipeline_common_finish: Pipeline_Stage_Definition[] = [ ["pipeline_payload_transport_ms", "像素传输", "浏览器收到元数据后，直到完整像素载荷到达的耗时。"], ["pipeline_canvas_upload_ms", "视频解码呈现", "浏览器 WebRTC 视频管线的解码与呈现尾部耗时。"], @@ -709,8 +771,10 @@ const pipeline_2d_definitions: Pipeline_Stage_Definition[] = [ ["pipeline_2d_paint_ms", "Blend2D 绘制", "二维 Paint 任务图清屏并写入 Blend2D 帧缓存的耗时。"], ["pipeline_2d_scene_coordination_ms", "2D Scene 编排", "Scene 渲染区间内除事件、Prepare、Paint 外的依赖图编排耗时。"], ["pipeline_2d_callback_ms", "2D 完成回调", "同步二维帧完成后回调到 Plot 发布线程的耗时。"], + ["pipeline_2d_encode_queue_ms", "2D 编码排队", "完成帧等待当前 Plot 编码串行域的耗时；多图满负载时可直接观察编码线程池竞争。"], ["pipeline_2d_encode_ms", "2D 像素封装", "原生 BGRA 帧只进行连续行打包；仅在浏览器不支持 WebGL2 时由 Blend2D 转换为 RGBA。"], - ["pipeline_2d_frame_handoff_ms", "2D 帧建立与发布调度", "Render_Frame 建立、进入 Scene 以及编码完成后生成元数据的调度间隙。"] + ["pipeline_2d_frame_handoff_ms", "2D 帧建立与发布调度", "Render_Frame 建立、进入 Scene 以及编码完成后生成元数据的调度间隙。"], + ["pipeline_2d_publish_tail_ms", "2D 媒体发布尾部", "上一完成帧从编码结束到元数据生成并移交消费者返回的耗时；用于识别媒体背压。"] ]; const pipeline_3d_definitions: Pipeline_Stage_Definition[] = [ pipeline_common_start, @@ -732,8 +796,10 @@ const pipeline_3d_definitions: Pipeline_Stage_Definition[] = [ ["pipeline_3d_gpu_sync_ms", "GPU 同步等待", "GPU 提交到完成区间扣除已测设备阶段后的 fence/调度时间。"], ["pipeline_3d_readback_ms", "3D CPU 回读", "GPU 完成后由 Datoviz 收集并复制 RGBA 像素的耗时。"], ["pipeline_3d_callback_ms", "3D 完成回调", "异步后端完成后回调到 Plot 发布线程的耗时。"], + ["pipeline_3d_encode_queue_ms", "3D 编码排队", "RGBA 完成帧等待当前 Plot 编码串行域的耗时；可识别多 Plot 共享工作线程导致的排队。"], ["pipeline_3d_encode_ms", "3D H.264 编码", "把连续 RGBA 像素转换为 YUV420P 并编码成 H.264 access unit 的耗时。"], - ["pipeline_3d_completion_handoff_ms", "3D 完成调度衔接", "GPU 回读、回调与发布边界之间尚未由 trace marker 单独覆盖的调度衔接时间。"] + ["pipeline_3d_completion_handoff_ms", "3D 完成调度衔接", "GPU 回读、回调与发布边界之间尚未由 trace marker 单独覆盖的调度衔接时间。"], + ["pipeline_3d_publish_tail_ms", "3D 媒体发布尾部", "上一完成帧从编码结束到元数据生成并移交消费者返回的耗时；用于识别媒体背压。"] ]; const pipeline_definitions = (dimension: Plot["dimension"], delivery: Frame_Delivery) => { const core = (dimension === "2D" ? pipeline_2d_definitions : pipeline_3d_definitions).map(definition => { @@ -829,16 +895,20 @@ function Frame_Diagnostics_View({diagnostics, dimension, on_reset}: {diagnostics const total_value = stage_values.reduce((sum, [, , , value]) => sum + (Number.isFinite(value) ? value : 0), 0); const statistic_labels: Record = {average: "滑动平均", variability: "波动", p95: "P95", p99: "P99"}; const pixel_delivery = displayed.metadata.delivery === "webrtc-video"; - const completion = pixel_delivery ? "视频帧元数据到达浏览器" : "诊断元数据到达浏览器"; + const completion = "诊断元数据到达浏览器"; const summaries: Array<[string, string, string]> = [ - [pixel_delivery ? "像素帧率" : "诊断频率", `${displayed.frame_rate_fps.toFixed(1)} FPS`, `当前统计区间内每秒完成的${pixel_delivery ? "完整像素帧" : "后台诊断帧"}数量。`], - ["端到端平均", `${displayed.request_to_pixels_average_ms.toFixed(2)} ms`, `服务端创建帧到${completion}的平均耗时。`], - ["端到端 P50", `${displayed.request_to_pixels_p50_ms.toFixed(2)} ms`, "一半样本不超过该端到端耗时。"], - ["端到端 P95", `${displayed.request_to_pixels_p95_ms.toFixed(2)} ms`, "95% 样本不超过该端到端耗时，用于观察长尾。"], - ["端到端 P99", `${displayed.request_to_pixels_p99_ms.toFixed(2)} ms`, "99% 样本不超过该端到端耗时，用于观察极端长尾。"], + ["Scene 完成帧率", `${displayed.frame_rate_fps.toFixed(1)} FPS`, "根据服务端帧序号跨度和创建时间计算；编码或媒体背压合并旧帧时仍反映真实 Scene 完成速率。"], + [pixel_delivery ? "视频投递帧率" : "诊断投递频率", `${displayed.delivery_frame_rate_fps.toFixed(1)} FPS`, `浏览器实际收到的${pixel_delivery ? "编码帧元数据" : "后台诊断帧"}频率；与 Scene 帧率的差值表示实时背压合并。`], + ["浏览器呈现帧率", `${displayed.video_playback.frame_rate_fps.toFixed(1)} FPS`, "由 requestVideoFrameCallback 统计的 video 元素真实呈现频率；服务端有帧而此项接近零时，问题位于关键帧、RTP、解码或呈现链路。"], + ["浏览器累计呈现", displayed.video_playback.presented_frames.toLocaleString("zh-CN"), `video readyState=${displayed.video_playback.ready_state}，currentTime=${displayed.video_playback.current_time_seconds.toFixed(3)} s。`], + ["浏览器累计丢帧", displayed.video_playback.dropped_frames.toLocaleString("zh-CN"), "浏览器 VideoPlaybackQuality 报告的累计丢弃视频帧。"], + ["服务端完成平均", `${displayed.request_to_pixels_average_ms.toFixed(2)} ms`, `服务端创建帧到${completion}的平均耗时；不包含浏览器视频解码与真正呈现。`], + ["服务端完成 P50", `${displayed.request_to_pixels_p50_ms.toFixed(2)} ms`, "一半样本不超过该服务端完成耗时。"], + ["服务端完成 P95", `${displayed.request_to_pixels_p95_ms.toFixed(2)} ms`, "95% 样本不超过该服务端完成耗时，用于观察长尾。"], + ["服务端完成 P99", `${displayed.request_to_pixels_p99_ms.toFixed(2)} ms`, "99% 样本不超过该服务端完成耗时，用于观察极端长尾。"], ["帧间隔平均", `${displayed.frame_interval_average_ms.toFixed(2)} ms`, `相邻两次${completion}的平均间隔。`], ["帧间隔抖动 P95", `${displayed.interval_jitter_p95_ms.toFixed(2)} ms`, "帧间隔相对中位数偏差的第 95 百分位。"], - ["请求 ID 缺口", displayed.dropped_sequence_count.toLocaleString("zh-CN"), "相邻已完成请求 ID 之间缺失的数量；可能表示请求未形成完整样本。"] + ["投递帧序号缺口", displayed.dropped_sequence_count.toLocaleString("zh-CN"), "相邻已投递帧之间被实时编码或媒体背压合并的历史帧数量；Scene 完成回调本身仍逐帧闭环。"] ]; const open_context_menu = (event: React.MouseEvent) => { event.preventDefault(); @@ -861,7 +931,7 @@ function Frame_Diagnostics_View({diagnostics, dimension, on_reset}: {diagnostics
{(["value", "percentage"] as Stage_Unit[]).map(unit => )}
-
浏览器请求{definitions.map(([key, label, description]) => →{label})}→{pixel_delivery ? "浏览器呈现" : "浏览器收到诊断"}
+
服务端帧时钟建帧{definitions.map(([key, label, description]) => →{label})}→浏览器收到诊断
{stage_values.map(([key, label, description, value]) =>
{label}
{!Number.isFinite(value) ? "--" : stage_unit === "percentage" ? `${(total_value > 0 ? value / total_value * 100 : 0).toFixed(1)}%` : diagnostic_value(value, key)}
)}
@@ -963,7 +1033,7 @@ function Frame_Policy_Pane({plot, analysis, busy, on_refresh, on_update, on_manu }) { const fields = analysis?.fields.filter(field => field.editable) ?? []; return
-
{plot.dimension} 持续采样控制选中或可见时传输完整像素；离屏时继续真实渲染并只回传轻量诊断。2D 与 3D 使用各自的采样频率和阶段模型。
+
{plot.dimension} 后端帧策略渲染、采样、H.264 编码与 WebRTC 传输全部由当前 Plot 的后端策略控制；浏览器隐藏画面不会停采样或断流。默认全部开启，用于完整链路压测。
{analysis ?
{fields.map(field => on_update(analysis, field, value)}/>)}
:

正在读取帧策略…

}
; @@ -986,27 +1056,16 @@ const Plot_Card = memo(function Plot_Card({plot, selected, policy, on_policy, on }) { const video_ref = useRef(null); const card_ref = useRef(null); - const [onscreen, set_onscreen] = useState(false); - useEffect(() => { - const card = card_ref.current; - if (!card) return; - const observer = new IntersectionObserver(entries => set_onscreen(entries[0]?.isIntersecting ?? false), {threshold: 0.05}); - observer.observe(card); - return () => observer.disconnect(); - }, []); - const sampling = (policy.visible && onscreen) || policy.refresh_hidden; - const {status, metrics} = use_plot_stream(plot, video_ref, policy.visible && onscreen && policy.transfer_pixels, sampling); + const {status, metrics} = use_plot_stream(plot, video_ref); const generated_time = metrics ? new Date(metrics.generated_time_unix_ms).toLocaleTimeString("zh-CN", {hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: false, fractionalSecondDigits: 3}) : "尚未生成帧"; return
on_select(plot)} onPointerUpCapture={event => { if (event.target instanceof HTMLVideoElement) on_select(plot); - }}>
绘图组件 · {plot.dimension}

{plot_labels[plot.id] ?? plot.title}

{{IDLE: "已停止", CONNECTING: "重连中", LIVE: "实时", OFFLINE: "已离线"}[status]}{metrics?.delivery === "diagnostics" ? "无像素传输" : metrics ? enum_label(metrics.pacing_mode) : "等待策略"}
{metrics ? metrics.frame_rate_fps.toFixed(1) : "--.-"} FPSE2E {metrics ? metrics.request_to_pixels_ms.toFixed(1) : "--.-"} msP95 {metrics ? metrics.p95_request_to_pixels_ms.toFixed(1) : "--.-"} msP99 {metrics ? metrics.p99_request_to_pixels_ms.toFixed(1) : "--.-"} ms
+ }}>
绘图组件 · {plot.dimension}

{plot_labels[plot.id] ?? plot.title}

{{IDLE: "已停止", CONNECTING: "重连中", LIVE: "实时", OFFLINE: "已离线"}[status]}{metrics?.delivery === "diagnostics" ? "无像素传输" : metrics ? enum_label(metrics.pacing_mode) : "等待策略"}
服务端 {metrics ? metrics.frame_rate_fps.toFixed(1) : "--.-"} FPS呈现 {metrics ? metrics.video_playback.frame_rate_fps.toFixed(1) : "--.-"} FPS耗时 {metrics ? metrics.request_to_pixels_ms.toFixed(1) : "--.-"} msP95 {metrics ? metrics.p95_request_to_pixels_ms.toFixed(1) : "--.-"} ms
event.stopPropagation()}> - -
- {plot.description ?

{plot.description}

: null}
; + {plot.description ?

{plot.description}

: null}
; }); type Gallery_Breakpoint = "lg" | "md" | "sm" | "xs"; @@ -1209,8 +1268,6 @@ export function App() { method: "PUT", headers: {"Content-Type": "application/json"}, body: JSON.stringify(value)}); const result = await response.json(); if (!result.success) throw new Error(result.error ?? "属性提交失败"); - if (component.id === "frame-analysis" && ["pacing_mode", "fixed_rate_fps"].includes(field.key)) - window.dispatchEvent(new CustomEvent("aethera-frame-policy", {detail: {plot_id: selected.id, key: field.key as Frame_Policy_Event["key"], value: result.value}})); set_schema(current => !current ? current : component.id === "frame-analysis" ? {...current, frame_analysis: {...current.frame_analysis, fields: current.frame_analysis.fields.map(current_field => current_field.key === field.key ? {...current_field, value: result.value} : current_field)}} : {...current, components: current.components.map(item => item.id !== component.id ? item : {...item,