diff --git a/render_2D/render_2D/base/Frame_2D.cpp b/render_2D/render_2D/base/Frame_2D.cpp index e122bbc..b347beb 100644 --- a/render_2D/render_2D/base/Frame_2D.cpp +++ b/render_2D/render_2D/base/Frame_2D.cpp @@ -1,14 +1,68 @@ #include "Frame_2D.hpp" +#include +#include +#include #include namespace aethera::render_2d { +namespace { +const BLPixelConverter& rgba_converter() { + static const BLPixelConverter converter = [] { + BLFormatInfo source{}; + if (source.query(BL_FORMAT_PRGB32) != BL_SUCCESS) + throw std::runtime_error("failed to query Blend2D PRGB32 format"); + constexpr std::uint8_t sizes[4]{8, 8, 8, 8}; + constexpr std::uint8_t shifts[4]{0, 8, 16, 24}; + BLFormatInfo destination{}; + destination.init(32, BL_FORMAT_FLAG_RGBA, sizes, shifts); + if (destination.sanitize() != BL_SUCCESS) + throw std::runtime_error("failed to describe browser RGBA8 format"); + BLPixelConverter result; + if (result.create(destination, source) != BL_SUCCESS) + throw std::runtime_error("failed to create Blend2D RGBA8 converter"); + return result; + }(); + return converter; +} +} struct Frame_2D::Private { - Blend2D_Cache color{}; /* Scene 直接合成且由外部 Frame 生命周期持有的最终颜色层。 */ + Blend2D_Cache color{}; /* Scene 直接合成的原生 PRGB32 颜色层。 */ + Pixel_Format output_format{native_pixel_format}; /* 调用方为本帧选择的最终协议格式。 */ }; -Frame_2D::Frame_2D(Frame_Identity identity) : Render_Frame(identity), d(std::make_unique()) {} +Frame_2D::Frame_2D(Frame_Identity identity, Pixel_Format output_format) + : Render_Frame(identity), d(std::make_unique()) { + if (std::ranges::find(supported_pixel_formats, output_format) == + supported_pixel_formats.end()) + throw std::invalid_argument("Frame_2D output pixel format is unsupported"); + d->output_format = output_format; +} Frame_2D::~Frame_2D() = default; +Pixel_Format Frame_2D::output_format() const noexcept { + return d->output_format; +} Image_View Frame_2D::image() const { return d->color.view(); } +Pixel_Buffer Frame_2D::output_pixels() const { + const Image_View source = image(); + Pixel_Buffer result{{}, source.width, source.height, d->output_format}; + if (source.empty()) return result; + const std::size_t row_size = static_cast(source.width) * 4U; + result.bytes.resize(row_size * static_cast(source.height)); + if (d->output_format == native_pixel_format) { + for (int y = 0; y < source.height; ++y) + std::memcpy(result.bytes.data() + static_cast(y) * row_size, + source.data + static_cast(y) * source.stride, + row_size); + return result; + } + if (rgba_converter().convert_rect( + result.bytes.data(), static_cast(row_size), source.data, + source.stride, static_cast(source.width), + static_cast(source.height)) != BL_SUCCESS) + throw std::runtime_error("Blend2D failed to convert PRGB32 to RGBA8"); + return result; +} Blend2D_Cache& detail::Frame_2D_Access::render_target(Frame_2D* frame) { - if (!frame) throw std::invalid_argument("Render_Scene_2D requires a non-null external frame"); + if (!frame) + throw std::invalid_argument("Render_Scene_2D requires a non-null external frame"); return frame->d->color; } } diff --git a/render_2D/render_2D/base/Frame_2D.hpp b/render_2D/render_2D/base/Frame_2D.hpp index de6561c..afe65de 100644 --- a/render_2D/render_2D/base/Frame_2D.hpp +++ b/render_2D/render_2D/base/Frame_2D.hpp @@ -1,6 +1,7 @@ #pragma once #include "../render/Blend2D_Cache.hpp" #include +#include #include namespace aethera::render_2d { class Frame_2D; @@ -11,12 +12,20 @@ struct Frame_2D_Access { } class Frame_2D final : public Render_Frame { public: - explicit Frame_2D(Frame_Identity identity); + static constexpr Pixel_Format native_pixel_format{ + Pixel_Format::bgra8_premultiplied}; + static constexpr std::array supported_pixel_formats{ + native_pixel_format, Pixel_Format::rgba8}; + explicit Frame_2D(Frame_Identity identity, + Pixel_Format output_format = native_pixel_format); ~Frame_2D() override; + [[nodiscard]] Pixel_Format output_format() const noexcept; [[nodiscard]] Image_View image() const; + /* 按本帧声明的输出格式生成连续像素;原生格式只做必要的行打包。 */ + [[nodiscard]] Pixel_Buffer output_pixels() const; private: struct Private; - std::unique_ptr d; /* 本帧最终二维颜色结果的唯一所有权。 */ + std::unique_ptr d; /* 最终二维颜色层及本帧输出格式。 */ friend struct detail::Frame_2D_Access; }; } diff --git a/render_2D/render_2D/base/Types.hpp b/render_2D/render_2D/base/Types.hpp index e3be6b1..59a90f2 100644 --- a/render_2D/render_2D/base/Types.hpp +++ b/render_2D/render_2D/base/Types.hpp @@ -141,14 +141,23 @@ enum class Line_Interpolation_Mode : std::uint8_t { step_right, cubic_value }; -enum class Pixel_Format : std::uint8_t { premultiplied_32 }; +enum class Pixel_Format : std::uint8_t { + bgra8_premultiplied, + rgba8 +}; +struct Pixel_Buffer { + std::vector bytes{}; + int width{}; + int height{}; + Pixel_Format format{Pixel_Format::bgra8_premultiplied}; +}; /* 只读图像像素视图。 */ struct Image_View { const std::byte* data{}; /* 首行像素地址;不拥有内存。 */ int width{}; /* 图像宽度,单位为像素。 */ int height{}; /* 图像高度,单位为像素。 */ int stride{}; /* 相邻两行起点的字节距离。 */ - Pixel_Format format{Pixel_Format::premultiplied_32}; /* 像素存储格式。 */ + Pixel_Format format{Pixel_Format::bgra8_premultiplied}; /* 像素存储格式。 */ [[nodiscard]] bool empty() const noexcept { return data == nullptr || width <= 0 || height <= 0; } diff --git a/render_2D/render_2D/render/Blend2D_Cache.cpp b/render_2D/render_2D/render/Blend2D_Cache.cpp index 9d233b3..1974087 100644 --- a/render_2D/render_2D/render/Blend2D_Cache.cpp +++ b/render_2D/render_2D/render/Blend2D_Cache.cpp @@ -79,7 +79,7 @@ Image_View Blend2D_Cache::view() const { BLImageData data{}; Private::require_success(d->image.get_data(&data), "read Blend2D cache"); return {static_cast(data.pixel_data), data.size.w, data.size.h, - static_cast(data.stride), Pixel_Format::premultiplied_32}; + static_cast(data.stride), Pixel_Format::bgra8_premultiplied}; } namespace detail { void Painter::Private::load_font_face(BLFontFace& face, std::initializer_list candidates) { diff --git a/render_2D/tests/Frame_Pixel_Format_Test.cpp b/render_2D/tests/Frame_Pixel_Format_Test.cpp new file mode 100644 index 0000000..824ed54 --- /dev/null +++ b/render_2D/tests/Frame_Pixel_Format_Test.cpp @@ -0,0 +1,35 @@ +#include +#include +namespace { +using namespace aethera; +using namespace aethera::render_2d; +void paint_translucent_red(Frame_2D& frame) { + auto& target = detail::Frame_2D_Access::render_target(&frame); + detail::Painter painter(target, Size{1, 1}); + painter.rect(Rect_F{0.0, 0.0, 1.0, 1.0}, + Pen{Color::transparent(), 0.0, Line_Style::none}, + Brush{Color{255, 0, 0, 128}, Brush_Style::solid}); +} +} +TEST(frame_pixel_format, publishes_native_bgra_premultiplied_by_default) { + Frame_2D frame{Frame_Identity{1, 0}}; + paint_translucent_red(frame); + const auto pixels = frame.output_pixels(); + ASSERT_EQ(pixels.format, Pixel_Format::bgra8_premultiplied); + ASSERT_EQ(pixels.bytes.size(), 4U); + EXPECT_EQ(std::to_integer(pixels.bytes[0]), 0U); + EXPECT_EQ(std::to_integer(pixels.bytes[1]), 0U); + EXPECT_EQ(std::to_integer(pixels.bytes[2]), 128U); + EXPECT_EQ(std::to_integer(pixels.bytes[3]), 128U); +} +TEST(frame_pixel_format, uses_blend2d_to_publish_straight_rgba) { + Frame_2D frame{Frame_Identity{2, 0}, Pixel_Format::rgba8}; + paint_translucent_red(frame); + const auto pixels = frame.output_pixels(); + ASSERT_EQ(pixels.format, Pixel_Format::rgba8); + ASSERT_EQ(pixels.bytes.size(), 4U); + EXPECT_GE(std::to_integer(pixels.bytes[0]), 254U); + EXPECT_EQ(std::to_integer(pixels.bytes[1]), 0U); + EXPECT_EQ(std::to_integer(pixels.bytes[2]), 0U); + EXPECT_EQ(std::to_integer(pixels.bytes[3]), 128U); +} diff --git a/render_3D/render_3D/base/Frame_3D.cpp b/render_3D/render_3D/base/Frame_3D.cpp index a723579..a8e5c59 100644 --- a/render_3D/render_3D/base/Frame_3D.cpp +++ b/render_3D/render_3D/base/Frame_3D.cpp @@ -2,13 +2,16 @@ #include namespace aethera::render_3d { struct Frame_3D::Private { + Pixel_Format output_format{Frame_3D::native_pixel_format}; Extent extent{}; /* 本帧 GPU 读回图像尺寸。 */ std::shared_ptr> pixels{}; /* 合并提交共享的不可变 RGBA8 像素。 */ Frame_3D_Output output{Frame_3D_Output::pixels}; /* 调用方要求的最终帧输出。 */ }; -Frame_3D::Frame_3D(Frame_Identity identity, Frame_3D_Output output) +Frame_3D::Frame_3D(Frame_Identity identity, Frame_3D_Output output, + Pixel_Format output_format) : Render_Frame(identity), d(std::make_unique()) { d->output = output; + d->output_format = output_format; } Frame_3D::~Frame_3D() = default; Extent Frame_3D::extent() const noexcept { return d->extent; } @@ -17,6 +20,7 @@ std::span Frame_3D::pixels() const noexcept { return *d->pixels; } Frame_3D_Output Frame_3D::output() const noexcept { return d->output; } +Pixel_Format Frame_3D::output_format() const noexcept { return d->output_format; } void detail::Frame_3D_Access::assign_pixels( std::span frames, Extent extent, std::vector pixels) { diff --git a/render_3D/render_3D/base/Frame_3D.hpp b/render_3D/render_3D/base/Frame_3D.hpp index 16fec10..8f4373f 100644 --- a/render_3D/render_3D/base/Frame_3D.hpp +++ b/render_3D/render_3D/base/Frame_3D.hpp @@ -1,6 +1,7 @@ #pragma once #include "Types.hpp" #include +#include #include #include #include @@ -19,11 +20,16 @@ struct Frame_3D_Access { } class Frame_3D final : public Render_Frame { public: - explicit Frame_3D(Frame_Identity identity, Frame_3D_Output output = Frame_3D_Output::pixels); + static constexpr Pixel_Format native_pixel_format{Pixel_Format::rgba8_unorm}; + static constexpr std::array supported_pixel_formats{native_pixel_format}; + explicit Frame_3D(Frame_Identity identity, + Frame_3D_Output output = Frame_3D_Output::pixels, + Pixel_Format output_format = native_pixel_format); ~Frame_3D() override; [[nodiscard]] Extent extent() const noexcept; [[nodiscard]] std::span pixels() const noexcept; [[nodiscard]] Frame_3D_Output output() const noexcept; + [[nodiscard]] Pixel_Format output_format() const noexcept; private: struct Private; std::unique_ptr d; /* 本帧最终三维读回结果的唯一所有权。 */ diff --git a/render_3D/render_3D/base/Types.hpp b/render_3D/render_3D/base/Types.hpp index 04165b1..3acafb0 100644 --- a/render_3D/render_3D/base/Types.hpp +++ b/render_3D/render_3D/base/Types.hpp @@ -8,6 +8,10 @@ namespace aethera::render_3d { using Coordinate_3D = float; using Pixel_Distance = float; +/* Datoviz 离屏目标与读回缓冲之间的实际字节布局。 */ +enum class Pixel_Format : std::uint8_t { + rgba8_unorm +}; struct Vec2 { Coordinate_3D x{}; /* 水平分量。 */ Coordinate_3D y{}; /* 垂直分量。 */ diff --git a/web_server/src/Graph_WebSocket.cpp b/web_server/src/Graph_WebSocket.cpp index 0158d25..df6cb42 100644 --- a/web_server/src/Graph_WebSocket.cpp +++ b/web_server/src/Graph_WebSocket.cpp @@ -18,7 +18,7 @@ struct Graph_WebSocket::Private { }; Graph_WebSocket::Graph_WebSocket(drogon::WebSocketConnectionPtr connection, std::shared_ptr plot) : d(std::make_unique()) { d->connection = connection; d->plot = std::move(plot); d->owner = connection.get(); } Graph_WebSocket::~Graph_WebSocket() { close(); } -void Graph_WebSocket::start() { if (d->attached) return; const auto weak = weak_from_this(); d->plot->attach(d->owner, [weak](std::shared_ptr frame) { const auto socket = weak.lock(); if (!socket) return; const auto connection = socket->d->connection.lock(); if (!connection || !connection->connected()) return; connection->send(frame->metadata, drogon::WebSocketMessageType::Text); if (!frame->pixels.empty()) connection->send(frame->pixels.data(), frame->pixels.size(), drogon::WebSocketMessageType::Binary); }); d->attached = true; } +void Graph_WebSocket::start() { if (d->attached) return; const auto weak = weak_from_this(); d->plot->attach(d->owner, [weak](std::shared_ptr frame) { const auto socket = weak.lock(); if (!socket) return; const auto connection = socket->d->connection.lock(); if (!connection || !connection->connected()) return; connection->send(frame->metadata, drogon::WebSocketMessageType::Text); if (!frame->pixels.empty()) connection->send(reinterpret_cast(frame->pixels.data()), frame->pixels.size(), drogon::WebSocketMessageType::Binary); }); d->attached = true; } void Graph_WebSocket::receive(std::string_view message) { const auto json = nlohmann::json::parse(message, nullptr, false); if (json.is_discarded() || !json.is_object()) return; @@ -33,6 +33,16 @@ void Graph_WebSocket::receive(std::string_view message) { request.time_milliseconds = value->get(); if (const auto value = json.find("delivery"); value != json.end() && value->is_string()) request.delivery = *value == "diagnostics" ? Plot_Frame_Delivery::diagnostics : Plot_Frame_Delivery::pixels; + if (const auto value = json.find("pixel_format"); + value != json.end() && value->is_string()) { + if (*value == "bgra8-premultiplied") + request.pixel_format = render_2d::Pixel_Format::bgra8_premultiplied; + else if (*value == "rgba8") + request.pixel_format = render_2d::Pixel_Format::rgba8; + else if (*value == "rgba8-unorm") + request.pixel_format = render_3d::Pixel_Format::rgba8_unorm; + else return; + } if (const auto viewport = json.find("viewport"); viewport != json.end() && viewport->is_object()) { if (const auto value = viewport->find("width"); value != viewport->end() && value->is_number_unsigned()) request.width = std::clamp(value->get(), 160U, 1920U); diff --git a/web_server/src/Plot.cpp b/web_server/src/Plot.cpp index e126bb0..aa1b803 100644 --- a/web_server/src/Plot.cpp +++ b/web_server/src/Plot.cpp @@ -33,7 +33,7 @@ using Frequency_Axis_Object = Impl; using Numeric_Axis_Object = Impl; using Time_Axis_Object = Impl; using Selection_Object = Impl; -constexpr std::uint16_t frame_protocol_version{5}; +constexpr std::uint16_t frame_protocol_version{6}; enum class Frame_Pacing_Mode : std::uint32_t { manual, fixed_rate, @@ -77,6 +77,31 @@ std::string_view delivery_name(Plot_Frame_Delivery delivery) { } throw std::logic_error("unknown plot frame delivery"); } +std::string_view pixel_format_name(render_2d::Pixel_Format format) { + switch (format) { + case render_2d::Pixel_Format::bgra8_premultiplied: return "bgra8-premultiplied"; + case render_2d::Pixel_Format::rgba8: return "rgba8"; + } + throw std::logic_error("unknown 2D pixel format"); +} +std::string_view pixel_format_name(render_3d::Pixel_Format format) { + switch (format) { + case render_3d::Pixel_Format::rgba8_unorm: return "rgba8-unorm"; + } + throw std::logic_error("unknown 3D pixel format"); +} +nlohmann::json supported_2d_pixel_formats() { + auto result = nlohmann::json::array(); + for (const auto format : Frame_2D::supported_pixel_formats) + result.push_back(pixel_format_name(format)); + return result; +} +nlohmann::json supported_3d_pixel_formats() { + auto result = nlohmann::json::array(); + for (const auto format : Frame_3D::supported_pixel_formats) + result.push_back(pixel_format_name(format)); + return result; +} Frame_Pacing_Properties Frame_Policy::snapshot() const { std::lock_guard lock(mutex); return pacing; @@ -114,7 +139,13 @@ nlohmann::json Frame_Policy::write_prop(std::string_view key, const nlohmann::js } return {{"success", false}, {"error", "unknown frame runtime property"}}; } -nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, std::uint32_t height, std::size_t byte_length, Plot_Frame_Delivery delivery, const Frame_Pacing_Properties& pacing) { +nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, + std::uint32_t height, std::size_t byte_length, + std::string_view pixel_format, + std::string_view native_pixel_format, + nlohmann::json supported_pixel_formats, + Plot_Frame_Delivery delivery, + const Frame_Pacing_Properties& pacing) { nlohmann::json markers = nlohmann::json::object(); for (const auto& point : frame.trace_points()) markers[std::string(magic_enum::enum_name(point.marker))] = point.elapsed_ns; nlohmann::json measurements = nlohmann::json::object(); @@ -124,7 +155,11 @@ nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, std::uin {"kind", "frame_metadata"}, {"protocol", "aethera.frame"}, {"version", frame_protocol_version}, {"sequence", identity.sequence}, {"correlation_id", identity.correlation_id}, {"created_time_unix_ms", static_cast(frame.created_time_unix_ns()) / 1'000'000.0}, {"delivery", delivery_name(delivery)}, - {"pixel", {{"width", width}, {"height", height}, {"format", "rgba8"}, {"byte_length", byte_length}}}, + {"pixel", {{"width", width}, {"height", height}, + {"format", pixel_format}, + {"native_format", native_pixel_format}, + {"supported_formats", std::move(supported_pixel_formats)}, + {"byte_length", byte_length}}}, {"pacing", {{"mode", pacing_mode_name(pacing.mode)}, {"fixed_rate_fps", pacing.fixed_rate_fps}, {"minimum_latency_headroom", pacing.minimum_latency_headroom}}}, {"trace", {{"clock", "steady_elapsed_ns"}, {"markers", std::move(markers)}, {"measurements", std::move(measurements)}}} }; @@ -132,35 +167,32 @@ nlohmann::json frame_metadata(Render_Frame& frame, std::uint32_t width, std::uin std::shared_ptr encode_frame(Frame_2D* frame, Plot_Frame_Delivery delivery, const Frame_Pacing_Properties& pacing) { frame->mark(Frame_Trace_Marker::websocket_publish_started); const Image_View image = frame->image(); - std::string output; - if (delivery == Plot_Frame_Delivery::pixels) { - output.reserve(static_cast(image.width) * image.height * 4); - for (int y = 0; y < image.height; ++y) { - const auto* row = reinterpret_cast( - image.data + static_cast(y) * image.stride); - for (int x = 0; x < image.width; ++x) { - const auto* pixel = row + x * 4; - output.push_back(static_cast(pixel[2])); - output.push_back(static_cast(pixel[1])); - output.push_back(static_cast(pixel[0])); - output.push_back(static_cast(pixel[3])); - } - } - } + Pixel_Buffer output{{}, image.width, image.height, frame->output_format()}; + if (delivery == Plot_Frame_Delivery::pixels) output = frame->output_pixels(); frame->mark(Frame_Trace_Marker::websocket_publish_finished); auto message = std::make_shared(); - message->pixels = std::move(output); - message->metadata = frame_metadata(*frame, static_cast(image.width), static_cast(image.height), message->pixels.size(), delivery, pacing).dump(); + message->pixels = std::move(output.bytes); + message->metadata = frame_metadata( + *frame, static_cast(output.width), + static_cast(output.height), message->pixels.size(), + pixel_format_name(output.format), + pixel_format_name(Frame_2D::native_pixel_format), + supported_2d_pixel_formats(), delivery, pacing).dump(); return message; } std::shared_ptr encode_frame(Frame_3D* frame, Plot_Frame_Delivery delivery, const Frame_Pacing_Properties& pacing) { frame->mark(Frame_Trace_Marker::websocket_publish_started); const auto pixels = frame->pixels(); auto message = std::make_shared(); - if (delivery == Plot_Frame_Delivery::pixels) message->pixels.assign(reinterpret_cast(pixels.data()), pixels.size()); + if (delivery == Plot_Frame_Delivery::pixels) + message->pixels.assign(pixels.begin(), pixels.end()); frame->mark(Frame_Trace_Marker::websocket_publish_finished); const auto extent = frame->extent(); - message->metadata = frame_metadata(*frame, extent.width, extent.height, message->pixels.size(), delivery, pacing).dump(); + message->metadata = frame_metadata( + *frame, extent.width, extent.height, message->pixels.size(), + pixel_format_name(frame->output_format()), + pixel_format_name(Frame_3D::native_pixel_format), + supported_3d_pixel_formats(), delivery, pacing).dump(); return message; } struct Schema_Query { @@ -281,9 +313,19 @@ nlohmann::json Plot::Private::schema() const { } Plot::Private::Managed_Frame Plot::Private::make_frame(Frame_Submission submission) { const Frame_Identity identity{next_frame_sequence++, submission.request.correlation_id}; - if (std::holds_alternative>(scene)) return {submission.owner, std::move(submission.request), std::make_unique(identity)}; + if (std::holds_alternative>(scene)) { + const auto* requested = std::get_if( + &submission.request.pixel_format); + const auto format = requested ? *requested : Frame_2D::native_pixel_format; + return {submission.owner, std::move(submission.request), + std::make_unique(identity, format)}; + } const auto output = submission.request.delivery == Plot_Frame_Delivery::pixels ? Frame_3D_Output::pixels : Frame_3D_Output::diagnostics; - return {submission.owner, std::move(submission.request), std::make_unique(identity, output)}; + const auto* requested = std::get_if( + &submission.request.pixel_format); + const auto format = requested ? *requested : Frame_3D::native_pixel_format; + return {submission.owner, std::move(submission.request), + std::make_unique(identity, output, format)}; } void Plot::Private::request_frame(Frame_Submission submission) { auto frame = make_frame(std::move(submission)); diff --git a/web_server/src/Plot.hpp b/web_server/src/Plot.hpp index dad385b..4a86f58 100644 --- a/web_server/src/Plot.hpp +++ b/web_server/src/Plot.hpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include namespace aethera::web { struct Plot_Input_Event { Event_Type type{Event_Type::pointer_move}; @@ -34,10 +36,12 @@ struct Plot_Frame_Request { std::uint32_t width{720}; /* 请求帧宽度,单位为物理像素。 */ std::uint32_t height{420}; /* 请求帧高度,单位为物理像素。 */ Plot_Frame_Delivery delivery{Plot_Frame_Delivery::pixels}; /* 本次请求返回完整像素或仅返回诊断元数据。 */ + std::variant pixel_format{ + render_2d::Frame_2D::native_pixel_format}; /* 二维帧发布格式;三维固定为原生 RGBA8。 */ }; struct Plot_Frame_Message { std::string metadata{}; /* 单帧 JSON 元数据 WebSocket 文本消息。 */ - std::string pixels{}; /* 紧随元数据发送的纯 RGBA8 WebSocket 二进制消息。 */ + std::vector pixels{}; /* 格式由元数据声明的像素所有权。 */ }; class Plot final : public std::enable_shared_from_this { public: diff --git a/webapp_gallery/src/app.tsx b/webapp_gallery/src/app.tsx index 5c9a724..2aeddc0 100644 --- a/webapp_gallery/src/app.tsx +++ b/webapp_gallery/src/app.tsx @@ -28,9 +28,10 @@ type State_Histories = Record; type Stream_Status = "IDLE" | "CONNECTING" | "LIVE" | "OFFLINE"; type Frame_Pacing_Mode = "manual" | "fixed_rate" | "minimum_latency" | "maximum_rate"; type Frame_Delivery = "pixels" | "diagnostics"; -type Frame_Metadata = {kind: "frame_metadata"; protocol: "aethera.frame"; version: 5; sequence: number; correlation_id: number; +type Pixel_Format = "bgra8-premultiplied" | "rgba8" | "rgba8-unorm"; +type Frame_Metadata = {kind: "frame_metadata"; protocol: "aethera.frame"; version: 6; sequence: number; correlation_id: number; delivery: Frame_Delivery; - created_time_unix_ms: number; pixel: {width: number; height: number; format: "rgba8"; byte_length: number}; + created_time_unix_ms: number; pixel: {width: number; height: number; format: Pixel_Format; native_format: Pixel_Format; supported_formats: Pixel_Format[]; byte_length: number}; pacing: {mode: Frame_Pacing_Mode; fixed_rate_fps: number; minimum_latency_headroom: number}; trace: {clock: "steady_elapsed_ns"; markers: Record; measurements: Record}}; type Frame_Stage_Values = Record; @@ -57,22 +58,124 @@ function local_time_milliseconds() { function valid_frame_metadata(value: unknown): value is Frame_Metadata { if (!value || typeof value !== "object") return false; const frame = value as Partial; - return frame.kind === "frame_metadata" && frame.protocol === "aethera.frame" && frame.version === 5 + return frame.kind === "frame_metadata" && frame.protocol === "aethera.frame" && frame.version === 6 && typeof frame.sequence === "number" && typeof frame.correlation_id === "number" && (frame.delivery === "pixels" || frame.delivery === "diagnostics") - && Boolean(frame.pixel) && frame.pixel?.format === "rgba8" && Boolean(frame.trace); + && Boolean(frame.pixel) && (frame.pixel?.format === "rgba8" || frame.pixel?.format === "bgra8-premultiplied" || frame.pixel?.format === "rgba8-unorm") + && (frame.pixel?.native_format === "rgba8" || frame.pixel?.native_format === "bgra8-premultiplied" || frame.pixel?.native_format === "rgba8-unorm") + && Array.isArray(frame.pixel?.supported_formats) + && frame.pixel!.supported_formats.includes(frame.pixel!.format) + && Boolean(frame.trace); } -function draw_pixels(canvas: HTMLCanvasElement, bytes: ArrayBuffer, metadata: Frame_Metadata) { - const {width, height, byte_length} = metadata.pixel; - if (bytes.byteLength !== byte_length || byte_length !== width * height * 4) return null; - const started = performance.now(); - const pixels = new Uint8ClampedArray(bytes); - if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; } +type Pixel_Renderer = { + preferred_2d_format: Pixel_Format; + draw: (bytes: ArrayBuffer, metadata: Frame_Metadata) => number | null; +}; + +function shader(gl: WebGL2RenderingContext, type: number, source: string) { + const value = gl.createShader(type); + if (!value) return null; + gl.shaderSource(value, source); + gl.compileShader(value); + if (gl.getShaderParameter(value, gl.COMPILE_STATUS)) return value; + gl.deleteShader(value); + return null; +} + +function webgl_pixel_renderer(canvas: HTMLCanvasElement, gl: WebGL2RenderingContext): Pixel_Renderer | null { + const vertex = shader(gl, gl.VERTEX_SHADER, `#version 300 es + precision highp float; + const vec2 positions[3] = vec2[3](vec2(-1.0, -1.0), vec2(3.0, -1.0), vec2(-1.0, 3.0)); + out vec2 texture_coordinate; + void main() { + vec2 position = positions[gl_VertexID]; + gl_Position = vec4(position, 0.0, 1.0); + texture_coordinate = vec2(position.x * 0.5 + 0.5, 1.0 - (position.y * 0.5 + 0.5)); + }`); + const fragment = shader(gl, gl.FRAGMENT_SHADER, `#version 300 es + precision highp float; + uniform sampler2D frame_texture; + uniform bool source_is_bgra; + in vec2 texture_coordinate; + out vec4 output_color; + void main() { + vec4 color = texture(frame_texture, texture_coordinate); + output_color = source_is_bgra ? color.bgra : color; + }`); + if (!vertex || !fragment) return null; + const program = gl.createProgram(); + if (!program) return null; + gl.attachShader(program, vertex); + gl.attachShader(program, fragment); + gl.linkProgram(program); + gl.deleteShader(vertex); + gl.deleteShader(fragment); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + gl.deleteProgram(program); + return null; + } + const texture = gl.createTexture(); + const vertex_array = gl.createVertexArray(); + if (!texture || !vertex_array) return null; + const format_location = gl.getUniformLocation(program, "source_is_bgra"); + gl.useProgram(program); + gl.uniform1i(gl.getUniformLocation(program, "frame_texture"), 0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1); + gl.pixelStorei(gl.UNPACK_COLORSPACE_CONVERSION_WEBGL, gl.NONE); + gl.disable(gl.BLEND); + return { + preferred_2d_format: "bgra8-premultiplied", + draw: (bytes, metadata) => { + const {width, height, byte_length, format} = metadata.pixel; + if (bytes.byteLength !== byte_length || byte_length !== width * height * 4) return null; + const started = performance.now(); + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + gl.viewport(0, 0, width, height); + gl.useProgram(program); + gl.bindVertexArray(vertex_array); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, + gl.UNSIGNED_BYTE, new Uint8Array(bytes)); + gl.uniform1i(format_location, format === "bgra8-premultiplied" ? 1 : 0); + gl.drawArrays(gl.TRIANGLES, 0, 3); + return performance.now() - started; + } + }; +} + +function pixel_renderer(canvas: HTMLCanvasElement): Pixel_Renderer | null { + const gl = canvas.getContext("webgl2", { + alpha: false, antialias: false, depth: false, stencil: false, + premultipliedAlpha: true, preserveDrawingBuffer: false + }); + if (gl) return webgl_pixel_renderer(canvas, gl); const context = canvas.getContext("2d", {alpha: false}); if (!context) return null; - context.putImageData(new ImageData(pixels, width, height), 0, 0); - return performance.now() - started; + return { + preferred_2d_format: "rgba8", + draw: (bytes, metadata) => { + const {width, height, byte_length, format} = metadata.pixel; + if ((format !== "rgba8" && format !== "rgba8-unorm") || bytes.byteLength !== byte_length || + byte_length !== width * height * 4) return null; + const started = performance.now(); + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + context.putImageData(new ImageData(new Uint8ClampedArray(bytes), width, height), 0, 0); + return performance.now() - started; + } + }; } function percentile(values: number[], ratio: number) { @@ -271,6 +374,15 @@ function use_plot_stream(plot: Plot, canvas_ref: React.RefObject { const pair = pending_metadata; pending_metadata = null; - const canvas = canvas_ref.current; - if (!pair || !canvas) return; + if (!pair) return; const completed_at = performance.now(); - const canvas_upload_ms = draw_pixels(canvas, bytes, pair.value); + const canvas_upload_ms = renderer.draw(bytes, pair.value); if (canvas_upload_ms !== null) complete_frame(pair, completed_at, canvas_upload_ms, true); else { frame_pending = false; @@ -777,7 +889,7 @@ const pipeline_2d_definitions: Pipeline_Stage_Definition[] = [ ["pipeline_2d_paint_ms", "Blend2D 绘制", "二维 Paint 任务图清屏并写入 Blend2D 帧缓存的耗时。"], ["pipeline_2d_scene_coordination_ms", "2D Scene 编排", "Scene 渲染区间内除事件、Prepare、Paint 外的依赖图编排耗时。"], ["pipeline_2d_callback_ms", "2D 完成回调", "同步二维帧完成后回调到 Plot 发布线程的耗时。"], - ["pipeline_2d_encode_ms", "BGRA→RGBA 编码", "逐行把 Blend2D BGRA 帧缓存转换成 WebSocket RGBA 载荷的耗时。"], + ["pipeline_2d_encode_ms", "2D 像素封装", "原生 BGRA 帧只进行连续行打包;仅在浏览器不支持 WebGL2 时由 Blend2D 转换为 RGBA。"], ["pipeline_2d_frame_handoff_ms", "2D 帧建立与发布调度", "Render_Frame 建立、进入 Scene 以及编码完成后生成元数据的调度间隙。"] ]; const pipeline_3d_definitions: Pipeline_Stage_Definition[] = [ @@ -1069,7 +1181,7 @@ const Plot_Card = memo(function Plot_Card({plot, selected, policy, on_policy, on
event.stopPropagation()}> - +
{plot.description ?

{plot.description}

: null}
{!policy.visible ?
画面不可见{policy.refresh_hidden ? "后端仍在渲染与采样" : "后端帧请求已停止"}
: null}
; });