taskflow 接入异步 datoviz性能监测

Kernel/src/renderive/real_time_data/Double_Buffer_Frame_Observer.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "Frame_Strategy_Observer.hpp"
+
+struct Double_Buffer_Frame_Observer {
+    static constexpr bool enabled = true;
+
+    template <class Target>
+    decltype(auto) bind(Target& target) {
+        return observer_.bind(target);
+    }
+
+    template <class Observation>
+    void observe(const Observation& observation) noexcept {
+        if (observation.event != Observation::Event::cache_updated)
+            return;
+        observer_.observe({
+            Real_Time_Data_Observation_Event::updated,
+            {this, Real_Time_Data_Retention::latest,
+             observation.cache_update_count, observation.time_ns,
+             observation.cache_update_count, 1}});
+    }
+
+private:
+    Frame_Strategy_Real_Time_Data_Observer observer_;
+};

Kernel/src/renderive/real_time_data/Real_Time_Data.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include "base/Real_Time_Data_Base.hpp"
 #include "Attach_Real_Time_Data.hpp"
+#include "Double_Buffer_Frame_Observer.hpp"
 #include "Frame_Strategy_Observer.hpp"
 #include "History_Real_Time_Data.hpp"
 #include "Observation.hpp"

Kernel/src/renderive/render_graph/External_Operation.cpp

@@ -0,0 +1,125 @@
+#include "External_Operation.hpp"
+
+#include <mutex>
+#include <stdexcept>
+#include <utility>
+
+struct External_Operation::State {
+    enum class Status {
+        pending,
+        completed,
+        failed
+    };
+
+    std::mutex mutex;
+    Completion completion;
+    std::exception_ptr error;
+    Status status{Status::pending};
+    bool subscribed{};
+};
+
+External_Operation::External_Operation(std::shared_ptr<State> state) noexcept
+    : state_(std::move(state)) {}
+
+void External_Operation::on_complete(Completion completion) const {
+    if (!state_)
+        throw std::logic_error("external operation is empty");
+    if (!completion)
+        throw std::invalid_argument("external operation completion is empty");
+
+    std::exception_ptr error;
+    bool invoke{};
+    {
+        std::lock_guard lock(state_->mutex);
+        if (state_->subscribed)
+            throw std::logic_error(
+                "external operation already has a completion subscriber");
+        state_->subscribed = true;
+        if (state_->status == State::Status::pending) {
+            state_->completion = std::move(completion);
+            return;
+        }
+        error = state_->error;
+        invoke = true;
+    }
+    if (invoke)
+        completion(std::move(error));
+}
+
+External_Operation::operator bool() const noexcept {
+    return static_cast<bool>(state_);
+}
+
+External_Operation_Source::External_Operation_Source()
+    : state_(std::make_shared<External_Operation::State>()) {}
+
+External_Operation External_Operation_Source::operation() const noexcept {
+    return External_Operation(state_);
+}
+
+bool External_Operation_Source::complete() noexcept {
+    return finish(false, {});
+}
+
+bool External_Operation_Source::fail(std::exception_ptr error) noexcept {
+    if (!error) {
+        try {
+            throw std::runtime_error("external operation failed");
+        } catch (...) {
+            error = std::current_exception();
+        }
+    }
+    return finish(true, std::move(error));
+}
+
+bool External_Operation_Source::finish(
+    bool failed, std::exception_ptr error) noexcept {
+    if (!state_)
+        return false;
+
+    External_Operation::Completion completion;
+    std::exception_ptr completion_error;
+    {
+        std::lock_guard lock(state_->mutex);
+        if (state_->status != External_Operation::State::Status::pending)
+            return false;
+        state_->status = failed
+            ? External_Operation::State::Status::failed
+            : External_Operation::State::Status::completed;
+        state_->error = std::move(error);
+        completion_error = state_->error;
+        completion = std::move(state_->completion);
+    }
+    if (completion) {
+        try {
+            completion(std::move(completion_error));
+        } catch (...) {
+        }
+    }
+    return true;
+}
+
+Node_Execution_Result::Node_Execution_Result(
+    std::optional<External_Operation> operation) noexcept
+    : operation_(std::move(operation)) {}
+
+Node_Execution_Result Node_Execution_Result::completed() noexcept {
+    return Node_Execution_Result(std::nullopt);
+}
+
+Node_Execution_Result Node_Execution_Result::external(
+    External_Operation operation) {
+    if (!operation)
+        throw std::invalid_argument("external node result has no operation");
+    return Node_Execution_Result(std::move(operation));
+}
+
+bool Node_Execution_Result::is_external() const noexcept {
+    return operation_.has_value();
+}
+
+const External_Operation& Node_Execution_Result::operation() const {
+    if (!operation_)
+        throw std::logic_error("completed node result has no external operation");
+    return *operation_;
+}

Kernel/src/renderive/render_graph/External_Operation.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <exception>
+#include <functional>
+#include <memory>
+#include <optional>
+
+class External_Operation_Source;
+
+class External_Operation {
+public:
+    using Completion = std::function<void(std::exception_ptr)>;
+
+    External_Operation() = default;
+
+    void on_complete(Completion completion) const;
+    [[nodiscard]] explicit operator bool() const noexcept;
+
+private:
+    struct State;
+
+    explicit External_Operation(std::shared_ptr<State> state) noexcept;
+
+    std::shared_ptr<State> state_;
+
+    friend class External_Operation_Source;
+};
+
+class External_Operation_Source {
+public:
+    External_Operation_Source();
+    External_Operation_Source(const External_Operation_Source&) = delete;
+    External_Operation_Source& operator=(const External_Operation_Source&) = delete;
+    External_Operation_Source(External_Operation_Source&&) noexcept = default;
+    External_Operation_Source& operator=(External_Operation_Source&&) noexcept = default;
+
+    [[nodiscard]] External_Operation operation() const noexcept;
+    [[nodiscard]] bool complete() noexcept;
+    [[nodiscard]] bool fail(std::exception_ptr error) noexcept;
+
+private:
+    [[nodiscard]] bool finish(bool failed, std::exception_ptr error) noexcept;
+    std::shared_ptr<External_Operation::State> state_;
+};
+
+class Node_Execution_Result {
+public:
+    [[nodiscard]] static Node_Execution_Result completed() noexcept;
+    [[nodiscard]] static Node_Execution_Result external(
+        External_Operation operation);
+
+    [[nodiscard]] bool is_external() const noexcept;
+    [[nodiscard]] const External_Operation& operation() const;
+
+private:
+    explicit Node_Execution_Result(
+        std::optional<External_Operation> operation) noexcept;
+
+    std::optional<External_Operation> operation_;
+};

Kernel/src/renderive/render_graph/Frame_Analysis.cpp

@@ -80,11 +80,14 @@ Frame_Analysis analyze_frame(const Render_Plan& plan, const Frame_Snapshot& fram
         auto& analysis = result.nodes[node.execution_index];
         analysis.node_id = node.node_id;
         analysis.duration_ns = execution.duration_ns();
+        analysis.cpu_duration_ns = execution.cpu_duration_ns();
+        analysis.external_duration_ns = execution.external_duration_ns();
         analysis.start_offset_ns = subtract_saturated(execution.start_time_ns,
                                                       frame.render_start_ns);
         analysis.end_offset_ns = subtract_saturated(execution.end_time_ns,
                                                     frame.render_start_ns);
-        result.total_work_duration_ns += analysis.duration_ns;
+        result.total_work_duration_ns += analysis.cpu_duration_ns;
+        result.total_external_duration_ns += analysis.external_duration_ns;
     }
 
     std::vector<std::vector<std::size_t>> predecessors(plan.graph.nodes.size());
@@ -116,20 +119,18 @@ Frame_Analysis analyze_frame(const Render_Plan& plan, const Frame_Snapshot& fram
     std::vector<std::size_t> path_parent(plan.graph.nodes.size(),
                                          std::numeric_limits<std::size_t>::max());
     for (std::size_t index : topological) {
-        std::uint64_t dependency_ready = frame.render_start_ns;
         std::uint64_t parent_path{};
         for (std::size_t predecessor : predecessors[index]) {
-            dependency_ready = std::max(dependency_ready,
-                frame.node_executions[predecessor].end_time_ns);
             if (path_duration[predecessor] > parent_path) {
                 parent_path = path_duration[predecessor];
                 path_parent[index] = predecessor;
             }
         }
         auto& analysis = result.nodes[index];
-        analysis.dependency_ready_time_ns = dependency_ready;
-        analysis.scheduler_wait_ns = frame.node_executions[index].start_time_ns >= dependency_ready
-            ? frame.node_executions[index].start_time_ns - dependency_ready
+        const auto& execution = frame.node_executions[index];
+        analysis.dependency_ready_time_ns = execution.ready_time_ns;
+        analysis.scheduler_wait_ns = execution.start_time_ns >= execution.ready_time_ns
+            ? execution.start_time_ns - execution.ready_time_ns
             : 0;
         path_duration[index] = parent_path + analysis.duration_ns;
     }
@@ -147,7 +148,8 @@ Frame_Analysis analyze_frame(const Render_Plan& plan, const Frame_Snapshot& fram
     for (auto& node : result.nodes) {
         node.work_contribution = result.total_work_duration_ns == 0
             ? 0.0
-            : static_cast<double>(node.duration_ns) / result.total_work_duration_ns;
+            : static_cast<double>(node.cpu_duration_ns) /
+                  result.total_work_duration_ns;
         node.critical_path_contribution = !node.on_critical_path ||
                                           result.critical_path_duration_ns == 0
             ? 0.0
@@ -162,11 +164,11 @@ Frame_Analysis analyze_frame(const Render_Plan& plan, const Frame_Snapshot& fram
     std::unordered_map<std::uint32_t, std::uint64_t> worker_work;
     events.reserve(frame.node_executions.size() * 2);
     for (const auto& execution : frame.node_executions) {
-        if (execution.duration_ns() == 0)
+        if (execution.cpu_duration_ns() == 0)
             continue;
         events.push_back({execution.start_time_ns, 1});
-        events.push_back({execution.end_time_ns, -1});
-        worker_work[execution.worker_id] += execution.duration_ns();
+        events.push_back({execution.cpu_end_time_ns, -1});
+        worker_work[execution.worker_id] += execution.cpu_duration_ns();
     }
     std::sort(events.begin(), events.end(), [](const Event& left, const Event& right) {
         return left.time < right.time || (left.time == right.time && left.delta < right.delta);

Kernel/src/renderive/render_graph/Frame_Analysis.hpp

@@ -9,6 +9,8 @@
 struct Node_Frame_Analysis {
     Render_Node_Id node_id{};
     std::uint64_t duration_ns{};
+    std::uint64_t cpu_duration_ns{};
+    std::uint64_t external_duration_ns{};
     std::uint64_t start_offset_ns{};
     std::uint64_t end_offset_ns{};
     std::uint64_t dependency_ready_time_ns{};
@@ -30,6 +32,7 @@ struct Frame_Analysis {
     std::uint64_t total_render_duration_ns{};
     std::uint64_t critical_path_duration_ns{};
     std::uint64_t total_work_duration_ns{};
+    std::uint64_t total_external_duration_ns{};
     std::uint64_t parallel_overlap_ns{};
     std::size_t peak_parallelism{};
     double average_parallelism{};

Kernel/src/renderive/render_graph/detail/Render_Graph_Runtime.cpp

@@ -0,0 +1,354 @@
+#include "Render_Graph_Runtime.hpp"
+
+#include <condition_variable>
+#include <exception>
+#include <limits>
+#include <mutex>
+#include <stdexcept>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace renderive::render_graph::detail {
+
+struct Render_Graph_Runtime::State
+    : std::enable_shared_from_this<Render_Graph_Runtime::State> {
+    struct Node_State {
+        std::size_t remaining_predecessors{};
+        Node_Execution_Status status{Node_Execution_Status::pending};
+    };
+
+    State(const Render_Plan& plan,
+          std::span<Node_Execution* const> execution_slots,
+          Execute_Node execute,
+          Schedule scheduler,
+          Current_Worker_Id worker_id)
+        : nodes(plan.graph.nodes.size()),
+          successors(plan.graph.nodes.size()),
+          executions(execution_slots.begin(), execution_slots.end()),
+          execute_node(std::move(execute)),
+          schedule(std::move(scheduler)),
+          current_worker_id(std::move(worker_id)),
+          unfinished_nodes(plan.graph.nodes.size()) {
+        if (!execute_node)
+            throw std::invalid_argument("render graph node executor is empty");
+        if (!schedule)
+            throw std::invalid_argument("render graph scheduler is empty");
+        if (!current_worker_id)
+            throw std::invalid_argument("render graph worker id source is empty");
+        if (!executions.empty() && executions.size() != nodes.size())
+            throw std::invalid_argument(
+                "render graph execution slot count differs from plan");
+        if (executions.empty())
+            executions.resize(nodes.size());
+
+        std::vector<bool> occupied(nodes.size());
+        std::unordered_map<Render_Node_Id, std::size_t> indices;
+        indices.reserve(nodes.size());
+        for (const auto& node : plan.graph.nodes) {
+            if (node.execution_index >= nodes.size() ||
+                occupied[node.execution_index])
+                throw std::invalid_argument(
+                    "render plan execution indices are not dense and unique");
+            occupied[node.execution_index] = true;
+            if (!indices.emplace(node.node_id, node.execution_index).second)
+                throw std::invalid_argument(
+                    "render plan contains duplicate node ids");
+        }
+        for (const auto& edge : plan.graph.edges) {
+            const auto from = indices.find(edge.from);
+            const auto to = indices.find(edge.to);
+            if (from == indices.end() || to == indices.end())
+                throw std::invalid_argument(
+                    "render plan edge references an unknown node");
+            successors[from->second].push_back(to->second);
+            ++nodes[to->second].remaining_predecessors;
+        }
+
+        std::vector<std::size_t> remaining;
+        remaining.reserve(nodes.size());
+        std::vector<std::size_t> topological;
+        topological.reserve(nodes.size());
+        for (const auto& node : nodes)
+            remaining.push_back(node.remaining_predecessors);
+        for (std::size_t index = 0; index < remaining.size(); ++index) {
+            if (remaining[index] == 0)
+                topological.push_back(index);
+        }
+        for (std::size_t cursor = 0; cursor < topological.size(); ++cursor) {
+            for (const std::size_t successor : successors[topological[cursor]]) {
+                if (--remaining[successor] == 0)
+                    topological.push_back(successor);
+            }
+        }
+        if (topological.size() != nodes.size())
+            throw std::invalid_argument("render plan contains a cycle");
+    }
+
+    void execute() {
+        std::vector<std::size_t> ready;
+        {
+            std::lock_guard lock(mutex);
+            if (started)
+                throw std::logic_error("render graph runtime already executed");
+            started = true;
+            if (nodes.empty()) {
+                terminal = true;
+            } else {
+                for (std::size_t index = 0; index < nodes.size(); ++index) {
+                    if (nodes[index].remaining_predecessors == 0)
+                        make_ready_locked(index, ready);
+                }
+                if (ready.empty()) {
+                    fail_graph_locked(std::make_exception_ptr(
+                        std::logic_error("render plan contains a cycle")));
+                }
+            }
+        }
+        submit(ready);
+
+        std::exception_ptr error;
+        {
+            std::unique_lock lock(mutex);
+            completion.wait(lock, [this] { return terminal; });
+            error = first_exception;
+        }
+        if (error)
+            std::rethrow_exception(error);
+    }
+
+    void make_ready_locked(std::size_t index,
+                           std::vector<std::size_t>& ready) {
+        auto& node = nodes.at(index);
+        if (failed || node.status != Node_Execution_Status::pending)
+            return;
+        node.status = Node_Execution_Status::ready;
+        if (auto* execution = executions[index]) {
+            execution->ready_time_ns = render_clock_now_ns();
+            execution->status = Node_Execution_Status::ready;
+        }
+        ready.push_back(index);
+    }
+
+    void submit(const std::vector<std::size_t>& ready) noexcept {
+        for (const std::size_t index : ready)
+            submit(index);
+    }
+
+    void submit(std::size_t index) noexcept {
+        {
+            std::lock_guard lock(mutex);
+            if (failed || nodes[index].status != Node_Execution_Status::ready) {
+                finish_if_terminal_locked();
+                return;
+            }
+            ++active_tasks;
+        }
+        try {
+            auto self = shared_from_this();
+            schedule([self = std::move(self), index] {
+                self->run_node(index);
+            });
+        } catch (...) {
+            std::lock_guard lock(mutex);
+            --active_tasks;
+            fail_node_locked(index, std::current_exception(),
+                             render_clock_now_ns());
+            finish_if_terminal_locked();
+        }
+    }
+
+    void run_node(std::size_t index) noexcept {
+        Node_Execution_Metrics* metrics{};
+        {
+            std::lock_guard lock(mutex);
+            if (failed || nodes[index].status != Node_Execution_Status::ready) {
+                --active_tasks;
+                finish_if_terminal_locked();
+                return;
+            }
+            nodes[index].status = Node_Execution_Status::running;
+            if (auto* execution = executions[index]) {
+                execution->start_time_ns = render_clock_now_ns();
+                try {
+                    execution->worker_id = current_worker_id();
+                } catch (...) {
+                    execution->worker_id =
+                        std::numeric_limits<std::uint32_t>::max();
+                }
+                execution->status = Node_Execution_Status::running;
+                metrics = &execution->metrics;
+            }
+        }
+
+        Node_Execution_Result result = Node_Execution_Result::completed();
+        std::exception_ptr error;
+        try {
+            result = execute_node(index, metrics);
+        } catch (...) {
+            error = std::current_exception();
+        }
+        const std::uint64_t cpu_end = render_clock_now_ns();
+
+        if (error) {
+            std::lock_guard lock(mutex);
+            --active_tasks;
+            fail_node_locked(index, std::move(error), cpu_end);
+            finish_if_terminal_locked();
+            return;
+        }
+        if (!result.is_external()) {
+            complete_synchronous(index, cpu_end);
+            return;
+        }
+
+        {
+            std::lock_guard lock(mutex);
+            --active_tasks;
+            auto& node = nodes[index];
+            node.status = Node_Execution_Status::waiting_external;
+            ++waiting_external;
+            if (auto* execution = executions[index]) {
+                execution->cpu_end_time_ns = cpu_end;
+                execution->external_start_time_ns = cpu_end;
+                execution->status = Node_Execution_Status::waiting_external;
+            }
+        }
+        try {
+            auto self = shared_from_this();
+            result.operation().on_complete(
+                [self = std::move(self), index](std::exception_ptr completion_error) {
+                    self->complete_external(index, std::move(completion_error));
+                });
+        } catch (...) {
+            complete_external(index, std::current_exception());
+        }
+    }
+
+    void complete_synchronous(std::size_t index,
+                              std::uint64_t cpu_end) noexcept {
+        std::vector<std::size_t> ready;
+        {
+            std::lock_guard lock(mutex);
+            --active_tasks;
+            auto& node = nodes[index];
+            node.status = Node_Execution_Status::complete;
+            if (auto* execution = executions[index]) {
+                execution->cpu_end_time_ns = cpu_end;
+                execution->end_time_ns = cpu_end;
+                execution->status = Node_Execution_Status::complete;
+            }
+            --unfinished_nodes;
+            unlock_successors_locked(index, ready);
+            finish_if_terminal_locked();
+        }
+        submit(ready);
+    }
+
+    void complete_external(std::size_t index,
+                           std::exception_ptr error) noexcept {
+        std::vector<std::size_t> ready;
+        {
+            std::lock_guard lock(mutex);
+            auto& node = nodes[index];
+            if (node.status != Node_Execution_Status::waiting_external)
+                return;
+            --waiting_external;
+            const std::uint64_t end = render_clock_now_ns();
+            if (auto* execution = executions[index]) {
+                execution->external_end_time_ns = end;
+                execution->end_time_ns = end;
+            }
+            if (error) {
+                fail_node_locked(index, std::move(error), end);
+            } else {
+                node.status = Node_Execution_Status::complete;
+                if (auto* execution = executions[index])
+                    execution->status = Node_Execution_Status::complete;
+                --unfinished_nodes;
+                unlock_successors_locked(index, ready);
+            }
+            finish_if_terminal_locked();
+        }
+        submit(ready);
+    }
+
+    void unlock_successors_locked(std::size_t index,
+                                  std::vector<std::size_t>& ready) {
+        if (failed)
+            return;
+        for (const std::size_t successor : successors[index]) {
+            auto& state = nodes[successor];
+            if (state.remaining_predecessors == 0)
+                continue;
+            --state.remaining_predecessors;
+            if (state.remaining_predecessors == 0)
+                make_ready_locked(successor, ready);
+        }
+    }
+
+    void fail_node_locked(std::size_t index, std::exception_ptr error,
+                          std::uint64_t end) {
+        auto& node = nodes[index];
+        node.status = Node_Execution_Status::failed;
+        if (auto* execution = executions[index]) {
+            if (execution->start_time_ns == 0)
+                execution->start_time_ns = end;
+            if (execution->cpu_end_time_ns == 0)
+                execution->cpu_end_time_ns = end;
+            execution->end_time_ns = end;
+            execution->status = Node_Execution_Status::failed;
+        }
+        fail_graph_locked(std::move(error));
+    }
+
+    void fail_graph_locked(std::exception_ptr error) {
+        failed = true;
+        if (!first_exception)
+            first_exception = std::move(error);
+    }
+
+    void finish_if_terminal_locked() {
+        if (terminal)
+            return;
+        if ((!failed && unfinished_nodes == 0) ||
+            (failed && active_tasks == 0 && waiting_external == 0)) {
+            terminal = true;
+            completion.notify_all();
+        }
+    }
+
+    std::vector<Node_State> nodes;
+    std::vector<std::vector<std::size_t>> successors;
+    std::vector<Node_Execution*> executions;
+    Execute_Node execute_node;
+    Schedule schedule;
+    Current_Worker_Id current_worker_id;
+    std::mutex mutex;
+    std::condition_variable completion;
+    std::exception_ptr first_exception;
+    std::size_t unfinished_nodes{};
+    std::size_t active_tasks{};
+    std::size_t waiting_external{};
+    bool started{};
+    bool failed{};
+    bool terminal{};
+};
+
+Render_Graph_Runtime::Render_Graph_Runtime(
+    const Render_Plan& plan,
+    std::span<Node_Execution* const> executions,
+    Execute_Node execute_node,
+    Schedule schedule,
+    Current_Worker_Id current_worker_id)
+    : state_(std::make_shared<State>(
+          plan, executions, std::move(execute_node), std::move(schedule),
+          std::move(current_worker_id))) {}
+
+Render_Graph_Runtime::~Render_Graph_Runtime() = default;
+
+void Render_Graph_Runtime::execute() {
+    state_->execute();
+}
+
+} // namespace renderive::render_graph::detail

Kernel/src/renderive/render_graph/detail/Render_Graph_Runtime.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <span>
+
+#include "renderive/render_graph/External_Operation.hpp"
+#include "renderive/render_graph/Render_Plan.hpp"
+#include "renderive/scene/base/Abstract_Frame.hpp"
+
+namespace renderive::render_graph::detail {
+
+class Render_Graph_Runtime final {
+public:
+    using Execute_Node = std::function<Node_Execution_Result(
+        std::size_t execution_index, Node_Execution_Metrics* metrics)>;
+    using Schedule = std::function<void(std::function<void()>)>;
+    using Current_Worker_Id = std::function<std::uint32_t()>;
+
+    Render_Graph_Runtime(const Render_Plan& plan,
+                         std::span<Node_Execution* const> executions,
+                         Execute_Node execute_node,
+                         Schedule schedule,
+                         Current_Worker_Id current_worker_id);
+    ~Render_Graph_Runtime();
+
+    Render_Graph_Runtime(const Render_Graph_Runtime&) = delete;
+    Render_Graph_Runtime& operator=(const Render_Graph_Runtime&) = delete;
+    Render_Graph_Runtime(Render_Graph_Runtime&&) = delete;
+    Render_Graph_Runtime& operator=(Render_Graph_Runtime&&) = delete;
+
+    void execute();
+
+private:
+    struct State;
+    std::shared_ptr<State> state_;
+};
+
+} // namespace renderive::render_graph::detail

Kernel/src/renderive/scene/base/Abstract_Frame.cpp

@@ -23,6 +23,18 @@ std::uint64_t Node_Execution::duration_ns() const noexcept {
     return end_time_ns >= start_time_ns ? end_time_ns - start_time_ns : 0;
 }
 
+std::uint64_t Node_Execution::cpu_duration_ns() const noexcept {
+    return cpu_end_time_ns >= start_time_ns
+        ? cpu_end_time_ns - start_time_ns
+        : 0;
+}
+
+std::uint64_t Node_Execution::external_duration_ns() const noexcept {
+    return external_end_time_ns >= external_start_time_ns
+        ? external_end_time_ns - external_start_time_ns
+        : 0;
+}
+
 std::uint64_t Frame_Snapshot::render_duration_ns() const noexcept {
     return render_end_ns >= render_start_ns ? render_end_ns - render_start_ns : 0;
 }

Kernel/src/renderive/scene/base/Abstract_Frame.hpp

@@ -8,7 +8,9 @@
 
 enum class Node_Execution_Status : std::uint8_t {
     pending,
+    ready,
     running,
+    waiting_external,
     complete,
     failed
 };
@@ -19,9 +21,26 @@ enum class Node_Metric_Kind : std::uint8_t {
     prepared_cells,
     pixel_count,
     primitive_count,
+    queue_wait_ns,
+    apply_duration_ns,
+    plan_emit_duration_ns,
+    backend_execute_duration_ns,
+    submit_duration_ns,
+    gpu_fence_wait_ns,
+    gpu_render_duration_ns,
+    gpu_copy_duration_ns,
+    readback_duration_ns,
+    backend_resource_version,
+    backend_frame_index,
+    backend_artifact_status,
+    backend_validation_code,
+    backend_validation_command,
+    backend_artifact_json_bytes,
     count
 };
 
+static_assert(static_cast<std::size_t>(Node_Metric_Kind::count) <= 32);
+
 struct Node_Execution_Metrics {
     void set(Node_Metric_Kind kind, std::uint64_t value) noexcept;
     [[nodiscard]] bool contains(Node_Metric_Kind kind) const noexcept;
@@ -32,12 +51,18 @@ struct Node_Execution_Metrics {
 
 struct Node_Execution {
     Render_Node_Id node_id{};
+    std::uint64_t ready_time_ns{};
     std::uint64_t start_time_ns{};
+    std::uint64_t cpu_end_time_ns{};
+    std::uint64_t external_start_time_ns{};
+    std::uint64_t external_end_time_ns{};
     std::uint64_t end_time_ns{};
     std::uint32_t worker_id{};
     Node_Execution_Status status{Node_Execution_Status::pending};
     Node_Execution_Metrics metrics;
     [[nodiscard]] std::uint64_t duration_ns() const noexcept;
+    [[nodiscard]] std::uint64_t cpu_duration_ns() const noexcept;
+    [[nodiscard]] std::uint64_t external_duration_ns() const noexcept;
 };
 
 struct Frame_Snapshot {

Kernel/src/renderive/scene/base/Scene_Base.cpp

@@ -19,6 +19,7 @@
 #include "renderive/renderable/Renderive_Id_Allocator.hpp"
 #include "renderive/renderable/base/Renderable_Base_p.hpp"
 #include "renderive/renderable/color/Color_Cache.hpp"
+#include "renderive/render_graph/detail/Render_Graph_Runtime.hpp"
 #include "renderive/state/base/State_Strategy_Base.hpp"
 
 static_assert(TF_VERSION == 400100, "Renderive requires Taskflow 4.1.0");
@@ -923,7 +924,7 @@ void Scene_Base::render_loop() {
                        task.snapshot, task.plan});
         std::exception_ptr exception;
         try {
-            execute_taskflow(task);
+            execute_render_graph(task);
             d_func().dispatch({Observation_Event::render_completed, d_func().now_ns(),
                            snapshot.render_sequence,
                            snapshot.scene_state_revision_,
@@ -1108,7 +1109,7 @@ std::shared_ptr<const Render_Plan> Scene_Base::compile_render_plan(
             Execution_Binding{{}, std::nullopt,
                 Scene_Render_Node_Function{
                     [renderer](const Scene_Render_Context& context) {
-                        renderer->render_scene(context);
+                        return renderer->render_scene(context);
                     }}});
         for (const Render_Node_Id terminal : terminals)
             append_edge(terminal, scene_render_node_id_);
@@ -1128,7 +1129,7 @@ std::shared_ptr<const Render_Plan> Scene_Base::compile_render_plan(
     return plan;
 }
 
-void Scene_Base::execute_taskflow(Render_Task& task) {
+void Scene_Base::execute_render_graph(Render_Task& task) {
     Abstract_Frame* const frame = std::visit(
         []<class Frame>(Frame& value) -> Abstract_Frame* {
             if constexpr (std::same_as<Frame, std::shared_ptr<Abstract_Frame>>)
@@ -1148,93 +1149,75 @@ void Scene_Base::execute_taskflow(Render_Task& task) {
                 state.paint_buffer_->clear();
         }
 
-        tf::Taskflow taskflow;
-        std::vector<tf::Task> tasks;
-        tasks.reserve(task.plan->graph.nodes.size());
         const bool capture = capture_ticket.capture;
         frame->begin_render(snapshot.render_sequence, *task.plan, capture,
                             capture ? render_clock_now_ns() : 0);
         frame_active = true;
-        for (const auto& node : task.plan->graph.nodes) {
-            tasks.push_back(taskflow.emplace([
-                this, &task, &snapshot, frame, node
-            ](tf::Runtime& runtime) {
-                Node_Execution* execution =
-                    frame->execution_slot(node.execution_index);
-                Node_Execution_Metrics* metrics =
-                    execution ? &execution->metrics : nullptr;
-                const auto run = [&] {
-                    const auto& binding =
-                        task.execution_bindings.at(node.execution_index);
-                    switch (node.kind) {
-                    case Render_Node_Kind::prepare: {
-                        const auto& state = snapshot.renderables.at(
-                            binding.renderable_index.value());
-                        std::get<Prepare_Render_Node_Function>(binding.function)(
-                            Prepare_Render_Context{snapshot, state, metrics});
-                        break;
-                    }
-                    case Render_Node_Kind::paint: {
-                        const auto& state = snapshot.renderables.at(
-                            binding.renderable_index.value());
-                        if (!state.paint_buffer_)
-                            throw std::logic_error(
-                                "paint node has no frame color cache");
-                        std::get<Paint_Render_Node_Function>(binding.function)(
-                            Paint_Render_Context{snapshot, state,
-                                                 *state.paint_buffer_, metrics});
-                        break;
-                    }
-                    case Render_Node_Kind::composite: {
-                        const Renderable_Frame_State* state =
-                            binding.renderable_index
-                                ? &snapshot.renderables.at(
-                                      *binding.renderable_index)
-                                : nullptr;
-                        const Color_Cache* cache =
-                            state && state->paint_buffer_
-                                ? state->paint_buffer_.get()
-                                : nullptr;
-                        std::get<Composite_Render_Node_Function>(binding.function)(
-                            Composite_Render_Context{snapshot, state, cache,
-                                                     metrics});
-                        break;
-                    }
-                    case Render_Node_Kind::render:
-                        std::get<Scene_Render_Node_Function>(binding.function)(
-                            Scene_Render_Context{snapshot, metrics});
-                        break;
-                    }
-                };
-                if (!execution) {
-                    Render_Execution_Scope scope(*this);
-                    run();
-                    return;
-                }
-                execution->worker_id =
-                    static_cast<std::uint32_t>(runtime.worker().id());
-                execution->start_time_ns = render_clock_now_ns();
-                execution->status = Node_Execution_Status::running;
-                Render_Execution_Scope scope(*this);
-                try {
-                    run();
-                    execution->end_time_ns = render_clock_now_ns();
-                    execution->status = Node_Execution_Status::complete;
-                } catch (...) {
-                    execution->end_time_ns = render_clock_now_ns();
-                    execution->status = Node_Execution_Status::failed;
-                    throw;
-                }
-            }).name(node.name));
-        }
-        std::unordered_map<Render_Node_Id, std::size_t> node_indices;
-        node_indices.reserve(task.plan->graph.nodes.size());
+        std::vector<Node_Execution*> execution_slots(
+            task.plan->graph.nodes.size());
         for (const auto& node : task.plan->graph.nodes)
-            node_indices.emplace(node.node_id, node.execution_index);
-        for (const auto& edge : task.plan->graph.edges)
-            tasks.at(node_indices.at(edge.from))
-                .precede(tasks.at(node_indices.at(edge.to)));
-        Execution_Context::executor().run(taskflow).get();
+            execution_slots[node.execution_index] =
+                frame->execution_slot(node.execution_index);
+
+        auto& executor = Execution_Context::executor();
+        renderive::render_graph::detail::Render_Graph_Runtime runtime(
+            *task.plan, execution_slots,
+            [this, &task, &snapshot](
+                std::size_t execution_index,
+                Node_Execution_Metrics* metrics) -> Node_Execution_Result {
+                const auto& node = task.plan->graph.nodes.at(execution_index);
+                const auto& binding =
+                    task.execution_bindings.at(execution_index);
+                Render_Execution_Scope scope(*this);
+                switch (node.kind) {
+                case Render_Node_Kind::prepare: {
+                    const auto& state = snapshot.renderables.at(
+                        binding.renderable_index.value());
+                    std::get<Prepare_Render_Node_Function>(binding.function)(
+                        Prepare_Render_Context{snapshot, state, metrics});
+                    return Node_Execution_Result::completed();
+                }
+                case Render_Node_Kind::paint: {
+                    const auto& state = snapshot.renderables.at(
+                        binding.renderable_index.value());
+                    if (!state.paint_buffer_)
+                        throw std::logic_error(
+                            "paint node has no frame color cache");
+                    std::get<Paint_Render_Node_Function>(binding.function)(
+                        Paint_Render_Context{snapshot, state,
+                                             *state.paint_buffer_, metrics});
+                    return Node_Execution_Result::completed();
+                }
+                case Render_Node_Kind::composite: {
+                    const Renderable_Frame_State* state =
+                        binding.renderable_index
+                            ? &snapshot.renderables.at(
+                                  *binding.renderable_index)
+                            : nullptr;
+                    const Color_Cache* cache = state && state->paint_buffer_
+                        ? state->paint_buffer_.get()
+                        : nullptr;
+                    std::get<Composite_Render_Node_Function>(binding.function)(
+                        Composite_Render_Context{snapshot, state, cache,
+                                                 metrics});
+                    return Node_Execution_Result::completed();
+                }
+                case Render_Node_Kind::render:
+                    return std::get<Scene_Render_Node_Function>(binding.function)(
+                        Scene_Render_Context{snapshot, metrics});
+                }
+                throw std::logic_error("unknown render node kind");
+            },
+            [&executor](std::function<void()> function) {
+                executor.silent_async(std::move(function));
+            },
+            [&executor] {
+                const int worker = executor.this_worker_id();
+                return worker < 0
+                    ? std::numeric_limits<std::uint32_t>::max()
+                    : static_cast<std::uint32_t>(worker);
+            });
+        runtime.execute();
 
         for (const auto& state : snapshot.renderables) {
             if (state.prepare_required)

Kernel/src/renderive/scene/base/Scene_Base.hpp

@@ -23,6 +23,7 @@
 #include "renderive/renderable/Renderable_Graph_Builder.hpp"
 #include "renderive/renderable/base/Renderable_Base.hpp"
 #include "renderive/renderable/base/Renderive_Owner.hpp"
+#include "renderive/render_graph/External_Operation.hpp"
 #include "renderive/render_graph/Render_Plan.hpp"
 #include "renderive/scene/Inheritance.hpp"
 #include "renderive/scene/dependency/Dependency_Resolver.hpp"
@@ -56,7 +57,8 @@ public:
 class Scene_Renderer {
 public:
     virtual ~Scene_Renderer() = default;
-    virtual void render_scene(const Scene_Render_Context& context) = 0;
+    virtual Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) = 0;
 };
 
 class Scene_Base {
@@ -215,7 +217,7 @@ private:
     using Composite_Render_Node_Function =
         std::function<void(const Composite_Render_Context&)>;
     using Scene_Render_Node_Function =
-        std::function<void(const Scene_Render_Context&)>;
+        std::function<Node_Execution_Result(const Scene_Render_Context&)>;
     using Execution_Function =
         std::variant<Prepare_Render_Node_Function, Paint_Render_Node_Function,
                      Composite_Render_Node_Function,
@@ -265,7 +267,7 @@ private:
     std::shared_ptr<const Render_Plan> compile_render_plan(
         const Frame_Render_Snapshot& snapshot, Render_Task& task);
     void render_loop();
-    void execute_taskflow(Render_Task& task);
+    void execute_render_graph(Render_Task& task);
     void validate_renderable_scene(const Renderable_Base& renderable) const;
     [[nodiscard]] bool is_renderable_attached_locked(
         const Renderable& renderable) const;

Kernel/tests/renderive/render_graph/Render_DAG_Test.cpp

@@ -67,18 +67,21 @@ TEST(render_dag_test, analysis_derives_wait_critical_path_and_parallel_overlap_f
     snapshot.render_start_ns = 100;
     snapshot.render_end_ns = 200;
     snapshot.node_executions.resize(plan->graph.nodes.size());
-    auto complete = [&snapshot, &plan](std::size_t slot, std::uint64_t start,
-                                      std::uint64_t end, std::uint32_t worker) {
+    auto complete = [&snapshot, &plan](std::size_t slot, std::uint64_t ready,
+                                      std::uint64_t start, std::uint64_t end,
+                                      std::uint32_t worker) {
         auto* execution = &snapshot.node_executions.at(slot);
         execution->node_id = plan->graph.nodes.at(slot).node_id;
+        execution->ready_time_ns = ready;
         execution->start_time_ns = start;
+        execution->cpu_end_time_ns = end;
         execution->end_time_ns = end;
         execution->worker_id = worker;
         execution->status = Node_Execution_Status::complete;
     };
-    complete(0, 110, 150, 0);
-    complete(1, 115, 145, 1);
-    complete(2, 160, 190, 1);
+    complete(0, 100, 110, 150, 0);
+    complete(1, 100, 115, 145, 1);
+    complete(2, 150, 160, 190, 1);
     const auto analysis = analyze_frame(*plan, snapshot);
 
     ASSERT_EQ(analysis.nodes.size(), 3u);
@@ -124,7 +127,9 @@ TEST(render_dag_test, capture_controller_and_repository_capture_exact_requested_
         for (std::size_t slot = 0; slot < plan->graph.nodes.size(); ++slot) {
             auto* execution = &snapshot->node_executions.at(slot);
             execution->node_id = plan->graph.nodes.at(slot).node_id;
+            execution->ready_time_ns = frame_id * 100;
             execution->start_time_ns = frame_id * 100 + slot * 10;
+            execution->cpu_end_time_ns = execution->start_time_ns + 5;
             execution->end_time_ns = execution->start_time_ns + 5;
             execution->worker_id = static_cast<std::uint32_t>(slot);
             execution->status = Node_Execution_Status::complete;
@@ -141,6 +146,43 @@ TEST(render_dag_test, capture_controller_and_repository_capture_exact_requested_
     EXPECT_EQ(session->frames[1].snapshot->frame_id, 2u);
 }
 
+TEST(render_dag_test,
+     external_wait_contributes_to_wall_path_but_not_cpu_worker_utilization) {
+    Render_Graph_Builder builder;
+    builder.emplace(201, 20, "GPU Render", Render_Node_Kind::render);
+    Render_Plan_History history;
+    const auto plan = history.publish(std::move(builder).finish());
+
+    Frame_Snapshot snapshot;
+    snapshot.frame_id = 7;
+    snapshot.render_plan_version = plan->version;
+    snapshot.render_start_ns = 1'000;
+    snapshot.render_end_ns = 1'200;
+    snapshot.node_executions.resize(1);
+    auto& execution = snapshot.node_executions.front();
+    execution.node_id = 201;
+    execution.ready_time_ns = 1'000;
+    execution.start_time_ns = 1'000;
+    execution.cpu_end_time_ns = 1'020;
+    execution.external_start_time_ns = 1'020;
+    execution.external_end_time_ns = 1'200;
+    execution.end_time_ns = 1'200;
+    execution.worker_id = 3;
+    execution.status = Node_Execution_Status::complete;
+
+    const auto analysis = analyze_frame(*plan, snapshot);
+    ASSERT_EQ(analysis.nodes.size(), 1U);
+    EXPECT_EQ(analysis.nodes.front().duration_ns, 200U);
+    EXPECT_EQ(analysis.nodes.front().cpu_duration_ns, 20U);
+    EXPECT_EQ(analysis.nodes.front().external_duration_ns, 180U);
+    EXPECT_EQ(analysis.total_work_duration_ns, 20U);
+    EXPECT_EQ(analysis.total_external_duration_ns, 180U);
+    EXPECT_EQ(analysis.critical_path_duration_ns, 200U);
+    ASSERT_EQ(analysis.workers.size(), 1U);
+    EXPECT_EQ(analysis.workers.front().work_duration_ns, 20U);
+    EXPECT_DOUBLE_EQ(analysis.workers.front().utilization, 0.1);
+}
+
 TEST(render_dag_test, failed_capture_ticket_is_retried_until_the_requested_frame_completes) {
     Capture_Controller controller;
     const auto session_id = controller.capture_next_frame();

Kernel/tests/renderive/render_graph/Render_Graph_Runtime_Test.cpp

@@ -0,0 +1,234 @@
+#include <gtest/gtest.h>
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <deque>
+#include <exception>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <stdexcept>
+#include <thread>
+#include <vector>
+
+#include "renderive/render_graph/External_Operation.hpp"
+#include "renderive/render_graph/Render_Plan.hpp"
+#include "renderive/render_graph/detail/Render_Graph_Runtime.hpp"
+
+namespace {
+
+class Single_Worker_Scheduler final {
+public:
+    Single_Worker_Scheduler()
+        : worker_([this] { run(); }) {}
+
+    ~Single_Worker_Scheduler() {
+        {
+            std::lock_guard lock(mutex_);
+            stopping_ = true;
+        }
+        ready_.notify_all();
+        worker_.join();
+    }
+
+    void schedule(std::function<void()> function) {
+        {
+            std::lock_guard lock(mutex_);
+            if (stopping_)
+                throw std::logic_error("scheduler is stopping");
+            queue_.push_back(std::move(function));
+        }
+        ready_.notify_one();
+    }
+
+private:
+    void run() {
+        for (;;) {
+            std::function<void()> function;
+            {
+                std::unique_lock lock(mutex_);
+                ready_.wait(lock, [this] {
+                    return stopping_ || !queue_.empty();
+                });
+                if (stopping_ && queue_.empty())
+                    return;
+                function = std::move(queue_.front());
+                queue_.pop_front();
+            }
+            function();
+        }
+    }
+
+    std::mutex mutex_;
+    std::condition_variable ready_;
+    std::deque<std::function<void()>> queue_;
+    std::thread worker_;
+    bool stopping_{};
+};
+
+std::shared_ptr<const Render_Plan> two_node_plan() {
+    Render_Graph_Builder builder;
+    const auto submit = builder.emplace(
+        301, 30, "Submit GPU", Render_Node_Kind::render);
+    const auto publish = builder.emplace(
+        302, 30, "Publish Frame", Render_Node_Kind::composite);
+    builder.precede(submit, publish);
+    Render_Plan_History history;
+    return history.publish(std::move(builder).finish());
+}
+
+std::vector<Node_Execution*> execution_slots(
+    const Render_Plan& plan, std::vector<Node_Execution>& storage) {
+    storage.resize(plan.graph.nodes.size());
+    std::vector<Node_Execution*> result(storage.size());
+    for (const auto& node : plan.graph.nodes) {
+        storage[node.execution_index].node_id = node.node_id;
+        result[node.execution_index] = &storage[node.execution_index];
+    }
+    return result;
+}
+
+} // namespace
+
+TEST(external_operation_test, completion_before_subscription_is_delivered_once) {
+    External_Operation_Source source;
+    const auto operation = source.operation();
+    EXPECT_TRUE(source.complete());
+    EXPECT_FALSE(source.complete());
+
+    int completion_count{};
+    operation.on_complete([&](std::exception_ptr error) {
+        EXPECT_FALSE(error);
+        ++completion_count;
+    });
+    EXPECT_EQ(completion_count, 1);
+}
+
+TEST(render_graph_runtime_test,
+     external_successor_stays_blocked_until_operation_completes) {
+    const auto plan = two_node_plan();
+    Single_Worker_Scheduler scheduler;
+    External_Operation_Source source;
+    std::vector<Node_Execution> execution_storage;
+    auto slots = execution_slots(*plan, execution_storage);
+    std::atomic<bool> submit_started{};
+    std::atomic<bool> publish_executed{};
+
+    renderive::render_graph::detail::Render_Graph_Runtime runtime(
+        *plan, slots,
+        [&](std::size_t index, Node_Execution_Metrics*) {
+            if (index == 0) {
+                submit_started.store(true, std::memory_order_release);
+                submit_started.notify_all();
+                return Node_Execution_Result::external(source.operation());
+            }
+            publish_executed.store(true, std::memory_order_release);
+            return Node_Execution_Result::completed();
+        },
+        [&](std::function<void()> function) {
+            scheduler.schedule(std::move(function));
+        },
+        [] { return 0U; });
+
+    std::thread execution([&] { runtime.execute(); });
+    submit_started.wait(false, std::memory_order_acquire);
+    EXPECT_FALSE(publish_executed.load(std::memory_order_acquire));
+    EXPECT_EQ(execution_storage[0].status,
+              Node_Execution_Status::waiting_external);
+    EXPECT_TRUE(source.complete());
+    execution.join();
+
+    EXPECT_TRUE(publish_executed.load(std::memory_order_acquire));
+    EXPECT_EQ(execution_storage[0].status, Node_Execution_Status::complete);
+    EXPECT_EQ(execution_storage[1].status, Node_Execution_Status::complete);
+    EXPECT_GT(execution_storage[0].cpu_end_time_ns, 0U);
+    EXPECT_GE(execution_storage[0].external_end_time_ns,
+              execution_storage[0].external_start_time_ns);
+}
+
+TEST(render_graph_runtime_test,
+     failed_external_node_prevents_its_successor_from_running) {
+    const auto plan = two_node_plan();
+    Single_Worker_Scheduler scheduler;
+    External_Operation_Source source;
+    std::atomic<bool> submit_started{};
+    std::atomic<bool> publish_executed{};
+    std::exception_ptr graph_error;
+
+    renderive::render_graph::detail::Render_Graph_Runtime runtime(
+        *plan, {},
+        [&](std::size_t index, Node_Execution_Metrics*) {
+            if (index == 0) {
+                submit_started.store(true, std::memory_order_release);
+                submit_started.notify_all();
+                return Node_Execution_Result::external(source.operation());
+            }
+            publish_executed.store(true, std::memory_order_release);
+            return Node_Execution_Result::completed();
+        },
+        [&](std::function<void()> function) {
+            scheduler.schedule(std::move(function));
+        },
+        [] { return 0U; });
+
+    std::thread execution([&] {
+        try {
+            runtime.execute();
+        } catch (...) {
+            graph_error = std::current_exception();
+        }
+    });
+    submit_started.wait(false, std::memory_order_acquire);
+    EXPECT_TRUE(source.fail(
+        std::make_exception_ptr(std::runtime_error("GPU submission failed"))));
+    execution.join();
+
+    ASSERT_TRUE(graph_error);
+    EXPECT_THROW(std::rethrow_exception(graph_error), std::runtime_error);
+    EXPECT_FALSE(publish_executed.load(std::memory_order_acquire));
+}
+
+TEST(render_graph_runtime_test,
+     pending_external_operation_does_not_occupy_scheduler_worker) {
+    Render_Graph_Builder builder;
+    builder.emplace(401, 40, "GPU Fence", Render_Node_Kind::render);
+    Render_Plan_History history;
+    const auto plan = history.publish(std::move(builder).finish());
+    Single_Worker_Scheduler scheduler;
+    External_Operation_Source source;
+    std::atomic<bool> external_started{};
+    std::mutex probe_mutex;
+    std::condition_variable probe_completed;
+    bool probe_ran{};
+
+    renderive::render_graph::detail::Render_Graph_Runtime runtime(
+        *plan, {},
+        [&](std::size_t, Node_Execution_Metrics*) {
+            external_started.store(true, std::memory_order_release);
+            external_started.notify_all();
+            return Node_Execution_Result::external(source.operation());
+        },
+        [&](std::function<void()> function) {
+            scheduler.schedule(std::move(function));
+        },
+        [] { return 0U; });
+
+    std::thread execution([&] { runtime.execute(); });
+    external_started.wait(false, std::memory_order_acquire);
+    scheduler.schedule([&] {
+        {
+            std::lock_guard lock(probe_mutex);
+            probe_ran = true;
+        }
+        probe_completed.notify_one();
+    });
+    {
+        std::unique_lock lock(probe_mutex);
+        EXPECT_TRUE(probe_completed.wait_for(
+            lock, std::chrono::seconds(1), [&] { return probe_ran; }));
+    }
+    EXPECT_TRUE(source.complete());
+    execution.join();
+}

Kernel/tests/renderive/scene/Scene3D_Context_Test.cpp

@@ -40,8 +40,10 @@ struct Scene3D_Prepared_Data_Renderable : Renderable_Test_Harness {
 
 struct Scene3D_Render_Node_Test_Scene final
     : Scene3D_Context<>, Scene_Renderer {
-    void render_scene(const Scene_Render_Context& context) override {
+    Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) override {
         prepared = context.prepared<Prepared_Mesh_Count>(owner);
+        return Node_Execution_Result::completed();
     }
 
     Renderable_Id owner{};
@@ -96,9 +98,11 @@ struct Scene3D_Playback_Snapshot_Test_Scene final
         wait_for_render();
     }
 
-    void render_scene(const Scene_Render_Context& context) override {
+    Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) override {
         rendered.push_back(
             context.frame.scene_state<Scene3D_Historical_State>().value);
+        return Node_Execution_Result::completed();
     }
 
     std::vector<int> rendered;

render_2D/render_2D/plottable/Plottable_Real_Time_Data.h

@@ -1,36 +1,14 @@
 #pragma once
 #include "../renderable/Renderable.h"
 #include <renderive/base/observer/Observer.hpp>
+#include <renderive/real_time_data/Double_Buffer_Frame_Observer.hpp>
 #include <renderive/real_time_data/Frame_Strategy_Observer.hpp>
 #include <renderive/real_time_data/History_Real_Time_Data.hpp>
 #include <mutex>
 namespace renderive::detail {
 using Plottable_Real_Time_Data_Observer = Observer_State<Frame_Strategy_Real_Time_Data_Observer>;
-struct Double_Buffer_Real_Time_Data_Observer {
-    static constexpr bool enabled = true;
-
-    template <class Target>
-    decltype(auto) bind(Target& target) {
-        return observer_.bind(target);
-    }
-
-    template <class Observation>
-    void observe(const Observation& observation) noexcept {
-        if (observation.event != Observation::Event::cache_updated)
-            return;
-        observer_.observe({
-            Real_Time_Data_Observation_Event::updated,
-            {this, Real_Time_Data_Retention::latest,
-             observation.cache_update_count, observation.time_ns,
-             observation.cache_update_count, 1}
-        });
-    }
-
-private:
-    Frame_Strategy_Real_Time_Data_Observer observer_;
-};
 using Plottable_Double_Buffer_Observer =
-Observer_State<Double_Buffer_Real_Time_Data_Observer>;
+Observer_State<::Double_Buffer_Frame_Observer>;
 inline Plottable_Real_Time_Data_Observer observe_real_time_data(
     ::renderive::Renderable& renderable) {
     return Plottable_Real_Time_Data_Observer(Frame_Strategy_Real_Time_Data_Observer(renderable));

render_2D/render_2D/renderable/Renderable.cpp

@@ -44,14 +44,6 @@ void Renderable::Impl::prepare_frame(const Prepare_Render_Context&) {}
 void Renderable::Impl::paint(detail::Painter&,
                              const Paint_Render_Context&) {}
 
-void Renderable::Impl::build_prepare_graph(
-    Renderable_Graph_Builder& builder) {
-    add_prepare_task(builder, "prepare", "Prepare",
-        [this](const Prepare_Render_Context& context) {
-            execute_prepare(context);
-        });
-}
-
 void Renderable::Impl::build_paint_graph(
     Renderable_Graph_Builder& builder) {
     const auto paint_task = add_paint_task(

render_2D/render_2D/renderable/Renderable_p.h

@@ -78,7 +78,6 @@ protected:
     virtual void prepare_frame(const Prepare_Render_Context& context);
     virtual void paint(detail::Painter& painter,
                        const Paint_Render_Context& context);
-    void build_prepare_graph(Renderable_Graph_Builder& builder) override;
     void build_paint_graph(Renderable_Graph_Builder& builder) override;
     void prepare(const Prepare_Render_Context& context) override;
     Renderable_Graph_Builder::Task add_prepare_task(

render_3D/render_3D/Point_Scene.cpp

@@ -1,6 +1,7 @@
 #include "Point_Scene.h"
 
 #include "detail/Datoviz_Visual_Backend.h"
+#include "detail/Gpu_Completion_Service.h"
 #include "detail/Point_Core.h"
 #include "detail/Render_Domain.h"
 
@@ -9,7 +10,9 @@
 
 #include <algorithm>
 #include <atomic>
+#include <chrono>
 #include <cmath>
+#include <exception>
 #include <memory>
 #include <mutex>
 #include <stdexcept>
@@ -37,6 +40,41 @@ float datoviz_wheel_step(float pixel_delta, float angle_delta) noexcept {
                                : pixel_delta / 100.0F;
 }
 
+void publish_trace(Node_Execution_Metrics* metrics,
+                   const detail::Datoviz_Frame_Trace& trace) {
+    if (!metrics)
+        return;
+    metrics->set(Node_Metric_Kind::queue_wait_ns,
+                 trace.render_domain_queue_wait_ns);
+    metrics->set(Node_Metric_Kind::apply_duration_ns, trace.apply_ns);
+    metrics->set(Node_Metric_Kind::plan_emit_duration_ns, trace.emit_ns);
+    metrics->set(Node_Metric_Kind::backend_execute_duration_ns,
+                 trace.execute_ns);
+    metrics->set(Node_Metric_Kind::submit_duration_ns, trace.submit_ns);
+    metrics->set(Node_Metric_Kind::gpu_fence_wait_ns,
+                 trace.gpu_fence_wait_ns);
+    metrics->set(Node_Metric_Kind::readback_duration_ns,
+                 trace.readback_ns);
+    if (trace.gpu) {
+        metrics->set(Node_Metric_Kind::gpu_render_duration_ns,
+                     trace.gpu->render_ns);
+        metrics->set(Node_Metric_Kind::gpu_copy_duration_ns,
+                     trace.gpu->copy_ns);
+    }
+    metrics->set(Node_Metric_Kind::backend_resource_version,
+                 trace.artifact_resource_version);
+    metrics->set(Node_Metric_Kind::backend_frame_index,
+                 trace.artifact_frame_index);
+    metrics->set(Node_Metric_Kind::backend_artifact_status,
+                 trace.artifact_status);
+    metrics->set(Node_Metric_Kind::backend_validation_code,
+                 trace.validation_code);
+    metrics->set(Node_Metric_Kind::backend_validation_command,
+                 trace.validation_command_index);
+    metrics->set(Node_Metric_Kind::backend_artifact_json_bytes,
+                 trace.artifact_json.size());
+}
+
 struct Scene_Model {
     virtual ~Scene_Model() = default;
     virtual void resize(Extent extent) = 0;
@@ -91,6 +129,7 @@ struct Basic_Point_Scene final
 
     ~Basic_Point_Scene() override {
         this->shutdown();
+        gpu_completion.shutdown();
         render_domain.invoke([this] { backend.reset(); });
     }
 
@@ -250,30 +289,122 @@ struct Basic_Point_Scene final
         return *this;
     }
 
-    void render_scene(const Scene_Render_Context& context) override {
+    Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) override {
         const auto prepared = context.prepared<detail::Prepared_Point>(point_id);
         if (!prepared)
             throw std::logic_error("Point_Visual did not publish prepared data");
-        const auto& scene_state =
+        const detail::Scene_State scene_state =
             context.frame.scene_state<detail::Scene_State>();
-        auto frame = render_domain.invoke([this, &scene_state, &context,
-                                           &prepared] {
-            return backend->render(scene_state,
-                                   context.frame.scene_state_revision(),
-                                   *prepared, context.frame.render_sequence);
-        });
-        if (context.metrics && frame)
-            context.metrics->set(
-                Node_Metric_Kind::pixel_count,
-                static_cast<std::uint64_t>(frame->extent.width) *
-                    frame->extent.height);
-        std::lock_guard lock(frame_mutex);
-        latest = std::move(frame);
+        const std::uint64_t scene_revision =
+            context.frame.scene_state_revision();
+        const std::uint64_t frame_sequence =
+            context.frame.render_sequence;
+        Node_Execution_Metrics* const metrics = context.metrics;
+        auto source = std::make_shared<External_Operation_Source>();
+        const auto operation = source->operation();
+        const auto queued_at = std::chrono::steady_clock::now();
+
+        try {
+            render_domain.post(
+                [this, scene_state, scene_revision, frame_sequence, prepared,
+                 metrics, source, queued_at] {
+                    try {
+                        const auto queue_wait =
+                            std::chrono::duration_cast<std::chrono::nanoseconds>(
+                                std::chrono::steady_clock::now() - queued_at)
+                                .count();
+                        auto pending = backend->submit(
+                            scene_state, scene_revision, *prepared,
+                            frame_sequence, metrics != nullptr);
+                        if (!pending) {
+                            {
+                                std::lock_guard lock(frame_mutex);
+                                latest.reset();
+                            }
+                            static_cast<void>(source->complete());
+                            return;
+                        }
+                        pending->trace.render_domain_queue_wait_ns =
+                            queue_wait > 0
+                                ? static_cast<std::uint64_t>(queue_wait)
+                                : 0;
+                        gpu_completion.watch(
+                            pending->device, pending->fence,
+                            [this, pending = *pending, metrics, source](
+                                detail::Gpu_Completion_Service::Result result)
+                                mutable {
+                                pending.trace.gpu_fence_wait_ns =
+                                    result.wait_duration_ns;
+                                try {
+                                    render_domain.post(
+                                        [this, pending = std::move(pending),
+                                         metrics, source,
+                                         wait_error = std::move(result.error)]()
+                                            mutable {
+                                            if (wait_error) {
+                                                try {
+                                                    backend->discard(
+                                                        std::move(pending));
+                                                    static_cast<void>(
+                                                        source->fail(std::move(
+                                                            wait_error)));
+                                                } catch (...) {
+                                                    static_cast<void>(
+                                                        source->fail(
+                                                            std::current_exception()));
+                                                }
+                                                return;
+                                            }
+                                            try {
+                                                auto completed =
+                                                    backend->collect(
+                                                        std::move(pending));
+                                                auto frame =
+                                                    std::move(completed.frame);
+                                                if (metrics && frame) {
+                                                    metrics->set(
+                                                        Node_Metric_Kind::
+                                                            pixel_count,
+                                                        static_cast<
+                                                            std::uint64_t>(
+                                                            frame->extent.width) *
+                                                            frame->extent.height);
+                                                }
+                                                publish_trace(
+                                                    metrics, completed.trace);
+                                                {
+                                                    std::lock_guard lock(
+                                                        frame_mutex);
+                                                    latest = std::move(frame);
+                                                }
+                                                static_cast<void>(
+                                                    source->complete());
+                                            } catch (...) {
+                                                static_cast<void>(source->fail(
+                                                    std::current_exception()));
+                                            }
+                                        });
+                                } catch (...) {
+                                    static_cast<void>(source->fail(
+                                        std::current_exception()));
+                                }
+                            });
+                    } catch (...) {
+                        static_cast<void>(
+                            source->fail(std::current_exception()));
+                    }
+                });
+        } catch (...) {
+            static_cast<void>(source->fail(std::current_exception()));
+        }
+        return Node_Execution_Result::external(operation);
     }
 
     std::shared_ptr<Point_Visual> visual;
     Renderable_Id point_id{};
     detail::Render_Domain render_domain;
+    detail::Gpu_Completion_Service gpu_completion;
     std::unique_ptr<detail::Datoviz_Visual_Backend> backend;
     mutable std::mutex frame_mutex;
     std::shared_ptr<const Pixel_Frame> latest;

render_3D/render_3D/Point_Visual.cpp

@@ -4,7 +4,7 @@
 #include "renderable/Renderable_p.h"
 
 #include <renderive/real_time_data/Double_Buffer_Strategy.hpp>
-#include <renderive/real_time_data/Frame_Strategy_Observer.hpp>
+#include <renderive/real_time_data/Double_Buffer_Frame_Observer.hpp>
 #include <renderive/scene/base/Abstract_Frame.hpp>
 #include <datoviz/scene/enums.h>
 
@@ -34,32 +34,10 @@ using Point_Payload = std::shared_ptr<const std::vector<Point>>;
 using Point_Buffer_Layout = ::Double_Buffer_Layout<
     ::Buffered_Data<Point_Payload_Tag, Point_Payload>>;
 
-struct Point_Buffer_Observer {
-    static constexpr bool enabled = true;
-
-    template <class Target>
-    decltype(auto) bind(Target& target) {
-        return observer_.bind(target);
-    }
-
-    template <class Observation>
-    void observe(const Observation& observation) noexcept {
-        if (observation.event != Observation::Event::cache_updated)
-            return;
-        observer_.observe({
-            Real_Time_Data_Observation_Event::updated,
-            {this, Real_Time_Data_Retention::latest,
-             observation.cache_update_count, observation.time_ns,
-             observation.cache_update_count, 1}});
-    }
-
-private:
-    Frame_Strategy_Real_Time_Data_Observer observer_;
-};
-
 using Point_Buffer_Strategy =
     ::Multi_Double_Buffer_Strategy<
-        Point_Buffer_Layout, std::mutex, Observer_State<Point_Buffer_Observer>>;
+        Point_Buffer_Layout, std::mutex,
+        Observer_State<::Double_Buffer_Frame_Observer>>;
 
 struct Point_Data_Observation {
     std::uint64_t revision{};
@@ -111,8 +89,8 @@ struct Point_Visual::Impl : next_Impl<Impl>, Point_Buffer_Strategy {
     void edit_points(const std::function<void(std::vector<Point>&)>& edit) {
         if (!edit)
             throw std::invalid_argument("Point_Visual point edit is empty");
-        const auto& published = points();
-        auto next = published ? *published : std::vector<Point>{};
+        const auto& cached = cache_buffer_value<Point_Payload_Tag>();
+        auto next = cached ? *cached : std::vector<Point>{};
         edit(next);
         update_points(std::move(next));
     }

render_3D/render_3D/detail/Datoviz_Frame_Trace.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <cstdint>
+#include <optional>
+#include <string>
+
+namespace renderive::render_3d::detail {
+
+struct Datoviz_Gpu_Timing {
+    std::uint64_t render_ns{};
+    std::uint64_t copy_ns{};
+};
+
+struct Datoviz_Frame_Trace {
+    std::uint64_t render_sequence{};
+    std::uint64_t render_domain_queue_wait_ns{};
+    std::uint64_t apply_ns{};
+    std::uint64_t emit_ns{};
+    std::uint64_t execute_ns{};
+    std::uint64_t submit_ns{};
+    std::uint64_t gpu_fence_wait_ns{};
+    std::uint64_t readback_ns{};
+    std::optional<Datoviz_Gpu_Timing> gpu;
+    std::uint64_t artifact_resource_version{};
+    std::uint64_t artifact_frame_index{};
+    std::uint32_t artifact_status{};
+    std::uint32_t validation_code{};
+    std::uint64_t validation_command_index{};
+    bool validation_ok{};
+    std::string artifact_json;
+};
+
+} // namespace renderive::render_3d::detail

render_3D/render_3D/detail/Datoviz_Visual_Backend.cpp

@@ -1,9 +1,12 @@
 #include "Datoviz_Visual_Backend.h"
 
+#include <datoviz/drp2/stream.h>
 #include <datoviz/scene/frame_plan.h>
+#include <volk.h>
 
 #include <algorithm>
 #include <array>
+#include <chrono>
 #include <cmath>
 #include <cstring>
 #include <limits>
@@ -16,6 +19,12 @@ namespace {
 
 constexpr std::uint64_t color_target_id = 0x5256504f494e54ULL;
 
+std::uint64_t trace_now_ns() noexcept {
+    return static_cast<std::uint64_t>(
+        std::chrono::duration_cast<std::chrono::nanoseconds>(
+            std::chrono::steady_clock::now().time_since_epoch()).count());
+}
+
 template <class Resource, class Allocate>
 Resource* allocate_wrapper(Allocate allocate, const char* message) {
     Resource* resource = allocate();
@@ -164,6 +173,11 @@ void configure_glyph_text(DvzScene* scene, DvzVisual* visual, const char* text)
 
 class Datoviz_Visual_Backend::Frame_Target final {
 public:
+    struct Collection {
+        std::vector<std::byte> pixels;
+        std::optional<Datoviz_Gpu_Timing> gpu_timing;
+    };
+
     Frame_Target(DvzGpuCtx* gpu_context, Extent extent, std::uint64_t generation)
         : gpu_context_(gpu_context), extent_(extent), generation_(generation) {
         if (gpu_context == nullptr || extent.empty())
@@ -224,6 +238,7 @@ public:
             dvz_buffer_usage(readback_, VK_BUFFER_USAGE_TRANSFER_DST_BIT);
             if (dvz_buffer_create(readback_) != 0)
                 throw std::runtime_error("failed to create Datoviz readback buffer");
+            initialize_timestamps(device, queue);
         } catch (...) {
             destroy();
             throw;
@@ -233,8 +248,8 @@ public:
     ~Frame_Target() { destroy(); }
 
     void begin() {
-        if (!dvz_fence_wait(fence_))
-            throw std::runtime_error("failed to wait for Datoviz frame fence");
+        if (in_flight_)
+            throw std::logic_error("Datoviz frame target is still in flight");
         dvz_cmd_reset(commands_);
         if (dvz_cmd_begin_result(commands_) != 0)
             throw std::runtime_error("failed to begin Datoviz command buffer");
@@ -263,6 +278,15 @@ public:
         dvz_barrier_image_mip(image_barrier, 0, 1);
         dvz_barrier_image_layers(image_barrier, 0, 1);
         dvz_cmd_barriers(commands_, &barriers);
+        if (timestamps_supported_) {
+            const VkCommandBuffer command_buffer =
+                dvz_commands_handle(commands_);
+            vkCmdResetQueryPool(command_buffer, query_pool_, 0, 4);
+            vkCmdWriteTimestamp(
+                command_buffer,
+                VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+                query_pool_, 0);
+        }
         recording_ = true;
     }
 
@@ -287,9 +311,16 @@ public:
         return frame;
     }
 
-    [[nodiscard]] std::vector<std::byte> finish() {
+    void submit() {
         if (!recording_)
             throw std::logic_error("Datoviz frame target is not recording");
+        const VkCommandBuffer command_buffer = dvz_commands_handle(commands_);
+        if (timestamps_supported_) {
+            vkCmdWriteTimestamp(
+                command_buffer,
+                VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+                query_pool_, 1);
+        }
         DvzBarriers image_barriers{};
         dvz_barriers(&image_barriers);
         auto* image_barrier =
@@ -307,6 +338,10 @@ public:
         dvz_barrier_image_mip(image_barrier, 0, 1);
         dvz_barrier_image_layers(image_barrier, 0, 1);
         dvz_cmd_barriers(commands_, &image_barriers);
+        if (timestamps_supported_) {
+            vkCmdWriteTimestamp(command_buffer, VK_PIPELINE_STAGE_TRANSFER_BIT,
+                                query_pool_, 2);
+        }
 
         DvzImageRegion region{};
         dvz_image_region(&region);
@@ -325,6 +360,11 @@ public:
         dvz_barrier_buffer_access(buffer_barrier, VK_ACCESS_2_TRANSFER_WRITE_BIT,
                                   VK_ACCESS_2_HOST_READ_BIT);
         dvz_cmd_barriers(commands_, &buffer_barriers);
+        if (timestamps_supported_) {
+            vkCmdWriteTimestamp(command_buffer,
+                                VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
+                                query_pool_, 3);
+        }
 
         if (dvz_cmd_end_result(commands_) != 0)
             throw std::runtime_error("failed to end Datoviz command buffer");
@@ -334,14 +374,44 @@ public:
         dvz_submit_command(submit_, dvz_commands_handle(commands_));
         DvzQueue* queue = dvz_gpu_ctx_queue(gpu_context_, DVZ_QUEUE_MAIN);
         if (dvz_submit_send(submit_, dvz_queue_handle(queue),
-                            dvz_fence_handle(fence_)) != VK_SUCCESS ||
-            !dvz_fence_wait(fence_))
+                            dvz_fence_handle(fence_)) != VK_SUCCESS)
             throw std::runtime_error("failed to submit Datoviz point frame");
+        in_flight_ = true;
         completed_layout_ = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
+    }
 
-        std::vector<std::byte> pixels(static_cast<std::size_t>(byte_size_));
-        dvz_buffer_download(readback_, 0, byte_size_, pixels.data());
-        return pixels;
+    [[nodiscard]] Collection collect() {
+        if (!in_flight_)
+            throw std::logic_error("Datoviz frame target has no pending frame");
+        Collection result;
+        try {
+            result.pixels.resize(static_cast<std::size_t>(byte_size_));
+            dvz_buffer_download(readback_, 0, byte_size_, result.pixels.data());
+            result.gpu_timing = collect_gpu_timing();
+        } catch (...) {
+            in_flight_ = false;
+            throw;
+        }
+        in_flight_ = false;
+        return result;
+    }
+
+    void discard_after_completion() {
+        if (!in_flight_)
+            throw std::logic_error("Datoviz frame target has no pending frame");
+        in_flight_ = false;
+    }
+
+    [[nodiscard]] VkDevice device() const {
+        return dvz_device_handle(dvz_gpu_ctx_device(gpu_context_));
+    }
+
+    [[nodiscard]] VkFence fence() const {
+        return dvz_fence_handle(fence_);
+    }
+
+    [[nodiscard]] std::uint64_t generation() const noexcept {
+        return generation_;
     }
 
     void abort() noexcept {
@@ -351,9 +421,94 @@ public:
     }
 
 private:
+    void initialize_timestamps(DvzDevice* device, DvzQueue* queue) noexcept {
+        if (device == nullptr || queue == nullptr ||
+            vkGetPhysicalDeviceQueueFamilyProperties == nullptr ||
+            vkGetPhysicalDeviceProperties == nullptr ||
+            vkCreateQueryPool == nullptr ||
+            vkCmdResetQueryPool == nullptr ||
+            vkCmdWriteTimestamp == nullptr ||
+            vkGetQueryPoolResults == nullptr)
+            return;
+        const VkPhysicalDevice physical =
+            dvz_device_physical_device(device);
+        const VkDevice logical = dvz_device_handle(device);
+        if (physical == VK_NULL_HANDLE || logical == VK_NULL_HANDLE)
+            return;
+
+        std::uint32_t family_count{};
+        vkGetPhysicalDeviceQueueFamilyProperties(
+            physical, &family_count, nullptr);
+        if (family_count == 0)
+            return;
+        std::vector<VkQueueFamilyProperties> families(family_count);
+        vkGetPhysicalDeviceQueueFamilyProperties(
+            physical, &family_count, families.data());
+        const std::uint32_t family = dvz_queue_family(queue);
+        if (family >= family_count ||
+            families[family].timestampValidBits == 0)
+            return;
+
+        VkPhysicalDeviceProperties properties{};
+        vkGetPhysicalDeviceProperties(physical, &properties);
+        VkQueryPoolCreateInfo configuration{
+            VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO};
+        configuration.queryType = VK_QUERY_TYPE_TIMESTAMP;
+        configuration.queryCount = 4;
+        if (vkCreateQueryPool(logical, &configuration, nullptr,
+                              &query_pool_) != VK_SUCCESS) {
+            query_pool_ = VK_NULL_HANDLE;
+            return;
+        }
+        timestamp_period_ns_ = properties.limits.timestampPeriod;
+        timestamp_valid_bits_ = families[family].timestampValidBits;
+        timestamps_supported_ = timestamp_period_ns_ > 0.0F;
+    }
+
+    [[nodiscard]] std::optional<Datoviz_Gpu_Timing>
+    collect_gpu_timing() const noexcept {
+        if (!timestamps_supported_ || query_pool_ == VK_NULL_HANDLE)
+            return std::nullopt;
+        const VkDevice device =
+            dvz_device_handle(dvz_gpu_ctx_device(gpu_context_));
+        std::array<std::uint64_t, 4> timestamps{};
+        if (vkGetQueryPoolResults(
+                device, query_pool_, 0,
+                static_cast<std::uint32_t>(timestamps.size()),
+                sizeof(timestamps), timestamps.data(), sizeof(std::uint64_t),
+                VK_QUERY_RESULT_64_BIT) != VK_SUCCESS)
+            return std::nullopt;
+
+        const auto elapsed = [this](std::uint64_t begin,
+                                    std::uint64_t end) noexcept {
+            std::uint64_t ticks = end - begin;
+            if (timestamp_valid_bits_ < 64) {
+                const std::uint64_t mask =
+                    (std::uint64_t{1} << timestamp_valid_bits_) - 1;
+                ticks &= mask;
+            }
+            const long double nanoseconds =
+                static_cast<long double>(ticks) * timestamp_period_ns_;
+            return nanoseconds >=
+                    static_cast<long double>(
+                        std::numeric_limits<std::uint64_t>::max())
+                ? std::numeric_limits<std::uint64_t>::max()
+                : static_cast<std::uint64_t>(nanoseconds);
+        };
+        return Datoviz_Gpu_Timing{
+            elapsed(timestamps[0], timestamps[1]),
+            elapsed(timestamps[2], timestamps[3])};
+    }
+
     void destroy() noexcept {
-        if (gpu_context_ != nullptr && fence_ != nullptr)
-            (void)dvz_fence_wait(fence_);
+        if (in_flight_)
+            std::terminate();
+        if (query_pool_ != VK_NULL_HANDLE && gpu_context_ != nullptr) {
+            vkDestroyQueryPool(
+                dvz_device_handle(dvz_gpu_ctx_device(gpu_context_)),
+                query_pool_, nullptr);
+            query_pool_ = VK_NULL_HANDLE;
+        }
         if (readback_ != nullptr) {
             dvz_buffer_destroy(readback_);
             dvz_buffer_free(readback_);
@@ -395,8 +550,13 @@ private:
     DvzFence* fence_{};
     DvzSubmit* submit_{};
     DvzBuffer* readback_{};
+    VkQueryPool query_pool_{VK_NULL_HANDLE};
     VkImageLayout completed_layout_{VK_IMAGE_LAYOUT_UNDEFINED};
+    float timestamp_period_ns_{};
+    std::uint32_t timestamp_valid_bits_{};
     bool recording_{};
+    bool in_flight_{};
+    bool timestamps_supported_{};
 };
 
 Datoviz_Visual_Backend::Datoviz_Visual_Backend(
@@ -859,13 +1019,19 @@ DvzSceneFrameArtifact* Datoviz_Visual_Backend::emit(
     return artifact;
 }
 
-std::shared_ptr<const Pixel_Frame> Datoviz_Visual_Backend::render(
+std::optional<Datoviz_Visual_Backend::Pending_Frame>
+Datoviz_Visual_Backend::submit(
     const Scene_State& scene, std::uint64_t scene_revision,
-    const Prepared_Point& point, std::uint64_t frame_sequence) {
+    const Prepared_Point& point, std::uint64_t frame_sequence,
+    bool capture_trace) {
     require_domain();
     if (scene.viewport.empty())
-        return {};
+        return std::nullopt;
+    Datoviz_Frame_Trace trace;
+    trace.render_sequence = frame_sequence;
+    auto phase_started = trace_now_ns();
     apply(scene, scene_revision, point);
+    trace.apply_ns = trace_now_ns() - phase_started;
 
     if (target_ == nullptr || target_extent_ != scene.viewport) {
         target_.reset();
@@ -877,19 +1043,39 @@ std::shared_ptr<const Pixel_Frame> Datoviz_Visual_Backend::render(
     target_->begin();
     DvzSceneFrameArtifact* artifact{};
     try {
+        phase_started = trace_now_ns();
         artifact = emit(scene);
+        trace.emit_ns = trace_now_ns() - phase_started;
     } catch (...) {
         target_->abort();
         throw;
     }
+    trace.artifact_status = static_cast<std::uint32_t>(
+        dvz_scene_frame_artifact_status(artifact));
+    trace.artifact_resource_version =
+        dvz_scene_frame_artifact_resource_version(artifact);
+    trace.artifact_frame_index =
+        dvz_scene_frame_artifact_frame_index(artifact);
     const DvzDrp2CommandStream* stream = dvz_scene_frame_artifact_stream(artifact);
     const DvzStreamFrame target_frame = target_->stream_frame();
+    phase_started = trace_now_ns();
     const bool attached = stream != nullptr &&
                           dvz_drp2_runtime_attach_frame_target(
                               runtime_, color_target_id, &target_frame);
     const DvzDrp2ValidationResult result =
         attached ? dvz_drp2_runtime_execute(runtime_, stream)
                  : DvzDrp2ValidationResult{};
+    trace.execute_ns = trace_now_ns() - phase_started;
+    trace.validation_ok = attached && result.ok;
+    trace.validation_code = static_cast<std::uint32_t>(result.code);
+    trace.validation_command_index = result.command_index;
+    if (capture_trace || !trace.validation_ok) {
+        if (char* json = dvz_scene_frame_artifact_json(
+                artifact, "renderive_frame")) {
+            trace.artifact_json = json;
+            dvz_drp2_stream_json_destroy(json);
+        }
+    }
     dvz_scene_frame_artifact_destroy(artifact);
     if (!attached) {
         target_->abort();
@@ -897,17 +1083,49 @@ std::shared_ptr<const Pixel_Frame> Datoviz_Visual_Backend::render(
     }
     if (!result.ok) {
         target_->abort();
-        throw std::runtime_error(
+        std::string message =
             "failed to execute Datoviz point frame: validation code " +
             std::to_string(static_cast<unsigned>(result.code)) +
-            ", command " + std::to_string(result.command_index));
+            ", command " + std::to_string(result.command_index);
+        if (!trace.artifact_json.empty()) {
+            message += ", artifact " +
+                trace.artifact_json.substr(
+                    0, std::min<std::size_t>(trace.artifact_json.size(), 2048));
+        }
+        throw std::runtime_error(std::move(message));
     }
 
+    phase_started = trace_now_ns();
+    target_->submit();
+    trace.submit_ns = trace_now_ns() - phase_started;
+    return Pending_Frame{target_->device(), target_->fence(), scene.viewport,
+                         frame_sequence, target_->generation(),
+                         std::move(trace)};
+}
+
+Datoviz_Visual_Backend::Completed_Frame Datoviz_Visual_Backend::collect(
+    Pending_Frame pending) {
+    require_domain();
+    if (target_ == nullptr ||
+        target_->generation() != pending.target_generation)
+        throw std::logic_error("Datoviz pending frame target no longer exists");
+    const std::uint64_t readback_started = trace_now_ns();
+    auto collection = target_->collect();
+    pending.trace.readback_ns = trace_now_ns() - readback_started;
+    pending.trace.gpu = std::move(collection.gpu_timing);
     auto output = std::make_shared<Pixel_Frame>();
-    output->extent = scene.viewport;
-    output->sequence = frame_sequence;
-    output->rgba8 = target_->finish();
-    return output;
+    output->extent = pending.extent;
+    output->sequence = pending.sequence;
+    output->rgba8 = std::move(collection.pixels);
+    return {std::move(output), std::move(pending.trace)};
+}
+
+void Datoviz_Visual_Backend::discard(Pending_Frame pending) {
+    require_domain();
+    if (target_ == nullptr ||
+        target_->generation() != pending.target_generation)
+        throw std::logic_error("Datoviz pending frame target no longer exists");
+    target_->discard_after_completion();
 }
 
 void Datoviz_Visual_Backend::destroy() noexcept {

render_3D/render_3D/detail/Datoviz_Visual_Backend.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "Datoviz_Frame_Trace.h"
 #include "Point_Core.h"
 
 #include <datoviz/drp2/runtime.h>
@@ -12,6 +13,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <optional>
 #include <thread>
 #include <vector>
 
@@ -19,6 +21,20 @@ namespace renderive::render_3d::detail {
 
 class Datoviz_Visual_Backend final {
 public:
+    struct Pending_Frame {
+        VkDevice device{VK_NULL_HANDLE};
+        VkFence fence{VK_NULL_HANDLE};
+        Extent extent{};
+        std::uint64_t sequence{};
+        std::uint64_t target_generation{};
+        Datoviz_Frame_Trace trace;
+    };
+
+    struct Completed_Frame {
+        std::shared_ptr<const Pixel_Frame> frame;
+        Datoviz_Frame_Trace trace;
+    };
+
     Datoviz_Visual_Backend(std::uint32_t gpu_index, bool validation_enabled,
                           const Scene_State& initial_scene);
     ~Datoviz_Visual_Backend();
@@ -26,9 +42,12 @@ public:
     Datoviz_Visual_Backend(const Datoviz_Visual_Backend&) = delete;
     Datoviz_Visual_Backend& operator=(const Datoviz_Visual_Backend&) = delete;
 
-    [[nodiscard]] std::shared_ptr<const Pixel_Frame> render(
+    [[nodiscard]] std::optional<Pending_Frame> submit(
         const Scene_State& scene, std::uint64_t scene_revision,
-        const Prepared_Point& point, std::uint64_t frame_sequence);
+        const Prepared_Point& point, std::uint64_t frame_sequence,
+        bool capture_trace);
+    [[nodiscard]] Completed_Frame collect(Pending_Frame pending);
+    void discard(Pending_Frame pending);
 
     void dispatch_pointer(::renderive::Event_Type type, float x, float y,
                           ::renderive::Mouse_Button button,

render_3D/render_3D/detail/Gpu_Completion_Service.cpp

@@ -0,0 +1,84 @@
+#include "Gpu_Completion_Service.h"
+
+#include <chrono>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+namespace renderive::render_3d::detail {
+
+Gpu_Completion_Service::Gpu_Completion_Service()
+    : thread_([this] { run(); }) {}
+
+Gpu_Completion_Service::~Gpu_Completion_Service() {
+    shutdown();
+}
+
+void Gpu_Completion_Service::watch(
+    VkDevice device, VkFence fence, Completion completion) {
+    if (device == VK_NULL_HANDLE || fence == VK_NULL_HANDLE)
+        throw std::invalid_argument("GPU completion fence is invalid");
+    if (!completion)
+        throw std::invalid_argument("GPU completion callback is empty");
+    {
+        std::lock_guard lock(mutex_);
+        if (stopping_)
+            throw std::runtime_error("GPU completion service is stopping");
+        pending_.push({device, fence, std::move(completion)});
+    }
+    ready_.notify_one();
+}
+
+void Gpu_Completion_Service::shutdown() noexcept {
+    {
+        std::lock_guard lock(mutex_);
+        if (stopping_ && !thread_.joinable())
+            return;
+        stopping_ = true;
+    }
+    ready_.notify_one();
+    if (thread_.joinable())
+        thread_.join();
+}
+
+void Gpu_Completion_Service::run() noexcept {
+    for (;;) {
+        Pending_Fence pending;
+        {
+            std::unique_lock lock(mutex_);
+            ready_.wait(lock, [this] {
+                return stopping_ || !pending_.empty();
+            });
+            if (stopping_ && pending_.empty())
+                return;
+            pending = std::move(pending_.front());
+            pending_.pop();
+        }
+
+        const auto wait_started = std::chrono::steady_clock::now();
+        const VkResult result = vkWaitForFences(
+            pending.device, 1, &pending.fence, VK_TRUE, UINT64_MAX);
+        const auto wait_duration =
+            std::chrono::duration_cast<std::chrono::nanoseconds>(
+                std::chrono::steady_clock::now() - wait_started).count();
+        Result completion_result;
+        completion_result.wait_duration_ns = wait_duration > 0
+            ? static_cast<std::uint64_t>(wait_duration)
+            : 0;
+        if (result != VK_SUCCESS) {
+            try {
+                throw std::runtime_error(
+                    "GPU fence wait failed with Vulkan result " +
+                    std::to_string(static_cast<int>(result)));
+            } catch (...) {
+                completion_result.error = std::current_exception();
+            }
+        }
+        try {
+            pending.completion(std::move(completion_result));
+        } catch (...) {
+        }
+    }
+}
+
+} // namespace renderive::render_3d::detail

render_3D/render_3D/detail/Gpu_Completion_Service.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <volk.h>
+
+#include <condition_variable>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <mutex>
+#include <queue>
+#include <thread>
+
+namespace renderive::render_3d::detail {
+
+class Gpu_Completion_Service final {
+public:
+    struct Result {
+        std::exception_ptr error;
+        std::uint64_t wait_duration_ns{};
+    };
+
+    using Completion = std::function<void(Result)>;
+
+    Gpu_Completion_Service();
+    ~Gpu_Completion_Service();
+
+    Gpu_Completion_Service(const Gpu_Completion_Service&) = delete;
+    Gpu_Completion_Service& operator=(const Gpu_Completion_Service&) = delete;
+
+    void watch(VkDevice device, VkFence fence, Completion completion);
+    void shutdown() noexcept;
+
+private:
+    struct Pending_Fence {
+        VkDevice device{VK_NULL_HANDLE};
+        VkFence fence{VK_NULL_HANDLE};
+        Completion completion;
+    };
+
+    void run() noexcept;
+
+    std::mutex mutex_;
+    std::condition_variable ready_;
+    std::queue<Pending_Fence> pending_;
+    std::thread thread_;
+    bool stopping_{};
+};
+
+} // namespace renderive::render_3d::detail

render_3D/render_3D/detail/Render_Domain.h

@@ -29,19 +29,25 @@ public:
     Render_Domain(const Render_Domain&) = delete;
     Render_Domain& operator=(const Render_Domain&) = delete;
 
+    void post(std::function<void()> function) {
+        if (!function)
+            throw std::invalid_argument("render domain task is empty");
+        {
+            std::lock_guard lock(mutex_);
+            if (stopping_)
+                throw std::runtime_error("Point_Scene render domain is stopping");
+            tasks_.push(std::move(function));
+        }
+        condition_.notify_one();
+    }
+
     template <class Function>
     auto invoke(Function&& function) -> std::invoke_result_t<Function> {
         using Result = std::invoke_result_t<Function>;
         auto task = std::make_shared<std::packaged_task<Result()>>(
             std::forward<Function>(function));
         auto result = task->get_future();
-        {
-            std::lock_guard lock(mutex_);
-            if (stopping_)
-                throw std::runtime_error("Point_Scene render domain is stopping");
-            tasks_.emplace([task] { (*task)(); });
-        }
-        condition_.notify_one();
+        post([task] { (*task)(); });
         if constexpr (std::is_void_v<Result>)
             result.get();
         else

render_3D/render_3D/visuals/Basic_Visual.h

@@ -2,8 +2,11 @@
 
 #include "Visual_Types.h"
 #include "../renderable/Inheritance.h"
+#include "../renderable/Renderable_p.h"
 
+#include <renderive/real_time_data/Double_Buffer_Frame_Observer.hpp>
 #include <renderive/real_time_data/Double_Buffer_Strategy.hpp>
+#include <renderive/scene/base/Abstract_Frame.hpp>
 
 #include <functional>
 #include <memory>
@@ -31,7 +34,7 @@ struct Basic_Visual : Renderable<Basic_Visual<Spec>> {
     using Payload = std::shared_ptr<const std::vector<Item>>;
     using Buffer = ::Multi_Double_Buffer_Strategy<
         ::Double_Buffer_Layout<::Buffered_Data<Visual_Data_Tag<Spec>, Payload>>,
-        std::mutex>;
+        std::mutex, Observer_State<::Double_Buffer_Frame_Observer>>;
 
     struct State : Base::template next_State<State>, Spec::Settings {
         Matrix4 transform;
@@ -68,10 +71,19 @@ struct Basic_Visual : Renderable<Basic_Visual<Spec>> {
     template <class Product, class Properties>
     using Business_Builder = Renderable_Builder<Product, Properties, State_Validator>;
 
+    struct Frame_Data {
+        State state;
+        Payload items;
+        std::uint64_t state_revision{};
+        std::uint64_t data_revision{};
+    };
+
     struct Impl : Base::template next_Impl<Impl>, Buffer {
         struct Observer : Base::template next_Impl<Impl>::next_Observer {
             static void handle(Impl& impl,
                                const Renderable_Event_View& observation) noexcept {
+                if (observation.event == Renderable_Observer_Event::Published)
+                    impl.Buffer::publish();
                 if (const auto* data = observation.template payload_if<
                         Visual_Data_Observation<Spec>>())
                     impl.observe_data(data->revision, data->item_count);
@@ -80,6 +92,8 @@ struct Basic_Visual : Renderable<Basic_Visual<Spec>> {
 
         explicit Impl(std::vector<Item> items) { update(std::move(items)); }
 
+        std::shared_ptr<void> real_time_data_binding;
+
         void update(std::vector<Item> items) {
             for (const auto& item : items) {
                 if (!Spec::valid(item))
@@ -92,11 +106,44 @@ struct Basic_Visual : Renderable<Basic_Visual<Spec>> {
             this->report_observation(Visual_Data_Observation<Spec>{
                 this->template cache_revision<Visual_Data_Tag<Spec>>(), count});
         }
+
+        [[nodiscard]] const Payload& published_items() const noexcept {
+            return this->template render_buffer_value<Visual_Data_Tag<Spec>>();
+        }
+
+        [[nodiscard]] std::uint64_t dependent_state_revision() const noexcept {
+            return this->Buffer::revision();
+        }
+
+        void capture_frame_data(Frame_Render_Snapshot& snapshot) const override {
+            const auto& control = static_cast<const Basic_Visual&>(this->owner());
+            snapshot.capture<Frame_Data>(
+                this->renderable_id,
+                std::make_shared<const Frame_Data>(Frame_Data{
+                    control.template render_state<State>(), published_items(),
+                    control.template state_revision<State>(),
+                    this->template buffer_revision<Visual_Data_Tag<Spec>>() }));
+        }
+
+        void prepare(const Prepare_Render_Context& context) override {
+            const auto input = context.template captured<Frame_Data>();
+            if (!input)
+                throw std::logic_error(
+                    "Datoviz visual frame input was not captured");
+            if (context.metrics) {
+                const auto count = input->items ? input->items->size() : 0U;
+                context.metrics->set(Node_Metric_Kind::input_count, count);
+                context.metrics->set(Node_Metric_Kind::primitive_count, count);
+            }
+            context.template publish<Frame_Data>(input);
+        }
     };
 
     explicit Basic_Visual(const State& state, std::vector<Item> items = {})
         : Base(With_Attached_Impl<Impl>{}, state, std::move(items)) {
         State_Validator{}(state);
+        this->template d_func<Impl>().real_time_data_binding =
+            this->template d_func<Impl>().Buffer::bind(*this);
     }
     ~Basic_Visual() override = default;
 
@@ -108,17 +155,17 @@ struct Basic_Visual : Renderable<Basic_Visual<Spec>> {
     void edit_items(const std::function<void(std::vector<Item>&)>& edit) {
         if (!edit)
             throw std::invalid_argument("visual item edit is empty");
-        const auto& current = this->template d_func<Impl>()
-                                  .template cache_buffer_value<Visual_Data_Tag<Spec>>();
-        auto next = current ? *current : std::vector<Item>{};
+        const auto& cached = this->template d_func<Impl>()
+                                 .template cache_buffer_value<Visual_Data_Tag<Spec>>();
+        auto next = cached ? *cached : std::vector<Item>{};
         edit(next);
         update_items(std::move(next));
     }
 
     [[nodiscard]] std::size_t item_count() const noexcept {
-        const auto& data = this->template d_func<Impl>()
-                               .template cache_buffer_value<Visual_Data_Tag<Spec>>();
-        return data ? data->size() : 0U;
+        const auto& cached = this->template d_func<Impl>()
+                                 .template cache_buffer_value<Visual_Data_Tag<Spec>>();
+        return cached ? cached->size() : 0U;
     }
 
     [[nodiscard]] std::uint64_t data_revision() const noexcept {

render_3D/tests/Datoviz_Visual_Family_Tests.cpp

@@ -1,25 +1,98 @@
 #include "render_3D/Datoviz_Visuals.h"
 
+#include <renderive/scene/Scene.hpp>
+
 #include <gtest/gtest.h>
 
+#include <memory>
+#include <stdexcept>
 #include <type_traits>
 
 namespace renderive::render_3d {
 namespace {
 
+template <class Visual>
+struct Visual_Frame_Test_Scene final : Scene3D_Context<>, Scene_Renderer {
+    using Frame_Data = typename Visual::Frame_Data;
+
+    explicit Visual_Frame_Test_Scene(
+        const std::shared_ptr<Visual>& visual)
+        : owner(visual->renderable_id()) {
+        auto builder = attach_builder();
+        builder.attach(renderive_Owner<Visual>(visual));
+    }
+
+    ~Visual_Frame_Test_Scene() override { shutdown(); }
+
+    void render_frame() {
+        publish_frame_state();
+        render();
+        wait_for_render();
+    }
+
+    Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) override {
+        prepared = context.template prepared<Frame_Data>(owner);
+        if (!prepared)
+            throw std::logic_error(
+                "Datoviz visual did not publish frame-local data");
+        return Node_Execution_Result::completed();
+    }
+
+    Renderable_Id owner{};
+    std::shared_ptr<const Frame_Data> prepared;
+};
+
 template <class Visual, class Item>
 void exercise(Item item) {
     static_assert(std::derived_from<Visual, Renderable>);
     auto visual = typename Visual::Builder{}.build(std::vector<Item>{item});
     ASSERT_NE(visual, nullptr);
     EXPECT_EQ(visual->item_count(), 1U);
-    const auto revision = visual->data_revision();
+    EXPECT_EQ(visual->observation().event,
+              Renderable_Observer_Event::Data_Updated);
+    EXPECT_EQ(visual->observation().data_revision,
+              visual->data_revision());
+
+    const auto initial_revision = visual->data_revision();
     visual->edit_items([](auto& items) { items.push_back(items.front()); });
     EXPECT_EQ(visual->item_count(), 2U);
+    EXPECT_GT(visual->data_revision(), initial_revision);
+    EXPECT_EQ(visual->observation().event,
+              Renderable_Observer_Event::Data_Updated);
+    EXPECT_EQ(visual->observation().item_count, 2U);
+    EXPECT_EQ(visual->observation().data_revision,
+              visual->data_revision());
+
+    Visual_Frame_Test_Scene<Visual> scene(visual);
+    scene.render_frame();
+    ASSERT_TRUE(scene.prepared);
+    ASSERT_TRUE(scene.prepared->items);
+    EXPECT_EQ(scene.prepared->items->size(), 2U);
+    EXPECT_EQ(visual->observation().event,
+              Renderable_Observer_Event::Published);
+
+    const auto first_frame = scene.prepared;
+    const auto first_published_revision = first_frame->data_revision;
+    const auto revision = visual->data_revision();
+    visual->update_items(std::vector<Item>{item, item, item});
+    EXPECT_EQ(visual->item_count(), 3U);
     EXPECT_GT(visual->data_revision(), revision);
-    const auto observation = visual->observation();
-    EXPECT_EQ(observation.item_count, 2U);
-    EXPECT_EQ(observation.data_revision, visual->data_revision());
+    EXPECT_EQ(visual->observation().event,
+              Renderable_Observer_Event::Data_Updated);
+    EXPECT_EQ(visual->observation().item_count, 3U);
+    EXPECT_EQ(visual->observation().data_revision,
+              visual->data_revision());
+    ASSERT_TRUE(first_frame->items);
+    EXPECT_EQ(first_frame->items->size(), 2U);
+
+    scene.render_frame();
+    ASSERT_TRUE(scene.prepared);
+    ASSERT_TRUE(scene.prepared->items);
+    EXPECT_EQ(scene.prepared->items->size(), 3U);
+    EXPECT_GT(scene.prepared->data_revision, first_published_revision);
+    EXPECT_EQ(visual->observation().event,
+              Renderable_Observer_Event::Published);
 }
 
 TEST(DatovizVisualFamilies, EveryPublicFamilyUsesRenderableStorage) {

render_3D/tests/Point_Render_Integration_Tests.cpp

@@ -1,5 +1,7 @@
 #include "render_3D/Point_Demo.h"
 
+#include <renderive/scene/base/Scene_Base.hpp>
+
 #include <gtest/gtest.h>
 
 #include <algorithm>
@@ -101,5 +103,48 @@ TEST(PointRenderIntegration, KernelEventsReachDatovizArcballOnItsDomain) {
     }
 }
 
+TEST(PointRenderIntegration,
+     CapturesExternalGpuLifetimeAndSeparatedBackendPhases) {
+    try {
+        auto demo = make_point_demo(Scene_Options{.viewport = {320, 200}});
+        auto& kernel_scene = demo.scene->render_scene();
+        const Capture_Session_Id capture = kernel_scene.capture_next_frame();
+        ASSERT_TRUE(demo.scene->request_frame());
+
+        const auto session = kernel_scene.capture_session(capture);
+        ASSERT_TRUE(session);
+        ASSERT_EQ(session->captured_count(), 1U);
+        const auto& snapshot = *session->frames.front().snapshot;
+        const auto plan = kernel_scene.find_render_plan(
+            snapshot.render_plan_version);
+        ASSERT_TRUE(plan);
+        const auto render_node = std::find_if(
+            plan->graph.nodes.begin(), plan->graph.nodes.end(),
+            [](const Render_Node& node) {
+                return node.kind == Render_Node_Kind::render;
+            });
+        ASSERT_NE(render_node, plan->graph.nodes.end());
+        const auto& execution = snapshot.node_executions.at(
+            render_node->execution_index);
+        EXPECT_EQ(execution.status, Node_Execution_Status::complete);
+        EXPECT_GT(execution.external_start_time_ns, 0U);
+        EXPECT_GE(execution.external_end_time_ns,
+                  execution.external_start_time_ns);
+        EXPECT_GE(execution.duration_ns(), execution.cpu_duration_ns());
+        EXPECT_TRUE(execution.metrics.contains(
+            Node_Metric_Kind::queue_wait_ns));
+        EXPECT_TRUE(execution.metrics.contains(
+            Node_Metric_Kind::backend_execute_duration_ns));
+        EXPECT_TRUE(execution.metrics.contains(
+            Node_Metric_Kind::gpu_fence_wait_ns));
+        EXPECT_TRUE(execution.metrics.contains(
+            Node_Metric_Kind::readback_duration_ns));
+        EXPECT_GT(execution.metrics.get(
+            Node_Metric_Kind::backend_artifact_json_bytes), 0U);
+    } catch (const std::exception& error) {
+        GTEST_SKIP() << "Vulkan/Datoviz unavailable: " << error.what();
+    }
+}
+
 } // namespace
 } // namespace renderive::render_3d

render_3D/tests/Point_State_Tests.cpp

@@ -10,6 +10,7 @@
 #include <array>
 #include <atomic>
 #include <memory>
+#include <stdexcept>
 #include <thread>
 #include <vector>
 
@@ -43,11 +44,14 @@ struct Test_Scene final
         wait_for_render();
     }
 
-    void render_scene(const Scene_Render_Context& context) override {
+    Node_Execution_Result render_scene(
+        const Scene_Render_Context& context) override {
         prepared = context.prepared<detail::Prepared_Point>(point_id);
-        ASSERT_TRUE(prepared);
+        if (!prepared)
+            throw std::logic_error("Point_Visual did not publish prepared data");
         EXPECT_EQ(context.frame.scene_state<detail::Scene_State>().viewport,
                   (Extent{320, 180}));
+        return Node_Execution_Result::completed();
     }
 
     Renderable_Id point_id{};

web_server/app/Gallery_Capture_Json.h

@@ -38,8 +38,12 @@ inline const char* execution_status(Node_Execution_Status status) noexcept {
     switch (status) {
     case Node_Execution_Status::pending:
         return "pending";
+    case Node_Execution_Status::ready:
+        return "ready";
     case Node_Execution_Status::running:
         return "running";
+    case Node_Execution_Status::waiting_external:
+        return "waiting_external";
     case Node_Execution_Status::complete:
         return "complete";
     case Node_Execution_Status::failed:
@@ -50,8 +54,18 @@ inline const char* execution_status(Node_Execution_Status status) noexcept {
 
 inline Json metrics_json(const Node_Execution_Metrics& metrics) {
     Json result = Json::object();
-    constexpr std::array names{"input_count", "chunk_size", "prepared_cells",
-                               "pixel_count", "primitive_count"};
+    constexpr std::array names{
+        "input_count", "chunk_size", "prepared_cells", "pixel_count",
+        "primitive_count", "queue_wait_ns", "apply_duration_ns",
+        "plan_emit_duration_ns", "backend_execute_duration_ns",
+        "submit_duration_ns", "gpu_fence_wait_ns",
+        "gpu_render_duration_ns", "gpu_copy_duration_ns",
+        "readback_duration_ns", "backend_resource_version",
+        "backend_frame_index", "backend_artifact_status",
+        "backend_validation_code", "backend_validation_command",
+        "backend_artifact_json_bytes"};
+    static_assert(names.size() ==
+                  static_cast<std::size_t>(Node_Metric_Kind::count));
     for (std::size_t index = 0; index < names.size(); ++index) {
         const auto kind = static_cast<Node_Metric_Kind>(index);
         if (metrics.contains(kind))
@@ -135,13 +149,27 @@ inline Json execution_json(const Node_Execution& execution,
                            const Frame_Snapshot& frame) {
     return {
         {"node_id", execution.node_id},
+        {"ready_time_ns", execution.ready_time_ns},
         {"start_time_ns", execution.start_time_ns},
+        {"cpu_end_time_ns", execution.cpu_end_time_ns},
+        {"external_start_time_ns", execution.external_start_time_ns},
+        {"external_end_time_ns", execution.external_end_time_ns},
         {"end_time_ns", execution.end_time_ns},
         {"start_offset_ns", execution.start_time_ns >= frame.render_start_ns
             ? execution.start_time_ns - frame.render_start_ns : 0},
+        {"cpu_end_offset_ns", execution.cpu_end_time_ns >= frame.render_start_ns
+            ? execution.cpu_end_time_ns - frame.render_start_ns : 0},
+        {"external_start_offset_ns",
+         execution.external_start_time_ns >= frame.render_start_ns
+             ? execution.external_start_time_ns - frame.render_start_ns : 0},
+        {"external_end_offset_ns",
+         execution.external_end_time_ns >= frame.render_start_ns
+             ? execution.external_end_time_ns - frame.render_start_ns : 0},
         {"end_offset_ns", execution.end_time_ns >= frame.render_start_ns
             ? execution.end_time_ns - frame.render_start_ns : 0},
         {"duration_ns", execution.duration_ns()},
+        {"cpu_duration_ns", execution.cpu_duration_ns()},
+        {"external_duration_ns", execution.external_duration_ns()},
         {"worker_id", execution.worker_id},
         {"status", execution_status(execution.status)},
         {"metrics", metrics_json(execution.metrics)}
@@ -152,6 +180,8 @@ inline Json node_analysis_json(const Node_Frame_Analysis& node) {
     return {
         {"node_id", node.node_id},
         {"duration_ns", node.duration_ns},
+        {"cpu_duration_ns", node.cpu_duration_ns},
+        {"external_duration_ns", node.external_duration_ns},
         {"start_offset_ns", node.start_offset_ns},
         {"end_offset_ns", node.end_offset_ns},
         {"dependency_ready_time_ns", node.dependency_ready_time_ns},
@@ -186,6 +216,8 @@ inline Json frame_json(const Captured_Frame& frame) {
             {"total_render_duration_ns", frame.analysis.total_render_duration_ns},
             {"critical_path_duration_ns", frame.analysis.critical_path_duration_ns},
             {"total_work_duration_ns", frame.analysis.total_work_duration_ns},
+            {"total_external_duration_ns",
+             frame.analysis.total_external_duration_ns},
             {"parallel_overlap_ns", frame.analysis.parallel_overlap_ns},
             {"peak_parallelism", frame.analysis.peak_parallelism},
             {"average_parallelism", frame.analysis.average_parallelism},

webapp_gallery/src/capture/frame_summary.tsx

@@ -1,2 +1,16 @@
-import {Metric_Grid} from "../common/metric_grid";import type {Gallery_Captured_Frame} from "../protocol/gallery_types";import {format_nanoseconds} from "../protocol/format";
-export function Frame_Summary({frame}:{frame:Gallery_Captured_Frame}) {const workers=new Set(frame.node_executions.map(item=>item.worker_id));return <Metric_Grid values={[["Frame",`#${frame.frame_id}`],["Plan",`v${frame.render_plan_version}`],["Render",format_nanoseconds(frame.render_duration_ns)],["Nodes",frame.node_executions.length],["Workers",workers.size]]}/>;}
+import {Metric_Grid} from "../common/metric_grid";
+import type {Gallery_Captured_Frame} from "../protocol/gallery_types";
+import {format_nanoseconds} from "../protocol/format";
+
+export function Frame_Summary({frame}:{frame:Gallery_Captured_Frame}) {
+    const workers=new Set(frame.node_executions.filter(item=>item.cpu_duration_ns>0).map(item=>item.worker_id));
+    return <Metric_Grid values={[
+        ["Frame",`#${frame.frame_id}`],
+        ["Plan",`v${frame.render_plan_version}`],
+        ["Wall",format_nanoseconds(frame.render_duration_ns)],
+        ["CPU work",format_nanoseconds(frame.analysis?.total_work_duration_ns)],
+        ["External",format_nanoseconds(frame.analysis?.total_external_duration_ns)],
+        ["Nodes",frame.node_executions.length],
+        ["Workers",workers.size]
+    ]}/>;
+}

webapp_gallery/src/capture/node_detail.tsx

@@ -1,2 +1,27 @@
-import {Stack,Typography} from "@mui/material";import type {Gallery_Capture_Session,Gallery_Captured_Frame,Gallery_Render_Plan} from "../protocol/gallery_types";import {Metric_Grid} from "../common/metric_grid";import {Json_Viewer} from "../common/json_viewer";import {format_nanoseconds} from "../protocol/format";
-export function Node_Detail({frame,plan,session,selected_node_id}:{frame:Gallery_Captured_Frame;plan:Gallery_Render_Plan;session:Gallery_Capture_Session;selected_node_id:number|null}) {const node=plan.nodes.find(item=>item.node_id===selected_node_id),execution=frame.node_executions.find(item=>item.node_id===selected_node_id),analysis=frame.analysis?.nodes.find(item=>item.node_id===selected_node_id),history=session.node_statistics.find(item=>item.node_id===selected_node_id);if(!node||!execution)return <Typography color="text.secondary">选择 DAG 节点或时间线区间查看详情。</Typography>;return <Stack spacing={2}><Metric_Grid values={[["Node",`${node.owner} / ${node.name}`],["Kind",node.kind],["Current",format_nanoseconds(execution.duration_ns)],["Scheduler wait",format_nanoseconds(analysis?.scheduler_wait_ns)],["Worker",execution.worker_id],["Critical",`${((analysis?.critical_path_contribution??0)*100).toFixed(2)}%`],["Moving avg",format_nanoseconds(history?.moving_average_ns)],["P95",format_nanoseconds(history?.p95_ns)],["P99",format_nanoseconds(history?.p99_ns)]]}/>{execution.metrics&&<Json_Viewer value={execution.metrics}/>}</Stack>;}
+import {Stack,Typography} from "@mui/material";
+import type {Gallery_Capture_Session,Gallery_Captured_Frame,Gallery_Render_Plan} from "../protocol/gallery_types";
+import {Metric_Grid} from "../common/metric_grid";
+import {Json_Viewer} from "../common/json_viewer";
+import {format_nanoseconds} from "../protocol/format";
+
+export function Node_Detail({frame,plan,session,selected_node_id}:{frame:Gallery_Captured_Frame;plan:Gallery_Render_Plan;session:Gallery_Capture_Session;selected_node_id:number|null}) {
+    const node=plan.nodes.find(item=>item.node_id===selected_node_id);
+    const execution=frame.node_executions.find(item=>item.node_id===selected_node_id);
+    const analysis=frame.analysis?.nodes.find(item=>item.node_id===selected_node_id);
+    const history=session.node_statistics.find(item=>item.node_id===selected_node_id);
+    if(!node||!execution)return <Typography color="text.secondary">Select a DAG node or timeline interval to inspect it.</Typography>;
+    return <Stack spacing={2}><Metric_Grid values={[
+        ["Node",`${node.owner} / ${node.name}`],
+        ["Kind",node.kind],
+        ["Status",execution.status],
+        ["Wall",format_nanoseconds(execution.duration_ns)],
+        ["CPU",format_nanoseconds(execution.cpu_duration_ns)],
+        ["External",format_nanoseconds(execution.external_duration_ns)],
+        ["Scheduler wait",format_nanoseconds(analysis?.scheduler_wait_ns)],
+        ["Worker",execution.worker_id],
+        ["Critical",`${((analysis?.critical_path_contribution??0)*100).toFixed(2)}%`],
+        ["Moving avg",format_nanoseconds(history?.moving_average_ns)],
+        ["P95",format_nanoseconds(history?.p95_ns)],
+        ["P99",format_nanoseconds(history?.p99_ns)]
+    ]}/>{execution.metrics&&<Json_Viewer value={execution.metrics}/>}</Stack>;
+}

webapp_gallery/src/capture/worker_timeline.tsx

@@ -1,2 +1,17 @@
-import {Box,Tooltip,Typography} from "@mui/material";import type {Gallery_Captured_Frame,Gallery_Render_Plan} from "../protocol/gallery_types";import {format_nanoseconds} from "../protocol/format";
-export function Worker_Timeline({frame,plan,selected_node_id,on_select_node}:{frame:Gallery_Captured_Frame;plan:Gallery_Render_Plan;selected_node_id:number|null;on_select_node:(id:number)=>void}) {const duration=Math.max(1,frame.render_duration_ns);const workers=[...new Set(frame.node_executions.map(item=>item.worker_id))].toSorted((a,b)=>a-b);const names=new Map(plan.nodes.map(node=>[node.node_id,node.name]));return <Box sx={{display:"grid",gridTemplateColumns:"76px 1fr",gap:1,alignItems:"center"}}>{workers.map(worker=><Box key={worker} sx={{display:"contents"}}><Typography variant="caption">Worker {worker}</Typography><Box sx={{position:"relative",height:34,bgcolor:"rgba(255,255,255,.04)",borderRadius:1}}>{frame.node_executions.filter(item=>item.worker_id===worker).map(item=><Tooltip key={`${item.node_id}:${item.start_offset_ns}`} title={`${names.get(item.node_id)??item.node_id} · ${format_nanoseconds(item.duration_ns)}`}><Box onClick={()=>on_select_node(item.node_id)} sx={{position:"absolute",left:`${item.start_offset_ns/duration*100}%`,width:`${Math.max(.4,item.duration_ns/duration*100)}%`,top:4,bottom:4,borderRadius:.5,cursor:"pointer",bgcolor:item.node_id===selected_node_id?"secondary.main":"primary.main"}}/></Tooltip>)}</Box></Box>)}</Box>;}
+import {Box,Tooltip,Typography} from "@mui/material";
+import type {Gallery_Captured_Frame,Gallery_Render_Plan} from "../protocol/gallery_types";
+import {format_nanoseconds} from "../protocol/format";
+
+export function Worker_Timeline({frame,plan,selected_node_id,on_select_node}:{frame:Gallery_Captured_Frame;plan:Gallery_Render_Plan;selected_node_id:number|null;on_select_node:(id:number)=>void}) {
+    const duration=Math.max(1,frame.render_duration_ns);
+    const cpu_nodes=frame.node_executions.filter(item=>item.cpu_duration_ns>0);
+    const external_nodes=frame.node_executions.filter(item=>item.external_duration_ns>0);
+    const workers=[...new Set(cpu_nodes.map(item=>item.worker_id))].toSorted((a,b)=>a-b);
+    const names=new Map(plan.nodes.map(node=>[node.node_id,node.name]));
+    const lane=(nodes:Gallery_Captured_Frame["node_executions"],external=false)=><Box sx={{position:"relative",height:34,bgcolor:"rgba(255,255,255,.04)",borderRadius:1}}>{nodes.map(item=>{
+        const start=external?item.external_start_offset_ns:item.start_offset_ns;
+        const span=external?item.external_duration_ns:item.cpu_duration_ns;
+        return <Tooltip key={`${external?"external":"cpu"}:${item.node_id}:${start}`} title={`${names.get(item.node_id)??item.node_id} · ${external?"External":"CPU"} ${format_nanoseconds(span)}`}><Box onClick={()=>on_select_node(item.node_id)} sx={{position:"absolute",left:`${start/duration*100}%`,width:`${Math.max(.4,span/duration*100)}%`,top:4,bottom:4,borderRadius:.5,cursor:"pointer",bgcolor:item.node_id===selected_node_id?"secondary.main":external?"warning.main":"primary.main",opacity:external ? .65 : 1}}/></Tooltip>;
+    })}</Box>;
+    return <Box sx={{display:"grid",gridTemplateColumns:"76px 1fr",gap:1,alignItems:"center"}}>{workers.map(worker=><Box key={worker} sx={{display:"contents"}}><Typography variant="caption">Worker {worker}</Typography>{lane(cpu_nodes.filter(item=>item.worker_id===worker))}</Box>)}{external_nodes.length>0&&<Box sx={{display:"contents"}}><Typography variant="caption">External</Typography>{lane(external_nodes,true)}</Box>}</Box>;
+}

webapp_gallery/src/protocol/gallery_types.ts

@@ -31,9 +31,9 @@ export interface Gallery_Render_Node {node_id: number; id?: string | number; nam
 export interface Gallery_Render_Edge {from: number | string; to: number | string;}
 export interface Gallery_Renderable_Cache {owner_id: number; name: string; prepare_cache: string; paint_cache: string;}
 export interface Gallery_Render_Plan {version: number; nodes: Gallery_Render_Node[]; edges: Gallery_Render_Edge[]; renderables: Gallery_Renderable_Cache[];}
-export interface Gallery_Node_Execution {node_id: number; worker_id: number; start_offset_ns: number; end_offset_ns: number; duration_ns: number; metrics?: Record<string, Json_Primitive>;}
-export interface Gallery_Node_Analysis {node_id: number; scheduler_wait_ns: number; critical_path_contribution: number; on_critical_path: boolean;}
-export interface Gallery_Captured_Frame {frame_id: number; render_plan_version: number; render_duration_ns: number; node_executions: Gallery_Node_Execution[]; analysis?: {nodes: Gallery_Node_Analysis[]};}
+export interface Gallery_Node_Execution {node_id: number; worker_id: number; start_offset_ns: number; cpu_end_offset_ns: number; external_start_offset_ns: number; external_end_offset_ns: number; end_offset_ns: number; duration_ns: number; cpu_duration_ns: number; external_duration_ns: number; status: "pending"|"ready"|"running"|"waiting_external"|"complete"|"failed"; metrics?: Record<string, Json_Primitive>;}
+export interface Gallery_Node_Analysis {node_id: number; duration_ns: number; cpu_duration_ns: number; external_duration_ns: number; scheduler_wait_ns: number; critical_path_contribution: number; on_critical_path: boolean;}
+export interface Gallery_Captured_Frame {frame_id: number; render_plan_version: number; render_duration_ns: number; node_executions: Gallery_Node_Execution[]; analysis?: {total_work_duration_ns?: number; total_external_duration_ns?: number; nodes: Gallery_Node_Analysis[]};}
 export interface Gallery_Node_Statistics {node_id: number; moving_average_ns: number; p95_ns: number; p99_ns: number; critical_path_frequency: number;}
 export interface Gallery_Plan_Statistics {render_plan_version: number; frame_count: number; render_average_ns: number; render_p95_ns: number; average_parallelism: number; peak_parallelism: number; scheduler_wait_average_ns: number; nodes: Gallery_Node_Statistics[];}
 export interface Gallery_Capture_Session {session_id: number; active: boolean; requested_count: number; captured_count: number; frames: Gallery_Captured_Frame[]; node_statistics: Gallery_Node_Statistics[]; plan_statistics: Gallery_Plan_Statistics[]; summary?: Record<string, Json_Value>;}

webapp_gallery/tests/dag/dag_model.test.ts

@@ -1,2 +1,2 @@
 import {describe,expect,it} from "vitest";import {build_dag_model} from "../../src/dag/dag_model";
-describe("DAG view model",()=>{it("uses node_id for current plan and capture overlays",()=>{const plan={version:2,nodes:[{node_id:7,name:"paint",owner:"plot",kind:"paint"}],edges:[],renderables:[]};const frame={frame_id:1,render_plan_version:2,render_duration_ns:10,node_executions:[{node_id:7,worker_id:3,start_offset_ns:0,end_offset_ns:10,duration_ns:10}],analysis:{nodes:[{node_id:7,scheduler_wait_ns:1,critical_path_contribution:1,on_critical_path:true}]}};const model=build_dag_model(plan,frame,[],7);expect(model.nodes[0].id).toBe("7");expect(model.nodes[0].data).toMatchObject({duration_ns:10,worker_id:3,critical:true,selected:true});});});
+describe("DAG view model",()=>{it("uses node_id for current plan and capture overlays",()=>{const plan={version:2,nodes:[{node_id:7,name:"paint",owner:"plot",kind:"paint"}],edges:[],renderables:[]};const frame={frame_id:1,render_plan_version:2,render_duration_ns:10,node_executions:[{node_id:7,worker_id:3,start_offset_ns:0,cpu_end_offset_ns:10,external_start_offset_ns:0,external_end_offset_ns:0,end_offset_ns:10,duration_ns:10,cpu_duration_ns:10,external_duration_ns:0,status:"complete" as const}],analysis:{total_work_duration_ns:10,total_external_duration_ns:0,nodes:[{node_id:7,duration_ns:10,cpu_duration_ns:10,external_duration_ns:0,scheduler_wait_ns:1,critical_path_contribution:1,on_critical_path:true}]}};const model=build_dag_model(plan,frame,[],7);expect(model.nodes[0].id).toBe("7");expect(model.nodes[0].data).toMatchObject({duration_ns:10,worker_id:3,critical:true,selected:true});});});