优化了 还是卡
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
#include "detail/Gpu_Completion_Service.hpp"
|
||||
|
||||
namespace aethera::render_3d {
|
||||
Gpu_Completion_State gpu_completion_state() noexcept {
|
||||
Gpu_Completion_State gpu_completion_state() {
|
||||
return detail::Gpu_Completion_Service::instance().state();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,12 +27,11 @@ struct Gpu_Completion_State : State_Type<Gpu_Completion_State_Tag> {
|
||||
std::uint64_t callback_max_ns{};
|
||||
std::uint64_t callback_failure_count{};
|
||||
std::uint64_t backpressure_count{};
|
||||
std::uint64_t backpressure_wait_ns{};
|
||||
std::uint64_t fault_count{};
|
||||
std::uint64_t abandoned_count{};
|
||||
bool stopping{};
|
||||
bool operator==(const Gpu_Completion_State&) const = default;
|
||||
};
|
||||
|
||||
[[nodiscard]] Gpu_Completion_State gpu_completion_state() noexcept;
|
||||
[[nodiscard]] Gpu_Completion_State gpu_completion_state();
|
||||
}
|
||||
|
||||
@@ -1,440 +1,439 @@
|
||||
#include "Gpu_Completion_Service.hpp"
|
||||
#include "Gpu_Completion_Service.hpp"
|
||||
#include "Exception.hpp"
|
||||
#include <algorithm>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace aethera::render_3d::detail {
|
||||
namespace {
|
||||
std::uint64_t elapsed_nanoseconds(
|
||||
std::chrono::steady_clock::time_point started) noexcept {
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - started).count();
|
||||
return elapsed > 0 ? static_cast<std::uint64_t>(elapsed) : 0ULL;
|
||||
}
|
||||
|
||||
template <typename Value>
|
||||
void update_peak(Value& peak, Value value) noexcept {
|
||||
peak = std::max(peak, value);
|
||||
}
|
||||
}
|
||||
|
||||
Gpu_Completion_Service& Gpu_Completion_Service::instance() {
|
||||
static Gpu_Completion_Service service;
|
||||
return service;
|
||||
}
|
||||
|
||||
Gpu_Completion_Service::Gpu_Completion_Service() {
|
||||
publish_state(0, 0);
|
||||
thread_ = std::thread([this] {
|
||||
run();
|
||||
});
|
||||
state_.current->capacity = static_cast<std::size_t>(default_capacity);
|
||||
state_.advance();
|
||||
thread_ = std::thread([this] { run(); });
|
||||
}
|
||||
|
||||
Gpu_Completion_Service::~Gpu_Completion_Service() {
|
||||
stopping_.store(true, std::memory_order_release);
|
||||
wake();
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
state_.current->stopping = true;
|
||||
++wake_generation_;
|
||||
state_.advance();
|
||||
}
|
||||
wake_condition_.notify_one();
|
||||
if (thread_.joinable()) thread_.join();
|
||||
}
|
||||
|
||||
Gpu_Completion_Service::Reservation::Reservation(
|
||||
std::shared_ptr<Pending_Fence> pending) noexcept : pending_(std::move(pending)) {}
|
||||
std::shared_ptr<Pending_Fence> pending) noexcept
|
||||
: pending_(std::move(pending)) {}
|
||||
|
||||
Gpu_Completion_Service::Reservation::~Reservation() noexcept {
|
||||
try { cancel(); }
|
||||
try {
|
||||
cancel();
|
||||
}
|
||||
catch (...) {}
|
||||
}
|
||||
Gpu_Completion_Service::Reservation::Reservation(Reservation&& other) noexcept : pending_(std::exchange(other.pending_, {})) {}
|
||||
|
||||
Gpu_Completion_Service::Reservation::Reservation(
|
||||
Reservation&& other) noexcept
|
||||
: pending_(std::exchange(other.pending_, {})) {}
|
||||
|
||||
Gpu_Completion_Service::Prepare_Result::operator bool() const noexcept {
|
||||
return result == Admission_Result::none;
|
||||
}
|
||||
|
||||
void Gpu_Completion_Service::Reservation::watch(VkDevice device,
|
||||
VkFence fence) {
|
||||
if (!pending_ || device == VK_NULL_HANDLE || fence == VK_NULL_HANDLE) throw std::logic_error("GPU completion reservation or fence is invalid");
|
||||
auto pending = std::exchange(pending_, {});
|
||||
auto* const service = pending->service;
|
||||
{
|
||||
std::lock_guard lock(pending->mutex);
|
||||
if (pending->status != Pending_Fence::Status::reserved) throw std::logic_error("GPU completion reservation is not reserved");
|
||||
pending->device = device;
|
||||
pending->fence = fence;
|
||||
// This timestamp is part of correctness, not only observability: it
|
||||
// bounds the lifetime of a submission whose fence never signals.
|
||||
pending->watched_at = std::chrono::steady_clock::now();
|
||||
const std::size_t watched =
|
||||
service->watched_.fetch_add(1, std::memory_order_relaxed) + 1;
|
||||
update_peak(service->peak_watched_, watched);
|
||||
pending->status = Pending_Fence::Status::watched;
|
||||
}
|
||||
service->wake();
|
||||
if (!pending_ || device == VK_NULL_HANDLE || fence == VK_NULL_HANDLE)
|
||||
throw std::logic_error(
|
||||
"GPU completion reservation or fence is invalid");
|
||||
pending_->service->watch(pending_, device, fence);
|
||||
pending_.reset();
|
||||
}
|
||||
|
||||
void Gpu_Completion_Service::Reservation::cancel() {
|
||||
if (!pending_) return;
|
||||
auto pending = std::exchange(pending_, {});
|
||||
cancel_reserved(pending);
|
||||
pending->service->wake();
|
||||
pending_->service->cancel(pending_);
|
||||
pending_.reset();
|
||||
}
|
||||
void Gpu_Completion_Service::update_peak(std::atomic_size_t& peak,
|
||||
std::size_t value) noexcept {
|
||||
std::size_t current = peak.load(std::memory_order_relaxed);
|
||||
while (current < value &&
|
||||
!peak.compare_exchange_weak(current, value,
|
||||
std::memory_order_relaxed)) {}
|
||||
}
|
||||
void Gpu_Completion_Service::update_max(std::atomic_uint64_t& maximum,
|
||||
std::uint64_t value) noexcept {
|
||||
std::uint64_t current = maximum.load(std::memory_order_relaxed);
|
||||
while (current < value &&
|
||||
!maximum.compare_exchange_weak(current, value,
|
||||
std::memory_order_relaxed)) {}
|
||||
}
|
||||
void Gpu_Completion_Service::cancel_reserved(
|
||||
const std::shared_ptr<Pending_Fence>& pending) {
|
||||
if (!pending) return;
|
||||
std::lock_guard lock(pending->mutex);
|
||||
if (pending->status == Pending_Fence::Status::reserved) pending->status = Pending_Fence::Status::canceled;
|
||||
}
|
||||
bool Gpu_Completion_Service::acquire_slot() noexcept {
|
||||
std::size_t current = in_flight_.load(std::memory_order_relaxed);
|
||||
while (current < static_cast<std::size_t>(default_capacity)) {
|
||||
if (in_flight_.compare_exchange_weak(
|
||||
current, current + 1,
|
||||
std::memory_order_acq_rel,
|
||||
std::memory_order_relaxed)) {
|
||||
update_peak(peak_in_flight_, current + 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Gpu_Completion_Service::watch(
|
||||
const std::shared_ptr<Pending_Fence>& pending,
|
||||
VkDevice device, VkFence fence) {
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
if (!pending || pending->service != this ||
|
||||
pending->status != Pending_Fence::Status::reserved)
|
||||
throw std::logic_error(
|
||||
"GPU completion reservation is not reserved");
|
||||
pending->device = device;
|
||||
pending->fence = fence;
|
||||
pending->watched_at = std::chrono::steady_clock::now();
|
||||
pending->status = Pending_Fence::Status::watched;
|
||||
++state_.current->watched;
|
||||
update_peak(state_.current->peak_watched,
|
||||
state_.current->watched);
|
||||
++wake_generation_;
|
||||
}
|
||||
backpressure_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
return false;
|
||||
wake_condition_.notify_one();
|
||||
}
|
||||
void Gpu_Completion_Service::release_slot() noexcept {
|
||||
in_flight_.fetch_sub(1, std::memory_order_relaxed);
|
||||
|
||||
void Gpu_Completion_Service::cancel(
|
||||
const std::shared_ptr<Pending_Fence>& pending) {
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
if (!pending || pending->service != this) return;
|
||||
if (pending->status == Pending_Fence::Status::reserved)
|
||||
pending->status = Pending_Fence::Status::canceled;
|
||||
++wake_generation_;
|
||||
}
|
||||
wake_condition_.notify_one();
|
||||
}
|
||||
|
||||
Gpu_Completion_Service::Prepare_Result Gpu_Completion_Service::prepare(
|
||||
Completion completion, Exception_Handler on_exception, bool observe) {
|
||||
if (!completion) throw std::invalid_argument("GPU completion callback is empty");
|
||||
if (!on_exception) throw std::invalid_argument("GPU completion exception handler is empty");
|
||||
if (stopping_.load(std::memory_order_acquire)) return {{}, Admission_Result::stopping};
|
||||
if (!completion)
|
||||
throw std::invalid_argument("GPU completion callback is empty");
|
||||
if (!on_exception)
|
||||
throw std::invalid_argument(
|
||||
"GPU completion exception handler is empty");
|
||||
|
||||
auto pending = std::make_shared<Pending_Fence>();
|
||||
pending->completion = std::move(completion);
|
||||
pending->on_exception = std::move(on_exception);
|
||||
pending->observe = observe;
|
||||
pending->service = this;
|
||||
/* 唯一 GPU Submit 域绝不等待 completion 容量。满载时把背压作为
|
||||
* 准入结果立即反馈给 Scene，由上游下一帧策略自然重试。 */
|
||||
if (!acquire_slot())
|
||||
return {{}, Admission_Result::capacity_exhausted};
|
||||
if (stopping_.load(std::memory_order_acquire)) {
|
||||
release_slot();
|
||||
return {{}, Admission_Result::stopping};
|
||||
}
|
||||
try {
|
||||
std::lock_guard lock(pending_mutex_);
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
auto& state = *state_.current;
|
||||
if (state.stopping)
|
||||
return {{}, Admission_Result::stopping};
|
||||
if (state.in_flight >= state.capacity) {
|
||||
++state.backpressure_count;
|
||||
return {{}, Admission_Result::capacity_exhausted};
|
||||
}
|
||||
pending_.push_back(pending);
|
||||
++state.in_flight;
|
||||
update_peak(state.peak_in_flight, state.in_flight);
|
||||
++state.reservation_count;
|
||||
++wake_generation_;
|
||||
}
|
||||
catch (...) {
|
||||
release_slot();
|
||||
throw;
|
||||
}
|
||||
reservation_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
wake();
|
||||
wake_condition_.notify_one();
|
||||
return {Reservation(std::move(pending)), Admission_Result::none};
|
||||
}
|
||||
Gpu_Completion_State Gpu_Completion_Service::state() const noexcept {
|
||||
std::lock_guard lock(state_mutex_);
|
||||
|
||||
Gpu_Completion_State Gpu_Completion_Service::state() const {
|
||||
std::lock_guard lock(service_mutex_);
|
||||
return *state_.pending;
|
||||
}
|
||||
void Gpu_Completion_Service::publish_state(std::size_t active_fences,
|
||||
std::size_t pending_fences) noexcept {
|
||||
std::lock_guard lock(state_mutex_);
|
||||
auto& state = *state_.current;
|
||||
state.capacity = static_cast<std::size_t>(default_capacity);
|
||||
state.in_flight = in_flight_.load(std::memory_order_relaxed);
|
||||
state.peak_in_flight = peak_in_flight_.load(std::memory_order_relaxed);
|
||||
state.watched = watched_.load(std::memory_order_relaxed);
|
||||
state.peak_watched = peak_watched_.load(std::memory_order_relaxed);
|
||||
state.active_fences = active_fences;
|
||||
state.pending_fences = pending_fences;
|
||||
state.reservation_count = reservation_count_.load(std::memory_order_relaxed);
|
||||
state.completion_count = completion_count_.load(std::memory_order_relaxed);
|
||||
state.cancellation_count = cancellation_count_.load(std::memory_order_relaxed);
|
||||
state.fence_probe_count = fence_probe_count_.load(std::memory_order_relaxed);
|
||||
state.fence_wait_count = fence_wait_count_.load(std::memory_order_relaxed);
|
||||
state.fence_wait_timeout_count = fence_wait_timeout_count_.load(std::memory_order_relaxed);
|
||||
state.fence_wait_total_ns = fence_wait_total_ns_.load(std::memory_order_relaxed);
|
||||
state.fence_wait_max_ns = fence_wait_max_ns_.load(std::memory_order_relaxed);
|
||||
state.callback_total_ns = callback_total_ns_.load(std::memory_order_relaxed);
|
||||
state.callback_max_ns = callback_max_ns_.load(std::memory_order_relaxed);
|
||||
state.callback_failure_count = callback_failure_count_.load(std::memory_order_relaxed);
|
||||
state.backpressure_count = backpressure_count_.load(std::memory_order_relaxed);
|
||||
state.backpressure_wait_ns = backpressure_wait_ns_.load(std::memory_order_relaxed);
|
||||
state.fault_count = fault_count_.load(std::memory_order_relaxed);
|
||||
state.abandoned_count = abandoned_count_.load(std::memory_order_relaxed);
|
||||
state.stopping = stopping_.load(std::memory_order_relaxed);
|
||||
|
||||
void Gpu_Completion_Service::publish_state(
|
||||
std::size_t active_fences, std::size_t pending_fences) {
|
||||
std::lock_guard lock(service_mutex_);
|
||||
state_.current->active_fences = active_fences;
|
||||
state_.current->pending_fences = pending_fences;
|
||||
state_.advance();
|
||||
}
|
||||
void Gpu_Completion_Service::wake() noexcept {
|
||||
wake_generation_.fetch_add(1, std::memory_order_release);
|
||||
wake_condition_.notify_one();
|
||||
}
|
||||
|
||||
void Gpu_Completion_Service::run() noexcept {
|
||||
struct Device_Fences {
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
std::vector<VkFence> fences;
|
||||
std::vector<VkFence> fences{};
|
||||
};
|
||||
std::vector<std::shared_ptr<Pending_Fence>> active;
|
||||
|
||||
try {
|
||||
active.reserve(static_cast<std::size_t>(default_capacity));
|
||||
std::size_t wait_group_index{};
|
||||
auto last_state_publication = std::chrono::steady_clock::time_point{};
|
||||
const auto wait_age_ns = [](std::chrono::steady_clock::time_point started) {
|
||||
const auto elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - started)
|
||||
.count();
|
||||
return elapsed > 0 ? static_cast<std::uint64_t>(elapsed) : 0ULL;
|
||||
};
|
||||
const auto finish = [this, &wait_age_ns](
|
||||
const std::shared_ptr<Pending_Fence>& pending,
|
||||
VkResult result, Completion_Error error) noexcept {
|
||||
struct Slot_Release {
|
||||
Gpu_Completion_Service* service;
|
||||
~Slot_Release() { service->release_slot(); }
|
||||
} slot_release{this};
|
||||
watched_.fetch_sub(1, std::memory_order_relaxed);
|
||||
try {
|
||||
Completion completion;
|
||||
Exception_Handler on_exception;
|
||||
std::chrono::steady_clock::time_point watched_at{};
|
||||
bool observe{};
|
||||
{
|
||||
std::lock_guard lock(pending->mutex);
|
||||
completion = std::move(pending->completion);
|
||||
on_exception = std::move(pending->on_exception);
|
||||
watched_at = pending->watched_at;
|
||||
observe = pending->observe;
|
||||
pending->status = Pending_Fence::Status::canceled;
|
||||
}
|
||||
Result completion_result;
|
||||
completion_result.error = error;
|
||||
completion_result.vulkan_result = result;
|
||||
if (observe) completion_result.wait_duration_ns = wait_age_ns(watched_at);
|
||||
if (error != Completion_Error::none) {
|
||||
fault_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
abandoned_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
const auto callback_started = std::chrono::steady_clock::now();
|
||||
try { completion(std::move(completion_result)); }
|
||||
catch (...) {
|
||||
callback_failure_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
try {
|
||||
on_exception(contextual_exception(
|
||||
"delivering GPU completion", std::current_exception()));
|
||||
}
|
||||
catch (...) {}
|
||||
}
|
||||
const auto callback_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - callback_started).count();
|
||||
if (callback_elapsed > 0) {
|
||||
const auto elapsed = static_cast<std::uint64_t>(callback_elapsed);
|
||||
callback_total_ns_.fetch_add(elapsed, std::memory_order_relaxed);
|
||||
update_max(callback_max_ns_, elapsed);
|
||||
}
|
||||
completion_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
catch (...) {
|
||||
try {
|
||||
std::lock_guard lock(pending->mutex);
|
||||
if (pending->on_exception)
|
||||
pending->on_exception(std::current_exception());
|
||||
pending->status = Pending_Fence::Status::canceled;
|
||||
}
|
||||
catch (...) {}
|
||||
}
|
||||
};
|
||||
for (;;) {
|
||||
const std::uint64_t wake_generation =
|
||||
wake_generation_.load(std::memory_order_acquire);
|
||||
{ std::lock_guard lock(pending_mutex_); while (!pending_.empty()) { active.push_back(std::move(pending_.front())); pending_.pop_front(); } }
|
||||
std::vector<Device_Fences> groups;
|
||||
std::size_t reserved_count{};
|
||||
for (auto iterator = active.begin(); iterator != active.end();) {
|
||||
Pending_Fence::Status status;
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
VkFence fence{VK_NULL_HANDLE};
|
||||
{
|
||||
std::lock_guard lock((*iterator)->mutex);
|
||||
status = (*iterator)->status;
|
||||
device = (*iterator)->device;
|
||||
fence = (*iterator)->fence;
|
||||
}
|
||||
if (status == Pending_Fence::Status::canceled ||
|
||||
(status == Pending_Fence::Status::reserved &&
|
||||
stopping_.load(std::memory_order_acquire))) {
|
||||
cancel_reserved(*iterator);
|
||||
iterator = active.erase(iterator);
|
||||
cancellation_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
release_slot();
|
||||
continue;
|
||||
}
|
||||
if (status == Pending_Fence::Status::reserved) ++reserved_count;
|
||||
if (status == Pending_Fence::Status::watched) {
|
||||
auto group = std::find_if(
|
||||
groups.begin(), groups.end(),
|
||||
[device](const Device_Fences& item) {
|
||||
return item.device == device;
|
||||
});
|
||||
if (group == groups.end()) {
|
||||
groups.push_back(Device_Fences{device, {}});
|
||||
group = groups.end() - 1;
|
||||
}
|
||||
group->fences.push_back(fence);
|
||||
}
|
||||
++iterator;
|
||||
}
|
||||
const auto publication_time = std::chrono::steady_clock::now();
|
||||
if (last_state_publication == std::chrono::steady_clock::time_point{} ||
|
||||
publication_time - last_state_publication >= state_publication_interval) {
|
||||
// 域内只按诊断粒度发布双缓冲 State；逐 fence 热路径只更新原子计数。
|
||||
publish_state(active.size(), reserved_count);
|
||||
last_state_publication = publication_time;
|
||||
}
|
||||
bool pending_empty;
|
||||
{ std::lock_guard lock(pending_mutex_); pending_empty = pending_.empty(); }
|
||||
if (stopping_.load(std::memory_order_acquire) && active.empty() && pending_empty) {
|
||||
publish_state(0, 0);
|
||||
return;
|
||||
}
|
||||
// Probe every watched fence first. A permanently unsignaled fence is
|
||||
// converted into a logical failure after a bounded interval. The
|
||||
// submission is explicitly marked abandoned so its owner can
|
||||
// quarantine, rather than recycle, the referenced GPU resources.
|
||||
bool completed_any = false;
|
||||
for (auto iterator = active.begin(); iterator != active.end();) {
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
VkFence fence{VK_NULL_HANDLE};
|
||||
Pending_Fence::Status status;
|
||||
std::chrono::steady_clock::time_point watched_at{};
|
||||
{
|
||||
std::lock_guard lock((*iterator)->mutex);
|
||||
status = (*iterator)->status;
|
||||
device = (*iterator)->device;
|
||||
fence = (*iterator)->fence;
|
||||
watched_at = (*iterator)->watched_at;
|
||||
}
|
||||
if (status != Pending_Fence::Status::watched) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
if (stopping_.load(std::memory_order_acquire)) {
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, VK_TIMEOUT, Completion_Error::fence_abandoned);
|
||||
completed_any = true;
|
||||
continue;
|
||||
}
|
||||
if (wait_age_ns(watched_at) >= maximum_fence_age_ns) {
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, VK_TIMEOUT, Completion_Error::fence_abandoned);
|
||||
completed_any = true;
|
||||
continue;
|
||||
}
|
||||
fence_probe_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
const VkResult result = vkGetFenceStatus(device, fence);
|
||||
if (result == VK_NOT_READY) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, result, result == VK_SUCCESS
|
||||
? Completion_Error::none
|
||||
: Completion_Error::vulkan_failure);
|
||||
completed_any = true;
|
||||
}
|
||||
if (completed_any) continue;
|
||||
if (groups.empty()) {
|
||||
std::unique_lock lock(wait_mutex_);
|
||||
if (wake_generation_.load(std::memory_order_acquire) ==
|
||||
wake_generation) {
|
||||
wake_condition_.wait(lock, [this, wake_generation] {
|
||||
return wake_generation_.load(std::memory_order_acquire) !=
|
||||
wake_generation;
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
wait_group_index %= groups.size();
|
||||
const Device_Fences& group = groups[wait_group_index++];
|
||||
const auto wait_started = std::chrono::steady_clock::now();
|
||||
fence_wait_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
const VkResult wait_result = vkWaitForFences(
|
||||
group.device, static_cast<std::uint32_t>(group.fences.size()),
|
||||
group.fences.data(), VK_FALSE, fence_wait_timeout_ns);
|
||||
const auto wait_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - wait_started).count();
|
||||
if (wait_elapsed > 0) {
|
||||
const auto elapsed = static_cast<std::uint64_t>(wait_elapsed);
|
||||
fence_wait_total_ns_.fetch_add(elapsed, std::memory_order_relaxed);
|
||||
update_max(fence_wait_max_ns_, elapsed);
|
||||
}
|
||||
if (wait_result == VK_TIMEOUT)
|
||||
fence_wait_timeout_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
for (auto iterator = active.begin(); iterator != active.end();) {
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
VkFence fence{VK_NULL_HANDLE};
|
||||
Pending_Fence::Status status;
|
||||
{
|
||||
std::lock_guard lock((*iterator)->mutex);
|
||||
status = (*iterator)->status;
|
||||
device = (*iterator)->device;
|
||||
fence = (*iterator)->fence;
|
||||
}
|
||||
if (status != Pending_Fence::Status::watched ||
|
||||
device != group.device) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
VkResult result = wait_result;
|
||||
if (wait_result == VK_SUCCESS || wait_result == VK_TIMEOUT) {
|
||||
fence_probe_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
result = vkGetFenceStatus(device, fence);
|
||||
}
|
||||
if (result == VK_NOT_READY) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, result, result == VK_SUCCESS
|
||||
? Completion_Error::none
|
||||
: Completion_Error::vulkan_failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
stopping_.store(true, std::memory_order_release);
|
||||
const auto service_failure = std::current_exception();
|
||||
const auto fail_pending = [this, &service_failure](
|
||||
const std::shared_ptr<Pending_Fence>& pending) noexcept {
|
||||
if (!pending) return;
|
||||
active.reserve(static_cast<std::size_t>(default_capacity));
|
||||
std::size_t wait_group_index{};
|
||||
auto last_state_publication =
|
||||
std::chrono::steady_clock::time_point{};
|
||||
|
||||
const auto finish = [this](
|
||||
const std::shared_ptr<Pending_Fence>& pending,
|
||||
VkResult vulkan_result, Completion_Error error) noexcept {
|
||||
Completion completion;
|
||||
Exception_Handler on_exception;
|
||||
bool watched{};
|
||||
try {
|
||||
std::lock_guard lock(pending->mutex);
|
||||
watched = pending->status == Pending_Fence::Status::watched;
|
||||
Result result{};
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
if (pending->status != Pending_Fence::Status::watched)
|
||||
return;
|
||||
completion = std::move(pending->completion);
|
||||
on_exception = std::move(pending->on_exception);
|
||||
pending->completion = {};
|
||||
result.error = error;
|
||||
result.vulkan_result = vulkan_result;
|
||||
if (pending->observe)
|
||||
result.wait_duration_ns =
|
||||
elapsed_nanoseconds(pending->watched_at);
|
||||
pending->status = Pending_Fence::Status::canceled;
|
||||
auto& state = *state_.current;
|
||||
if (state.watched != 0) --state.watched;
|
||||
if (error != Completion_Error::none) {
|
||||
++state.fault_count;
|
||||
if (error == Completion_Error::fence_abandoned)
|
||||
++state.abandoned_count;
|
||||
}
|
||||
}
|
||||
catch (...) {}
|
||||
if (watched)
|
||||
watched_.fetch_sub(1, std::memory_order_relaxed);
|
||||
cancellation_count_.fetch_add(1, std::memory_order_relaxed);
|
||||
if (on_exception) {
|
||||
try { on_exception(service_failure); }
|
||||
|
||||
const auto callback_started =
|
||||
std::chrono::steady_clock::now();
|
||||
std::exception_ptr callback_failure;
|
||||
try {
|
||||
completion(std::move(result));
|
||||
}
|
||||
catch (...) {
|
||||
callback_failure = std::current_exception();
|
||||
}
|
||||
const auto callback_ns =
|
||||
elapsed_nanoseconds(callback_started);
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
auto& state = *state_.current;
|
||||
state.callback_total_ns += callback_ns;
|
||||
update_peak(state.callback_max_ns, callback_ns);
|
||||
if (callback_failure) ++state.callback_failure_count;
|
||||
++state.completion_count;
|
||||
if (state.in_flight != 0) --state.in_flight;
|
||||
}
|
||||
if (callback_failure) {
|
||||
try {
|
||||
on_exception(contextual_exception(
|
||||
"delivering GPU completion",
|
||||
std::move(callback_failure)));
|
||||
}
|
||||
catch (...) {}
|
||||
}
|
||||
release_slot();
|
||||
};
|
||||
for (const auto& pending : active) fail_pending(pending);
|
||||
|
||||
for (;;) {
|
||||
std::shared_ptr<Pending_Fence> pending;
|
||||
try {
|
||||
std::lock_guard lock(pending_mutex_);
|
||||
if (pending_.empty()) break;
|
||||
pending = std::move(pending_.front());
|
||||
std::uint64_t wake_generation{};
|
||||
bool stopping{};
|
||||
std::size_t reserved_count{};
|
||||
std::vector<Device_Fences> groups;
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
wake_generation = wake_generation_;
|
||||
stopping = state_.current->stopping;
|
||||
while (!pending_.empty()) {
|
||||
active.push_back(std::move(pending_.front()));
|
||||
pending_.pop_front();
|
||||
}
|
||||
for (auto iterator = active.begin();
|
||||
iterator != active.end();) {
|
||||
const auto status = (*iterator)->status;
|
||||
if (status == Pending_Fence::Status::canceled ||
|
||||
(status == Pending_Fence::Status::reserved &&
|
||||
stopping)) {
|
||||
iterator = active.erase(iterator);
|
||||
++state_.current->cancellation_count;
|
||||
if (state_.current->in_flight != 0)
|
||||
--state_.current->in_flight;
|
||||
continue;
|
||||
}
|
||||
if (status == Pending_Fence::Status::reserved) {
|
||||
++reserved_count;
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
auto group = std::ranges::find(
|
||||
groups, (*iterator)->device,
|
||||
&Device_Fences::device);
|
||||
if (group == groups.end()) {
|
||||
groups.push_back(
|
||||
Device_Fences{(*iterator)->device, {}});
|
||||
group = groups.end() - 1;
|
||||
}
|
||||
group->fences.push_back((*iterator)->fence);
|
||||
++iterator;
|
||||
}
|
||||
}
|
||||
|
||||
const auto publication_time =
|
||||
std::chrono::steady_clock::now();
|
||||
if (last_state_publication ==
|
||||
std::chrono::steady_clock::time_point{} ||
|
||||
publication_time - last_state_publication >=
|
||||
state_publication_interval) {
|
||||
publish_state(active.size(), reserved_count);
|
||||
last_state_publication = publication_time;
|
||||
}
|
||||
|
||||
bool pending_empty{};
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
stopping = state_.current->stopping;
|
||||
pending_empty = pending_.empty();
|
||||
}
|
||||
if (stopping && active.empty() && pending_empty) {
|
||||
publish_state(0, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
bool completed_any{};
|
||||
for (auto iterator = active.begin();
|
||||
iterator != active.end();) {
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
VkFence fence{VK_NULL_HANDLE};
|
||||
std::chrono::steady_clock::time_point watched_at{};
|
||||
Pending_Fence::Status status{};
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
status = (*iterator)->status;
|
||||
device = (*iterator)->device;
|
||||
fence = (*iterator)->fence;
|
||||
watched_at = (*iterator)->watched_at;
|
||||
stopping = state_.current->stopping;
|
||||
}
|
||||
if (status != Pending_Fence::Status::watched) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
if (stopping ||
|
||||
elapsed_nanoseconds(watched_at) >=
|
||||
maximum_fence_age_ns) {
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, VK_TIMEOUT,
|
||||
Completion_Error::fence_abandoned);
|
||||
completed_any = true;
|
||||
continue;
|
||||
}
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
++state_.current->fence_probe_count;
|
||||
}
|
||||
const VkResult result = vkGetFenceStatus(device, fence);
|
||||
if (result == VK_NOT_READY) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, result,
|
||||
result == VK_SUCCESS
|
||||
? Completion_Error::none
|
||||
: Completion_Error::vulkan_failure);
|
||||
completed_any = true;
|
||||
}
|
||||
if (completed_any) continue;
|
||||
|
||||
if (groups.empty()) {
|
||||
std::unique_lock lock(service_mutex_);
|
||||
wake_condition_.wait(lock, [this, wake_generation] {
|
||||
return wake_generation_ != wake_generation ||
|
||||
state_.current->stopping;
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
wait_group_index %= groups.size();
|
||||
const auto& group = groups[wait_group_index++];
|
||||
const auto wait_started = std::chrono::steady_clock::now();
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
++state_.current->fence_wait_count;
|
||||
}
|
||||
const VkResult wait_result = vkWaitForFences(
|
||||
group.device,
|
||||
static_cast<std::uint32_t>(group.fences.size()),
|
||||
group.fences.data(), VK_FALSE,
|
||||
fence_wait_timeout_ns);
|
||||
const auto wait_ns = elapsed_nanoseconds(wait_started);
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
auto& state = *state_.current;
|
||||
state.fence_wait_total_ns += wait_ns;
|
||||
update_peak(state.fence_wait_max_ns, wait_ns);
|
||||
if (wait_result == VK_TIMEOUT)
|
||||
++state.fence_wait_timeout_count;
|
||||
}
|
||||
|
||||
for (auto iterator = active.begin();
|
||||
iterator != active.end();) {
|
||||
VkDevice device{VK_NULL_HANDLE};
|
||||
VkFence fence{VK_NULL_HANDLE};
|
||||
Pending_Fence::Status status{};
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
status = (*iterator)->status;
|
||||
device = (*iterator)->device;
|
||||
fence = (*iterator)->fence;
|
||||
}
|
||||
if (status != Pending_Fence::Status::watched ||
|
||||
device != group.device) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
VkResult result = wait_result;
|
||||
if (wait_result == VK_SUCCESS ||
|
||||
wait_result == VK_TIMEOUT) {
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
++state_.current->fence_probe_count;
|
||||
}
|
||||
result = vkGetFenceStatus(device, fence);
|
||||
}
|
||||
if (result == VK_NOT_READY) {
|
||||
++iterator;
|
||||
continue;
|
||||
}
|
||||
auto pending = *iterator;
|
||||
iterator = active.erase(iterator);
|
||||
finish(pending, result,
|
||||
result == VK_SUCCESS
|
||||
? Completion_Error::none
|
||||
: Completion_Error::vulkan_failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (...) {
|
||||
const auto service_failure = std::current_exception();
|
||||
std::vector<std::shared_ptr<Pending_Fence>> failed =
|
||||
std::move(active);
|
||||
{
|
||||
std::lock_guard lock(service_mutex_);
|
||||
while (!pending_.empty()) {
|
||||
failed.push_back(std::move(pending_.front()));
|
||||
pending_.pop_front();
|
||||
}
|
||||
catch (...) { break; }
|
||||
fail_pending(pending);
|
||||
auto& state = *state_.current;
|
||||
state.stopping = true;
|
||||
for (const auto& pending : failed) {
|
||||
if (pending->status == Pending_Fence::Status::watched &&
|
||||
state.watched != 0)
|
||||
--state.watched;
|
||||
pending->status = Pending_Fence::Status::canceled;
|
||||
++state.cancellation_count;
|
||||
if (state.in_flight != 0) --state.in_flight;
|
||||
}
|
||||
state.active_fences = 0;
|
||||
state.pending_fences = 0;
|
||||
state_.advance();
|
||||
}
|
||||
for (auto& pending : failed) {
|
||||
if (!pending->on_exception) continue;
|
||||
try { pending->on_exception(service_failure); }
|
||||
catch (...) {}
|
||||
}
|
||||
publish_state(0, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
#include "../Gpu_Completion_State.hpp"
|
||||
#include <double_buffer/mechanism.hpp>
|
||||
#include <volk.h>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
@@ -28,9 +27,9 @@ public:
|
||||
vulkan_failure
|
||||
};
|
||||
struct Result {
|
||||
Completion_Error error{}; /* 归一化完成结果。 */
|
||||
VkResult vulkan_result{VK_SUCCESS}; /* Vulkan 原始结果码。 */
|
||||
std::uint64_t wait_duration_ns{}; /* fence 等待时间，单位为纳秒。 */
|
||||
Completion_Error error{}; /* 归一化完成结果。 */
|
||||
VkResult vulkan_result{VK_SUCCESS}; /* Vulkan 原始结果码。 */
|
||||
std::uint64_t wait_duration_ns{}; /* fence 等待时间，单位为纳秒。 */
|
||||
};
|
||||
using Completion = std::function<void(Result)>;
|
||||
using Exception_Handler = std::function<void(std::exception_ptr)>;
|
||||
@@ -46,12 +45,12 @@ public:
|
||||
private:
|
||||
explicit Reservation(std::shared_ptr<Pending_Fence> pending) noexcept;
|
||||
void cancel();
|
||||
std::shared_ptr<Pending_Fence> pending_; /* 尚未 watch 或 cancel 的准入记录。 */
|
||||
std::shared_ptr<Pending_Fence> pending_; /* 尚未 watch 或 cancel 的准入记录。 */
|
||||
friend class Gpu_Completion_Service;
|
||||
};
|
||||
struct Prepare_Result {
|
||||
Reservation reservation; /* 成功时返回的 fence reservation。 */
|
||||
Admission_Result result{}; /* 准入结果。 */
|
||||
Reservation reservation; /* 成功时返回的 fence reservation。 */
|
||||
Admission_Result result{}; /* 准入结果。 */
|
||||
[[nodiscard]] explicit operator bool() const noexcept;
|
||||
};
|
||||
static Gpu_Completion_Service& instance();
|
||||
@@ -60,7 +59,7 @@ public:
|
||||
[[nodiscard]] Prepare_Result prepare(Completion completion,
|
||||
Exception_Handler on_exception,
|
||||
bool observe);
|
||||
[[nodiscard]] Gpu_Completion_State state() const noexcept;
|
||||
[[nodiscard]] Gpu_Completion_State state() const;
|
||||
private:
|
||||
struct Pending_Fence {
|
||||
enum class Status {
|
||||
@@ -68,61 +67,34 @@ private:
|
||||
watched,
|
||||
canceled
|
||||
};
|
||||
std::mutex mutex; /* 保护本条记录的状态和回调移动。 */
|
||||
VkDevice device{VK_NULL_HANDLE}; /* fence 所属 Vulkan Device。 */
|
||||
VkFence fence{VK_NULL_HANDLE}; /* 受监视的 Vulkan fence。 */
|
||||
Completion completion; /* 完成或隔离后的交付回调。 */
|
||||
Exception_Handler on_exception; /* 回调异常的隔离入口。 */
|
||||
std::chrono::steady_clock::time_point watched_at{}; /* 开始监视的单调时钟时刻。 */
|
||||
Gpu_Completion_Service* service{}; /* 不拥有的服务实例。 */
|
||||
Status status{Status::reserved}; /* reservation 生命周期状态。 */
|
||||
bool observe{}; /* 是否采集 fence 等待耗时。 */
|
||||
VkDevice device{VK_NULL_HANDLE}; /* fence 所属 Vulkan Device。 */
|
||||
VkFence fence{VK_NULL_HANDLE}; /* 受监视的 Vulkan fence。 */
|
||||
Completion completion; /* 完成或隔离后的交付回调。 */
|
||||
Exception_Handler on_exception; /* 回调异常的隔离入口。 */
|
||||
std::chrono::steady_clock::time_point watched_at{}; /* 开始监视的单调时钟时刻。 */
|
||||
Gpu_Completion_Service* service{}; /* 不拥有的服务实例。 */
|
||||
Status status{Status::reserved}; /* reservation 生命周期状态。 */
|
||||
bool observe{}; /* 是否采集 fence 等待耗时。 */
|
||||
};
|
||||
Gpu_Completion_Service();
|
||||
~Gpu_Completion_Service();
|
||||
static void update_peak(std::atomic_size_t& peak, std::size_t value) noexcept;
|
||||
static void update_max(std::atomic_uint64_t& maximum,
|
||||
std::uint64_t value) noexcept;
|
||||
static void cancel_reserved(const std::shared_ptr<Pending_Fence>& pending);
|
||||
[[nodiscard]] bool acquire_slot() noexcept;
|
||||
void release_slot() noexcept;
|
||||
void watch(const std::shared_ptr<Pending_Fence>& pending,
|
||||
VkDevice device, VkFence fence);
|
||||
void cancel(const std::shared_ptr<Pending_Fence>& pending);
|
||||
void publish_state(std::size_t active_fences,
|
||||
std::size_t pending_fences) noexcept;
|
||||
void wake() noexcept;
|
||||
std::size_t pending_fences);
|
||||
void run() noexcept;
|
||||
static constexpr std::ptrdiff_t default_capacity = 1024;
|
||||
static constexpr std::uint64_t fence_wait_timeout_ns = 1'000'000;
|
||||
static constexpr std::uint64_t maximum_fence_age_ns = 30'000'000'000ULL;
|
||||
static constexpr auto state_publication_interval = std::chrono::milliseconds(100);
|
||||
std::mutex pending_mutex_; /* 保护新登记 fence 队列。 */
|
||||
std::deque<std::shared_ptr<Pending_Fence>> pending_; /* 完成线程尚未分组的 fence。 */
|
||||
std::mutex wait_mutex_; /* 保护完成线程的条件等待。 */
|
||||
std::condition_variable wake_condition_; /* 新 fence 或停止请求的唤醒源。 */
|
||||
std::atomic_uint64_t wake_generation_{}; /* 防止丢失唤醒的版本。 */
|
||||
std::atomic_size_t in_flight_{}; /* 当前 reservation 数。 */
|
||||
std::atomic_size_t peak_in_flight_{}; /* 历史最大 reservation 数。 */
|
||||
std::atomic_size_t watched_{}; /* 当前受监视 fence 数。 */
|
||||
std::atomic_size_t peak_watched_{}; /* 历史最大受监视 fence 数。 */
|
||||
std::atomic_uint64_t reservation_count_{};
|
||||
std::atomic_uint64_t completion_count_{};
|
||||
std::atomic_uint64_t cancellation_count_{};
|
||||
std::atomic_uint64_t fence_probe_count_{};
|
||||
std::atomic_uint64_t fence_wait_count_{};
|
||||
std::atomic_uint64_t fence_wait_timeout_count_{};
|
||||
std::atomic_uint64_t fence_wait_total_ns_{};
|
||||
std::atomic_uint64_t fence_wait_max_ns_{};
|
||||
std::atomic_uint64_t callback_total_ns_{};
|
||||
std::atomic_uint64_t callback_max_ns_{};
|
||||
std::atomic_uint64_t callback_failure_count_{};
|
||||
std::atomic_uint64_t backpressure_count_{}; /* 准入背压累计次数。 */
|
||||
std::atomic_uint64_t backpressure_wait_ns_{}; /* 准入背压累计纳秒。 */
|
||||
std::atomic_uint64_t fault_count_{}; /* 完成故障累计次数。 */
|
||||
std::atomic_uint64_t abandoned_count_{}; /* 放弃 fence 累计次数。 */
|
||||
std::atomic_bool stopping_{}; /* 服务是否正在停止。 */
|
||||
mutable std::mutex state_mutex_; /* 只保护域级 State 的指针交换。 */
|
||||
double_buffer::Publish_Double_Buffer<Gpu_Completion_State> state_{};
|
||||
std::thread thread_; /* 专用 fence 完成线程。 */
|
||||
/* GPU completion 的队列、reservation 生命周期和 State 指针交换是
|
||||
* 同一个一致性域；这把锁是该域唯一的外部同步边界。 */
|
||||
mutable std::mutex service_mutex_{};
|
||||
std::condition_variable wake_condition_{}; /* 专用完成线程的休眠唤醒源。 */
|
||||
std::uint64_t wake_generation_{}; /* service_mutex_ 下的防丢唤醒版本。 */
|
||||
std::deque<std::shared_ptr<Pending_Fence>> pending_{}; /* 尚未转入完成线程本地集合的记录。 */
|
||||
double_buffer::Publish_Double_Buffer<Gpu_Completion_State> state_{}; /* GPU completion 运行状态的唯一权威源。 */
|
||||
std::thread thread_; /* 专用 fence 完成线程。 */
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user