ae/ECAP_Server
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

内存优化

Browse Source
This commit is contained in:
种太阳的热带美短
2026-08-28 11:49:06 +08:00
parent e26f7eb52f
commit fa1e7e24fc
21 changed files with 722 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/Local_Server/Data_Source/Data_Source.cpp
+111 -33
View File
@@ -1,4 +1,5 @@
#include "Data_Source.h"
#include <algorithm>
#include <codecvt>
#include <string_view>
#include "../server/Global.h"
@@ -155,6 +156,18 @@ std::string bin_format(std::string_view hex, time_t t) {
SSR::MLAT_timestamp a(sec, 0);
return create_Binary_Format_memory(hex, 0, &a);
}
static int hex_digit_value(unsigned char c) {
if (c >= '0' && c <= '9')
return c - '0';
if (c >= 'a' && c <= 'f')
return c - 'a' + 10;
if (c >= 'A' && c <= 'F')
return c - 'A' + 10;
return -1;
}
static bool is_hex_separator(unsigned char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
std::optional<std::string> File_Data_Source::get_raw_line(int& ret_index) {
if (index == 0 && part_infos.empty()) {
ret_index = -1;
@@ -204,31 +217,35 @@ std::string get_true_from_raw_line(Data_Source* ds, File_Data_Type data_type, in
// 10 01 AA 14 61 7E 5E 02 E6 0F 38 7D 87 08 1A 32 10 01 AD B5 64 F7 63 5D 78
// 0F 9D BA 93 BA
else if (data_type == File_Data_Type::BIN_Blank_Text) {
std::string info(log_info);
std::string result = ds->last_char;
ds->last_char = "";
for (char c : info) {
if (std::isxdigit(c)) {
// 会判断字符 c 是否是十六进制数字字符
result += c;
int high = -1;
char high_char{};
ret.reserve((log_info.size() + ds->last_char.size()) / 2);
if (!ds->last_char.empty()) {
high_char = ds->last_char[0];
high = hex_digit_value(static_cast<unsigned char>(high_char));
}
ds->last_char.clear();
for (char c : log_info) {
const auto value = hex_digit_value(static_cast<unsigned char>(c));
if (value >= 0) {
if (high < 0) {
high = value;
high_char = c;
}
else {
ret.push_back(static_cast<char>((high << 4) | value));
high = -1;
}
}
else if (std::isspace(c)) {
//
}
else {
else if (!is_hex_separator(static_cast<unsigned char>(c))) {
std::ostringstream oss;
oss << "存在非法字符[" << c << "]: value:" << (int)c << " in：" << log_info << LOG_POS;
server_logger->c_debug("非法字符：", {}, oss.str());
return "";
}
}
auto size = result.size();
if (size % 2 != 0) {
// 保存最后一个字符
ds->last_char = result.empty() ? "" : std::string(1, result.back());
result.pop_back();
}
ret = hex2mem(result);
if (high >= 0)
ds->last_char.assign(1, high_char);
}
else if (data_type == File_Data_Type::BIN_Blank_One_Line_With_Escape) {
auto size = log_info.size();
@@ -406,25 +423,86 @@ void File_Data_Source::handle_mode_s(std::shared_ptr<SSR::Mode_S_Msg>& msg) {
}
}
void Dll_Data_Source::origin_data_transform_mode_data(std::string& mode_data) {
auto start = std::chrono::steady_clock::now();
if (read_func_ptr == nullptr)
return;
const auto config = specific.read([](const auto& value) { return value; });
auto buf = reinterpret_cast<char*>(buffer.data());
auto len = read_func_ptr(buf, config.buffer_size);
read_statistics.update(len);
std::string data(buf, len);
if (data.empty()) {
auto end = std::chrono::steady_clock::now();
auto cost = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// std::cout << "Dll_Data_Source::read cost: " << cost << " us\n";
return;
const auto normal_read_size = config.buffer_size;
const auto drain_read_size = std::max(normal_read_size, backlog_read_size);
std::size_t raw_size = 0;
drop_mode_ac_for_batch_ = false;
auto read = [&](std::size_t request_size) {
const auto required_size = raw_size + request_size;
if (buffer.size() < required_size)
buffer.resize(required_size);
const auto length = read_func_ptr(reinterpret_cast<char*>(buffer.data() + raw_size), request_size);
read_statistics.update(length);
raw_size += length;
last_read_request_bytes_ = request_size;
return length;
};
if (backlog_active_.load(std::memory_order_relaxed)) {
drop_mode_ac_for_batch_ = true;
const auto length = read(drain_read_size);
backlog_active_.store(length == drain_read_size, std::memory_order_relaxed);
}
auto ret = get_true_from_raw_line(this, config.data_type, -1, data);
auto end = std::chrono::steady_clock::now();
auto cost = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
// std::cout << "Dll_Data_Source::read cost: " << cost << " us\n";
mode_data = ret;
else {
const auto first_length = read(normal_read_size);
if (first_length == normal_read_size) {
const auto probe_length = read(drain_read_size);
if (probe_length >= normal_read_size) {
drop_mode_ac_for_batch_ = true;
backlog_active_.store(probe_length == drain_read_size, std::memory_order_relaxed);
backlog_events_total_.fetch_add(1, std::memory_order_relaxed);
}
}
}
last_batch_raw_bytes_.store(raw_size, std::memory_order_relaxed);
auto largest = largest_batch_raw_bytes_.load(std::memory_order_relaxed);
while (largest < raw_size && !largest_batch_raw_bytes_.compare_exchange_weak(largest, raw_size, std::memory_order_relaxed)) {
}
if (drop_mode_ac_for_batch_) {
backlog_batches_total_.fetch_add(1, std::memory_order_relaxed);
backlog_raw_bytes_total_.fetch_add(raw_size, std::memory_order_relaxed);
}
if (raw_size == 0)
return;
mode_data = get_true_from_raw_line(this, config.data_type, -1,
std::string_view(reinterpret_cast<const char*>(buffer.data()), raw_size));
}
bool Dll_Data_Source::discard_input_packet(std::string_view packet) {
if (!drop_mode_ac_for_batch_ || packet.size() < 2 || packet[1] != SSR::Msg::AC)
return false;
dropped_mode_ac_frames_total_.fetch_add(1, std::memory_order_relaxed);
dropped_mode_ac_bytes_total_.fetch_add(packet.size(), std::memory_order_relaxed);
return true;
}
Psc::JSON Dll_Data_Source::backlog_state_json() const {
const auto normal_read_size = specific.member<&Dll_Data_Source_Data::buffer_size>().read([](const auto& value) {
return value;
});
Psc::JSON result = Psc::JSON::object();
result.append({"active", backlog_active_.load(std::memory_order_relaxed)});
result.append({"normal_read_bytes", normal_read_size});
result.append({"drain_read_bytes", std::max(normal_read_size, backlog_read_size)});
result.append({"last_read_request_bytes", last_read_request_bytes_.load(std::memory_order_relaxed)});
result.append({"last_batch_raw_bytes", last_batch_raw_bytes_.load(std::memory_order_relaxed)});
result.append({"largest_batch_raw_bytes", largest_batch_raw_bytes_.load(std::memory_order_relaxed)});
result.append({"events_total", backlog_events_total_.load(std::memory_order_relaxed)});
result.append({"batches_total", backlog_batches_total_.load(std::memory_order_relaxed)});
result.append({"raw_bytes_total", backlog_raw_bytes_total_.load(std::memory_order_relaxed)});
result.append({"dropped_mode_ac_frames_total", dropped_mode_ac_frames_total_.load(std::memory_order_relaxed)});
result.append({"dropped_mode_ac_bytes_total", dropped_mode_ac_bytes_total_.load(std::memory_order_relaxed)});
return result;
}
Psc::JSON Dll_Data_Source::get_custom_state_json() {
Psc::JSON result = Psc::JSON::object();
{
std::lock_guard lock(state_mtx);
result.append({"state", state});
}
result.append({"library_read", read_statistics.to_json()});
result.append({"backlog", backlog_state_json()});
return result;
}
asio::awaitable<void> Shared_Memory_Data_Source::_open() {
const auto config = specific.read([](const auto& value) { return value; });
module/Local_Server/Data_Source/Data_Source.h
+20 -8
View File
@@ -1,4 +1,5 @@
#pragma once
#include <memory_resource>
#include <string_view>
#include "Data_Source_Handler.h"
namespace Psc {
@@ -176,6 +177,9 @@ public:
virtual asio::awaitable<void> handle_in_loop_coro() {
co_return;
}
virtual bool discard_input_packet(std::string_view) {
return false;
}
Frequency_Limit statistic_fl;
Psc::JSON statistic_json() {
Psc::JSON ret = With_Loop_Coro::to_base_json();
@@ -459,19 +463,14 @@ struct Type_Descriptor<Dll_Data_Source_Data> {
class Dll_Data_Source : public Data_Source {
public:
adminive::Managed_Value<Dll_Data_Source_Data> specific;
Psc::JSON get_custom_state_json() override {
std::lock_guard lock(state_mtx);
Psc::JSON ret = Psc::JSON::object();
ret.append({"state", state});
ret.append({"library_read", read_statistics.to_json()});
return ret;
}
Psc::JSON get_custom_state_json() override;
Psc::JSON backlog_state_json() const;
Psc::Throughput_Statistics read_statistics;
Dll_Data_Source() {
type = "Dll_Data_Source";
}
~Dll_Data_Source() override = default;
std::vector<std::uint8_t> buffer;
std::pmr::vector<std::uint8_t> buffer;
void from_json(const Psc::JSON* that_json) override {
Data_Source::from_json(that_json);
const auto buffer_size = specific.write([&](auto& value) {
@@ -521,7 +520,20 @@ public:
}
co_return;
}
bool discard_input_packet(std::string_view packet) override;
void origin_data_transform_mode_data(std::string& data) override;
private:
static constexpr std::size_t backlog_read_size = 8 * 1024 * 1024;
bool drop_mode_ac_for_batch_{};
std::atomic_bool backlog_active_{};
std::atomic<std::uint64_t> backlog_events_total_{};
std::atomic<std::uint64_t> backlog_batches_total_{};
std::atomic<std::uint64_t> backlog_raw_bytes_total_{};
std::atomic<std::uint64_t> dropped_mode_ac_frames_total_{};
std::atomic<std::uint64_t> dropped_mode_ac_bytes_total_{};
std::atomic<std::size_t> last_batch_raw_bytes_{};
std::atomic<std::size_t> largest_batch_raw_bytes_{};
std::atomic<std::size_t> last_read_request_bytes_{};
};
class Shared_Memory_Data_Source_Data {
public:
module/Local_Server/Data_Source/Data_Source_Handler.cpp
+20 -53
View File
@@ -15,14 +15,16 @@ std::shared_ptr<SSR::Msg> Data_Source_Handler::create_msg(std::string_view packe
std::cout << "first:" << mem2hex(std::string(packet)) << std::endl;
return nullptr;
}
std::string error_len = source->key + " " + LOG_POS_SIMPLE + std::string(" ") + "mode_s_error_length";
auto error_len = [&] {
return source->key + " " + LOG_POS_SIMPLE + std::string(" ") + "mode_s_error_length";
};
if (packet.size() < 2)
return nullptr;
auto mt = packet[1];
if (mt == SSR::Msg::AC) {
if (packet.size() != SSR::Msg::AC_len) {
auto packet_hex = mem2hex(std::string(packet));
SSR::mode_s_logger->c_debug(error_len, {},
SSR::mode_s_logger->c_debug(error_len(), {},
std::to_string(mt) + " size:" + std::to_string(packet.size()) +
" hex:" + packet_hex + " should:" + std::to_string(SSR::Msg::AC_len));
mode_ac_statistic.add_length_error();
@@ -34,7 +36,7 @@ std::shared_ptr<SSR::Msg> Data_Source_Handler::create_msg(std::string_view packe
if (mt == SSR::Msg::S7) {
if (packet.size() != SSR::Mode_S_Msg::S7_len) {
auto packet_hex = mem2hex(std::string(packet));
SSR::mode_s_logger->c_debug(error_len, {},
SSR::mode_s_logger->c_debug(error_len(), {},
std::to_string(mt) + " size:" + std::to_string(packet.size()) +
" hex:" + packet_hex + " should:" + std::to_string(SSR::Msg::S7_len));
mode_s_statistic.add_length_error("[length error] Msg::S7 handle_mode_s_source");
@@ -45,7 +47,7 @@ std::shared_ptr<SSR::Msg> Data_Source_Handler::create_msg(std::string_view packe
if (mt == SSR::Msg::S14) {
if (packet.size() != SSR::Msg::S14_len) {
auto packet_hex = mem2hex(std::string(packet));
SSR::mode_s_logger->c_debug(error_len, {},
SSR::mode_s_logger->c_debug(error_len(), {},
std::to_string(mt) + " size:" + std::to_string(packet.size()) +
" hex:" + packet_hex + " should:" + std::to_string(SSR::Msg::S14_len));
mode_s_statistic.add_length_error("[length error] Msg::S14 handle_mode_s_source");
@@ -57,7 +59,7 @@ std::shared_ptr<SSR::Msg> Data_Source_Handler::create_msg(std::string_view packe
return nullptr; // 不知道如何解析跳过
if (packet.size() != SSR::Msg::Radarcape_status_len) {
auto packet_hex = mem2hex(std::string(packet));
SSR::mode_s_logger->c_debug(error_len, {},
SSR::mode_s_logger->c_debug(error_len(), {},
std::to_string(mt) + " size:" + std::to_string(packet.size()) + " hex:" +
packet_hex + " should:" + std::to_string(SSR::Msg::Radarcape_status_len));
mode_s_statistic.add_length_error("[length error] Msg::Radarcape_status_len handle_mode_s_source");
@@ -70,7 +72,7 @@ std::shared_ptr<SSR::Msg> Data_Source_Handler::create_msg(std::string_view packe
auto packet_hex = mem2hex(std::string(packet));
// 找不到协议 size:23 hex:1A34195F0000001500B502FF06F423CE50000090000000
// should:5
SSR::mode_s_logger->c_debug(error_len, {},
SSR::mode_s_logger->c_debug(error_len(), {},
Psc::to_string(source->type) + " HULC size:" + std::to_string(packet.size()) +
" hex:" + packet_hex);
return nullptr;
@@ -94,14 +96,9 @@ size_t Data_Source_Handler::process_mode_acs_data(std::string_view origin_data)
std::cout << " after origin_data_transform_mode_data " << VAR_STR_2(name, mode_data.size()) << key
<< std::endl;
}
static Value_Growth_Multi_T mt;
auto t = key;
if (data_source_debug) {
std::cout << name << " 读取到:" << VAR_STR_2(key, mode_data.size()) << " " << std::endl;
}
if (mt.test(key, mode_data.size())) {
std::cout << VAR_STR_2(key, mode_data.size()) << " 数据增长过快，可能内存积压" << std::endl;
}
size_t ret = 0;
if (mode_data.empty()) {
return ret;
@@ -120,6 +117,8 @@ size_t Data_Source_Handler::process_mode_acs_data(std::string_view origin_data)
}
auto handle_packet = [this, source, &ret](std::string& packet) {
ret++;
if (source->discard_input_packet(packet))
return;
auto msg = create_msg(packet);
if (!msg)
return;
@@ -145,30 +144,14 @@ size_t Data_Source_Handler::process_mode_acs_data(std::string_view origin_data)
source->push_to_feed(msg);
}
};
auto f = [this, source, handle_packet](std::string& packet) {
const auto wait = Global::instance()->mode_acs.settings.member<&Mode_ACS_Config_Data::wait_process_msg>().read([](const auto& value) { return value; });
if (wait) {
handle_packet(packet);
}
else {
auto co = Coro::instance();
auto executor = co->process_data->get_executor();
asio::post(executor, [handle_packet, packet = std::move(packet)]() mutable { handle_packet(packet); });
}
auto f = [handle_packet](std::string& packet) {
handle_packet(packet);
};
SSR::Binary_Format_handle_buffer(source->buffer, mode_data, f);
return ret;
}
void Data_Source_Handler::handle_mode_s(std::shared_ptr<SSR::Mode_S_Msg>& mode_s_msg) {
auto source = ds(this);
std::string t =
mode_s_msg->mlat_timestamp.to_memory() + mode_s_msg->signal_level + Psc::hex2mem(mode_s_msg->msg_hex);
auto p = mode_s_msg->packet.substr(2);
if (t != p) {
auto data = std::format("编解码出问题了(组合的数据，拼接的数据) hex:[{}] [{}] bin:[{}] [{}]",
mem2hex(t, true, " "), mem2hex(mode_s_msg->packet, true, " "), t, mode_s_msg->packet);
Psc::fail_fast(data);
}
const auto mode_config = Global::instance()->mode_acs.settings.read([](const auto& value) {
return std::tuple{value.time_space_filter, value.speed_filter, value.aircraft_change_list_adsb_range_filter, value.aircraft_change_list_adsb_range_factor, value.max_speed_m_s, value.air_pos_timeout, value.surface_pos_timeout};
});
@@ -181,9 +164,11 @@ void Data_Source_Handler::handle_mode_s(std::shared_ptr<SSR::Mode_S_Msg>& mode_s
if (!base_station_pos && base_station_has_valid_position)
base_station_pos = SSR::Position_3D{lat, lon, alt};
SSR::ADS_B_T::Constraint air_constraint{max_speed_m_s, air_pos_timeout, SSR::cpr_cb, time_space_filter,
speed_filter, use_system_time, base_station_pos, alt, range_filter, range_factor};
speed_filter, use_system_time, base_station_pos, alt, range_filter, range_factor,
false};
SSR::ADS_B_T::Constraint surface_constraint{max_speed_m_s, surface_pos_timeout, SSR::cpr_cb, time_space_filter,
speed_filter, use_system_time, base_station_pos, alt, range_filter, range_factor};
speed_filter, use_system_time, base_station_pos, alt, range_filter, range_factor,
false};
SSR::parse_mode_s_bin(source.get(), mode_s_msg, base_station_pos, air_constraint, surface_constraint);
auto base = source->get_aircraft(mode_s_msg->icao);
if (base) {
@@ -271,7 +256,7 @@ void Data_Source_Handler::refresh_data_feed_key_list() {
std::vector<Cached_Source_Info> next;
for (const auto& feed : Global::instance()->mode_acs.data_feed_config.map.list()) {
if (feed->enabled() && feed->source_key == key)
next.push_back({feed->key});
next.push_back({feed->key, feed});
}
std::lock_guard<std::mutex> lock(cdf_mtx);
for (auto& item : next) {
@@ -287,14 +272,8 @@ void Data_Source_Handler::refresh_data_feed_key_list() {
}
std::optional<std::string> convert_to_send_format(Data_Source_Handler* ds, const std::shared_ptr<Data_Feed>& feed,
const std::shared_ptr<SSR::Msg>& msg) {
const std::shared_ptr<SSR::Msg>& msg) {
auto& type = msg->type;
auto fs = dynamic_cast<File_Data_Source*>(msg->source.get());
if (fs) {
if (fs->specific.member<&File_Data_Source_Data::play_mode>().read([](const auto& value) { return value; }) == Play_Mode::analysis) {
// std::cout << fs->key << " analysis i ==" << i << std::endl;
}
}
// 出口输出的条件
const auto feed_format = feed->output_format.read([](const auto& value) { return value; });
const auto output_format = feed_format.type;
@@ -409,20 +388,8 @@ void Data_Source_Handler::push_to_feed(const std::shared_ptr<SSR::Msg>& msg) {
const auto [other_size, s_size] = Global::instance()->mode_acs.settings.read([](const auto& value) {
return std::pair{value.mode_other_max_num, value.mode_s_max_num};
});
std::vector<std::string> feed_keys;
{
std::lock_guard lock(cdf_mtx);
feed_keys.reserve(cached_data_feed_key_list.size());
for (const auto& item : cached_data_feed_key_list)
feed_keys.push_back(item.key);
}
for (const auto& key : feed_keys) {
auto opt_feed = Global::instance()->mode_acs.data_feed_config.map.get(key);
if (!opt_feed.has_value()) {
std::cout << "wrong data feed: " << key << std::endl;
break;
}
std::shared_ptr<Data_Feed>& feed = opt_feed.value();
for (const auto& item : cached_data_feed_key_list) {
const auto& feed = item.feed;
if (!feed->enabled())
continue;
const bool mode_s = msg->type == SSR::Mode_Msg::T::S7 || msg->type == SSR::Mode_Msg::T::S14;
module/Local_Server/Data_Source/Data_Source_Handler.h
+2
View File
@@ -10,6 +10,7 @@
#include "Psc_Cpp_Core/Statistics/Statistics.h"
#include "Psc_Cpp_Core/transmit_protocol/core/statistics.h"
#include "global.h"
class Data_Feed;
// 这个函数专门装cpu密集函数 放到线程池去处理
class Data_Source_Handler : public DataBase {
public:
@@ -28,6 +29,7 @@ public:
void refresh_data_feed_key_list();
struct Cached_Source_Info {
std::string key;
std::shared_ptr<Data_Feed> feed;
Psc::Value_Statistics mode_s_cache_num;
Psc::Value_Statistics mode_other_cache_num;
};
module/Local_Server/Data_Source/Database.cpp
+7 -9
View File
@@ -417,17 +417,15 @@ DataBase::create_aircraft(std::string_view icao)
return ret;
}
void DataBase::delete_timeout_aircraft()
{
aircraft_map.remove_cond([](const std::shared_ptr<SSR::Aircraft_Info>& item)
{
if (item->timestamp == 0)
{
void DataBase::delete_timeout_aircraft() {
const auto now = std::time(nullptr);
const auto timeout_seconds = Global::instance()->mode_acs.settings.member<&Mode_ACS_Config_Data::timeout_seconds>().read(
[](const auto& value) { return value; });
aircraft_map.remove_cond([now, timeout_seconds](const std::shared_ptr<SSR::Aircraft_Info>& item) {
if (item->timestamp == 0) {
std::cout << "未初始化的 timestamp " << LOG_POS << std::endl;
}
long long time = std::time(nullptr) - item->timestamp;
return time > Global::instance()->mode_acs.settings.member<&Mode_ACS_Config_Data::timeout_seconds>().read(
[](const auto& value) { return value; });
return now - item->timestamp > timeout_seconds;
});
}