ae/ECAP_Server
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

加协程

Browse Source
This commit is contained in:
种太阳的热带美短
2026-06-22 17:48:50 +08:00
parent 068ef188e7
commit 42d1cd17c1
4 changed files with 606 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/Local_Server/External_Database/External_Database.cpp
+489 -128
View File
@@ -8,44 +8,103 @@
#include <SQLiteCpp/Statement.h>
#include <SQLiteCpp/Transaction.h>
#include <drogon/MultiPart.h>
#include <trantor/utils/ConcurrentTaskQueue.h>
#include <chrono>
#include <iostream>
#include <set>
#include <stdexcept>
#include <utility>
#include <variant>
namespace {
Psc::JSON external_resource_status_to_json(const External_Resource_Status& status) {
return Psc::JSON::object({
{"name", status.name},
{"row_count", status.row_count},
{"downloaded", status.downloaded},
{"imported", status.imported},
{"message", status.message},
});
}
Psc::JSON external_resource_status_list_to_json(
const std::vector<External_Resource_Status>& status_list) {
auto result = Psc::JSON::array();
for (const auto& status : status_list) {
result.append(external_resource_status_to_json(status));
namespace
{
Psc::JSON external_resource_status_to_json(const External_Resource_Status& status)
{
return Psc::JSON::object({
{"name", status.name},
{"row_count", status.row_count},
{"downloaded", status.downloaded},
{"imported", status.imported},
{"message", status.message},
});
}
return result;
}
} // 命名空间
Psc::JSON external_resource_status_list_to_json(
const std::vector<External_Resource_Status>& status_list)
{
auto result = Psc::JSON::array();
for (const auto& status : status_list)
{
result.append(external_resource_status_to_json(status));
}
return result;
}
std::optional<std::string> External_Database_Row::get(const std::string& column) const {
trantor::ConcurrentTaskQueue& external_database_task_queue()
{
static trantor::ConcurrentTaskQueue queue(2, "external_database");
return queue;
}
template <typename T, typename Work, typename Callback>
void run_external_database_async(Work&& work, Callback&& callback)
{
auto callback_holder = std::make_shared<std::decay_t<Callback>>(std::forward<Callback>(callback));
external_database_task_queue().runTaskInQueue(
[work = std::forward<Work>(work), callback_holder]() mutable
{
try
{
(*callback_holder)(nullptr, work());
}
catch (...)
{
(*callback_holder)(std::current_exception(), T{});
}
});
}
template <typename T, typename Starter>
ucoro::awaitable<T> await_external_database_callback(Starter starter)
{
using Result = ucoro::traits::exception_with_result_t<T>;
auto result = co_await ucoro::callback_awaitable<Result>(
[starter = std::move(starter)](auto handler) mutable
{
starter([handler = std::move(handler)](std::exception_ptr exception,
T value) mutable
{
if (exception)
{
handler(Result{exception});
return;
}
handler(Result{std::move(value)});
});
});
if (std::holds_alternative<std::exception_ptr>(result))
{
std::rethrow_exception(std::get<std::exception_ptr>(result));
}
co_return std::move(std::get<T>(result));
}
} // 命名空间
std::optional<std::string> External_Database_Row::get(const std::string& column) const
{
const auto iter = columns.find(column);
return iter == columns.end() ? std::nullopt : iter->second;
}
Psc::JSON External_Database_Row::to_json() const {
Psc::JSON External_Database_Row::to_json() const
{
auto result = Psc::JSON::object();
for (const auto& [column, value] : columns) {
for (const auto& [column, value] : columns)
{
result.append({column, value ? Psc::JSON(*value) : Psc::JSON(nullptr)});
}
return result;
@@ -60,34 +119,37 @@ External_Resources::External_Resources(std::string name,
cache_file_name(std::move(cache_file_name)),
download_url(std::move(download_url)),
description(std::move(description)),
update_interval_seconds(update_interval_seconds) {}
update_interval_seconds(update_interval_seconds)
{
}
void External_Resources::mark_updated() {
void External_Resources::mark_updated()
{
last_updated_at = std::time(nullptr);
}
void External_Resources::clear_last_updated_at() {
void External_Resources::clear_last_updated_at()
{
last_updated_at = 0;
}
std::size_t External_Resources::row_count(SQLite::Database& db) const {
std::size_t External_Resources::row_count(SQLite::Database& db) const
{
return static_cast<std::size_t>(
db.execAndGet("SELECT COUNT(*) FROM " +
External_Database_Utils::quote_identifier(name)).getInt64());
External_Database_Utils::quote_identifier(name)).getInt64());
}
std::filesystem::path External_Resources::source_file(
const std::filesystem::path& cache_pos) const {
const std::filesystem::path& cache_pos) const
{
return cache_pos / cache_file_name;
}
bool External_Resources::fetch(const std::filesystem::path& cache_pos,
const bool force_download) const {
const bool force_download) const
{
const auto target_file = source_file(cache_pos);
if (!force_download) return false;
if (download_url.empty()) return false;
@@ -99,24 +161,29 @@ External_Resource_Status External_Resources::update(
SQLite::Database& db,
const std::filesystem::path& cache_pos,
const bool force_download,
const bool only_when_empty) {
const bool only_when_empty)
{
create_table(db);
External_Resource_Status result{name, row_count(db), false, false, ""};
if (only_when_empty && result.row_count != 0) {
if (only_when_empty && result.row_count != 0)
{
result.message = "table already populated";
return result;
}
std::filesystem::create_directories(cache_pos);
result.downloaded = fetch(cache_pos, force_download);
if (cache_file_name.empty()) {
if (cache_file_name.empty())
{
result.message = "dynamic query required";
return result;
}
const auto local_file = source_file(cache_pos);
if (!std::filesystem::exists(local_file)) {
result.message = download_url.empty() ? "offline import or dynamic query required"
: "cache file unavailable";
if (!std::filesystem::exists(local_file))
{
result.message = download_url.empty()
? "offline import or dynamic query required"
: "cache file unavailable";
return result;
}
@@ -129,7 +196,8 @@ External_Resource_Status External_Resources::update(
External_Resource_Status External_Resources::import_offline(
SQLite::Database& db,
const std::filesystem::path& source_file) {
const std::filesystem::path& source_file)
{
create_table(db);
External_Resource_Status result{name, 0, false, false, ""};
result.row_count = import_file(db, source_file);
@@ -141,19 +209,22 @@ External_Resource_Status External_Resources::import_offline(
External_Resource_Status External_Resources::clear_table(
SQLite::Database& db,
const std::filesystem::path& cache_pos) {
const std::filesystem::path& cache_pos)
{
SQLite::Transaction transaction(db);
db.exec("DROP TABLE IF EXISTS " +
External_Database_Utils::quote_identifier(name));
External_Database_Utils::quote_identifier(name));
create_table(db);
transaction.commit();
if (!cache_file_name.empty()) {
if (!cache_file_name.empty())
{
std::error_code error;
std::filesystem::remove(source_file(cache_pos), error);
if (error) {
if (error)
{
throw std::runtime_error("Cannot remove cache file for resource " + name +
": " + error.message());
": " + error.message());
}
}
@@ -163,41 +234,50 @@ External_Resource_Status External_Resources::clear_table(
std::optional<External_Database_Row> External_Resources::query_one(
SQLite::Database& db,
const std::vector<std::string>& primary_key_values) const {
const std::vector<std::string>& primary_key_values) const
{
const auto& columns = primary_key_columns();
if (columns.size() != primary_key_values.size()) {
if (columns.size() != primary_key_values.size())
{
throw std::invalid_argument("primary key value count mismatch for resource: " + name);
}
std::string sql = "SELECT * FROM " +
External_Database_Utils::quote_identifier(name) + " WHERE ";
for (std::size_t index = 0; index < columns.size(); ++index) {
External_Database_Utils::quote_identifier(name) + " WHERE ";
for (std::size_t index = 0; index < columns.size(); ++index)
{
if (index != 0) sql += " AND ";
sql += External_Database_Utils::quote_identifier(columns[index]) +
" = ? COLLATE NOCASE";
" = ? COLLATE NOCASE";
}
sql += " LIMIT 1";
SQLite::Statement statement(db, sql);
for (std::size_t index = 0; index < primary_key_values.size(); ++index) {
for (std::size_t index = 0; index < primary_key_values.size(); ++index)
{
statement.bind(static_cast<int>(index + 1),
External_Database_Utils::normalize_key(primary_key_values[index]));
}
if (!statement.executeStep()) return std::nullopt;
External_Database_Row row;
for (int index = 0; index < statement.getColumnCount(); ++index) {
for (int index = 0; index < statement.getColumnCount(); ++index)
{
const auto name = statement.getColumnName(index);
if (statement.isColumnNull(index)) {
if (statement.isColumnNull(index))
{
row.columns.emplace(name, std::nullopt);
} else {
}
else
{
row.columns.emplace(name, statement.getColumn(index).getString());
}
}
return row;
}
External_Resources_Manager::External_Resources_Manager() {
External_Resources_Manager::External_Resources_Manager()
{
register_external_resource(make_tar1090_db_aircraft_csv_gz_resource());
register_external_resource(make_wiedehopf_tar1090_db_resource());
register_external_resource(make_mictronics_aircraft_database_resource());
@@ -211,15 +291,18 @@ External_Resources_Manager::External_Resources_Manager() {
register_external_resource(make_opensky_flightdata_api_resource());
}
void External_Resources_Manager::init(const Psc::JSON* that_json) {
void External_Resources_Manager::init(const Psc::JSON* that_json)
{
if (!that_json) return;
const auto list = that_json->get("list");
if (!list) return;
std::lock_guard lock(mtx_);
for (const auto& config : list->children) {
for (const auto& config : list->children)
{
const auto name = config.try_get_string("name");
if (!name) continue;
for (const auto& resource : resources_) {
for (const auto& resource : resources_)
{
if (resource->name != *name) continue;
resource->init(&config);
break;
@@ -228,23 +311,29 @@ void External_Resources_Manager::init(const Psc::JSON* that_json) {
Get_J(ecap_sqlite_path)
}
Psc::JSON External_Resources_Manager::to_json() const {
Psc::JSON External_Resources_Manager::to_json() const
{
std::lock_guard lock(mtx_);
auto ret = Psc::JSON::object();
Ret_J(ecap_sqlite_path)
auto list = Psc::JSON::array();
for (const std::unique_ptr<External_Resources>& resource : resources_) {
for (const std::unique_ptr<External_Resources>& resource : resources_)
{
list.children.emplace_back(resource->to_json());
}
ret.children.emplace_back(Psc::JSON{"list", list});
return ret;
}
void External_Resources_Manager::load_sqlite_db(std::string path) {
try {
void External_Resources_Manager::load_sqlite_db(std::string path)
{
try
{
db_ = std::make_unique<SQLite::Database>(
path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE);
} catch (SQLite::Exception& error) {
}
catch (SQLite::Exception& error)
{
Psc::fail_fast(std::string("load_sqlite_db error: ") + error.what());
}
@@ -255,167 +344,415 @@ void External_Resources_Manager::load_sqlite_db(std::string path) {
}
std::vector<External_Resource_Status>
External_Resources_Manager::refresh_external_databases(const bool force_download) {
External_Resources_Manager::refresh_external_databases(const bool force_download)
{
return refresh_all(require_db(), force_download);
}
External_Resource_Status External_Resources_Manager::refresh_external_database(
const std::string& resource_name,
const bool force_download) {
const bool force_download)
{
return refresh_one(require_db(), resource_name, force_download);
}
External_Resource_Status External_Resources_Manager::import_external_database(
const std::string& resource_name,
const std::filesystem::path& source_file) {
const std::filesystem::path& source_file)
{
return import_offline(require_db(), resource_name, source_file);
}
External_Resource_Status External_Resources_Manager::clear_external_database_table(
const std::string& resource_name) {
const std::string& resource_name)
{
return clear_table(require_db(), resource_name);
}
std::optional<External_Database_Row>
External_Resources_Manager::query_external_database(
const std::string& resource_name,
const std::vector<std::string>& primary_key_values) const {
const std::vector<std::string>& primary_key_values) const
{
return query_one(require_db(), resource_name, primary_key_values);
}
std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_aircraft_external_databases(
const std::string& icao24) const {
const std::string& icao24) const
{
return query_aircraft(require_db(), icao24);
}
std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_callsign_external_databases(
const std::optional<std::string>& callsign) const {
const std::optional<std::string>& callsign) const
{
return query_callsign(require_db(), callsign);
}
std::vector<External_Resource_Status>
External_Resources_Manager::external_database_status() const {
External_Resources_Manager::external_database_status() const
{
return status(require_db());
}
void External_Resources_Manager::server(Global* g) {
void External_Resources_Manager::async_refresh_external_databases(const bool force_download,
Status_List_Callback callback)
{
run_external_database_async<std::vector<External_Resource_Status>>(
[this, force_download]() { return refresh_external_databases(force_download); },
std::move(callback));
}
void External_Resources_Manager::async_refresh_external_database(
std::string resource_name,
const bool force_download,
Status_Callback callback)
{
run_external_database_async<External_Resource_Status>(
[this, resource_name = std::move(resource_name), force_download]()
{
return refresh_external_database(resource_name, force_download);
},
std::move(callback));
}
void External_Resources_Manager::async_import_external_database(
std::string resource_name,
std::filesystem::path source_file,
Status_Callback callback)
{
run_external_database_async<External_Resource_Status>(
[this,
resource_name = std::move(resource_name),
source_file = std::move(source_file)]()
{
return import_external_database(resource_name, source_file);
},
std::move(callback));
}
void External_Resources_Manager::async_clear_external_database_table(
std::string resource_name,
Status_Callback callback)
{
run_external_database_async<External_Resource_Status>(
[this, resource_name = std::move(resource_name)]()
{
return clear_external_database_table(resource_name);
},
std::move(callback));
}
void External_Resources_Manager::async_query_external_database(
std::string resource_name,
std::vector<std::string> primary_key_values,
Row_Callback callback) const
{
run_external_database_async<std::optional<External_Database_Row>>(
[this,
resource_name = std::move(resource_name),
primary_key_values = std::move(primary_key_values)]()
{
return query_external_database(resource_name, primary_key_values);
},
std::move(callback));
}
void External_Resources_Manager::async_query_aircraft_external_databases(
std::string icao24,
Row_Map_Callback callback) const
{
run_external_database_async<
std::map<std::string, std::shared_ptr<External_Database_Row>>>(
[this, icao24 = std::move(icao24)]()
{
return query_aircraft_external_databases(icao24);
},
std::move(callback));
}
void External_Resources_Manager::async_query_callsign_external_databases(
std::optional<std::string> callsign,
Row_Map_Callback callback) const
{
run_external_database_async<
std::map<std::string, std::shared_ptr<External_Database_Row>>>(
[this, callsign = std::move(callsign)]()
{
return query_callsign_external_databases(callsign);
},
std::move(callback));
}
void External_Resources_Manager::async_external_database_status(
Status_List_Callback callback) const
{
run_external_database_async<std::vector<External_Resource_Status>>(
[this]() { return external_database_status(); },
std::move(callback));
}
ucoro::awaitable<std::vector<External_Resource_Status>>
External_Resources_Manager::refresh_external_databases_coro(
const bool force_download)
{
co_return co_await await_external_database_callback<
std::vector<External_Resource_Status>>(
[this, force_download](Status_List_Callback callback)
{
async_refresh_external_databases(force_download, std::move(callback));
});
}
ucoro::awaitable<External_Resource_Status>
External_Resources_Manager::refresh_external_database_coro(
std::string resource_name,
const bool force_download)
{
co_return co_await await_external_database_callback<External_Resource_Status>(
[this,
resource_name = std::move(resource_name),
force_download](Status_Callback callback) mutable
{
async_refresh_external_database(
std::move(resource_name), force_download, std::move(callback));
});
}
ucoro::awaitable<External_Resource_Status>
External_Resources_Manager::import_external_database_coro(
std::string resource_name,
std::filesystem::path source_file)
{
co_return co_await await_external_database_callback<External_Resource_Status>(
[this,
resource_name = std::move(resource_name),
source_file = std::move(source_file)](Status_Callback callback) mutable
{
async_import_external_database(
std::move(resource_name), std::move(source_file), std::move(callback));
});
}
ucoro::awaitable<External_Resource_Status>
External_Resources_Manager::clear_external_database_table_coro(
std::string resource_name)
{
co_return co_await await_external_database_callback<External_Resource_Status>(
[this, resource_name = std::move(resource_name)](Status_Callback callback) mutable
{
async_clear_external_database_table(
std::move(resource_name), std::move(callback));
});
}
ucoro::awaitable<std::optional<External_Database_Row>>
External_Resources_Manager::query_external_database_coro(
std::string resource_name,
std::vector<std::string> primary_key_values) const
{
co_return co_await await_external_database_callback<
std::optional<External_Database_Row>>(
[this,
resource_name = std::move(resource_name),
primary_key_values = std::move(primary_key_values)](Row_Callback callback) mutable
{
async_query_external_database(
std::move(resource_name),
std::move(primary_key_values),
std::move(callback));
});
}
ucoro::awaitable<std::map<std::string, std::shared_ptr<External_Database_Row>>>
External_Resources_Manager::query_aircraft_external_databases_coro(
std::string icao24) const
{
co_return co_await await_external_database_callback<
std::map<std::string, std::shared_ptr<External_Database_Row>>>(
[this, icao24 = std::move(icao24)](Row_Map_Callback callback) mutable
{
async_query_aircraft_external_databases(
std::move(icao24), std::move(callback));
});
}
ucoro::awaitable<std::map<std::string, std::shared_ptr<External_Database_Row>>>
External_Resources_Manager::query_callsign_external_databases_coro(
std::optional<std::string> callsign) const
{
co_return co_await await_external_database_callback<
std::map<std::string, std::shared_ptr<External_Database_Row>>>(
[this, callsign = std::move(callsign)](Row_Map_Callback callback) mutable
{
async_query_callsign_external_databases(
std::move(callsign), std::move(callback));
});
}
ucoro::awaitable<std::vector<External_Resource_Status>>
External_Resources_Manager::external_database_status_coro() const
{
co_return co_await await_external_database_callback<
std::vector<External_Resource_Status>>(
[this](Status_List_Callback callback)
{
async_external_database_status(std::move(callback));
});
}
void External_Resources_Manager::server(Global* g)
{
auto& svr = g->svr;
const auto& api = g->api;
svr.Post(api + "get_external_database_config", [this](HTTP_Param) {
svr.Post(api + "get_external_database_config", [this](HTTP_Param)
{
auto t = warp(to_json()).to_json_string();
res->setBody(t);
});
svr.Post(api + "get_external_database_status", [this](HTTP_Param) {
svr.PostCoro(api + "get_external_database_status", [this](HTTP_Param) -> drogon::Task<>
{
auto status_list =
co_await Ecap_Coro::to_drogon(external_database_status_coro());
res->setBody(
warp(external_resource_status_list_to_json(external_database_status()))
.to_json_string());
warp(external_resource_status_list_to_json(status_list))
.to_json_string());
co_return;
});
svr.Post(api + "refresh_external_database", [this, g](HTTP_Param) {
svr.PostCoro(api + "refresh_external_database", [this, g](HTTP_Param) -> drogon::Task<>
{
CHECK_JSON_PARAM
HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
const auto result = refresh_external_database(name, true);
const auto result =
co_await Ecap_Coro::to_drogon(refresh_external_database_coro(name, true));
g->save();
res->setBody(warp(external_resource_status_to_json(result)).to_json_string());
co_return;
});
svr.Post(api + "refresh_external_databases", [this, g](HTTP_Param) {
const auto result = refresh_external_databases(true);
svr.PostCoro(api + "refresh_external_databases", [this, g](HTTP_Param) -> drogon::Task<>
{
const auto result =
co_await Ecap_Coro::to_drogon(refresh_external_databases_coro(true));
g->save();
res->setBody(warp(external_resource_status_list_to_json(result)).to_json_string());
co_return;
});
svr.Post(api + "import_external_database", [this, g](HTTP_Param) {
svr.PostCoro(api + "import_external_database", [this, g](HTTP_Param) -> drogon::Task<>
{
CHECK_JSON_PARAM
HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
HTTP_REQUIRE_VALUE(source_file, params.try_get_string("source_file"))
const auto result = import_external_database(name, source_file);
const auto result =
co_await Ecap_Coro::to_drogon(import_external_database_coro(name, source_file));
g->save();
res->setBody(warp(external_resource_status_to_json(result)).to_json_string());
co_return;
});
svr.Post(api + "upload_external_database", [this, g](HTTP_Param) {
svr.PostCoro(api + "upload_external_database", [this, g](HTTP_Param) -> drogon::Task<>
{
drogon::MultiPartParser parser;
if (parser.parse(req) != 0) {
if (parser.parse(req) != 0)
{
throw_invalid_http_param("multipart");
}
const auto name = parser.getOptionalParameter<std::string>("name");
if (!name || parser.getFiles().size() != 1) {
if (!name || parser.getFiles().size() != 1)
{
throw_invalid_http_param("name or file");
}
const auto& upload = parser.getFiles().front();
const auto serial = std::chrono::steady_clock::now().time_since_epoch().count();
auto extension = std::string(upload.getFileExtension());
if (!extension.empty() && extension.front() != '.') {
if (!extension.empty() && extension.front() != '.')
{
extension.insert(extension.begin(), '.');
}
const auto saved_name =
"external_database_" + std::to_string(serial) + extension;
if (upload.saveAs(saved_name) != 0) {
if (upload.saveAs(saved_name) != 0)
{
throw std::runtime_error("Cannot save uploaded external database file");
}
const auto source_file =
std::filesystem::path(drogon::app().getUploadPath()) / saved_name;
const auto result = import_external_database(*name, source_file);
const auto result = co_await Ecap_Coro::to_drogon(
import_external_database_coro(*name, source_file));
g->save();
res->setBody(warp(external_resource_status_to_json(result)).to_json_string());
co_return;
});
svr.Post(api + "clear_external_database_table", [this, g](HTTP_Param) {
svr.PostCoro(api + "clear_external_database_table", [this, g](HTTP_Param) -> drogon::Task<>
{
CHECK_JSON_PARAM
HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
const auto result = clear_external_database_table(name);
const auto result =
co_await Ecap_Coro::to_drogon(clear_external_database_table_coro(name));
g->save();
res->setBody(warp(external_resource_status_to_json(result)).to_json_string());
co_return;
});
svr.Post(api + "query_external_database", [this](HTTP_Param) {
svr.PostCoro(api + "query_external_database", [this](HTTP_Param) -> drogon::Task<>
{
CHECK_JSON_PARAM
HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
HTTP_REQUIRE_PTR(primary_key_values_json, params.get("primary_key_values"))
std::vector<std::string> primary_key_values;
for (const auto& value : primary_key_values_json->children) {
if (value.valueType != Psc::JsonType::String) {
for (const auto& value : primary_key_values_json->children)
{
if (value.valueType != Psc::JsonType::String)
{
throw_invalid_http_param("primary_key_values");
}
primary_key_values.push_back(value.val);
}
const auto row = query_external_database(name, primary_key_values);
res->setBody(row ? warp(row->to_json()).to_json_string()
const auto row = co_await Ecap_Coro::to_drogon(
query_external_database_coro(name, std::move(primary_key_values)));
res->setBody(row
? warp(row->to_json()).to_json_string()
: Psc::JSON(nullptr).to_json_string());
co_return;
});
}
void External_Resources_Manager::register_external_resource(
std::unique_ptr<External_Resources> external_res) {
std::unique_ptr<External_Resources> external_res)
{
if (!external_res) throw std::invalid_argument("external resource is null");
resources_.push_back(std::move(external_res));
}
void External_Resources_Manager::initialize(SQLite::Database& db,
std::filesystem::path cache_pos) {
std::filesystem::path cache_pos)
{
std::lock_guard lock(mtx_);
cache_pos_ = std::move(cache_pos);
for (const auto& resource : resources_) resource->create_table(db);
}
std::vector<External_Resource_Status> External_Resources_Manager::sync_missing(
SQLite::Database& db) {
SQLite::Database& db)
{
return update_all(db, false, true);
}
std::vector<External_Resource_Status> External_Resources_Manager::refresh_all(
SQLite::Database& db,
const bool force_download) {
const bool force_download)
{
return update_all(db, force_download, false);
}
External_Resource_Status External_Resources_Manager::refresh_one(
SQLite::Database& db,
const std::string& resource_name,
const bool force_download) {
const bool force_download)
{
std::lock_guard lock(mtx_);
return get_resource(resource_name).update(db, cache_pos_, force_download, false);
}
@@ -423,14 +760,16 @@ External_Resource_Status External_Resources_Manager::refresh_one(
External_Resource_Status External_Resources_Manager::import_offline(
SQLite::Database& db,
const std::string& resource_name,
const std::filesystem::path& source_file) {
const std::filesystem::path& source_file)
{
std::lock_guard lock(mtx_);
return get_resource(resource_name).import_offline(db, source_file);
}
External_Resource_Status External_Resources_Manager::clear_table(
SQLite::Database& db,
const std::string& resource_name) {
const std::string& resource_name)
{
std::lock_guard lock(mtx_);
return get_resource(resource_name).clear_table(db, cache_pos_);
}
@@ -438,14 +777,16 @@ External_Resource_Status External_Resources_Manager::clear_table(
std::optional<External_Database_Row> External_Resources_Manager::query_one(
SQLite::Database& db,
const std::string& resource_name,
const std::vector<std::string>& primary_key_values) const {
const std::vector<std::string>& primary_key_values) const
{
std::lock_guard lock(mtx_);
return get_resource(resource_name).query_one(db, primary_key_values);
}
std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_aircraft(SQLite::Database& db,
const std::string& icao24) const {
const std::string& icao24) const
{
Scope_Timer timer(
"external_database.query_aircraft",
Global::instance()->http_monitor_config.slow_scope_threshold_ms);
@@ -460,7 +801,8 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db,
std::map<std::string, std::shared_ptr<External_Database_Row>> result;
const auto add_row = [&](const std::string& resource_name,
const std::vector<std::string>& primary_key_values,
const std::string& result_name = std::string{}) {
const std::string& result_name = std::string{})
{
auto& resource = get_resource(resource_name);
auto row = resource.query_one(db, primary_key_values);
if (!row) return std::shared_ptr<External_Database_Row>{};
@@ -470,9 +812,11 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db,
};
std::optional<std::string> type_code;
for (const auto& resource_name : kAircraftResourceKeys) {
for (const auto& resource_name : kAircraftResourceKeys)
{
const auto row = add_row(resource_name, {icao24});
if (row && !type_code) {
if (row && !type_code)
{
type_code = row->get("type_code");
if (!type_code) type_code = row->get("typecode");
}
@@ -485,7 +829,8 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db,
std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_callsign(
SQLite::Database& db,
const std::optional<std::string>& callsign) const {
const std::optional<std::string>& callsign) const
{
Scope_Timer timer(
"external_database.query_callsign",
Global::instance()->http_monitor_config.slow_scope_threshold_ms);
@@ -499,7 +844,8 @@ External_Resources_Manager::query_callsign(
std::map<std::string, std::shared_ptr<External_Database_Row>> result;
const auto add_row = [&](const std::string& resource_name,
const std::vector<std::string>& primary_key_values,
const std::string& result_name = std::string{}) {
const std::string& result_name = std::string{})
{
auto& resource = get_resource(resource_name);
auto row = resource.query_one(db, primary_key_values);
if (!row) return std::shared_ptr<External_Database_Row>{};
@@ -509,15 +855,18 @@ External_Resources_Manager::query_callsign(
};
std::set<std::string> airport_codes;
if (callsign && !callsign->empty()) {
for (const auto& resource_name : kRouteResourceKeys) {
if (callsign && !callsign->empty())
{
for (const auto& resource_name : kRouteResourceKeys)
{
const auto row = add_row(resource_name, {*callsign});
if (!row) continue;
const auto airports = row->get("AirportCodes");
if (!airports) continue;
std::size_t start = 0;
while (start < airports->size()) {
while (start < airports->size())
{
const auto end = airports->find('-', start);
airport_codes.emplace(airports->substr(start, end - start));
if (end == std::string::npos) break;
@@ -526,7 +875,8 @@ External_Resources_Manager::query_callsign(
}
}
for (const auto& airport_code : airport_codes) {
for (const auto& airport_code : airport_codes)
{
if (airport_code.empty()) continue;
add_row("vrs_airports", {airport_code}, "vrs_airports:" + airport_code);
}
@@ -534,25 +884,31 @@ External_Resources_Manager::query_callsign(
}
std::vector<External_Resource_Status> External_Resources_Manager::status(
SQLite::Database& db) const {
SQLite::Database& db) const
{
std::lock_guard lock(mtx_);
std::vector<External_Resource_Status> result;
for (const auto& resource : resources_) {
for (const auto& resource : resources_)
{
result.push_back({resource->name, resource->row_count(db), false, false, ""});
}
return result;
}
SQLite::Database& External_Resources_Manager::require_db() const {
if (!db_) {
SQLite::Database& External_Resources_Manager::require_db() const
{
if (!db_)
{
throw std::runtime_error("DB not loaded. Call load_sqlite_db() first.");
}
return *db_;
}
External_Resources& External_Resources_Manager::get_resource(
const std::string& resource_name) const {
for (const auto& resource : resources_) {
const std::string& resource_name) const
{
for (const auto& resource : resources_)
{
if (resource->name == resource_name) return *resource;
}
throw std::invalid_argument("unknown external resource: " + resource_name);
@@ -561,18 +917,23 @@ External_Resources& External_Resources_Manager::get_resource(
std::vector<External_Resource_Status> External_Resources_Manager::update_all(
SQLite::Database& db,
const bool force_download,
const bool only_when_empty) {
const bool only_when_empty)
{
std::lock_guard lock(mtx_);
std::vector<External_Resource_Status> result;
for (const auto& resource : resources_) {
try {
for (const auto& resource : resources_)
{
try
{
auto status = resource->update(db, cache_pos_, force_download, only_when_empty);
std::cout << "External database [" << status.name << "] "
<< status.message << ", rows=" << status.row_count << std::endl;
<< status.message << ", rows=" << status.row_count << std::endl;
result.push_back(std::move(status));
} catch (const std::exception& error) {
}
catch (const std::exception& error)
{
std::cerr << "External database [" << resource->name
<< "] update failed: " << error.what() << std::endl;
<< "] update failed: " << error.what() << std::endl;
result.push_back(
{resource->name, resource->row_count(db), false, false, error.what()});
}
module/Local_Server/External_Database/global.h
+49
View File
@@ -6,13 +6,16 @@
#include <cstddef>
#include <cstdint>
#include <ctime>
#include <exception>
#include <filesystem>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <vector>
#include <ucoro/awaitable.hpp>
class Global;
@@ -98,6 +101,13 @@ protected:
class External_Resources_Manager {
public:
using Status_Callback = std::function<void(std::exception_ptr, External_Resource_Status)>;
using Status_List_Callback = std::function<void(std::exception_ptr, std::vector<External_Resource_Status>)>;
using Row_Callback = std::function<void(std::exception_ptr, std::optional<External_Database_Row>)>;
using Row_Map_Callback = std::function<void(
std::exception_ptr,
std::map<std::string, std::shared_ptr<External_Database_Row>>)>;
External_Resources_Manager();
void init(const Psc::JSON* that_json);
@@ -121,6 +131,45 @@ public:
const std::optional<std::string>& callsign) const;
[[nodiscard]] std::vector<External_Resource_Status> external_database_status() const;
void async_refresh_external_databases(bool force_download,
Status_List_Callback callback);
void async_refresh_external_database(std::string resource_name,
bool force_download,
Status_Callback callback);
void async_import_external_database(std::string resource_name,
std::filesystem::path source_file,
Status_Callback callback);
void async_clear_external_database_table(std::string resource_name,
Status_Callback callback);
void async_query_external_database(std::string resource_name,
std::vector<std::string> primary_key_values,
Row_Callback callback) const;
void async_query_aircraft_external_databases(std::string icao24,
Row_Map_Callback callback) const;
void async_query_callsign_external_databases(std::optional<std::string> callsign,
Row_Map_Callback callback) const;
void async_external_database_status(Status_List_Callback callback) const;
[[nodiscard]] ucoro::awaitable<std::vector<External_Resource_Status>>
refresh_external_databases_coro(bool force_download = true);
[[nodiscard]] ucoro::awaitable<External_Resource_Status>
refresh_external_database_coro(std::string resource_name,
bool force_download = true);
[[nodiscard]] ucoro::awaitable<External_Resource_Status>
import_external_database_coro(std::string resource_name,
std::filesystem::path source_file);
[[nodiscard]] ucoro::awaitable<External_Resource_Status>
clear_external_database_table_coro(std::string resource_name);
[[nodiscard]] ucoro::awaitable<std::optional<External_Database_Row>>
query_external_database_coro(std::string resource_name,
std::vector<std::string> primary_key_values) const;
[[nodiscard]] ucoro::awaitable<std::map<std::string, std::shared_ptr<External_Database_Row>>>
query_aircraft_external_databases_coro(std::string icao24) const;
[[nodiscard]] ucoro::awaitable<std::map<std::string, std::shared_ptr<External_Database_Row>>>
query_callsign_external_databases_coro(std::optional<std::string> callsign) const;
[[nodiscard]] ucoro::awaitable<std::vector<External_Resource_Status>>
external_database_status_coro() const;
void register_external_resource(std::unique_ptr<External_Resources> external_res);
void initialize(SQLite::Database& db, std::filesystem::path cache_pos);
std::vector<External_Resource_Status> sync_missing(SQLite::Database& db);
module/Local_Server/server/Global.h
+62
View File
@@ -6,6 +6,7 @@
#include "../Data_Source/export.h"
#include "../State_Report/State_Report.h"
#include "../global_include.h"
#include "Ucoro_Drogon_Glue.h"
#include "Config.h"
#include <drogon/drogon.h>
@@ -72,6 +73,7 @@ public:
std::string rise = "rise_";
std::string fall = "fall_";
using Handler = std::function<void(const drogon::HttpRequestPtr&, const drogon::HttpResponsePtr&)>;
using CoroHandler = std::function<drogon::Task<>(const drogon::HttpRequestPtr&, const drogon::HttpResponsePtr&)>;
bool listen(const std::string& host, int port);
~Web_Server() = default;
@@ -103,6 +105,36 @@ public:
return *this;
}
Web_Server& PostCoro(const std::string& pattern, CoroHandler handler) {
drogon::app().registerHandler(
pattern,
[handler = std::move(handler)](
const drogon::HttpRequestPtr& request,
std::function<void(const drogon::HttpResponsePtr&)>&& callback) {
auto resp = drogon::HttpResponse::newHttpResponse();
resp->setContentTypeCode(drogon::CT_APPLICATION_JSON);
drogon::async_run(
[handler, request, resp, callback = std::move(callback)]() mutable -> drogon::Task<> {
try {
co_await handler(request, resp);
} catch (const std::exception& error) {
resp->setStatusCode(drogon::k500InternalServerError);
resp->setBody(Psc::JSON::object({{"error", std::string(error.what())}}).to_json_string());
} catch (...) {
resp->setStatusCode(drogon::k500InternalServerError);
resp->setBody(Psc::JSON::object({{"error", "unknown error"}}).to_json_string());
}
callback(resp);
co_return;
});
},
{drogon::Post}
);
return *this;
}
Web_Server& Get(const std::string& pattern, Handler handler) {
drogon::app().registerHandlerViaRegex(
pattern,
@@ -120,6 +152,36 @@ public:
return *this;
}
Web_Server& GetCoro(const std::string& pattern, CoroHandler handler) {
drogon::app().registerHandlerViaRegex(
pattern,
[handler = std::move(handler)](
const drogon::HttpRequestPtr& request,
std::function<void(const drogon::HttpResponsePtr&)>&& callback) {
auto resp = drogon::HttpResponse::newHttpResponse();
resp->setContentTypeCode(drogon::CT_APPLICATION_JSON);
drogon::async_run(
[handler, request, resp, callback = std::move(callback)]() mutable -> drogon::Task<> {
try {
co_await handler(request, resp);
} catch (const std::exception& error) {
resp->setStatusCode(drogon::k500InternalServerError);
resp->setBody(Psc::JSON::object({{"error", std::string(error.what())}}).to_json_string());
} catch (...) {
resp->setStatusCode(drogon::k500InternalServerError);
resp->setBody(Psc::JSON::object({{"error", "unknown error"}}).to_json_string());
}
callback(resp);
co_return;
});
},
{drogon::Get}
);
return *this;
}
};