diff --git a/main.cmake b/main.cmake index ba36148..0cb4a46 100644 --- a/main.cmake +++ b/main.cmake @@ -15,14 +15,17 @@ message(STATUS "CMAKE_FIND_ROOT_PATH_MODE_PACKAGE=${CMAKE_FIND_ROOT_PATH_MODE_PA find_package(Jsoncpp CONFIG) +list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") +if(MSVC AND NOT TARGET std::filesystem) + add_library(std::filesystem INTERFACE IMPORTED GLOBAL) + set_property(TARGET std::filesystem APPEND PROPERTY INTERFACE_COMPILE_FEATURES cxx_std_17) +endif() find_package(Drogon) set(Core Core_Static) if(1) - - set(md "${CMAKE_CURRENT_LIST_DIR}/module") @@ -71,6 +74,7 @@ if(1) _depend(ecap_server SSR) _attach_include_dir(ecap_server "${md}") _attach_source_dir(ecap_server "${md}/Local_Server") + target_include_directories(ecap_server PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../../core_library/Core/3rd/ucoro/include") target_link_libraries(ecap_server PRIVATE Drogon::Drogon) target_link_libraries(ecap_server PRIVATE SQLiteCpp) target_link_libraries(ecap_server PRIVATE ${ZLIB_LIBRARY}) diff --git a/module/Local_Server/External_Database/External_Database.cpp b/module/Local_Server/External_Database/External_Database.cpp index 151c8fa..f98f349 100644 --- a/module/Local_Server/External_Database/External_Database.cpp +++ b/module/Local_Server/External_Database/External_Database.cpp @@ -8,44 +8,103 @@ #include #include #include +#include #include #include #include #include #include +#include -namespace { - -Psc::JSON external_resource_status_to_json(const External_Resource_Status& status) { - return Psc::JSON::object({ - {"name", status.name}, - {"row_count", status.row_count}, - {"downloaded", status.downloaded}, - {"imported", status.imported}, - {"message", status.message}, - }); -} - -Psc::JSON external_resource_status_list_to_json( - const std::vector& status_list) { - auto result = Psc::JSON::array(); - for (const auto& status : status_list) { - result.append(external_resource_status_to_json(status)); +namespace +{ + Psc::JSON external_resource_status_to_json(const External_Resource_Status& status) + { + return Psc::JSON::object({ + {"name", status.name}, + {"row_count", status.row_count}, + {"downloaded", status.downloaded}, + {"imported", status.imported}, + {"message", status.message}, + }); } - return result; -} -} // 命名空间 + Psc::JSON external_resource_status_list_to_json( + const std::vector& status_list) + { + auto result = Psc::JSON::array(); + for (const auto& status : status_list) + { + result.append(external_resource_status_to_json(status)); + } + return result; + } -std::optional External_Database_Row::get(const std::string& column) const { + trantor::ConcurrentTaskQueue& external_database_task_queue() + { + static trantor::ConcurrentTaskQueue queue(2, "external_database"); + return queue; + } + + template + void run_external_database_async(Work&& work, Callback&& callback) + { + auto callback_holder = std::make_shared>(std::forward(callback)); + + external_database_task_queue().runTaskInQueue( + [work = std::forward(work), callback_holder]() mutable + { + try + { + (*callback_holder)(nullptr, work()); + } + catch (...) + { + (*callback_holder)(std::current_exception(), T{}); + } + }); + } + + template + ucoro::awaitable await_external_database_callback(Starter starter) + { + using Result = ucoro::traits::exception_with_result_t; + auto result = co_await ucoro::callback_awaitable( + [starter = std::move(starter)](auto handler) mutable + { + starter([handler = std::move(handler)](std::exception_ptr exception, + T value) mutable + { + if (exception) + { + handler(Result{exception}); + return; + } + handler(Result{std::move(value)}); + }); + }); + + if (std::holds_alternative(result)) + { + std::rethrow_exception(std::get(result)); + } + + co_return std::move(std::get(result)); + } +} // 命名空间 + +std::optional External_Database_Row::get(const std::string& column) const +{ const auto iter = columns.find(column); return iter == columns.end() ? std::nullopt : iter->second; } -Psc::JSON External_Database_Row::to_json() const { +Psc::JSON External_Database_Row::to_json() const +{ auto result = Psc::JSON::object(); - for (const auto& [column, value] : columns) { + for (const auto& [column, value] : columns) + { result.append({column, value ? Psc::JSON(*value) : Psc::JSON(nullptr)}); } return result; @@ -60,34 +119,37 @@ External_Resources::External_Resources(std::string name, cache_file_name(std::move(cache_file_name)), download_url(std::move(download_url)), description(std::move(description)), - update_interval_seconds(update_interval_seconds) {} + update_interval_seconds(update_interval_seconds) +{ +} - - - - -void External_Resources::mark_updated() { +void External_Resources::mark_updated() +{ last_updated_at = std::time(nullptr); } -void External_Resources::clear_last_updated_at() { +void External_Resources::clear_last_updated_at() +{ last_updated_at = 0; } -std::size_t External_Resources::row_count(SQLite::Database& db) const { +std::size_t External_Resources::row_count(SQLite::Database& db) const +{ return static_cast( db.execAndGet("SELECT COUNT(*) FROM " + - External_Database_Utils::quote_identifier(name)).getInt64()); + External_Database_Utils::quote_identifier(name)).getInt64()); } std::filesystem::path External_Resources::source_file( - const std::filesystem::path& cache_pos) const { + const std::filesystem::path& cache_pos) const +{ return cache_pos / cache_file_name; } bool External_Resources::fetch(const std::filesystem::path& cache_pos, - const bool force_download) const { + const bool force_download) const +{ const auto target_file = source_file(cache_pos); if (!force_download) return false; if (download_url.empty()) return false; @@ -99,24 +161,29 @@ External_Resource_Status External_Resources::update( SQLite::Database& db, const std::filesystem::path& cache_pos, const bool force_download, - const bool only_when_empty) { + const bool only_when_empty) +{ create_table(db); External_Resource_Status result{name, row_count(db), false, false, ""}; - if (only_when_empty && result.row_count != 0) { + if (only_when_empty && result.row_count != 0) + { result.message = "table already populated"; return result; } std::filesystem::create_directories(cache_pos); result.downloaded = fetch(cache_pos, force_download); - if (cache_file_name.empty()) { + if (cache_file_name.empty()) + { result.message = "dynamic query required"; return result; } const auto local_file = source_file(cache_pos); - if (!std::filesystem::exists(local_file)) { - result.message = download_url.empty() ? "offline import or dynamic query required" - : "cache file unavailable"; + if (!std::filesystem::exists(local_file)) + { + result.message = download_url.empty() + ? "offline import or dynamic query required" + : "cache file unavailable"; return result; } @@ -129,7 +196,8 @@ External_Resource_Status External_Resources::update( External_Resource_Status External_Resources::import_offline( SQLite::Database& db, - const std::filesystem::path& source_file) { + const std::filesystem::path& source_file) +{ create_table(db); External_Resource_Status result{name, 0, false, false, ""}; result.row_count = import_file(db, source_file); @@ -141,19 +209,22 @@ External_Resource_Status External_Resources::import_offline( External_Resource_Status External_Resources::clear_table( SQLite::Database& db, - const std::filesystem::path& cache_pos) { + const std::filesystem::path& cache_pos) +{ SQLite::Transaction transaction(db); db.exec("DROP TABLE IF EXISTS " + - External_Database_Utils::quote_identifier(name)); + External_Database_Utils::quote_identifier(name)); create_table(db); transaction.commit(); - if (!cache_file_name.empty()) { + if (!cache_file_name.empty()) + { std::error_code error; std::filesystem::remove(source_file(cache_pos), error); - if (error) { + if (error) + { throw std::runtime_error("Cannot remove cache file for resource " + name + - ": " + error.message()); + ": " + error.message()); } } @@ -163,41 +234,50 @@ External_Resource_Status External_Resources::clear_table( std::optional External_Resources::query_one( SQLite::Database& db, - const std::vector& primary_key_values) const { + const std::vector& primary_key_values) const +{ const auto& columns = primary_key_columns(); - if (columns.size() != primary_key_values.size()) { + if (columns.size() != primary_key_values.size()) + { throw std::invalid_argument("primary key value count mismatch for resource: " + name); } std::string sql = "SELECT * FROM " + - External_Database_Utils::quote_identifier(name) + " WHERE "; - for (std::size_t index = 0; index < columns.size(); ++index) { + External_Database_Utils::quote_identifier(name) + " WHERE "; + for (std::size_t index = 0; index < columns.size(); ++index) + { if (index != 0) sql += " AND "; sql += External_Database_Utils::quote_identifier(columns[index]) + - " = ? COLLATE NOCASE"; + " = ? COLLATE NOCASE"; } sql += " LIMIT 1"; SQLite::Statement statement(db, sql); - for (std::size_t index = 0; index < primary_key_values.size(); ++index) { + for (std::size_t index = 0; index < primary_key_values.size(); ++index) + { statement.bind(static_cast(index + 1), External_Database_Utils::normalize_key(primary_key_values[index])); } if (!statement.executeStep()) return std::nullopt; External_Database_Row row; - for (int index = 0; index < statement.getColumnCount(); ++index) { + for (int index = 0; index < statement.getColumnCount(); ++index) + { const auto name = statement.getColumnName(index); - if (statement.isColumnNull(index)) { + if (statement.isColumnNull(index)) + { row.columns.emplace(name, std::nullopt); - } else { + } + else + { row.columns.emplace(name, statement.getColumn(index).getString()); } } return row; } -External_Resources_Manager::External_Resources_Manager() { +External_Resources_Manager::External_Resources_Manager() +{ register_external_resource(make_tar1090_db_aircraft_csv_gz_resource()); register_external_resource(make_wiedehopf_tar1090_db_resource()); register_external_resource(make_mictronics_aircraft_database_resource()); @@ -211,15 +291,18 @@ External_Resources_Manager::External_Resources_Manager() { register_external_resource(make_opensky_flightdata_api_resource()); } -void External_Resources_Manager::init(const Psc::JSON* that_json) { +void External_Resources_Manager::init(const Psc::JSON* that_json) +{ if (!that_json) return; const auto list = that_json->get("list"); if (!list) return; std::lock_guard lock(mtx_); - for (const auto& config : list->children) { + for (const auto& config : list->children) + { const auto name = config.try_get_string("name"); if (!name) continue; - for (const auto& resource : resources_) { + for (const auto& resource : resources_) + { if (resource->name != *name) continue; resource->init(&config); break; @@ -228,23 +311,29 @@ void External_Resources_Manager::init(const Psc::JSON* that_json) { Get_J(ecap_sqlite_path) } -Psc::JSON External_Resources_Manager::to_json() const { +Psc::JSON External_Resources_Manager::to_json() const +{ std::lock_guard lock(mtx_); auto ret = Psc::JSON::object(); Ret_J(ecap_sqlite_path) auto list = Psc::JSON::array(); - for (const std::unique_ptr& resource : resources_) { + for (const std::unique_ptr& resource : resources_) + { list.children.emplace_back(resource->to_json()); } ret.children.emplace_back(Psc::JSON{"list", list}); return ret; } -void External_Resources_Manager::load_sqlite_db(std::string path) { - try { +void External_Resources_Manager::load_sqlite_db(std::string path) +{ + try + { db_ = std::make_unique( path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE); - } catch (SQLite::Exception& error) { + } + catch (SQLite::Exception& error) + { Psc::fail_fast(std::string("load_sqlite_db error: ") + error.what()); } @@ -255,167 +344,415 @@ void External_Resources_Manager::load_sqlite_db(std::string path) { } std::vector -External_Resources_Manager::refresh_external_databases(const bool force_download) { +External_Resources_Manager::refresh_external_databases(const bool force_download) +{ return refresh_all(require_db(), force_download); } External_Resource_Status External_Resources_Manager::refresh_external_database( const std::string& resource_name, - const bool force_download) { + const bool force_download) +{ return refresh_one(require_db(), resource_name, force_download); } External_Resource_Status External_Resources_Manager::import_external_database( const std::string& resource_name, - const std::filesystem::path& source_file) { + const std::filesystem::path& source_file) +{ return import_offline(require_db(), resource_name, source_file); } External_Resource_Status External_Resources_Manager::clear_external_database_table( - const std::string& resource_name) { + const std::string& resource_name) +{ return clear_table(require_db(), resource_name); } std::optional External_Resources_Manager::query_external_database( const std::string& resource_name, - const std::vector& primary_key_values) const { + const std::vector& primary_key_values) const +{ return query_one(require_db(), resource_name, primary_key_values); } std::map> External_Resources_Manager::query_aircraft_external_databases( - const std::string& icao24) const { + const std::string& icao24) const +{ return query_aircraft(require_db(), icao24); } std::map> External_Resources_Manager::query_callsign_external_databases( - const std::optional& callsign) const { + const std::optional& callsign) const +{ return query_callsign(require_db(), callsign); } std::vector -External_Resources_Manager::external_database_status() const { +External_Resources_Manager::external_database_status() const +{ return status(require_db()); } -void External_Resources_Manager::server(Global* g) { +void External_Resources_Manager::async_refresh_external_databases(const bool force_download, + Status_List_Callback callback) +{ + run_external_database_async>( + [this, force_download]() { return refresh_external_databases(force_download); }, + std::move(callback)); +} + +void External_Resources_Manager::async_refresh_external_database( + std::string resource_name, + const bool force_download, + Status_Callback callback) +{ + run_external_database_async( + [this, resource_name = std::move(resource_name), force_download]() + { + return refresh_external_database(resource_name, force_download); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_import_external_database( + std::string resource_name, + std::filesystem::path source_file, + Status_Callback callback) +{ + run_external_database_async( + [this, + resource_name = std::move(resource_name), + source_file = std::move(source_file)]() + { + return import_external_database(resource_name, source_file); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_clear_external_database_table( + std::string resource_name, + Status_Callback callback) +{ + run_external_database_async( + [this, resource_name = std::move(resource_name)]() + { + return clear_external_database_table(resource_name); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_query_external_database( + std::string resource_name, + std::vector primary_key_values, + Row_Callback callback) const +{ + run_external_database_async>( + [this, + resource_name = std::move(resource_name), + primary_key_values = std::move(primary_key_values)]() + { + return query_external_database(resource_name, primary_key_values); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_query_aircraft_external_databases( + std::string icao24, + Row_Map_Callback callback) const +{ + run_external_database_async< + std::map>>( + [this, icao24 = std::move(icao24)]() + { + return query_aircraft_external_databases(icao24); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_query_callsign_external_databases( + std::optional callsign, + Row_Map_Callback callback) const +{ + run_external_database_async< + std::map>>( + [this, callsign = std::move(callsign)]() + { + return query_callsign_external_databases(callsign); + }, + std::move(callback)); +} + +void External_Resources_Manager::async_external_database_status( + Status_List_Callback callback) const +{ + run_external_database_async>( + [this]() { return external_database_status(); }, + std::move(callback)); +} + +ucoro::awaitable> +External_Resources_Manager::refresh_external_databases_coro( + const bool force_download) +{ + co_return co_await await_external_database_callback< + std::vector>( + [this, force_download](Status_List_Callback callback) + { + async_refresh_external_databases(force_download, std::move(callback)); + }); +} + +ucoro::awaitable +External_Resources_Manager::refresh_external_database_coro( + std::string resource_name, + const bool force_download) +{ + co_return co_await await_external_database_callback( + [this, + resource_name = std::move(resource_name), + force_download](Status_Callback callback) mutable + { + async_refresh_external_database( + std::move(resource_name), force_download, std::move(callback)); + }); +} + +ucoro::awaitable +External_Resources_Manager::import_external_database_coro( + std::string resource_name, + std::filesystem::path source_file) +{ + co_return co_await await_external_database_callback( + [this, + resource_name = std::move(resource_name), + source_file = std::move(source_file)](Status_Callback callback) mutable + { + async_import_external_database( + std::move(resource_name), std::move(source_file), std::move(callback)); + }); +} + +ucoro::awaitable +External_Resources_Manager::clear_external_database_table_coro( + std::string resource_name) +{ + co_return co_await await_external_database_callback( + [this, resource_name = std::move(resource_name)](Status_Callback callback) mutable + { + async_clear_external_database_table( + std::move(resource_name), std::move(callback)); + }); +} + +ucoro::awaitable> +External_Resources_Manager::query_external_database_coro( + std::string resource_name, + std::vector primary_key_values) const +{ + co_return co_await await_external_database_callback< + std::optional>( + [this, + resource_name = std::move(resource_name), + primary_key_values = std::move(primary_key_values)](Row_Callback callback) mutable + { + async_query_external_database( + std::move(resource_name), + std::move(primary_key_values), + std::move(callback)); + }); +} + +ucoro::awaitable>> +External_Resources_Manager::query_aircraft_external_databases_coro( + std::string icao24) const +{ + co_return co_await await_external_database_callback< + std::map>>( + [this, icao24 = std::move(icao24)](Row_Map_Callback callback) mutable + { + async_query_aircraft_external_databases( + std::move(icao24), std::move(callback)); + }); +} + +ucoro::awaitable>> +External_Resources_Manager::query_callsign_external_databases_coro( + std::optional callsign) const +{ + co_return co_await await_external_database_callback< + std::map>>( + [this, callsign = std::move(callsign)](Row_Map_Callback callback) mutable + { + async_query_callsign_external_databases( + std::move(callsign), std::move(callback)); + }); +} + +ucoro::awaitable> +External_Resources_Manager::external_database_status_coro() const +{ + co_return co_await await_external_database_callback< + std::vector>( + [this](Status_List_Callback callback) + { + async_external_database_status(std::move(callback)); + }); +} + +void External_Resources_Manager::server(Global* g) +{ auto& svr = g->svr; const auto& api = g->api; - svr.Post(api + "get_external_database_config", [this](HTTP_Param) { - - + svr.Post(api + "get_external_database_config", [this](HTTP_Param) + { auto t = warp(to_json()).to_json_string(); res->setBody(t); }); - svr.Post(api + "get_external_database_status", [this](HTTP_Param) { + svr.PostCoro(api + "get_external_database_status", [this](HTTP_Param) -> drogon::Task<> + { + auto status_list = + co_await Ecap_Coro::to_drogon(external_database_status_coro()); res->setBody( - warp(external_resource_status_list_to_json(external_database_status())) - .to_json_string()); + warp(external_resource_status_list_to_json(status_list)) + .to_json_string()); + co_return; }); - svr.Post(api + "refresh_external_database", [this, g](HTTP_Param) { + svr.PostCoro(api + "refresh_external_database", [this, g](HTTP_Param) -> drogon::Task<> + { CHECK_JSON_PARAM HTTP_REQUIRE_VALUE(name, params.try_get_string("name")) - const auto result = refresh_external_database(name, true); + const auto result = + co_await Ecap_Coro::to_drogon(refresh_external_database_coro(name, true)); g->save(); res->setBody(warp(external_resource_status_to_json(result)).to_json_string()); + co_return; }); - svr.Post(api + "refresh_external_databases", [this, g](HTTP_Param) { - const auto result = refresh_external_databases(true); + svr.PostCoro(api + "refresh_external_databases", [this, g](HTTP_Param) -> drogon::Task<> + { + const auto result = + co_await Ecap_Coro::to_drogon(refresh_external_databases_coro(true)); g->save(); res->setBody(warp(external_resource_status_list_to_json(result)).to_json_string()); + co_return; }); - svr.Post(api + "import_external_database", [this, g](HTTP_Param) { + svr.PostCoro(api + "import_external_database", [this, g](HTTP_Param) -> drogon::Task<> + { CHECK_JSON_PARAM HTTP_REQUIRE_VALUE(name, params.try_get_string("name")) HTTP_REQUIRE_VALUE(source_file, params.try_get_string("source_file")) - const auto result = import_external_database(name, source_file); + const auto result = + co_await Ecap_Coro::to_drogon(import_external_database_coro(name, source_file)); g->save(); res->setBody(warp(external_resource_status_to_json(result)).to_json_string()); + co_return; }); - svr.Post(api + "upload_external_database", [this, g](HTTP_Param) { + svr.PostCoro(api + "upload_external_database", [this, g](HTTP_Param) -> drogon::Task<> + { drogon::MultiPartParser parser; - if (parser.parse(req) != 0) { + if (parser.parse(req) != 0) + { throw_invalid_http_param("multipart"); } const auto name = parser.getOptionalParameter("name"); - if (!name || parser.getFiles().size() != 1) { + if (!name || parser.getFiles().size() != 1) + { throw_invalid_http_param("name or file"); } const auto& upload = parser.getFiles().front(); const auto serial = std::chrono::steady_clock::now().time_since_epoch().count(); auto extension = std::string(upload.getFileExtension()); - if (!extension.empty() && extension.front() != '.') { + if (!extension.empty() && extension.front() != '.') + { extension.insert(extension.begin(), '.'); } const auto saved_name = "external_database_" + std::to_string(serial) + extension; - if (upload.saveAs(saved_name) != 0) { + if (upload.saveAs(saved_name) != 0) + { throw std::runtime_error("Cannot save uploaded external database file"); } const auto source_file = std::filesystem::path(drogon::app().getUploadPath()) / saved_name; - const auto result = import_external_database(*name, source_file); + const auto result = co_await Ecap_Coro::to_drogon( + import_external_database_coro(*name, source_file)); g->save(); res->setBody(warp(external_resource_status_to_json(result)).to_json_string()); + co_return; }); - svr.Post(api + "clear_external_database_table", [this, g](HTTP_Param) { + svr.PostCoro(api + "clear_external_database_table", [this, g](HTTP_Param) -> drogon::Task<> + { CHECK_JSON_PARAM HTTP_REQUIRE_VALUE(name, params.try_get_string("name")) - const auto result = clear_external_database_table(name); + const auto result = + co_await Ecap_Coro::to_drogon(clear_external_database_table_coro(name)); g->save(); res->setBody(warp(external_resource_status_to_json(result)).to_json_string()); + co_return; }); - svr.Post(api + "query_external_database", [this](HTTP_Param) { + svr.PostCoro(api + "query_external_database", [this](HTTP_Param) -> drogon::Task<> + { CHECK_JSON_PARAM HTTP_REQUIRE_VALUE(name, params.try_get_string("name")) HTTP_REQUIRE_PTR(primary_key_values_json, params.get("primary_key_values")) std::vector primary_key_values; - for (const auto& value : primary_key_values_json->children) { - if (value.valueType != Psc::JsonType::String) { + for (const auto& value : primary_key_values_json->children) + { + if (value.valueType != Psc::JsonType::String) + { throw_invalid_http_param("primary_key_values"); } primary_key_values.push_back(value.val); } - const auto row = query_external_database(name, primary_key_values); - res->setBody(row ? warp(row->to_json()).to_json_string() + const auto row = co_await Ecap_Coro::to_drogon( + query_external_database_coro(name, std::move(primary_key_values))); + res->setBody(row + ? warp(row->to_json()).to_json_string() : Psc::JSON(nullptr).to_json_string()); + co_return; }); } void External_Resources_Manager::register_external_resource( - std::unique_ptr external_res) { + std::unique_ptr external_res) +{ if (!external_res) throw std::invalid_argument("external resource is null"); resources_.push_back(std::move(external_res)); } void External_Resources_Manager::initialize(SQLite::Database& db, - std::filesystem::path cache_pos) { + std::filesystem::path cache_pos) +{ std::lock_guard lock(mtx_); cache_pos_ = std::move(cache_pos); for (const auto& resource : resources_) resource->create_table(db); } std::vector External_Resources_Manager::sync_missing( - SQLite::Database& db) { + SQLite::Database& db) +{ return update_all(db, false, true); } std::vector External_Resources_Manager::refresh_all( SQLite::Database& db, - const bool force_download) { + const bool force_download) +{ return update_all(db, force_download, false); } External_Resource_Status External_Resources_Manager::refresh_one( SQLite::Database& db, const std::string& resource_name, - const bool force_download) { + const bool force_download) +{ std::lock_guard lock(mtx_); return get_resource(resource_name).update(db, cache_pos_, force_download, false); } @@ -423,14 +760,16 @@ External_Resource_Status External_Resources_Manager::refresh_one( External_Resource_Status External_Resources_Manager::import_offline( SQLite::Database& db, const std::string& resource_name, - const std::filesystem::path& source_file) { + const std::filesystem::path& source_file) +{ std::lock_guard lock(mtx_); return get_resource(resource_name).import_offline(db, source_file); } External_Resource_Status External_Resources_Manager::clear_table( SQLite::Database& db, - const std::string& resource_name) { + const std::string& resource_name) +{ std::lock_guard lock(mtx_); return get_resource(resource_name).clear_table(db, cache_pos_); } @@ -438,14 +777,16 @@ External_Resource_Status External_Resources_Manager::clear_table( std::optional External_Resources_Manager::query_one( SQLite::Database& db, const std::string& resource_name, - const std::vector& primary_key_values) const { + const std::vector& primary_key_values) const +{ std::lock_guard lock(mtx_); return get_resource(resource_name).query_one(db, primary_key_values); } std::map> External_Resources_Manager::query_aircraft(SQLite::Database& db, - const std::string& icao24) const { + const std::string& icao24) const +{ Scope_Timer timer( "external_database.query_aircraft", Global::instance()->http_monitor_config.slow_scope_threshold_ms); @@ -460,7 +801,8 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db, std::map> result; const auto add_row = [&](const std::string& resource_name, const std::vector& primary_key_values, - const std::string& result_name = std::string{}) { + const std::string& result_name = std::string{}) + { auto& resource = get_resource(resource_name); auto row = resource.query_one(db, primary_key_values); if (!row) return std::shared_ptr{}; @@ -470,9 +812,11 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db, }; std::optional type_code; - for (const auto& resource_name : kAircraftResourceKeys) { + for (const auto& resource_name : kAircraftResourceKeys) + { const auto row = add_row(resource_name, {icao24}); - if (row && !type_code) { + if (row && !type_code) + { type_code = row->get("type_code"); if (!type_code) type_code = row->get("typecode"); } @@ -485,7 +829,8 @@ External_Resources_Manager::query_aircraft(SQLite::Database& db, std::map> External_Resources_Manager::query_callsign( SQLite::Database& db, - const std::optional& callsign) const { + const std::optional& callsign) const +{ Scope_Timer timer( "external_database.query_callsign", Global::instance()->http_monitor_config.slow_scope_threshold_ms); @@ -499,7 +844,8 @@ External_Resources_Manager::query_callsign( std::map> result; const auto add_row = [&](const std::string& resource_name, const std::vector& primary_key_values, - const std::string& result_name = std::string{}) { + const std::string& result_name = std::string{}) + { auto& resource = get_resource(resource_name); auto row = resource.query_one(db, primary_key_values); if (!row) return std::shared_ptr{}; @@ -509,15 +855,18 @@ External_Resources_Manager::query_callsign( }; std::set airport_codes; - if (callsign && !callsign->empty()) { - for (const auto& resource_name : kRouteResourceKeys) { + if (callsign && !callsign->empty()) + { + for (const auto& resource_name : kRouteResourceKeys) + { const auto row = add_row(resource_name, {*callsign}); if (!row) continue; const auto airports = row->get("AirportCodes"); if (!airports) continue; std::size_t start = 0; - while (start < airports->size()) { + while (start < airports->size()) + { const auto end = airports->find('-', start); airport_codes.emplace(airports->substr(start, end - start)); if (end == std::string::npos) break; @@ -526,7 +875,8 @@ External_Resources_Manager::query_callsign( } } - for (const auto& airport_code : airport_codes) { + for (const auto& airport_code : airport_codes) + { if (airport_code.empty()) continue; add_row("vrs_airports", {airport_code}, "vrs_airports:" + airport_code); } @@ -534,25 +884,31 @@ External_Resources_Manager::query_callsign( } std::vector External_Resources_Manager::status( - SQLite::Database& db) const { + SQLite::Database& db) const +{ std::lock_guard lock(mtx_); std::vector result; - for (const auto& resource : resources_) { + for (const auto& resource : resources_) + { result.push_back({resource->name, resource->row_count(db), false, false, ""}); } return result; } -SQLite::Database& External_Resources_Manager::require_db() const { - if (!db_) { +SQLite::Database& External_Resources_Manager::require_db() const +{ + if (!db_) + { throw std::runtime_error("DB not loaded. Call load_sqlite_db() first."); } return *db_; } External_Resources& External_Resources_Manager::get_resource( - const std::string& resource_name) const { - for (const auto& resource : resources_) { + const std::string& resource_name) const +{ + for (const auto& resource : resources_) + { if (resource->name == resource_name) return *resource; } throw std::invalid_argument("unknown external resource: " + resource_name); @@ -561,18 +917,23 @@ External_Resources& External_Resources_Manager::get_resource( std::vector External_Resources_Manager::update_all( SQLite::Database& db, const bool force_download, - const bool only_when_empty) { + const bool only_when_empty) +{ std::lock_guard lock(mtx_); std::vector result; - for (const auto& resource : resources_) { - try { + for (const auto& resource : resources_) + { + try + { auto status = resource->update(db, cache_pos_, force_download, only_when_empty); std::cout << "External database [" << status.name << "] " - << status.message << ", rows=" << status.row_count << std::endl; + << status.message << ", rows=" << status.row_count << std::endl; result.push_back(std::move(status)); - } catch (const std::exception& error) { + } + catch (const std::exception& error) + { std::cerr << "External database [" << resource->name - << "] update failed: " << error.what() << std::endl; + << "] update failed: " << error.what() << std::endl; result.push_back( {resource->name, resource->row_count(db), false, false, error.what()}); } diff --git a/module/Local_Server/External_Database/global.h b/module/Local_Server/External_Database/global.h index 03965ce..f0bc28b 100644 --- a/module/Local_Server/External_Database/global.h +++ b/module/Local_Server/External_Database/global.h @@ -6,13 +6,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include +#include class Global; @@ -98,6 +101,13 @@ protected: class External_Resources_Manager { public: + using Status_Callback = std::function; + using Status_List_Callback = std::function)>; + using Row_Callback = std::function)>; + using Row_Map_Callback = std::function>)>; + External_Resources_Manager(); void init(const Psc::JSON* that_json); @@ -121,6 +131,45 @@ public: const std::optional& callsign) const; [[nodiscard]] std::vector external_database_status() const; + void async_refresh_external_databases(bool force_download, + Status_List_Callback callback); + void async_refresh_external_database(std::string resource_name, + bool force_download, + Status_Callback callback); + void async_import_external_database(std::string resource_name, + std::filesystem::path source_file, + Status_Callback callback); + void async_clear_external_database_table(std::string resource_name, + Status_Callback callback); + void async_query_external_database(std::string resource_name, + std::vector primary_key_values, + Row_Callback callback) const; + void async_query_aircraft_external_databases(std::string icao24, + Row_Map_Callback callback) const; + void async_query_callsign_external_databases(std::optional callsign, + Row_Map_Callback callback) const; + void async_external_database_status(Status_List_Callback callback) const; + + [[nodiscard]] ucoro::awaitable> + refresh_external_databases_coro(bool force_download = true); + [[nodiscard]] ucoro::awaitable + refresh_external_database_coro(std::string resource_name, + bool force_download = true); + [[nodiscard]] ucoro::awaitable + import_external_database_coro(std::string resource_name, + std::filesystem::path source_file); + [[nodiscard]] ucoro::awaitable + clear_external_database_table_coro(std::string resource_name); + [[nodiscard]] ucoro::awaitable> + query_external_database_coro(std::string resource_name, + std::vector primary_key_values) const; + [[nodiscard]] ucoro::awaitable>> + query_aircraft_external_databases_coro(std::string icao24) const; + [[nodiscard]] ucoro::awaitable>> + query_callsign_external_databases_coro(std::optional callsign) const; + [[nodiscard]] ucoro::awaitable> + external_database_status_coro() const; + void register_external_resource(std::unique_ptr external_res); void initialize(SQLite::Database& db, std::filesystem::path cache_pos); std::vector sync_missing(SQLite::Database& db); diff --git a/module/Local_Server/server/Global.h b/module/Local_Server/server/Global.h index 21e52be..1d1b25c 100644 --- a/module/Local_Server/server/Global.h +++ b/module/Local_Server/server/Global.h @@ -6,6 +6,7 @@ #include "../Data_Source/export.h" #include "../State_Report/State_Report.h" #include "../global_include.h" +#include "Ucoro_Drogon_Glue.h" #include "Config.h" #include @@ -72,6 +73,7 @@ public: std::string rise = "rise_"; std::string fall = "fall_"; using Handler = std::function; + using CoroHandler = std::function(const drogon::HttpRequestPtr&, const drogon::HttpResponsePtr&)>; bool listen(const std::string& host, int port); ~Web_Server() = default; @@ -103,6 +105,36 @@ public: return *this; } + Web_Server& PostCoro(const std::string& pattern, CoroHandler handler) { + drogon::app().registerHandler( + pattern, + [handler = std::move(handler)]( + const drogon::HttpRequestPtr& request, + std::function&& callback) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + + drogon::async_run( + [handler, request, resp, callback = std::move(callback)]() mutable -> drogon::Task<> { + try { + co_await handler(request, resp); + } catch (const std::exception& error) { + resp->setStatusCode(drogon::k500InternalServerError); + resp->setBody(Psc::JSON::object({{"error", std::string(error.what())}}).to_json_string()); + } catch (...) { + resp->setStatusCode(drogon::k500InternalServerError); + resp->setBody(Psc::JSON::object({{"error", "unknown error"}}).to_json_string()); + } + callback(resp); + co_return; + }); + }, + {drogon::Post} + ); + + return *this; + } + Web_Server& Get(const std::string& pattern, Handler handler) { drogon::app().registerHandlerViaRegex( pattern, @@ -120,6 +152,36 @@ public: return *this; } + + Web_Server& GetCoro(const std::string& pattern, CoroHandler handler) { + drogon::app().registerHandlerViaRegex( + pattern, + [handler = std::move(handler)]( + const drogon::HttpRequestPtr& request, + std::function&& callback) { + auto resp = drogon::HttpResponse::newHttpResponse(); + resp->setContentTypeCode(drogon::CT_APPLICATION_JSON); + + drogon::async_run( + [handler, request, resp, callback = std::move(callback)]() mutable -> drogon::Task<> { + try { + co_await handler(request, resp); + } catch (const std::exception& error) { + resp->setStatusCode(drogon::k500InternalServerError); + resp->setBody(Psc::JSON::object({{"error", std::string(error.what())}}).to_json_string()); + } catch (...) { + resp->setStatusCode(drogon::k500InternalServerError); + resp->setBody(Psc::JSON::object({{"error", "unknown error"}}).to_json_string()); + } + callback(resp); + co_return; + }); + }, + {drogon::Get} + ); + + return *this; + } };