ae/ECAP_Server
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

首次提交

Browse Source
This commit is contained in:
种太阳的热带美短
2026-06-16 11:18:22 +08:00
commit 068ef188e7
201 changed files with 57503 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/Local_Server/External_Database/Resource_Utils.cpp
+458
View File
@@ -0,0 +1,458 @@
#include "Resource_Utils.h"
#include <SQLiteCpp/Transaction.h>
#include <drogon/HttpClient.h>
#include <drogon/HttpRequest.h>
#include <drogon/HttpResponse.h>
#include <zlib.h>
#include <algorithm>
#include <cctype>
#include <fstream>
#include <map>
#include <sstream>
#include <stdexcept>
#include <system_error>
#include <utility>
namespace External_Database_Utils {
namespace {
constexpr const char* kTar1090Url =
"https://raw.githubusercontent.com/wiedehopf/tar1090-db/refs/heads/csv/aircraft.csv.gz";
const std::vector<std::string> kTar1090Columns = {
"icao24", "registration", "type_code", "subtype",
"type_name", "year", "operator_info"
};
const std::vector<std::string> kTar1090PrimaryKey = {"icao24"};
std::pair<std::string, std::string> split_url(const std::string& url) {
const auto scheme_end = url.find("://");
if (scheme_end == std::string::npos) {
throw std::runtime_error("Invalid download URL: " + url);
}
const auto path_begin = url.find('/', scheme_end + 3);
if (path_begin == std::string::npos) return {url, "/"};
return {url.substr(0, path_begin), url.substr(path_begin)};
}
void replace_file(const std::filesystem::path& temp_file,
const std::filesystem::path& target_file) {
std::error_code error;
std::filesystem::remove(target_file, error);
error.clear();
std::filesystem::rename(temp_file, target_file, error);
if (error) {
throw std::runtime_error("Cannot replace cache file " + target_file.string() +
": " + error.message());
}
}
std::string create_csv_table_sql(const std::string& table_name,
const std::vector<std::string>& columns,
const std::vector<std::string>& primary_key_columns) {
std::ostringstream sql;
sql << "CREATE TABLE IF NOT EXISTS " << quote_identifier(table_name) << " (";
for (std::size_t index = 0; index < columns.size(); ++index) {
if (index != 0) sql << ',';
sql << quote_identifier(columns[index]) << " TEXT";
}
if (!primary_key_columns.empty()) {
sql << ", PRIMARY KEY (";
for (std::size_t index = 0; index < primary_key_columns.size(); ++index) {
if (index != 0) sql << ',';
sql << quote_identifier(primary_key_columns[index]);
}
sql << ')';
}
sql << ");";
return sql.str();
}
class Header_Csv_Resource final : public External_Resources {
public:
explicit Header_Csv_Resource(Header_Csv_Resource_Config config)
: External_Resources(config.name,
config.cache_file_name,
config.download_url,
config.description,
config.update_interval_seconds),
config_(std::move(config)) {}
void create_table(SQLite::Database& db) const override {
db.exec(create_csv_table_sql(name, config_.columns,
config_.primary_key_columns));
}
protected:
std::size_t import_file(SQLite::Database& db,
const std::filesystem::path& source_file) const override {
return import_header_csv(db, source_file, name, config_.columns,
config_.primary_key_columns, config_.delimiter);
}
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
return config_.primary_key_columns;
}
private:
Header_Csv_Resource_Config config_;
};
class Schema_Only_Resource final : public External_Resources {
public:
explicit Schema_Only_Resource(Schema_Only_Resource_Config config)
: External_Resources(config.name,
"",
"",
config.description,
config.update_interval_seconds),
config_(std::move(config)) {}
void create_table(SQLite::Database& db) const override {
db.exec(config_.create_table_sql);
}
protected:
std::size_t import_file(SQLite::Database&,
const std::filesystem::path&) const override {
throw std::runtime_error("This resource is updated by its API-specific adapter");
}
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
return config_.primary_key_columns;
}
private:
Schema_Only_Resource_Config config_;
};
class Tar1090_Aircraft_Resource final : public External_Resources {
public:
Tar1090_Aircraft_Resource(std::string name,
std::string description,
const std::uint64_t update_interval_seconds)
: External_Resources(std::move(name),
"aircraft.csv",
kTar1090Url,
std::move(description),
update_interval_seconds) {}
void create_table(SQLite::Database& db) const override {
db.exec(create_csv_table_sql(name, kTar1090Columns, kTar1090PrimaryKey));
}
protected:
bool fetch(const std::filesystem::path& cache_pos,
const bool force_download) const override {
const auto csv_file = source_file(cache_pos);
if (!force_download) return false;
const auto gzip_file = cache_pos / "aircraft.csv.gz";
download_to_file(download_url, gzip_file);
decompress_gzip(gzip_file, csv_file);
return true;
}
std::size_t import_file(SQLite::Database& db,
const std::filesystem::path& source_file) const override {
if (source_file.extension() == ".gz") {
auto csv_file = source_file;
csv_file.replace_extension(".csv");
decompress_gzip(source_file, csv_file);
return import_file(db, csv_file);
}
std::ifstream input(source_file, std::ios::binary);
if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string());
SQLite::Transaction transaction(db);
db.exec("DELETE FROM " + quote_identifier(name));
SQLite::Statement statement(
db, make_upsert_sql(name, kTar1090Columns, kTar1090PrimaryKey));
std::vector<std::string> fields;
std::size_t row_count = 0;
while (read_csv_row(input, fields, ';')) {
if (fields.empty()) continue;
const auto icao = normalize_key(fields[0]);
if (icao.empty()) continue;
statement.reset();
statement.clearBindings();
statement.bind(1, icao);
for (std::size_t index = 1; index < kTar1090Columns.size(); ++index) {
std::optional<std::string> value;
if (index < fields.size()) value = normalize_field(fields[index]);
bind_optional(statement, static_cast<int>(index + 1), value);
}
statement.exec();
++row_count;
}
transaction.commit();
return row_count;
}
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
return kTar1090PrimaryKey;
}
};
} // 命名空间
std::string quote_identifier(const std::string& value) {
std::string result = "\"";
for (const char ch : value) {
result += ch;
if (ch == '"') result += '"';
}
result += '"';
return result;
}
std::string normalize_key(std::string value) {
const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; };
value.erase(value.begin(), std::find_if(value.begin(), value.end(),
[&](char ch) { return !is_space(ch); }));
value.erase(std::find_if(value.rbegin(), value.rend(),
[&](char ch) { return !is_space(ch); }).base(),
value.end());
std::transform(value.begin(), value.end(), value.begin(),
[](unsigned char ch) { return static_cast<char>(std::toupper(ch)); });
return value;
}
std::optional<std::string> normalize_field(std::string value) {
const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; };
value.erase(value.begin(), std::find_if(value.begin(), value.end(),
[&](char ch) { return !is_space(ch); }));
value.erase(std::find_if(value.rbegin(), value.rend(),
[&](char ch) { return !is_space(ch); }).base(),
value.end());
if (value.empty()) return std::nullopt;
return value;
}
void bind_optional(SQLite::Statement& statement,
const int index,
const std::optional<std::string>& value) {
if (value) statement.bind(index, *value);
else statement.bind(index);
}
bool read_csv_row(std::istream& input,
std::vector<std::string>& fields,
const char delimiter) {
fields.clear();
std::string field;
char quote = '\0';
bool have_data = false;
while (true) {
const int next = input.get();
if (next == EOF) {
if (!have_data && field.empty() && fields.empty()) return false;
fields.push_back(std::move(field));
return true;
}
have_data = true;
const char ch = static_cast<char>(next);
if (quote != '\0') {
if (ch == quote) {
if (input.peek() == quote) {
input.get();
field += quote;
} else {
quote = '\0';
}
} else {
field += ch;
}
} else if ((ch == '"' || ch == '\'') && field.empty()) {
quote = ch;
} else if (ch == delimiter) {
fields.push_back(std::move(field));
field.clear();
} else if (ch == '\n') {
fields.push_back(std::move(field));
return true;
} else if (ch != '\r') {
field += ch;
}
}
}
std::string make_upsert_sql(const std::string& table_name,
const std::vector<std::string>& columns,
const std::vector<std::string>& primary_key_columns) {
std::ostringstream sql;
sql << "INSERT INTO " << quote_identifier(table_name) << " (";
for (std::size_t index = 0; index < columns.size(); ++index) {
if (index != 0) sql << ',';
sql << quote_identifier(columns[index]);
}
sql << ") VALUES (";
for (std::size_t index = 0; index < columns.size(); ++index) {
if (index != 0) sql << ',';
sql << '?';
}
sql << ')';
if (!primary_key_columns.empty()) {
sql << " ON CONFLICT (";
for (std::size_t index = 0; index < primary_key_columns.size(); ++index) {
if (index != 0) sql << ',';
sql << quote_identifier(primary_key_columns[index]);
}
sql << ") DO UPDATE SET ";
bool first = true;
for (const auto& column : columns) {
if (std::find(primary_key_columns.begin(), primary_key_columns.end(), column) !=
primary_key_columns.end()) continue;
if (!first) sql << ',';
first = false;
sql << quote_identifier(column) << "=excluded." << quote_identifier(column);
}
}
return sql.str();
}
void download_to_file(std::string url, const std::filesystem::path& target_file) {
std::filesystem::create_directories(target_file.parent_path());
constexpr int kMaxRedirects = 5;
for (int redirect = 0; redirect <= kMaxRedirects; ++redirect) {
const auto [origin, path] = split_url(url);
auto client = drogon::HttpClient::newHttpClient(origin);
auto request = drogon::HttpRequest::newHttpRequest();
request->setMethod(drogon::Get);
request->setPath(path);
request->addHeader("User-Agent", "ECAP_Server external database updater");
const auto [result, response] = client->sendRequest(request, 600.0);
if (result != drogon::ReqResult::Ok || !response) {
throw std::runtime_error("Download failed: " + url);
}
const int status = static_cast<int>(response->getStatusCode());
if (status >= 300 && status < 400) {
const auto location = response->getHeader("location");
if (location.empty()) throw std::runtime_error("Redirect has no location: " + url);
url = location.front() == '/' ? origin + location : location;
continue;
}
if (status != 200) {
throw std::runtime_error("Download returned HTTP " + std::to_string(status) +
": " + url);
}
const auto temp_file = target_file.string() + ".tmp";
std::ofstream output(temp_file, std::ios::binary | std::ios::trunc);
if (!output) throw std::runtime_error("Cannot write cache file: " + temp_file);
const auto body = response->getBody();
output.write(body.data(), static_cast<std::streamsize>(body.size()));
output.close();
if (!output) throw std::runtime_error("Cannot finish cache file: " + temp_file);
replace_file(temp_file, target_file);
return;
}
throw std::runtime_error("Too many redirects while downloading: " + url);
}
void decompress_gzip(const std::filesystem::path& gzip_file,
const std::filesystem::path& target_file) {
const auto temp_file = target_file.string() + ".tmp";
gzFile input = gzopen(gzip_file.string().c_str(), "rb");
if (!input) throw std::runtime_error("Cannot open gzip file: " + gzip_file.string());
std::ofstream output(temp_file, std::ios::binary | std::ios::trunc);
if (!output) {
gzclose(input);
throw std::runtime_error("Cannot write decompressed file: " + temp_file);
}
char buffer[64 * 1024];
int size = 0;
while ((size = gzread(input, buffer, sizeof(buffer))) > 0) output.write(buffer, size);
const int close_result = gzclose(input);
output.close();
if (size < 0 || close_result != Z_OK || !output) {
std::filesystem::remove(temp_file);
throw std::runtime_error("Cannot decompress gzip file: " + gzip_file.string());
}
replace_file(temp_file, target_file);
}
std::size_t import_header_csv(SQLite::Database& db,
const std::filesystem::path& source_file,
const std::string& table_name,
const std::vector<std::string>& columns,
const std::vector<std::string>& primary_key_columns,
const char delimiter) {
std::ifstream input(source_file, std::ios::binary);
if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string());
std::vector<std::string> header;
if (!read_csv_row(input, header, delimiter)) {
throw std::runtime_error("CSV file is empty: " + source_file.string());
}
if (!header.empty() && header.front().size() >= 3 &&
static_cast<unsigned char>(header.front()[0]) == 0xef &&
static_cast<unsigned char>(header.front()[1]) == 0xbb &&
static_cast<unsigned char>(header.front()[2]) == 0xbf) {
header.front().erase(0, 3);
}
std::map<std::string, std::size_t> header_index;
for (std::size_t index = 0; index < header.size(); ++index) {
header_index.emplace(header[index], index);
}
for (const auto& column : columns) {
if (header_index.count(column) == 0) {
throw std::runtime_error("CSV header is missing column '" + column + "'");
}
}
SQLite::Transaction transaction(db);
db.exec("DELETE FROM " + quote_identifier(table_name));
SQLite::Statement statement(db, make_upsert_sql(table_name, columns, primary_key_columns));
std::vector<std::string> fields;
std::size_t row_count = 0;
while (read_csv_row(input, fields, delimiter)) {
bool primary_key_ok = true;
for (const auto& column : primary_key_columns) {
const auto index = header_index.at(column);
if (index >= fields.size() || normalize_key(fields[index]).empty()) {
primary_key_ok = false;
break;
}
}
if (!primary_key_ok) continue;
statement.reset();
statement.clearBindings();
for (std::size_t index = 0; index < columns.size(); ++index) {
std::optional<std::string> value;
const auto source_index = header_index.at(columns[index]);
if (source_index < fields.size()) value = normalize_field(fields[source_index]);
if (value &&
std::find(primary_key_columns.begin(), primary_key_columns.end(),
columns[index]) != primary_key_columns.end()) {
value = normalize_key(*value);
}
bind_optional(statement, static_cast<int>(index + 1), value);
}
statement.exec();
++row_count;
}
transaction.commit();
return row_count;
}
std::unique_ptr<External_Resources> make_header_csv_resource(
Header_Csv_Resource_Config config) {
return std::make_unique<Header_Csv_Resource>(std::move(config));
}
std::unique_ptr<External_Resources> make_schema_only_resource(
Schema_Only_Resource_Config config) {
return std::make_unique<Schema_Only_Resource>(std::move(config));
}
std::unique_ptr<External_Resources> make_tar1090_aircraft_resource(
std::string name,
std::string description,
const std::uint64_t update_interval_seconds) {
return std::make_unique<Tar1090_Aircraft_Resource>(
std::move(name), std::move(description), update_interval_seconds);
}
} // 命名空间 External_Database_Utils