459 lines
18 KiB
C++
459 lines
18 KiB
C++
#include "Resource_Utils.h"
|
|
|
|
#include <SQLiteCpp/Transaction.h>
|
|
#include <drogon/HttpClient.h>
|
|
#include <drogon/HttpRequest.h>
|
|
#include <drogon/HttpResponse.h>
|
|
#include <zlib.h>
|
|
|
|
#include <algorithm>
|
|
#include <cctype>
|
|
#include <fstream>
|
|
#include <map>
|
|
#include <sstream>
|
|
#include <stdexcept>
|
|
#include <system_error>
|
|
#include <utility>
|
|
|
|
namespace External_Database_Utils {
|
|
namespace {
|
|
|
|
constexpr const char* kTar1090Url =
|
|
"https://raw.githubusercontent.com/wiedehopf/tar1090-db/refs/heads/csv/aircraft.csv.gz";
|
|
|
|
const std::vector<std::string> kTar1090Columns = {
|
|
"icao24", "registration", "type_code", "subtype",
|
|
"type_name", "year", "operator_info"
|
|
};
|
|
const std::vector<std::string> kTar1090PrimaryKey = {"icao24"};
|
|
|
|
std::pair<std::string, std::string> split_url(const std::string& url) {
|
|
const auto scheme_end = url.find("://");
|
|
if (scheme_end == std::string::npos) {
|
|
throw std::runtime_error("Invalid download URL: " + url);
|
|
}
|
|
const auto path_begin = url.find('/', scheme_end + 3);
|
|
if (path_begin == std::string::npos) return {url, "/"};
|
|
return {url.substr(0, path_begin), url.substr(path_begin)};
|
|
}
|
|
|
|
void replace_file(const std::filesystem::path& temp_file,
|
|
const std::filesystem::path& target_file) {
|
|
std::error_code error;
|
|
std::filesystem::remove(target_file, error);
|
|
error.clear();
|
|
std::filesystem::rename(temp_file, target_file, error);
|
|
if (error) {
|
|
throw std::runtime_error("Cannot replace cache file " + target_file.string() +
|
|
": " + error.message());
|
|
}
|
|
}
|
|
|
|
std::string create_csv_table_sql(const std::string& table_name,
|
|
const std::vector<std::string>& columns,
|
|
const std::vector<std::string>& primary_key_columns) {
|
|
std::ostringstream sql;
|
|
sql << "CREATE TABLE IF NOT EXISTS " << quote_identifier(table_name) << " (";
|
|
for (std::size_t index = 0; index < columns.size(); ++index) {
|
|
if (index != 0) sql << ',';
|
|
sql << quote_identifier(columns[index]) << " TEXT";
|
|
}
|
|
if (!primary_key_columns.empty()) {
|
|
sql << ", PRIMARY KEY (";
|
|
for (std::size_t index = 0; index < primary_key_columns.size(); ++index) {
|
|
if (index != 0) sql << ',';
|
|
sql << quote_identifier(primary_key_columns[index]);
|
|
}
|
|
sql << ')';
|
|
}
|
|
sql << ");";
|
|
return sql.str();
|
|
}
|
|
|
|
class Header_Csv_Resource final : public External_Resources {
|
|
public:
|
|
explicit Header_Csv_Resource(Header_Csv_Resource_Config config)
|
|
: External_Resources(config.name,
|
|
config.cache_file_name,
|
|
config.download_url,
|
|
config.description,
|
|
config.update_interval_seconds),
|
|
config_(std::move(config)) {}
|
|
|
|
void create_table(SQLite::Database& db) const override {
|
|
db.exec(create_csv_table_sql(name, config_.columns,
|
|
config_.primary_key_columns));
|
|
}
|
|
|
|
protected:
|
|
std::size_t import_file(SQLite::Database& db,
|
|
const std::filesystem::path& source_file) const override {
|
|
return import_header_csv(db, source_file, name, config_.columns,
|
|
config_.primary_key_columns, config_.delimiter);
|
|
}
|
|
|
|
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
|
|
return config_.primary_key_columns;
|
|
}
|
|
|
|
private:
|
|
Header_Csv_Resource_Config config_;
|
|
};
|
|
|
|
class Schema_Only_Resource final : public External_Resources {
|
|
public:
|
|
explicit Schema_Only_Resource(Schema_Only_Resource_Config config)
|
|
: External_Resources(config.name,
|
|
"",
|
|
"",
|
|
config.description,
|
|
config.update_interval_seconds),
|
|
config_(std::move(config)) {}
|
|
|
|
void create_table(SQLite::Database& db) const override {
|
|
db.exec(config_.create_table_sql);
|
|
}
|
|
|
|
protected:
|
|
std::size_t import_file(SQLite::Database&,
|
|
const std::filesystem::path&) const override {
|
|
throw std::runtime_error("This resource is updated by its API-specific adapter");
|
|
}
|
|
|
|
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
|
|
return config_.primary_key_columns;
|
|
}
|
|
|
|
private:
|
|
Schema_Only_Resource_Config config_;
|
|
};
|
|
|
|
class Tar1090_Aircraft_Resource final : public External_Resources {
|
|
public:
|
|
Tar1090_Aircraft_Resource(std::string name,
|
|
std::string description,
|
|
const std::uint64_t update_interval_seconds)
|
|
: External_Resources(std::move(name),
|
|
"aircraft.csv",
|
|
kTar1090Url,
|
|
std::move(description),
|
|
update_interval_seconds) {}
|
|
|
|
void create_table(SQLite::Database& db) const override {
|
|
db.exec(create_csv_table_sql(name, kTar1090Columns, kTar1090PrimaryKey));
|
|
}
|
|
|
|
protected:
|
|
bool fetch(const std::filesystem::path& cache_pos,
|
|
const bool force_download) const override {
|
|
const auto csv_file = source_file(cache_pos);
|
|
if (!force_download) return false;
|
|
const auto gzip_file = cache_pos / "aircraft.csv.gz";
|
|
download_to_file(download_url, gzip_file);
|
|
decompress_gzip(gzip_file, csv_file);
|
|
return true;
|
|
}
|
|
|
|
std::size_t import_file(SQLite::Database& db,
|
|
const std::filesystem::path& source_file) const override {
|
|
if (source_file.extension() == ".gz") {
|
|
auto csv_file = source_file;
|
|
csv_file.replace_extension(".csv");
|
|
decompress_gzip(source_file, csv_file);
|
|
return import_file(db, csv_file);
|
|
}
|
|
|
|
std::ifstream input(source_file, std::ios::binary);
|
|
if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string());
|
|
|
|
SQLite::Transaction transaction(db);
|
|
db.exec("DELETE FROM " + quote_identifier(name));
|
|
SQLite::Statement statement(
|
|
db, make_upsert_sql(name, kTar1090Columns, kTar1090PrimaryKey));
|
|
std::vector<std::string> fields;
|
|
std::size_t row_count = 0;
|
|
while (read_csv_row(input, fields, ';')) {
|
|
if (fields.empty()) continue;
|
|
const auto icao = normalize_key(fields[0]);
|
|
if (icao.empty()) continue;
|
|
|
|
statement.reset();
|
|
statement.clearBindings();
|
|
statement.bind(1, icao);
|
|
for (std::size_t index = 1; index < kTar1090Columns.size(); ++index) {
|
|
std::optional<std::string> value;
|
|
if (index < fields.size()) value = normalize_field(fields[index]);
|
|
bind_optional(statement, static_cast<int>(index + 1), value);
|
|
}
|
|
statement.exec();
|
|
++row_count;
|
|
}
|
|
transaction.commit();
|
|
return row_count;
|
|
}
|
|
|
|
[[nodiscard]] const std::vector<std::string>& primary_key_columns() const override {
|
|
return kTar1090PrimaryKey;
|
|
}
|
|
};
|
|
|
|
} // 命名空间
|
|
|
|
std::string quote_identifier(const std::string& value) {
|
|
std::string result = "\"";
|
|
for (const char ch : value) {
|
|
result += ch;
|
|
if (ch == '"') result += '"';
|
|
}
|
|
result += '"';
|
|
return result;
|
|
}
|
|
|
|
std::string normalize_key(std::string value) {
|
|
const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; };
|
|
value.erase(value.begin(), std::find_if(value.begin(), value.end(),
|
|
[&](char ch) { return !is_space(ch); }));
|
|
value.erase(std::find_if(value.rbegin(), value.rend(),
|
|
[&](char ch) { return !is_space(ch); }).base(),
|
|
value.end());
|
|
std::transform(value.begin(), value.end(), value.begin(),
|
|
[](unsigned char ch) { return static_cast<char>(std::toupper(ch)); });
|
|
return value;
|
|
}
|
|
|
|
std::optional<std::string> normalize_field(std::string value) {
|
|
const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; };
|
|
value.erase(value.begin(), std::find_if(value.begin(), value.end(),
|
|
[&](char ch) { return !is_space(ch); }));
|
|
value.erase(std::find_if(value.rbegin(), value.rend(),
|
|
[&](char ch) { return !is_space(ch); }).base(),
|
|
value.end());
|
|
if (value.empty()) return std::nullopt;
|
|
return value;
|
|
}
|
|
|
|
void bind_optional(SQLite::Statement& statement,
|
|
const int index,
|
|
const std::optional<std::string>& value) {
|
|
if (value) statement.bind(index, *value);
|
|
else statement.bind(index);
|
|
}
|
|
|
|
bool read_csv_row(std::istream& input,
|
|
std::vector<std::string>& fields,
|
|
const char delimiter) {
|
|
fields.clear();
|
|
std::string field;
|
|
char quote = '\0';
|
|
bool have_data = false;
|
|
while (true) {
|
|
const int next = input.get();
|
|
if (next == EOF) {
|
|
if (!have_data && field.empty() && fields.empty()) return false;
|
|
fields.push_back(std::move(field));
|
|
return true;
|
|
}
|
|
have_data = true;
|
|
const char ch = static_cast<char>(next);
|
|
if (quote != '\0') {
|
|
if (ch == quote) {
|
|
if (input.peek() == quote) {
|
|
input.get();
|
|
field += quote;
|
|
} else {
|
|
quote = '\0';
|
|
}
|
|
} else {
|
|
field += ch;
|
|
}
|
|
} else if ((ch == '"' || ch == '\'') && field.empty()) {
|
|
quote = ch;
|
|
} else if (ch == delimiter) {
|
|
fields.push_back(std::move(field));
|
|
field.clear();
|
|
} else if (ch == '\n') {
|
|
fields.push_back(std::move(field));
|
|
return true;
|
|
} else if (ch != '\r') {
|
|
field += ch;
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string make_upsert_sql(const std::string& table_name,
|
|
const std::vector<std::string>& columns,
|
|
const std::vector<std::string>& primary_key_columns) {
|
|
std::ostringstream sql;
|
|
sql << "INSERT INTO " << quote_identifier(table_name) << " (";
|
|
for (std::size_t index = 0; index < columns.size(); ++index) {
|
|
if (index != 0) sql << ',';
|
|
sql << quote_identifier(columns[index]);
|
|
}
|
|
sql << ") VALUES (";
|
|
for (std::size_t index = 0; index < columns.size(); ++index) {
|
|
if (index != 0) sql << ',';
|
|
sql << '?';
|
|
}
|
|
sql << ')';
|
|
if (!primary_key_columns.empty()) {
|
|
sql << " ON CONFLICT (";
|
|
for (std::size_t index = 0; index < primary_key_columns.size(); ++index) {
|
|
if (index != 0) sql << ',';
|
|
sql << quote_identifier(primary_key_columns[index]);
|
|
}
|
|
sql << ") DO UPDATE SET ";
|
|
bool first = true;
|
|
for (const auto& column : columns) {
|
|
if (std::find(primary_key_columns.begin(), primary_key_columns.end(), column) !=
|
|
primary_key_columns.end()) continue;
|
|
if (!first) sql << ',';
|
|
first = false;
|
|
sql << quote_identifier(column) << "=excluded." << quote_identifier(column);
|
|
}
|
|
}
|
|
return sql.str();
|
|
}
|
|
|
|
void download_to_file(std::string url, const std::filesystem::path& target_file) {
|
|
std::filesystem::create_directories(target_file.parent_path());
|
|
constexpr int kMaxRedirects = 5;
|
|
for (int redirect = 0; redirect <= kMaxRedirects; ++redirect) {
|
|
const auto [origin, path] = split_url(url);
|
|
auto client = drogon::HttpClient::newHttpClient(origin);
|
|
auto request = drogon::HttpRequest::newHttpRequest();
|
|
request->setMethod(drogon::Get);
|
|
request->setPath(path);
|
|
request->addHeader("User-Agent", "ECAP_Server external database updater");
|
|
const auto [result, response] = client->sendRequest(request, 600.0);
|
|
if (result != drogon::ReqResult::Ok || !response) {
|
|
throw std::runtime_error("Download failed: " + url);
|
|
}
|
|
const int status = static_cast<int>(response->getStatusCode());
|
|
if (status >= 300 && status < 400) {
|
|
const auto location = response->getHeader("location");
|
|
if (location.empty()) throw std::runtime_error("Redirect has no location: " + url);
|
|
url = location.front() == '/' ? origin + location : location;
|
|
continue;
|
|
}
|
|
if (status != 200) {
|
|
throw std::runtime_error("Download returned HTTP " + std::to_string(status) +
|
|
": " + url);
|
|
}
|
|
const auto temp_file = target_file.string() + ".tmp";
|
|
std::ofstream output(temp_file, std::ios::binary | std::ios::trunc);
|
|
if (!output) throw std::runtime_error("Cannot write cache file: " + temp_file);
|
|
const auto body = response->getBody();
|
|
output.write(body.data(), static_cast<std::streamsize>(body.size()));
|
|
output.close();
|
|
if (!output) throw std::runtime_error("Cannot finish cache file: " + temp_file);
|
|
replace_file(temp_file, target_file);
|
|
return;
|
|
}
|
|
throw std::runtime_error("Too many redirects while downloading: " + url);
|
|
}
|
|
|
|
void decompress_gzip(const std::filesystem::path& gzip_file,
|
|
const std::filesystem::path& target_file) {
|
|
const auto temp_file = target_file.string() + ".tmp";
|
|
gzFile input = gzopen(gzip_file.string().c_str(), "rb");
|
|
if (!input) throw std::runtime_error("Cannot open gzip file: " + gzip_file.string());
|
|
std::ofstream output(temp_file, std::ios::binary | std::ios::trunc);
|
|
if (!output) {
|
|
gzclose(input);
|
|
throw std::runtime_error("Cannot write decompressed file: " + temp_file);
|
|
}
|
|
char buffer[64 * 1024];
|
|
int size = 0;
|
|
while ((size = gzread(input, buffer, sizeof(buffer))) > 0) output.write(buffer, size);
|
|
const int close_result = gzclose(input);
|
|
output.close();
|
|
if (size < 0 || close_result != Z_OK || !output) {
|
|
std::filesystem::remove(temp_file);
|
|
throw std::runtime_error("Cannot decompress gzip file: " + gzip_file.string());
|
|
}
|
|
replace_file(temp_file, target_file);
|
|
}
|
|
|
|
std::size_t import_header_csv(SQLite::Database& db,
|
|
const std::filesystem::path& source_file,
|
|
const std::string& table_name,
|
|
const std::vector<std::string>& columns,
|
|
const std::vector<std::string>& primary_key_columns,
|
|
const char delimiter) {
|
|
std::ifstream input(source_file, std::ios::binary);
|
|
if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string());
|
|
std::vector<std::string> header;
|
|
if (!read_csv_row(input, header, delimiter)) {
|
|
throw std::runtime_error("CSV file is empty: " + source_file.string());
|
|
}
|
|
if (!header.empty() && header.front().size() >= 3 &&
|
|
static_cast<unsigned char>(header.front()[0]) == 0xef &&
|
|
static_cast<unsigned char>(header.front()[1]) == 0xbb &&
|
|
static_cast<unsigned char>(header.front()[2]) == 0xbf) {
|
|
header.front().erase(0, 3);
|
|
}
|
|
std::map<std::string, std::size_t> header_index;
|
|
for (std::size_t index = 0; index < header.size(); ++index) {
|
|
header_index.emplace(header[index], index);
|
|
}
|
|
for (const auto& column : columns) {
|
|
if (header_index.count(column) == 0) {
|
|
throw std::runtime_error("CSV header is missing column '" + column + "'");
|
|
}
|
|
}
|
|
|
|
SQLite::Transaction transaction(db);
|
|
db.exec("DELETE FROM " + quote_identifier(table_name));
|
|
SQLite::Statement statement(db, make_upsert_sql(table_name, columns, primary_key_columns));
|
|
std::vector<std::string> fields;
|
|
std::size_t row_count = 0;
|
|
while (read_csv_row(input, fields, delimiter)) {
|
|
bool primary_key_ok = true;
|
|
for (const auto& column : primary_key_columns) {
|
|
const auto index = header_index.at(column);
|
|
if (index >= fields.size() || normalize_key(fields[index]).empty()) {
|
|
primary_key_ok = false;
|
|
break;
|
|
}
|
|
}
|
|
if (!primary_key_ok) continue;
|
|
statement.reset();
|
|
statement.clearBindings();
|
|
for (std::size_t index = 0; index < columns.size(); ++index) {
|
|
std::optional<std::string> value;
|
|
const auto source_index = header_index.at(columns[index]);
|
|
if (source_index < fields.size()) value = normalize_field(fields[source_index]);
|
|
if (value &&
|
|
std::find(primary_key_columns.begin(), primary_key_columns.end(),
|
|
columns[index]) != primary_key_columns.end()) {
|
|
value = normalize_key(*value);
|
|
}
|
|
bind_optional(statement, static_cast<int>(index + 1), value);
|
|
}
|
|
statement.exec();
|
|
++row_count;
|
|
}
|
|
transaction.commit();
|
|
return row_count;
|
|
}
|
|
|
|
std::unique_ptr<External_Resources> make_header_csv_resource(
|
|
Header_Csv_Resource_Config config) {
|
|
return std::make_unique<Header_Csv_Resource>(std::move(config));
|
|
}
|
|
|
|
std::unique_ptr<External_Resources> make_schema_only_resource(
|
|
Schema_Only_Resource_Config config) {
|
|
return std::make_unique<Schema_Only_Resource>(std::move(config));
|
|
}
|
|
|
|
std::unique_ptr<External_Resources> make_tar1090_aircraft_resource(
|
|
std::string name,
|
|
std::string description,
|
|
const std::uint64_t update_interval_seconds) {
|
|
return std::make_unique<Tar1090_Aircraft_Resource>(
|
|
std::move(name), std::move(description), update_interval_seconds);
|
|
}
|
|
|
|
} // 命名空间 External_Database_Utils
|