#include "Resource_Utils.h" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace External_Database_Utils { namespace { constexpr const char* kTar1090Url = "https://raw.githubusercontent.com/wiedehopf/tar1090-db/refs/heads/csv/aircraft.csv.gz"; const std::vector kTar1090Columns = { "icao24", "registration", "type_code", "subtype", "type_name", "year", "operator_info" }; const std::vector kTar1090PrimaryKey = {"icao24"}; std::pair split_url(const std::string& url) { const auto scheme_end = url.find("://"); if (scheme_end == std::string::npos) { throw std::runtime_error("Invalid download URL: " + url); } const auto path_begin = url.find('/', scheme_end + 3); if (path_begin == std::string::npos) return {url, "/"}; return {url.substr(0, path_begin), url.substr(path_begin)}; } void replace_file(const std::filesystem::path& temp_file, const std::filesystem::path& target_file) { std::error_code error; std::filesystem::remove(target_file, error); error.clear(); std::filesystem::rename(temp_file, target_file, error); if (error) { throw std::runtime_error("Cannot replace cache file " + target_file.string() + ": " + error.message()); } } std::string create_csv_table_sql(const std::string& table_name, const std::vector& columns, const std::vector& primary_key_columns) { std::ostringstream sql; sql << "CREATE TABLE IF NOT EXISTS " << quote_identifier(table_name) << " ("; for (std::size_t index = 0; index < columns.size(); ++index) { if (index != 0) sql << ','; sql << quote_identifier(columns[index]) << " TEXT"; } if (!primary_key_columns.empty()) { sql << ", PRIMARY KEY ("; for (std::size_t index = 0; index < primary_key_columns.size(); ++index) { if (index != 0) sql << ','; sql << quote_identifier(primary_key_columns[index]); } sql << ')'; } sql << ");"; return sql.str(); } class Header_Csv_Resource final : public External_Resources { public: explicit Header_Csv_Resource(Header_Csv_Resource_Config config) : External_Resources(config.name, config.cache_file_name, config.download_url, config.description, config.update_interval_seconds), config_(std::move(config)) {} void create_table(SQLite::Database& db) const override { db.exec(create_csv_table_sql(name, config_.columns, config_.primary_key_columns)); } protected: std::size_t import_file(SQLite::Database& db, const std::filesystem::path& source_file) const override { return import_header_csv(db, source_file, name, config_.columns, config_.primary_key_columns, config_.delimiter); } [[nodiscard]] const std::vector& primary_key_columns() const override { return config_.primary_key_columns; } private: Header_Csv_Resource_Config config_; }; class Schema_Only_Resource final : public External_Resources { public: explicit Schema_Only_Resource(Schema_Only_Resource_Config config) : External_Resources(config.name, "", "", config.description, config.update_interval_seconds), config_(std::move(config)) {} void create_table(SQLite::Database& db) const override { db.exec(config_.create_table_sql); } protected: std::size_t import_file(SQLite::Database&, const std::filesystem::path&) const override { throw std::runtime_error("This resource is updated by its API-specific adapter"); } [[nodiscard]] const std::vector& primary_key_columns() const override { return config_.primary_key_columns; } private: Schema_Only_Resource_Config config_; }; class Tar1090_Aircraft_Resource final : public External_Resources { public: Tar1090_Aircraft_Resource(std::string name, std::string description, const std::uint64_t update_interval_seconds) : External_Resources(std::move(name), "aircraft.csv", kTar1090Url, std::move(description), update_interval_seconds) {} void create_table(SQLite::Database& db) const override { db.exec(create_csv_table_sql(name, kTar1090Columns, kTar1090PrimaryKey)); } protected: bool fetch(const std::filesystem::path& cache_pos, const bool force_download) const override { const auto csv_file = source_file(cache_pos); if (!force_download) return false; const auto gzip_file = cache_pos / "aircraft.csv.gz"; download_to_file(download_url, gzip_file); decompress_gzip(gzip_file, csv_file); return true; } std::size_t import_file(SQLite::Database& db, const std::filesystem::path& source_file) const override { if (source_file.extension() == ".gz") { auto csv_file = source_file; csv_file.replace_extension(".csv"); decompress_gzip(source_file, csv_file); return import_file(db, csv_file); } std::ifstream input(source_file, std::ios::binary); if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string()); SQLite::Transaction transaction(db); db.exec("DELETE FROM " + quote_identifier(name)); SQLite::Statement statement( db, make_upsert_sql(name, kTar1090Columns, kTar1090PrimaryKey)); std::vector fields; std::size_t row_count = 0; while (read_csv_row(input, fields, ';')) { if (fields.empty()) continue; const auto icao = normalize_key(fields[0]); if (icao.empty()) continue; statement.reset(); statement.clearBindings(); statement.bind(1, icao); for (std::size_t index = 1; index < kTar1090Columns.size(); ++index) { std::optional value; if (index < fields.size()) value = normalize_field(fields[index]); bind_optional(statement, static_cast(index + 1), value); } statement.exec(); ++row_count; } transaction.commit(); return row_count; } [[nodiscard]] const std::vector& primary_key_columns() const override { return kTar1090PrimaryKey; } }; } // 命名空间 std::string quote_identifier(const std::string& value) { std::string result = "\""; for (const char ch : value) { result += ch; if (ch == '"') result += '"'; } result += '"'; return result; } std::string normalize_key(std::string value) { const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; }; value.erase(value.begin(), std::find_if(value.begin(), value.end(), [&](char ch) { return !is_space(ch); })); value.erase(std::find_if(value.rbegin(), value.rend(), [&](char ch) { return !is_space(ch); }).base(), value.end()); std::transform(value.begin(), value.end(), value.begin(), [](unsigned char ch) { return static_cast(std::toupper(ch)); }); return value; } std::optional normalize_field(std::string value) { const auto is_space = [](unsigned char ch) { return std::isspace(ch) != 0; }; value.erase(value.begin(), std::find_if(value.begin(), value.end(), [&](char ch) { return !is_space(ch); })); value.erase(std::find_if(value.rbegin(), value.rend(), [&](char ch) { return !is_space(ch); }).base(), value.end()); if (value.empty()) return std::nullopt; return value; } void bind_optional(SQLite::Statement& statement, const int index, const std::optional& value) { if (value) statement.bind(index, *value); else statement.bind(index); } bool read_csv_row(std::istream& input, std::vector& fields, const char delimiter) { fields.clear(); std::string field; char quote = '\0'; bool have_data = false; while (true) { const int next = input.get(); if (next == EOF) { if (!have_data && field.empty() && fields.empty()) return false; fields.push_back(std::move(field)); return true; } have_data = true; const char ch = static_cast(next); if (quote != '\0') { if (ch == quote) { if (input.peek() == quote) { input.get(); field += quote; } else { quote = '\0'; } } else { field += ch; } } else if ((ch == '"' || ch == '\'') && field.empty()) { quote = ch; } else if (ch == delimiter) { fields.push_back(std::move(field)); field.clear(); } else if (ch == '\n') { fields.push_back(std::move(field)); return true; } else if (ch != '\r') { field += ch; } } } std::string make_upsert_sql(const std::string& table_name, const std::vector& columns, const std::vector& primary_key_columns) { std::ostringstream sql; sql << "INSERT INTO " << quote_identifier(table_name) << " ("; for (std::size_t index = 0; index < columns.size(); ++index) { if (index != 0) sql << ','; sql << quote_identifier(columns[index]); } sql << ") VALUES ("; for (std::size_t index = 0; index < columns.size(); ++index) { if (index != 0) sql << ','; sql << '?'; } sql << ')'; if (!primary_key_columns.empty()) { sql << " ON CONFLICT ("; for (std::size_t index = 0; index < primary_key_columns.size(); ++index) { if (index != 0) sql << ','; sql << quote_identifier(primary_key_columns[index]); } sql << ") DO UPDATE SET "; bool first = true; for (const auto& column : columns) { if (std::find(primary_key_columns.begin(), primary_key_columns.end(), column) != primary_key_columns.end()) continue; if (!first) sql << ','; first = false; sql << quote_identifier(column) << "=excluded." << quote_identifier(column); } } return sql.str(); } void download_to_file(std::string url, const std::filesystem::path& target_file) { std::filesystem::create_directories(target_file.parent_path()); constexpr int kMaxRedirects = 5; for (int redirect = 0; redirect <= kMaxRedirects; ++redirect) { const auto [origin, path] = split_url(url); auto client = drogon::HttpClient::newHttpClient(origin); auto request = drogon::HttpRequest::newHttpRequest(); request->setMethod(drogon::Get); request->setPath(path); request->addHeader("User-Agent", "ECAP_Server external database updater"); const auto [result, response] = client->sendRequest(request, 600.0); if (result != drogon::ReqResult::Ok || !response) { throw std::runtime_error("Download failed: " + url); } const int status = static_cast(response->getStatusCode()); if (status >= 300 && status < 400) { const auto location = response->getHeader("location"); if (location.empty()) throw std::runtime_error("Redirect has no location: " + url); url = location.front() == '/' ? origin + location : location; continue; } if (status != 200) { throw std::runtime_error("Download returned HTTP " + std::to_string(status) + ": " + url); } const auto temp_file = target_file.string() + ".tmp"; std::ofstream output(temp_file, std::ios::binary | std::ios::trunc); if (!output) throw std::runtime_error("Cannot write cache file: " + temp_file); const auto body = response->getBody(); output.write(body.data(), static_cast(body.size())); output.close(); if (!output) throw std::runtime_error("Cannot finish cache file: " + temp_file); replace_file(temp_file, target_file); return; } throw std::runtime_error("Too many redirects while downloading: " + url); } void decompress_gzip(const std::filesystem::path& gzip_file, const std::filesystem::path& target_file) { const auto temp_file = target_file.string() + ".tmp"; gzFile input = gzopen(gzip_file.string().c_str(), "rb"); if (!input) throw std::runtime_error("Cannot open gzip file: " + gzip_file.string()); std::ofstream output(temp_file, std::ios::binary | std::ios::trunc); if (!output) { gzclose(input); throw std::runtime_error("Cannot write decompressed file: " + temp_file); } char buffer[64 * 1024]; int size = 0; while ((size = gzread(input, buffer, sizeof(buffer))) > 0) output.write(buffer, size); const int close_result = gzclose(input); output.close(); if (size < 0 || close_result != Z_OK || !output) { std::filesystem::remove(temp_file); throw std::runtime_error("Cannot decompress gzip file: " + gzip_file.string()); } replace_file(temp_file, target_file); } std::size_t import_header_csv(SQLite::Database& db, const std::filesystem::path& source_file, const std::string& table_name, const std::vector& columns, const std::vector& primary_key_columns, const char delimiter) { std::ifstream input(source_file, std::ios::binary); if (!input) throw std::runtime_error("Cannot open CSV file: " + source_file.string()); std::vector header; if (!read_csv_row(input, header, delimiter)) { throw std::runtime_error("CSV file is empty: " + source_file.string()); } if (!header.empty() && header.front().size() >= 3 && static_cast(header.front()[0]) == 0xef && static_cast(header.front()[1]) == 0xbb && static_cast(header.front()[2]) == 0xbf) { header.front().erase(0, 3); } std::map header_index; for (std::size_t index = 0; index < header.size(); ++index) { header_index.emplace(header[index], index); } for (const auto& column : columns) { if (header_index.count(column) == 0) { throw std::runtime_error("CSV header is missing column '" + column + "'"); } } SQLite::Transaction transaction(db); db.exec("DELETE FROM " + quote_identifier(table_name)); SQLite::Statement statement(db, make_upsert_sql(table_name, columns, primary_key_columns)); std::vector fields; std::size_t row_count = 0; while (read_csv_row(input, fields, delimiter)) { bool primary_key_ok = true; for (const auto& column : primary_key_columns) { const auto index = header_index.at(column); if (index >= fields.size() || normalize_key(fields[index]).empty()) { primary_key_ok = false; break; } } if (!primary_key_ok) continue; statement.reset(); statement.clearBindings(); for (std::size_t index = 0; index < columns.size(); ++index) { std::optional value; const auto source_index = header_index.at(columns[index]); if (source_index < fields.size()) value = normalize_field(fields[source_index]); if (value && std::find(primary_key_columns.begin(), primary_key_columns.end(), columns[index]) != primary_key_columns.end()) { value = normalize_key(*value); } bind_optional(statement, static_cast(index + 1), value); } statement.exec(); ++row_count; } transaction.commit(); return row_count; } std::unique_ptr make_header_csv_resource( Header_Csv_Resource_Config config) { return std::make_unique(std::move(config)); } std::unique_ptr make_schema_only_resource( Schema_Only_Resource_Config config) { return std::make_unique(std::move(config)); } std::unique_ptr make_tar1090_aircraft_resource( std::string name, std::string description, const std::uint64_t update_interval_seconds) { return std::make_unique( std::move(name), std::move(description), update_interval_seconds); } } // 命名空间 External_Database_Utils