ECAP_Server/module/Local_Server/External_Database/External_Database.cpp

#include "global.h"

#include "../server/Global.h"
#include "../server/Performance_Monitor.h"
#include "Core/Base/Coro_Result.h"
#include "Resource_Utils.h"
#include "Resources.h"

#include <SQLiteCpp/Statement.h>
#include <SQLiteCpp/Transaction.h>
#include <drogon/MultiPart.h>
#include <trantor/utils/ConcurrentTaskQueue.h>

#include <chrono>
#include <iostream>
#include <set>
#include <stdexcept>
#include <utility>
#include <variant>

namespace {
Psc::JSON
external_resource_status_to_json(const External_Resource_Status &status) {
  return Psc::JSON::object({
      {"name", status.name},
      {"row_count", status.row_count},
      {"downloaded", status.downloaded},
      {"imported", status.imported},
      {"message", status.message},
  });
}

Psc::JSON external_resource_status_list_to_json(
    const std::vector<External_Resource_Status> &status_list) {
  auto result = Psc::JSON::array();
  for (const auto &status : status_list) {
    result.append(external_resource_status_to_json(status));
  }
  return result;
}

trantor::ConcurrentTaskQueue &external_database_task_queue() {
  static trantor::ConcurrentTaskQueue queue(2, "external_database");
  return queue;
}

template <typename T, typename Work, typename Callback>
void run_external_database_async(Work &&work, Callback &&callback) {
  auto callback_holder = std::make_shared<std::decay_t<Callback>>(
      std::forward<Callback>(callback));

  external_database_task_queue().runTaskInQueue(
      [work = std::forward<Work>(work), callback_holder]() mutable {
        try {
          (*callback_holder)(nullptr, work());
        } catch (...) {
          (*callback_holder)(std::current_exception(), T{});
        }
      });
}

template <typename T, typename Starter>
concurrencpp::result<T> await_external_database_callback(Starter starter) {
  auto result = co_await Psc::coro::callback_result<T>(
      [starter = std::move(starter)](auto done) mutable {
        starter([done = std::move(done)](std::exception_ptr exception,
                                         T value) mutable {
          if (exception) {
            done.set_exception(exception);
            return;
          }
          done(std::move(value));
        });
      });
  co_return std::move(result);
}
} // namespace

std::optional<std::string>
External_Database_Row::get(const std::string &column) const {
  const auto iter = columns.find(column);
  return iter == columns.end() ? std::nullopt : iter->second;
}

Psc::JSON External_Database_Row::to_json() const {
  auto result = Psc::JSON::object();
  for (const auto &[column, value] : columns) {
    result.append({column, value ? Psc::JSON(*value) : Psc::JSON(nullptr)});
  }
  return result;
}

External_Resources::External_Resources(
    std::string name, std::string cache_file_name, std::string download_url,
    std::string description, const std::uint64_t update_interval_seconds)
    : name(std::move(name)), cache_file_name(std::move(cache_file_name)),
      download_url(std::move(download_url)),
      description(std::move(description)),
      update_interval_seconds(update_interval_seconds) {}

void External_Resources::mark_updated() {
  last_updated_at = std::time(nullptr);
}

void External_Resources::clear_last_updated_at() { last_updated_at = 0; }

std::size_t External_Resources::row_count(SQLite::Database &db) const {
  return static_cast<std::size_t>(
      db.execAndGet("SELECT COUNT(*) FROM " +
                    External_Database_Utils::quote_identifier(name))
          .getInt64());
}

std::filesystem::path
External_Resources::source_file(const std::filesystem::path &cache_pos) const {
  return cache_pos / cache_file_name;
}

bool External_Resources::fetch(const std::filesystem::path &cache_pos,
                               const bool force_download) const {
  const auto target_file = source_file(cache_pos);
  if (!force_download)
    return false;
  if (download_url.empty())
    return false;
  External_Database_Utils::download_to_file(download_url, target_file);
  return true;
}

External_Resource_Status External_Resources::update(
    SQLite::Database &db, const std::filesystem::path &cache_pos,
    const bool force_download, const bool only_when_empty) {
  create_table(db);
  External_Resource_Status result{name, row_count(db), false, false, ""};
  if (only_when_empty && result.row_count != 0) {
    result.message = "table already populated";
    return result;
  }

  std::filesystem::create_directories(cache_pos);
  result.downloaded = fetch(cache_pos, force_download);
  if (cache_file_name.empty()) {
    result.message = "dynamic query required";
    return result;
  }
  const auto local_file = source_file(cache_pos);
  if (!std::filesystem::exists(local_file)) {
    result.message = download_url.empty()
                         ? "offline import or dynamic query required"
                         : "cache file unavailable";
    return result;
  }

  result.row_count = import_file(db, local_file);
  result.imported = true;
  result.message = "ok";
  mark_updated();
  return result;
}

External_Resource_Status
External_Resources::import_offline(SQLite::Database &db,
                                   const std::filesystem::path &source_file) {
  create_table(db);
  External_Resource_Status result{name, 0, false, false, ""};
  result.row_count = import_file(db, source_file);
  result.imported = true;
  result.message = "ok";
  mark_updated();
  return result;
}

External_Resource_Status
External_Resources::clear_table(SQLite::Database &db,
                                const std::filesystem::path &cache_pos) {
  SQLite::Transaction transaction(db);
  db.exec("DROP TABLE IF EXISTS " +
          External_Database_Utils::quote_identifier(name));
  create_table(db);
  transaction.commit();

  if (!cache_file_name.empty()) {
    std::error_code error;
    std::filesystem::remove(source_file(cache_pos), error);
    if (error) {
      throw std::runtime_error("Cannot remove cache file for resource " + name +
                               ": " + error.message());
    }
  }

  clear_last_updated_at();
  return {name, 0, false, false, "table cleared"};
}

std::optional<External_Database_Row> External_Resources::query_one(
    SQLite::Database &db,
    const std::vector<std::string> &primary_key_values) const {
  const auto &columns = primary_key_columns();
  if (columns.size() != primary_key_values.size()) {
    throw std::invalid_argument(
        "primary key value count mismatch for resource: " + name);
  }

  std::string sql = "SELECT * FROM " +
                    External_Database_Utils::quote_identifier(name) + " WHERE ";
  for (std::size_t index = 0; index < columns.size(); ++index) {
    if (index != 0)
      sql += " AND ";
    sql += External_Database_Utils::quote_identifier(columns[index]) +
           " = ? COLLATE NOCASE";
  }
  sql += " LIMIT 1";

  SQLite::Statement statement(db, sql);
  for (std::size_t index = 0; index < primary_key_values.size(); ++index) {
    statement.bind(
        static_cast<int>(index + 1),
        External_Database_Utils::normalize_key(primary_key_values[index]));
  }
  if (!statement.executeStep())
    return std::nullopt;

  External_Database_Row row;
  for (int index = 0; index < statement.getColumnCount(); ++index) {
    const auto name = statement.getColumnName(index);
    if (statement.isColumnNull(index)) {
      row.columns.emplace(name, std::nullopt);
    } else {
      row.columns.emplace(name, statement.getColumn(index).getString());
    }
  }
  return row;
}

External_Resources_Manager::External_Resources_Manager() {
  register_external_resource(make_tar1090_db_aircraft_csv_gz_resource());
  register_external_resource(make_wiedehopf_tar1090_db_resource());
  register_external_resource(make_mictronics_aircraft_database_resource());
  register_external_resource(make_opensky_aircraft_database_resource());
  register_external_resource(make_faa_aircraft_registry_resource());
  register_external_resource(make_icao_doc_8643_resource());
  register_external_resource(make_vrs_routes_resource());
  register_external_resource(make_vrs_airports_resource());
  register_external_resource(make_adsblol_vrs_standing_data_resource());
  register_external_resource(make_vradarserver_standing_data_resource());
  register_external_resource(make_opensky_flightdata_api_resource());
}

void External_Resources_Manager::init(const Psc::JSON *that_json) {
  if (!that_json)
    return;
  const auto list = that_json->get("list");
  if (!list)
    return;
  std::lock_guard lock(mtx_);
  for (const auto &config : list->children) {
    const auto name = config.try_get_string("name");
    if (!name)
      continue;
    for (const auto &resource : resources_) {
      if (resource->name != *name)
        continue;
      resource->init(&config);
      break;
    }
  }
  Get_J(ecap_sqlite_path)
}

Psc::JSON External_Resources_Manager::to_json() const {
  std::lock_guard lock(mtx_);
  auto ret = Psc::JSON::object();
  Ret_J(ecap_sqlite_path) auto list = Psc::JSON::array();
  for (const std::unique_ptr<External_Resources> &resource : resources_) {
    list.children.emplace_back(resource->to_json());
  }
  ret.children.emplace_back(Psc::JSON{"list", list});
  return ret;
}

void External_Resources_Manager::load_sqlite_db(std::string path) {
  try {
    db_ = std::make_unique<SQLite::Database>(path, SQLite::OPEN_READWRITE |
                                                       SQLite::OPEN_CREATE);
  } catch (SQLite::Exception &error) {
    Psc::fail_fast(std::string("load_sqlite_db error: ") + error.what());
  }

  db_->exec("PRAGMA journal_mode=WAL;");
  db_->exec("PRAGMA synchronous=NORMAL;");
  initialize(*db_, std::filesystem::path(path).parent_path());
  sync_missing(*db_);
}

std::vector<External_Resource_Status>
External_Resources_Manager::refresh_external_databases(
    const bool force_download) {
  return refresh_all(require_db(), force_download);
}

External_Resource_Status External_Resources_Manager::refresh_external_database(
    const std::string &resource_name, const bool force_download) {
  return refresh_one(require_db(), resource_name, force_download);
}

External_Resource_Status External_Resources_Manager::import_external_database(
    const std::string &resource_name,
    const std::filesystem::path &source_file) {
  return import_offline(require_db(), resource_name, source_file);
}

External_Resource_Status
External_Resources_Manager::clear_external_database_table(
    const std::string &resource_name) {
  return clear_table(require_db(), resource_name);
}

std::optional<External_Database_Row>
External_Resources_Manager::query_external_database(
    const std::string &resource_name,
    const std::vector<std::string> &primary_key_values) const {
  return query_one(require_db(), resource_name, primary_key_values);
}

std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_aircraft_external_databases(
    const std::string &icao24) const {
  return query_aircraft(require_db(), icao24);
}

std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_callsign_external_databases(
    const std::optional<std::string> &callsign) const {
  return query_callsign(require_db(), callsign);
}

std::vector<External_Resource_Status>
External_Resources_Manager::external_database_status() const {
  return status(require_db());
}

void External_Resources_Manager::async_refresh_external_databases(
    const bool force_download, Status_List_Callback callback) {
  run_external_database_async<std::vector<External_Resource_Status>>(
      [this, force_download]() {
        return refresh_external_databases(force_download);
      },
      std::move(callback));
}

void External_Resources_Manager::async_refresh_external_database(
    std::string resource_name, const bool force_download,
    Status_Callback callback) {
  run_external_database_async<External_Resource_Status>(
      [this, resource_name = std::move(resource_name), force_download]() {
        return refresh_external_database(resource_name, force_download);
      },
      std::move(callback));
}

void External_Resources_Manager::async_import_external_database(
    std::string resource_name, std::filesystem::path source_file,
    Status_Callback callback) {
  run_external_database_async<External_Resource_Status>(
      [this, resource_name = std::move(resource_name),
       source_file = std::move(source_file)]() {
        return import_external_database(resource_name, source_file);
      },
      std::move(callback));
}

void External_Resources_Manager::async_clear_external_database_table(
    std::string resource_name, Status_Callback callback) {
  run_external_database_async<External_Resource_Status>(
      [this, resource_name = std::move(resource_name)]() {
        return clear_external_database_table(resource_name);
      },
      std::move(callback));
}

void External_Resources_Manager::async_query_external_database(
    std::string resource_name, std::vector<std::string> primary_key_values,
    Row_Callback callback) const {
  run_external_database_async<std::optional<External_Database_Row>>(
      [this, resource_name = std::move(resource_name),
       primary_key_values = std::move(primary_key_values)]() {
        return query_external_database(resource_name, primary_key_values);
      },
      std::move(callback));
}

void External_Resources_Manager::async_query_aircraft_external_databases(
    std::string icao24, Row_Map_Callback callback) const {
  run_external_database_async<
      std::map<std::string, std::shared_ptr<External_Database_Row>>>(
      [this, icao24 = std::move(icao24)]() {
        return query_aircraft_external_databases(icao24);
      },
      std::move(callback));
}

void External_Resources_Manager::async_query_callsign_external_databases(
    std::optional<std::string> callsign, Row_Map_Callback callback) const {
  run_external_database_async<
      std::map<std::string, std::shared_ptr<External_Database_Row>>>(
      [this, callsign = std::move(callsign)]() {
        return query_callsign_external_databases(callsign);
      },
      std::move(callback));
}

void External_Resources_Manager::async_external_database_status(
    Status_List_Callback callback) const {
  run_external_database_async<std::vector<External_Resource_Status>>(
      [this]() { return external_database_status(); }, std::move(callback));
}

concurrencpp::result<std::vector<External_Resource_Status>>
External_Resources_Manager::refresh_external_databases_coro(
    const bool force_download) {
  co_return co_await await_external_database_callback<
      std::vector<External_Resource_Status>>(
      [this, force_download](Status_List_Callback callback) {
        async_refresh_external_databases(force_download, std::move(callback));
      });
}

concurrencpp::result<External_Resource_Status>
External_Resources_Manager::refresh_external_database_coro(
    std::string resource_name, const bool force_download) {
  co_return co_await await_external_database_callback<External_Resource_Status>(
      [this, resource_name = std::move(resource_name),
       force_download](Status_Callback callback) mutable {
        async_refresh_external_database(std::move(resource_name),
                                        force_download, std::move(callback));
      });
}

concurrencpp::result<External_Resource_Status>
External_Resources_Manager::import_external_database_coro(
    std::string resource_name, std::filesystem::path source_file) {
  co_return co_await await_external_database_callback<External_Resource_Status>(
      [this, resource_name = std::move(resource_name),
       source_file = std::move(source_file)](Status_Callback callback) mutable {
        async_import_external_database(std::move(resource_name),
                                       std::move(source_file),
                                       std::move(callback));
      });
}

concurrencpp::result<External_Resource_Status>
External_Resources_Manager::clear_external_database_table_coro(
    std::string resource_name) {
  co_return co_await await_external_database_callback<External_Resource_Status>(
      [this, resource_name =
                 std::move(resource_name)](Status_Callback callback) mutable {
        async_clear_external_database_table(std::move(resource_name),
                                            std::move(callback));
      });
}

concurrencpp::result<std::optional<External_Database_Row>>
External_Resources_Manager::query_external_database_coro(
    std::string resource_name,
    std::vector<std::string> primary_key_values) const {
  co_return co_await await_external_database_callback<
      std::optional<External_Database_Row>>(
      [this, resource_name = std::move(resource_name),
       primary_key_values =
           std::move(primary_key_values)](Row_Callback callback) mutable {
        async_query_external_database(std::move(resource_name),
                                      std::move(primary_key_values),
                                      std::move(callback));
      });
}

concurrencpp::result<std::shared_ptr<External_Database_Row_Map>>
External_Resources_Manager::query_aircraft_external_databases_coro(
    std::string icao24) const {
  auto result = co_await Psc::coro::callback_result<
      std::shared_ptr<External_Database_Row_Map>>(
      [this, icao24 = std::move(icao24)](auto done) mutable {
        async_query_aircraft_external_databases(
            std::move(icao24),
            [done = std::move(done)](std::exception_ptr exception,
                                     External_Database_Row_Map value) mutable {
              if (exception) {
                done.set_exception(exception);
                return;
              }
              done(std::make_shared<External_Database_Row_Map>(
                  std::move(value)));
            });
      });
  co_return result;
}

concurrencpp::result<std::shared_ptr<External_Database_Row_Map>>
External_Resources_Manager::query_callsign_external_databases_coro(
    std::optional<std::string> callsign) const {
  auto result = co_await Psc::coro::callback_result<
      std::shared_ptr<External_Database_Row_Map>>(
      [this, callsign = std::move(callsign)](auto done) mutable {
        async_query_callsign_external_databases(
            std::move(callsign),
            [done = std::move(done)](std::exception_ptr exception,
                                     External_Database_Row_Map value) mutable {
              if (exception) {
                done.set_exception(exception);
                return;
              }
              done(std::make_shared<External_Database_Row_Map>(
                  std::move(value)));
            });
      });
  co_return result;
}

concurrencpp::result<std::vector<External_Resource_Status>>
External_Resources_Manager::external_database_status_coro() const {
  co_return co_await await_external_database_callback<
      std::vector<External_Resource_Status>>(
      [this](Status_List_Callback callback) {
        async_external_database_status(std::move(callback));
      });
}

void External_Resources_Manager::server(Global *g) {
  auto &svr = g->svr;
  const auto &api = g->api;

  svr.Post(api + "get_external_database_config", [this](HTTP_Param) {
    auto t = warp(to_json()).to_json_string();

    res->setBody(t);
  });
  svr.PostCoro(
      api + "get_external_database_status",
      [this](HTTP_Param) -> drogon::Task<> {
        auto status_list =
            co_await Ecap_Coro::to_drogon(external_database_status_coro());

        res->setBody(warp(external_resource_status_list_to_json(status_list))
                         .to_json_string());
        co_return;
      });
  svr.PostCoro(
      api + "refresh_external_database",
      [this, g](HTTP_Param) -> drogon::Task<> {
        CHECK_JSON_PARAM
        HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
        const auto result = co_await Ecap_Coro::to_drogon(
            refresh_external_database_coro(name, true));
        g->save();
        res->setBody(
            warp(external_resource_status_to_json(result)).to_json_string());
        co_return;
      });
  svr.PostCoro(api + "refresh_external_databases",
               [this, g](HTTP_Param) -> drogon::Task<> {
                 const auto result = co_await Ecap_Coro::to_drogon(
                     refresh_external_databases_coro(true));
                 g->save();
                 res->setBody(
                     warp(external_resource_status_list_to_json(result))
                         .to_json_string());
                 co_return;
               });
  svr.PostCoro(
      api + "import_external_database",
      [this, g](HTTP_Param) -> drogon::Task<> {
        CHECK_JSON_PARAM
        HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
        HTTP_REQUIRE_VALUE(source_file, params.try_get_string("source_file"))
        const auto result = co_await Ecap_Coro::to_drogon(
            import_external_database_coro(name, source_file));
        g->save();
        res->setBody(
            warp(external_resource_status_to_json(result)).to_json_string());
        co_return;
      });
  svr.PostCoro(
      api + "upload_external_database",
      [this, g](HTTP_Param) -> drogon::Task<> {
        drogon::MultiPartParser parser;
        if (parser.parse(req) != 0) {
          throw_invalid_http_param("multipart");
        }
        const auto name = parser.getOptionalParameter<std::string>("name");
        if (!name || parser.getFiles().size() != 1) {
          throw_invalid_http_param("name or file");
        }

        const auto &upload = parser.getFiles().front();
        const auto serial =
            std::chrono::steady_clock::now().time_since_epoch().count();
        auto extension = std::string(upload.getFileExtension());
        if (!extension.empty() && extension.front() != '.') {
          extension.insert(extension.begin(), '.');
        }
        const auto saved_name =
            "external_database_" + std::to_string(serial) + extension;
        if (upload.saveAs(saved_name) != 0) {
          throw std::runtime_error(
              "Cannot save uploaded external database file");
        }

        const auto source_file =
            std::filesystem::path(drogon::app().getUploadPath()) / saved_name;
        const auto result = co_await Ecap_Coro::to_drogon(
            import_external_database_coro(*name, source_file));
        g->save();
        res->setBody(
            warp(external_resource_status_to_json(result)).to_json_string());
        co_return;
      });
  svr.PostCoro(
      api + "clear_external_database_table",
      [this, g](HTTP_Param) -> drogon::Task<> {
        CHECK_JSON_PARAM
        HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
        const auto result = co_await Ecap_Coro::to_drogon(
            clear_external_database_table_coro(name));
        g->save();
        res->setBody(
            warp(external_resource_status_to_json(result)).to_json_string());
        co_return;
      });
  svr.PostCoro(
      api + "query_external_database", [this](HTTP_Param) -> drogon::Task<> {
        CHECK_JSON_PARAM
        HTTP_REQUIRE_VALUE(name, params.try_get_string("name"))
        HTTP_REQUIRE_PTR(primary_key_values_json,
                         params.get("primary_key_values"))
        std::vector<std::string> primary_key_values;
        for (const auto &value : primary_key_values_json->children) {
          if (value.valueType != Psc::JsonType::String) {
            throw_invalid_http_param("primary_key_values");
          }
          primary_key_values.push_back(value.val);
        }
        const auto row = co_await Ecap_Coro::to_drogon(
            query_external_database_coro(name, std::move(primary_key_values)));
        res->setBody(row ? warp(row->to_json()).to_json_string()
                         : Psc::JSON(nullptr).to_json_string());
        co_return;
      });
}

void External_Resources_Manager::register_external_resource(
    std::unique_ptr<External_Resources> external_res) {
  if (!external_res)
    throw std::invalid_argument("external resource is null");
  resources_.push_back(std::move(external_res));
}

void External_Resources_Manager::initialize(SQLite::Database &db,
                                            std::filesystem::path cache_pos) {
  std::lock_guard lock(mtx_);
  cache_pos_ = std::move(cache_pos);
  for (const auto &resource : resources_)
    resource->create_table(db);
}

std::vector<External_Resource_Status>
External_Resources_Manager::sync_missing(SQLite::Database &db) {
  return update_all(db, false, true);
}

std::vector<External_Resource_Status>
External_Resources_Manager::refresh_all(SQLite::Database &db,
                                        const bool force_download) {
  return update_all(db, force_download, false);
}

External_Resource_Status
External_Resources_Manager::refresh_one(SQLite::Database &db,
                                        const std::string &resource_name,
                                        const bool force_download) {
  std::lock_guard lock(mtx_);
  return get_resource(resource_name)
      .update(db, cache_pos_, force_download, false);
}

External_Resource_Status External_Resources_Manager::import_offline(
    SQLite::Database &db, const std::string &resource_name,
    const std::filesystem::path &source_file) {
  std::lock_guard lock(mtx_);
  return get_resource(resource_name).import_offline(db, source_file);
}

External_Resource_Status
External_Resources_Manager::clear_table(SQLite::Database &db,
                                        const std::string &resource_name) {
  std::lock_guard lock(mtx_);
  return get_resource(resource_name).clear_table(db, cache_pos_);
}

std::optional<External_Database_Row> External_Resources_Manager::query_one(
    SQLite::Database &db, const std::string &resource_name,
    const std::vector<std::string> &primary_key_values) const {
  std::lock_guard lock(mtx_);
  return get_resource(resource_name).query_one(db, primary_key_values);
}

std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_aircraft(SQLite::Database &db,
                                           const std::string &icao24) const {
  Scope_Timer timer(
      "external_database.query_aircraft",
      Global::instance()->http_monitor_config.slow_scope_threshold_ms);
  static const std::vector<std::string> kAircraftResourceKeys = {
      "tar1090_db_aircraft",          "wiedehopf_tar1090_db_aircraft",
      "mictronics_aircraft_database", "opensky_aircraft_database",
      "faa_aircraft_registry",
  };
  std::lock_guard lock(mtx_);
  std::map<std::string, std::shared_ptr<External_Database_Row>> result;
  const auto add_row = [&](const std::string &resource_name,
                           const std::vector<std::string> &primary_key_values,
                           const std::string &result_name = std::string{}) {
    auto &resource = get_resource(resource_name);
    auto row = resource.query_one(db, primary_key_values);
    if (!row)
      return std::shared_ptr<External_Database_Row>{};
    auto shared_row = std::make_shared<External_Database_Row>(std::move(*row));
    result.emplace(result_name.empty() ? resource_name : result_name,
                   shared_row);
    return shared_row;
  };

  std::optional<std::string> type_code;
  for (const auto &resource_name : kAircraftResourceKeys) {
    const auto row = add_row(resource_name, {icao24});
    if (row && !type_code) {
      type_code = row->get("type_code");
      if (!type_code)
        type_code = row->get("typecode");
    }
  }

  if (type_code)
    add_row("icao_doc_8643_aircraft_type_designators", {*type_code});
  return result;
}

std::map<std::string, std::shared_ptr<External_Database_Row>>
External_Resources_Manager::query_callsign(
    SQLite::Database &db, const std::optional<std::string> &callsign) const {
  Scope_Timer timer(
      "external_database.query_callsign",
      Global::instance()->http_monitor_config.slow_scope_threshold_ms);
  static const std::vector<std::string> kRouteResourceKeys = {
      "vrs_routes",
      "adsblol_vrs_standing_data_routes",
      "vradarserver_standing_data_routes",
  };

  std::lock_guard lock(mtx_);
  std::map<std::string, std::shared_ptr<External_Database_Row>> result;
  const auto add_row = [&](const std::string &resource_name,
                           const std::vector<std::string> &primary_key_values,
                           const std::string &result_name = std::string{}) {
    auto &resource = get_resource(resource_name);
    auto row = resource.query_one(db, primary_key_values);
    if (!row)
      return std::shared_ptr<External_Database_Row>{};
    auto shared_row = std::make_shared<External_Database_Row>(std::move(*row));
    result.emplace(result_name.empty() ? resource_name : result_name,
                   shared_row);
    return shared_row;
  };

  std::set<std::string> airport_codes;
  if (callsign && !callsign->empty()) {
    for (const auto &resource_name : kRouteResourceKeys) {
      const auto row = add_row(resource_name, {*callsign});
      if (!row)
        continue;
      const auto airports = row->get("AirportCodes");
      if (!airports)
        continue;

      std::size_t start = 0;
      while (start < airports->size()) {
        const auto end = airports->find('-', start);
        airport_codes.emplace(airports->substr(start, end - start));
        if (end == std::string::npos)
          break;
        start = end + 1;
      }
    }
  }

  for (const auto &airport_code : airport_codes) {
    if (airport_code.empty())
      continue;
    add_row("vrs_airports", {airport_code}, "vrs_airports:" + airport_code);
  }
  return result;
}

std::vector<External_Resource_Status>
External_Resources_Manager::status(SQLite::Database &db) const {
  std::lock_guard lock(mtx_);
  std::vector<External_Resource_Status> result;
  for (const auto &resource : resources_) {
    result.push_back(
        {resource->name, resource->row_count(db), false, false, ""});
  }
  return result;
}

SQLite::Database &External_Resources_Manager::require_db() const {
  if (!db_) {
    throw std::runtime_error("DB not loaded. Call load_sqlite_db() first.");
  }
  return *db_;
}

External_Resources &External_Resources_Manager::get_resource(
    const std::string &resource_name) const {
  for (const auto &resource : resources_) {
    if (resource->name == resource_name)
      return *resource;
  }
  throw std::invalid_argument("unknown external resource: " + resource_name);
}

std::vector<External_Resource_Status>
External_Resources_Manager::update_all(SQLite::Database &db,
                                       const bool force_download,
                                       const bool only_when_empty) {
  std::lock_guard lock(mtx_);
  std::vector<External_Resource_Status> result;
  for (const auto &resource : resources_) {
    try {
      auto status =
          resource->update(db, cache_pos_, force_download, only_when_empty);
      std::cout << "External database [" << status.name << "] "
                << status.message << ", rows=" << status.row_count << std::endl;
      result.push_back(std::move(status));
    } catch (const std::exception &error) {
      std::cerr << "External database [" << resource->name
                << "] update failed: " << error.what() << std::endl;
      result.push_back({resource->name, resource->row_count(db), false, false,
                        error.what()});
    }
  }
  return result;
}