#pragma once #include #include "Psc_Cpp_Core/Base/JSON.h" #include "Psc_Cpp_Core/Statistics/Frequency_Limit.h" #include #include #include #include #include #include class Crawler { public: using Call_back = std::function; explicit Crawler(std::string_view db_path) : db_(std::string(db_path), SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE) { aircraft_json_table_create_if_not_exists(); // 可选:性能更好(仍然是同步) db_.exec("PRAGMA journal_mode=WAL;"); db_.exec("PRAGMA synchronous=NORMAL;"); } void aircraft_json_table_create_if_not_exists() { db_.exec(R"sql( CREATE TABLE IF NOT EXISTS aircraft_json ( icao TEXT PRIMARY KEY, json TEXT ); )sql"); } bool check_aircraft_exists(SQLite::Database& db, std::string_view icao) { // 查询是否存在该icao static auto stmt_check_ = SQLite::Statement(db, R"sql( SELECT COUNT(*) FROM aircraft_json WHERE icao = ?; )sql"); stmt_check_.reset(); stmt_check_.clearBindings(); stmt_check_.bind(1, std::string(icao)); int count = 0; if (stmt_check_.executeStep()) { count = stmt_check_.getColumn(0).getInt(); } // 如果存在记录,返回 true return count > 0; } void insert_aircraft_json(std::string_view icao, const Psc::JSON& j) { static auto stmt_upsert_ = SQLite::Statement(db_, R"sql( INSERT INTO aircraft_json(icao, json) VALUES(?, ?) ON CONFLICT(icao) DO UPDATE SET json = excluded.json; )sql"); const std::string json_str = j.to_json_string(); std::lock_guard g(mtx); SQLite::Transaction tx(db_); // BEGIN stmt_upsert_.reset(); stmt_upsert_.clearBindings(); stmt_upsert_.bind(1, std::string(icao)); stmt_upsert_.bind(2, json_str); stmt_upsert_.exec(); tx.commit(); } void req_all_insert_aircraft_json() { std::atomic idx = 0; handle([this, &idx](std::string_view hex, std::string_view icaoType, Psc::JSON& json) { // 直接同步写库(最简单) try { static Frequency_Limit fl; idx++; if (fl.test()) { std::cout << idx << "/618325 条数据处理成功! " << std::endl; } insert_aircraft_json(hex, json); } catch (const std::exception& e) { // 用你自己的 LOG_ERROR std::cerr << "sqlite insert failed: " << e.what() << "\n"; } }); } private: SQLite::Database db_; std::mutex mtx; void handle(Call_back cb); // 你自己实现/已有 };