84 lines
2.9 KiB
C++
84 lines
2.9 KiB
C++
#pragma once
|
|
#include <string_view>
|
|
#include "Psc_Cpp_Core/Base/JSON.h"
|
|
#include "Psc_Cpp_Core/Statistics/Frequency_Limit.h"
|
|
#include <SQLiteCpp/Database.h>
|
|
#include <SQLiteCpp/Statement.h>
|
|
#include <SQLiteCpp/Transaction.h>
|
|
#include <functional>
|
|
#include <iostream>
|
|
#include <string>
|
|
class Crawler {
|
|
public:
|
|
using Call_back = std::function<void(std::string_view hex, std::string_view icaoType, Psc::JSON& json)>;
|
|
explicit Crawler(std::string_view db_path) :
|
|
db_(std::string(db_path), SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE) {
|
|
aircraft_json_table_create_if_not_exists();
|
|
// 可选：性能更好（仍然是同步）
|
|
db_.exec("PRAGMA journal_mode=WAL;");
|
|
db_.exec("PRAGMA synchronous=NORMAL;");
|
|
}
|
|
void aircraft_json_table_create_if_not_exists() {
|
|
db_.exec(R"sql(
|
|
CREATE TABLE IF NOT EXISTS aircraft_json (
|
|
icao TEXT PRIMARY KEY,
|
|
json TEXT
|
|
);
|
|
)sql");
|
|
}
|
|
bool check_aircraft_exists(SQLite::Database& db, std::string_view icao) {
|
|
// 查询是否存在该icao
|
|
static auto stmt_check_ = SQLite::Statement(db, R"sql(
|
|
SELECT COUNT(*) FROM aircraft_json WHERE icao = ?;
|
|
)sql");
|
|
stmt_check_.reset();
|
|
stmt_check_.clearBindings();
|
|
stmt_check_.bind(1, std::string(icao));
|
|
int count = 0;
|
|
if (stmt_check_.executeStep()) {
|
|
count = stmt_check_.getColumn(0).getInt();
|
|
}
|
|
// 如果存在记录，返回 true
|
|
return count > 0;
|
|
}
|
|
void insert_aircraft_json(std::string_view icao, const Psc::JSON& j) {
|
|
static auto stmt_upsert_ = SQLite::Statement(db_, R"sql(
|
|
INSERT INTO aircraft_json(icao, json)
|
|
VALUES(?, ?)
|
|
ON CONFLICT(icao) DO UPDATE SET
|
|
json = excluded.json;
|
|
)sql");
|
|
const std::string json_str = j.to_json_string();
|
|
std::lock_guard g(mtx);
|
|
SQLite::Transaction tx(db_); // BEGIN
|
|
stmt_upsert_.reset();
|
|
stmt_upsert_.clearBindings();
|
|
stmt_upsert_.bind(1, std::string(icao));
|
|
stmt_upsert_.bind(2, json_str);
|
|
stmt_upsert_.exec();
|
|
tx.commit();
|
|
}
|
|
void req_all_insert_aircraft_json() {
|
|
std::atomic<size_t> idx = 0;
|
|
handle([this, &idx](std::string_view hex, std::string_view icaoType, Psc::JSON& json) {
|
|
// 直接同步写库（最简单）
|
|
try {
|
|
static Frequency_Limit fl;
|
|
idx++;
|
|
if (fl.test()) {
|
|
std::cout << idx << "/618325 条数据处理成功！ " << std::endl;
|
|
}
|
|
insert_aircraft_json(hex, json);
|
|
}
|
|
catch (const std::exception& e) {
|
|
// 用你自己的 LOG_ERROR
|
|
std::cerr << "sqlite insert failed: " << e.what() << "\n";
|
|
}
|
|
});
|
|
}
|
|
private:
|
|
SQLite::Database db_;
|
|
std::mutex mtx;
|
|
void handle(Call_back cb); // 你自己实现/已有
|
|
};
|