93 lines
2.7 KiB
C++
93 lines
2.7 KiB
C++
#include "Crawler.h"
|
||
|
||
#include "Local_Server/server/Global.h"
|
||
|
||
void Crawler::handle(Call_back call_back) {
|
||
using namespace drogon;
|
||
int idx = 0;
|
||
|
||
SQLite::Statement aircraft_rows(
|
||
db_, "SELECT icao24, type_code FROM mictronics_aircraft_database");
|
||
while (aircraft_rows.executeStep()) {
|
||
idx++;
|
||
|
||
if (idx > 10)
|
||
break;
|
||
if (idx % 1 == 0) {
|
||
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
|
||
}
|
||
{
|
||
static Frequency_Limit fl;
|
||
if (fl.test()) {
|
||
std::cout << idx++ << "/618325 条数据已发出！ " << std::endl;
|
||
}
|
||
}
|
||
const auto hex = aircraft_rows.getColumn(0).getString();
|
||
const auto icaoType = aircraft_rows.isColumnNull(1)
|
||
? std::string{}
|
||
: aircraft_rows.getColumn(1).getString();
|
||
|
||
bool ok = check_aircraft_exists(db_, hex);
|
||
if (!ok) {
|
||
static Frequency_Limit fl;
|
||
if (fl.test()) {
|
||
std::cout << idx << "/618325 数据存在跳过！ " << std::endl;
|
||
}
|
||
}
|
||
auto client = HttpClient::newHttpClient("https://planespotters.live");
|
||
|
||
std::string path = "/api/aircraft/hex/" + hex;
|
||
|
||
// GET https://planespotters.live/api/aircraft/hex/780F0E
|
||
// User-Agent: IntelliJ HTTP Client/CLion 2025.2.6
|
||
// Accept-Encoding: br, deflate, gzip, x-gzip
|
||
// Accept: */*
|
||
|
||
auto req = HttpRequest::newHttpRequest();
|
||
req->setMethod(Get);
|
||
req->setPath(path);
|
||
|
||
// query 参数：?icaoType=B763
|
||
req->setParameter("icaoType", icaoType);
|
||
|
||
req->addHeader("Accept-Language", "en-US,en;q=0.9");
|
||
req->addHeader("User-Agent", "IntelliJ HTTP Client/CLion 2025.2.6");
|
||
req->addHeader("Accept", "*/*");
|
||
req->addHeader("Accept-Encoding", "br, deflate, gzip, x-gzip");
|
||
|
||
client->sendRequest(req, [hex, icaoType, call_back](
|
||
ReqResult r, const HttpResponsePtr &resp) {
|
||
if (r != ReqResult::Ok || !resp) {
|
||
LOG_ERROR << "request failed, ReqResult=" << (int)r;
|
||
return;
|
||
}
|
||
|
||
if (resp->getStatusCode() != k200OK) {
|
||
static Frequency_Limit_Multi fl;
|
||
auto tt = to_string(resp->getStatusCode());
|
||
if (fl.test(tt)) {
|
||
std::cout << tt << std::endl;
|
||
}
|
||
|
||
return;
|
||
}
|
||
|
||
// 解析 JSON（如果响应是 JSON）
|
||
auto json = resp->getJsonObject();
|
||
if (!json) {
|
||
LOG_ERROR << "not json, body=" << resp->getBody();
|
||
return;
|
||
}
|
||
|
||
auto t = try_parse_json(json->toStyledString());
|
||
|
||
if (!t) {
|
||
std::cout << "解析失败？？？" << t->to_json_string() << std::endl;
|
||
Psc::fail_fast_core_dump();
|
||
return;
|
||
}
|
||
call_back(hex, icaoType, t.value());
|
||
});
|
||
}
|
||
}
|