#include "Crawler.h" #include "Local_Server/server/Global.h" void Crawler::handle(Call_back call_back) { using namespace drogon; int idx = 0; SQLite::Statement aircraft_rows( db_, "SELECT icao24, type_code FROM mictronics_aircraft_database"); while (aircraft_rows.executeStep()) { idx++; if (idx > 10) break; if (idx % 1 == 0) { std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } { static Frequency_Limit fl; if (fl.test()) { std::cout << idx++ << "/618325 条数据已发出! " << std::endl; } } const auto hex = aircraft_rows.getColumn(0).getString(); const auto icaoType = aircraft_rows.isColumnNull(1) ? std::string{} : aircraft_rows.getColumn(1).getString(); bool ok = check_aircraft_exists(db_, hex); if (!ok) { static Frequency_Limit fl; if (fl.test()) { std::cout << idx << "/618325 数据存在跳过! " << std::endl; } } auto client = HttpClient::newHttpClient("https://planespotters.live"); std::string path = "/api/aircraft/hex/" + hex; // GET https://planespotters.live/api/aircraft/hex/780F0E // User-Agent: IntelliJ HTTP Client/CLion 2025.2.6 // Accept-Encoding: br, deflate, gzip, x-gzip // Accept: */* auto req = HttpRequest::newHttpRequest(); req->setMethod(Get); req->setPath(path); // query 参数:?icaoType=B763 req->setParameter("icaoType", icaoType); req->addHeader("Accept-Language", "en-US,en;q=0.9"); req->addHeader("User-Agent", "IntelliJ HTTP Client/CLion 2025.2.6"); req->addHeader("Accept", "*/*"); req->addHeader("Accept-Encoding", "br, deflate, gzip, x-gzip"); client->sendRequest(req, [hex, icaoType, call_back]( ReqResult r, const HttpResponsePtr &resp) { if (r != ReqResult::Ok || !resp) { LOG_ERROR << "request failed, ReqResult=" << (int)r; return; } if (resp->getStatusCode() != k200OK) { static Frequency_Limit_Multi fl; auto tt = to_string(resp->getStatusCode()); if (fl.test(tt)) { std::cout << tt << std::endl; } return; } // 解析 JSON(如果响应是 JSON) auto json = resp->getJsonObject(); if (!json) { LOG_ERROR << "not json, body=" << resp->getBody(); return; } auto t = try_parse_json(json->toStyledString()); if (!t) { std::cout << "解析失败???" << t->to_json_string() << std::endl; Psc::fail_fast_core_dump(); return; } call_back(hex, icaoType, t.value()); }); } }