首次提交

2026-06-16 11:18:22 +08:00
commit 068ef188e7
201 changed files with 57503 additions and 0 deletions
@@ -0,0 +1,96 @@
+#include "Crawler.h"
+
+#include "Local_Server/server/Global.h"
+
+void Crawler::handle(Call_back call_back) {
+  using namespace drogon;
+  int idx = 0;
+
+  SQLite::Statement aircraft_rows(
+      db_, "SELECT icao24, type_code FROM mictronics_aircraft_database");
+  while (aircraft_rows.executeStep()) {
+    idx++;
+
+    if (idx > 10) break;
+    if (idx % 1 == 0) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    }
+    {
+      static Frequency_Limit fl;
+      if (fl.test()) {
+        std::cout << idx++ << "/618325  条数据已发出！ " << std::endl;
+      }
+    }
+    const auto hex = aircraft_rows.getColumn(0).getString();
+    const auto icaoType = aircraft_rows.isColumnNull(1)
+                              ? std::string{}
+                              : aircraft_rows.getColumn(1).getString();
+
+    bool ok = check_aircraft_exists(db_, hex);
+    if (!ok) {
+      static Frequency_Limit fl;
+      if (fl.test()) {
+        std::cout << idx << "/618325  数据存在跳过！ " << std::endl;
+      }
+    }
+    auto client = HttpClient::newHttpClient("https://planespotters.live");
+
+    std::string path = "/api/aircraft/hex/" + hex;
+
+    // GET https://planespotters.live/api/aircraft/hex/780F0E
+    // User-Agent: IntelliJ HTTP Client/CLion 2025.2.6
+    // Accept-Encoding: br, deflate, gzip, x-gzip
+    // Accept: */*
+
+    auto req = HttpRequest::newHttpRequest();
+    req->setMethod(Get);
+    req->setPath(path);
+
+    // query 参数：?icaoType=B763
+    req->setParameter("icaoType", icaoType);
+
+    req->addHeader("Accept-Language", "en-US,en;q=0.9");
+    req->addHeader("User-Agent", "IntelliJ HTTP Client/CLion 2025.2.6");
+    req->addHeader("Accept", "*/*");
+    req->addHeader("Accept-Encoding", "br, deflate, gzip, x-gzip");
+
+
+
+
+
+    client->sendRequest(req, [hex, icaoType, call_back](
+                                 ReqResult r, const HttpResponsePtr &resp) {
+      if (r != ReqResult::Ok || !resp) {
+        LOG_ERROR << "request failed, ReqResult=" << (int)r;
+        return;
+      }
+
+      if (resp->getStatusCode() != k200OK) {
+        static Frequency_Limit_Multi fl;
+        auto tt = to_string(resp->getStatusCode());
+        if (fl.test(tt)) {
+          std::cout << tt << std::endl;
+
+        }
+
+        return;
+      }
+
+      // 解析 JSON（如果响应是 JSON）
+      auto json = resp->getJsonObject();
+      if (!json) {
+        LOG_ERROR << "not json, body=" << resp->getBody();
+        return;
+      }
+
+      auto t = try_parse_json(json->toStyledString());
+
+      if (!t) {
+        std::cout << "解析失败？？？" << t->to_json_string() << std::endl;
+        Psc::fail_fast_core_dump();
+        return;
+      }
+      call_back(hex, icaoType, t.value());
+    });
+  }
+}