首次提交

2026-06-16 11:18:22 +08:00
commit 068ef188e7
201 changed files with 57503 additions and 0 deletions
@@ -0,0 +1,96 @@
+#include "Crawler.h"
+
+#include "Local_Server/server/Global.h"
+
+void Crawler::handle(Call_back call_back) {
+  using namespace drogon;
+  int idx = 0;
+
+  SQLite::Statement aircraft_rows(
+      db_, "SELECT icao24, type_code FROM mictronics_aircraft_database");
+  while (aircraft_rows.executeStep()) {
+    idx++;
+
+    if (idx > 10) break;
+    if (idx % 1 == 0) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+    }
+    {
+      static Frequency_Limit fl;
+      if (fl.test()) {
+        std::cout << idx++ << "/618325  条数据已发出！ " << std::endl;
+      }
+    }
+    const auto hex = aircraft_rows.getColumn(0).getString();
+    const auto icaoType = aircraft_rows.isColumnNull(1)
+                              ? std::string{}
+                              : aircraft_rows.getColumn(1).getString();
+
+    bool ok = check_aircraft_exists(db_, hex);
+    if (!ok) {
+      static Frequency_Limit fl;
+      if (fl.test()) {
+        std::cout << idx << "/618325  数据存在跳过！ " << std::endl;
+      }
+    }
+    auto client = HttpClient::newHttpClient("https://planespotters.live");
+
+    std::string path = "/api/aircraft/hex/" + hex;
+
+    // GET https://planespotters.live/api/aircraft/hex/780F0E
+    // User-Agent: IntelliJ HTTP Client/CLion 2025.2.6
+    // Accept-Encoding: br, deflate, gzip, x-gzip
+    // Accept: */*
+
+    auto req = HttpRequest::newHttpRequest();
+    req->setMethod(Get);
+    req->setPath(path);
+
+    // query 参数：?icaoType=B763
+    req->setParameter("icaoType", icaoType);
+
+    req->addHeader("Accept-Language", "en-US,en;q=0.9");
+    req->addHeader("User-Agent", "IntelliJ HTTP Client/CLion 2025.2.6");
+    req->addHeader("Accept", "*/*");
+    req->addHeader("Accept-Encoding", "br, deflate, gzip, x-gzip");
+
+
+
+
+
+    client->sendRequest(req, [hex, icaoType, call_back](
+                                 ReqResult r, const HttpResponsePtr &resp) {
+      if (r != ReqResult::Ok || !resp) {
+        LOG_ERROR << "request failed, ReqResult=" << (int)r;
+        return;
+      }
+
+      if (resp->getStatusCode() != k200OK) {
+        static Frequency_Limit_Multi fl;
+        auto tt = to_string(resp->getStatusCode());
+        if (fl.test(tt)) {
+          std::cout << tt << std::endl;
+
+        }
+
+        return;
+      }
+
+      // 解析 JSON（如果响应是 JSON）
+      auto json = resp->getJsonObject();
+      if (!json) {
+        LOG_ERROR << "not json, body=" << resp->getBody();
+        return;
+      }
+
+      auto t = try_parse_json(json->toStyledString());
+
+      if (!t) {
+        std::cout << "解析失败？？？" << t->to_json_string() << std::endl;
+        Psc::fail_fast_core_dump();
+        return;
+      }
+      call_back(hex, icaoType, t.value());
+    });
+  }
+}
@@ -0,0 +1,96 @@
+#pragma once
+
+#include "../../../../../core_library/Core/Core/Base/JSON.h"
+#include "Core/Statistics/Frequency_Limit.h"
+
+#include <SQLiteCpp/Database.h>
+#include <SQLiteCpp/Statement.h>
+#include <SQLiteCpp/Transaction.h>
+#include <functional>
+#include <iostream>
+#include <string>
+
+
+class Crawler {
+public:
+  using Call_back = std::function<void(std::string hex, std::string icaoType,
+                                       Psc::JSON &json)>;
+
+  explicit Crawler(const std::string &db_path)
+      : db_(db_path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE) {
+    aircraft_json_table_create_if_not_exists();
+    // 可选：性能更好（仍然是同步）
+    db_.exec("PRAGMA journal_mode=WAL;");
+    db_.exec("PRAGMA synchronous=NORMAL;");
+  }
+
+  void aircraft_json_table_create_if_not_exists() {
+    db_.exec(R"sql(
+      CREATE TABLE IF NOT EXISTS aircraft_json (
+        icao TEXT PRIMARY KEY,
+        json TEXT
+      );
+    )sql");
+  }
+
+  bool check_aircraft_exists(SQLite::Database& db, const std::string& icao) {
+    // 查询是否存在该icao
+    static auto stmt_check_ = SQLite::Statement(db, R"sql(
+        SELECT COUNT(*) FROM aircraft_json WHERE icao = ?;
+    )sql");
+
+    stmt_check_.reset();
+    stmt_check_.clearBindings();
+    stmt_check_.bind(1, icao);
+
+    int count = 0;
+    if (stmt_check_.executeStep()) {
+      count = stmt_check_.getColumn(0).getInt();
+    }
+
+    // 如果存在记录，返回 true
+    return count > 0;
+  }
+
+  void insert_aircraft_json(const std::string &icao, const Psc::JSON &j) {
+    static auto stmt_upsert_ = SQLite::Statement(db_, R"sql(
+          INSERT INTO aircraft_json(icao, json)
+          VALUES(?, ?)
+          ON CONFLICT(icao) DO UPDATE SET
+            json = excluded.json;
+        )sql");
+    const std::string json_str = j.to_json_string();
+    std::lock_guard g(mtx);
+    SQLite::Transaction tx(db_); // BEGIN
+    stmt_upsert_.reset();
+    stmt_upsert_.clearBindings();
+    stmt_upsert_.bind(1, icao);
+    stmt_upsert_.bind(2, json_str);
+    stmt_upsert_.exec();
+    tx.commit();
+  }
+
+  void req_all_insert_aircraft_json() {
+    std::atomic<size_t> idx = 0;
+    handle(
+        [this, &idx](std::string hex, std::string icaoType, Psc::JSON &json) {
+          // 直接同步写库（最简单）
+          try {
+            static Frequency_Limit fl;
+            idx++;
+            if (fl.test()) {
+              std::cout << idx << "/618325  条数据处理成功！ " << std::endl;
+            }
+            insert_aircraft_json(hex, json);
+          } catch (const std::exception &e) {
+            // 用你自己的 LOG_ERROR
+            std::cerr << "sqlite insert failed: " << e.what() << "\n";
+          }
+        });
+  }
+
+private:
+  SQLite::Database db_;
+  std::mutex mtx;
+  void handle(Call_back cb); // 你自己实现/已有
+};