diff --git a/orotool.conf b/orotool.conf index d34ba7c..f348eb8 100644 --- a/orotool.conf +++ b/orotool.conf @@ -6,3 +6,4 @@ backend=sqlite [options] fetch_extra_delay=30 api_key=my_key_here +store_raw_json=true diff --git a/src/app_config.cpp b/src/app_config.cpp index 0f7ad70..b3e7e65 100644 --- a/src/app_config.cpp +++ b/src/app_config.cpp @@ -24,6 +24,8 @@ #include #include #include +#include +#include namespace duck { @@ -45,6 +47,32 @@ namespace { constexpr const char g_api_key[] = "api_key"; constexpr const char g_api_key_def[] = ""; + constexpr const char g_store_raw_json_sect[] = "options"; + constexpr const char g_store_raw_json[] = "store_raw_json"; + constexpr const char g_store_raw_json_def[] = "false"; + + bool equal (std::string_view a, std::string_view b) { + return a.size() == b.size() and std::equal( + a.begin(), a.end(), + b.begin(), + [](auto c1, auto c2) {return std::tolower(c1) == std::tolower(c2); } + ); + } + + bool to_bool (std::string_view a) { + std::array yes_list { + "true", "1", "yes", "on", "ok", "enable", "enabled", "sure", "aye", "one" + }; + + return ( + std::find_if( + yes_list.begin(), + yes_list.end(), + [a](std::string_view b) -> bool { return equal(a, b); } + ) != yes_list.end() + ); + } + std::string whole_ini() { std::ifstream input(g_config_file_path); input >> std::noskipws; @@ -132,4 +160,10 @@ std::string_view AppConfig::backend() const { return value_ifp(m_ini, g_backend_sect, g_backend, g_def_backend_name, false); } +bool AppConfig::store_raw_json() const { + std::string_view val = value_ifp(m_ini, g_store_raw_json_sect, g_store_raw_json, g_store_raw_json_def, false); + + return to_bool(val); +} + } //namespace duck diff --git a/src/app_config.hpp b/src/app_config.hpp index dfe9f96..9b1f533 100644 --- a/src/app_config.hpp +++ b/src/app_config.hpp @@ -33,6 +33,7 @@ public: std::size_t worker_threads() const; std::size_t fetch_extra_delay() const; std::string_view backend() const; + bool store_raw_json() const; private: kamokan::IniFile m_ini; diff --git a/src/evloop.cpp b/src/evloop.cpp index a097bab..fec22c2 100644 --- a/src/evloop.cpp +++ b/src/evloop.cpp @@ -56,7 +56,7 @@ namespace { }; } //unnamed namespace -void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count) { +void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json) { typedef TimerOroApi TimerShops; typedef TimerOroApi TimerItems; typedef TimerOroApi TimerIcons; @@ -71,11 +71,12 @@ void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_ const double ed = static_cast(extra_delay); auto sig_int = worker.make_event(&worker); + const bool& rj = store_raw_json; - auto timer_items = worker.make_event(TSet{0.0, ed}, &pool, api, db); - auto timer_icons = worker.make_event(TSet{5.0, ed}, &pool, api, db); - auto timer_shops = worker.make_event(TSet{10.0, ed}, &pool, api, db); - auto timer_creat = worker.make_event(TSet{15.0, ed}, &pool, api, db); + auto timer_items = worker.make_event(TSet{0.0, ed, rj}, &pool, api, db); + auto timer_icons = worker.make_event(TSet{5.0, ed, rj}, &pool, api, db); + auto timer_shops = worker.make_event(TSet{10.0, ed, rj}, &pool, api, db); + auto timer_creat = worker.make_event(TSet{15.0, ed, rj}, &pool, api, db); worker.wait(); #if !defined(NDEBUG) diff --git a/src/evloop.hpp b/src/evloop.hpp index ad1e0a8..444c59e 100644 --- a/src/evloop.hpp +++ b/src/evloop.hpp @@ -26,6 +26,6 @@ namespace oro { namespace duck { -void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count); +void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json); } //namespace duck diff --git a/src/main.cpp b/src/main.cpp index 3b2d8ba..f9c43ce 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -29,18 +29,18 @@ namespace { void print_ping(oro::Api& oro_api) { - auto ping = oro_api.ping(); + auto ping = oro_api.ping(false); - std::cout << "date: " << ping.first.date << '\n'; - std::cout << "rate limit: " << ping.first.rate_limit << '\n'; - std::cout << "remaining: " << ping.first.rate_limit_remaining << '\n'; - std::cout << "reset: " << ping.first.rate_limit_reset << '\n'; - std::cout << "retry after: " << ping.first.retry_after << '\n'; - std::cout << "server: " << ping.first.server << '\n'; + std::cout << "date: " << ping.header.date << '\n'; + std::cout << "rate limit: " << ping.header.rate_limit << '\n'; + std::cout << "remaining: " << ping.header.rate_limit_remaining << '\n'; + std::cout << "reset: " << ping.header.rate_limit_reset << '\n'; + std::cout << "retry after: " << ping.header.retry_after << '\n'; + std::cout << "server: " << ping.header.server << '\n'; std::cout << "-----\n"; - std::cout << "timestamp: " << ping.second.generation_timestamp << '\n'; - std::cout << "answer: " << ping.second.message << '\n'; - std::cout << "version: " << ping.second.version << '\n'; + std::cout << "timestamp: " << ping.data.generation_timestamp << '\n'; + std::cout << "answer: " << ping.data.message << '\n'; + std::cout << "version: " << ping.data.version << '\n'; } constexpr auto app_version() { @@ -82,7 +82,8 @@ int main(int argc, char* argv[]) { oro_api.get(), db.get(), app_conf.fetch_extra_delay(), - app_conf.worker_threads() + app_conf.worker_threads(), + app_conf.store_raw_json() ); } #if defined(OROTOOL_WITH_RESTCCPP) diff --git a/src/nap/http_response.hpp b/src/nap/http_response.hpp index 9c1eb63..f69949e 100644 --- a/src/nap/http_response.hpp +++ b/src/nap/http_response.hpp @@ -27,6 +27,7 @@ namespace nap { struct HttpResponse { std::unique_ptr raw; std::vector> header_list; + std::string_view header; std::string_view body; std::string_view http_ver; std::string_view code_desc; diff --git a/src/nap/page_fetch.cpp b/src/nap/page_fetch.cpp index dda1d2f..6ed6b0c 100644 --- a/src/nap/page_fetch.cpp +++ b/src/nap/page_fetch.cpp @@ -101,15 +101,14 @@ HttpResponse page_fetch ( HttpResponse resp; resp.code = easy.get_info().get(); - std::string_view head; { auto [raw, head_tmp, body] = make_raw_string(header_oss, body_oss, body_padding); - head = head_tmp; resp.raw = std::move(raw); + resp.header = head_tmp; resp.body = body; } - auto parsed_header = header_parse(head); + auto parsed_header = header_parse(resp.header); resp.header_list = std::move(parsed_header.fields); assert(resp.code == parsed_header.code); resp.http_ver = parsed_header.version; diff --git a/src/oro/api.hpp b/src/oro/api.hpp index 77c1009..aafcbd6 100644 --- a/src/oro/api.hpp +++ b/src/oro/api.hpp @@ -61,6 +61,13 @@ struct Header { }; +template +struct ApiOutput { + Header header; + T data; + std::string raw_response; +}; + class Api { public: Api ( @@ -71,12 +78,12 @@ public: ); virtual ~Api() noexcept; - virtual std::pair ping() = 0; - virtual std::pair who_am_i() = 0; - virtual std::pair items_list() = 0; - virtual std::pair items_icons() = 0; - virtual std::pair market_list() = 0; - virtual std::pair fame_list() = 0; + virtual ApiOutput ping(bool with_raw) = 0; + virtual ApiOutput who_am_i(bool with_raw) = 0; + virtual ApiOutput items_list(bool with_raw) = 0; + virtual ApiOutput items_icons(bool with_raw) = 0; + virtual ApiOutput market_list(bool with_raw) = 0; + virtual ApiOutput fame_list(bool with_raw) = 0; protected: std::string m_prefix; diff --git a/src/oro/api_nap.cpp b/src/oro/api_nap.cpp index 2437e76..5b77629 100644 --- a/src/oro/api_nap.cpp +++ b/src/oro/api_nap.cpp @@ -23,6 +23,7 @@ #include #include #include +#include namespace sjd = simdjson::dom; namespace sj = simdjson; @@ -115,25 +116,27 @@ ApiNap::ApiNap ( ApiNap::~ApiNap() noexcept = default; -std::pair ApiNap::ping() { +ApiOutput ApiNap::ping(bool with_raw) { return fetch_and_parse( g_endpoint_ping, [](const simdjson::dom::element& doc, Ping& out) { out.message = doc["message"]; - } + }, + with_raw ); } -std::pair ApiNap::who_am_i() { +ApiOutput ApiNap::who_am_i(bool with_raw) { return fetch_and_parse( g_endpoint_whoami, [](const simdjson::dom::element& doc, WhoAmI& out) { out.master_id = static_cast(doc["master_id"].get_uint64()); - } + }, + with_raw ); } -std::pair ApiNap::items_list() { +ApiOutput ApiNap::items_list(bool with_raw) { return fetch_and_parse( g_endpoint_items_list, [](const simdjson::dom::element& doc, Items& out) { @@ -150,11 +153,12 @@ std::pair ApiNap::items_list() { new_entry.slots = get_optional(item["slots"]); out.items.push_back(std::move(new_entry)); } - } + }, + with_raw ); } -std::pair ApiNap::items_icons() { +ApiOutput ApiNap::items_icons(bool with_raw) { return fetch_and_parse( g_endpoint_items_icons, [](const simdjson::dom::element& doc, Icons& out) { @@ -166,11 +170,12 @@ std::pair ApiNap::items_icons() { new_entry.icon = icon["icon"]; out.icons.push_back(std::move(new_entry)); } - } + }, + with_raw ); } -std::pair ApiNap::market_list() { +ApiOutput ApiNap::market_list(bool with_raw) { return fetch_and_parse( g_endpoint_market_list, [](const simdjson::dom::element& doc, Shops& out) { @@ -216,22 +221,24 @@ std::pair ApiNap::market_list() { } out.shops.push_back(std::move(new_shop)); } - } + }, + with_raw ); } -std::pair ApiNap::fame_list() { +ApiOutput ApiNap::fame_list(bool with_raw) { return fetch_and_parse( g_endpoint_fame_list, [](const simdjson::dom::element& doc, Creators& out) { fill_creator_list(doc["brewers"], out.brewers); fill_creator_list(doc["forgers"], out.brewers); - } + }, + with_raw ); } template -std::pair ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill) { +ApiOutput ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw) { auto resp = m_qrest.fetch(m_prefix + endpoint); if (200 != resp.code) @@ -249,7 +256,16 @@ std::pair ApiNap::fetch_and_parse (const char* endpoint, F&& data_fil data_fill(doc, dataret); } - return {to_header(resp), std::move(dataret)}; + std::string raw_resp; + if (with_raw) { + assert(not resp.header.empty() and resp.header[resp.header.size() - 1] == '\n'); + raw_resp.reserve(resp.header.size() + 1 + resp.body.size()); + raw_resp.append(resp.header); + raw_resp.append("\n"); + raw_resp.append(resp.body); + } + + return {to_header(resp), std::move(dataret), std::move(raw_resp)}; } } //namespace oro diff --git a/src/oro/originsdb.hpp b/src/oro/originsdb.hpp index 98d22d4..7aa9c70 100644 --- a/src/oro/originsdb.hpp +++ b/src/oro/originsdb.hpp @@ -19,7 +19,7 @@ #include "datatypes.hpp" #include "oro/dboperation.hpp" -#include +#include "oro/source.hpp" #include namespace oro { @@ -34,10 +34,10 @@ public: OriginsDB() = default; virtual ~OriginsDB() noexcept = default; - virtual void update (const Items& items, const oro::Timestamp& next_update) = 0; - virtual void update (const Icons& icons, const oro::Timestamp& next_update) = 0; - virtual void update (const Shops& shops, const oro::Timestamp& next_update) = 0; - virtual void update (const Creators& creat, const oro::Timestamp& next_update) = 0; + virtual void update (const Items& items, const oro::Timestamp& next_update, const Source& source) = 0; + virtual void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) = 0; + virtual void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) = 0; + virtual void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) = 0; virtual Timestamp next_access_time (DBOperation op) const = 0; diff --git a/src/oro/private/api_nap.hpp b/src/oro/private/api_nap.hpp index c855eee..9d1698f 100644 --- a/src/oro/private/api_nap.hpp +++ b/src/oro/private/api_nap.hpp @@ -20,7 +20,6 @@ #include "oro/api.hpp" #include "nap/quick_rest.hpp" #include -#include namespace oro { @@ -35,16 +34,16 @@ public: virtual ~ApiNap() noexcept; - virtual std::pair ping() override; - virtual std::pair who_am_i() override; - virtual std::pair items_list() override; - virtual std::pair items_icons() override; - virtual std::pair market_list() override; - virtual std::pair fame_list() override; + virtual ApiOutput ping(bool with_raw) override; + virtual ApiOutput who_am_i(bool with_raw) override; + virtual ApiOutput items_list(bool with_raw) override; + virtual ApiOutput items_icons(bool with_raw) override; + virtual ApiOutput market_list(bool with_raw) override; + virtual ApiOutput fame_list(bool with_raw) override; private: template - std::pair fetch_and_parse (const char* endpoint, F&& data_fill); + ApiOutput fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw); std::mutex m_json_mutex; simdjson::dom::parser m_json; diff --git a/src/oro/private/originsdb_sqlite.cpp b/src/oro/private/originsdb_sqlite.cpp index 94baa81..7c479a4 100644 --- a/src/oro/private/originsdb_sqlite.cpp +++ b/src/oro/private/originsdb_sqlite.cpp @@ -52,6 +52,7 @@ namespace { ", master_id INTEGER UNIQUE" ", name TEXT" ", points INTEGER" + ", source_id INTEGER NOT NULL" ")"; constexpr const char g_create_slotted_cards[] = "CREATE TABLE IF NOT EXISTS slotted_cards(" @@ -83,6 +84,7 @@ namespace { ", hash TEXT NOT NULL" ", discovery_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP" ", last_seen_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP" + ", source_id INTEGER NOT NULL" ", FOREIGN KEY(shop_id) REFERENCES shops(id)" ")"; constexpr const char g_create_shops[] = @@ -108,6 +110,7 @@ namespace { ", npc_price INTEGER" ", slots TINYINT" ", removal_date TEXT" + ", source_id INTEGER NOT NULL" ")"; constexpr const char g_create_icons_a[] = "CREATE TABLE IF NOT EXISTS "; constexpr const char g_create_icons_b[] = @@ -115,6 +118,7 @@ namespace { "id INTEGER PRIMARY KEY NOT NULL" ", item_id INTEGER NOT NULL" ", icon TEXT" + ", source_id INTEGER NOT NULL" ", FOREIGN KEY(item_id) REFERENCES items(id)" ")"; constexpr const char g_create_access[] = @@ -123,6 +127,12 @@ namespace { ", operation TINYINT PRIMARY KEY NOT NULL" ", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'" ")"; + constexpr const char g_create_source_store[] = + "CREATE TABLE IF NOT EXISTS source_store(" + "id INTEGER PRIMARY KEY NOT NULL" + ", source TEXT" + ", format TEXT" + ")"; std::string to_string (int num) { auto ary = dhandy::int_to_ary(num); @@ -322,6 +332,17 @@ namespace { return (prefix + values + suffix); } + + SQLiteID insert_source_ifn (SQLite::Database& db, const Source& source) { + if (not source.data) + return 0; + + db.exec(g_create_source_store); + SQLite::Statement query(db, "INSERT INTO source_store(source, format) VALUES(?, ?)"); + bind_all(query, no_copy(*source.data), static_cast(source.format)); + query.exec(); + return db.getLastInsertRowid(); + } } //unnamed namespace OriginsDBSQLite::OriginsDBSQLite (std::string_view path) : @@ -336,7 +357,7 @@ OriginsDBSQLite::OriginsDBSQLite (std::string_view path) : OriginsDBSQLite::~OriginsDBSQLite() noexcept = default; -void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update) { +void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update, const Source& source) { auto& db = *m_db; std::unique_lock lock(m_db_mutex); update_last_access(db, DBOperation::Items, next_update); @@ -360,14 +381,16 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd SQLite::Statement query(db, "INSERT INTO items_staging " - "(item_id, unique_name, name, type, subtype, npc_price, slots) " - "VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7)" + "(item_id, unique_name, name, type, subtype, npc_price, slots, source_id) " + "VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)" ); SQLite::Transaction transaction(db); + const SQLiteID source_id = insert_source_ifn(db, source); for (const auto& item : items.items) { bind_all(query, item.item_id, no_copy(item.unique_name), no_copy(item.name), static_cast(item.type.value), - cast(item.subtype), item.npc_price, item.slots + cast(item.subtype), item.npc_price, item.slots, + source_id ); query.exec(); @@ -384,19 +407,19 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd ")" ); db.exec( - "INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots) " - "SELECT item_id, unique_name, name, type, subtype, npc_price, slots FROM items_staging WHERE true " + "INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots, source_id) " + "SELECT item_id, unique_name, name, type, subtype, npc_price, slots, source_id FROM items_staging WHERE true " "ON CONFLICT(item_id, ifnull(removal_date, 0)) DO UPDATE SET " "unique_name=excluded.unique_name, " "name=excluded.name, type=excluded.type, subtype=excluded.subtype, " - "npc_price=excluded.npc_price, slots=excluded.slots" + "npc_price=excluded.npc_price, slots=excluded.slots, source_id=excluded.source_id" ); db.exec("DROP TABLE items_staging"); transaction.commit(); } -void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update) { +void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) { //example: //{ // "item_id":501, @@ -411,33 +434,34 @@ void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_upd db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b); db.exec("CREATE UNIQUE INDEX IF NOT EXISTS icons_item_id_idx ON icons(item_id)"); SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)"); - SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon) VALUES(?, ?)"); + SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon, source_id) VALUES(?, ?, ?)"); ItemIdToTableId item_id_to_table_id(db, false); SQLite::Transaction transaction(db); + const SQLiteID source_id = insert_source_ifn(db, source); for (const auto& ico : icons.icons) { bind_all(ins_empty_item, ico.item_id); ins_empty_item.exec(); ins_empty_item.reset(); - bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon)); + bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon), source_id); query.exec(); query.reset(); //base64_decode(std::string_view(ico.icon).substr(std::string_view("data:image/png;base64,").size())); } db.exec( - "INSERT INTO icons (item_id, icon) " - "SELECT item_id, icon FROM icons_staging WHERE true " + "INSERT INTO icons (item_id, icon, source_id) " + "SELECT item_id, icon, source_id FROM icons_staging WHERE true " "ON CONFLICT(item_id) DO UPDATE SET " - "icon=excluded.icon" + "icon=excluded.icon, source_id=excluded.source_id" ); db.exec("DROP TABLE icons_staging"); transaction.commit(); } -void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update) { +void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) { //example: //{ // "title":"• B\u003ePoison Bottle •", @@ -463,7 +487,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd db.exec(g_create_shop_items); db.exec(g_create_slotted_cards); - SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash) VALUES(?, ?)"); + SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash, source_id) VALUES(?, ?, ?)"); SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)"); SQLite::Statement sel_sshot(db, "SELECT id FROM shop_snapshots WHERE shop_id = ? AND hash = ?"); SQLite::Statement ins_item(db, "INSERT INTO shop_items(snapshot_id, item_id, amount, price, refine, star_crumbs, element, creator, beloved) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); @@ -473,6 +497,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd InsertShopIfn insert_shop_ifn(db); ItemIdToTableId item_id_to_table_id(db); SQLite::Transaction transaction(db); + const SQLiteID source_id = insert_source_ifn(db, source); for (const auto& shop : shops.shops) { { //insert new shop or get its id if already there @@ -490,7 +515,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd } //snapshot is new, insert it - bind_all(ins_sshot, shop_id_ins.first, no_copy(hash)); + bind_all(ins_sshot, shop_id_ins.first, no_copy(hash), source_id); ins_sshot.exec(); ins_sshot.reset(); } @@ -520,26 +545,27 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd transaction.commit(); } -void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update) { +void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) { std::unique_lock lock(m_db_mutex); auto& db = *m_db; update_last_access(db, DBOperation::Creators, next_update); db.exec(g_create_fame_list); - auto insert_creators = [](SQLite::Statement& query, const std::vector& creats, int type) { + auto insert_creators = [](SQLite::Statement& query, const std::vector& creats, int type, SQLiteID source_id) { for (const auto& creator : creats) { - bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points); + bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points, source_id); query.exec(); query.reset(); } }; - SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points) VALUES (?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points"); + SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points, source_id) VALUES (?, ?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points"); SQLite::Transaction transaction(db); - insert_creators(query, creat.brewers, 1); - insert_creators(query, creat.forgers, 2); + const SQLiteID source_id = insert_source_ifn(db, source); + insert_creators(query, creat.brewers, 1, source_id); + insert_creators(query, creat.forgers, 2, source_id); transaction.commit(); } diff --git a/src/oro/private/originsdb_sqlite.hpp b/src/oro/private/originsdb_sqlite.hpp index 5fcd704..2bc88f0 100644 --- a/src/oro/private/originsdb_sqlite.hpp +++ b/src/oro/private/originsdb_sqlite.hpp @@ -34,10 +34,10 @@ public: explicit OriginsDBSQLite(std::string_view path); virtual ~OriginsDBSQLite() noexcept; - void update (const Items& items, const oro::Timestamp& next_update) override; - void update (const Icons& icons, const oro::Timestamp& next_update) override; - void update (const Shops& shops, const oro::Timestamp& next_update) override; - void update (const Creators& creat, const oro::Timestamp& next_update) override; + void update (const Items& items, const oro::Timestamp& next_update, const Source& source) override; + void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) override; + void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) override; + void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) override; Timestamp next_access_time (DBOperation op) const override; diff --git a/src/oro/source.hpp b/src/oro/source.hpp new file mode 100644 index 0000000..04b27a6 --- /dev/null +++ b/src/oro/source.hpp @@ -0,0 +1,38 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#pragma once + +#include +#include + +namespace oro { + +//The raw json source data complete with http response header. It can be +//anything really, but it's intended to be stored in the DB. Entries in the +//other tables will have a reference to the source. + +enum class SourceFormat { + Plain +}; + +struct Source { + std::optional data; + SourceFormat format; +}; + +} //namespace oro diff --git a/src/timer_base.cpp b/src/timer_base.cpp index ac3a2b3..0c51a25 100644 --- a/src/timer_base.cpp +++ b/src/timer_base.cpp @@ -64,14 +64,15 @@ TimerBase::TimerBase ( m_extra_delay(settings.extra_delay), m_pool(pool), m_oro_api(oro_api), - m_db(db) + m_db(db), + m_store_raw_response(settings.store_raw_json) { assert(m_pool); assert(m_oro_api); } void TimerBase::on_timer() { - m_pool->submit(&TimerBase::fetch_data, this); + m_pool->submit(&TimerBase::fetch_data, this, m_store_raw_response); } void TimerBase::set_next_timer (const oro::Header& header) { @@ -95,20 +96,20 @@ oro::OriginsDB& TimerBase::db() { return *m_db; } -void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) { - db().update(shops, calc_next_update(header, m_extra_delay)); +void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source) { + db().update(shops, calc_next_update(header, m_extra_delay), source); } -void TimerBase::update_db (const oro::Items& items, const oro::Header& header) { - db().update(items, calc_next_update(header, m_extra_delay)); +void TimerBase::update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source) { + db().update(items, calc_next_update(header, m_extra_delay), source); } -void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) { - db().update(icons, calc_next_update(header, m_extra_delay)); +void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source) { + db().update(icons, calc_next_update(header, m_extra_delay), source); } -void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) { - db().update(creators, calc_next_update(header, m_extra_delay)); +void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source) { + db().update(creators, calc_next_update(header, m_extra_delay), source); } } //namespace duck diff --git a/src/timer_base.hpp b/src/timer_base.hpp index e00e458..8cbbd41 100644 --- a/src/timer_base.hpp +++ b/src/timer_base.hpp @@ -20,6 +20,7 @@ #include "eventia/private/context.hpp" #include "eventia/timer.hpp" #include "oro/dboperation.hpp" +#include "oro/source.hpp" namespace roar11 { class ThreadPool; @@ -38,13 +39,15 @@ namespace oro { namespace duck { struct TimerSettings { - TimerSettings (double min_wait, double extra_delay) : + TimerSettings (double min_wait, double extra_delay, bool with_raw_json) : min_wait(min_wait), - extra_delay(extra_delay) + extra_delay(extra_delay), + store_raw_json(with_raw_json) { } double min_wait; double extra_delay; + bool store_raw_json; }; class TimerBase : public eve::Timer { @@ -63,21 +66,22 @@ public: protected: void set_next_timer (const oro::Header& header); - void update_db (const oro::Shops& shops, const oro::Header& header); - void update_db (const oro::Items& items, const oro::Header& header); - void update_db (const oro::Icons& icons, const oro::Header& header); - void update_db (const oro::Creators& creators, const oro::Header& header); + void update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source); + void update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source); + void update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source); + void update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source); roar11::ThreadPool& pool(); oro::Api& oro_api(); oro::OriginsDB& db(); private: - virtual void fetch_data() = 0; + virtual void fetch_data(bool with_raw) = 0; double m_extra_delay; roar11::ThreadPool* m_pool; oro::Api* m_oro_api; oro::OriginsDB* m_db; + bool m_store_raw_response; }; } //namespace duck diff --git a/src/timer_oro_api.cpp b/src/timer_oro_api.cpp index 8e8c6ec..cda1c8b 100644 --- a/src/timer_oro_api.cpp +++ b/src/timer_oro_api.cpp @@ -28,26 +28,26 @@ namespace duck { namespace { - template auto invoke_api_func (oro::Api& api); + template auto invoke_api_func (oro::Api& api, bool with_raw); template <> - inline auto invoke_api_func (oro::Api& api) { - return api.items_icons(); + inline auto invoke_api_func (oro::Api& api, bool with_raw) { + return api.items_icons(with_raw); } template <> - inline auto invoke_api_func (oro::Api& api) { - return api.items_list(); + inline auto invoke_api_func (oro::Api& api, bool with_raw) { + return api.items_list(with_raw); } template <> - inline auto invoke_api_func (oro::Api& api) { - return api.fame_list(); + inline auto invoke_api_func (oro::Api& api, bool with_raw) { + return api.fame_list(with_raw); } template <> - inline auto invoke_api_func (oro::Api& api) { - return api.market_list(); + inline auto invoke_api_func (oro::Api& api, bool with_raw) { + return api.market_list(with_raw); } } //unnamed namespace @@ -64,12 +64,18 @@ inline TimerOroApi::TimerOroApi ( } template -inline void TimerOroApi::fetch_data() { +inline void TimerOroApi::fetch_data (bool with_raw) { int status_code = 200; try { - auto results = invoke_api_func(oro_api()); - set_next_timer(results.first); - this->update_db(results.second, results.first); + auto results = invoke_api_func(oro_api(), with_raw); + set_next_timer(results.header); + + oro::Source raw_src; + if (with_raw) { + raw_src.data = std::move(results.raw_response); + raw_src.format = oro::SourceFormat::Plain; + } + this->update_db(results.data, results.header, raw_src); } #if defined(OROTOOL_WITH_RESTCCPP) catch (const restc_cpp::RequestFailedWithErrorException& err) { diff --git a/src/timer_oro_api.hpp b/src/timer_oro_api.hpp index 8473d52..38a3a5f 100644 --- a/src/timer_oro_api.hpp +++ b/src/timer_oro_api.hpp @@ -38,7 +38,7 @@ public: ); private: - virtual void fetch_data() override; + virtual void fetch_data (bool with_raw) override; }; } //namespace duck