Add store_raw_json option to config
This commit is contained in:
parent
3b071727c3
commit
9d4d52bed0
19 changed files with 246 additions and 111 deletions
|
|
@ -6,3 +6,4 @@ backend=sqlite
|
|||
[options]
|
||||
fetch_extra_delay=30
|
||||
api_key=my_key_here
|
||||
store_raw_json=true
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@
|
|||
#include <algorithm>
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
#include <cctype>
|
||||
#include <array>
|
||||
|
||||
namespace duck {
|
||||
|
||||
|
|
@ -45,6 +47,32 @@ namespace {
|
|||
constexpr const char g_api_key[] = "api_key";
|
||||
constexpr const char g_api_key_def[] = "";
|
||||
|
||||
constexpr const char g_store_raw_json_sect[] = "options";
|
||||
constexpr const char g_store_raw_json[] = "store_raw_json";
|
||||
constexpr const char g_store_raw_json_def[] = "false";
|
||||
|
||||
bool equal (std::string_view a, std::string_view b) {
|
||||
return a.size() == b.size() and std::equal(
|
||||
a.begin(), a.end(),
|
||||
b.begin(),
|
||||
[](auto c1, auto c2) {return std::tolower(c1) == std::tolower(c2); }
|
||||
);
|
||||
}
|
||||
|
||||
bool to_bool (std::string_view a) {
|
||||
std::array<std::string_view, 10> yes_list {
|
||||
"true", "1", "yes", "on", "ok", "enable", "enabled", "sure", "aye", "one"
|
||||
};
|
||||
|
||||
return (
|
||||
std::find_if(
|
||||
yes_list.begin(),
|
||||
yes_list.end(),
|
||||
[a](std::string_view b) -> bool { return equal(a, b); }
|
||||
) != yes_list.end()
|
||||
);
|
||||
}
|
||||
|
||||
std::string whole_ini() {
|
||||
std::ifstream input(g_config_file_path);
|
||||
input >> std::noskipws;
|
||||
|
|
@ -132,4 +160,10 @@ std::string_view AppConfig::backend() const {
|
|||
return value_ifp(m_ini, g_backend_sect, g_backend, g_def_backend_name, false);
|
||||
}
|
||||
|
||||
bool AppConfig::store_raw_json() const {
|
||||
std::string_view val = value_ifp(m_ini, g_store_raw_json_sect, g_store_raw_json, g_store_raw_json_def, false);
|
||||
|
||||
return to_bool(val);
|
||||
}
|
||||
|
||||
} //namespace duck
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ public:
|
|||
std::size_t worker_threads() const;
|
||||
std::size_t fetch_extra_delay() const;
|
||||
std::string_view backend() const;
|
||||
bool store_raw_json() const;
|
||||
|
||||
private:
|
||||
kamokan::IniFile m_ini;
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ namespace {
|
|||
};
|
||||
} //unnamed namespace
|
||||
|
||||
void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count) {
|
||||
void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json) {
|
||||
typedef TimerOroApi<oro::DBOperation::Shops> TimerShops;
|
||||
typedef TimerOroApi<oro::DBOperation::Items> TimerItems;
|
||||
typedef TimerOroApi<oro::DBOperation::Icons> TimerIcons;
|
||||
|
|
@ -71,11 +71,12 @@ void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_
|
|||
const double ed = static_cast<double>(extra_delay);
|
||||
|
||||
auto sig_int = worker.make_event<SignalInt>(&worker);
|
||||
const bool& rj = store_raw_json;
|
||||
|
||||
auto timer_items = worker.make_event<TimerItems>(TSet{0.0, ed}, &pool, api, db);
|
||||
auto timer_icons = worker.make_event<TimerIcons>(TSet{5.0, ed}, &pool, api, db);
|
||||
auto timer_shops = worker.make_event<TimerShops>(TSet{10.0, ed}, &pool, api, db);
|
||||
auto timer_creat = worker.make_event<TimerCreators>(TSet{15.0, ed}, &pool, api, db);
|
||||
auto timer_items = worker.make_event<TimerItems>(TSet{0.0, ed, rj}, &pool, api, db);
|
||||
auto timer_icons = worker.make_event<TimerIcons>(TSet{5.0, ed, rj}, &pool, api, db);
|
||||
auto timer_shops = worker.make_event<TimerShops>(TSet{10.0, ed, rj}, &pool, api, db);
|
||||
auto timer_creat = worker.make_event<TimerCreators>(TSet{15.0, ed, rj}, &pool, api, db);
|
||||
|
||||
worker.wait();
|
||||
#if !defined(NDEBUG)
|
||||
|
|
|
|||
|
|
@ -26,6 +26,6 @@ namespace oro {
|
|||
|
||||
namespace duck {
|
||||
|
||||
void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count);
|
||||
void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json);
|
||||
|
||||
} //namespace duck
|
||||
|
|
|
|||
25
src/main.cpp
25
src/main.cpp
|
|
@ -29,18 +29,18 @@
|
|||
|
||||
namespace {
|
||||
void print_ping(oro::Api& oro_api) {
|
||||
auto ping = oro_api.ping();
|
||||
|
||||
std::cout << "date: " << ping.first.date << '\n';
|
||||
std::cout << "rate limit: " << ping.first.rate_limit << '\n';
|
||||
std::cout << "remaining: " << ping.first.rate_limit_remaining << '\n';
|
||||
std::cout << "reset: " << ping.first.rate_limit_reset << '\n';
|
||||
std::cout << "retry after: " << ping.first.retry_after << '\n';
|
||||
std::cout << "server: " << ping.first.server << '\n';
|
||||
auto ping = oro_api.ping(false);
|
||||
|
||||
std::cout << "date: " << ping.header.date << '\n';
|
||||
std::cout << "rate limit: " << ping.header.rate_limit << '\n';
|
||||
std::cout << "remaining: " << ping.header.rate_limit_remaining << '\n';
|
||||
std::cout << "reset: " << ping.header.rate_limit_reset << '\n';
|
||||
std::cout << "retry after: " << ping.header.retry_after << '\n';
|
||||
std::cout << "server: " << ping.header.server << '\n';
|
||||
std::cout << "-----\n";
|
||||
std::cout << "timestamp: " << ping.second.generation_timestamp << '\n';
|
||||
std::cout << "answer: " << ping.second.message << '\n';
|
||||
std::cout << "version: " << ping.second.version << '\n';
|
||||
std::cout << "timestamp: " << ping.data.generation_timestamp << '\n';
|
||||
std::cout << "answer: " << ping.data.message << '\n';
|
||||
std::cout << "version: " << ping.data.version << '\n';
|
||||
}
|
||||
|
||||
constexpr auto app_version() {
|
||||
|
|
@ -82,7 +82,8 @@ int main(int argc, char* argv[]) {
|
|||
oro_api.get(),
|
||||
db.get(),
|
||||
app_conf.fetch_extra_delay(),
|
||||
app_conf.worker_threads()
|
||||
app_conf.worker_threads(),
|
||||
app_conf.store_raw_json()
|
||||
);
|
||||
}
|
||||
#if defined(OROTOOL_WITH_RESTCCPP)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ namespace nap {
|
|||
struct HttpResponse {
|
||||
std::unique_ptr<char[]> raw;
|
||||
std::vector<std::pair<std::string_view, std::string_view>> header_list;
|
||||
std::string_view header;
|
||||
std::string_view body;
|
||||
std::string_view http_ver;
|
||||
std::string_view code_desc;
|
||||
|
|
|
|||
|
|
@ -101,15 +101,14 @@ HttpResponse page_fetch (
|
|||
HttpResponse resp;
|
||||
resp.code = easy.get_info<CURLINFO_RESPONSE_CODE>().get();
|
||||
|
||||
std::string_view head;
|
||||
{
|
||||
auto [raw, head_tmp, body] = make_raw_string(header_oss, body_oss, body_padding);
|
||||
head = head_tmp;
|
||||
resp.raw = std::move(raw);
|
||||
resp.header = head_tmp;
|
||||
resp.body = body;
|
||||
}
|
||||
|
||||
auto parsed_header = header_parse(head);
|
||||
auto parsed_header = header_parse(resp.header);
|
||||
resp.header_list = std::move(parsed_header.fields);
|
||||
assert(resp.code == parsed_header.code);
|
||||
resp.http_ver = parsed_header.version;
|
||||
|
|
|
|||
|
|
@ -61,6 +61,13 @@ struct Header {
|
|||
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct ApiOutput {
|
||||
Header header;
|
||||
T data;
|
||||
std::string raw_response;
|
||||
};
|
||||
|
||||
class Api {
|
||||
public:
|
||||
Api (
|
||||
|
|
@ -71,12 +78,12 @@ public:
|
|||
);
|
||||
virtual ~Api() noexcept;
|
||||
|
||||
virtual std::pair<Header, Ping> ping() = 0;
|
||||
virtual std::pair<Header, WhoAmI> who_am_i() = 0;
|
||||
virtual std::pair<Header, Items> items_list() = 0;
|
||||
virtual std::pair<Header, Icons> items_icons() = 0;
|
||||
virtual std::pair<Header, Shops> market_list() = 0;
|
||||
virtual std::pair<Header, Creators> fame_list() = 0;
|
||||
virtual ApiOutput<Ping> ping(bool with_raw) = 0;
|
||||
virtual ApiOutput<WhoAmI> who_am_i(bool with_raw) = 0;
|
||||
virtual ApiOutput<Items> items_list(bool with_raw) = 0;
|
||||
virtual ApiOutput<Icons> items_icons(bool with_raw) = 0;
|
||||
virtual ApiOutput<Shops> market_list(bool with_raw) = 0;
|
||||
virtual ApiOutput<Creators> fame_list(bool with_raw) = 0;
|
||||
|
||||
protected:
|
||||
std::string m_prefix;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
#include <optional>
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cassert>
|
||||
|
||||
namespace sjd = simdjson::dom;
|
||||
namespace sj = simdjson;
|
||||
|
|
@ -115,25 +116,27 @@ ApiNap::ApiNap (
|
|||
|
||||
ApiNap::~ApiNap() noexcept = default;
|
||||
|
||||
std::pair<Header, Ping> ApiNap::ping() {
|
||||
ApiOutput<Ping> ApiNap::ping(bool with_raw) {
|
||||
return fetch_and_parse<Ping>(
|
||||
g_endpoint_ping,
|
||||
[](const simdjson::dom::element& doc, Ping& out) {
|
||||
out.message = doc["message"];
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
std::pair<Header, WhoAmI> ApiNap::who_am_i() {
|
||||
ApiOutput<WhoAmI> ApiNap::who_am_i(bool with_raw) {
|
||||
return fetch_and_parse<WhoAmI>(
|
||||
g_endpoint_whoami,
|
||||
[](const simdjson::dom::element& doc, WhoAmI& out) {
|
||||
out.master_id = static_cast<unsigned int>(doc["master_id"].get_uint64());
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
std::pair<Header, Items> ApiNap::items_list() {
|
||||
ApiOutput<Items> ApiNap::items_list(bool with_raw) {
|
||||
return fetch_and_parse<Items>(
|
||||
g_endpoint_items_list,
|
||||
[](const simdjson::dom::element& doc, Items& out) {
|
||||
|
|
@ -150,11 +153,12 @@ std::pair<Header, Items> ApiNap::items_list() {
|
|||
new_entry.slots = get_optional<unsigned int, uint64_t>(item["slots"]);
|
||||
out.items.push_back(std::move(new_entry));
|
||||
}
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
std::pair<Header, Icons> ApiNap::items_icons() {
|
||||
ApiOutput<Icons> ApiNap::items_icons(bool with_raw) {
|
||||
return fetch_and_parse<Icons>(
|
||||
g_endpoint_items_icons,
|
||||
[](const simdjson::dom::element& doc, Icons& out) {
|
||||
|
|
@ -166,11 +170,12 @@ std::pair<Header, Icons> ApiNap::items_icons() {
|
|||
new_entry.icon = icon["icon"];
|
||||
out.icons.push_back(std::move(new_entry));
|
||||
}
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
std::pair<Header, Shops> ApiNap::market_list() {
|
||||
ApiOutput<Shops> ApiNap::market_list(bool with_raw) {
|
||||
return fetch_and_parse<Shops>(
|
||||
g_endpoint_market_list,
|
||||
[](const simdjson::dom::element& doc, Shops& out) {
|
||||
|
|
@ -216,22 +221,24 @@ std::pair<Header, Shops> ApiNap::market_list() {
|
|||
}
|
||||
out.shops.push_back(std::move(new_shop));
|
||||
}
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
std::pair<Header, Creators> ApiNap::fame_list() {
|
||||
ApiOutput<Creators> ApiNap::fame_list(bool with_raw) {
|
||||
return fetch_and_parse<Creators>(
|
||||
g_endpoint_fame_list,
|
||||
[](const simdjson::dom::element& doc, Creators& out) {
|
||||
fill_creator_list(doc["brewers"], out.brewers);
|
||||
fill_creator_list(doc["forgers"], out.brewers);
|
||||
}
|
||||
},
|
||||
with_raw
|
||||
);
|
||||
}
|
||||
|
||||
template <typename T, typename F>
|
||||
std::pair<Header, T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill) {
|
||||
ApiOutput<T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw) {
|
||||
auto resp = m_qrest.fetch(m_prefix + endpoint);
|
||||
|
||||
if (200 != resp.code)
|
||||
|
|
@ -249,7 +256,16 @@ std::pair<Header, T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fil
|
|||
data_fill(doc, dataret);
|
||||
}
|
||||
|
||||
return {to_header(resp), std::move(dataret)};
|
||||
std::string raw_resp;
|
||||
if (with_raw) {
|
||||
assert(not resp.header.empty() and resp.header[resp.header.size() - 1] == '\n');
|
||||
raw_resp.reserve(resp.header.size() + 1 + resp.body.size());
|
||||
raw_resp.append(resp.header);
|
||||
raw_resp.append("\n");
|
||||
raw_resp.append(resp.body);
|
||||
}
|
||||
|
||||
return {to_header(resp), std::move(dataret), std::move(raw_resp)};
|
||||
}
|
||||
|
||||
} //namespace oro
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
#include "datatypes.hpp"
|
||||
#include "oro/dboperation.hpp"
|
||||
#include <string_view>
|
||||
#include "oro/source.hpp"
|
||||
#include <memory>
|
||||
|
||||
namespace oro {
|
||||
|
|
@ -34,10 +34,10 @@ public:
|
|||
OriginsDB() = default;
|
||||
virtual ~OriginsDB() noexcept = default;
|
||||
|
||||
virtual void update (const Items& items, const oro::Timestamp& next_update) = 0;
|
||||
virtual void update (const Icons& icons, const oro::Timestamp& next_update) = 0;
|
||||
virtual void update (const Shops& shops, const oro::Timestamp& next_update) = 0;
|
||||
virtual void update (const Creators& creat, const oro::Timestamp& next_update) = 0;
|
||||
virtual void update (const Items& items, const oro::Timestamp& next_update, const Source& source) = 0;
|
||||
virtual void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) = 0;
|
||||
virtual void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) = 0;
|
||||
virtual void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) = 0;
|
||||
|
||||
virtual Timestamp next_access_time (DBOperation op) const = 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@
|
|||
#include "oro/api.hpp"
|
||||
#include "nap/quick_rest.hpp"
|
||||
#include <mutex>
|
||||
#include <utility>
|
||||
|
||||
namespace oro {
|
||||
|
||||
|
|
@ -35,16 +34,16 @@ public:
|
|||
|
||||
virtual ~ApiNap() noexcept;
|
||||
|
||||
virtual std::pair<Header, Ping> ping() override;
|
||||
virtual std::pair<Header, WhoAmI> who_am_i() override;
|
||||
virtual std::pair<Header, Items> items_list() override;
|
||||
virtual std::pair<Header, Icons> items_icons() override;
|
||||
virtual std::pair<Header, Shops> market_list() override;
|
||||
virtual std::pair<Header, Creators> fame_list() override;
|
||||
virtual ApiOutput<Ping> ping(bool with_raw) override;
|
||||
virtual ApiOutput<WhoAmI> who_am_i(bool with_raw) override;
|
||||
virtual ApiOutput<Items> items_list(bool with_raw) override;
|
||||
virtual ApiOutput<Icons> items_icons(bool with_raw) override;
|
||||
virtual ApiOutput<Shops> market_list(bool with_raw) override;
|
||||
virtual ApiOutput<Creators> fame_list(bool with_raw) override;
|
||||
|
||||
private:
|
||||
template <typename T, typename F>
|
||||
std::pair<Header, T> fetch_and_parse (const char* endpoint, F&& data_fill);
|
||||
ApiOutput<T> fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw);
|
||||
|
||||
std::mutex m_json_mutex;
|
||||
simdjson::dom::parser m_json;
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ namespace {
|
|||
", master_id INTEGER UNIQUE"
|
||||
", name TEXT"
|
||||
", points INTEGER"
|
||||
", source_id INTEGER NOT NULL"
|
||||
")";
|
||||
constexpr const char g_create_slotted_cards[] =
|
||||
"CREATE TABLE IF NOT EXISTS slotted_cards("
|
||||
|
|
@ -83,6 +84,7 @@ namespace {
|
|||
", hash TEXT NOT NULL"
|
||||
", discovery_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP"
|
||||
", last_seen_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP"
|
||||
", source_id INTEGER NOT NULL"
|
||||
", FOREIGN KEY(shop_id) REFERENCES shops(id)"
|
||||
")";
|
||||
constexpr const char g_create_shops[] =
|
||||
|
|
@ -108,6 +110,7 @@ namespace {
|
|||
", npc_price INTEGER"
|
||||
", slots TINYINT"
|
||||
", removal_date TEXT"
|
||||
", source_id INTEGER NOT NULL"
|
||||
")";
|
||||
constexpr const char g_create_icons_a[] = "CREATE TABLE IF NOT EXISTS ";
|
||||
constexpr const char g_create_icons_b[] =
|
||||
|
|
@ -115,6 +118,7 @@ namespace {
|
|||
"id INTEGER PRIMARY KEY NOT NULL"
|
||||
", item_id INTEGER NOT NULL"
|
||||
", icon TEXT"
|
||||
", source_id INTEGER NOT NULL"
|
||||
", FOREIGN KEY(item_id) REFERENCES items(id)"
|
||||
")";
|
||||
constexpr const char g_create_access[] =
|
||||
|
|
@ -123,6 +127,12 @@ namespace {
|
|||
", operation TINYINT PRIMARY KEY NOT NULL"
|
||||
", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'"
|
||||
")";
|
||||
constexpr const char g_create_source_store[] =
|
||||
"CREATE TABLE IF NOT EXISTS source_store("
|
||||
"id INTEGER PRIMARY KEY NOT NULL"
|
||||
", source TEXT"
|
||||
", format TEXT"
|
||||
")";
|
||||
|
||||
std::string to_string (int num) {
|
||||
auto ary = dhandy::int_to_ary<int>(num);
|
||||
|
|
@ -322,6 +332,17 @@ namespace {
|
|||
|
||||
return (prefix + values + suffix);
|
||||
}
|
||||
|
||||
SQLiteID insert_source_ifn (SQLite::Database& db, const Source& source) {
|
||||
if (not source.data)
|
||||
return 0;
|
||||
|
||||
db.exec(g_create_source_store);
|
||||
SQLite::Statement query(db, "INSERT INTO source_store(source, format) VALUES(?, ?)");
|
||||
bind_all(query, no_copy(*source.data), static_cast<int>(source.format));
|
||||
query.exec();
|
||||
return db.getLastInsertRowid();
|
||||
}
|
||||
} //unnamed namespace
|
||||
|
||||
OriginsDBSQLite::OriginsDBSQLite (std::string_view path) :
|
||||
|
|
@ -336,7 +357,7 @@ OriginsDBSQLite::OriginsDBSQLite (std::string_view path) :
|
|||
|
||||
OriginsDBSQLite::~OriginsDBSQLite() noexcept = default;
|
||||
|
||||
void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update) {
|
||||
void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update, const Source& source) {
|
||||
auto& db = *m_db;
|
||||
std::unique_lock<std::mutex> lock(m_db_mutex);
|
||||
update_last_access(db, DBOperation::Items, next_update);
|
||||
|
|
@ -360,14 +381,16 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd
|
|||
|
||||
SQLite::Statement query(db,
|
||||
"INSERT INTO items_staging "
|
||||
"(item_id, unique_name, name, type, subtype, npc_price, slots) "
|
||||
"VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7)"
|
||||
"(item_id, unique_name, name, type, subtype, npc_price, slots, source_id) "
|
||||
"VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"
|
||||
);
|
||||
SQLite::Transaction transaction(db);
|
||||
const SQLiteID source_id = insert_source_ifn(db, source);
|
||||
for (const auto& item : items.items) {
|
||||
bind_all(query, item.item_id, no_copy(item.unique_name),
|
||||
no_copy(item.name), static_cast<int>(item.type.value),
|
||||
cast<int>(item.subtype), item.npc_price, item.slots
|
||||
cast<int>(item.subtype), item.npc_price, item.slots,
|
||||
source_id
|
||||
);
|
||||
|
||||
query.exec();
|
||||
|
|
@ -384,19 +407,19 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd
|
|||
")"
|
||||
);
|
||||
db.exec(
|
||||
"INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots) "
|
||||
"SELECT item_id, unique_name, name, type, subtype, npc_price, slots FROM items_staging WHERE true "
|
||||
"INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots, source_id) "
|
||||
"SELECT item_id, unique_name, name, type, subtype, npc_price, slots, source_id FROM items_staging WHERE true "
|
||||
"ON CONFLICT(item_id, ifnull(removal_date, 0)) DO UPDATE SET "
|
||||
"unique_name=excluded.unique_name, "
|
||||
"name=excluded.name, type=excluded.type, subtype=excluded.subtype, "
|
||||
"npc_price=excluded.npc_price, slots=excluded.slots"
|
||||
"npc_price=excluded.npc_price, slots=excluded.slots, source_id=excluded.source_id"
|
||||
);
|
||||
db.exec("DROP TABLE items_staging");
|
||||
|
||||
transaction.commit();
|
||||
}
|
||||
|
||||
void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update) {
|
||||
void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) {
|
||||
//example:
|
||||
//{
|
||||
// "item_id":501,
|
||||
|
|
@ -411,33 +434,34 @@ void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_upd
|
|||
db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b);
|
||||
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS icons_item_id_idx ON icons(item_id)");
|
||||
SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)");
|
||||
SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon) VALUES(?, ?)");
|
||||
SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon, source_id) VALUES(?, ?, ?)");
|
||||
|
||||
ItemIdToTableId item_id_to_table_id(db, false);
|
||||
SQLite::Transaction transaction(db);
|
||||
const SQLiteID source_id = insert_source_ifn(db, source);
|
||||
for (const auto& ico : icons.icons) {
|
||||
bind_all(ins_empty_item, ico.item_id);
|
||||
ins_empty_item.exec();
|
||||
ins_empty_item.reset();
|
||||
|
||||
bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon));
|
||||
bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon), source_id);
|
||||
query.exec();
|
||||
query.reset();
|
||||
|
||||
//base64_decode(std::string_view(ico.icon).substr(std::string_view("data:image/png;base64,").size()));
|
||||
}
|
||||
db.exec(
|
||||
"INSERT INTO icons (item_id, icon) "
|
||||
"SELECT item_id, icon FROM icons_staging WHERE true "
|
||||
"INSERT INTO icons (item_id, icon, source_id) "
|
||||
"SELECT item_id, icon, source_id FROM icons_staging WHERE true "
|
||||
"ON CONFLICT(item_id) DO UPDATE SET "
|
||||
"icon=excluded.icon"
|
||||
"icon=excluded.icon, source_id=excluded.source_id"
|
||||
);
|
||||
db.exec("DROP TABLE icons_staging");
|
||||
|
||||
transaction.commit();
|
||||
}
|
||||
|
||||
void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update) {
|
||||
void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) {
|
||||
//example:
|
||||
//{
|
||||
// "title":"• B\u003ePoison Bottle •",
|
||||
|
|
@ -463,7 +487,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
|
|||
db.exec(g_create_shop_items);
|
||||
db.exec(g_create_slotted_cards);
|
||||
|
||||
SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash) VALUES(?, ?)");
|
||||
SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash, source_id) VALUES(?, ?, ?)");
|
||||
SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)");
|
||||
SQLite::Statement sel_sshot(db, "SELECT id FROM shop_snapshots WHERE shop_id = ? AND hash = ?");
|
||||
SQLite::Statement ins_item(db, "INSERT INTO shop_items(snapshot_id, item_id, amount, price, refine, star_crumbs, element, creator, beloved) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
||||
|
|
@ -473,6 +497,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
|
|||
InsertShopIfn insert_shop_ifn(db);
|
||||
ItemIdToTableId item_id_to_table_id(db);
|
||||
SQLite::Transaction transaction(db);
|
||||
const SQLiteID source_id = insert_source_ifn(db, source);
|
||||
for (const auto& shop : shops.shops) {
|
||||
{
|
||||
//insert new shop or get its id if already there
|
||||
|
|
@ -490,7 +515,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
|
|||
}
|
||||
|
||||
//snapshot is new, insert it
|
||||
bind_all(ins_sshot, shop_id_ins.first, no_copy(hash));
|
||||
bind_all(ins_sshot, shop_id_ins.first, no_copy(hash), source_id);
|
||||
ins_sshot.exec();
|
||||
ins_sshot.reset();
|
||||
}
|
||||
|
|
@ -520,26 +545,27 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
|
|||
transaction.commit();
|
||||
}
|
||||
|
||||
void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update) {
|
||||
void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) {
|
||||
std::unique_lock<std::mutex> lock(m_db_mutex);
|
||||
auto& db = *m_db;
|
||||
update_last_access(db, DBOperation::Creators, next_update);
|
||||
|
||||
db.exec(g_create_fame_list);
|
||||
|
||||
auto insert_creators = [](SQLite::Statement& query, const std::vector<oro::Creator>& creats, int type) {
|
||||
auto insert_creators = [](SQLite::Statement& query, const std::vector<oro::Creator>& creats, int type, SQLiteID source_id) {
|
||||
for (const auto& creator : creats) {
|
||||
bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points);
|
||||
bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points, source_id);
|
||||
query.exec();
|
||||
query.reset();
|
||||
}
|
||||
};
|
||||
|
||||
SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points) VALUES (?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points");
|
||||
SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points, source_id) VALUES (?, ?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points");
|
||||
|
||||
SQLite::Transaction transaction(db);
|
||||
insert_creators(query, creat.brewers, 1);
|
||||
insert_creators(query, creat.forgers, 2);
|
||||
const SQLiteID source_id = insert_source_ifn(db, source);
|
||||
insert_creators(query, creat.brewers, 1, source_id);
|
||||
insert_creators(query, creat.forgers, 2, source_id);
|
||||
transaction.commit();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,10 +34,10 @@ public:
|
|||
explicit OriginsDBSQLite(std::string_view path);
|
||||
virtual ~OriginsDBSQLite() noexcept;
|
||||
|
||||
void update (const Items& items, const oro::Timestamp& next_update) override;
|
||||
void update (const Icons& icons, const oro::Timestamp& next_update) override;
|
||||
void update (const Shops& shops, const oro::Timestamp& next_update) override;
|
||||
void update (const Creators& creat, const oro::Timestamp& next_update) override;
|
||||
void update (const Items& items, const oro::Timestamp& next_update, const Source& source) override;
|
||||
void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) override;
|
||||
void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) override;
|
||||
void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) override;
|
||||
|
||||
Timestamp next_access_time (DBOperation op) const override;
|
||||
|
||||
|
|
|
|||
38
src/oro/source.hpp
Normal file
38
src/oro/source.hpp
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/* Copyright 2020, Michele Santullo
|
||||
* This file is part of orotool.
|
||||
*
|
||||
* Orotool is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Orotool is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Orotool. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
|
||||
namespace oro {
|
||||
|
||||
//The raw json source data complete with http response header. It can be
|
||||
//anything really, but it's intended to be stored in the DB. Entries in the
|
||||
//other tables will have a reference to the source.
|
||||
|
||||
enum class SourceFormat {
|
||||
Plain
|
||||
};
|
||||
|
||||
struct Source {
|
||||
std::optional<std::string> data;
|
||||
SourceFormat format;
|
||||
};
|
||||
|
||||
} //namespace oro
|
||||
|
|
@ -64,14 +64,15 @@ TimerBase::TimerBase (
|
|||
m_extra_delay(settings.extra_delay),
|
||||
m_pool(pool),
|
||||
m_oro_api(oro_api),
|
||||
m_db(db)
|
||||
m_db(db),
|
||||
m_store_raw_response(settings.store_raw_json)
|
||||
{
|
||||
assert(m_pool);
|
||||
assert(m_oro_api);
|
||||
}
|
||||
|
||||
void TimerBase::on_timer() {
|
||||
m_pool->submit(&TimerBase::fetch_data, this);
|
||||
m_pool->submit(&TimerBase::fetch_data, this, m_store_raw_response);
|
||||
}
|
||||
|
||||
void TimerBase::set_next_timer (const oro::Header& header) {
|
||||
|
|
@ -95,20 +96,20 @@ oro::OriginsDB& TimerBase::db() {
|
|||
return *m_db;
|
||||
}
|
||||
|
||||
void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) {
|
||||
db().update(shops, calc_next_update(header, m_extra_delay));
|
||||
void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source) {
|
||||
db().update(shops, calc_next_update(header, m_extra_delay), source);
|
||||
}
|
||||
|
||||
void TimerBase::update_db (const oro::Items& items, const oro::Header& header) {
|
||||
db().update(items, calc_next_update(header, m_extra_delay));
|
||||
void TimerBase::update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source) {
|
||||
db().update(items, calc_next_update(header, m_extra_delay), source);
|
||||
}
|
||||
|
||||
void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) {
|
||||
db().update(icons, calc_next_update(header, m_extra_delay));
|
||||
void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source) {
|
||||
db().update(icons, calc_next_update(header, m_extra_delay), source);
|
||||
}
|
||||
|
||||
void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) {
|
||||
db().update(creators, calc_next_update(header, m_extra_delay));
|
||||
void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source) {
|
||||
db().update(creators, calc_next_update(header, m_extra_delay), source);
|
||||
}
|
||||
|
||||
} //namespace duck
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
#include "eventia/private/context.hpp"
|
||||
#include "eventia/timer.hpp"
|
||||
#include "oro/dboperation.hpp"
|
||||
#include "oro/source.hpp"
|
||||
|
||||
namespace roar11 {
|
||||
class ThreadPool;
|
||||
|
|
@ -38,13 +39,15 @@ namespace oro {
|
|||
namespace duck {
|
||||
|
||||
struct TimerSettings {
|
||||
TimerSettings (double min_wait, double extra_delay) :
|
||||
TimerSettings (double min_wait, double extra_delay, bool with_raw_json) :
|
||||
min_wait(min_wait),
|
||||
extra_delay(extra_delay)
|
||||
extra_delay(extra_delay),
|
||||
store_raw_json(with_raw_json)
|
||||
{ }
|
||||
|
||||
double min_wait;
|
||||
double extra_delay;
|
||||
bool store_raw_json;
|
||||
};
|
||||
|
||||
class TimerBase : public eve::Timer {
|
||||
|
|
@ -63,21 +66,22 @@ public:
|
|||
|
||||
protected:
|
||||
void set_next_timer (const oro::Header& header);
|
||||
void update_db (const oro::Shops& shops, const oro::Header& header);
|
||||
void update_db (const oro::Items& items, const oro::Header& header);
|
||||
void update_db (const oro::Icons& icons, const oro::Header& header);
|
||||
void update_db (const oro::Creators& creators, const oro::Header& header);
|
||||
void update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source);
|
||||
void update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source);
|
||||
void update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source);
|
||||
void update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source);
|
||||
roar11::ThreadPool& pool();
|
||||
oro::Api& oro_api();
|
||||
oro::OriginsDB& db();
|
||||
|
||||
private:
|
||||
virtual void fetch_data() = 0;
|
||||
virtual void fetch_data(bool with_raw) = 0;
|
||||
|
||||
double m_extra_delay;
|
||||
roar11::ThreadPool* m_pool;
|
||||
oro::Api* m_oro_api;
|
||||
oro::OriginsDB* m_db;
|
||||
bool m_store_raw_response;
|
||||
};
|
||||
|
||||
} //namespace duck
|
||||
|
|
|
|||
|
|
@ -28,26 +28,26 @@
|
|||
namespace duck {
|
||||
|
||||
namespace {
|
||||
template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api);
|
||||
template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api, bool with_raw);
|
||||
|
||||
template <>
|
||||
inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api) {
|
||||
return api.items_icons();
|
||||
inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api, bool with_raw) {
|
||||
return api.items_icons(with_raw);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api) {
|
||||
return api.items_list();
|
||||
inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api, bool with_raw) {
|
||||
return api.items_list(with_raw);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api) {
|
||||
return api.fame_list();
|
||||
inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api, bool with_raw) {
|
||||
return api.fame_list(with_raw);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api) {
|
||||
return api.market_list();
|
||||
inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api, bool with_raw) {
|
||||
return api.market_list(with_raw);
|
||||
}
|
||||
} //unnamed namespace
|
||||
|
||||
|
|
@ -64,12 +64,18 @@ inline TimerOroApi<Op>::TimerOroApi (
|
|||
}
|
||||
|
||||
template<oro::DBOperation Op>
|
||||
inline void TimerOroApi<Op>::fetch_data() {
|
||||
inline void TimerOroApi<Op>::fetch_data (bool with_raw) {
|
||||
int status_code = 200;
|
||||
try {
|
||||
auto results = invoke_api_func<Op>(oro_api());
|
||||
set_next_timer(results.first);
|
||||
this->update_db(results.second, results.first);
|
||||
auto results = invoke_api_func<Op>(oro_api(), with_raw);
|
||||
set_next_timer(results.header);
|
||||
|
||||
oro::Source raw_src;
|
||||
if (with_raw) {
|
||||
raw_src.data = std::move(results.raw_response);
|
||||
raw_src.format = oro::SourceFormat::Plain;
|
||||
}
|
||||
this->update_db(results.data, results.header, raw_src);
|
||||
}
|
||||
#if defined(OROTOOL_WITH_RESTCCPP)
|
||||
catch (const restc_cpp::RequestFailedWithErrorException& err) {
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ public:
|
|||
);
|
||||
|
||||
private:
|
||||
virtual void fetch_data() override;
|
||||
virtual void fetch_data (bool with_raw) override;
|
||||
};
|
||||
|
||||
} //namespace duck
|
||||
|
|
|
|||
Loading…
Reference in a new issue