diff --git a/.gitmodules b/.gitmodules index f67262f..0a32d9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "subprojects/duckhandy"] path = subprojects/duckhandy url = http://alarmpi.no-ip.org/gitan/King_DuckZ/duckhandy.git +[submodule "subprojects/magic_enum"] + path = subprojects/magic_enum + url = https://github.com/Neargye/magic_enum.git diff --git a/meson.build b/meson.build index edc03c9..8cdf05f 100644 --- a/meson.build +++ b/meson.build @@ -7,5 +7,6 @@ project('orotool', 'cpp', 'c', duckhandy_inc = include_directories('subprojects/duckhandy/include') date_inc = include_directories('subprojects/date/include') +magic_enum_inc = include_directories('subprojects/magic_enum/include') subdir('src') diff --git a/src/eventia/timer.cpp b/src/eventia/timer.cpp index 1370f60..80bdfba 100644 --- a/src/eventia/timer.cpp +++ b/src/eventia/timer.cpp @@ -85,6 +85,9 @@ void Timer::set_timer (double delay) { m_local->timer.stop(); ev_now_update(*m_local->context.loop); m_local->timer.start(m_local->timer_value, 0.0); +#if !defined(NDEBUG) + std::cout << "Timer::set_timer(" << delay << ")\n"; +#endif m_local->context.async->send(); } diff --git a/src/evloop.cpp b/src/evloop.cpp index f93b6ea..c2caf5e 100644 --- a/src/evloop.cpp +++ b/src/evloop.cpp @@ -20,7 +20,7 @@ #include "eventia/eventia.hpp" #include "eventia/signal.hpp" #include "roar11/ThreadPool.hpp" -#include "oro/api.hpp" +#include "oro/dboperation.hpp" #include namespace duck { @@ -54,26 +54,26 @@ namespace { private: eve::Eventia* m_eventia; }; - } //unnamed namespace void test(oro::Api* api, oro::OriginsDB* db, std::size_t thread_count) { - typedef TimerOroApi<&oro::Api::market_list> TimerShops; - typedef TimerOroApi<&oro::Api::items_list> TimerItems; - typedef TimerOroApi<&oro::Api::items_icons> TimerIcons; - typedef TimerOroApi<&oro::Api::fame_list> TimerCreators; + typedef TimerOroApi TimerShops; + typedef TimerOroApi TimerItems; + typedef TimerOroApi TimerIcons; + typedef TimerOroApi TimerCreators; std::cout << "Running with " << thread_count << " worker threads\n"; roar11::ThreadPool pool(thread_count); eve::Eventia worker; pool.submit(worker.event_functor()); + auto sig_int = worker.make_event(&worker); - auto timer_items = worker.make_event(0.5, &pool, api, db); - auto timer_icons = worker.make_event(1.0, &pool, api, db); - auto timer_shops = worker.make_event(1.5, &pool, api, db); - auto timer_creat = worker.make_event(2.0, &pool, api, db); + auto timer_items = worker.make_event(0.0, &pool, api, db); + auto timer_icons = worker.make_event(5.0, &pool, api, db); + auto timer_shops = worker.make_event(10.0, &pool, api, db); + auto timer_creat = worker.make_event(15.0, &pool, api, db); worker.wait(); #if !defined(NDEBUG) diff --git a/src/meson.build b/src/meson.build index 761ddab..d7a0d00 100644 --- a/src/meson.build +++ b/src/meson.build @@ -61,9 +61,14 @@ executable(meson.project_name(), 'oro/private/tiger.cpp', 'eventia/signal.cpp', 'eventia/event.cpp', + 'timer_oro_api.cpp', project_config_file, install: true, dependencies: lib_deps, - include_directories: [date_inc, duckhandy_inc], + include_directories: [ + date_inc, + duckhandy_inc, + magic_enum_inc, + ], cpp_args: ['-DEV_USE_STDEXCEPT'], ) diff --git a/src/oro/dboperation.hpp b/src/oro/dboperation.hpp new file mode 100644 index 0000000..1cf2dec --- /dev/null +++ b/src/oro/dboperation.hpp @@ -0,0 +1,26 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#pragma once + +namespace oro { + +enum class DBOperation { + Creators, Shops, Items, Icons +}; + +} //namespace oro diff --git a/src/oro/originsdb.cpp b/src/oro/originsdb.cpp index 52bd707..9f74707 100644 --- a/src/oro/originsdb.cpp +++ b/src/oro/originsdb.cpp @@ -28,6 +28,8 @@ #include "oro/private/tiger.hpp" #include "oro/private/sqlite_helpers.hpp" #include "duckhandy/int_conv.hpp" +#include "duckhandy/string_bt.hpp" +#include "magic_enum.hpp" #include #include #include @@ -112,6 +114,17 @@ namespace { ", icon TEXT" ", FOREIGN KEY(item_id) REFERENCES items(id)" ")"; + constexpr const char g_create_access[] = + "CREATE TABLE IF NOT EXISTS access(" + "last TEXT NOT NULL DEFAULT '1970-01-01 00:00:00'" + ", operation TINYINT PRIMARY KEY NOT NULL" + ", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'" + ")"; + + std::string to_string (int num) { + auto ary = dhandy::int_to_ary(num); + return std::string(ary.begin(), ary.begin() + ary.size() - 1); + } class ItemIdToTableId { public: @@ -132,7 +145,7 @@ namespace { m_query.bind(1, item_id); if (not m_query.executeStep()) { m_query.reset(); - throw std::runtime_error("No item_id " + std::to_string(item_id) + " found in table items"); + throw std::runtime_error("No item_id " + to_string(item_id) + " found in table items"); } const int64_t table_id = m_query.getColumn(0); @@ -256,19 +269,74 @@ namespace { } return mchlib::tiger_to_string(th); } + + void update_last_access (SQLite::Database& db, DBOperation op, const oro::Timestamp& next_update) { + db.exec( + std::string("INSERT INTO access (operation, last, next_update) VALUES(").append( + dhandy::int_to_ary(static_cast(op)).to_string_view() + ) + + ", CURRENT_TIMESTAMP, " + + "'" + to_sqlite_string(next_update) + "'" + + ") ON CONFLICT(operation) DO UPDATE " + "SET last=CURRENT_TIMESTAMP" + ", next_update='" + to_sqlite_string(next_update) + "'" + ); + } + + template + constexpr auto make_sel_last_acces_b(std::index_sequence) { + return ( + ( + dhandy::bt::string<(N==0 ? 0 : 3)+1, char>(N == 0 ? "" : "),(") + + dhandy::bt::string< + dhandy::int_to_ary(static_cast(Ops)).size(), + char + >(dhandy::int_to_ary(static_cast(Ops)).data()) + ) + ... + + dhandy::bt::make_string("") + ); + } + + template + constexpr auto make_sel_last_acces_a (std::index_sequence seq) { + return make_sel_last_acces_b()[Indices]...>(seq); + } + + constexpr auto make_select_last_acces_str() { + using dhandy::bt::make_string; + using std::make_index_sequence; + using magic_enum::enum_count; + using oro::DBOperation; + constexpr auto prefix = make_string("WITH list(operation) AS (VALUES("); + constexpr auto suffix = make_string("))" + " SELECT IFNULL(next_update, '1970-01-01 00:00:00') AS next_update" + " FROM list" + " LEFT JOIN access USING(operation)" + " WHERE operation=" + ); + constexpr auto values = + make_sel_last_acces_a(make_index_sequence()>{}); + + return (prefix + values + suffix); + } } //unnamed namespace OriginsDB::OriginsDB (std::string_view path) : m_path(path.data(), path.size()), m_db(std::make_unique(m_path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) { + { + std::unique_lock lock(m_db_mutex); + m_db->exec(g_create_access); + } } OriginsDB::~OriginsDB() noexcept = default; -void OriginsDB::update (const Items& items) { +void OriginsDB::update (const Items& items, const oro::Timestamp& next_update) { auto& db = *m_db; std::unique_lock lock(m_db_mutex); + update_last_access(db, DBOperation::Items, next_update); //example: //{ @@ -324,7 +392,7 @@ void OriginsDB::update (const Items& items) { transaction.commit(); } -void OriginsDB::update (const Icons& icons) { +void OriginsDB::update (const Icons& icons, const oro::Timestamp& next_update) { //example: //{ // "item_id":501, @@ -333,6 +401,7 @@ void OriginsDB::update (const Icons& icons) { std::unique_lock lock(m_db_mutex); auto& db = *m_db; + update_last_access(db, DBOperation::Icons, next_update); db.exec(std::string(g_create_icons_a).append("icons_staging") + g_create_icons_b); db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b); @@ -359,7 +428,7 @@ void OriginsDB::update (const Icons& icons) { transaction.commit(); } -void OriginsDB::update (const Shops& shops) { +void OriginsDB::update (const Shops& shops, const oro::Timestamp& next_update) { //example: //{ // "title":"• B\u003ePoison Bottle •", @@ -376,6 +445,7 @@ void OriginsDB::update (const Shops& shops) { auto& db = *m_db; std::unique_lock lock(m_db_mutex); + update_last_access(db, DBOperation::Shops, next_update); db.exec(g_create_shops); db.exec("CREATE UNIQUE INDEX IF NOT EXISTS shops_owner_idx ON shops(owner, creation_date)"); @@ -436,9 +506,10 @@ void OriginsDB::update (const Shops& shops) { transaction.commit(); } -void OriginsDB::update (const Creators& creat) { +void OriginsDB::update (const Creators& creat, const oro::Timestamp& next_update) { std::unique_lock lock(m_db_mutex); auto& db = *m_db; + update_last_access(db, DBOperation::Creators, next_update); db.exec(g_create_fame_list); @@ -458,4 +529,20 @@ void OriginsDB::update (const Creators& creat) { transaction.commit(); } +Timestamp OriginsDB::next_access_time (DBOperation op) const { + constexpr auto select_last_acces_str = make_select_last_acces_str(); + SQLite::Statement sel_last( + *m_db, + std::string(select_last_acces_str.data()).append( + dhandy::int_to_ary(static_cast(op)).to_string_view() + ) + ); + + auto info = exec_first_res(sel_last); + if (not info) + throw std::runtime_error("Query returned no value for access.operation " + to_string(static_cast(op))); + + return from_sqlite_timestamp(*info); +} + } //namespace oro diff --git a/src/oro/originsdb.hpp b/src/oro/originsdb.hpp index 14d46b7..0fa765f 100644 --- a/src/oro/originsdb.hpp +++ b/src/oro/originsdb.hpp @@ -17,6 +17,8 @@ #pragma once +#include "datatypes.hpp" +#include "oro/dboperation.hpp" #include #include #include @@ -38,10 +40,12 @@ public: explicit OriginsDB(std::string_view path); ~OriginsDB() noexcept; - void update (const Items& items); - void update (const Icons& icons); - void update (const Shops& shops); - void update (const Creators& creat); + void update (const Items& items, const oro::Timestamp& next_update); + void update (const Icons& icons, const oro::Timestamp& next_update); + void update (const Shops& shops, const oro::Timestamp& next_update); + void update (const Creators& creat, const oro::Timestamp& next_update); + + Timestamp next_access_time (DBOperation op) const; private: std::string m_path; diff --git a/src/oro/private/sqlite_helpers.hpp b/src/oro/private/sqlite_helpers.hpp index a1e020d..967eb6c 100644 --- a/src/oro/private/sqlite_helpers.hpp +++ b/src/oro/private/sqlite_helpers.hpp @@ -24,6 +24,13 @@ namespace oro { namespace detail { + template struct RowRetInfo; + template class Ret> + struct RowRetInfo> { + typedef Ret type; + constexpr static const std::size_t count = sizeof...(R); + }; + template struct NoCopy { explicit NoCopy (T& v) : @@ -125,4 +132,17 @@ inline std::optional exec_first_res (SQLite::Statement& st, Args&&... args) { st.reset(); return retval; } + +template +inline std::optional exec_first_row (SQLite::Statement& st, Args&&... args) { + detail::bind_all(st, std::make_integer_sequence{}, std::forward(args)...); + + std::optional retval; + if (st.executeStep()) { + retval = st.getColumns::count>(); + } + + st.reset(); + return retval; +} } //namespace oro diff --git a/src/timer_base.cpp b/src/timer_base.cpp index 1a01f4d..39b8e09 100644 --- a/src/timer_base.cpp +++ b/src/timer_base.cpp @@ -21,17 +21,48 @@ #include "oro/originsdb.hpp" #include #include +#include namespace duck { +namespace { + constexpr const unsigned long g_extra_wait = 30; + + [[gnu::pure]] + unsigned long time_interval (const oro::Header& header) { + return header.retry_after / header.rate_limit; + } + + oro::Timestamp calc_next_update (const oro::Header& header) { + oro::Timestamp ret; + ret.ts = + std::chrono::time_point_cast(std::chrono::system_clock::now()) + + std::chrono::seconds(time_interval(header)); + return ret; + } + + double initial_timer (oro::OriginsDB& db, oro::DBOperation op, double min_wait) { + using std::chrono::duration_cast; + using std::chrono::duration; + using std::chrono::seconds; + + auto now = std::chrono::system_clock::now(); + auto next_time = db.next_access_time(op); + duration diff = duration_cast(next_time.ts - now); + const auto retval = std::max(min_wait, diff.count()); + return retval; + } +} //unnamed namespace + TimerBase::TimerBase ( const eve::Context& ctx, - double timeout, + oro::DBOperation type, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db ) : - eve::Timer(timeout, ctx), + eve::Timer(initial_timer(*db, type, min_wait), ctx), m_pool(pool), m_oro_api(oro_api), m_db(db) @@ -45,7 +76,7 @@ void TimerBase::on_timer() { } void TimerBase::set_next_timer (const oro::Header& header) { - const int next_timer = header.retry_after / header.rate_limit; + const unsigned long next_timer = time_interval(header) + g_extra_wait; std::cout << "Next timer in " << next_timer << " secs\n"; this->set_timer(static_cast(next_timer)); } @@ -65,20 +96,20 @@ oro::OriginsDB& TimerBase::db() { return *m_db; } -void TimerBase::update_db (const oro::Shops& shops) { - db().update(shops); +void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) { + db().update(shops, calc_next_update(header)); } -void TimerBase::update_db (const oro::Items& items) { - db().update(items); +void TimerBase::update_db (const oro::Items& items, const oro::Header& header) { + db().update(items, calc_next_update(header)); } -void TimerBase::update_db (const oro::Icons& icons) { - db().update(icons); +void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) { + db().update(icons, calc_next_update(header)); } -void TimerBase::update_db (const oro::Creators& creators) { - db().update(creators); +void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) { + db().update(creators, calc_next_update(header)); } } //namespace duck diff --git a/src/timer_base.hpp b/src/timer_base.hpp index ff67235..b861c72 100644 --- a/src/timer_base.hpp +++ b/src/timer_base.hpp @@ -19,6 +19,7 @@ #include "eventia/private/context.hpp" #include "eventia/timer.hpp" +#include "oro/dboperation.hpp" namespace roar11 { class ThreadPool; @@ -40,7 +41,8 @@ class TimerBase : public eve::Timer { public: TimerBase ( const eve::Context& ctx, - double timeout, + oro::DBOperation type, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db @@ -51,10 +53,10 @@ public: protected: void set_next_timer (const oro::Header& header); - void update_db (const oro::Shops& shops); - void update_db (const oro::Items& items); - void update_db (const oro::Icons& icons); - void update_db (const oro::Creators& creators); + void update_db (const oro::Shops& shops, const oro::Header& header); + void update_db (const oro::Items& items, const oro::Header& header); + void update_db (const oro::Icons& icons, const oro::Header& header); + void update_db (const oro::Creators& creators, const oro::Header& header); roar11::ThreadPool& pool(); oro::Api& oro_api(); oro::OriginsDB& db(); diff --git a/src/timer_oro_api.cpp b/src/timer_oro_api.cpp new file mode 100644 index 0000000..519cdce --- /dev/null +++ b/src/timer_oro_api.cpp @@ -0,0 +1,21 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#include "timer_oro_api.hpp" + +namespace duck { +} //namespace duck diff --git a/src/timer_oro_api.hpp b/src/timer_oro_api.hpp index 5c7da11..a8d1391 100644 --- a/src/timer_oro_api.hpp +++ b/src/timer_oro_api.hpp @@ -18,6 +18,8 @@ #pragma once #include "timer_base.hpp" +#include "oro/dboperation.hpp" +#include "oro/api.hpp" #include namespace oro { @@ -26,12 +28,12 @@ namespace oro { namespace duck { -template +template class TimerOroApi : TimerBase { public: TimerOroApi ( const eve::Context& ctx, - double timeout, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db @@ -41,24 +43,48 @@ private: virtual void fetch_data() override; }; -template -inline TimerOroApi::TimerOroApi ( +namespace detail { + template auto invoke_api_func (oro::Api& api); + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.items_icons(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.items_list(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.fame_list(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.market_list(); + } +} //namespace detail + +template +inline TimerOroApi::TimerOroApi ( const eve::Context& ctx, - double timeout, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db ) : - TimerBase(ctx, timeout, pool, oro_api, db) + TimerBase(ctx, Op, min_wait, pool, oro_api, db) { } -template -inline void TimerOroApi::fetch_data() { +template +inline void TimerOroApi::fetch_data() { try { - auto results = (oro_api().*Method)(); + auto results = detail::invoke_api_func(oro_api()); set_next_timer(results.first); - this->update_db(results.second); + this->update_db(results.second, results.first); } catch (...) { this->set_exception(std::current_exception()); diff --git a/subprojects/magic_enum b/subprojects/magic_enum new file mode 160000 index 0000000..d73a985 --- /dev/null +++ b/subprojects/magic_enum @@ -0,0 +1 @@ +Subproject commit d73a985f520aa25a432d1bbf5525663b93c189a0