From b06456e4b09490ff76399d2e9448bae227677c27 Mon Sep 17 00:00:00 2001 From: King_DuckZ Date: Sun, 16 Aug 2020 06:28:58 +0100 Subject: [PATCH] Restore timeouts from previous run This is quite a big update, the new feature is that when the program starts it loads the next update time from the db if present. If so it resumes the timers from there. Maximum wait time is currently capped at 24h. TimerOroApi is not templated on methods anymore, rather on the new DBOperation enum. This is so that it has that enum value to pass it to other functions. This could've been a separate commit, but wth... Magic enum allows me to iterate over an enum's values at build time, it's useful for building the query in OriginsDB for the SELECT in the access table (see make_select_last_access_str()). --- .gitmodules | 3 + meson.build | 1 + src/eventia/timer.cpp | 3 + src/evloop.cpp | 20 +++--- src/meson.build | 7 ++- src/oro/dboperation.hpp | 26 ++++++++ src/oro/originsdb.cpp | 97 ++++++++++++++++++++++++++++-- src/oro/originsdb.hpp | 12 ++-- src/oro/private/sqlite_helpers.hpp | 20 ++++++ src/timer_base.cpp | 53 ++++++++++++---- src/timer_base.hpp | 12 ++-- src/timer_oro_api.cpp | 21 +++++++ src/timer_oro_api.hpp | 46 +++++++++++--- subprojects/magic_enum | 1 + 14 files changed, 276 insertions(+), 46 deletions(-) create mode 100644 src/oro/dboperation.hpp create mode 100644 src/timer_oro_api.cpp create mode 160000 subprojects/magic_enum diff --git a/.gitmodules b/.gitmodules index f67262f..0a32d9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "subprojects/duckhandy"] path = subprojects/duckhandy url = http://alarmpi.no-ip.org/gitan/King_DuckZ/duckhandy.git +[submodule "subprojects/magic_enum"] + path = subprojects/magic_enum + url = https://github.com/Neargye/magic_enum.git diff --git a/meson.build b/meson.build index edc03c9..8cdf05f 100644 --- a/meson.build +++ b/meson.build @@ -7,5 +7,6 @@ project('orotool', 'cpp', 'c', duckhandy_inc = include_directories('subprojects/duckhandy/include') date_inc = include_directories('subprojects/date/include') +magic_enum_inc = include_directories('subprojects/magic_enum/include') subdir('src') diff --git a/src/eventia/timer.cpp b/src/eventia/timer.cpp index 1370f60..80bdfba 100644 --- a/src/eventia/timer.cpp +++ b/src/eventia/timer.cpp @@ -85,6 +85,9 @@ void Timer::set_timer (double delay) { m_local->timer.stop(); ev_now_update(*m_local->context.loop); m_local->timer.start(m_local->timer_value, 0.0); +#if !defined(NDEBUG) + std::cout << "Timer::set_timer(" << delay << ")\n"; +#endif m_local->context.async->send(); } diff --git a/src/evloop.cpp b/src/evloop.cpp index f93b6ea..c2caf5e 100644 --- a/src/evloop.cpp +++ b/src/evloop.cpp @@ -20,7 +20,7 @@ #include "eventia/eventia.hpp" #include "eventia/signal.hpp" #include "roar11/ThreadPool.hpp" -#include "oro/api.hpp" +#include "oro/dboperation.hpp" #include namespace duck { @@ -54,26 +54,26 @@ namespace { private: eve::Eventia* m_eventia; }; - } //unnamed namespace void test(oro::Api* api, oro::OriginsDB* db, std::size_t thread_count) { - typedef TimerOroApi<&oro::Api::market_list> TimerShops; - typedef TimerOroApi<&oro::Api::items_list> TimerItems; - typedef TimerOroApi<&oro::Api::items_icons> TimerIcons; - typedef TimerOroApi<&oro::Api::fame_list> TimerCreators; + typedef TimerOroApi TimerShops; + typedef TimerOroApi TimerItems; + typedef TimerOroApi TimerIcons; + typedef TimerOroApi TimerCreators; std::cout << "Running with " << thread_count << " worker threads\n"; roar11::ThreadPool pool(thread_count); eve::Eventia worker; pool.submit(worker.event_functor()); + auto sig_int = worker.make_event(&worker); - auto timer_items = worker.make_event(0.5, &pool, api, db); - auto timer_icons = worker.make_event(1.0, &pool, api, db); - auto timer_shops = worker.make_event(1.5, &pool, api, db); - auto timer_creat = worker.make_event(2.0, &pool, api, db); + auto timer_items = worker.make_event(0.0, &pool, api, db); + auto timer_icons = worker.make_event(5.0, &pool, api, db); + auto timer_shops = worker.make_event(10.0, &pool, api, db); + auto timer_creat = worker.make_event(15.0, &pool, api, db); worker.wait(); #if !defined(NDEBUG) diff --git a/src/meson.build b/src/meson.build index 761ddab..d7a0d00 100644 --- a/src/meson.build +++ b/src/meson.build @@ -61,9 +61,14 @@ executable(meson.project_name(), 'oro/private/tiger.cpp', 'eventia/signal.cpp', 'eventia/event.cpp', + 'timer_oro_api.cpp', project_config_file, install: true, dependencies: lib_deps, - include_directories: [date_inc, duckhandy_inc], + include_directories: [ + date_inc, + duckhandy_inc, + magic_enum_inc, + ], cpp_args: ['-DEV_USE_STDEXCEPT'], ) diff --git a/src/oro/dboperation.hpp b/src/oro/dboperation.hpp new file mode 100644 index 0000000..1cf2dec --- /dev/null +++ b/src/oro/dboperation.hpp @@ -0,0 +1,26 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#pragma once + +namespace oro { + +enum class DBOperation { + Creators, Shops, Items, Icons +}; + +} //namespace oro diff --git a/src/oro/originsdb.cpp b/src/oro/originsdb.cpp index 52bd707..9f74707 100644 --- a/src/oro/originsdb.cpp +++ b/src/oro/originsdb.cpp @@ -28,6 +28,8 @@ #include "oro/private/tiger.hpp" #include "oro/private/sqlite_helpers.hpp" #include "duckhandy/int_conv.hpp" +#include "duckhandy/string_bt.hpp" +#include "magic_enum.hpp" #include #include #include @@ -112,6 +114,17 @@ namespace { ", icon TEXT" ", FOREIGN KEY(item_id) REFERENCES items(id)" ")"; + constexpr const char g_create_access[] = + "CREATE TABLE IF NOT EXISTS access(" + "last TEXT NOT NULL DEFAULT '1970-01-01 00:00:00'" + ", operation TINYINT PRIMARY KEY NOT NULL" + ", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'" + ")"; + + std::string to_string (int num) { + auto ary = dhandy::int_to_ary(num); + return std::string(ary.begin(), ary.begin() + ary.size() - 1); + } class ItemIdToTableId { public: @@ -132,7 +145,7 @@ namespace { m_query.bind(1, item_id); if (not m_query.executeStep()) { m_query.reset(); - throw std::runtime_error("No item_id " + std::to_string(item_id) + " found in table items"); + throw std::runtime_error("No item_id " + to_string(item_id) + " found in table items"); } const int64_t table_id = m_query.getColumn(0); @@ -256,19 +269,74 @@ namespace { } return mchlib::tiger_to_string(th); } + + void update_last_access (SQLite::Database& db, DBOperation op, const oro::Timestamp& next_update) { + db.exec( + std::string("INSERT INTO access (operation, last, next_update) VALUES(").append( + dhandy::int_to_ary(static_cast(op)).to_string_view() + ) + + ", CURRENT_TIMESTAMP, " + + "'" + to_sqlite_string(next_update) + "'" + + ") ON CONFLICT(operation) DO UPDATE " + "SET last=CURRENT_TIMESTAMP" + ", next_update='" + to_sqlite_string(next_update) + "'" + ); + } + + template + constexpr auto make_sel_last_acces_b(std::index_sequence) { + return ( + ( + dhandy::bt::string<(N==0 ? 0 : 3)+1, char>(N == 0 ? "" : "),(") + + dhandy::bt::string< + dhandy::int_to_ary(static_cast(Ops)).size(), + char + >(dhandy::int_to_ary(static_cast(Ops)).data()) + ) + ... + + dhandy::bt::make_string("") + ); + } + + template + constexpr auto make_sel_last_acces_a (std::index_sequence seq) { + return make_sel_last_acces_b()[Indices]...>(seq); + } + + constexpr auto make_select_last_acces_str() { + using dhandy::bt::make_string; + using std::make_index_sequence; + using magic_enum::enum_count; + using oro::DBOperation; + constexpr auto prefix = make_string("WITH list(operation) AS (VALUES("); + constexpr auto suffix = make_string("))" + " SELECT IFNULL(next_update, '1970-01-01 00:00:00') AS next_update" + " FROM list" + " LEFT JOIN access USING(operation)" + " WHERE operation=" + ); + constexpr auto values = + make_sel_last_acces_a(make_index_sequence()>{}); + + return (prefix + values + suffix); + } } //unnamed namespace OriginsDB::OriginsDB (std::string_view path) : m_path(path.data(), path.size()), m_db(std::make_unique(m_path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE)) { + { + std::unique_lock lock(m_db_mutex); + m_db->exec(g_create_access); + } } OriginsDB::~OriginsDB() noexcept = default; -void OriginsDB::update (const Items& items) { +void OriginsDB::update (const Items& items, const oro::Timestamp& next_update) { auto& db = *m_db; std::unique_lock lock(m_db_mutex); + update_last_access(db, DBOperation::Items, next_update); //example: //{ @@ -324,7 +392,7 @@ void OriginsDB::update (const Items& items) { transaction.commit(); } -void OriginsDB::update (const Icons& icons) { +void OriginsDB::update (const Icons& icons, const oro::Timestamp& next_update) { //example: //{ // "item_id":501, @@ -333,6 +401,7 @@ void OriginsDB::update (const Icons& icons) { std::unique_lock lock(m_db_mutex); auto& db = *m_db; + update_last_access(db, DBOperation::Icons, next_update); db.exec(std::string(g_create_icons_a).append("icons_staging") + g_create_icons_b); db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b); @@ -359,7 +428,7 @@ void OriginsDB::update (const Icons& icons) { transaction.commit(); } -void OriginsDB::update (const Shops& shops) { +void OriginsDB::update (const Shops& shops, const oro::Timestamp& next_update) { //example: //{ // "title":"• B\u003ePoison Bottle •", @@ -376,6 +445,7 @@ void OriginsDB::update (const Shops& shops) { auto& db = *m_db; std::unique_lock lock(m_db_mutex); + update_last_access(db, DBOperation::Shops, next_update); db.exec(g_create_shops); db.exec("CREATE UNIQUE INDEX IF NOT EXISTS shops_owner_idx ON shops(owner, creation_date)"); @@ -436,9 +506,10 @@ void OriginsDB::update (const Shops& shops) { transaction.commit(); } -void OriginsDB::update (const Creators& creat) { +void OriginsDB::update (const Creators& creat, const oro::Timestamp& next_update) { std::unique_lock lock(m_db_mutex); auto& db = *m_db; + update_last_access(db, DBOperation::Creators, next_update); db.exec(g_create_fame_list); @@ -458,4 +529,20 @@ void OriginsDB::update (const Creators& creat) { transaction.commit(); } +Timestamp OriginsDB::next_access_time (DBOperation op) const { + constexpr auto select_last_acces_str = make_select_last_acces_str(); + SQLite::Statement sel_last( + *m_db, + std::string(select_last_acces_str.data()).append( + dhandy::int_to_ary(static_cast(op)).to_string_view() + ) + ); + + auto info = exec_first_res(sel_last); + if (not info) + throw std::runtime_error("Query returned no value for access.operation " + to_string(static_cast(op))); + + return from_sqlite_timestamp(*info); +} + } //namespace oro diff --git a/src/oro/originsdb.hpp b/src/oro/originsdb.hpp index 14d46b7..0fa765f 100644 --- a/src/oro/originsdb.hpp +++ b/src/oro/originsdb.hpp @@ -17,6 +17,8 @@ #pragma once +#include "datatypes.hpp" +#include "oro/dboperation.hpp" #include #include #include @@ -38,10 +40,12 @@ public: explicit OriginsDB(std::string_view path); ~OriginsDB() noexcept; - void update (const Items& items); - void update (const Icons& icons); - void update (const Shops& shops); - void update (const Creators& creat); + void update (const Items& items, const oro::Timestamp& next_update); + void update (const Icons& icons, const oro::Timestamp& next_update); + void update (const Shops& shops, const oro::Timestamp& next_update); + void update (const Creators& creat, const oro::Timestamp& next_update); + + Timestamp next_access_time (DBOperation op) const; private: std::string m_path; diff --git a/src/oro/private/sqlite_helpers.hpp b/src/oro/private/sqlite_helpers.hpp index a1e020d..967eb6c 100644 --- a/src/oro/private/sqlite_helpers.hpp +++ b/src/oro/private/sqlite_helpers.hpp @@ -24,6 +24,13 @@ namespace oro { namespace detail { + template struct RowRetInfo; + template class Ret> + struct RowRetInfo> { + typedef Ret type; + constexpr static const std::size_t count = sizeof...(R); + }; + template struct NoCopy { explicit NoCopy (T& v) : @@ -125,4 +132,17 @@ inline std::optional exec_first_res (SQLite::Statement& st, Args&&... args) { st.reset(); return retval; } + +template +inline std::optional exec_first_row (SQLite::Statement& st, Args&&... args) { + detail::bind_all(st, std::make_integer_sequence{}, std::forward(args)...); + + std::optional retval; + if (st.executeStep()) { + retval = st.getColumns::count>(); + } + + st.reset(); + return retval; +} } //namespace oro diff --git a/src/timer_base.cpp b/src/timer_base.cpp index 1a01f4d..39b8e09 100644 --- a/src/timer_base.cpp +++ b/src/timer_base.cpp @@ -21,17 +21,48 @@ #include "oro/originsdb.hpp" #include #include +#include namespace duck { +namespace { + constexpr const unsigned long g_extra_wait = 30; + + [[gnu::pure]] + unsigned long time_interval (const oro::Header& header) { + return header.retry_after / header.rate_limit; + } + + oro::Timestamp calc_next_update (const oro::Header& header) { + oro::Timestamp ret; + ret.ts = + std::chrono::time_point_cast(std::chrono::system_clock::now()) + + std::chrono::seconds(time_interval(header)); + return ret; + } + + double initial_timer (oro::OriginsDB& db, oro::DBOperation op, double min_wait) { + using std::chrono::duration_cast; + using std::chrono::duration; + using std::chrono::seconds; + + auto now = std::chrono::system_clock::now(); + auto next_time = db.next_access_time(op); + duration diff = duration_cast(next_time.ts - now); + const auto retval = std::max(min_wait, diff.count()); + return retval; + } +} //unnamed namespace + TimerBase::TimerBase ( const eve::Context& ctx, - double timeout, + oro::DBOperation type, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db ) : - eve::Timer(timeout, ctx), + eve::Timer(initial_timer(*db, type, min_wait), ctx), m_pool(pool), m_oro_api(oro_api), m_db(db) @@ -45,7 +76,7 @@ void TimerBase::on_timer() { } void TimerBase::set_next_timer (const oro::Header& header) { - const int next_timer = header.retry_after / header.rate_limit; + const unsigned long next_timer = time_interval(header) + g_extra_wait; std::cout << "Next timer in " << next_timer << " secs\n"; this->set_timer(static_cast(next_timer)); } @@ -65,20 +96,20 @@ oro::OriginsDB& TimerBase::db() { return *m_db; } -void TimerBase::update_db (const oro::Shops& shops) { - db().update(shops); +void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) { + db().update(shops, calc_next_update(header)); } -void TimerBase::update_db (const oro::Items& items) { - db().update(items); +void TimerBase::update_db (const oro::Items& items, const oro::Header& header) { + db().update(items, calc_next_update(header)); } -void TimerBase::update_db (const oro::Icons& icons) { - db().update(icons); +void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) { + db().update(icons, calc_next_update(header)); } -void TimerBase::update_db (const oro::Creators& creators) { - db().update(creators); +void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) { + db().update(creators, calc_next_update(header)); } } //namespace duck diff --git a/src/timer_base.hpp b/src/timer_base.hpp index ff67235..b861c72 100644 --- a/src/timer_base.hpp +++ b/src/timer_base.hpp @@ -19,6 +19,7 @@ #include "eventia/private/context.hpp" #include "eventia/timer.hpp" +#include "oro/dboperation.hpp" namespace roar11 { class ThreadPool; @@ -40,7 +41,8 @@ class TimerBase : public eve::Timer { public: TimerBase ( const eve::Context& ctx, - double timeout, + oro::DBOperation type, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db @@ -51,10 +53,10 @@ public: protected: void set_next_timer (const oro::Header& header); - void update_db (const oro::Shops& shops); - void update_db (const oro::Items& items); - void update_db (const oro::Icons& icons); - void update_db (const oro::Creators& creators); + void update_db (const oro::Shops& shops, const oro::Header& header); + void update_db (const oro::Items& items, const oro::Header& header); + void update_db (const oro::Icons& icons, const oro::Header& header); + void update_db (const oro::Creators& creators, const oro::Header& header); roar11::ThreadPool& pool(); oro::Api& oro_api(); oro::OriginsDB& db(); diff --git a/src/timer_oro_api.cpp b/src/timer_oro_api.cpp new file mode 100644 index 0000000..519cdce --- /dev/null +++ b/src/timer_oro_api.cpp @@ -0,0 +1,21 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#include "timer_oro_api.hpp" + +namespace duck { +} //namespace duck diff --git a/src/timer_oro_api.hpp b/src/timer_oro_api.hpp index 5c7da11..a8d1391 100644 --- a/src/timer_oro_api.hpp +++ b/src/timer_oro_api.hpp @@ -18,6 +18,8 @@ #pragma once #include "timer_base.hpp" +#include "oro/dboperation.hpp" +#include "oro/api.hpp" #include namespace oro { @@ -26,12 +28,12 @@ namespace oro { namespace duck { -template +template class TimerOroApi : TimerBase { public: TimerOroApi ( const eve::Context& ctx, - double timeout, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db @@ -41,24 +43,48 @@ private: virtual void fetch_data() override; }; -template -inline TimerOroApi::TimerOroApi ( +namespace detail { + template auto invoke_api_func (oro::Api& api); + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.items_icons(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.items_list(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.fame_list(); + } + + template <> + inline auto invoke_api_func (oro::Api& api) { + return api.market_list(); + } +} //namespace detail + +template +inline TimerOroApi::TimerOroApi ( const eve::Context& ctx, - double timeout, + double min_wait, roar11::ThreadPool* pool, oro::Api* oro_api, oro::OriginsDB* db ) : - TimerBase(ctx, timeout, pool, oro_api, db) + TimerBase(ctx, Op, min_wait, pool, oro_api, db) { } -template -inline void TimerOroApi::fetch_data() { +template +inline void TimerOroApi::fetch_data() { try { - auto results = (oro_api().*Method)(); + auto results = detail::invoke_api_func(oro_api()); set_next_timer(results.first); - this->update_db(results.second); + this->update_db(results.second, results.first); } catch (...) { this->set_exception(std::current_exception()); diff --git a/subprojects/magic_enum b/subprojects/magic_enum new file mode 160000 index 0000000..d73a985 --- /dev/null +++ b/subprojects/magic_enum @@ -0,0 +1 @@ +Subproject commit d73a985f520aa25a432d1bbf5525663b93c189a0