Restore timeouts from previous run

This is quite a big update, the new feature is that when the
program starts it loads the next update time from the db if
present. If so it resumes the timers from there. Maximum wait
time is currently capped at 24h.

TimerOroApi is not templated on methods anymore, rather on
the new DBOperation enum. This is so that it has that enum
value to pass it to other functions. This could've been a
separate commit, but wth...

Magic enum allows me to iterate over an enum's values at
build time, it's useful for building the query in OriginsDB
for the SELECT in the access table
(see make_select_last_access_str()).

.gitmodules

@@ -13,3 +13,6 @@
 [submodule "subprojects/duckhandy"]
 	path = subprojects/duckhandy
 	url = http://alarmpi.no-ip.org/gitan/King_DuckZ/duckhandy.git
+[submodule "subprojects/magic_enum"]
+	path = subprojects/magic_enum
+	url = https://github.com/Neargye/magic_enum.git

meson.build

@@ -7,5 +7,6 @@ project('orotool', 'cpp', 'c',
 
 duckhandy_inc = include_directories('subprojects/duckhandy/include')
 date_inc = include_directories('subprojects/date/include')
+magic_enum_inc = include_directories('subprojects/magic_enum/include')
 
 subdir('src')

src/eventia/timer.cpp

@@ -85,6 +85,9 @@ void Timer::set_timer (double delay) {
 	m_local->timer.stop();
 	ev_now_update(*m_local->context.loop);
 	m_local->timer.start(m_local->timer_value, 0.0);
+#if !defined(NDEBUG)
+	std::cout << "Timer::set_timer(" << delay << ")\n";
+#endif
 	m_local->context.async->send();
 }
 

src/evloop.cpp

@@ -20,7 +20,7 @@
 #include "eventia/eventia.hpp"
 #include "eventia/signal.hpp"
 #include "roar11/ThreadPool.hpp"
-#include "oro/api.hpp"
+#include "oro/dboperation.hpp"
 #include <iostream>
 
 namespace duck {
@@ -54,26 +54,26 @@ namespace {
 	private:
 		eve::Eventia* m_eventia;
 	};
-
 } //unnamed namespace
 
 void test(oro::Api* api, oro::OriginsDB* db, std::size_t thread_count) {
-	typedef TimerOroApi<&oro::Api::market_list> TimerShops;
-	typedef TimerOroApi<&oro::Api::items_list> TimerItems;
-	typedef TimerOroApi<&oro::Api::items_icons> TimerIcons;
-	typedef TimerOroApi<&oro::Api::fame_list> TimerCreators;
+	typedef TimerOroApi<oro::DBOperation::Shops> TimerShops;
+	typedef TimerOroApi<oro::DBOperation::Items> TimerItems;
+	typedef TimerOroApi<oro::DBOperation::Icons> TimerIcons;
+	typedef TimerOroApi<oro::DBOperation::Creators> TimerCreators;
 
 	std::cout << "Running with " << thread_count << " worker threads\n";
 	roar11::ThreadPool pool(thread_count);
 	eve::Eventia worker;
 	pool.submit(worker.event_functor());
 
+
 	auto sig_int = worker.make_event<SignalInt>(&worker);
 
-	auto timer_items = worker.make_event<TimerItems>(0.5, &pool, api, db);
-	auto timer_icons = worker.make_event<TimerIcons>(1.0, &pool, api, db);
-	auto timer_shops = worker.make_event<TimerShops>(1.5, &pool, api, db);
-	auto timer_creat = worker.make_event<TimerCreators>(2.0, &pool, api, db);
+	auto timer_items = worker.make_event<TimerItems>(0.0, &pool, api, db);
+	auto timer_icons = worker.make_event<TimerIcons>(5.0, &pool, api, db);
+	auto timer_shops = worker.make_event<TimerShops>(10.0, &pool, api, db);
+	auto timer_creat = worker.make_event<TimerCreators>(15.0, &pool, api, db);
 
 	worker.wait();
 #if !defined(NDEBUG)

src/meson.build

@@ -61,9 +61,14 @@ executable(meson.project_name(),
   'oro/private/tiger.cpp',
   'eventia/signal.cpp',
   'eventia/event.cpp',
+  'timer_oro_api.cpp',
   project_config_file,
   install: true,
   dependencies: lib_deps,
-  include_directories: [date_inc, duckhandy_inc],
+  include_directories: [
+	date_inc,
+	duckhandy_inc,
+	magic_enum_inc,
+  ],
   cpp_args: ['-DEV_USE_STDEXCEPT'],
 )

src/oro/dboperation.hpp

@@ -0,0 +1,26 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+namespace oro {
+
+enum class DBOperation {
+	Creators, Shops, Items, Icons
+};
+
+} //namespace oro

src/oro/originsdb.cpp

@@ -28,6 +28,8 @@
 #include "oro/private/tiger.hpp"
 #include "oro/private/sqlite_helpers.hpp"
 #include "duckhandy/int_conv.hpp"
+#include "duckhandy/string_bt.hpp"
+#include "magic_enum.hpp"
 #include <mutex>
 #include <optional>
 #include <type_traits>
@@ -112,6 +114,17 @@ namespace {
 			", icon TEXT"
 			", FOREIGN KEY(item_id) REFERENCES items(id)"
 		")";
+	constexpr const char g_create_access[] =
+		"CREATE TABLE IF NOT EXISTS access("
+			"last TEXT NOT NULL DEFAULT '1970-01-01 00:00:00'"
+			", operation TINYINT PRIMARY KEY NOT NULL"
+			", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'"
+		")";
+
+	std::string to_string (int num) {
+		auto ary = dhandy::int_to_ary<int>(num);
+		return std::string(ary.begin(), ary.begin() + ary.size() - 1);
+	}
 
 	class ItemIdToTableId {
 	public:
@@ -132,7 +145,7 @@ namespace {
 			m_query.bind(1, item_id);
 			if (not m_query.executeStep()) {
 				m_query.reset();
-				throw std::runtime_error("No item_id " + std::to_string(item_id) + " found in table items");
+				throw std::runtime_error("No item_id " + to_string(item_id) + " found in table items");
 			}
 
 			const int64_t table_id = m_query.getColumn(0);
@@ -256,19 +269,74 @@ namespace {
 		}
 		return  mchlib::tiger_to_string(th);
 	}
+
+	void update_last_access (SQLite::Database& db, DBOperation op, const oro::Timestamp& next_update) {
+		db.exec(
+			std::string("INSERT INTO access (operation, last, next_update) VALUES(").append(
+				dhandy::int_to_ary<int>(static_cast<int>(op)).to_string_view()
+			) +
+			", CURRENT_TIMESTAMP, " +
+			"'" + to_sqlite_string(next_update) + "'" +
+			") ON CONFLICT(operation) DO UPDATE "
+			"SET last=CURRENT_TIMESTAMP"
+			", next_update='" + to_sqlite_string(next_update) + "'"
+		);
+	}
+
+	template <oro::DBOperation... Ops, std::size_t... N>
+	constexpr auto make_sel_last_acces_b(std::index_sequence<N...>) {
+		return (
+			(
+				dhandy::bt::string<(N==0 ? 0 : 3)+1, char>(N == 0 ? "" : "),(") +
+				dhandy::bt::string<
+					dhandy::int_to_ary(static_cast<int>(Ops)).size(),
+					char
+				>(dhandy::int_to_ary(static_cast<int>(Ops)).data())
+			) + ... +
+			dhandy::bt::make_string("")
+		);
+	}
+
+	template <std::size_t... Indices>
+	constexpr auto make_sel_last_acces_a (std::index_sequence<Indices...> seq) {
+		return make_sel_last_acces_b<magic_enum::enum_values<oro::DBOperation>()[Indices]...>(seq);
+	}
+
+	constexpr auto make_select_last_acces_str() {
+		using dhandy::bt::make_string;
+		using std::make_index_sequence;
+		using magic_enum::enum_count;
+		using oro::DBOperation;
+		constexpr auto prefix = make_string("WITH list(operation) AS (VALUES(");
+		constexpr auto suffix = make_string("))"
+			" SELECT IFNULL(next_update, '1970-01-01 00:00:00') AS next_update"
+			" FROM list"
+			" LEFT JOIN access USING(operation)"
+			" WHERE operation="
+		);
+		constexpr auto values =
+			make_sel_last_acces_a(make_index_sequence<enum_count<DBOperation>()>{});
+
+		return (prefix + values + suffix);
+	}
 } //unnamed namespace
 
 OriginsDB::OriginsDB (std::string_view path) :
 	m_path(path.data(), path.size()),
 	m_db(std::make_unique<SQLite::Database>(m_path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE))
 {
+	{
+		std::unique_lock<std::mutex> lock(m_db_mutex);
+		m_db->exec(g_create_access);
+	}
 }
 
 OriginsDB::~OriginsDB() noexcept = default;
 
-void OriginsDB::update (const Items& items) {
+void OriginsDB::update (const Items& items, const oro::Timestamp& next_update) {
 	auto& db = *m_db;
 	std::unique_lock<std::mutex> lock(m_db_mutex);
+	update_last_access(db, DBOperation::Items, next_update);
 
 	//example:
 	//{
@@ -324,7 +392,7 @@ void OriginsDB::update (const Items& items) {
 	transaction.commit();
 }
 
-void OriginsDB::update (const Icons& icons) {
+void OriginsDB::update (const Icons& icons, const oro::Timestamp& next_update) {
 	//example:
 	//{
 	//	"item_id":501,
@@ -333,6 +401,7 @@ void OriginsDB::update (const Icons& icons) {
 
 	std::unique_lock<std::mutex> lock(m_db_mutex);
 	auto& db = *m_db;
+	update_last_access(db, DBOperation::Icons, next_update);
 
 	db.exec(std::string(g_create_icons_a).append("icons_staging") + g_create_icons_b);
 	db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b);
@@ -359,7 +428,7 @@ void OriginsDB::update (const Icons& icons) {
 	transaction.commit();
 }
 
-void OriginsDB::update (const Shops& shops) {
+void OriginsDB::update (const Shops& shops, const oro::Timestamp& next_update) {
 	//example:
 	//{
 	//	"title":"• B\u003ePoison Bottle •",
@@ -376,6 +445,7 @@ void OriginsDB::update (const Shops& shops) {
 
 	auto& db = *m_db;
 	std::unique_lock<std::mutex> lock(m_db_mutex);
+	update_last_access(db, DBOperation::Shops, next_update);
 
 	db.exec(g_create_shops);
 	db.exec("CREATE UNIQUE INDEX IF NOT EXISTS shops_owner_idx ON shops(owner, creation_date)");
@@ -436,9 +506,10 @@ void OriginsDB::update (const Shops& shops) {
 	transaction.commit();
 }
 
-void OriginsDB::update (const Creators& creat) {
+void OriginsDB::update (const Creators& creat, const oro::Timestamp& next_update) {
 	std::unique_lock<std::mutex> lock(m_db_mutex);
 	auto& db = *m_db;
+	update_last_access(db, DBOperation::Creators, next_update);
 
 	db.exec(g_create_fame_list);
 
@@ -458,4 +529,20 @@ void OriginsDB::update (const Creators& creat) {
 	transaction.commit();
 }
 
+Timestamp OriginsDB::next_access_time (DBOperation op) const {
+	constexpr auto select_last_acces_str = make_select_last_acces_str();
+	SQLite::Statement sel_last(
+		*m_db,
+		std::string(select_last_acces_str.data()).append(
+			dhandy::int_to_ary<int>(static_cast<int>(op)).to_string_view()
+		)
+	);
+
+	auto info = exec_first_res<std::string>(sel_last);
+	if (not info)
+		throw std::runtime_error("Query returned no value for access.operation " + to_string(static_cast<int>(op)));
+
+	return from_sqlite_timestamp(*info);
+}
+
 } //namespace oro

src/oro/originsdb.hpp

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "datatypes.hpp"
+#include "oro/dboperation.hpp"
 #include <mutex>
 #include <string_view>
 #include <string>
@@ -38,10 +40,12 @@ public:
 	explicit OriginsDB(std::string_view path);
 	~OriginsDB() noexcept;
 
-	void update (const Items& items);
-	void update (const Icons& icons);
-	void update (const Shops& shops);
-	void update (const Creators& creat);
+	void update (const Items& items, const oro::Timestamp& next_update);
+	void update (const Icons& icons, const oro::Timestamp& next_update);
+	void update (const Shops& shops, const oro::Timestamp& next_update);
+	void update (const Creators& creat, const oro::Timestamp& next_update);
+
+	Timestamp next_access_time (DBOperation op) const;
 
 private:
 	std::string m_path;

src/oro/private/sqlite_helpers.hpp

@@ -24,6 +24,13 @@
 
 namespace oro {
 namespace detail {
+	template <typename T> struct RowRetInfo;
+	template <typename... R, template <typename...> class Ret>
+	struct RowRetInfo<Ret<R...>> {
+		typedef Ret<R...> type;
+		constexpr static const std::size_t count = sizeof...(R);
+	};
+
 	template <typename T>
 	struct NoCopy {
 		explicit NoCopy (T& v) :
@@ -125,4 +132,17 @@ inline std::optional<T> exec_first_res (SQLite::Statement& st, Args&&... args) {
 	st.reset();
 	return retval;
 }
+
+template <typename Ret, typename... Args>
+inline std::optional<Ret> exec_first_row (SQLite::Statement& st, Args&&... args) {
+	detail::bind_all(st, std::make_integer_sequence<int, sizeof...(Args)>{}, std::forward<Args>(args)...);
+
+	std::optional<Ret> retval;
+	if (st.executeStep()) {
+		retval = st.getColumns<Ret, detail::RowRetInfo<Ret>::count>();
+	}
+
+	st.reset();
+	return retval;
+}
 } //namespace oro

src/timer_base.cpp

@@ -21,17 +21,48 @@
 #include "oro/originsdb.hpp"
 #include <cassert>
 #include <iostream>
+#include <chrono>
 
 namespace duck {
 
+namespace {
+	constexpr const unsigned long g_extra_wait = 30;
+
+	[[gnu::pure]]
+	unsigned long time_interval (const oro::Header& header) {
+		return header.retry_after / header.rate_limit;
+	}
+
+	oro::Timestamp calc_next_update (const oro::Header& header) {
+		oro::Timestamp ret;
+		ret.ts =
+			std::chrono::time_point_cast<oro::timestamp_t::duration>(std::chrono::system_clock::now()) +
+			std::chrono::seconds(time_interval(header));
+		return ret;
+	}
+
+	double initial_timer (oro::OriginsDB& db, oro::DBOperation op, double min_wait) {
+		using std::chrono::duration_cast;
+		using std::chrono::duration;
+		using std::chrono::seconds;
+
+		auto now = std::chrono::system_clock::now();
+		auto next_time = db.next_access_time(op);
+		duration<double> diff = duration_cast<seconds>(next_time.ts - now);
+		const auto retval = std::max(min_wait, diff.count());
+		return retval;
+	}
+} //unnamed namespace
+
 TimerBase::TimerBase (
 	const eve::Context& ctx,
-	double timeout,
+	oro::DBOperation type,
+	double min_wait,
 	roar11::ThreadPool* pool,
 	oro::Api* oro_api,
 	oro::OriginsDB* db
 ) :
-	eve::Timer(timeout, ctx),
+	eve::Timer(initial_timer(*db, type, min_wait), ctx),
 	m_pool(pool),
 	m_oro_api(oro_api),
 	m_db(db)
@@ -45,7 +76,7 @@ void TimerBase::on_timer() {
 }
 
 void TimerBase::set_next_timer (const oro::Header& header) {
-	const int next_timer = header.retry_after / header.rate_limit;
+	const unsigned long next_timer = time_interval(header) + g_extra_wait;
 	std::cout << "Next timer in " << next_timer << " secs\n";
 	this->set_timer(static_cast<double>(next_timer));
 }
@@ -65,20 +96,20 @@ oro::OriginsDB& TimerBase::db() {
 	return *m_db;
 }
 
-void TimerBase::update_db (const oro::Shops& shops) {
-	db().update(shops);
+void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) {
+	db().update(shops, calc_next_update(header));
 }
 
-void TimerBase::update_db (const oro::Items& items) {
-	db().update(items);
+void TimerBase::update_db (const oro::Items& items, const oro::Header& header) {
+	db().update(items, calc_next_update(header));
 }
 
-void TimerBase::update_db (const oro::Icons& icons) {
-	db().update(icons);
+void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) {
+	db().update(icons, calc_next_update(header));
 }
 
-void TimerBase::update_db (const oro::Creators& creators) {
-	db().update(creators);
+void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) {
+	db().update(creators, calc_next_update(header));
 }
 
 } //namespace duck

src/timer_base.hpp

@@ -19,6 +19,7 @@
 
 #include "eventia/private/context.hpp"
 #include "eventia/timer.hpp"
+#include "oro/dboperation.hpp"
 
 namespace roar11 {
 	class ThreadPool;
@@ -40,7 +41,8 @@ class TimerBase : public eve::Timer {
 public:
 	TimerBase (
 		const eve::Context& ctx,
-		double timeout,
+		oro::DBOperation type,
+		double min_wait,
 		roar11::ThreadPool* pool,
 		oro::Api* oro_api,
 		oro::OriginsDB* db
@@ -51,10 +53,10 @@ public:
 
 protected:
 	void set_next_timer (const oro::Header& header);
-	void update_db (const oro::Shops& shops);
-	void update_db (const oro::Items& items);
-	void update_db (const oro::Icons& icons);
-	void update_db (const oro::Creators& creators);
+	void update_db (const oro::Shops& shops, const oro::Header& header);
+	void update_db (const oro::Items& items, const oro::Header& header);
+	void update_db (const oro::Icons& icons, const oro::Header& header);
+	void update_db (const oro::Creators& creators, const oro::Header& header);
 	roar11::ThreadPool& pool();
 	oro::Api& oro_api();
 	oro::OriginsDB& db();

src/timer_oro_api.cpp

@@ -0,0 +1,21 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "timer_oro_api.hpp"
+
+namespace duck {
+} //namespace duck

src/timer_oro_api.hpp

@@ -18,6 +18,8 @@
 #pragma once
 
 #include "timer_base.hpp"
+#include "oro/dboperation.hpp"
+#include "oro/api.hpp"
 #include <exception>
 
 namespace oro {
@@ -26,12 +28,12 @@ namespace oro {
 
 namespace duck {
 
-template<auto Method>
+template<oro::DBOperation Op>
 class TimerOroApi : TimerBase {
 public:
 	TimerOroApi (
 		const eve::Context& ctx,
-		double timeout,
+		double min_wait,
 		roar11::ThreadPool* pool,
 		oro::Api* oro_api,
 		oro::OriginsDB* db
@@ -41,24 +43,48 @@ private:
 	virtual void fetch_data() override;
 };
 
-template<auto Method>
-inline TimerOroApi<Method>::TimerOroApi (
+namespace detail {
+	template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api);
+
+	template <>
+	inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api) {
+		return api.items_icons();
+	}
+
+	template <>
+	inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api) {
+		return api.items_list();
+	}
+
+	template <>
+	inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api) {
+		return api.fame_list();
+	}
+
+	template <>
+	inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api) {
+		return api.market_list();
+	}
+} //namespace detail
+
+template<oro::DBOperation Op>
+inline TimerOroApi<Op>::TimerOroApi (
 	const eve::Context& ctx,
-	double timeout,
+	double min_wait,
 	roar11::ThreadPool* pool,
 	oro::Api* oro_api,
 	oro::OriginsDB* db
 ) :
-	TimerBase(ctx, timeout, pool, oro_api, db)
+	TimerBase(ctx, Op, min_wait, pool, oro_api, db)
 {
 }
 
-template<auto Method>
-inline void TimerOroApi<Method>::fetch_data() {
+template<oro::DBOperation Op>
+inline void TimerOroApi<Op>::fetch_data() {
 	try {
-		auto results = (oro_api().*Method)();
+		auto results = detail::invoke_api_func<Op>(oro_api());
 		set_next_timer(results.first);
-		this->update_db(results.second);
+		this->update_db(results.second, results.first);
 	}
 	catch (...) {
 		this->set_exception(std::current_exception());

subprojects/magic_enum

@@ -0,0 +1 @@
+Subproject commit d73a985f520aa25a432d1bbf5525663b93c189a0