Add store_raw_json option to config

orotool.conf

@@ -6,3 +6,4 @@ backend=sqlite
 [options]
 fetch_extra_delay=30
 api_key=my_key_here
+store_raw_json=true

src/app_config.cpp

@@ -24,6 +24,8 @@
 #include <algorithm>
 #include <stdexcept>
 #include <iostream>
+#include <cctype>
+#include <array>
 
 namespace duck {
 
@@ -45,6 +47,32 @@ namespace {
 	constexpr const char g_api_key[] = "api_key";
 	constexpr const char g_api_key_def[] = "";
 
+	constexpr const char g_store_raw_json_sect[] = "options";
+	constexpr const char g_store_raw_json[] = "store_raw_json";
+	constexpr const char g_store_raw_json_def[] = "false";
+
+	bool equal (std::string_view a, std::string_view b) {
+		return a.size() == b.size() and std::equal(
+			a.begin(), a.end(),
+			b.begin(),
+			[](auto c1, auto c2) {return std::tolower(c1) == std::tolower(c2); }
+		);
+	}
+
+	bool to_bool (std::string_view a) {
+		std::array<std::string_view, 10> yes_list {
+			"true", "1", "yes", "on", "ok", "enable", "enabled", "sure", "aye", "one"
+		};
+
+		return (
+			std::find_if(
+				yes_list.begin(),
+				yes_list.end(),
+				[a](std::string_view b) -> bool { return equal(a, b); }
+			) != yes_list.end()
+		);
+	}
+
 	std::string whole_ini() {
 		std::ifstream input(g_config_file_path);
 		input >> std::noskipws;
@@ -132,4 +160,10 @@ std::string_view AppConfig::backend() const {
 	return value_ifp(m_ini, g_backend_sect, g_backend, g_def_backend_name, false);
 }
 
+bool AppConfig::store_raw_json() const {
+	std::string_view val = value_ifp(m_ini, g_store_raw_json_sect, g_store_raw_json, g_store_raw_json_def, false);
+
+	return to_bool(val);
+}
+
 } //namespace duck

src/app_config.hpp

@@ -33,6 +33,7 @@ public:
 	std::size_t worker_threads() const;
 	std::size_t fetch_extra_delay() const;
 	std::string_view backend() const;
+	bool store_raw_json() const;
 
 private:
 	kamokan::IniFile m_ini;

src/evloop.cpp

@@ -56,7 +56,7 @@ namespace {
 	};
 } //unnamed namespace
 
-void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count) {
+void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json) {
 	typedef TimerOroApi<oro::DBOperation::Shops> TimerShops;
 	typedef TimerOroApi<oro::DBOperation::Items> TimerItems;
 	typedef TimerOroApi<oro::DBOperation::Icons> TimerIcons;
@@ -71,11 +71,12 @@ void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_
 	const double ed = static_cast<double>(extra_delay);
 
 	auto sig_int = worker.make_event<SignalInt>(&worker);
+	const bool& rj = store_raw_json;
 
-	auto timer_items = worker.make_event<TimerItems>(TSet{0.0, ed}, &pool, api, db);
+	auto timer_items = worker.make_event<TimerItems>(TSet{0.0, ed, rj}, &pool, api, db);
-	auto timer_icons = worker.make_event<TimerIcons>(TSet{5.0, ed}, &pool, api, db);
+	auto timer_icons = worker.make_event<TimerIcons>(TSet{5.0, ed, rj}, &pool, api, db);
-	auto timer_shops = worker.make_event<TimerShops>(TSet{10.0, ed}, &pool, api, db);
+	auto timer_shops = worker.make_event<TimerShops>(TSet{10.0, ed, rj}, &pool, api, db);
-	auto timer_creat = worker.make_event<TimerCreators>(TSet{15.0, ed}, &pool, api, db);
+	auto timer_creat = worker.make_event<TimerCreators>(TSet{15.0, ed, rj}, &pool, api, db);
 
 	worker.wait();
 #if !defined(NDEBUG)

src/evloop.hpp

@@ -26,6 +26,6 @@ namespace oro {
 
 namespace duck {
 
-void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count);
+void test(oro::Api* api, oro::OriginsDB* db, std::size_t extra_delay, std::size_t thread_count, bool store_raw_json);
 
 } //namespace duck

src/main.cpp

@@ -29,18 +29,18 @@
 
 namespace {
 	void print_ping(oro::Api& oro_api) {
-		auto ping = oro_api.ping();
+		auto ping = oro_api.ping(false);
 
-		std::cout << "date: " << ping.first.date << '\n';
+		std::cout << "date: " << ping.header.date << '\n';
-		std::cout << "rate limit: " << ping.first.rate_limit << '\n';
+		std::cout << "rate limit: " << ping.header.rate_limit << '\n';
-		std::cout << "remaining: " << ping.first.rate_limit_remaining << '\n';
+		std::cout << "remaining: " << ping.header.rate_limit_remaining << '\n';
-		std::cout << "reset: " << ping.first.rate_limit_reset << '\n';
+		std::cout << "reset: " << ping.header.rate_limit_reset << '\n';
-		std::cout << "retry after: " << ping.first.retry_after << '\n';
+		std::cout << "retry after: " << ping.header.retry_after << '\n';
-		std::cout << "server: " << ping.first.server << '\n';
+		std::cout << "server: " << ping.header.server << '\n';
 		std::cout << "-----\n";
-		std::cout << "timestamp: " << ping.second.generation_timestamp << '\n';
+		std::cout << "timestamp: " << ping.data.generation_timestamp << '\n';
-		std::cout << "answer: " << ping.second.message << '\n';
+		std::cout << "answer: " << ping.data.message << '\n';
-		std::cout << "version: " << ping.second.version << '\n';
+		std::cout << "version: " << ping.data.version << '\n';
 	}
 
 	constexpr auto app_version() {
@@ -82,7 +82,8 @@ int main(int argc, char* argv[]) {
 			oro_api.get(),
 			db.get(),
 			app_conf.fetch_extra_delay(),
-			app_conf.worker_threads()
+			app_conf.worker_threads(),
+			app_conf.store_raw_json()
 		);
 	}
 #if defined(OROTOOL_WITH_RESTCCPP)

src/nap/http_response.hpp

@@ -27,6 +27,7 @@ namespace nap {
 struct HttpResponse {
 	std::unique_ptr<char[]> raw;
 	std::vector<std::pair<std::string_view, std::string_view>> header_list;
+	std::string_view header;
 	std::string_view body;
 	std::string_view http_ver;
 	std::string_view code_desc;

src/nap/page_fetch.cpp

@@ -101,15 +101,14 @@ HttpResponse page_fetch (
 	HttpResponse resp;
 	resp.code = easy.get_info<CURLINFO_RESPONSE_CODE>().get();
 
-	std::string_view head;
 	{
 		auto [raw, head_tmp, body] = make_raw_string(header_oss, body_oss, body_padding);
-		head = head_tmp;
 		resp.raw = std::move(raw);
+		resp.header = head_tmp;
 		resp.body = body;
 	}
 
-	auto parsed_header = header_parse(head);
+	auto parsed_header = header_parse(resp.header);
 	resp.header_list = std::move(parsed_header.fields);
 	assert(resp.code == parsed_header.code);
 	resp.http_ver = parsed_header.version;

src/oro/api.hpp

@@ -61,6 +61,13 @@ struct Header {
 
 };
 
+template <typename T>
+struct ApiOutput {
+	Header header;
+	T data;
+	std::string raw_response;
+};
+
 class Api {
 public:
 	Api (
@@ -71,12 +78,12 @@ public:
 	);
 	virtual ~Api() noexcept;
 
-	virtual std::pair<Header, Ping> ping() = 0;
+	virtual ApiOutput<Ping> ping(bool with_raw) = 0;
-	virtual std::pair<Header, WhoAmI> who_am_i() = 0;
+	virtual ApiOutput<WhoAmI> who_am_i(bool with_raw) = 0;
-	virtual std::pair<Header, Items> items_list() = 0;
+	virtual ApiOutput<Items> items_list(bool with_raw) = 0;
-	virtual std::pair<Header, Icons> items_icons() = 0;
+	virtual ApiOutput<Icons> items_icons(bool with_raw) = 0;
-	virtual std::pair<Header, Shops> market_list() = 0;
+	virtual ApiOutput<Shops> market_list(bool with_raw) = 0;
-	virtual std::pair<Header, Creators> fame_list() = 0;
+	virtual ApiOutput<Creators> fame_list(bool with_raw) = 0;
 
 protected:
 	std::string m_prefix;

src/oro/api_nap.cpp

@@ -23,6 +23,7 @@
 #include <optional>
 #include <algorithm>
 #include <cctype>
+#include <cassert>
 
 namespace sjd = simdjson::dom;
 namespace sj = simdjson;
@@ -115,25 +116,27 @@ ApiNap::ApiNap (
 
 ApiNap::~ApiNap() noexcept = default;
 
-std::pair<Header, Ping> ApiNap::ping() {
+ApiOutput<Ping> ApiNap::ping(bool with_raw) {
 	return fetch_and_parse<Ping>(
 		g_endpoint_ping,
 		[](const simdjson::dom::element& doc, Ping& out) {
 			out.message = doc["message"];
-		}
+		},
+		with_raw
 	);
 }
 
-std::pair<Header, WhoAmI> ApiNap::who_am_i() {
+ApiOutput<WhoAmI> ApiNap::who_am_i(bool with_raw) {
 	return fetch_and_parse<WhoAmI>(
 		g_endpoint_whoami,
 		[](const simdjson::dom::element& doc, WhoAmI& out) {
 			out.master_id = static_cast<unsigned int>(doc["master_id"].get_uint64());
-		}
+		},
+		with_raw
 	);
 }
 
-std::pair<Header, Items> ApiNap::items_list() {
+ApiOutput<Items> ApiNap::items_list(bool with_raw) {
 	return fetch_and_parse<Items>(
 		g_endpoint_items_list,
 		[](const simdjson::dom::element& doc, Items& out) {
@@ -150,11 +153,12 @@ std::pair<Header, Items> ApiNap::items_list() {
 				new_entry.slots = get_optional<unsigned int, uint64_t>(item["slots"]);
 				out.items.push_back(std::move(new_entry));
 			}
-		}
+		},
+		with_raw
 	);
 }
 
-std::pair<Header, Icons> ApiNap::items_icons() {
+ApiOutput<Icons> ApiNap::items_icons(bool with_raw) {
 	return fetch_and_parse<Icons>(
 		g_endpoint_items_icons,
 		[](const simdjson::dom::element& doc, Icons& out) {
@@ -166,11 +170,12 @@ std::pair<Header, Icons> ApiNap::items_icons() {
 				new_entry.icon = icon["icon"];
 				out.icons.push_back(std::move(new_entry));
 			}
-		}
+		},
+		with_raw
 	);
 }
 
-std::pair<Header, Shops> ApiNap::market_list() {
+ApiOutput<Shops> ApiNap::market_list(bool with_raw) {
 	return fetch_and_parse<Shops>(
 		g_endpoint_market_list,
 		[](const simdjson::dom::element& doc, Shops& out) {
@@ -216,22 +221,24 @@ std::pair<Header, Shops> ApiNap::market_list() {
 				}
 				out.shops.push_back(std::move(new_shop));
 			}
-		}
+		},
+		with_raw
 	);
 }
 
-std::pair<Header, Creators> ApiNap::fame_list() {
+ApiOutput<Creators> ApiNap::fame_list(bool with_raw) {
 	return fetch_and_parse<Creators>(
 		g_endpoint_fame_list,
 		[](const simdjson::dom::element& doc, Creators& out) {
 			fill_creator_list(doc["brewers"], out.brewers);
 			fill_creator_list(doc["forgers"], out.brewers);
-		}
+		},
+		with_raw
 	);
 }
 
 template <typename T, typename F>
-std::pair<Header, T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill) {
+ApiOutput<T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw) {
 	auto resp = m_qrest.fetch(m_prefix + endpoint);
 
 	if (200 != resp.code)
@@ -249,7 +256,16 @@ std::pair<Header, T> ApiNap::fetch_and_parse (const char* endpoint, F&& data_fil
 		data_fill(doc, dataret);
 	}
 
-	return {to_header(resp), std::move(dataret)};
+	std::string raw_resp;
+	if (with_raw) {
+		assert(not resp.header.empty() and resp.header[resp.header.size() - 1] == '\n');
+		raw_resp.reserve(resp.header.size() + 1 + resp.body.size());
+		raw_resp.append(resp.header);
+		raw_resp.append("\n");
+		raw_resp.append(resp.body);
+	}
+
+	return {to_header(resp), std::move(dataret), std::move(raw_resp)};
 }
 
 } //namespace oro

src/oro/originsdb.hpp

@@ -19,7 +19,7 @@
 
 #include "datatypes.hpp"
 #include "oro/dboperation.hpp"
-#include <string_view>
+#include "oro/source.hpp"
 #include <memory>
 
 namespace oro {
@@ -34,10 +34,10 @@ public:
 	OriginsDB() = default;
 	virtual ~OriginsDB() noexcept = default;
 
-	virtual void update (const Items& items, const oro::Timestamp& next_update) = 0;
+	virtual void update (const Items& items, const oro::Timestamp& next_update, const Source& source) = 0;
-	virtual void update (const Icons& icons, const oro::Timestamp& next_update) = 0;
+	virtual void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) = 0;
-	virtual void update (const Shops& shops, const oro::Timestamp& next_update) = 0;
+	virtual void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) = 0;
-	virtual void update (const Creators& creat, const oro::Timestamp& next_update) = 0;
+	virtual void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) = 0;
 
 	virtual Timestamp next_access_time (DBOperation op) const = 0;
 

src/oro/private/api_nap.hpp

@@ -20,7 +20,6 @@
 #include "oro/api.hpp"
 #include "nap/quick_rest.hpp"
 #include <mutex>
-#include <utility>
 
 namespace oro {
 
@@ -35,16 +34,16 @@ public:
 
 	virtual ~ApiNap() noexcept;
 
-	virtual std::pair<Header, Ping> ping() override;
+	virtual ApiOutput<Ping> ping(bool with_raw) override;
-	virtual std::pair<Header, WhoAmI> who_am_i() override;
+	virtual ApiOutput<WhoAmI> who_am_i(bool with_raw) override;
-	virtual std::pair<Header, Items> items_list() override;
+	virtual ApiOutput<Items> items_list(bool with_raw) override;
-	virtual std::pair<Header, Icons> items_icons() override;
+	virtual ApiOutput<Icons> items_icons(bool with_raw) override;
-	virtual std::pair<Header, Shops> market_list() override;
+	virtual ApiOutput<Shops> market_list(bool with_raw) override;
-	virtual std::pair<Header, Creators> fame_list() override;
+	virtual ApiOutput<Creators> fame_list(bool with_raw) override;
 
 private:
 	template <typename T, typename F>
-	std::pair<Header, T> fetch_and_parse (const char* endpoint, F&& data_fill);
+	ApiOutput<T> fetch_and_parse (const char* endpoint, F&& data_fill, bool with_raw);
 
 	std::mutex m_json_mutex;
 	simdjson::dom::parser m_json;

src/oro/private/originsdb_sqlite.cpp

@@ -52,6 +52,7 @@ namespace {
 			", master_id INTEGER UNIQUE"
 			", name TEXT"
 			", points INTEGER"
+			", source_id INTEGER NOT NULL"
 		")";
 	constexpr const char g_create_slotted_cards[] =
 		"CREATE TABLE IF NOT EXISTS slotted_cards("
@@ -83,6 +84,7 @@ namespace {
 			", hash TEXT NOT NULL"
 			", discovery_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP"
 			", last_seen_date TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP"
+			", source_id INTEGER NOT NULL"
 			", FOREIGN KEY(shop_id) REFERENCES shops(id)"
 		")";
 	constexpr const char g_create_shops[] =
@@ -108,6 +110,7 @@ namespace {
 			", npc_price INTEGER"
 			", slots TINYINT"
 			", removal_date TEXT"
+			", source_id INTEGER NOT NULL"
 		")";
 	constexpr const char g_create_icons_a[] = "CREATE TABLE IF NOT EXISTS ";
 	constexpr const char g_create_icons_b[] =
@@ -115,6 +118,7 @@ namespace {
 			"id INTEGER PRIMARY KEY NOT NULL"
 			", item_id INTEGER NOT NULL"
 			", icon TEXT"
+			", source_id INTEGER NOT NULL"
 			", FOREIGN KEY(item_id) REFERENCES items(id)"
 		")";
 	constexpr const char g_create_access[] =
@@ -123,6 +127,12 @@ namespace {
 			", operation TINYINT PRIMARY KEY NOT NULL"
 			", next_update INTEGER NOT NULL DEFAULT '1970-01-01 00:00:00'"
 		")";
+	constexpr const char g_create_source_store[] =
+		"CREATE TABLE IF NOT EXISTS source_store("
+			"id INTEGER PRIMARY KEY NOT NULL"
+			", source TEXT"
+			", format TEXT"
+		")";
 
 	std::string to_string (int num) {
 		auto ary = dhandy::int_to_ary<int>(num);
@@ -322,6 +332,17 @@ namespace {
 
 		return (prefix + values + suffix);
 	}
+
+	SQLiteID insert_source_ifn (SQLite::Database& db, const Source& source) {
+		if (not source.data)
+			return 0;
+
+		db.exec(g_create_source_store);
+		SQLite::Statement query(db, "INSERT INTO source_store(source, format) VALUES(?, ?)");
+		bind_all(query, no_copy(*source.data), static_cast<int>(source.format));
+		query.exec();
+		return db.getLastInsertRowid();
+	}
 } //unnamed namespace
 
 OriginsDBSQLite::OriginsDBSQLite (std::string_view path) :
@@ -336,7 +357,7 @@ OriginsDBSQLite::OriginsDBSQLite (std::string_view path) :
 
 OriginsDBSQLite::~OriginsDBSQLite() noexcept = default;
 
-void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update) {
+void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_update, const Source& source) {
 	auto& db = *m_db;
 	std::unique_lock<std::mutex> lock(m_db_mutex);
 	update_last_access(db, DBOperation::Items, next_update);
@@ -360,14 +381,16 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd
 
 	SQLite::Statement query(db,
 		"INSERT INTO items_staging "
-			"(item_id, unique_name, name, type, subtype, npc_price, slots) "
+			"(item_id, unique_name, name, type, subtype, npc_price, slots, source_id) "
-			"VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7)"
+			"VALUES(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"
 	);
 	SQLite::Transaction transaction(db);
+	const SQLiteID source_id = insert_source_ifn(db, source);
 	for (const auto& item : items.items) {
 		bind_all(query, item.item_id, no_copy(item.unique_name),
 			no_copy(item.name), static_cast<int>(item.type.value),
-			cast<int>(item.subtype), item.npc_price, item.slots
+			cast<int>(item.subtype), item.npc_price, item.slots,
+			source_id
 		);
 
 		query.exec();
@@ -384,19 +407,19 @@ void OriginsDBSQLite::update (const Items& items, const oro::Timestamp& next_upd
 		")"
 	);
 	db.exec(
-		"INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots) "
+		"INSERT INTO items (item_id, unique_name, name, type, subtype, npc_price, slots, source_id) "
-		"SELECT item_id, unique_name, name, type, subtype, npc_price, slots FROM items_staging WHERE true "
+		"SELECT item_id, unique_name, name, type, subtype, npc_price, slots, source_id FROM items_staging WHERE true "
 		"ON CONFLICT(item_id, ifnull(removal_date, 0)) DO UPDATE SET "
 		"unique_name=excluded.unique_name, "
 		"name=excluded.name, type=excluded.type, subtype=excluded.subtype, "
-		"npc_price=excluded.npc_price, slots=excluded.slots"
+		"npc_price=excluded.npc_price, slots=excluded.slots, source_id=excluded.source_id"
 	);
 	db.exec("DROP TABLE items_staging");
 
 	transaction.commit();
 }
 
-void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update) {
+void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) {
 	//example:
 	//{
 	//	"item_id":501,
@@ -411,33 +434,34 @@ void OriginsDBSQLite::update (const Icons& icons, const oro::Timestamp& next_upd
 	db.exec(std::string(g_create_icons_a).append("icons") + g_create_icons_b);
 	db.exec("CREATE UNIQUE INDEX IF NOT EXISTS icons_item_id_idx ON icons(item_id)");
 	SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)");
-	SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon) VALUES(?, ?)");
+	SQLite::Statement query(db, "INSERT INTO icons_staging(item_id, icon, source_id) VALUES(?, ?, ?)");
 
 	ItemIdToTableId item_id_to_table_id(db, false);
 	SQLite::Transaction transaction(db);
+	const SQLiteID source_id = insert_source_ifn(db, source);
 	for (const auto& ico : icons.icons) {
 		bind_all(ins_empty_item, ico.item_id);
 		ins_empty_item.exec();
 		ins_empty_item.reset();
 
-		bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon));
+		bind_all(query, item_id_to_table_id(ico.item_id), no_copy(ico.icon), source_id);
 		query.exec();
 		query.reset();
 
 		//base64_decode(std::string_view(ico.icon).substr(std::string_view("data:image/png;base64,").size()));
 	}
 	db.exec(
-		"INSERT INTO icons (item_id, icon) "
+		"INSERT INTO icons (item_id, icon, source_id) "
-		"SELECT item_id, icon FROM icons_staging WHERE true "
+		"SELECT item_id, icon, source_id FROM icons_staging WHERE true "
 		"ON CONFLICT(item_id) DO UPDATE SET "
-		"icon=excluded.icon"
+		"icon=excluded.icon, source_id=excluded.source_id"
 	);
 	db.exec("DROP TABLE icons_staging");
 
 	transaction.commit();
 }
 
-void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update) {
+void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) {
 	//example:
 	//{
 	//	"title":"• B\u003ePoison Bottle •",
@@ -463,7 +487,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
 	db.exec(g_create_shop_items);
 	db.exec(g_create_slotted_cards);
 
-	SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash) VALUES(?, ?)");
+	SQLite::Statement ins_sshot(db, "INSERT OR IGNORE INTO shop_snapshots(shop_id, hash, source_id) VALUES(?, ?, ?)");
 	SQLite::Statement ins_empty_item(db, "INSERT OR IGNORE INTO items(item_id) VALUES(?)");
 	SQLite::Statement sel_sshot(db, "SELECT id FROM shop_snapshots WHERE shop_id = ? AND hash = ?");
 	SQLite::Statement ins_item(db, "INSERT INTO shop_items(snapshot_id, item_id, amount, price, refine, star_crumbs, element, creator, beloved) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)");
@@ -473,6 +497,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
 	InsertShopIfn insert_shop_ifn(db);
 	ItemIdToTableId item_id_to_table_id(db);
 	SQLite::Transaction transaction(db);
+	const SQLiteID source_id = insert_source_ifn(db, source);
 	for (const auto& shop : shops.shops) {
 		{
 			//insert new shop or get its id if already there
@@ -490,7 +515,7 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
 			}
 
 			//snapshot is new, insert it
-			bind_all(ins_sshot, shop_id_ins.first, no_copy(hash));
+			bind_all(ins_sshot, shop_id_ins.first, no_copy(hash), source_id);
 			ins_sshot.exec();
 			ins_sshot.reset();
 		}
@@ -520,26 +545,27 @@ void OriginsDBSQLite::update (const Shops& shops, const oro::Timestamp& next_upd
 	transaction.commit();
 }
 
-void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update) {
+void OriginsDBSQLite::update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) {
 	std::unique_lock<std::mutex> lock(m_db_mutex);
 	auto& db = *m_db;
 	update_last_access(db, DBOperation::Creators, next_update);
 
 	db.exec(g_create_fame_list);
 
-	auto insert_creators = [](SQLite::Statement& query, const std::vector<oro::Creator>& creats, int type) {
+	auto insert_creators = [](SQLite::Statement& query, const std::vector<oro::Creator>& creats, int type, SQLiteID source_id) {
 		for (const auto& creator : creats) {
-			bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points);
+			bind_all(query, type, creator.char_id, no_copy(creator.name), creator.points, source_id);
 			query.exec();
 			query.reset();
 		}
 	};
 
-	SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points) VALUES (?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points");
+	SQLite::Statement query(db, "INSERT INTO fame_list(type, master_id, name, points, source_id) VALUES (?, ?, ?, ?, ?) ON CONFLICT(master_id) DO UPDATE SET points=excluded.points");
 
 	SQLite::Transaction transaction(db);
-	insert_creators(query, creat.brewers, 1);
+	const SQLiteID source_id = insert_source_ifn(db, source);
-	insert_creators(query, creat.forgers, 2);
+	insert_creators(query, creat.brewers, 1, source_id);
+	insert_creators(query, creat.forgers, 2, source_id);
 	transaction.commit();
 }
 

src/oro/private/originsdb_sqlite.hpp

@@ -34,10 +34,10 @@ public:
 	explicit OriginsDBSQLite(std::string_view path);
 	virtual ~OriginsDBSQLite() noexcept;
 
-	void update (const Items& items, const oro::Timestamp& next_update) override;
+	void update (const Items& items, const oro::Timestamp& next_update, const Source& source) override;
-	void update (const Icons& icons, const oro::Timestamp& next_update) override;
+	void update (const Icons& icons, const oro::Timestamp& next_update, const Source& source) override;
-	void update (const Shops& shops, const oro::Timestamp& next_update) override;
+	void update (const Shops& shops, const oro::Timestamp& next_update, const Source& source) override;
-	void update (const Creators& creat, const oro::Timestamp& next_update) override;
+	void update (const Creators& creat, const oro::Timestamp& next_update, const Source& source) override;
 
 	Timestamp next_access_time (DBOperation op) const override;
 

src/oro/source.hpp

@@ -0,0 +1,38 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+#include <optional>
+
+namespace oro {
+
+//The raw json source data complete with http response header. It can be
+//anything really, but it's intended to be stored in the DB. Entries in the
+//other tables will have a reference to the source.
+
+enum class SourceFormat {
+	Plain
+};
+
+struct Source {
+	std::optional<std::string> data;
+	SourceFormat format;
+};
+
+} //namespace oro

src/timer_base.cpp

@@ -64,14 +64,15 @@ TimerBase::TimerBase (
 	m_extra_delay(settings.extra_delay),
 	m_pool(pool),
 	m_oro_api(oro_api),
-	m_db(db)
+	m_db(db),
+	m_store_raw_response(settings.store_raw_json)
 {
 	assert(m_pool);
 	assert(m_oro_api);
 }
 
 void TimerBase::on_timer() {
-	m_pool->submit(&TimerBase::fetch_data, this);
+	m_pool->submit(&TimerBase::fetch_data, this, m_store_raw_response);
 }
 
 void TimerBase::set_next_timer (const oro::Header& header) {
@@ -95,20 +96,20 @@ oro::OriginsDB& TimerBase::db() {
 	return *m_db;
 }
 
-void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header) {
+void TimerBase::update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source) {
-	db().update(shops, calc_next_update(header, m_extra_delay));
+	db().update(shops, calc_next_update(header, m_extra_delay), source);
 }
 
-void TimerBase::update_db (const oro::Items& items, const oro::Header& header) {
+void TimerBase::update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source) {
-	db().update(items, calc_next_update(header, m_extra_delay));
+	db().update(items, calc_next_update(header, m_extra_delay), source);
 }
 
-void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header) {
+void TimerBase::update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source) {
-	db().update(icons, calc_next_update(header, m_extra_delay));
+	db().update(icons, calc_next_update(header, m_extra_delay), source);
 }
 
-void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header) {
+void TimerBase::update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source) {
-	db().update(creators, calc_next_update(header, m_extra_delay));
+	db().update(creators, calc_next_update(header, m_extra_delay), source);
 }
 
 } //namespace duck

src/timer_base.hpp

@@ -20,6 +20,7 @@
 #include "eventia/private/context.hpp"
 #include "eventia/timer.hpp"
 #include "oro/dboperation.hpp"
+#include "oro/source.hpp"
 
 namespace roar11 {
 	class ThreadPool;
@@ -38,13 +39,15 @@ namespace oro {
 namespace duck {
 
 struct TimerSettings {
-	TimerSettings (double min_wait, double extra_delay) :
+	TimerSettings (double min_wait, double extra_delay, bool with_raw_json) :
 		min_wait(min_wait),
-		extra_delay(extra_delay)
+		extra_delay(extra_delay),
+		store_raw_json(with_raw_json)
 	{ }
 
 	double min_wait;
 	double extra_delay;
+	bool store_raw_json;
 };
 
 class TimerBase : public eve::Timer {
@@ -63,21 +66,22 @@ public:
 
 protected:
 	void set_next_timer (const oro::Header& header);
-	void update_db (const oro::Shops& shops, const oro::Header& header);
+	void update_db (const oro::Shops& shops, const oro::Header& header, const oro::Source& source);
-	void update_db (const oro::Items& items, const oro::Header& header);
+	void update_db (const oro::Items& items, const oro::Header& header, const oro::Source& source);
-	void update_db (const oro::Icons& icons, const oro::Header& header);
+	void update_db (const oro::Icons& icons, const oro::Header& header, const oro::Source& source);
-	void update_db (const oro::Creators& creators, const oro::Header& header);
+	void update_db (const oro::Creators& creators, const oro::Header& header, const oro::Source& source);
 	roar11::ThreadPool& pool();
 	oro::Api& oro_api();
 	oro::OriginsDB& db();
 
 private:
-	virtual void fetch_data() = 0;
+	virtual void fetch_data(bool with_raw) = 0;
 
 	double m_extra_delay;
 	roar11::ThreadPool* m_pool;
 	oro::Api* m_oro_api;
 	oro::OriginsDB* m_db;
+	bool m_store_raw_response;
 };
 
 } //namespace duck

src/timer_oro_api.cpp

@@ -28,26 +28,26 @@
 namespace duck {
 
 namespace {
-	template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api);
+	template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api, bool with_raw);
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api) {
+	inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api, bool with_raw) {
-		return api.items_icons();
+		return api.items_icons(with_raw);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api) {
+	inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api, bool with_raw) {
-		return api.items_list();
+		return api.items_list(with_raw);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api) {
+	inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api, bool with_raw) {
-		return api.fame_list();
+		return api.fame_list(with_raw);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api) {
+	inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api, bool with_raw) {
-		return api.market_list();
+		return api.market_list(with_raw);
 	}
 } //unnamed namespace
 
@@ -64,12 +64,18 @@ inline TimerOroApi<Op>::TimerOroApi (
 }
 
 template<oro::DBOperation Op>
-inline void TimerOroApi<Op>::fetch_data() {
+inline void TimerOroApi<Op>::fetch_data (bool with_raw) {
 	int status_code = 200;
 	try {
-		auto results = invoke_api_func<Op>(oro_api());
+		auto results = invoke_api_func<Op>(oro_api(), with_raw);
-		set_next_timer(results.first);
+		set_next_timer(results.header);
-		this->update_db(results.second, results.first);
+
+		oro::Source raw_src;
+		if (with_raw) {
+			raw_src.data = std::move(results.raw_response);
+			raw_src.format = oro::SourceFormat::Plain;
+		}
+		this->update_db(results.data, results.header, raw_src);
 	}
 #if defined(OROTOOL_WITH_RESTCCPP)
 	catch (const restc_cpp::RequestFailedWithErrorException& err) {

src/timer_oro_api.hpp

@@ -38,7 +38,7 @@ public:
 	);
 
 private:
-	virtual void fetch_data() override;
+	virtual void fetch_data (bool with_raw) override;
 };
 
 } //namespace duck