Add optional lzma support for stored json responses

When enabled entries in the DB are stored as base64
encoded xz data. To uncompress save to a file
my_json.base64 and run:

cat my_json.base64 | base64 --decode | unxz --decompress - > my_json.txt

meson_options.txt

@@ -3,3 +3,4 @@ option('def_sqlite_db_name', type: 'string', value: 'originsro.db3')
 option('tests', type: 'feature', value: 'enabled')
 option('with_sqlite', type: 'feature', value: 'auto')
 option('rest_lib', type: 'combo', choices: ['nap', 'restc-cpp'], value: 'nap')
+option('with_lzma', type: 'feature', value: 'enabled')

orotool.conf

@@ -7,6 +7,7 @@ backend=sqlite
 fetch_extra_delay=2
 api_key=my_key_here
 store_raw_json=true
+json_store_mode=xz
 
 [timing]
 items=604800

src/app_config.cpp

@@ -54,6 +54,10 @@ namespace {
 	constexpr const char g_store_raw_json[] = "store_raw_json";
 	constexpr const char g_store_raw_json_def[] = "false";
 
+	constexpr const char g_json_store_mode_sect[] = "options";
+	constexpr const char g_json_store_mode[] = "json_store_mode";
+	constexpr const char g_json_store_mode_def[] = "plain";
+
 	constexpr const char g_items_time_sect[] = "timing";
 	constexpr const char g_items_time[] = "items";
 	constexpr const char g_items_time_def[] = "604800";
@@ -205,4 +209,30 @@ std::size_t AppConfig::creators_timeout() const {
 	return std::max(dhandy::int_conv<std::size_t>(val), g_min_update_timeout);
 }
 
+oro::SourceFormat AppConfig::json_store_mode() const {
+	std::string_view val = value_ifp(m_ini, g_json_store_mode_sect, g_json_store_mode, g_json_store_mode_def, false);
+
+	if (equal(val, "plain")) {
+		return oro::SourceFormat::Plain;
+	}
+	else if (equal(val, "xz")) {
+#if defined(OROTOOL_WITH_LZMA)
+		return oro::SourceFormat::Base64_xz;
+#else
+		throw std::runtime_error(
+			std::string("This version of ") + g_project_name +
+			" was compiled without lzma support so " + g_json_store_mode +
+			" can't be set to xz"
+		);
+#endif
+	}
+	else {
+		throw std::runtime_error(
+			"Invalid value \"" + std::string(val) +
+			"\" for option [" + g_json_store_mode_sect + "] " +
+			g_json_store_mode
+		);
+	}
+}
+
 } //namespace duck

src/app_config.hpp

@@ -18,6 +18,7 @@
 #pragma once
 
 #include "ini_file.hpp"
+#include "oro/source.hpp"
 #include <string_view>
 #include <cstddef>
 
@@ -39,6 +40,7 @@ public:
 	std::size_t icons_timeout() const;
 	std::size_t shops_timeout() const;
 	std::size_t creators_timeout() const;
+	oro::SourceFormat json_store_mode() const;
 
 private:
 	kamokan::IniFile m_ini;

src/base64.cpp

@@ -21,11 +21,16 @@
 #include "gnulib/lib/base64.h"
 #include <cassert>
 #include <stdexcept>
+#include <cstring>
 
 #if defined(base64_decode)
 #	undef base64_decode
 #endif
 
+#if defined(base64_encode)
+#	undef base64_encode
+#endif
+
 namespace duck {
 
 std::vector<char> base64_decode (std::string_view text) {
@@ -43,6 +48,20 @@ std::vector<char> base64_decode (std::string_view text) {
 	return retval;
 }
 
+std::string base64_encode (const std::vector<uint8_t>& data) {
+	const std::size_t size = (data.size() + 2) / 3 * 4 + 1;
+	std::string ret(size, '@');
+	::base64_encode(
+		reinterpret_cast<const char*>(data.data()),
+		data.size(),
+		ret.data(),
+		ret.size()
+	);
+
+	ret.resize(std::strlen(ret.data()));
+	return ret;
+}
+
 } //namespace duck
 
 #undef restrict

src/base64.hpp

@@ -19,9 +19,11 @@
 
 #include <string_view>
 #include <vector>
+#include <string>
 
 namespace duck {
 
 std::vector<char> base64_decode (std::string_view text);
+std::string base64_encode (const std::vector<uint8_t>& data);
 
 } //namespace duck

src/config.hpp.in

@@ -29,6 +29,8 @@ constexpr const unsigned short int g_version_major = @PROJECT_VERSION_MAJOR@;
 constexpr const unsigned short int g_version_minor = @PROJECT_VERSION_MINOR@;
 constexpr const unsigned short int g_version_patch = @PROJECT_VERSION_PATCH@;
 
+#mesondefine OROTOOL_WITH_LZMA
+
 #mesondefine OROTOOL_WITH_SQLITE
 
 #mesondefine OROTOOL_WITH_NAP

src/evloop.cpp

@@ -72,17 +72,18 @@ void test(oro::Api* api, oro::OriginsDB* db, const AppConfig& app_conf) {
 	const double ed = static_cast<double>(app_conf.fetch_extra_delay());
 
 	auto sig_int = worker.make_event<SignalInt>(&worker);
-	const bool rj = app_conf.store_raw_json();
+	const oro::SourceFormat sf =
+		(app_conf.store_raw_json() ? app_conf.json_store_mode() : oro::SourceFormat::None);
 
 	const double w1 = static_cast<double>(app_conf.items_timeout());
 	const double w2 = static_cast<double>(app_conf.icons_timeout());
 	const double w3 = static_cast<double>(app_conf.shops_timeout());
 	const double w4 = static_cast<double>(app_conf.creators_timeout());
 
-	auto timer_items = worker.make_event<TimerItems>(TSet{w1, ed, rj}, &pool, api, db);
-	auto timer_icons = worker.make_event<TimerIcons>(TSet{w2, ed, rj}, &pool, api, db);
-	auto timer_shops = worker.make_event<TimerShops>(TSet{w3, ed, rj}, &pool, api, db);
-	auto timer_creat = worker.make_event<TimerCreators>(TSet{w4, ed, rj}, &pool, api, db);
+	auto timer_items = worker.make_event<TimerItems>(TSet{w1, ed, sf}, &pool, api, db);
+	auto timer_icons = worker.make_event<TimerIcons>(TSet{w2, ed, sf}, &pool, api, db);
+	auto timer_shops = worker.make_event<TimerShops>(TSet{w3, ed, sf}, &pool, api, db);
+	auto timer_creat = worker.make_event<TimerCreators>(TSet{w4, ed, sf}, &pool, api, db);
 
 	worker.wait();
 #if !defined(NDEBUG)

src/lzma.cpp

@@ -0,0 +1,107 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//Code based on the public domain example here:
+//https://fossies.org/linux/xz/doc/examples/01_compress_easy.c
+
+#include "lzma.hpp"
+#include <lzma.h>
+#include <stdexcept>
+#include <string>
+#include <array>
+
+namespace duck {
+
+struct Lzma::LocalData {
+	LocalData() :
+		stream(LZMA_STREAM_INIT)
+	{
+	}
+
+	::lzma_stream stream;
+};
+
+Lzma::Lzma() :
+	m_local(std::make_unique<LocalData>())
+{
+	constexpr const uint32_t preset = 9 | LZMA_PRESET_EXTREME;
+
+	const lzma_ret init_ret = lzma_easy_encoder(&m_local->stream, preset, LZMA_CHECK_CRC64);
+	if (LZMA_OK != init_ret) {
+		const char* msg;
+		switch (init_ret) {
+		case LZMA_MEM_ERROR:
+			msg = "memory allocation failed";
+			break;
+		case LZMA_OPTIONS_ERROR:
+			msg = "specified preset is not supported";
+			break;
+		case LZMA_UNSUPPORTED_CHECK:
+			msg = "specified integrity check is not supported";
+			break;
+		default:
+			msg = "unknown error";
+			break;
+		}
+		throw std::runtime_error(std::string("lzma initialisation failed: ") +
+			msg + " (" + std::to_string(init_ret) + ")");
+	}
+}
+
+Lzma::~Lzma() noexcept {
+	::lzma_end(&m_local->stream);
+}
+
+std::vector<uint8_t> Lzma::compress (std::string_view input) {
+	std::array<uint8_t, BUFSIZ> outbuf;
+	auto& strm = m_local->stream;
+
+	strm.next_in = reinterpret_cast<const uint8_t*>(input.data());
+	strm.avail_in = input.size();
+	strm.next_out = outbuf.data();
+	strm.avail_out = outbuf.size();
+
+	std::vector<uint8_t> out;
+
+	::lzma_ret code;
+	do {
+		const ::lzma_action action = (strm.avail_in ? LZMA_RUN : LZMA_FINISH);
+		code = ::lzma_code(&strm, action);
+		if (strm.avail_out == 0 or LZMA_STREAM_END == code) {
+			out.insert(out.end(), outbuf.data(), outbuf.data() + (outbuf.size() - strm.avail_out));
+			strm.next_out = outbuf.data();
+			strm.avail_out = outbuf.size();
+		}
+	} while (LZMA_OK == code);
+
+	static const std::string err_prefix("lzma compress failed: ");
+	switch (code) {
+	case LZMA_OK:
+	case LZMA_STREAM_END:
+		break;
+	case LZMA_MEM_ERROR:
+		throw std::runtime_error(err_prefix + "memory allocation failed (" + std::to_string(code) + ")");
+	case LZMA_DATA_ERROR:
+		throw std::runtime_error(err_prefix + "output size limit exceeded (" + std::to_string(code) + ")");
+	default:
+		throw std::runtime_error(err_prefix + "unknown error (" + std::to_string(code) + ")");
+	}
+
+	return out;
+}
+
+} //namespace duck

src/lzma.hpp

@@ -0,0 +1,40 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <string_view>
+#include <cstdint>
+
+namespace duck {
+
+class Lzma {
+public:
+	Lzma();
+	~Lzma() noexcept;
+
+	std::vector<uint8_t> compress (std::string_view input);
+
+private:
+	struct LocalData;
+
+	std::unique_ptr<LocalData> m_local;
+};
+
+} //namespace duck

src/meson.build

@@ -15,6 +15,8 @@ curlcpp_dep = dependency('curlcpp', version: '>=1.4',
   ],
 )
 
+lzma_dep = dependency('liblzma', required: get_option('with_lzma'), version: '>=5.2.5')
+
 simdjson_dep = dependency('simdjson', version: '>=0.5.0',
   fallback: ['simdjson', 'simdjson_dep'],
 )
@@ -56,6 +58,7 @@ conf.set('PROJECT_VERSION_MAJOR', version_arr[0])
 conf.set('PROJECT_VERSION_MINOR', version_arr[1])
 conf.set('PROJECT_VERSION_PATCH', version_arr[2])
 conf.set('OROTOOL_WITH_SQLITE', sqlitecpp_dep.found())
+conf.set('OROTOOL_WITH_LZMA', lzma_dep.found())
 conf.set('OROTOOL_WITH_NAP', get_option('rest_lib') == 'nap')
 conf.set('OROTOOL_WITH_RESTCCPP', get_option('rest_lib') == 'restc-cpp')
 project_config_file = configure_file(
@@ -86,20 +89,25 @@ lib_deps = [
   boost_dep,
   curlcpp_dep,
   simdjson_dep,
+  lzma_dep,
 ] + backend_libs
 
 if get_option('rest_lib') == 'nap'
-  oro_rest_sources = [
+  optional_sources = [
 	'nap/page_fetch.cpp',
 	'nap/http_header_parse.cpp',
 	'nap/quick_rest.cpp',
 	'oro/api_nap.cpp',
   ]
 elif get_option('rest_lib') == 'restc-cpp'
-  oro_rest_sources = ['oro/api_restccpp.cpp']
+  optional_sources = ['oro/api_restccpp.cpp']
   lib_deps += [restc_cpp_dep]
 endif
 
+if lzma_dep.found()
+  optional_sources += ['lzma.cpp']
+endif
+
 executable(meson.project_name(),
   'main.cpp',
   'ini_file.cpp',
@@ -123,7 +131,7 @@ executable(meson.project_name(),
   'oro/originsdb.cpp',
   'oro/api.cpp',
   'oro/api_nap_exception.cpp',
-  oro_rest_sources,
+  optional_sources,
   project_config_file,
   install: true,
   dependencies: lib_deps,

src/oro/source.hpp

@@ -27,12 +27,14 @@ namespace oro {
 //other tables will have a reference to the source.
 
 enum class SourceFormat {
-	Plain
+	None,
+	Plain,
+	Base64_xz
 };
 
 struct Source {
 	std::optional<std::string> data;
-	SourceFormat format;
+	SourceFormat format {SourceFormat::None};
 };
 
 } //namespace oro

src/timer_base.cpp

@@ -79,14 +79,14 @@ TimerBase::TimerBase (
 	m_pool(pool),
 	m_oro_api(oro_api),
 	m_db(db),
-	m_store_raw_response(settings.store_raw_json)
+	m_source_store_mode(settings.src_store_mode)
 {
 	assert(m_pool);
 	assert(m_oro_api);
 }
 
 void TimerBase::on_timer() {
-	m_pool->submit(&TimerBase::fetch_data, this, m_store_raw_response);
+	m_pool->submit(&TimerBase::fetch_data, this, m_source_store_mode);
 }
 
 void TimerBase::set_next_timer (const oro::Header& header) {

src/timer_base.hpp

@@ -39,15 +39,15 @@ namespace oro {
 namespace duck {
 
 struct TimerSettings {
-	TimerSettings (double min_wait, double extra_delay, bool with_raw_json) :
+	TimerSettings (double min_wait, double extra_delay, oro::SourceFormat store_mode) :
 		min_wait(min_wait),
 		extra_delay(extra_delay),
-		store_raw_json(with_raw_json)
+		src_store_mode(store_mode)
 	{ }
 
 	double min_wait;
 	double extra_delay;
-	bool store_raw_json;
+	oro::SourceFormat src_store_mode;
 };
 
 class TimerBase : public eve::Timer {
@@ -76,14 +76,14 @@ protected:
 	void reset_db_access_time (oro::DBOperation op);
 
 private:
-	virtual void fetch_data(bool with_raw) = 0;
+	virtual void fetch_data(oro::SourceFormat store_mode) = 0;
 
 	double m_extra_delay;
 	double m_min_wait;
 	roar11::ThreadPool* m_pool;
 	oro::Api* m_oro_api;
 	oro::OriginsDB* m_db;
-	bool m_store_raw_response;
+	oro::SourceFormat m_source_store_mode;
 };
 
 } //namespace duck

src/timer_oro_api.cpp

@@ -18,6 +18,10 @@
 #include "timer_oro_api.hpp"
 #include "orotool_config.hpp"
 #include "oro/api.hpp"
+#if defined(OROTOOL_WITH_LZMA)
+#	include "lzma.hpp"
+#endif
+#include "base64.hpp"
 #if defined(OROTOOL_WITH_RESTCCPP)
 #	include <restc-cpp/error.h>
 #elif defined(OROTOOL_WITH_NAP)
@@ -28,26 +32,26 @@
 namespace duck {
 
 namespace {
-	template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api, bool with_raw);
+	template <oro::DBOperation Op> auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode);
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api, bool with_raw) {
-		return api.items_icons(with_raw);
+	inline auto invoke_api_func<oro::DBOperation::Icons> (oro::Api& api, oro::SourceFormat store_mode) {
+		return api.items_icons(oro::SourceFormat::None != store_mode);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api, bool with_raw) {
-		return api.items_list(with_raw);
+	inline auto invoke_api_func<oro::DBOperation::Items> (oro::Api& api, oro::SourceFormat store_mode) {
+		return api.items_list(oro::SourceFormat::None != store_mode);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api, bool with_raw) {
-		return api.fame_list(with_raw);
+	inline auto invoke_api_func<oro::DBOperation::Creators> (oro::Api& api, oro::SourceFormat store_mode) {
+		return api.fame_list(oro::SourceFormat::None != store_mode);
 	}
 
 	template <>
-	inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api, bool with_raw) {
-		return api.market_list(with_raw);
+	inline auto invoke_api_func<oro::DBOperation::Shops> (oro::Api& api, oro::SourceFormat store_mode) {
+		return api.market_list(oro::SourceFormat::None != store_mode);
 	}
 } //unnamed namespace
 
@@ -64,16 +68,28 @@ inline TimerOroApi<Op>::TimerOroApi (
 }
 
 template<oro::DBOperation Op>
-inline void TimerOroApi<Op>::fetch_data (bool with_raw) {
+inline void TimerOroApi<Op>::fetch_data (oro::SourceFormat store_mode) {
 	int status_code = 200;
 	try {
-		auto results = invoke_api_func<Op>(oro_api(), with_raw);
+		auto results = invoke_api_func<Op>(oro_api(), store_mode);
 		set_next_timer(results.header);
 
 		oro::Source raw_src;
-		if (with_raw) {
+		raw_src.format = store_mode;
+		switch (store_mode) {
+		case oro::SourceFormat::Plain:
 			raw_src.data = std::move(results.raw_response);
-			raw_src.format = oro::SourceFormat::Plain;
+			break;
+		case oro::SourceFormat::Base64_xz:
+#if defined(OROTOOL_WITH_LZMA)
+			{
+				Lzma lzma;
+				raw_src.data = base64_encode(lzma.compress(results.raw_response));
+			}
+			break;
+#endif
+		default:
+			raw_src.format = oro::SourceFormat::None;
 		}
 		this->update_db(results.data, results.header, raw_src);
 	}

src/timer_oro_api.hpp

@@ -38,7 +38,7 @@ public:
 	);
 
 private:
-	virtual void fetch_data (bool with_raw) override;
+	virtual void fetch_data (oro::SourceFormat store_mode) override;
 };
 
 } //namespace duck