diff --git a/meson_options.txt b/meson_options.txt index 94ce1f3..4996dea 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -3,3 +3,4 @@ option('def_sqlite_db_name', type: 'string', value: 'originsro.db3') option('tests', type: 'feature', value: 'enabled') option('with_sqlite', type: 'feature', value: 'auto') option('rest_lib', type: 'combo', choices: ['nap', 'restc-cpp'], value: 'nap') +option('with_lzma', type: 'feature', value: 'enabled') diff --git a/orotool.conf b/orotool.conf index 8ba323f..7b71d65 100644 --- a/orotool.conf +++ b/orotool.conf @@ -7,6 +7,7 @@ backend=sqlite fetch_extra_delay=2 api_key=my_key_here store_raw_json=true +json_store_mode=xz [timing] items=604800 diff --git a/src/app_config.cpp b/src/app_config.cpp index f06e085..97a91b1 100644 --- a/src/app_config.cpp +++ b/src/app_config.cpp @@ -54,6 +54,10 @@ namespace { constexpr const char g_store_raw_json[] = "store_raw_json"; constexpr const char g_store_raw_json_def[] = "false"; + constexpr const char g_json_store_mode_sect[] = "options"; + constexpr const char g_json_store_mode[] = "json_store_mode"; + constexpr const char g_json_store_mode_def[] = "plain"; + constexpr const char g_items_time_sect[] = "timing"; constexpr const char g_items_time[] = "items"; constexpr const char g_items_time_def[] = "604800"; @@ -205,4 +209,30 @@ std::size_t AppConfig::creators_timeout() const { return std::max(dhandy::int_conv(val), g_min_update_timeout); } +oro::SourceFormat AppConfig::json_store_mode() const { + std::string_view val = value_ifp(m_ini, g_json_store_mode_sect, g_json_store_mode, g_json_store_mode_def, false); + + if (equal(val, "plain")) { + return oro::SourceFormat::Plain; + } + else if (equal(val, "xz")) { +#if defined(OROTOOL_WITH_LZMA) + return oro::SourceFormat::Base64_xz; +#else + throw std::runtime_error( + std::string("This version of ") + g_project_name + + " was compiled without lzma support so " + g_json_store_mode + + " can't be set to xz" + ); +#endif + } + else { + throw std::runtime_error( + "Invalid value \"" + std::string(val) + + "\" for option [" + g_json_store_mode_sect + "] " + + g_json_store_mode + ); + } +} + } //namespace duck diff --git a/src/app_config.hpp b/src/app_config.hpp index 207cd0a..0dcde2d 100644 --- a/src/app_config.hpp +++ b/src/app_config.hpp @@ -18,6 +18,7 @@ #pragma once #include "ini_file.hpp" +#include "oro/source.hpp" #include #include @@ -39,6 +40,7 @@ public: std::size_t icons_timeout() const; std::size_t shops_timeout() const; std::size_t creators_timeout() const; + oro::SourceFormat json_store_mode() const; private: kamokan::IniFile m_ini; diff --git a/src/base64.cpp b/src/base64.cpp index 8978e2b..7a045b3 100644 --- a/src/base64.cpp +++ b/src/base64.cpp @@ -21,11 +21,16 @@ #include "gnulib/lib/base64.h" #include #include +#include #if defined(base64_decode) # undef base64_decode #endif +#if defined(base64_encode) +# undef base64_encode +#endif + namespace duck { std::vector base64_decode (std::string_view text) { @@ -43,6 +48,20 @@ std::vector base64_decode (std::string_view text) { return retval; } +std::string base64_encode (const std::vector& data) { + const std::size_t size = (data.size() + 2) / 3 * 4 + 1; + std::string ret(size, '@'); + ::base64_encode( + reinterpret_cast(data.data()), + data.size(), + ret.data(), + ret.size() + ); + + ret.resize(std::strlen(ret.data())); + return ret; +} + } //namespace duck #undef restrict diff --git a/src/base64.hpp b/src/base64.hpp index 7f20296..728d329 100644 --- a/src/base64.hpp +++ b/src/base64.hpp @@ -19,9 +19,11 @@ #include #include +#include namespace duck { std::vector base64_decode (std::string_view text); +std::string base64_encode (const std::vector& data); } //namespace duck diff --git a/src/config.hpp.in b/src/config.hpp.in index c94cc0a..6544a4f 100644 --- a/src/config.hpp.in +++ b/src/config.hpp.in @@ -29,6 +29,8 @@ constexpr const unsigned short int g_version_major = @PROJECT_VERSION_MAJOR@; constexpr const unsigned short int g_version_minor = @PROJECT_VERSION_MINOR@; constexpr const unsigned short int g_version_patch = @PROJECT_VERSION_PATCH@; +#mesondefine OROTOOL_WITH_LZMA + #mesondefine OROTOOL_WITH_SQLITE #mesondefine OROTOOL_WITH_NAP diff --git a/src/evloop.cpp b/src/evloop.cpp index e918810..43c288e 100644 --- a/src/evloop.cpp +++ b/src/evloop.cpp @@ -72,17 +72,18 @@ void test(oro::Api* api, oro::OriginsDB* db, const AppConfig& app_conf) { const double ed = static_cast(app_conf.fetch_extra_delay()); auto sig_int = worker.make_event(&worker); - const bool rj = app_conf.store_raw_json(); + const oro::SourceFormat sf = + (app_conf.store_raw_json() ? app_conf.json_store_mode() : oro::SourceFormat::None); const double w1 = static_cast(app_conf.items_timeout()); const double w2 = static_cast(app_conf.icons_timeout()); const double w3 = static_cast(app_conf.shops_timeout()); const double w4 = static_cast(app_conf.creators_timeout()); - auto timer_items = worker.make_event(TSet{w1, ed, rj}, &pool, api, db); - auto timer_icons = worker.make_event(TSet{w2, ed, rj}, &pool, api, db); - auto timer_shops = worker.make_event(TSet{w3, ed, rj}, &pool, api, db); - auto timer_creat = worker.make_event(TSet{w4, ed, rj}, &pool, api, db); + auto timer_items = worker.make_event(TSet{w1, ed, sf}, &pool, api, db); + auto timer_icons = worker.make_event(TSet{w2, ed, sf}, &pool, api, db); + auto timer_shops = worker.make_event(TSet{w3, ed, sf}, &pool, api, db); + auto timer_creat = worker.make_event(TSet{w4, ed, sf}, &pool, api, db); worker.wait(); #if !defined(NDEBUG) diff --git a/src/lzma.cpp b/src/lzma.cpp new file mode 100644 index 0000000..718e639 --- /dev/null +++ b/src/lzma.cpp @@ -0,0 +1,107 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +//Code based on the public domain example here: +//https://fossies.org/linux/xz/doc/examples/01_compress_easy.c + +#include "lzma.hpp" +#include +#include +#include +#include + +namespace duck { + +struct Lzma::LocalData { + LocalData() : + stream(LZMA_STREAM_INIT) + { + } + + ::lzma_stream stream; +}; + +Lzma::Lzma() : + m_local(std::make_unique()) +{ + constexpr const uint32_t preset = 9 | LZMA_PRESET_EXTREME; + + const lzma_ret init_ret = lzma_easy_encoder(&m_local->stream, preset, LZMA_CHECK_CRC64); + if (LZMA_OK != init_ret) { + const char* msg; + switch (init_ret) { + case LZMA_MEM_ERROR: + msg = "memory allocation failed"; + break; + case LZMA_OPTIONS_ERROR: + msg = "specified preset is not supported"; + break; + case LZMA_UNSUPPORTED_CHECK: + msg = "specified integrity check is not supported"; + break; + default: + msg = "unknown error"; + break; + } + throw std::runtime_error(std::string("lzma initialisation failed: ") + + msg + " (" + std::to_string(init_ret) + ")"); + } +} + +Lzma::~Lzma() noexcept { + ::lzma_end(&m_local->stream); +} + +std::vector Lzma::compress (std::string_view input) { + std::array outbuf; + auto& strm = m_local->stream; + + strm.next_in = reinterpret_cast(input.data()); + strm.avail_in = input.size(); + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); + + std::vector out; + + ::lzma_ret code; + do { + const ::lzma_action action = (strm.avail_in ? LZMA_RUN : LZMA_FINISH); + code = ::lzma_code(&strm, action); + if (strm.avail_out == 0 or LZMA_STREAM_END == code) { + out.insert(out.end(), outbuf.data(), outbuf.data() + (outbuf.size() - strm.avail_out)); + strm.next_out = outbuf.data(); + strm.avail_out = outbuf.size(); + } + } while (LZMA_OK == code); + + static const std::string err_prefix("lzma compress failed: "); + switch (code) { + case LZMA_OK: + case LZMA_STREAM_END: + break; + case LZMA_MEM_ERROR: + throw std::runtime_error(err_prefix + "memory allocation failed (" + std::to_string(code) + ")"); + case LZMA_DATA_ERROR: + throw std::runtime_error(err_prefix + "output size limit exceeded (" + std::to_string(code) + ")"); + default: + throw std::runtime_error(err_prefix + "unknown error (" + std::to_string(code) + ")"); + } + + return out; +} + +} //namespace duck diff --git a/src/lzma.hpp b/src/lzma.hpp new file mode 100644 index 0000000..d6782a4 --- /dev/null +++ b/src/lzma.hpp @@ -0,0 +1,40 @@ +/* Copyright 2020, Michele Santullo + * This file is part of orotool. + * + * Orotool is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Orotool is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Orotool. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +namespace duck { + +class Lzma { +public: + Lzma(); + ~Lzma() noexcept; + + std::vector compress (std::string_view input); + +private: + struct LocalData; + + std::unique_ptr m_local; +}; + +} //namespace duck diff --git a/src/meson.build b/src/meson.build index 4aa844a..6b86f33 100644 --- a/src/meson.build +++ b/src/meson.build @@ -15,6 +15,8 @@ curlcpp_dep = dependency('curlcpp', version: '>=1.4', ], ) +lzma_dep = dependency('liblzma', required: get_option('with_lzma'), version: '>=5.2.5') + simdjson_dep = dependency('simdjson', version: '>=0.5.0', fallback: ['simdjson', 'simdjson_dep'], ) @@ -56,6 +58,7 @@ conf.set('PROJECT_VERSION_MAJOR', version_arr[0]) conf.set('PROJECT_VERSION_MINOR', version_arr[1]) conf.set('PROJECT_VERSION_PATCH', version_arr[2]) conf.set('OROTOOL_WITH_SQLITE', sqlitecpp_dep.found()) +conf.set('OROTOOL_WITH_LZMA', lzma_dep.found()) conf.set('OROTOOL_WITH_NAP', get_option('rest_lib') == 'nap') conf.set('OROTOOL_WITH_RESTCCPP', get_option('rest_lib') == 'restc-cpp') project_config_file = configure_file( @@ -86,20 +89,25 @@ lib_deps = [ boost_dep, curlcpp_dep, simdjson_dep, + lzma_dep, ] + backend_libs if get_option('rest_lib') == 'nap' - oro_rest_sources = [ + optional_sources = [ 'nap/page_fetch.cpp', 'nap/http_header_parse.cpp', 'nap/quick_rest.cpp', 'oro/api_nap.cpp', ] elif get_option('rest_lib') == 'restc-cpp' - oro_rest_sources = ['oro/api_restccpp.cpp'] + optional_sources = ['oro/api_restccpp.cpp'] lib_deps += [restc_cpp_dep] endif +if lzma_dep.found() + optional_sources += ['lzma.cpp'] +endif + executable(meson.project_name(), 'main.cpp', 'ini_file.cpp', @@ -123,7 +131,7 @@ executable(meson.project_name(), 'oro/originsdb.cpp', 'oro/api.cpp', 'oro/api_nap_exception.cpp', - oro_rest_sources, + optional_sources, project_config_file, install: true, dependencies: lib_deps, diff --git a/src/oro/source.hpp b/src/oro/source.hpp index 04b27a6..c1b38a0 100644 --- a/src/oro/source.hpp +++ b/src/oro/source.hpp @@ -27,12 +27,14 @@ namespace oro { //other tables will have a reference to the source. enum class SourceFormat { - Plain + None, + Plain, + Base64_xz }; struct Source { std::optional data; - SourceFormat format; + SourceFormat format {SourceFormat::None}; }; } //namespace oro diff --git a/src/timer_base.cpp b/src/timer_base.cpp index 5dbd7e0..5aafd9b 100644 --- a/src/timer_base.cpp +++ b/src/timer_base.cpp @@ -79,14 +79,14 @@ TimerBase::TimerBase ( m_pool(pool), m_oro_api(oro_api), m_db(db), - m_store_raw_response(settings.store_raw_json) + m_source_store_mode(settings.src_store_mode) { assert(m_pool); assert(m_oro_api); } void TimerBase::on_timer() { - m_pool->submit(&TimerBase::fetch_data, this, m_store_raw_response); + m_pool->submit(&TimerBase::fetch_data, this, m_source_store_mode); } void TimerBase::set_next_timer (const oro::Header& header) { diff --git a/src/timer_base.hpp b/src/timer_base.hpp index 3ea75ee..d12ea70 100644 --- a/src/timer_base.hpp +++ b/src/timer_base.hpp @@ -39,15 +39,15 @@ namespace oro { namespace duck { struct TimerSettings { - TimerSettings (double min_wait, double extra_delay, bool with_raw_json) : + TimerSettings (double min_wait, double extra_delay, oro::SourceFormat store_mode) : min_wait(min_wait), extra_delay(extra_delay), - store_raw_json(with_raw_json) + src_store_mode(store_mode) { } double min_wait; double extra_delay; - bool store_raw_json; + oro::SourceFormat src_store_mode; }; class TimerBase : public eve::Timer { @@ -76,14 +76,14 @@ protected: void reset_db_access_time (oro::DBOperation op); private: - virtual void fetch_data(bool with_raw) = 0; + virtual void fetch_data(oro::SourceFormat store_mode) = 0; double m_extra_delay; double m_min_wait; roar11::ThreadPool* m_pool; oro::Api* m_oro_api; oro::OriginsDB* m_db; - bool m_store_raw_response; + oro::SourceFormat m_source_store_mode; }; } //namespace duck diff --git a/src/timer_oro_api.cpp b/src/timer_oro_api.cpp index b94f959..fd56b8c 100644 --- a/src/timer_oro_api.cpp +++ b/src/timer_oro_api.cpp @@ -18,6 +18,10 @@ #include "timer_oro_api.hpp" #include "orotool_config.hpp" #include "oro/api.hpp" +#if defined(OROTOOL_WITH_LZMA) +# include "lzma.hpp" +#endif +#include "base64.hpp" #if defined(OROTOOL_WITH_RESTCCPP) # include #elif defined(OROTOOL_WITH_NAP) @@ -28,26 +32,26 @@ namespace duck { namespace { - template auto invoke_api_func (oro::Api& api, bool with_raw); + template auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode); template <> - inline auto invoke_api_func (oro::Api& api, bool with_raw) { - return api.items_icons(with_raw); + inline auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode) { + return api.items_icons(oro::SourceFormat::None != store_mode); } template <> - inline auto invoke_api_func (oro::Api& api, bool with_raw) { - return api.items_list(with_raw); + inline auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode) { + return api.items_list(oro::SourceFormat::None != store_mode); } template <> - inline auto invoke_api_func (oro::Api& api, bool with_raw) { - return api.fame_list(with_raw); + inline auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode) { + return api.fame_list(oro::SourceFormat::None != store_mode); } template <> - inline auto invoke_api_func (oro::Api& api, bool with_raw) { - return api.market_list(with_raw); + inline auto invoke_api_func (oro::Api& api, oro::SourceFormat store_mode) { + return api.market_list(oro::SourceFormat::None != store_mode); } } //unnamed namespace @@ -64,16 +68,28 @@ inline TimerOroApi::TimerOroApi ( } template -inline void TimerOroApi::fetch_data (bool with_raw) { +inline void TimerOroApi::fetch_data (oro::SourceFormat store_mode) { int status_code = 200; try { - auto results = invoke_api_func(oro_api(), with_raw); + auto results = invoke_api_func(oro_api(), store_mode); set_next_timer(results.header); oro::Source raw_src; - if (with_raw) { + raw_src.format = store_mode; + switch (store_mode) { + case oro::SourceFormat::Plain: raw_src.data = std::move(results.raw_response); - raw_src.format = oro::SourceFormat::Plain; + break; + case oro::SourceFormat::Base64_xz: +#if defined(OROTOOL_WITH_LZMA) + { + Lzma lzma; + raw_src.data = base64_encode(lzma.compress(results.raw_response)); + } + break; +#endif + default: + raw_src.format = oro::SourceFormat::None; } this->update_db(results.data, results.header, raw_src); } diff --git a/src/timer_oro_api.hpp b/src/timer_oro_api.hpp index 38a3a5f..0f99d3b 100644 --- a/src/timer_oro_api.hpp +++ b/src/timer_oro_api.hpp @@ -38,7 +38,7 @@ public: ); private: - virtual void fetch_data (bool with_raw) override; + virtual void fetch_data (oro::SourceFormat store_mode) override; }; } //namespace duck