Initial version of a rest server

This is more of a proof of concept than the actual
feature. WiP

.gitignore

@@ -8,3 +8,4 @@ subprojects/SQLiteCpp/
 subprojects/simdjson
 subprojects/restc-cpp
 subprojects/lest
+subprojects/civetweb

meson_options.txt

@@ -4,3 +4,4 @@ option('tests', type: 'feature', value: 'enabled')
 option('with_sqlite', type: 'feature', value: 'auto')
 option('rest_lib', type: 'combo', choices: ['nap', 'restc-cpp'], value: 'nap')
 option('with_lzma', type: 'feature', value: 'enabled')
+option('with_webserver', type: 'feature', value: 'disabled')

orotool.conf

@@ -16,3 +16,6 @@ items=604800
 icons=604800
 shops=120
 creators=400
+
+[webserver]
+enabled=yes

src/app_config.cpp

@@ -54,6 +54,10 @@ namespace {
 	constexpr const char g_store_raw_json[] = "store_raw_json";
 	constexpr const char g_store_raw_json_def[] = "false";
 
+	constexpr const char g_enable_webserver_sect[] = "webserver";
+	constexpr const char g_enable_webserver[] = "enabled";
+	constexpr const char g_enable_webserver_def[] = "no";
+
 	constexpr const char g_json_store_mode_sect[] = "options";
 	constexpr const char g_json_store_mode[] = "json_store_mode";
 	constexpr const char g_json_store_mode_def[] = "plain";
@@ -197,6 +201,11 @@ bool AppConfig::store_raw_json() const {
 	return to_bool(val);
 }
 
+bool AppConfig::enable_webserver() const {
+	std::string_view val = value_ifp(m_ini, g_enable_webserver_sect, g_enable_webserver, g_enable_webserver_def, false);
+	return to_bool(val);
+}
+
 std::size_t AppConfig::items_timeout() const {
 	std::string_view val = value_ifp(m_ini, g_items_time_sect, g_items_time, g_items_time_def, false);
 	return std::max(dhandy::int_conv<std::size_t>(val), g_min_update_timeout);

src/app_config.hpp

@@ -35,6 +35,7 @@ public:
 	std::size_t fetch_extra_delay() const;
 	std::string_view backend() const;
 	bool store_raw_json() const;
+	bool enable_webserver() const;
 
 	std::size_t items_timeout() const;
 	std::size_t icons_timeout() const;

src/config.hpp.in

@@ -30,6 +30,7 @@ constexpr const unsigned short int g_version_minor = @PROJECT_VERSION_MINOR@;
 constexpr const unsigned short int g_version_patch = @PROJECT_VERSION_PATCH@;
 
 #mesondefine OROTOOL_WITH_LZMA
+#mesondefine OROTOOL_WITH_WEBSERVER
 
 #mesondefine OROTOOL_WITH_SQLITE
 

src/gnulib

@@ -1 +1 @@
-Subproject commit a5578a71ef65a9aed1ccc254b9d9a07b0b46c1b0
+Subproject commit 444acd314f6c4a1fb7ea7412a54b8ea5fc0ad7c2

src/main.cpp

@@ -22,6 +22,9 @@
 #include "app_config.hpp"
 #include "duckhandy/int_conv.hpp"
 #include "duckhandy/string_bt.hpp"
+#if defined(OROTOOL_WITH_WEBSERVER)
+#	include "webserver/rest_server.hpp"
+#endif
 #if !defined(NDEBUG)
 #	include <iostream>
 #endif
@@ -97,6 +100,8 @@ int main(int argc, char* argv[]) {
 
 		std::unique_ptr<oro::OriginsDB> db(oro::OriginsDB::make(app_conf.backend(), app_conf.db_path()));
 
+		duck::RestServer server(8171);
+
 		duck::test(oro_api.get(), db.get(), app_conf);
 	}
 #if defined(OROTOOL_WITH_RESTCCPP)

src/meson.build

@@ -15,6 +15,23 @@ curlcpp_dep = dependency('curlcpp', version: '>=1.4',
   ],
 )
 
+civetweb_dep = dependency('civetweb', version: '>=1.12',
+  required: get_option('with_webserver'),
+  fallback: ['civetweb', 'civetweb_dep'],
+  default_options: [
+	'civetweb_cxx=enabled',
+	'civetweb_ssl_dynamic_loading=disabled',
+	'civetweb_ssl=auto',
+	'civetweb_serve_files=false',
+	'civetweb_serve_filesystems=false',
+	'civetweb_enable_ipv6=true',
+	'civetweb_enable_server_executable=false',
+	'civetweb_enable_cgi=false',
+	'civetweb_enable_caching=true',
+	'default_library=static',
+  ],
+)
+
 lzma_dep = dependency('liblzma', required: get_option('with_lzma'), version: '>=5.2.5')
 
 simdjson_dep = dependency('simdjson', version: '>=0.5.0',
@@ -27,7 +44,7 @@ if not get_option('with_sqlite').disabled()
 	required: get_option('with_sqlite'),
   )
 else
-  sqlitecpp_dep = disabler()
+  sqlitecpp_dep = dependency('', required: false)
 endif
 
 cpp_comp = meson.get_compiler('cpp')
@@ -61,6 +78,7 @@ conf.set('OROTOOL_WITH_SQLITE', sqlitecpp_dep.found())
 conf.set('OROTOOL_WITH_LZMA', lzma_dep.found())
 conf.set('OROTOOL_WITH_NAP', get_option('rest_lib') == 'nap')
 conf.set('OROTOOL_WITH_RESTCCPP', get_option('rest_lib') == 'restc-cpp')
+conf.set('OROTOOL_WITH_WEBSERVER', civetweb_dep.found())
 project_config_file = configure_file(
 	input: 'config.hpp.in',
 	output: meson.project_name() + '_config.hpp',
@@ -90,6 +108,7 @@ lib_deps = [
   curlcpp_dep,
   simdjson_dep,
   lzma_dep,
+  civetweb_dep,
 ] + backend_libs
 
 if get_option('rest_lib') == 'nap'
@@ -108,6 +127,10 @@ if lzma_dep.found()
   optional_sources += ['lzma.cpp']
 endif
 
+if civetweb_dep.found()
+  optional_sources += ['webserver/rest_server.cpp']
+endif
+
 executable(meson.project_name(),
   'main.cpp',
   'ini_file.cpp',
@@ -135,6 +158,7 @@ executable(meson.project_name(),
   optional_sources,
   project_config_file,
   'eventia_thread_pool.cpp',
+  'webserver/private/dateconv.cpp',
   install: true,
   dependencies: lib_deps,
   include_directories: [

src/oro/api_nap_exception.cpp

@@ -36,7 +36,7 @@ std::size_t append_to_char_arary (std::string_view in, char* out, std::size_t of
 
 ServerError::ServerError (const nap::HttpResponse& resp) :
 	std::runtime_error(
-		std::string("Server replied: ") + std::to_string(resp.code) + " (HTTP " +
+		std::string("Server replied: ") + std::to_string(resp.code) + " (" +
 			std::string(resp.http_ver) + ", \"" + std::string(resp.code_desc) +
 			"\")"
 	),

src/oro/datatypes.cpp

@@ -27,6 +27,14 @@ namespace oro {
 		return *this;
 	}
 
+	Timestamp Timestamp::now() {
+		return Timestamp{
+			std::chrono::time_point_cast<oro::timestamp_t::duration>(
+				std::chrono::system_clock::now()
+			)
+		};
+	}
+
 	std::ostream& operator<<(std::ostream& os, const Timestamp& ts) {
 		using date::operator<<;
 		os << ts.ts;

src/oro/datatypes.hpp

@@ -27,6 +27,7 @@ namespace oro {
 	struct Timestamp {
 		Timestamp& operator= (std::string&& str);
 		operator timestamp_t() const { return ts; }
+		static Timestamp now();
 
 		timestamp_t ts;
 	};

src/timer_base.cpp

@@ -47,7 +47,7 @@ namespace {
 	oro::Timestamp calc_next_update (const oro::Header& header, double min_wait, double extra) {
 		oro::Timestamp ret;
 		ret.ts =
-			std::chrono::time_point_cast<oro::timestamp_t::duration>(std::chrono::system_clock::now()) +
+			oro::Timestamp::now().ts +
 			std::chrono::seconds(time_interval(header, min_wait, extra));
 		return ret;
 	}
@@ -114,7 +114,7 @@ oro::OriginsDB& TimerBase::db() {
 void TimerBase::reset_db_access_time (oro::DBOperation op) {
 	oro::Timestamp ts;
 	ts.ts =
-		std::chrono::time_point_cast<oro::timestamp_t::duration>(std::chrono::system_clock::now()) +
+		oro::Timestamp::now().ts +
 		std::chrono::seconds(static_cast<unsigned long>(m_min_wait));
 	db().update_access_time(ts, op);
 }

src/timer_oro_api.cpp

@@ -22,7 +22,6 @@
 #	include "lzma.hpp"
 #endif
 #include "base64.hpp"
-#include "magic_enum.hpp"
 #if defined(OROTOOL_WITH_RESTCCPP)
 #	include <restc-cpp/error.h>
 #elif defined(OROTOOL_WITH_NAP)
@@ -71,7 +70,7 @@ inline TimerOroApi<Op>::TimerOroApi (
 
 template<oro::DBOperation Op>
 inline void TimerOroApi<Op>::fetch_data (oro::SourceFormat store_mode) {
-	bool schedule_retry = false;
+	int status_code = 200;
 	try {
 		auto results = invoke_api_func<Op>(oro_api(), store_mode);
 		set_next_timer(results.header);
@@ -97,9 +96,8 @@ inline void TimerOroApi<Op>::fetch_data (oro::SourceFormat store_mode) {
 	}
 #if defined(OROTOOL_WITH_RESTCCPP)
 	catch (const restc_cpp::RequestFailedWithErrorException& err) {
-		const int status_code = err.http_response.status_code;
+		status_code = err.http_response.status_code;
 		if (429 == err.http_response.status_code) {
-			schedule_retry = true;
 			//server received too many requests, give up for now
 		}
 		else {
@@ -108,17 +106,10 @@ inline void TimerOroApi<Op>::fetch_data (oro::SourceFormat store_mode) {
 	}
 #elif defined(OROTOOL_WITH_NAP)
 	catch (const oro::ServerError& err) {
-		const int status_code = err.error_code();
+		status_code = err.error_code();
-		switch (status_code) {
+		if (429 == status_code) {
-		case 503:
+		}
-			std::cerr << "Server returned 503 during request for " <<
+		else {
-				magic_enum::enum_name(Op) << '\n';
-			schedule_retry = true;
-			break;
-		case 429:
-			schedule_retry = true;
-			break;
-		default:
 			throw;
 		}
 	}
@@ -143,9 +134,9 @@ inline void TimerOroApi<Op>::fetch_data (oro::SourceFormat store_mode) {
 #endif
 
 	m_retries_left = error_retries();
-	if (schedule_retry) {
+	if (429 == status_code) {
 		oro::Header head;
-		head.date.ts = std::chrono::time_point_cast<oro::timestamp_t::duration>(std::chrono::system_clock::now());
+		head.date.ts = oro::Timestamp::now().ts;
 		head.rate_limit = 1;
 		head.rate_limit_remaining = 0;
 		head.rate_limit_reset = 600;

src/webserver/private/dateconv.cpp

@@ -0,0 +1,34 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "dateconv.hpp"
+
+#define HAS_UNCAUGHT_EXCEPTIONS 1
+#include "date/date.h"
+
+namespace duck {
+
+std::string to_header_string (const oro::Timestamp& ts) {
+	using date::operator<<;
+	using std::chrono::seconds;
+	using date::floor;
+	using date::format;
+
+	return format("%a, %d %b %Y %T %Z", floor<seconds>(ts.ts));
+}
+
+} //namespace duck

src/webserver/private/dateconv.hpp

@@ -0,0 +1,25 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "oro/datatypes.hpp"
+#include <string>
+
+namespace duck {
+std::string to_header_string (const oro::Timestamp& ts);
+} //namespace duck

src/webserver/rest_server.cpp

@@ -0,0 +1,79 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "rest_server.hpp"
+#include "CivetServer.h"
+#include "duckhandy/int_conv.hpp"
+#include "private/dateconv.hpp"
+
+namespace duck {
+namespace {
+class V1ItemsHandler : public CivetHandler {
+public:
+	bool handleGet (CivetServer* server, struct mg_connection* conn) override {
+		mg_printf(conn,
+			"HTTP/1.1 200 OK\r\n"
+			"Content-Type: application/json; charset=utf-8\r\n"
+			"Connection: close\r\n"
+			"Date: %s\r\n"
+			"\r\n",
+			to_header_string(oro::Timestamp::now()).c_str()
+		);
+		mg_printf(conn, "Hi this is the items handler\r\n");
+		return true;
+	}
+};
+
+void init_civetweb_ifn() {
+	static bool initialised = false;
+	if (not initialised) {
+		initialised = true;
+		if (mg_check_feature(2))
+			mg_init_library(MG_FEATURES_SSL);
+		else
+			mg_init_library(0);
+	}
+}
+} //unnamed namespace
+
+struct RestServer::LocalData {
+	explicit LocalData (const char* options[]) :
+		server(options)
+	{
+		server.addHandler("/v1/items", v1_items_handler);
+	}
+
+	V1ItemsHandler v1_items_handler;
+	CivetServer server;
+};
+
+RestServer::RestServer (uint16_t port) {
+	init_civetweb_ifn();
+
+	const std::string port_str = dhandy::int_conv<std::string>(port);
+	const char* options[] = {
+		"decode_url", "yes",
+		"listening_ports", port_str.c_str(),
+		"request_timeout_ms", "10000",
+		nullptr
+	};
+
+	m_local = std::make_unique<LocalData>(options);
+}
+
+RestServer::~RestServer() noexcept = default;
+} //namespace duck

src/webserver/rest_server.hpp

@@ -0,0 +1,34 @@
+/* Copyright 2020, Michele Santullo
+ * This file is part of orotool.
+ *
+ * Orotool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Orotool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Orotool.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <memory>
+
+namespace duck {
+class RestServer {
+public:
+	RestServer (uint16_t port);
+	~RestServer() noexcept;
+
+private:
+	struct LocalData;
+
+	std::unique_ptr<LocalData> m_local;
+};
+} //namespace duck

subprojects/civetweb.wrap

@@ -0,0 +1,7 @@
+[wrap-git]
+url = https://github.com/civetweb/civetweb.git
+revision = 4b440a339979852d5a51fb11a822952712231c23
+patch_directory = civetweb
+
+[provide]
+dependency_names = civetweb-1.12

subprojects/date

@@ -1 +1 @@
-Subproject commit 3cbfa4318fc6b43c27e0d1302a2a59cfea4b6e68
+Subproject commit 569b2d678547985d0ead3b73ee93a28919000887

subprojects/magic_enum

@@ -1 +1 @@
-Subproject commit 3080a1d7269fb96b70186512d8e849b09c424daf
+Subproject commit d73a985f520aa25a432d1bbf5525663b93c189a0

subprojects/packagefiles/civetweb/meson.build

@@ -0,0 +1,37 @@
+project('civetweb', 'c',
+  version: '1.12',
+  meson_version: '>=0.49.2',
+  default_options: ['buildtype=release', 'c_std=c11', 'cpp_std=c++17', 'b_ndebug=if-release'],
+  license: 'MIT',
+)
+
+has_cpp = add_languages('cpp', required: get_option('civetweb_cxx'))
+
+compiler_opts = []
+if get_option('civetweb_enable_ipv6')
+  compiler_opts += ['-DUSE_IPV6']
+endif
+if get_option('civetweb_serve_files')
+  compiler_opts += ['-DUSE_SERVER_STATS']
+endif
+if not get_option('civetweb_serve_files')
+  compiler_opts += ['-DNO_FILES']
+endif
+if get_option('civetweb_ssl_openssl_api') == '1.0'
+  compiler_opts += ['-DOPENSSL_API_1_1']
+elif get_option('civetweb_ssl_openssl_api') == '1.1'
+  compiler_opts += ['-DOPENSSL_API_1_1']
+endif
+compiler_opts += ['-DUSE_STACK_SIZE=' + get_option('civetweb_thread_stack_size').to_string()]
+if not get_option('civetweb_serve_filesystems')
+  compiler_opts += ['-DNO_FILESYSTEMS']
+endif
+if not get_option('civetweb_enable_cgi')
+  compiler_opts += ['-DNO_CGI']
+endif
+if not get_option('civetweb_enable_caching')
+  compiler_opts += ['-DNO_CACHING']
+endif
+
+public_incl = include_directories('include')
+subdir('src')

subprojects/packagefiles/civetweb/meson_options.txt

@@ -0,0 +1,15 @@
+option('civetweb_cxx', type: 'feature', value: 'disabled', description: 'Enables the C++ wrapper library')
+option('civetweb_lua', type: 'feature', value: 'disabled', description: 'Enable Lua CGIs')
+option('civetweb_ssl', type: 'feature', value: 'enabled', description: 'Enables the secure socket layer')
+option('civetweb_ssl_dynamic_loading', type: 'feature', value: 'auto', description: 'Dynamically loads the SSL library rather than linking it')
+option('civetweb_serve_files', type: 'boolean', value: 'true', description: 'Configures the server to serve static files')
+option('civetweb_serve_filesystems', type: 'boolean', value: 'true')
+option('civetweb_enable_ipv6', type: 'boolean', value: 'false', description: 'Enables the IP version 6 support')
+option('civetweb_enable_websockets', type: 'boolean', value: 'false', description: 'Enable websockets connections')
+option('civetweb_enable_server_stats', type: 'boolean', value: 'false', description: 'Enable server statistics')
+option('civetweb_enable_server_executable', type: 'boolean', value: 'true', description: 'Enable building of the server executable')
+option('civetweb_ssl_openssl_api', type: 'combo', choices: ['1.0', '1.1'], value: '1.1', description: 'Version of the OpenSSL API to use')
+option('civetweb_thread_stack_size', type: 'integer', value: 102400, description: 'The stack size in bytes for each thread created')
+option('civetweb_install_executable', type: 'boolean', value: 'false', description: 'Enable installing CivetWeb executable')
+option('civetweb_enable_cgi', type: 'boolean', value: 'true', description: 'Enables CGI, so theserver will execute CGI scripts')
+option('civetweb_enable_caching', type: 'boolean', value: 'true', description: 'Enables caching, timegm is used')

subprojects/packagefiles/civetweb/src/meson.build

@@ -0,0 +1,57 @@
+dl_dep = meson.get_compiler('c').find_library('dl', required: get_option('civetweb_ssl_dynamic_loading'))
+threads_dep = dependency('threads')
+openssl_dep = dependency('openssl', required: get_option('civetweb_ssl'))
+lua_dep = dependency('lua', required: get_option('civetweb_lua'))
+
+if not openssl_dep.found()
+  compiler_opts += ['-DNO_SSL']
+endif
+if not dl_dep.found()
+  compiler_opts += ['-DNO_SSL_DL']
+endif
+if lua_dep.found()
+  compiler_opts += ['-DUSE_LUA']
+endif
+
+civetweb_lib = library('civetweb-c-library',
+  'civetweb.c',
+  install: true,
+  c_args: compiler_opts,
+  include_directories: public_incl,
+  dependencies: [
+	dl_dep,
+	threads_dep,
+	openssl_dep,
+	lua_dep,
+  ],
+)
+
+link_with_libs = [civetweb_lib]
+
+if has_cpp
+  civetwebpp_lib = library('civetweb-cpp',
+	'CivetServer.cpp',
+	cpp_args: compiler_opts + ['-DCIVETWEB_CXX_DLL_EXPORTS'],
+	include_directories: public_incl,
+	link_with: civetweb_lib,
+  )
+  link_with_libs += [civetwebpp_lib]
+endif
+
+civetweb_dep = declare_dependency(
+  link_with: link_with_libs,
+  include_directories: public_incl,
+  compile_args: compiler_opts,
+)
+
+if get_option('civetweb_enable_server_executable')
+  executable('civetweb-c-executable',
+	'main.c',
+	include_directories: public_incl,
+	install: get_option('civetweb_install_executable'),
+	compile_args: compiler_opts,
+	dependencies: [
+	  civetweb_dep,
+	]
+  )
+endif