Cache highlighted pastie when saving.

src/kamokan/main.cpp

@@ -178,8 +178,10 @@ int main (int parArgc, char* parArgv[], char* parEnvp[]) {
 			tawashi::cgi::drop_arguments(cgi_env->request_uri_relative()),
 			cgi_env->request_method()
 		);
-		if (response)
+		if (response) {
 			response->send();
+			response->join();
+		}
 	}
 	catch (const std::exception& e) {
 		statuslog->critical("Uncaught exception in main(): \"{}\"", e.what());

src/kamokan_impl/CMakeLists.txt

@@ -46,6 +46,7 @@ target_link_libraries(${PROJECT_NAME}
 	PUBLIC mstch
 	PRIVATE pthread
 	PUBLIC tawashi
+	PRIVATE pthread
 )
 target_compile_definitions(${PROJECT_NAME}
 	PRIVATE BOOST_SPIRIT_USE_PHOENIX_V3=1
@@ -67,6 +68,9 @@ configure_file(
 add_custom_command(
 	OUTPUT include/lua_scripts_for_redis.cpp
 	COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" -P ${CMAKE_CURRENT_SOURCE_DIR}/lua_to_cpp.cmake
-	DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/retrieve_pastie.lua ${CMAKE_CURRENT_SOURCE_DIR}/save_pastie.lua
+	DEPENDS
+		${CMAKE_CURRENT_SOURCE_DIR}/retrieve_pastie.lua
+		${CMAKE_CURRENT_SOURCE_DIR}/save_pastie.lua
+		${CMAKE_CURRENT_SOURCE_DIR}/add_highlighted_pastie.lua
 	COMMENT "Embedding save/load lua scripts into the c++ code"
 )

src/kamokan_impl/add_highlighted_pastie.lua

@@ -0,0 +1,13 @@
+local full_token = KEYS[1]
+local highlighted_text = ARGV[1]
+local comment = ARGV[2]
+
+local saved = redis.call("HMSET", full_token,
+	"hl_pastie", highlighted_text,
+	"hl_comment", comment
+)
+
+if not saved then
+	return redis.error_reply("PastieNotSaved")
+end
+return "1"

src/kamokan_impl/highlight_functions.cpp

@@ -31,13 +31,13 @@ namespace kamokan {
 			return parSettings.as<std::string>("highlight_css");
 		}
 
-		std::string lang_name_to_file_path (const std::string& parLang, const std::string& parLangmapDir) {
+		std::string lang_name_to_file_path (boost::string_view parLang, const std::string& parLangmapDir) {
 			if (parLang.empty())
 				return std::string();
 
 			srchilite::LangMap lang_map(parLangmapDir, "lang.map");
 			lang_map.open();
-			std::string lang_file = lang_map.getFileName(parLang);
+			std::string lang_file = lang_map.getFileName(std::string(parLang));
 			if (lang_file.empty())
 				lang_file = "default.lang";
 
@@ -89,7 +89,7 @@ namespace kamokan {
 		return boost::copy_range<HighlightLangList>(lang_range | boost::adaptors::map_keys);
 	}
 
-	SplitHighlightedPastie highlight_string (std::string&& parIn, const std::string& parLang, const SettingsBag& parSettings) {
+	SplitHighlightedPastie highlight_string (std::string&& parIn, boost::string_view parLang, const SettingsBag& parSettings) {
 		const std::string langmap_dir = parSettings.as<std::string>("langmap_dir");
 		srchilite::SourceHighlight highlighter;
 		highlighter.setDataDir(langmap_dir);
@@ -105,10 +105,16 @@ namespace kamokan {
 		highlighter.setStyleCssFile(highlight_css_path(parSettings));
 		highlighter.setGenerateLineNumbers(false);
 
-		std::istringstream iss(std::move(parIn));
-		std::ostringstream oss;
-		highlighter.highlight(iss, oss, lang_name_to_file_path(parLang, langmap_dir));
-		return strip_tags_from_highlighted(oss.str());
+		std::string lang_file_path = lang_name_to_file_path(parLang, langmap_dir);
+		if (not lang_file_path.empty()) {
+			std::istringstream iss(std::move(parIn));
+			std::ostringstream oss;
+			highlighter.highlight(iss, oss, lang_file_path);
+			return strip_tags_from_highlighted(oss.str());
+		}
+		else {
+			return SplitHighlightedPastie();
+		}
 	}
 } //namespace kamokan
 

src/kamokan_impl/highlight_functions.hpp

@@ -32,5 +32,5 @@ namespace kamokan {
 	};
 
 	HighlightLangList list_highlight_langs (const SettingsBag& parSettings);
-	SplitHighlightedPastie highlight_string (std::string&& parIn, const std::string& parLang, const SettingsBag& parSettings);
+	SplitHighlightedPastie highlight_string (std::string&& parIn, boost::string_view parLang, const SettingsBag& parSettings);
 } //namespace kamokan

src/kamokan_impl/lua_scripts_for_redis.hpp

@@ -22,6 +22,8 @@
 namespace kamokan {
 	extern const char g_save_script[];
 	extern const char g_load_script[];
+	extern const char g_add_highlighted_script[];
 	extern const std::size_t g_save_script_size;
 	extern const std::size_t g_load_script_size;
+	extern const std::size_t g_add_highlighted_script_size;
 } //namespace kamokan

src/kamokan_impl/lua_to_cpp.cmake

@@ -1,21 +1,28 @@
-file(READ ${SOURCE_DIR}/retrieve_pastie.lua retrieve_pastie)
-file(READ ${SOURCE_DIR}/save_pastie.lua save_pastie)
-string(REGEX REPLACE "[ \t]*=[ \t]*" "=" retrieve_pastie "${retrieve_pastie}")
-string(REGEX REPLACE "[ \t]*=[ \t]*" "=" save_pastie "${save_pastie}")
-string(REGEX REPLACE ",[ \t]*" "," retrieve_pastie "${retrieve_pastie}")
-string(REGEX REPLACE ",[ \t]*" "," save_pastie "${save_pastie}")
-string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" retrieve_pastie "${retrieve_pastie}")
-string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" save_pastie "${save_pastie}")
-string(LENGTH "${retrieve_pastie}" retrieve_pastie_length)
-string(LENGTH "${save_pastie}" save_pastie_length)
+function (load_and_strip_lua_script file_path out_prefix)
+	file(READ ${file_path} retval)
+	string(REGEX REPLACE "[ \t]*=[ \t]*" "=" retval "${retval}")
+	string(REGEX REPLACE ",[ \t]*" "," retval "${retval}")
+	string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" retval "${retval}")
+	string(LENGTH "${retval}" retval_length)
+	set(${out_prefix}_pastie "${retval}" PARENT_SCOPE)
+	set(${out_prefix}_pastie_length "${retval_length}" PARENT_SCOPE)
+	unset(retval)
+	unset(retval_length)
+endfunction()
+
+load_and_strip_lua_script(${SOURCE_DIR}/retrieve_pastie.lua retrieve)
+load_and_strip_lua_script(${SOURCE_DIR}/save_pastie.lua save)
+load_and_strip_lua_script(${SOURCE_DIR}/add_highlighted_pastie.lua add_highlighted)
 
 set(lua_scripts_for_redis_content "//File autogenerated by cmake, changes will be lost
 #include <cstddef>
 namespace kamokan {
 extern const char g_save_script[] = R\"lua(${save_pastie})lua\";
 extern const char g_load_script[] = R\"lua(${retrieve_pastie})lua\";
+extern const char g_add_highlighted_script[] = R\"lua(${add_highlighted_pastie})lua\";
 extern const std::size_t g_save_script_size = ${save_pastie_length};
 extern const std::size_t g_load_script_size = ${retrieve_pastie_length};
+extern const std::size_t g_add_highlighted_script_size = ${add_highlighted_pastie_length};
 } //namespace kamokan
 ")
 
@@ -27,5 +34,7 @@ file(WRITE
 unset(lua_scripts_for_redis_content)
 unset(save_pastie_length)
 unset(retrieve_pastie_length)
+unset(add_highlighted_pastie_length)
 unset(save_pastie)
 unset(retrieve_pastie)
+unset(add_highlighted_pastie)

src/kamokan_impl/response.cpp

@@ -290,4 +290,7 @@ namespace kamokan {
 	std::string Response::default_pastie_lang() {
 		return std::string();
 	}
+
+	void Response::join() {
+	}
 } //namespace kamokan

src/kamokan_impl/response.hpp

@@ -45,6 +45,7 @@ namespace kamokan {
 
 		void send();
 		void set_app_start_time (const std::chrono::time_point<std::chrono::steady_clock>& parTime);
+		virtual void join();
 
 	protected:
 		Response (

src/kamokan_impl/storage.cpp

@@ -30,10 +30,12 @@
 #include <utility>
 #include <algorithm>
 
-#define TOKEN_PREFIX "kamokan:{store:}"
+#define KAMOKAN_TOKEN_PREFIX "kamokan:"
+#define TOKEN_PREFIX KAMOKAN_TOKEN_PREFIX "{store:}"
 
 namespace kamokan {
 	namespace {
+		const char g_hl_token_prefix[] = KAMOKAN_TOKEN_PREFIX "hl:";
 		const char g_token_prefix[] = TOKEN_PREFIX;
 
 		redis::IncRedis make_incredis (const SettingsBag& parSettings) {
@@ -83,6 +85,10 @@ namespace kamokan {
 			}
 			return true;
 		}
+
+		std::string make_regular_pastie_token (const boost::string_view& parToken) {
+			return std::string(g_token_prefix) + std::string(parToken);
+		}
 	} //unnamed namespace
 
 	Storage::RetrievedPastie::RetrievedPastie() :
@@ -191,7 +197,7 @@ namespace kamokan {
 
 		redis::Script retrieve = m_redis->command().make_script(string_view(g_load_script, g_load_script_size));
 		auto batch = m_redis->command().make_batch();
-		std::string token_with_prefix = std::string(g_token_prefix) + std::string(parToken);
+		std::string token_with_prefix = make_regular_pastie_token(parToken);
 		retrieve.run(batch, std::make_tuple(token_with_prefix), std::tuple<>());
 		auto raw_replies = batch.replies();
 		if (raw_replies.empty())
@@ -232,6 +238,28 @@ namespace kamokan {
 		return *m_settings;
 	}
 #endif
+
+	void Storage::submit_highlighted_pastie (
+		const boost::string_view& parToken,
+		const boost::string_view& parText,
+		const boost::string_view& parComment,
+		uint32_t parMaxTokenLen
+	) const {
+		using boost::string_view;
+
+		const bool valid_token = is_valid_token(parToken, parMaxTokenLen);
+		if (not valid_token)
+			return;
+
+		redis::Script retrieve = m_redis->command().make_script(string_view(g_add_highlighted_script, g_add_highlighted_script_size));
+		auto batch = m_redis->command().make_batch();
+		std::string token_with_prefix(make_regular_pastie_token(parToken));
+		retrieve.run(
+			batch,
+			std::make_tuple(token_with_prefix),
+			std::make_tuple(parText, parComment)
+		);
+	}
 } //namespace kamokan
 
 #undef TOKEN_PREFIX

src/kamokan_impl/storage.hpp

@@ -64,6 +64,12 @@ namespace kamokan {
 			bool parSelfDestruct,
 			const std::string& parRemoteIP
 		) const;
+		kamokan_virtual_testing void submit_highlighted_pastie (
+			const boost::string_view& parToken,
+			const boost::string_view& parText,
+			const boost::string_view& parComment,
+			uint32_t parMaxTokenLen
+		) const;
 
 		kamokan_virtual_testing RetrievedPastie retrieve_pastie (const boost::string_view& parToken, uint32_t parMaxTokenLen) const;
 

src/kamokan_impl/submit_paste_response.cpp

@@ -24,12 +24,14 @@
 #include "tawashi_exception.hpp"
 #include "ip_utils.hpp"
 #include "string_conv.hpp"
+#include "highlight_functions.hpp"
 #include <ciso646>
 #include <algorithm>
 #include <boost/lexical_cast.hpp>
 #include <cstdint>
 #include <spdlog/spdlog.h>
 #include <utility>
+#include <thread>
 
 namespace kamokan {
 	namespace {
@@ -76,6 +78,12 @@ namespace kamokan {
 		}
 	} //unnamed namespace
 
+	struct SubmitPasteResponse::LocalData {
+		std::string pastie_token;
+		boost::string_view pastie_lang;
+		std::thread submit_thread;
+	};
+
 #if defined(KAMOKAN_WITH_TESTING)
 	SubmitPasteResponse::SubmitPasteResponse (
 		const Kakoune::SafePtr<SettingsBag>& parSettings,
@@ -83,7 +91,8 @@ namespace kamokan {
 		const Kakoune::SafePtr<cgi::Env>& parCgiEnv,
 		bool parInitStorage
 	) :
-		Response(parSettings, parStreamOut, parCgiEnv, parInitStorage)
+		Response(parSettings, parStreamOut, parCgiEnv, parInitStorage),
+		m_local(std::make_unique<LocalData>())
 	{
 	}
 #endif
@@ -93,10 +102,13 @@ namespace kamokan {
 		std::ostream* parStreamOut,
 		const Kakoune::SafePtr<cgi::Env>& parCgiEnv
 	) :
-		Response(parSettings, parStreamOut, parCgiEnv, true)
+		Response(parSettings, parStreamOut, parCgiEnv, true),
+		m_local(std::make_unique<LocalData>())
 	{
 	}
 
+	SubmitPasteResponse::~SubmitPasteResponse() = default;
+
 	tawashi::HttpHeader SubmitPasteResponse::on_process() {
 		using tawashi::ErrorReasons;
 
@@ -111,7 +123,7 @@ namespace kamokan {
 		try {
 			auto& post = this->cgi_post();
 			pastie = get_value_from_post(post, make_string_view(g_post_key));
-			m_pastie_lang = get_value_from_post_log_failure(post, make_string_view(g_language_key));
+			m_local->pastie_lang = get_value_from_post_log_failure(post, make_string_view(g_language_key));
 			duration = get_value_from_post_log_failure(post, make_string_view(g_duration_key));
 			self_destruct = string_conv<bool>(get_value_from_post_log_failure(post, make_string_view(g_self_destruct)));
 		}
@@ -143,18 +155,18 @@ namespace kamokan {
 		//TODO: replace boost's lexical_cast with mine when I have some checks
 		//over invalid inputs
 		const uint32_t duration_int = std::max(std::min((duration.empty() ? 86400U : boost::lexical_cast<uint32_t>(duration)), 2628000U), 1U);
-		StringOrHeader submit_result = submit_to_storage(pastie, duration_int, m_pastie_lang, self_destruct);
+		StringOrHeader submit_result = submit_to_storage(pastie, duration_int, m_local->pastie_lang, self_destruct);
 		const boost::optional<std::string>& token = submit_result.first;
 
 		if (token) {
-			m_pastie_token = std::move(*token);
+			m_local->pastie_token = std::move(*token);
 			std::ostringstream oss;
-			oss << m_pastie_token;
-			if (not m_pastie_lang.empty())
-				oss << '?' << m_pastie_lang;
+			oss << m_local->pastie_token;
+			if (not m_local->pastie_lang.empty())
+				oss << '?' << m_local->pastie_lang;
 
 			std::string redirect = oss.str();
-			statuslog->info("Pastie token=\"{}\" redirect=\"{}\"", m_pastie_token, redirect);
+			statuslog->info("Pastie token=\"{}\" redirect=\"{}\"", m_local->pastie_token, redirect);
 
 			if (self_destruct)
 				return tawashi::make_header_type_html();
@@ -173,13 +185,27 @@ namespace kamokan {
 		const boost::string_view& parLang,
 		bool parSelfDestruct
 	) -> StringOrHeader {
+		//Send to the storage
 		auto& storage = this->storage();
 		std::string remote_ip = tawashi::guess_real_remote_ip(cgi_env());
 		Storage::SubmissionResult submission_res = storage.submit_pastie(parText, parExpiry, parLang, parSelfDestruct, remote_ip);
-		if (not submission_res.error)
+
+		if (not submission_res.error) {
+			//if data was submitted successfully, start a separate thread to do
+			//the syntax highlighting and upload that to the storage asynchronously
+			//since it's likely to take a long time
+			assert(not m_local->submit_thread.joinable());
+			std::string lang(parLang);
+			std::string text(parText);
+			m_local->submit_thread = std::thread([&,text,token=submission_res.token,lang]() mutable {
+				this->store_highlighted_pastie(token, std::move(text), lang);
+			});
+
 			return std::make_pair(boost::make_optional(std::move(submission_res.token)), tawashi::HttpHeader());
-		else
+		}
+		else {
 			return std::make_pair(boost::optional<std::string>(), make_error_redirect(*submission_res.error));
+		}
 	}
 
 	tawashi::HttpHeader SubmitPasteResponse::make_success_response (std::string&& parPastieParam) {
@@ -188,7 +214,29 @@ namespace kamokan {
 	}
 
 	void SubmitPasteResponse::on_mustache_prepare (mstch::map& parContext) {
-		parContext["pastie_token"] = std::move(m_pastie_token);
-		parContext["pastie_lang"] = std::move(m_pastie_lang);
+		parContext["pastie_token"] = std::move(m_local->pastie_token);
+		parContext["pastie_lang"] = std::move(m_local->pastie_lang);
+	}
+
+	void SubmitPasteResponse::join() {
+		if (m_local->submit_thread.joinable()) {
+			m_local->submit_thread.join();
+		}
+	}
+
+	void SubmitPasteResponse::store_highlighted_pastie (boost::string_view parToken, std::string&& parText, boost::string_view parLang) {
+		if (parLang.empty() or parLang == "colourless")
+			return;
+
+		SplitHighlightedPastie highlighted = highlight_string(std::move(parText), parLang, settings());
+		if (highlighted.comment.empty() or highlighted.text.empty())
+			return;
+
+		storage().submit_highlighted_pastie(
+			parToken,
+			highlighted.text,
+			highlighted.comment,
+			settings().as<uint32_t>("max_token_length")
+		);
 	}
 } //namespace kamokan

src/kamokan_impl/submit_paste_response.hpp

@@ -23,6 +23,7 @@
 #include <boost/utility/string_view.hpp>
 #include <cassert>
 #include <utility>
+#include <memory>
 
 namespace kamokan {
 	class SubmitPasteResponse : public Response {
@@ -41,6 +42,9 @@ namespace kamokan {
 			std::ostream* parStreamOut,
 			const Kakoune::SafePtr<cgi::Env>& parCgiEnv
 		);
+		~SubmitPasteResponse();
+
+		virtual void join() override;
 
 	protected:
 		virtual boost::string_view page_basename() const override { return boost::string_view("saved"); }
@@ -48,6 +52,7 @@ namespace kamokan {
 
 	private:
 		typedef std::pair<boost::optional<std::string>, tawashi::HttpHeader> StringOrHeader;
+		struct LocalData;
 
 		virtual tawashi::HttpHeader on_process() override;
 		virtual void on_mustache_prepare (mstch::map& parContext) override;
@@ -57,8 +62,8 @@ namespace kamokan {
 			const boost::string_view& parLang,
 			bool parSelfDestruct
 		);
+		void store_highlighted_pastie (boost::string_view parToken, std::string&& parText, boost::string_view parLang);
 
-		std::string m_pastie_token;
-		boost::string_view m_pastie_lang;
+		std::unique_ptr<LocalData> m_local;
 	};
 } //namespace kamokan