diff --git a/src/kamokan/main.cpp b/src/kamokan/main.cpp index bc18543..d1db81b 100644 --- a/src/kamokan/main.cpp +++ b/src/kamokan/main.cpp @@ -178,8 +178,10 @@ int main (int parArgc, char* parArgv[], char* parEnvp[]) { tawashi::cgi::drop_arguments(cgi_env->request_uri_relative()), cgi_env->request_method() ); - if (response) + if (response) { response->send(); + response->join(); + } } catch (const std::exception& e) { statuslog->critical("Uncaught exception in main(): \"{}\"", e.what()); diff --git a/src/kamokan_impl/CMakeLists.txt b/src/kamokan_impl/CMakeLists.txt index db7200c..e7bc17c 100644 --- a/src/kamokan_impl/CMakeLists.txt +++ b/src/kamokan_impl/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries(${PROJECT_NAME} PUBLIC mstch PRIVATE pthread PUBLIC tawashi + PRIVATE pthread ) target_compile_definitions(${PROJECT_NAME} PRIVATE BOOST_SPIRIT_USE_PHOENIX_V3=1 @@ -67,6 +68,9 @@ configure_file( add_custom_command( OUTPUT include/lua_scripts_for_redis.cpp COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}" -P ${CMAKE_CURRENT_SOURCE_DIR}/lua_to_cpp.cmake - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/retrieve_pastie.lua ${CMAKE_CURRENT_SOURCE_DIR}/save_pastie.lua + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/retrieve_pastie.lua + ${CMAKE_CURRENT_SOURCE_DIR}/save_pastie.lua + ${CMAKE_CURRENT_SOURCE_DIR}/add_highlighted_pastie.lua COMMENT "Embedding save/load lua scripts into the c++ code" ) diff --git a/src/kamokan_impl/add_highlighted_pastie.lua b/src/kamokan_impl/add_highlighted_pastie.lua new file mode 100644 index 0000000..c97aafd --- /dev/null +++ b/src/kamokan_impl/add_highlighted_pastie.lua @@ -0,0 +1,13 @@ +local full_token = KEYS[1] +local highlighted_text = ARGV[1] +local comment = ARGV[2] + +local saved = redis.call("HMSET", full_token, + "hl_pastie", highlighted_text, + "hl_comment", comment +) + +if not saved then + return redis.error_reply("PastieNotSaved") +end +return "1" diff --git a/src/kamokan_impl/highlight_functions.cpp b/src/kamokan_impl/highlight_functions.cpp index c8344b3..772fe31 100644 --- a/src/kamokan_impl/highlight_functions.cpp +++ b/src/kamokan_impl/highlight_functions.cpp @@ -31,13 +31,13 @@ namespace kamokan { return parSettings.as("highlight_css"); } - std::string lang_name_to_file_path (const std::string& parLang, const std::string& parLangmapDir) { + std::string lang_name_to_file_path (boost::string_view parLang, const std::string& parLangmapDir) { if (parLang.empty()) return std::string(); srchilite::LangMap lang_map(parLangmapDir, "lang.map"); lang_map.open(); - std::string lang_file = lang_map.getFileName(parLang); + std::string lang_file = lang_map.getFileName(std::string(parLang)); if (lang_file.empty()) lang_file = "default.lang"; @@ -89,7 +89,7 @@ namespace kamokan { return boost::copy_range(lang_range | boost::adaptors::map_keys); } - SplitHighlightedPastie highlight_string (std::string&& parIn, const std::string& parLang, const SettingsBag& parSettings) { + SplitHighlightedPastie highlight_string (std::string&& parIn, boost::string_view parLang, const SettingsBag& parSettings) { const std::string langmap_dir = parSettings.as("langmap_dir"); srchilite::SourceHighlight highlighter; highlighter.setDataDir(langmap_dir); @@ -105,10 +105,16 @@ namespace kamokan { highlighter.setStyleCssFile(highlight_css_path(parSettings)); highlighter.setGenerateLineNumbers(false); - std::istringstream iss(std::move(parIn)); - std::ostringstream oss; - highlighter.highlight(iss, oss, lang_name_to_file_path(parLang, langmap_dir)); - return strip_tags_from_highlighted(oss.str()); + std::string lang_file_path = lang_name_to_file_path(parLang, langmap_dir); + if (not lang_file_path.empty()) { + std::istringstream iss(std::move(parIn)); + std::ostringstream oss; + highlighter.highlight(iss, oss, lang_file_path); + return strip_tags_from_highlighted(oss.str()); + } + else { + return SplitHighlightedPastie(); + } } } //namespace kamokan diff --git a/src/kamokan_impl/highlight_functions.hpp b/src/kamokan_impl/highlight_functions.hpp index 1eee70c..4307f21 100644 --- a/src/kamokan_impl/highlight_functions.hpp +++ b/src/kamokan_impl/highlight_functions.hpp @@ -32,5 +32,5 @@ namespace kamokan { }; HighlightLangList list_highlight_langs (const SettingsBag& parSettings); - SplitHighlightedPastie highlight_string (std::string&& parIn, const std::string& parLang, const SettingsBag& parSettings); + SplitHighlightedPastie highlight_string (std::string&& parIn, boost::string_view parLang, const SettingsBag& parSettings); } //namespace kamokan diff --git a/src/kamokan_impl/lua_scripts_for_redis.hpp b/src/kamokan_impl/lua_scripts_for_redis.hpp index c2c2cd3..01f9123 100644 --- a/src/kamokan_impl/lua_scripts_for_redis.hpp +++ b/src/kamokan_impl/lua_scripts_for_redis.hpp @@ -22,6 +22,8 @@ namespace kamokan { extern const char g_save_script[]; extern const char g_load_script[]; + extern const char g_add_highlighted_script[]; extern const std::size_t g_save_script_size; extern const std::size_t g_load_script_size; + extern const std::size_t g_add_highlighted_script_size; } //namespace kamokan diff --git a/src/kamokan_impl/lua_to_cpp.cmake b/src/kamokan_impl/lua_to_cpp.cmake index e921625..40f0a43 100644 --- a/src/kamokan_impl/lua_to_cpp.cmake +++ b/src/kamokan_impl/lua_to_cpp.cmake @@ -1,21 +1,28 @@ -file(READ ${SOURCE_DIR}/retrieve_pastie.lua retrieve_pastie) -file(READ ${SOURCE_DIR}/save_pastie.lua save_pastie) -string(REGEX REPLACE "[ \t]*=[ \t]*" "=" retrieve_pastie "${retrieve_pastie}") -string(REGEX REPLACE "[ \t]*=[ \t]*" "=" save_pastie "${save_pastie}") -string(REGEX REPLACE ",[ \t]*" "," retrieve_pastie "${retrieve_pastie}") -string(REGEX REPLACE ",[ \t]*" "," save_pastie "${save_pastie}") -string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" retrieve_pastie "${retrieve_pastie}") -string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" save_pastie "${save_pastie}") -string(LENGTH "${retrieve_pastie}" retrieve_pastie_length) -string(LENGTH "${save_pastie}" save_pastie_length) +function (load_and_strip_lua_script file_path out_prefix) + file(READ ${file_path} retval) + string(REGEX REPLACE "[ \t]*=[ \t]*" "=" retval "${retval}") + string(REGEX REPLACE ",[ \t]*" "," retval "${retval}") + string(REGEX REPLACE "(^|\n)[ \t]+" "\\1" retval "${retval}") + string(LENGTH "${retval}" retval_length) + set(${out_prefix}_pastie "${retval}" PARENT_SCOPE) + set(${out_prefix}_pastie_length "${retval_length}" PARENT_SCOPE) + unset(retval) + unset(retval_length) +endfunction() + +load_and_strip_lua_script(${SOURCE_DIR}/retrieve_pastie.lua retrieve) +load_and_strip_lua_script(${SOURCE_DIR}/save_pastie.lua save) +load_and_strip_lua_script(${SOURCE_DIR}/add_highlighted_pastie.lua add_highlighted) set(lua_scripts_for_redis_content "//File autogenerated by cmake, changes will be lost #include namespace kamokan { extern const char g_save_script[] = R\"lua(${save_pastie})lua\"; extern const char g_load_script[] = R\"lua(${retrieve_pastie})lua\"; +extern const char g_add_highlighted_script[] = R\"lua(${add_highlighted_pastie})lua\"; extern const std::size_t g_save_script_size = ${save_pastie_length}; extern const std::size_t g_load_script_size = ${retrieve_pastie_length}; +extern const std::size_t g_add_highlighted_script_size = ${add_highlighted_pastie_length}; } //namespace kamokan ") @@ -27,5 +34,7 @@ file(WRITE unset(lua_scripts_for_redis_content) unset(save_pastie_length) unset(retrieve_pastie_length) +unset(add_highlighted_pastie_length) unset(save_pastie) unset(retrieve_pastie) +unset(add_highlighted_pastie) diff --git a/src/kamokan_impl/response.cpp b/src/kamokan_impl/response.cpp index 2f3f036..c2a86a7 100644 --- a/src/kamokan_impl/response.cpp +++ b/src/kamokan_impl/response.cpp @@ -290,4 +290,7 @@ namespace kamokan { std::string Response::default_pastie_lang() { return std::string(); } + + void Response::join() { + } } //namespace kamokan diff --git a/src/kamokan_impl/response.hpp b/src/kamokan_impl/response.hpp index c19bc18..3c780be 100644 --- a/src/kamokan_impl/response.hpp +++ b/src/kamokan_impl/response.hpp @@ -45,6 +45,7 @@ namespace kamokan { void send(); void set_app_start_time (const std::chrono::time_point& parTime); + virtual void join(); protected: Response ( diff --git a/src/kamokan_impl/storage.cpp b/src/kamokan_impl/storage.cpp index 7df414f..c93c39d 100644 --- a/src/kamokan_impl/storage.cpp +++ b/src/kamokan_impl/storage.cpp @@ -30,10 +30,12 @@ #include #include -#define TOKEN_PREFIX "kamokan:{store:}" +#define KAMOKAN_TOKEN_PREFIX "kamokan:" +#define TOKEN_PREFIX KAMOKAN_TOKEN_PREFIX "{store:}" namespace kamokan { namespace { + const char g_hl_token_prefix[] = KAMOKAN_TOKEN_PREFIX "hl:"; const char g_token_prefix[] = TOKEN_PREFIX; redis::IncRedis make_incredis (const SettingsBag& parSettings) { @@ -83,6 +85,10 @@ namespace kamokan { } return true; } + + std::string make_regular_pastie_token (const boost::string_view& parToken) { + return std::string(g_token_prefix) + std::string(parToken); + } } //unnamed namespace Storage::RetrievedPastie::RetrievedPastie() : @@ -191,7 +197,7 @@ namespace kamokan { redis::Script retrieve = m_redis->command().make_script(string_view(g_load_script, g_load_script_size)); auto batch = m_redis->command().make_batch(); - std::string token_with_prefix = std::string(g_token_prefix) + std::string(parToken); + std::string token_with_prefix = make_regular_pastie_token(parToken); retrieve.run(batch, std::make_tuple(token_with_prefix), std::tuple<>()); auto raw_replies = batch.replies(); if (raw_replies.empty()) @@ -232,6 +238,28 @@ namespace kamokan { return *m_settings; } #endif + + void Storage::submit_highlighted_pastie ( + const boost::string_view& parToken, + const boost::string_view& parText, + const boost::string_view& parComment, + uint32_t parMaxTokenLen + ) const { + using boost::string_view; + + const bool valid_token = is_valid_token(parToken, parMaxTokenLen); + if (not valid_token) + return; + + redis::Script retrieve = m_redis->command().make_script(string_view(g_add_highlighted_script, g_add_highlighted_script_size)); + auto batch = m_redis->command().make_batch(); + std::string token_with_prefix(make_regular_pastie_token(parToken)); + retrieve.run( + batch, + std::make_tuple(token_with_prefix), + std::make_tuple(parText, parComment) + ); + } } //namespace kamokan #undef TOKEN_PREFIX diff --git a/src/kamokan_impl/storage.hpp b/src/kamokan_impl/storage.hpp index dadf905..e04dc31 100644 --- a/src/kamokan_impl/storage.hpp +++ b/src/kamokan_impl/storage.hpp @@ -64,6 +64,12 @@ namespace kamokan { bool parSelfDestruct, const std::string& parRemoteIP ) const; + kamokan_virtual_testing void submit_highlighted_pastie ( + const boost::string_view& parToken, + const boost::string_view& parText, + const boost::string_view& parComment, + uint32_t parMaxTokenLen + ) const; kamokan_virtual_testing RetrievedPastie retrieve_pastie (const boost::string_view& parToken, uint32_t parMaxTokenLen) const; diff --git a/src/kamokan_impl/submit_paste_response.cpp b/src/kamokan_impl/submit_paste_response.cpp index e27ee77..06aa7fe 100644 --- a/src/kamokan_impl/submit_paste_response.cpp +++ b/src/kamokan_impl/submit_paste_response.cpp @@ -24,12 +24,14 @@ #include "tawashi_exception.hpp" #include "ip_utils.hpp" #include "string_conv.hpp" +#include "highlight_functions.hpp" #include #include #include #include #include #include +#include namespace kamokan { namespace { @@ -76,6 +78,12 @@ namespace kamokan { } } //unnamed namespace + struct SubmitPasteResponse::LocalData { + std::string pastie_token; + boost::string_view pastie_lang; + std::thread submit_thread; + }; + #if defined(KAMOKAN_WITH_TESTING) SubmitPasteResponse::SubmitPasteResponse ( const Kakoune::SafePtr& parSettings, @@ -83,7 +91,8 @@ namespace kamokan { const Kakoune::SafePtr& parCgiEnv, bool parInitStorage ) : - Response(parSettings, parStreamOut, parCgiEnv, parInitStorage) + Response(parSettings, parStreamOut, parCgiEnv, parInitStorage), + m_local(std::make_unique()) { } #endif @@ -93,10 +102,13 @@ namespace kamokan { std::ostream* parStreamOut, const Kakoune::SafePtr& parCgiEnv ) : - Response(parSettings, parStreamOut, parCgiEnv, true) + Response(parSettings, parStreamOut, parCgiEnv, true), + m_local(std::make_unique()) { } + SubmitPasteResponse::~SubmitPasteResponse() = default; + tawashi::HttpHeader SubmitPasteResponse::on_process() { using tawashi::ErrorReasons; @@ -111,7 +123,7 @@ namespace kamokan { try { auto& post = this->cgi_post(); pastie = get_value_from_post(post, make_string_view(g_post_key)); - m_pastie_lang = get_value_from_post_log_failure(post, make_string_view(g_language_key)); + m_local->pastie_lang = get_value_from_post_log_failure(post, make_string_view(g_language_key)); duration = get_value_from_post_log_failure(post, make_string_view(g_duration_key)); self_destruct = string_conv(get_value_from_post_log_failure(post, make_string_view(g_self_destruct))); } @@ -143,18 +155,18 @@ namespace kamokan { //TODO: replace boost's lexical_cast with mine when I have some checks //over invalid inputs const uint32_t duration_int = std::max(std::min((duration.empty() ? 86400U : boost::lexical_cast(duration)), 2628000U), 1U); - StringOrHeader submit_result = submit_to_storage(pastie, duration_int, m_pastie_lang, self_destruct); + StringOrHeader submit_result = submit_to_storage(pastie, duration_int, m_local->pastie_lang, self_destruct); const boost::optional& token = submit_result.first; if (token) { - m_pastie_token = std::move(*token); + m_local->pastie_token = std::move(*token); std::ostringstream oss; - oss << m_pastie_token; - if (not m_pastie_lang.empty()) - oss << '?' << m_pastie_lang; + oss << m_local->pastie_token; + if (not m_local->pastie_lang.empty()) + oss << '?' << m_local->pastie_lang; std::string redirect = oss.str(); - statuslog->info("Pastie token=\"{}\" redirect=\"{}\"", m_pastie_token, redirect); + statuslog->info("Pastie token=\"{}\" redirect=\"{}\"", m_local->pastie_token, redirect); if (self_destruct) return tawashi::make_header_type_html(); @@ -173,13 +185,27 @@ namespace kamokan { const boost::string_view& parLang, bool parSelfDestruct ) -> StringOrHeader { + //Send to the storage auto& storage = this->storage(); std::string remote_ip = tawashi::guess_real_remote_ip(cgi_env()); Storage::SubmissionResult submission_res = storage.submit_pastie(parText, parExpiry, parLang, parSelfDestruct, remote_ip); - if (not submission_res.error) + + if (not submission_res.error) { + //if data was submitted successfully, start a separate thread to do + //the syntax highlighting and upload that to the storage asynchronously + //since it's likely to take a long time + assert(not m_local->submit_thread.joinable()); + std::string lang(parLang); + std::string text(parText); + m_local->submit_thread = std::thread([&,text,token=submission_res.token,lang]() mutable { + this->store_highlighted_pastie(token, std::move(text), lang); + }); + return std::make_pair(boost::make_optional(std::move(submission_res.token)), tawashi::HttpHeader()); - else + } + else { return std::make_pair(boost::optional(), make_error_redirect(*submission_res.error)); + } } tawashi::HttpHeader SubmitPasteResponse::make_success_response (std::string&& parPastieParam) { @@ -188,7 +214,29 @@ namespace kamokan { } void SubmitPasteResponse::on_mustache_prepare (mstch::map& parContext) { - parContext["pastie_token"] = std::move(m_pastie_token); - parContext["pastie_lang"] = std::move(m_pastie_lang); + parContext["pastie_token"] = std::move(m_local->pastie_token); + parContext["pastie_lang"] = std::move(m_local->pastie_lang); + } + + void SubmitPasteResponse::join() { + if (m_local->submit_thread.joinable()) { + m_local->submit_thread.join(); + } + } + + void SubmitPasteResponse::store_highlighted_pastie (boost::string_view parToken, std::string&& parText, boost::string_view parLang) { + if (parLang.empty() or parLang == "colourless") + return; + + SplitHighlightedPastie highlighted = highlight_string(std::move(parText), parLang, settings()); + if (highlighted.comment.empty() or highlighted.text.empty()) + return; + + storage().submit_highlighted_pastie( + parToken, + highlighted.text, + highlighted.comment, + settings().as("max_token_length") + ); } } //namespace kamokan diff --git a/src/kamokan_impl/submit_paste_response.hpp b/src/kamokan_impl/submit_paste_response.hpp index f2193df..3813e62 100644 --- a/src/kamokan_impl/submit_paste_response.hpp +++ b/src/kamokan_impl/submit_paste_response.hpp @@ -23,6 +23,7 @@ #include #include #include +#include namespace kamokan { class SubmitPasteResponse : public Response { @@ -41,6 +42,9 @@ namespace kamokan { std::ostream* parStreamOut, const Kakoune::SafePtr& parCgiEnv ); + ~SubmitPasteResponse(); + + virtual void join() override; protected: virtual boost::string_view page_basename() const override { return boost::string_view("saved"); } @@ -48,6 +52,7 @@ namespace kamokan { private: typedef std::pair, tawashi::HttpHeader> StringOrHeader; + struct LocalData; virtual tawashi::HttpHeader on_process() override; virtual void on_mustache_prepare (mstch::map& parContext) override; @@ -57,8 +62,8 @@ namespace kamokan { const boost::string_view& parLang, bool parSelfDestruct ); + void store_highlighted_pastie (boost::string_view parToken, std::string&& parText, boost::string_view parLang); - std::string m_pastie_token; - boost::string_view m_pastie_lang; + std::unique_ptr m_local; }; } //namespace kamokan