King_DuckZ/kamokan
1
0
Fork 0
mirror of https://github.com/KingDuckZ/kamokan.git synced 2024-12-27 21:35:41 +00:00
Code Issues Releases Wiki Activity

Replace nullchars with #.

Browse source 
There is also a glib-based implementation but I'm not
sure about it yet.
This commit is contained in:
King_DuckZ 2017-05-10 20:45:26 +01:00
parent ed1343bd7b
commit 13e46ab1e6
1 changed files with 35 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/tawashi_implem/sanitized_utf8.cpp
View file

@ -16,14 +16,47 @@
*/
#include "sanitized_utf8.hpp"
#include "utf8.h"
#include <iterator>
#define SANITIZE_WITH_UTFCPP
#if defined(SANITIZE_WITH_UTFCPP)
# include "utf8.h"
# include <algorithm>
#else
# include <glib.h>
# include <cassert>
# include <array>
#endif
namespace tawashi {
std::string sanitized_utf8 (const boost::string_ref& parStr) {
std::string sanitized;
sanitized.reserve(parStr.size());
utf8::replace_invalid(parStr.begin(), parStr.end(), std::back_inserter(sanitized));
#if defined(SANITIZE_WITH_UTFCPP)
utf8::replace_invalid(parStr.begin(), parStr.end(), std::back_inserter(sanitized), 0xFFFD);
std::replace(sanitized.begin(), sanitized.end(), '\0', '#');
#else
# error "untested code, don't enable in final builds"
std::array<char, 6> replacement;
const int replacement_len = g_unichar_to_utf8(0xFFFD, replacement.data());
std::size_t beg_offset = 0;
const char* end;
while (not g_utf8_validate(parStr.data() + beg_offset, parStr.size() - beg_offset, &end)) {
assert(beg_offset < parStr.size());
const std::size_t valid_chunk_size = end - (parStr.data() + beg_offset);
sanitized.append(parStr.data() + beg_offset, end);
if (*end)
sanitized.append(replacement.data(), replacement_len);
else
sanitized.append({ '#' });
beg_offset += valid_chunk_size + 1;
assert(beg_offset <= parStr.size());
}
sanitized.append(parStr.data() + beg_offset, end);
assert(g_utf8_validate(sanitized.data(), sanitized.size(), nullptr));
#endif
return sanitized;
}
} //namespace tawashi