King_DuckZ/utfcpp
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Removed std::distance from validate_next and (hopefully) made it work with input iterators. Also, did a major

Browse source 
refactoring of that function.


git-svn-id: http://svn.code.sf.net/p/utfcpp/code@91 a809a056-fc17-0410-9590-b4f493f8b08e
This commit is contained in:
ntrifunovic 2009-07-05 00:09:18 +00:00
parent 74be521392
commit e2799bdab6
1 changed files with 167 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

260
v2_0/source/utf8/core.h
View file

@ -100,118 +100,192 @@ namespace internal
return 0; return 0;
} }
bool is_overlong_sequence(uint32_t cp, int length)
{
if (cp < 0x80) {
if (length != 1)
return true;
}
else if (cp < 0x800) {
if (length != 2)
return true;
}
else if (cp < 0x10000) {
if (length != 3)
return true;
}
return false;
}
enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
/// get_sequence_x functions decode utf-8 sequences of the length x
template <typename octet_iterator>
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t* code_point)
{
if (it != end) {
if (code_point)
*code_point = mask8(*it);
return UTF8_OK;
}
return NOT_ENOUGH_ROOM;
}
template <typename octet_iterator>
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t* code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;
if (it != end) {
uint32_t cp = mask8(*it);
if (++it != end) {
if (is_trail(*it)) {
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
return ret_code;
}
template <typename octet_iterator>
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t* code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;
if (it != end) {
uint32_t cp = mask8(*it);
if (++it != end) {
if (is_trail(*it)) {
cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff);
if (++it != end) {
if (is_trail(*it)) {
cp += (*it) & 0x3f;
if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
return ret_code;
}
template <typename octet_iterator>
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t* code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;
if (it != end) {
uint32_t cp = mask8(*it);
if (++it != end) {
if (is_trail(*it)) {
cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff);
if (++it != end) {
if (is_trail(*it)) {
cp += (mask8(*it) << 6) & 0xfff;
if (++it != end) {
if (is_trail(*it)) {
cp += (*it) & 0x3f;
if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
return ret_code;
}
template <typename octet_iterator> template <typename octet_iterator>
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point) utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point)
{ {
// Save the original value of it so we can go back in case of failure
// Of course, it does not make much sense with i.e. stream iterators
octet_iterator original_it = it; octet_iterator original_it = it;
uint32_t cp = mask8(*it);
// Check the lead octet uint32_t cp = 0;
// Determine the sequence length based on the lead octet
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type; typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
octet_difference_type length = sequence_length(it); octet_difference_type length = sequence_length(it);
if (length == 0)
return INVALID_LEAD;
// "Shortcut" for ASCII characters // Now that we have a valid sequence length, get trail octets and calculate the code point
if (length == 1) { utf_error err = UTF8_OK;
if (std::distance(it, end) > 0) {
if (code_point)
*code_point = cp;
++it;
return UTF8_OK;
}
else
return NOT_ENOUGH_ROOM;
}
// Do we have enough memory?
if (std::distance(it, end) < length)
return NOT_ENOUGH_ROOM;
// Check trail octets and calculate the code point
switch (length) { switch (length) {
case 0: case 1:
return INVALID_LEAD; err = get_sequence_1(it, end, &cp);
break; break;
case 2: case 2:
if (is_trail(*(++it))) { err = get_sequence_2(it, end, &cp);
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
break; break;
case 3: case 3:
if (is_trail(*(++it))) { err = get_sequence_3(it, end, &cp);
cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff);
if (is_trail(*(++it))) {
cp += (*it) & 0x3f;
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
break; break;
case 4: case 4:
if (is_trail(*(++it))) { err = get_sequence_4(it, end, &cp);
cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff);
if (is_trail(*(++it))) {
cp += (mask8(*it) << 6) & 0xfff;
if (is_trail(*(++it))) {
cp += (*it) & 0x3f;
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
}
else {
it = original_it;
return INCOMPLETE_SEQUENCE;
}
break; break;
} }
// Is the code point valid?
if (!is_code_point_valid(cp)) { if (err == UTF8_OK) {
for (octet_difference_type i = 0; i < length - 1; ++i) // Decoding suceeded. Now, security checks...
it = original_it; if (is_code_point_valid(cp)) {
return INVALID_CODE_POINT; if (!is_overlong_sequence(cp, length)){
// Passed! Return here.
if (code_point)
*code_point = cp;
++it;
return UTF8_OK;
}
else
err = OVERLONG_SEQUENCE;
}
else
err = INVALID_CODE_POINT;
} }
if (code_point) // Failure branch - restore the original value of the iterator
*code_point = cp; it = original_it;
return err;
if (cp < 0x80) {
if (length != 1) {
it = original_it;
return OVERLONG_SEQUENCE;
}
}
else if (cp < 0x800) {
if (length != 2) {
it = original_it;
return OVERLONG_SEQUENCE;
}
}
else if (cp < 0x10000) {
if (length != 3) {
it = original_it;
return OVERLONG_SEQUENCE;
}
}
++it;
return UTF8_OK;
} }
template <typename octet_iterator> template <typename octet_iterator>