commit ac292859544459985f958e68b62cd726d42f51f5 Author: ntrifunovic Date: Fri Jul 7 11:48:37 2006 +0000 Initial import of the project tree git-svn-id: http://svn.code.sf.net/p/utfcpp/code@1 a809a056-fc17-0410-9590-b4f493f8b08e diff --git a/doc/.utf8cpp.html.html.swp b/doc/.utf8cpp.html.html.swp new file mode 100644 index 0000000..d4d7961 Binary files /dev/null and b/doc/.utf8cpp.html.html.swp differ diff --git a/doc/err.txt b/doc/err.txt new file mode 100644 index 0000000..0dbdfb2 --- /dev/null +++ b/doc/err.txt @@ -0,0 +1,10 @@ +line 2 column 1 - Warning: missing declaration +line 2 column 1 - Warning: inserting missing 'title' element +line 182 column 1 - Warning: trimming empty

+Info: Document content looks like HTML 3.2 +3 warnings, 0 errors were found! + +To learn more about HTML Tidy see http://tidy.sourceforge.net +Please send bug reports to html-tidy@w3.org +HTML and CSS specifications are available from http://www.w3.org/ +Lobby your company to join W3C, see http://www.w3.org/Consortium diff --git a/doc/utf8cpp.html b/doc/utf8cpp.html new file mode 100644 index 0000000..7354353 --- /dev/null +++ b/doc/utf8cpp.html @@ -0,0 +1,692 @@ + + + + + + + +

Introduction

Many C++ developers miss an easy and portable way of handling +Unicode encoded strings. C++ Standard is currently Unicode +agnostic, and while some work is being done to introduce Unicode to +the next incarnation called C++0x, for the moment nothing of the +sort is available. In the meantime, developers use 3rd party +libraries like ICU, OS specific capabilities, or simply roll out +their own solutions.

In order to easily handle UTF-8 encoded Unicode strings, I have +come up with a set of template functions. For anybody used to work +with STL algorithms, they should be easy and natural to use. The +code is freely available for any purpose - check out the license at +the beginning of the utf8.h file. Be aware, though, that while I +did some testing, this library has not been used in production yet. +If you run into bugs or performance issues, please let me know and +I'll do my best to address them.

The purpose of this article is not to offer an introduction to +Unicode in general, and UTF-8 in particular. If you are not +familiar with Unicode, be sure to check out Unicode Home Page or some other +source of information for Unicode. Also, it is not my aim to +advocate the use of UTF-8 encoded strings in C++ programs; if you +want to handle UTF-8 encoded strings from C++, I am sure you have +good reasons for it.

Examples of use

To illustrate the use of this utf8 library, we shall open a file +containing a line of UTF-8 encoded text, read the line into +std::string, convert the text to UTF-16, and write it +to another file:

+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+using namespace std;
+
+int main()
+{
+    // Open the file with a utf-8 encoded line of text in it
+    ifstream fs8("utf8.txt");
+    if (!fs8.is_open()) {
+        cout << "Could not open utf8.txt" << endl;
+        return 0;
+    }
+
+    // is there a utf8 marker? if yes, skip it.
+    fs8.seekg(0, ios::end);
+    ifstream::pos_type file_length = fs8.tellg();
+    fs8.seekg(0, ios::beg);
+    if (file_length > 3) {
+        char bom[3];
+        fs8.read(bom, 3);
+        if (!utf8::is_bom(bom))
+            fs8.seekg(0, ios::beg);        
+    }
+
+    // Read the line from the file
+    string text8;
+    getline(fs8, text8);
+
+    // Make sure it is valid utf-8
+    if (!utf8::is_valid(text8.begin(), text8.end())) {
+        cout << "Invalid utf-8 text";
+        return 0;
+    }
+
+    // Convert the text to utf-16
+    vector<unsigned short> text16;
+    text16.push_back(0xfeff); // bom
+    utf8::utf8to16(text8.begin(), text8.end(), back_inserter(text16));
+
+    // Create  the file for writing the utf-16 string
+    ofstream fs16("utf16.txt", ios_base::out | ios_base::binary);
+    if (!fs16.is_open()) {
+        cout << "Could not open utf16.txt" << endl;
+        return 0;
+    }
+    
+    // Write the utf16 text to the file
+    fs16.write(reinterpret_cast<const char*>(&text16[0]), text16.size() * sizeof (unsigned short));
+}
+

In the previous code sample, we have seen the use of 3 functions +from utf8 namespace: first we used is_bom +function to detect UTF-8 byte order mark at the beginning of the +file, then is_valid to make sure that the text we +loaded is valid UTF-8, and finally utf8to16 to convert +the text to UTF-16 encoding. Note that the use of +is_valid was optional in this case; +utf8to16 throws an exception in case of invalid UTF-8 +text.

Reference

Functions From utf8 Namespace

utf8::append

Encodes a 32 bit code point as a UTF-8 sequence of octets and +appends the sequence to a UTF-8 string.

template <typename octet_iterator> octet_iterator
+append(uint32_t cp, octet_iterator result);

cp: A 32 bit integer representing a code point to +append to the sequence.
+result: An output iterator to the place in the +sequence where to append the code point.
+Return value: An iterator pointing to the place after the +newly appended sequence.

Example of use:

+unsigned char u[5] = {0,0,0,0,0};
+
+unsigned char* end = append(0x0448, u);
+
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+

Note that append does not allocate any memory - it +is the burden of the caller to make sure there is enough memory +allocated for the operation. To make things more interesting, +append can add anywhere between 1 and 4 octets to the +sequence. In practice, you would most often want to use +std::back_inserter to ensure that the necessary memory +is allocated.

In case of an invalid code point, a +utf8::invalid_code_point exception is thrown.

utf8::next

Given the iterator to the beginning of the UTF-8 sequence, it +returns the code point and moves the iterator to the next +position.

template <typename octet_iterator> uint32_t
+next(octet_iterator& it, octet_iterator end);

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the beginning of the next +code point.
+end: end of the UTF-8 sequence to be processed. If +it gets equal to end during the +extraction of a code point, an utf8::not_enough_room +exception is thrown.
+Return value: the 32 bit representation of the processed +UTF-8 code point.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+int cp = next(w, twochars + 6);
+
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+

This function is typically used to iterate through a UTF-8 +encoded string.

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown.

utf8::previous

Given a reference to an iterator pointing to an octet in a UTF-8 +seqence, it decreases the iterator until it hits the beginning of +the previous UTF-8 encoded code point and returns the 32 bits +representation of the code point.

template <typename octet_iterator> uint32_t
+previous(octet_iterator& it, octet_iterator pass_start);

it: a reference pointing to an octet within a UTF-8 +encoded string. After the function returns, it is decremented to +point to the beginning of the previous code point.
+pass_start: an iterator to the point in the sequence +where the search for the beginning of a code point is aborted if no +result was reached. It is a safety measure to prevent passing the +beginning of the string in the search for a UTF-8 lead octet.
+Return value: the 32 bit representation of the previous code +point.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars + 3;
+
+int cp = previous (w, twochars - 1);
+
+assert (cp == 0x65e5);
+assert (w == twochars);
+

The primary purpose of this function is to iterate backwards +through a UTF-8 encoded string. Therefore, it will +typically point to the beginning of a code point, and +pass_start will point to the octet just before the +beginning of the string to ensure we don't go backwards too far. +it is decreased until it points to a lead UTF-8 octet, +and then the UTF-8 sequence beginning with that octet is decoded to +a 32 bit representation and returned.

In case pass_end is reached before a UTF-8 lead +octet is hit, or if an invalid UTF-8 sequence is started by the +lead octet, an invalid_utf8 exception is thrown

utf8::advance

Advances an iterator by the specified number of code points +within an UTF-8 sequence.

template <typename octet_iterator, typename
+distance_type> void advance (octet_iterator& it,
+distance_type n, octet_iterator end);

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the nth following code +point.
+n: a positive integer that shows how many code points +we want to advance.
+end: end of the UTF-8 sequence to be processed. If +it gets equal to end during the +extraction of a code point, an utf8::not_enough_room +exception is thrown.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+advance (w, 2, twochars + 6);
+
+assert (w == twochars + 5);
+

This function works only "forward". In case of a negative +n, there is no effect.

In case of an invalid code point, a +utf8::invalid_code_point exception is thrown.

utf8::distance

Given the iterators to two UTF-8 encoded code points in a +seqence, returns the number of code points between them.

template <typename octet_iterator> typename
+std::iterator_traits<octet_iterator>::difference_type
+distance (octet_iterator first, octet_iterator last);

first: an iterator to a beginning of a UTF-8 +encoded code point.
+last: an iterator to a "post-end" of the last UTF-8 +encoded code point in the sequence we are trying to determine the +length. It can be the beginning of a new code point, or not.
+Return value the distance between the iterators, in code +points.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+
+size_t dist = utf8::distance(twochars, twochars + 5);
+
+assert (dist == 2);
+

This function is used to find the length (in code points) of a +UTF-8 encoded string. The reason it is called distance, +rather than, say, length is mainly because developers are +used that length is an O(1) function. Computing the length +of an UTF-8 string is a linear operation, and it looked better to +model it after std::distance algorithm.

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

utf8::utf16to8

Converts a UTF-16 encoded string to UTF-8.

template <typename u16bit_iterator, typename
+octet_iterator> void utf16to8 (u16bit_iterator start,
+u16bit_iterator end, octet_iterator result);

start: an iterator pointing to the beginning of the +UTF-16 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-16 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

Example of use:

+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+
+utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+
+assert (utf8result.size() == 10);    
+

In case of invalid UTF-16 sequence, a +utf8::invalid_utf16 exception is thrown.

utf8::utf8to16

Converts an UTF-8 encoded string to UTF-16

template <typename u16bit_iterator, typename
+octet_iterator> void utf8to16 (octet_iterator start,
+octet_iterator end, u16bit_iterator result);

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert. < br /> end: an +iterator pointing to pass-the-end of the UTF-8 encoded string to +convert.
+result: an output iterator to the place in the UTF-16 +string where to append the result of conversion.

Example of use:

+unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88,
+    0xf0, 0x9d, 0x84, 0x9e};
+vector <unsigned short> utf16result;
+
+utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

utf8::utf32to8

Converts a UTF-32 encoded string to UTF-8.

template <typename octet_iterator, typename
+u32bit_iterator> void utf32to8 (u32bit_iterator start,
+u32bit_iterator end, octet_iterator result);

start: an iterator pointing to the beginning of the +UTF-32 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-32 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

Example of use:

+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+
+assert (utf8result.size() == 9);
+

In case of invalid UTF-32 string, a +utf8::invalid_code_point exception is thrown.

utf8::utf8to32

Converts a UTF-8 encoded string to UTF-32.

template <typename octet_iterator, typename
+u32bit_iterator> void utf8to32 (octet_iterator start,
+octet_iterator end, u32bit_iterator result);

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert.
+end: an iterator pointing to pass-the-end of the UTF-8 +encoded string to convert.
+result: an output iterator to the place in the UTF-32 +string where to append the result of conversion.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+vector<int> utf32result;
+
+utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+
+assert (utf32result.size() == 2);
+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

utf8::find_invalid

Detects an invalid sequence within a UTF-8 string.

template <typename octet_iterator> octet_iterator
+find_invalid(octet_iterator start, octet_iterator end);

start: an iterator pointing to the beginning of the +UTF-8 string to test for validity.
+end: an iterator pointing to pass-the-end of the UTF-8 +string to test for validity.
+Return value: an iterator pointing to the first invalid +octet in the UTF-8 string. In case none were found, equals +end.

Example of use:

+unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa};
+
+unsigned char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
+
+assert (invalid == utf_invalid + 5);
+

This function is typically used to make sure a UTF-8 string is +valid before processing it with other functions. It is especially +important to call it if before doing any of the unchecked +operations on it.

utf8::is_valid

Checks whether a sequence of octets is a valid UTF-8 string.

template <typename octet_iterator> bool
+is_valid(octet_iterator start, octet_iterator end);

start: an iterator pointing to the beginning of the +UTF-8 string to test for validity.
+end: an iterator pointing to pass-the-end of the UTF-8 +string to test for validity.
+Return value: true if the sequence is a valid +UTF-8 string; false if not.

+Example of use: +

+unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa};
+
+bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
+
+assert (bvalid == false);
+

is_valid is a shorthand for +find_invalid(start, end) == end;. You may want to use +it to make sure that a byte seqence is a valid UTF-8 string without +the need to know where it fails if it is not valid.

utf8::is_bom

Checks whether a sequence of three octets is a UTF-8 byte order +mark (BOM)

template <typename octet_iterator> bool is_bom
+(octet_iterator it);

it Beginning of the 3-octet sequence to check
+Return value: true if the sequence is UTF-8 +byte order mark; false if not.

Example of use:

+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+
+bool bbom = is_bom(byte_order_mark);
+
+assert (bbom == true);
+

The typical use of this function is to check the first three +bytes of a file. If they form the UTF-8 BOM, we want to skip them +before processing the actual UTF-8 encoded text.

Functions From utf8::unchecked Namespace

utf8::unchecked::append

Encodes a 32 bit code point as a UTF-8 sequence of octets and +appends the sequence to a UTF-8 string.

template <typename octet_iterator> octet_iterator
+append(uint32_t cp, octet_iterator result);

Example of use:

+unsigned char u[5] = {0,0,0,0,0};
+
+unsigned char* end = unchecked::append(0x0448, u);
+
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+

This is a quicker but less safe version of +utf8::append. It does not check for validity of the +supplied code point, and may produce an invalid UTF-8 sequence.

utf8::unchecked::next

Given the iterator to the beginning of a UTF-8 sequence, it +returns the code point and moves the iterator to the next +position.

template <typename octet_iterator> uint32_t
+next(octet_iterator& it);

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+int cp = unchecked::next(w);
+
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+

This is a quicker but less safe version of +utf8::next. It does not check for validity of the +supplied UTF-8 sequence.

utf8::unchecked::previous

template <typename octet_iterator> uint32_t
+previous(octet_iterator& it);

it: a reference pointing to an octet within a UTF-8 +encoded string. After the function returns, it is decremented to +point to the beginning of the previous code point.
+Return value: the 32 bit representation of the previous code +point.

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars + 3;
+
+int cp = unchecked::previous (w);
+
+assert (cp == 0x65e5);
+assert (w == twochars);
+

This is a quicker but less safe version of +utf8::previous. It does not check for validity of the +supplied UTF-8 sequence and offers no boundary checking.

utf8::unchecked::advance

Advances an iterator by the specified number of code points +within an UTF-8 sequence.

template <typename octet_iterator, typename
+distance_type> void advance (octet_iterator& it,
+distance_type n);

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+unchecked::advance (w, 2);
+
+assert (w == twochars + 5);
+

This function works only "forward". In case of a negative +n, there is no effect.

This is a quicker but less safe version of +utf8::advance. It does not check for validity of the +supplied UTF-8 sequence and offers no boundary checking.

utf8::unchecked::distance

Given the iterators to two UTF-8 encoded code points in a +seqence, returns the number of code points between them.

template <typename octet_iterator> typename
+std::iterator_traits<octet_iterator>::difference_type
+distance (octet_iterator first, octet_iterator last);

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+
+size_t dist = utf8::unchecked::distance(twochars, twochars + 5);
+
+assert (dist == 2);
+

This is a quicker but less safe version of +utf8::distance. It does not check for validity of the +supplied UTF-8 sequence.

utf8::unchecked::utf16to8

Converts a UTF-16 encoded string to UTF-8.

template <typename u16bit_iterator, typename
+octet_iterator> void utf16to8 (u16bit_iterator start,
+u16bit_iterator end, octet_iterator result);

Example of use:

+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+
+unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+
+assert (utf8result.size() == 10);    
+

This is a quicker but less safe version of +utf8::utf16to8. It does not check for validity of the +supplied UTF-16 sequence.

utf8::unchecked::utf8to16

Converts an UTF-8 encoded string to UTF-16

template <typename u16bit_iterator, typename
+octet_iterator> void utf8to16 (octet_iterator start,
+octet_iterator end, u16bit_iterator result);

Example of use:

+unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88,
+    0xf0, 0x9d, 0x84, 0x9e};
+vector <unsigned short> utf16result;
+
+unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+

This is a quicker but less safe version of +utf8::utf8to16. It does not check for validity of the +supplied UTF-8 sequence.

utf8::unchecked::utf32to8

Converts a UTF-32 encoded string to UTF-8.

template <typename octet_iterator, typename
+u32bit_iterator> void utf32to8 (u32bit_iterator start,
+u32bit_iterator end, octet_iterator result);

Example of use:

+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+
+assert (utf8result.size() == 9);
+

This is a quicker but less safe version of +utf8::utf32to8. It does not check for validity of the +supplied UTF-32 sequence.

utf8::unchecked::utf8to32

Converts a UTF-8 encoded string to UTF-32.

template <typename octet_iterator, typename
+u32bit_iterator> void utf8to32 (octet_iterator start,
+octet_iterator end, u32bit_iterator result);

Example of use:

+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+vector<int> utf32result;
+
+unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+
+assert (utf32result.size() == 2);
+

This is a quicker but less safe version of +utf8::utf8to32. It does not check for validity of the +supplied UTF-8 sequence.

Points of interest

Design goals and decisions

The library was designed to be:

Generic: for better or worse, there are many C++ string classes +out there, and the library should work with as many of them as +possible.
Portable: the library should be portable both accross different +platforms and compilers. The only non-portable code is a small +section that declares unsigned integers of different sizes: three +typedefs. They can be changed by the users of the library if they +don't match their platform. The default setting should work for +Windows (both 32 and 64 bit), and most 32 bit and 64 bit Unix +derivatives.
Lightweight: follow the "pay only for what you use" +guidline.
Unintrusive: avoid forcing any particular design or even +programming style on the user. This is a library, not a +framework.

Alternatives

In case you want to look into other means of working with UTF-8 +strings from C++, here is the list of solutions I am aware of:

ICU Library. It is +very powerful, complete, feature-rich, mature, and widely used. +Also big, intrusive, non-generic, and doesn't play well with the +Standard Library. I definitelly recommend looking at ICU even if +you don't plan to use it.
Glib::ustring. +A class specifically made to work with UTF-8 strings, and also feel +like std::string. If you prefer to have yet another +string class in your code, it may be worth a look. Be aware of the +licensing issues, though.
Platform dependent solutions: Windows and POSIX have functions +to convert strings from one encoding to another. That is only a +subset of what my library offers, but if that is all you need it +may be good enough, especially given the fact that these functions +are mature and tested in production.

Conclusion

Until Unicode becomes officially recognized by the C++ Standard +Library, we need to use other means to work with UTF-8 strings. +Template functions I describe in this article may be a good step in +this direction.

References

+ + diff --git a/source/utf8.h b/source/utf8.h new file mode 100644 index 0000000..77f3f35 --- /dev/null +++ b/source/utf8.h @@ -0,0 +1,504 @@ +// Copyright (c) 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public std::exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + const char* what() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public std::exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + const char* what() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public std::exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + const char* what() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public std::exception { + public: + const char* what() { return "Not enough space"; } + }; + + + + + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint32_t LEAD_SURROGATE_MIN = 0xd800; + const uint32_t LEAD_SURROGATE_MAX = 0xdbff; + const uint32_t TRAIL_SURROGATE_MIN = 0xdc00; + const uint32_t TRAIL_SURROGATE_MAX = 0xdfff; + const uint32_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000 - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffff; + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + /// Helper functions - not intended to be directly called by the library users + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + + + /// The library API - functions intended to be called by the users + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + if (mask8(*result) > 0xf4) + break; + if (mask8(*result) < 0x80) + ; + else if ((mask8(*result) >> 5) == 0x6) { + uint8_t lead = mask8(*result); + if (++result == end) + return (--result); + if (!is_trail(*result)) + return result; + switch (lead) { + case 0xe0: + if ((mask8(*result)) < 0xa0) + return result; + break; + case 0xed: + if ((mask8(*result)) > 0x9F) + return result; + break; + case 0xf0: + if ((mask8(*result)) < 0x90) + return result; + break; + } + } + else if ((mask8(*result) >> 4) == 0xe) { + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + } + + else if ((mask8(*result) >> 3) == 0x1e) { + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + } + else + break; + ++result; + } + return result; + } + + template + bool is_valid(octet_iterator start, octet_iterator end) + { + return (find_invalid(start, end) == end); + } + + template + bool is_bom (octet_iterator it) + { + return ( + (mask8(*it++)) == bom[0] && + (mask8(*it++)) == bom[1] && + (mask8(*it)) == bom[2] + ); + } + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + if (is_surrogate(cp)) + throw invalid_code_point(cp); + + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp <= CODE_POINT_MAX) { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast((cp >> 12) | 0x80); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else + throw invalid_code_point(cp); + + return result; + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = mask8(*it); + if (cp < 0x80) + ; + else if ((mask8(*it) >> 5) == 0x6) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else if ((mask8(*it) >> 4) == 0xe) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); + if (++it != end) { + if (is_trail(*it)) { + cp += (*it) & 0x3f; + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else if ((mask8(*it) >> 3) == 0x1e) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 18) & 0x1fffff) + (mask8(*it) << 12) & 0x3ffff; + if (++it != end) { + if (is_trail(*it)) { + cp += (mask8(*it) << 6) & 0xfff; + if (++it != end) { + if (is_trail(*it)) { + cp += (*it) & 0x3f; + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + ++it; + if (cp > CODE_POINT_MAX || is_surrogate(cp)) + throw invalid_code_point(cp); + + return cp; + } + + + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + next(first, last); + return dist; + } + + template + void utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = mask16(*start++); + // Take care of surrogate pairs first + if (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX) { + if (start != end) { + uint32_t trail_surrogate = mask16(*start++); + if (trail_surrogate >= TRAIL_SURROGATE_MIN && trail_surrogate <= TRAIL_SURROGATE_MAX) + cp = (cp << 10) + trail_surrogate + SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(*start)); + + } + *result = append(cp, result); + } + } + + template + void utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + } + + template + void utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + *result = append(*(start++), result); + } + + template + void utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = next(start, end); + } + + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast((cp >> 12) | 0x80); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = mask8(*it); + if (cp < 0x80) + ; + else if ((mask8(*it) >> 5) == 0x6) { + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + } + else if ((mask8(*it) >> 4) == 0xe) { + ++it; + cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + } + else if (((*it) >> 3) == 0x1e) { + ++it; + cp = ((cp << 18) & 0x1fffff) + (mask8(*it) << 12) & 0x3ffff; + ++it; + cp += (mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + } + ++it; + return cp; + } + + template + uint32_t previous(octet_iterator& it) + { + while (is_trail(*(--it))) ; + octet_iterator temp = it; + return next(temp); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + next(it); + } + + template + uint32_t get(octet_iterator it) + { + return next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + next(first); + return dist; + } + + template + void utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = mask16(*start++); + // Take care of surrogate pairs first + if (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX) { + uint32_t trail_surrogate = mask16(*start++); + cp = (cp << 10) + trail_surrogate + SURROGATE_OFFSET; + } + *result = append(cp, result); + } + } + + template + void utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + } + + template + void utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + *result = append(*(start++), result); + } + + template + void utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = next(start); + } + + } // namespace utf8::unchecked +} // namespace utf8 + +#endif // header guard diff --git a/test_data/negative/utf8_invalid.txt b/test_data/negative/utf8_invalid.txt new file mode 100755 index 0000000..cc03e7b --- /dev/null +++ b/test_data/negative/utf8_invalid.txt @@ -0,0 +1,271 @@ +UTF-8 decoder capability and stress test +---------------------------------------- + +Markus Kuhn - 2003-02-19 + +This test file can help you examine, how your UTF-8 decoder handles +various types of correct, malformed, or otherwise interesting UTF-8 +sequences. This file is not meant to be a conformance test. It does +not prescribes any particular outcome and therefore there is no way to +"pass" or "fail" this test file, even though the texts suggests a +preferable decoder behaviour at some places. The aim is instead to +help you think about and test the behaviour of your UTF-8 on a +systematic collection of unusual inputs. Experience so far suggests +that most first-time authors of UTF-8 decoders find at least one +serious problem in their decoder by using this file. + +The test lines below cover boundary conditions, malformed UTF-8 +sequences as well as correctly encoded UTF-8 sequences of Unicode code +points that should never occur in a correct UTF-8 file. + +According to ISO 10646-1:2000, sections D.7 and 2.3c, a device +receiving UTF-8 shall interpret a "malformed sequence in the same way +that it interprets a character that is outside the adopted subset" and +"characters that are not within the adopted subset shall be indicated +to the user" by a receiving device. A quite commonly used approach in +UTF-8 decoders is to replace any malformed UTF-8 sequence by a +replacement character (U+FFFD), which looks a bit like an inverted +question mark, or a similar symbol. It might be a good idea to +visually distinguish a malformed UTF-8 sequence from a correctly +encoded Unicode character that is just not available in the current +font but otherwise fully legal, even though ISO 10646-1 doesn't +mandate this. In any case, just ignoring malformed sequences or +unavailable characters does not conform to ISO 10646, will make +debugging more difficult, and can lead to user confusion. + +Please check, whether a malformed UTF-8 sequence is (1) represented at +all, (2) represented by exactly one single replacement character (or +equivalent signal), and (3) the following quotation mark after an +illegal UTF-8 sequence is correctly displayed, i.e. proper +resynchronization takes place immageately after any malformed +sequence. This file says "THE END" in the last line, so if you don't +see that, your decoder crashed somehow before, which should always be +cause for concern. + +All lines in this file are exactly 79 characters long (plus the line +feed). In addition, all lines end with "|", except for the two test +lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls +U+0000 and U+007F. If you display this file with a fixed-width font, +these "|" characters should all line up in column 79 (right margin). +This allows you to test quickly, whether your UTF-8 decoder finds the +correct number of characters in every line, that is whether each +malformed sequences is replaced by a single replacement character. + +Note that as an alternative to the notion of malformed sequence used +here, it is also a perfectly acceptable (and in some situations even +preferable) solution to represent each individual byte of a malformed +sequence by a replacement character. If you follow this strategy in +your decoder, then please ignore the "|" column. + + +Here come the tests: | + | +1 Some correct UTF-8 text | + | +You should see the Greek word 'kosme': "κόσμε" | + | +2 Boundary condition test cases | + | +2.1 First possible sequence of a certain length | + | +2.1.1 1 byte (U-00000000): "" +2.1.2 2 bytes (U-00000080): "" | +2.1.3 3 bytes (U-00000800): "ࠀ" | +2.1.4 4 bytes (U-00010000): "𐀀" | +2.1.5 5 bytes (U-00200000): "?" | +2.1.6 6 bytes (U-04000000): "?" | + | +2.2 Last possible sequence of a certain length | + | +2.2.1 1 byte (U-0000007F): "" +2.2.2 2 bytes (U-000007FF): "߿" | +2.2.3 3 bytes (U-0000FFFF): "?" | +2.2.4 4 bytes (U-001FFFFF): "?" | +2.2.5 5 bytes (U-03FFFFFF): "?" | +2.2.6 6 bytes (U-7FFFFFFF): "?" | + | +2.3 Other boundary conditions | + | +2.3.1 U-0000D7FF = ed 9f bf = "퟿" | +2.3.2 U-0000E000 = ee 80 80 = "" | +2.3.3 U-0000FFFD = ef bf bd = "�" | +2.3.4 U-0010FFFF = f4 8f bf bf = "􏿿" | +2.3.5 U-00110000 = f4 90 80 80 = "?" | + | +3 Malformed sequences | + | +3.1 Unexpected continuation bytes | + | +Each unexpected continuation byte should be separately signalled as a | +malformed sequence of its own. | + | +3.1.1 First continuation byte 0x80: "?" | +3.1.2 Last continuation byte 0xbf: "?" | + | +3.1.3 2 continuation bytes: "??" | +3.1.4 3 continuation bytes: "???" | +3.1.5 4 continuation bytes: "????" | +3.1.6 5 continuation bytes: "?????" | +3.1.7 6 continuation bytes: "??????" | +3.1.8 7 continuation bytes: "???????" | + | +3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | + | + "???????????????? | + ???????????????? | + ???????????????? | + ????????????????" | + | +3.2 Lonely start characters | + | +3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), | + each followed by a space character: | + | + "?? àĠŠƠǠȠɠʠˠ̠͠ΠϠ | + РѠҠӠԠՠ֠נؠ٠ڠ۠ܠݠޠߠ" | + | +3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), | + each followed by a space character: | + | + "ࠡ ⠣ 䠥栧蠩 ꠫ 젭  " | + | +3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), | + each followed by a space character: | + | + "𠱠򠳠??" | + | +3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), | + each followed by a space character: | + | + "? ? | + | +3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), | + each followed by a space character: | + | + "? | + | +3.3 Sequences with last continuation byte missing | + | +All bytes of an incomplete sequence should be signalled as a single | +malformed sequence, i.e., you should see only a single replacement | +character in each of the next 10 tests. (Characters as in section 2) | + | +3.3.1 2-byte sequence with last byte missing (U+0000): "? | +3.3.2 3-byte sequence with last byte missing (U+0000): "? | +3.3.3 4-byte sequence with last byte missing (U+0000): "? | +3.3.4 5-byte sequence with last byte missing (U+0000): "? | +3.3.5 6-byte sequence with last byte missing (U+0000): "? | +3.3.6 2-byte sequence with last byte missing (U-000007FF): "ߢ | +3.3.7 3-byte sequence with last byte missing (U-0000FFFF): "￢ | +3.3.8 4-byte sequence with last byte missing (U-001FFFFF): "? | +3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): "? | +3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "? | + | +3.4 Concatenation of incomplete sequences | + | +All the 10 sequences of 3.3 concatenated, you should see 10 malformed | +sequences being signalled: | + | + "???????????????" | + | +3.5 Impossible bytes | + | +The following two bytes cannot appear in a correct UTF-8 string | + | +3.5.1 fe = "? | +3.5.2 ff = "? | +3.5.3 fe fe ff ff = "? | + | +4 Overlong sequences | + | +The following sequences are not malformed according to the letter of | +the Unicode 2.0 standard. However, they are longer then necessary and | +a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 | +decoder" should reject them just like malformed sequences for two | +reasons: (1) It helps to debug applications if overlong sequences are | +not treated as valid representations of characters, because this helps | +to spot problems more quickly. (2) Overlong sequences provide | +alternative representations of characters, that could maliciously be | +used to bypass filters that check only for ASCII characters. For | +instance, a 2-byte encoded line feed (LF) would not be caught by a | +line counter that counts only 0x0a bytes, but it would still be | +processed as a line feed by an unsafe UTF-8 decoder later in the | +pipeline. From a security point of view, ASCII compatibility of UTF-8 | +sequences means also, that ASCII characters are *only* allowed to be | +represented by ASCII bytes in the range 0x00-0x7f. To ensure this | +aspect of ASCII compatibility, use only "safe UTF-8 decoders" that | +reject overlong UTF-8 sequences for which a shorter encoding exists. | + | +4.1 Examples of an overlong ASCII character | + | +With a safe UTF-8 decoder, all of the following five overlong | +representations of the ASCII character slash ("/") should be rejected | +like a malformed UTF-8 sequence, for instance by substituting it with | +a replacement character. If you see a slash below, you do not have a | +safe UTF-8 decoder! | + | +4.1.1 U+002F = c0 af = "?" | +4.1.2 U+002F = e0 80 af = "?" | +4.1.3 U+002F = f0 80 80 af = "?" | +4.1.4 U+002F = f8 80 80 80 af = "?" | +4.1.5 U+002F = fc 80 80 80 80 af = "?" | + | +4.2 Maximum overlong sequences | + | +Below you see the highest Unicode value that is still resulting in an | +overlong sequence if represented with the given number of bytes. This | +is a boundary test for safe UTF-8 decoders. All five characters should | +be rejected like malformed UTF-8 sequences. | + | +4.2.1 U-0000007F = c1 bf = "?" | +4.2.2 U-000007FF = e0 9f bf = "?" | +4.2.3 U-0000FFFF = f0 8f bf bf = "?" | +4.2.4 U-001FFFFF = f8 87 bf bf bf = "?" | +4.2.5 U-03FFFFFF = fc 83 bf bf bf bf = "?" | + | +4.3 Overlong representation of the NUL character | + | +The following five sequences should also be rejected like malformed | +UTF-8 sequences and should not be treated like the ASCII NUL | +character. | + | +4.3.1 U+0000 = c0 80 = "?" | +4.3.2 U+0000 = e0 80 80 = "?" | +4.3.3 U+0000 = f0 80 80 80 = "?" | +4.3.4 U+0000 = f8 80 80 80 80 = "?" | +4.3.5 U+0000 = fc 80 80 80 80 80 = "?" | + | +5 Illegal code positions | + | +The following UTF-8 sequences should be rejected like malformed | +sequences, because they never represent valid ISO 10646 characters and | +a UTF-8 decoder that accepts them might introduce security problems | +comparable to overlong UTF-8 sequences. | + | +5.1 Single UTF-16 surrogates | + | +5.1.1 U+D800 = ed a0 80 = "��" | +5.1.2 U+DB7F = ed ad bf = "��" | +5.1.3 U+DB80 = ed ae 80 = "��" | +5.1.4 U+DBFF = ed af bf = "��" | +5.1.5 U+DC00 = ed b0 80 = "��" | +5.1.6 U+DF80 = ed be 80 = "��" | +5.1.7 U+DFFF = ed bf bf = "��" | + | +5.2 Paired UTF-16 surrogates | + | +5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = "𐀀" | +5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = "𐏿" | +5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = "󯰀" | +5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = "󯿿" | +5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = "󰀀" | +5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = "󰏿" | +5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = "􏰀" | +5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = "􏿿" | + | +5.3 Other illegal code positions | + | +5.3.1 U+FFFE = ef bf be = "?" | +5.3.2 U+FFFF = ef bf bf = "?" | + | +THE END | diff --git a/test_drivers/smoke_test/test.cpp b/test_drivers/smoke_test/test.cpp new file mode 100644 index 0000000..0281656 --- /dev/null +++ b/test_drivers/smoke_test/test.cpp @@ -0,0 +1,144 @@ + +#include +#include +#include +#include "utf8.h" +using namespace utf8; +using namespace std; + +int main() +{ + //append + unsigned char u[5] = {0,0,0,0,0}; + + unsigned char* end = append(0x0448, u); + assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0); + + end = append(0x65e5, u); + assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0); + + end = append(0x10346, u); + assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0); + + //next + unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0}; + unsigned char* w = twochars; + int cp = next(w, twochars + 6); + assert (cp == 0x65e5); + assert (w == twochars + 3); + + //previous + cp = previous (w, twochars - 1); + assert (cp == 0x65e5); + assert (w == twochars); + + // advance + w = twochars; + advance (w, 2, twochars + 6); + assert (w == twochars + 5); + + // distance + size_t dist = utf8::distance(twochars, twochars + 5); + assert (dist == 2); + + // utf32to8 + int utf32string[] = {0x448, 0x65E5, 0x10346, 0}; + vector utf8result; + utf32to8(utf32string, utf32string + 3, back_inserter(utf8result)); + assert (utf8result.size() == 9); + + //utf8to32 + vector utf32result; + utf8to32(twochars, twochars + 5, back_inserter(utf32result)); + assert (utf32result.size() == 2); + + //utf16to8 + unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e}; + utf8result.clear(); + utf16to8(utf16string, utf16string + 5, back_inserter(utf8result)); + assert (utf8result.size() == 10); + + //utf8to16 + unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, + 0xf0, 0x9d, 0x84, 0x9e}; + vector utf16result; + utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result)); + assert (utf16result.size() == 4); + assert (utf16result[2] == 0xd834); + assert (utf16result[3] == 0xdd1e); + + //find_invalid + unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa}; + unsigned char* invalid = find_invalid(utf_invalid, utf_invalid + 6); + assert (invalid == utf_invalid + 5); + + //is_valid + bool bvalid = is_valid(utf_invalid, utf_invalid + 6); + assert (bvalid == false); + bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9); + assert (bvalid == true); + + //is_bom + unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf}; + bool bbom = is_bom(byte_order_mark); + assert (bbom == true); + + ////////////////////////////////////////////////////////// + //// Unchecked variants + ////////////////////////////////////////////////////////// + + //append + memset(u, 0, 5); + end = unchecked::append(0x0448, u); + assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0); + + end = unchecked::append(0x65e5, u); + assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0); + + end = unchecked::append(0x10346, u); + assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0); + + //next + w = twochars; + cp = unchecked::next(w); + assert (cp == 0x65e5); + assert (w == twochars + 3); + + //previous + cp = unchecked::previous (w); + assert (cp == 0x65e5); + assert (w == twochars); + + // advance + w = twochars; + unchecked::advance (w, 2); + assert (w == twochars + 5); + + // distance + dist = unchecked::distance(twochars, twochars + 5); + assert (dist == 2); + + // utf32to8 + utf8result.clear(); + unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result)); + assert (utf8result.size() == 9); + + //utf8to32 + utf32result.clear(); + unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result)); + assert (utf32result.size() == 2); + + //utf16to8 + utf8result.clear(); + unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result)); + assert (utf8result.size() == 10); + + //utf8to16 + utf16result.clear(); + unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result)); + assert (utf16result.size() == 4); + assert (utf16result[2] == 0xd834); + assert (utf16result[3] == 0xdd1e); +} + +