A minor documentation fix

git-svn-id: http://svn.code.sf.net/p/utfcpp/code@77 a809a056-fc17-0410-9590-b4f493f8b08e

boost/boost/utf8.hpp

@@ -0,0 +1,530 @@
+// utf8.hpp header file
+
+/*
+Copyright 2006 Nemanja Trifunovic
+
+Distributed under the Boost Software License, Version 1.0. (See
+accompanying file LICENSE_1_0.txt or copy at
+http://www.boost.org/LICENSE_1_0.txt)
+*/
+
+
+#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
+#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
+
+#include <exception>
+
+#include <boost/config.hpp>
+
+namespace boost {
+
+namespace utf8 {
+    
+    // Exceptions that may be thrown from the library functions.
+    class invalid_code_point : public std::exception {
+        uint32_t cp;
+    public:
+        invalid_code_point(uint32_t cp) : cp(cp) {}
+        virtual const char* what() const throw() { return "Invalid code point"; }
+        uint32_t code_point() const {return cp;}
+    };
+
+    class invalid_utf8 : public std::exception {
+        uint8_t u8;
+    public:
+        invalid_utf8 (uint8_t u) : u8(u) {}
+        virtual const char* what() const throw() { return "Invalid UTF-8"; }
+        uint8_t utf8_octet() const {return u8;}
+    };
+
+    class invalid_utf16 : public std::exception {
+        uint16_t u16;
+    public:
+        invalid_utf16 (uint16_t u) : u16(u) {}
+        virtual const char* what() const throw() { return "Invalid UTF-16"; }
+        uint16_t utf16_word() const {return u16;}
+    };
+
+    class not_enough_room : public std::exception {
+    public:
+        virtual const char* what() const throw() { return "Not enough space"; }
+    };
+
+
+// Helper code - not intended to be directly called by the library users. May be changed at any time
+namespace internal
+{    
+    // Unicode constants
+    // Leading (high) surrogates: 0xd800 - 0xdbff
+    // Trailing (low) surrogates: 0xdc00 - 0xdfff
+    const uint16_t LEAD_SURROGATE_MIN  = 0xd800u;
+    const uint16_t LEAD_SURROGATE_MAX  = 0xdbffu;
+    const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
+    const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
+    const uint16_t LEAD_OFFSET         = LEAD_SURROGATE_MIN - (0x10000 >> 10);
+    const uint32_t SURROGATE_OFFSET    = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
+
+    // Maximum valid value for a Unicode code point
+    const uint32_t CODE_POINT_MAX      = 0x0010ffffu;
+
+    template<typename octet_type>
+    inline uint8_t mask8(octet_type oc)
+    {
+        return static_cast<uint8_t>(0xff & oc);
+    }
+    template<typename u16_type>
+    inline uint16_t mask16(u16_type oc)
+    {
+        return static_cast<uint16_t>(0xffff & oc);
+    }
+    template<typename octet_type>
+    inline bool is_trail(octet_type oc)
+    {
+        return ((mask8(oc) >> 6) == 0x2);
+    }
+
+    template <typename u16>
+    inline bool is_surrogate(u16 cp)
+    {
+        return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
+    }
+
+    template <typename u32>
+    inline bool is_code_point_valid(u32 cp)
+    {
+        return (cp <= CODE_POINT_MAX && !is_surrogate(cp) && cp != 0xfffe && cp != 0xffff);
+    }  
+
+    template <typename octet_iterator>
+    inline typename std::iterator_traits<octet_iterator>::difference_type
+    sequence_length(octet_iterator lead_it)
+    {
+        uint8_t lead = mask8(*lead_it);
+        if (lead < 0x80) 
+            return 1;
+        else if ((lead >> 5) == 0x6)
+            return 2;
+        else if ((lead >> 4) == 0xe)
+            return 3;
+        else if ((lead >> 3) == 0x1e)
+            return 4;
+        else 
+            return 0;
+    }
+
+    enum utf_error {OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};
+
+    template <typename octet_iterator>
+    utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point = 0)
+    {
+        uint32_t cp = mask8(*it);
+        // Check the lead octet
+        typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
+        octet_difference_type length = sequence_length(it);
+
+        // "Shortcut" for ASCII characters
+        if (length == 1) {
+            if (end - it > 0) {
+                if (code_point)
+                    *code_point = cp;
+                ++it;
+                return OK;
+            }
+            else
+                return NOT_ENOUGH_ROOM;
+        }
+
+        // Do we have enough memory?     
+        if (end - it < length)
+            return NOT_ENOUGH_ROOM;
+        
+        // Check trail octets and calculate the code point
+        switch (length) {
+            case 0:
+                return INVALID_LEAD;
+                break;
+            case 2:
+                if (is_trail(*(++it))) { 
+                    cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
+                }
+                else {
+                    --it;
+                    return INCOMPLETE_SEQUENCE;
+                }
+            break;
+            case 3:
+                if (is_trail(*(++it))) {
+                    cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff);
+                    if (is_trail(*(++it))) {
+                        cp += (*it) & 0x3f;
+                    }
+                    else {
+                        --it; --it; 
+                        return INCOMPLETE_SEQUENCE;
+                    }
+                }
+                else {
+                    --it;
+                    return INCOMPLETE_SEQUENCE;
+                }
+            break;
+            case 4:
+                if (is_trail(*(++it))) {
+                    cp = ((cp << 18) & 0x1fffff) + ((mask8(*it) << 12) & 0x3ffff);                
+                    if (is_trail(*(++it))) {
+                        cp += (mask8(*it) << 6) & 0xfff;
+                        if (is_trail(*(++it))) {
+                            cp += (*it) & 0x3f; 
+                        }
+                        else {
+                            --it; --it; --it;
+                            return INCOMPLETE_SEQUENCE;
+                        }
+                    }
+                    else {
+                        --it; --it;
+                        return INCOMPLETE_SEQUENCE;
+                    }
+                }
+                else {
+                    --it;
+                    return INCOMPLETE_SEQUENCE;
+                }
+            break;
+        }
+        // Is the code point valid?
+        if (!is_code_point_valid(cp)) {
+            for (octet_difference_type i = 0; i < length - 1; ++i) 
+                --it;
+            return INVALID_CODE_POINT;
+        }
+            
+        if (code_point)
+            *code_point = cp;
+            
+        if (cp < 0x80) {
+            if (length != 1) {
+                for (octet_difference_type i = 0; i < length - 1; ++i)
+                    --it;
+                return OVERLONG_SEQUENCE;
+            }
+        }
+        else if (cp < 0x800) {
+            if (length != 2) {
+                for (octet_difference_type i = 0; i < length - 1; ++i)
+                    --it;
+                return OVERLONG_SEQUENCE;
+            }
+        }
+        else if (cp < 0x10000) {
+            if (length != 3) {
+                for (octet_difference_type i = 0; i < length - 1; ++i)
+                    --it;
+                return OVERLONG_SEQUENCE;
+            }
+        }
+           
+        ++it;
+        return OK;    
+    }
+
+} // namespace internal 
+    
+    /// The library API - functions intended to be called by the users
+ 
+    // Byte order mark
+    const uint8_t bom[] = {0xef, 0xbb, 0xbf}; 
+
+    template <typename octet_iterator>
+    octet_iterator find_invalid(octet_iterator start, octet_iterator end)
+    {
+        octet_iterator result = start;
+        while (result != end) {
+            internal::utf_error err_code = internal::validate_next(result, end);
+            if (err_code != internal::OK)
+                return result;
+        }
+        return result;
+    }
+
+    template <typename octet_iterator>
+    bool is_valid(octet_iterator start, octet_iterator end)
+    {
+        return (find_invalid(start, end) == end);
+    }
+
+    template <typename octet_iterator>
+    bool is_bom (octet_iterator it)
+    {
+        return (
+            (internal::mask8(*it++)) == bom[0] &&
+            (internal::mask8(*it++)) == bom[1] &&
+            (internal::mask8(*it))   == bom[2]
+           );
+    }
+    template <typename octet_iterator>
+    octet_iterator append(uint32_t cp, octet_iterator result)
+    {
+        if (!internal::is_code_point_valid(cp)) 
+            throw invalid_code_point(cp);
+
+        if (cp < 0x80)                        // one octet
+            *(result++) = static_cast<uint8_t>(cp);  
+        else if (cp < 0x800) {                // two octets
+            *(result++) = static_cast<uint8_t>((cp >> 6)            | 0xc0);
+            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
+        }
+        else if (cp < 0x10000) {              // three octets
+            *(result++) = static_cast<uint8_t>((cp >> 12)           | 0xe0);
+            *(result++) = static_cast<uint8_t>((cp >> 6) & 0x3f     | 0x80);
+            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
+        }
+        else if (cp <= internal::CODE_POINT_MAX) {      // four octets
+            *(result++) = static_cast<uint8_t>((cp >> 18)           | 0xf0);
+            *(result++) = static_cast<uint8_t>((cp >> 12)& 0x3f     | 0x80);
+            *(result++) = static_cast<uint8_t>((cp >> 6) & 0x3f     | 0x80);
+            *(result++) = static_cast<uint8_t>((cp & 0x3f)          | 0x80);
+        }
+        else
+            throw invalid_code_point(cp);
+
+        return result;
+    }
+
+    template <typename octet_iterator>
+    uint32_t next(octet_iterator& it, octet_iterator end)
+    {
+        uint32_t cp = 0;
+        internal::utf_error err_code = internal::validate_next(it, end, &cp);
+        switch (err_code) {
+            case internal::OK :
+                break;
+            case internal::NOT_ENOUGH_ROOM :
+                throw not_enough_room();
+            case internal::INVALID_LEAD :
+            case internal::INCOMPLETE_SEQUENCE :
+            case internal::OVERLONG_SEQUENCE :
+                throw invalid_utf8(*it);
+            case internal::INVALID_CODE_POINT :
+                throw invalid_code_point(cp);
+        }
+        return cp;        
+    }
+
+    template <typename octet_iterator>
+    uint32_t prior(octet_iterator& it, octet_iterator start)
+    {
+        octet_iterator end = it;
+        while (internal::is_trail(*(--it))) 
+            if (it < start)
+                throw invalid_utf8(*it); // error - no lead byte in the sequence
+        octet_iterator temp = it;
+        return next(temp, end);
+    }
+
+    template <typename octet_iterator, typename distance_type>
+    void advance (octet_iterator& it, distance_type n, octet_iterator end)
+    {
+        for (distance_type i = 0; i < n; ++i)
+            next(it, end);
+    }
+
+    template <typename octet_iterator>
+    typename std::iterator_traits<octet_iterator>::difference_type
+    distance (octet_iterator first, octet_iterator last)
+    {
+        typename std::iterator_traits<octet_iterator>::difference_type dist;
+        for (dist = 0; first < last; ++dist) 
+            next(first, last);
+        return dist;
+    }
+
+    template <typename u16bit_iterator, typename octet_iterator>
+    octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
+    {       
+        while (start != end) {
+            uint32_t cp = internal::mask16(*start++);
+            // Take care of surrogate pairs first
+            if (internal::is_surrogate(cp)) {
+                if (start != end) {
+                    uint32_t trail_surrogate = internal::mask16(*start++);
+                    if (trail_surrogate >= internal::TRAIL_SURROGATE_MIN && trail_surrogate <= internal::TRAIL_SURROGATE_MAX)
+                        cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;                    
+                    else 
+                        throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
+                }
+                else 
+                    throw invalid_utf16(static_cast<uint16_t>(*start));
+            
+            }
+            result = append(cp, result);
+        }
+        return result;        
+    }
+
+    template <typename u16bit_iterator, typename octet_iterator>
+    u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
+    {
+        while (start != end) {
+            uint32_t cp = next(start, end);
+            if (cp > 0xffff) { //make a surrogate pair
+                *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
+                *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
+            }
+            else
+                *result++ = static_cast<uint16_t>(cp);
+        }
+        return result;
+    }
+
+    template <typename octet_iterator, typename u32bit_iterator>
+    octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
+    {
+        while (start != end)
+            result = append(*(start++), result);
+
+        return result;
+    }
+
+    template <typename octet_iterator, typename u32bit_iterator>
+    u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
+    {
+        while (start < end)
+            (*result++) = next(start, end);
+
+        return result;
+    }
+
+    namespace unchecked 
+    {
+        template <typename octet_iterator>
+        octet_iterator append(uint32_t cp, octet_iterator result)
+        {
+            if (cp < 0x80)                        // one octet
+                *(result++) = static_cast<uint8_t>(cp);  
+            else if (cp < 0x800) {                // two octets
+                *(result++) = static_cast<uint8_t>((cp >> 6)          | 0xc0);
+                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
+            }
+            else if (cp < 0x10000) {              // three octets
+                *(result++) = static_cast<uint8_t>((cp >> 12)         | 0xe0);
+                *(result++) = static_cast<uint8_t>((cp >> 6) & 0x3f   | 0x80);
+                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
+            }
+            else {                                // four octets
+                *(result++) = static_cast<uint8_t>((cp >> 18)         | 0xf0);
+                *(result++) = static_cast<uint8_t>((cp >> 12)& 0x3f   | 0x80);
+                *(result++) = static_cast<uint8_t>((cp >> 6) & 0x3f   | 0x80);
+                *(result++) = static_cast<uint8_t>((cp & 0x3f)        | 0x80);
+            }
+            return result;
+        }
+        template <typename octet_iterator>
+        uint32_t next(octet_iterator& it)
+        {
+            uint32_t cp = internal::mask8(*it);
+            typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
+            switch (length) {
+                case 1:
+                    break;
+                case 2:
+                    it++;
+                    cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
+                    break;
+                case 3:
+                    ++it; 
+                    cp = ((cp << 12) & 0xffff) + ((internal::mask8(*it) << 6) & 0xfff);
+                    ++it;
+                    cp += (*it) & 0x3f;
+                    break;
+                case 4:
+                    ++it;
+                    cp = ((cp << 18) & 0x1fffff) + ((internal::mask8(*it) << 12) & 0x3ffff);                
+                    ++it;
+                    cp += (internal::mask8(*it) << 6) & 0xfff;
+                    ++it;
+                    cp += (*it) & 0x3f; 
+                    break;
+            }
+            ++it;
+            return cp;        
+        }
+
+        template <typename octet_iterator>
+        uint32_t prior(octet_iterator& it)
+        {
+            while (internal::is_trail(*(--it))) ;
+            octet_iterator temp = it;
+            return next(temp);
+        }
+
+        template <typename octet_iterator, typename distance_type>
+        void advance (octet_iterator& it, distance_type n)
+        {
+            for (distance_type i = 0; i < n; ++i)
+                next(it);
+        }
+
+        template <typename octet_iterator>
+        typename std::iterator_traits<octet_iterator>::difference_type
+        distance (octet_iterator first, octet_iterator last)
+        {
+            typename std::iterator_traits<octet_iterator>::difference_type dist;
+            for (dist = 0; first < last; ++dist) 
+                next(first);
+            return dist;
+        }
+
+        template <typename u16bit_iterator, typename octet_iterator>
+        octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
+        {       
+            while (start != end) {
+                uint32_t cp = internal::mask16(*start++);
+            // Take care of surrogate pairs first
+                if (internal::is_surrogate(cp)) {
+                    uint32_t trail_surrogate = internal::mask16(*start++);
+                    cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
+                }
+                result = append(cp, result);
+            }
+            return result;         
+        }
+
+        template <typename u16bit_iterator, typename octet_iterator>
+        u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
+        {
+            while (start != end) {
+                uint32_t cp = next(start);
+                if (cp > 0xffff) { //make a surrogate pair
+                    *result++ = static_cast<uint16_t>((cp >> 10)   + internal::LEAD_OFFSET);
+                    *result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
+                }
+                else
+                    *result++ = static_cast<uint16_t>(cp);
+            }
+            return result;
+        }
+
+        template <typename octet_iterator, typename u32bit_iterator>
+        octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
+        {
+            while (start != end)
+                result = append(*(start++), result);
+
+            return result;
+        }
+
+        template <typename octet_iterator, typename u32bit_iterator>
+        u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
+        {
+            while (start < end)
+                (*result++) = next(start);
+
+            return result;
+        }
+
+    } // namespace utf8::unchecked
+} // namespace utf8 
+} // namespace boost
+
+#endif // header guard

boost/libs/utf8/Jamfile

@@ -0,0 +1,17 @@
+# Boost UTF8 Library test Jamfile
+
+# Copyright Nemanja Trifunovic 2006
+
+# Use, modification, and distribution is subject to the Boost Software
+# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+
+# bring in rules for testing
+import testing ;
+
+{
+   test-suite "utf8"
+       : [ run test.cpp ]
+       ;
+}

boost/libs/utf8/test.cpp

@@ -0,0 +1,220 @@
+#include <vector>
+#include <iterator>
+#include "boost/utf8.hpp"
+#include <boost/detail/lightweight_test.hpp>
+using namespace boost::utf8;
+using namespace std;
+
+int main()
+{
+    //append
+    unsigned char u[5] = {0,0,0,0,0};
+
+    unsigned char* end = append(0x0448, u);
+    BOOST_TEST (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+    end = append(0x65e5, u);
+    BOOST_TEST (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
+
+    end = append(0x3044, u);
+    BOOST_TEST (u[0] == 0xe3 && u[1] == 0x81 && u[2] == 0x84 && u[3] == 0 && u[4] == 0);
+
+    end = append(0x10346, u);
+    BOOST_TEST (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
+
+    //next
+    char* twochars = "\xe6\x97\xa5\xd1\x88";
+    char* w = twochars;
+    int cp = next(w, twochars + 6);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == twochars + 3);
+
+    char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
+    w = threechars;
+    cp = next(w, threechars + 9);
+    BOOST_TEST (cp == 0x10346);
+    BOOST_TEST (w == threechars + 4);
+    cp = next(w, threechars + 9);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == threechars + 7);
+    cp = next(w, threechars + 9);
+    BOOST_TEST (cp == 0x0448);
+    BOOST_TEST (w == threechars + 9);
+
+
+    //prior
+    w = twochars + 3;
+    cp = prior (w, twochars - 1);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == twochars);
+
+    w = threechars + 9;
+    cp = prior(w, threechars - 1);
+    BOOST_TEST (cp == 0x0448);
+    BOOST_TEST (w == threechars + 7);
+    cp = prior(w, threechars -1);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == threechars + 4);
+    cp = prior(w, threechars - 1);
+    BOOST_TEST (cp == 0x10346);
+    BOOST_TEST (w == threechars); 
+
+    // advance
+    w = twochars;
+    advance (w, 2, twochars + 6);
+    BOOST_TEST (w == twochars + 5);
+
+    // distance
+    size_t dist = boost::utf8::distance(twochars, twochars + 5);
+    BOOST_TEST (dist == 2);
+
+    // utf32to8
+    int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+    vector<char> utf8result;
+    utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+    BOOST_TEST (utf8result.size() == 9);
+    // try it with the return value;
+    char* utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
+    BOOST_TEST (utf8_end == &utf8result[0] + 9);
+
+    //utf8to32
+    vector<int> utf32result;
+    utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+    BOOST_TEST (utf32result.size() == 2);
+    // try it with the return value;
+    int* utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
+    BOOST_TEST (utf32_end == &utf32result[0] + 2);
+
+    //utf16to8
+    unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+    utf8result.clear();
+    utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+    BOOST_TEST (utf8result.size() == 10);
+    // try it with the return value;
+    utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
+    BOOST_TEST (utf8_end == &utf8result[0] + 10);
+
+    //utf8to16
+    char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
+    vector <unsigned short> utf16result;
+    utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+    BOOST_TEST (utf16result.size() == 4);
+    BOOST_TEST (utf16result[2] == 0xd834);
+    BOOST_TEST (utf16result[3] == 0xdd1e);
+    // try it with the return value;
+    unsigned short* utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
+    BOOST_TEST (utf16_end == &utf16result[0] + 4);
+
+    //find_invalid
+    char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
+    char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
+    BOOST_TEST (invalid == utf_invalid + 5);
+
+    //is_valid
+    bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
+    BOOST_TEST (bvalid == false);
+    bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
+    BOOST_TEST (bvalid == true);
+
+    //is_bom
+    unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+    bool bbom = is_bom(byte_order_mark);
+    BOOST_TEST (bbom == true);
+
+    //////////////////////////////////////////////////////////
+    //// Unchecked variants
+    //////////////////////////////////////////////////////////
+
+    //append
+    memset(u, 0, 5);
+    end = unchecked::append(0x0448, u);
+    BOOST_TEST (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+    end = unchecked::append(0x65e5, u);
+    BOOST_TEST (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
+
+    end = unchecked::append(0x10346, u);
+    BOOST_TEST (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
+
+    //next
+    w = twochars;
+    cp = unchecked::next(w);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == twochars + 3);
+
+    w = threechars;
+    cp = unchecked::next(w);
+    BOOST_TEST (cp == 0x10346);
+    BOOST_TEST (w == threechars + 4);
+    cp = unchecked::next(w);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == threechars + 7);
+    cp = unchecked::next(w);
+    BOOST_TEST (cp == 0x0448);
+    BOOST_TEST (w == threechars + 9);
+
+
+    //prior
+    w = twochars + 3;
+    cp = unchecked::prior (w);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == twochars);
+
+    w = threechars + 9;
+    cp = unchecked::prior(w);
+    BOOST_TEST (cp == 0x0448);
+    BOOST_TEST (w == threechars + 7);
+    cp = unchecked::prior(w);
+    BOOST_TEST (cp == 0x65e5);
+    BOOST_TEST (w == threechars + 4);
+    cp = unchecked::prior(w);
+    BOOST_TEST (cp == 0x10346);
+    BOOST_TEST (w == threechars); 
+
+    // advance
+    w = twochars;
+    unchecked::advance (w, 2);
+    BOOST_TEST (w == twochars + 5);
+
+    // distance
+    dist = unchecked::distance(twochars, twochars + 5);
+    BOOST_TEST (dist == 2);
+
+    // utf32to8
+    utf8result.clear();
+    unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+    BOOST_TEST (utf8result.size() == 9);
+    // try it with the return value;
+    utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
+    BOOST_TEST(utf8_end == &utf8result[0] + 9);
+
+    //utf8to32
+    utf32result.clear();
+    unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+    BOOST_TEST (utf32result.size() == 2);
+    // try it with the return value;
+    utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
+    BOOST_TEST (utf32_end == &utf32result[0] + 2);
+
+    //utf16to8
+    utf8result.clear();
+    unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+    BOOST_TEST (utf8result.size() == 10);
+    // try it with the return value;
+    utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
+    BOOST_TEST (utf8_end == &utf8result[0] + 10);
+
+    //utf8to16
+    utf16result.clear();
+    unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+    BOOST_TEST (utf16result.size() == 4);
+    BOOST_TEST (utf16result[2] == 0xd834);
+    BOOST_TEST (utf16result[3] == 0xdd1e);
+    // try it with the return value;
+    utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
+    BOOST_TEST (utf16_end == &utf16result[0] + 4);
+
+    return boost::report_errors();
+}
+
+

doc/ReleaseNotes

@@ -1,14 +1,11 @@
 utf8 cpp library
-Release 1.0 Final
+Release 1.02 
 
-This is the first production release of the library.
+utf8::previous is deprecated in favor of utf8::prior
 
 Bug Fixes: 
-No bugs reported since Beta 3.
+[ 1599048 ] utf8::previous doesn't play well with Standard containers
 
-Other changes from Beta 3:
-- Performance tuning: The library is now biased towards texts with lots of ASCII characters.
-- Vaaarious refactorings to improve readibility and reduce code duplication
 
 Files: utf8.h utf8cpp.html ReleaseNotes
 

source/utf8.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2006 Nemanja Trifunovic
+// Copyright 2006 Nemanja Trifunovic
 
 /*
 Permission is hereby granted, free of charge, to any person or organization
@@ -45,7 +45,7 @@ namespace utf8
         uint32_t cp;
     public:
         invalid_code_point(uint32_t cp) : cp(cp) {}
-        const char* what() { return "Invalid code point"; }
+        virtual const char* what() const throw() { return "Invalid code point"; }
         uint32_t code_point() const {return cp;}
     };
 
@@ -53,7 +53,7 @@ namespace utf8
         uint8_t u8;
     public:
         invalid_utf8 (uint8_t u) : u8(u) {}
-        const char* what() { return "Invalid UTF-8"; }
+        virtual const char* what() const throw() { return "Invalid UTF-8"; }
         uint8_t utf8_octet() const {return u8;}
     };
 
@@ -61,13 +61,13 @@ namespace utf8
         uint16_t u16;
     public:
         invalid_utf16 (uint16_t u) : u16(u) {}
-        const char* what() { return "Invalid UTF-16"; }
+        virtual const char* what() const throw() { return "Invalid UTF-16"; }
         uint16_t utf16_word() const {return u16;}
     };
 
     class not_enough_room : public std::exception {
     public:
-        const char* what() { return "Not enough space"; }
+        virtual const char* what() const throw() { return "Not enough space"; }
     };
 
 
@@ -140,8 +140,8 @@ namespace internal
     {
         uint32_t cp = mask8(*it);
         // Check the lead octet
-        typedef typename std::iterator_traits<octet_iterator>::difference_type octet_differece_type;
-        octet_differece_type length = sequence_length(it);
+        typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
+        octet_difference_type length = sequence_length(it);
 
         // "Shortcut" for ASCII characters
         if (length == 1) {
@@ -215,7 +215,7 @@ namespace internal
         }
         // Is the code point valid?
         if (!is_code_point_valid(cp)) {
-            for (octet_differece_type i = 0; i < length - 1; ++i) 
+            for (octet_difference_type i = 0; i < length - 1; ++i) 
                 --it;
             return INVALID_CODE_POINT;
         }
@@ -225,21 +225,21 @@ namespace internal
             
         if (cp < 0x80) {
             if (length != 1) {
-                for (octet_differece_type i = 0; i < length - 1; ++i)
+                for (octet_difference_type i = 0; i < length - 1; ++i)
                     --it;
                 return OVERLONG_SEQUENCE;
             }
         }
         else if (cp < 0x800) {
             if (length != 2) {
-                for (octet_differece_type i = 0; i < length - 1; ++i)
+                for (octet_difference_type i = 0; i < length - 1; ++i)
                     --it;
                 return OVERLONG_SEQUENCE;
             }
         }
         else if (cp < 0x10000) {
             if (length != 3) {
-                for (octet_differece_type i = 0; i < length - 1; ++i)
+                for (octet_difference_type i = 0; i < length - 1; ++i)
                     --it;
                 return OVERLONG_SEQUENCE;
             }
@@ -332,7 +332,18 @@ namespace internal
         return cp;        
     }
 
+    template <typename octet_iterator>
+    uint32_t prior(octet_iterator& it, octet_iterator start)
+    {
+        octet_iterator end = it;
+        while (internal::is_trail(*(--it))) 
+            if (it < start)
+                throw invalid_utf8(*it); // error - no lead byte in the sequence
+        octet_iterator temp = it;
+        return next(temp, end);
+    }
 
+    /// Deprecated in versions that include "prior"
     template <typename octet_iterator>
     uint32_t previous(octet_iterator& it, octet_iterator pass_start)
     {
@@ -473,13 +484,20 @@ namespace internal
         }
 
         template <typename octet_iterator>
-        uint32_t previous(octet_iterator& it)
+        uint32_t prior(octet_iterator& it)
         {
             while (internal::is_trail(*(--it))) ;
             octet_iterator temp = it;
             return next(temp);
         }
 
+        // Deprecated in versions that include prior, but only for the sake of consistency (see utf8::previous)
+        template <typename octet_iterator>
+        inline uint32_t previous(octet_iterator& it)
+        {
+            return prior(it);
+        }
+        
         template <typename octet_iterator, typename distance_type>
         void advance (octet_iterator& it, distance_type n)
         {

test_drivers/smoke_test/test.cpp

@@ -42,8 +42,24 @@ int main()
     assert (cp == 0x0448);
     assert (w == threechars + 9);
 
+    //prior
+    w = twochars + 3;
+    cp = prior (w, twochars);
+    assert (cp == 0x65e5);
+    assert (w == twochars);
 
-    //previous
+    w = threechars + 9;
+    cp = prior(w, threechars);
+    assert (cp == 0x0448);
+    assert (w == threechars + 7);
+    cp = prior(w, threechars);
+    assert (cp == 0x65e5);
+    assert (w == threechars + 4);
+    cp = prior(w, threechars);
+    assert (cp == 0x10346);
+    assert (w == threechars); 
+
+    //previous (deprecated)
     w = twochars + 3;
     cp = previous (w, twochars - 1);
     assert (cp == 0x65e5);
@@ -155,7 +171,7 @@ int main()
     assert (w == threechars + 9);
 
 
-    //previous
+    //previous (calls prior internally)
     w = twochars + 3;
     cp = unchecked::previous (w);
     assert (cp == 0x65e5);