From ac292859544459985f958e68b62cd726d42f51f5 Mon Sep 17 00:00:00 2001 From: ntrifunovic Date: Fri, 7 Jul 2006 11:48:37 +0000 Subject: [PATCH] Initial import of the project tree git-svn-id: http://svn.code.sf.net/p/utfcpp/code@1 a809a056-fc17-0410-9590-b4f493f8b08e --- doc/.utf8cpp.html.html.swp | Bin 0 -> 139264 bytes doc/err.txt | 10 + doc/utf8cpp.html | 692 ++++++++++++++++++++++++++++ source/utf8.h | 504 ++++++++++++++++++++ test_data/negative/utf8_invalid.txt | 271 +++++++++++ test_drivers/smoke_test/test.cpp | 144 ++++++ 6 files changed, 1621 insertions(+) create mode 100644 doc/.utf8cpp.html.html.swp create mode 100644 doc/err.txt create mode 100644 doc/utf8cpp.html create mode 100644 source/utf8.h create mode 100755 test_data/negative/utf8_invalid.txt create mode 100644 test_drivers/smoke_test/test.cpp diff --git a/doc/.utf8cpp.html.html.swp b/doc/.utf8cpp.html.html.swp new file mode 100644 index 0000000000000000000000000000000000000000..d4d7961b36d96107a1d5c254301b55f6cd21fca6 GIT binary patch literal 139264 zcmeHwdyHgRd0zp+G0$KV)&|@7(1Uk-cKh8kO>a+6`(CHM_Lf^uvaOZP8h&0u|B0WkyrEXJ zeb;IQwsT}P)W@m6`$lKkZyHYOm$47u6}#bT;I|sR-fGin(4X7*d*#6IcKCm>YpOph z3RDy*jRJQ!H`doKEWD1r@;mRAcGK!c6$L5^R1~NvP*I?wKt+Ly0u==+3Y-oK_&rwp z7P|U;w3nkjPqx>|-((V?&t8DGjJ79_{~Uh5G=)A=2F{9^p3&wBEh&ZW0Riv_sQeG zfZz1}2jnrm7v2At^-@lf}uflKY-yg{1 z`|z9k_&Is}UGn&U$m8$EZ@T`g^7z&GP1ipU#3TB>293Udi9G%u{7vt3pFI9v`TYfX z{C)CxMIOHv$MpP%MAYd9Upy}EE=ZEd^0U0>L{zvExoU8S$~k|!9KH;fHq zVQ;s-_w?>+eJ}N-@GkA>E?&>DS)*fk-qnSLy*mdc>jiGl@l58ln15h-%y50H(J>kR zv>l)M4!&ubF2i*e0v4QR(C{tCrZ4W=R>NuHRGYO;+jNc25>9KfyLTR2-lk7`{ANMN za97x^7CUwV+}`B(Z5dswV;L^%TmFH%(-Nzj%nNV|?%p^s8;8sZe73tv-5wx8_;w@E z=vpC-@Qf5PZWbqZfQdXOa2uT0EW72nU4yd{{!$#U!mf2ZXNg(9V!-Y(GpsH#gwZ^58io(% z0NVzFWd*C5Ht}JTd5F-l+a9sr^@ko}J=bZwM%TN-EWvzz3f^In`S%{N5R>QQxO$Jy4hjp`U**4fnNJP9B!m(BkSj+0* zNsr*XK4cj6%?a8E^cxrU5AZFh(X_yXL=4~beB6V+h9K=&b=Po-s6wdLP16>yPeL-1 zsmG=t3Q9NN0@q>n033~X=$S5FrEA!jK$hnPCJ5B)fF^ha5F8IZw4FYu&#ebK9oBRh z=1v_QAvzh&rfYg0y<&GIo2?3kffirYvL93=bhMuHB#kf4_@;Seb{tU3V_nb)#E6)t z=eRy+h`w=5W6OmAjkDtF0V%S=@AwQvchhj2B#s(^>zcL??oyW+ZQH>>H^jOpxPXs8 z=V@G0H|YsY$L33#T0kLs+!`YcNzP5Y5G~Hfk~J-pnDh`VeEsecd+HhH z^~{FVvKr_}&!}Ui_HoJ*m*y5`^)YiDuox}Qc#h>Vr*DJtI{|gxI}QAGbE9xIkFRQp zwXxM&TWhtrT5v0-5*6QYl+{+RV-w7!+h~u}J4Sxk|+A5|BmLjw3shC{; zd*$s`Puyc+Z-R?=SFeS7d#-{Tt<%Z(*)lF04XAv(!QP{oe;vo5#fK|cv{-p;nO$aY zs_(7_;d2u7h^rHYa=&$VsW_^O)!^6cu?UC;Ha#yze$KO z?i6TBeo?FNi2PTp?e<)AueQtew|ZOQRr50aTY%Pc&*$)U4L=JwZ}q@5n)ux{eS^_l zSvH>wtfQ+74PaU1+#*eNp&<$28?)?gsG{c1=d6g^abT57n+vsA-KhDvn z@3^r`-(q!F5w}lg`j+1@_ufU5H8ZAucU7EK+m&6Ur|#7V|9=OZ|Nnsk0D%9O@#`Bf z0=RHK|2jFO(QX3cZ=!9WJ%IKDX#X9~Ik_zeW3dXwRYjH`o(Ch4#B>zk~KGXrD(A!f&GW z(T>o55bZ0n@6U%Hd(59*0bb=VWh!9!0g76DUCUeFTCu!+Xj%5+#f$7668_XuE^t~v z3MN1lU9>1tIg`r<{r^}ypgcaU#1*vpps9JXKH@dwmG2@evp@HpLZCGLmwz_@^Pl&Dl;t8A- z>st}69s3A`;AEwj=ct&ISJ3sH{Edmit_j#|vo&!kebS(TETNMiLpnK_nB}+ZBclUs zOvAp-UqE~=dctSA7=8#AerC?_lgz16=P{m78o(^vU?>bmv8RZaZ<{_D#ui)Nw&Wqn zCc=av*eWFDZLC(Dj&uzy2`~f^vl0gn(vmuWo341KdDvcD<6>-Q2ka{EPv0ls3%s|8 zcJX43zM~(y9uuWWA1|@Z*p;;S5wkwK`J)kus2lqo({96HTEdtOGuEb}Y~G7mQl{_h zz-+uv>o=#5>eyBOR=(Nkv=`+I4c`sZSbm2w1abwDVe>V(2w+{qJv7}na+@7tPVdm_ zk(q691m?%h*SSH8tmCu@@?J*^S{Xx=J)J%JZ97Y8PP4 z=mhW$l)$J%GPX47Jv0TT+ixLrN}$M$`wb0^t8r85c{Ws3;3NTBV}1!O6X6|)S)ERR z7a8y%a04wZKFUhF#QI?30~p}M^ftJB;}``TPrPp6RICq}6iE*PO_3$?4#XA_%oHaF z4i~-S^`{=aQHTj+$QzSyXGii`f^J$ET=%I$PZ zkV*9V#_|6aF!b9c{->Dnm&3pRP1pdwiuS+JK7r=I-`_=hKH7VL&)*9?{$s%3blne* zkG;o)>?i0l!J6WK8E0^-nH=)CC@PwKH!le`NWa6hJ{`s8t$IdMcEHe#7u4z5{T(_x z{ffzZA_7Tb8be}Ph)q8Mz>FSL!e&WxPYzx>i3U41f%*03W^so{pul6wPt6`9=*N|U zH!Hoi_z5G4BdwlJLXvX)TenTb{s0QhM;ebmC$w|}=`Dq6`^a(v_<1R)$>WO=R|a{E z1AxAOO5{Jq3bh%9H3&<=_?^{Nw}>&bpH{de>OL;Qc0177T*MPVK?W6?Gt_Hg7lJut zmM^V>R!C!Qnm!DMQroW|Lz5NZGGwaAj2j{Kmf}CaHrf3yTK{y%O4-k{pIAP7fI!{Q z4#&e`$fCgQ1z;3%OM+Ek9q?61J}G6gJ6bkr=D|6A#6vkfiWxyL7i>!$i#Om=xN_Ec;=pzKB0T9{!{ij8 z_or83c!+kPjJj7hY z3PuL@2d4p}7fZM)IhNG)7Cs+hf%UCt*M3xb!rsY zTe^aWu5&0j{(cS9gY0|loS4C7y-8M z$0y*Q|2W#;Mf(M`FUa%04%}b;R1~NvaEd7qTCB1)uF2Lvx^LdS^S(>VO^G`~^eogh zWdBgWz+!*}g(A>?d|D5YePJ1`VZQi)q^gH2f=k) z3m1B>!~j7w*;OX&b_jM^yL1Vm4gsB*%~DFcNZ117993-XitK69ZZFODXV0UPpf?!# zGKPY^V{J+Pd*_v@6M7UYu&$+`(aUC!LP@9lupM@dZEmnSMd?7pZ^QfNBGTIQ5CbYa zs@&$*b11NYT!h>TO@EP+>LF;9Y>+u1sEDB-xjhm`PlM2S%b?@Wme|PW3j7{oyMU_T z@J9BKBqb8ED1As+G8J8szC`giF}j23(SV!`vfmwP02Lnud$&cf8;ni?jMoT(PsqBs z52}#cwH^&tryve+cx`@tj5`$f(8|*=h#0AyQ^jUp1SOkBXWbM zXu{;7Xw}??^E`SqRBZK8GwOGgd2d=463wFF(v|For7moT1<@B6=~58YzxxR-s*5(h`9@ z%7x0y6^V$0NQxlrAV-@l5`wzQBn2Dn>_}hkKJ{Xs!@=Yr#l>*x`Ybae_cBRNc%AK) z91;|EKx;T=g)O7ViEPrAJ-9MXGIK>Oq@*r4p@;|;KT0}6%#5gmL{(3^X4gSwnPrNt z7nxKj7=v0qumFiAmRAzt_Mdo=4%vHfx@l7p8+8+P<7QZJgi5LKf+VQ1BcptH=@FDy zap(@@tyh(9&NA&A5GSah16#zgu$9$KzfS>y>)bqXNvtrKVFLzBD#X%r4OBmBAQp_O zNcDN{SQi;J7$Bs;GgXs{0)A#kLY^m*NTbbucKNIfGNI%Wuk%LLlSrep%?8R7QIRCc z(csRIA7`e~as1y#4F88^{6E#4e>HIb`{3^n(0sHzXwRVSqA|3OApZa7U=yHgetCw7 zD1i+Klw=QKk@V9N>xt(s?4>Wdp=M8pd%o`G)|iOdrV=8xz;veb#_8wI_R~@P3u-LV zdnWFX_Z`u1z}g+7qPqL=NueSV8Sh3N+Qg8B_Zf3c;dvOSf*ExvsZ!BVHYLgzTjxTV zxcGP%Oh30@5lI&k<@S zsu;xXpS^q(sZ4W>D4+zr%q4dU14|J`sRmo3H|nyYWF9H+W|J2$&F)$eNp)Fnru>Z( z*KkI|KQHRe&h9N|JKC%nPz_~{$zUzy*cJ6iX`0674dw655UmQe54MkeVX)YDo!KUX zrD=ZSb3lorLn|}|g(Bw{6+_ZjhJb~kU!vBn6fL0?k*h&WO6Razp}5lmJRJXj3^@K< zG5k-t|M$TVZ~&bDM=wB~KeUgby&vso(f%sh+tBEmm(g|A&l#aW%+irP+lx6+Gdvql zo#xZ>?w$8tfupJvBn327h@!$Y?F5usX6pq|icnewrxJQ&+%BZYq{*1*3QiRY0bSiT z38&&=q|m&$Kbx0pv*4E)Ku^;8G6HgHni|L{oOYwF%SF0=i8_!jX?qYc``D5NNJ5^#sLl52D$)I!>-UqmFu`lxW|AlTZdv z!nx$(PXk-&5H{l;LTt}rv`kd~$L0WPs~s9!0BCFp;PGO9Ll5Kq<5s;Wc0BBwVnFOg z0h~xdoOv$h88XJ1QjX)bNjTr+5xJ0k^|a#bc~Uf*TF()IV9|#x=kZ^)p13o*j0OwN~J(P4=Rw?fDjmN18Ttib0*HTVP0@%a^W;P+q9kJ}^ z6b+cJFg-H?>S1J>8vc1jr$x)t`3zE1^q)(h8s-1L5m>%~M*05~>i;#={Qn&4_J0Px ze)`<5F#Z|A_j7-Zn1MmYm0PXxWw+L1n|SoX-nHft?JP#=R#9|y{a9$GVlyansy!Pa zT0ralMtLFZBo}n+NX3H*GFqj*5Lqe5lv2%Sr*e%?>eH}*!v(HV&zcr(j@dAIC3(S< z6K%*4#fqrRvm*x^4`A;|zH#2n7WKq?F}sREukQ&jjvR@(tnTB_DlO5Y!luT5obDv% zI^&ETg|InD(*k_JFj3G34gxVnh5#~50BC{xu~RDd^e0Q3DZxtTaf=KL7;@2xZZczM zqYBtR0nugI3{);$%!!~X!i!oH;|`2iJ=a3n%xF7S?)}5?hv6Og8|E<;V@5H`*l@K; zSUoq|j5dKq{0aDQDaTz{^g6l|Z?+)_eQaS6i^t0jb%`4XY5L5vRBMD^CtzRQae&GO z(j*8XjO1^_mJdokrCNK0xtv%|hNPorDfVl9z{g(Ce4{DJeU8Igw2yiAOPy}s!n9K+ zGwjo(tYn;`d_rQ=3$pJD3D0RO7z;!d_G|G?DAA+PE55@!FKjA?L_CRa*bSPH42CYX8=YIG+ z*ZiLYj833eH26xP*D*q_If*=l;x54r zBRq>&C>P0qa2sNyMB4i}XOS!HfYyAnkP^rC>5K^Dsnv=d(Wb}JpRMR49F3Ws0!1z8D<mOP(15UCPKYn9Z~M+yJ`B(VH%6P^eDhwH!g zBE zz(o}*oxm6zf@Hr^>LiHwy4aZ(5w%M6J$s41d{?$?{wbW7V<3qbbl^tZBwiQ@NskoJ zQ>xZ?XfvYmXriqW>419!`% z;P?d86&GhF20M%j zzK1T4apQVnS}dB_TXiT8U0pkhT7JQ zjY?4g@kE#`!gc*+r$tsJQRQpy9C5&{OScYfo^Vrd+M+tyx$7p;M?K`lw5_uM6h z0{O^10No5^d3nb1BYt~*`qys2s^iUHy?M+H$yy&GgiOg+r*2J%nU zJ_{nI9y6%pIER!BW%_)pNNm0Uot+o15SvGiPg&iSlH20Kxivc{2+IGr9oO;?x-+zT z6aN2~@a2D-#Q#+P|04eTIr#p6ANv4&679Wc??LOJJrC{WunoKzwgI~4Tc;2CCwiQ~ z;d#*lMf0~D^iNYphtqY=y}EEA#fQ$L2PVJGaOirt21RRD1kIIbNUo}sltkLt`c`&_ z%2h$KpHnp6x>98}K8pd|J0EDi?K?0Y4P>4xWS%q;P?+H`T0iH=oYSvZ7aRp8>KP4Y zrI5#*0LlX(rhQlwZk}!cZX3xK421$nS2Cm`^9i1>wG!nFdEtwWTLG0UA7YBHQ0Um= zMT#U_U*qLWrD1Q84}wq}As^^dsYO1-uw6l7BPTjn-tQva;|Li&j$49nbI5y|$k-_1 z0ny5+YIj7hQdMqEXx?$nWy~}iBnh~FoR`#Q7?=3g7i*U;@y#!27ZbiA#`fkV{=2zu zp3?m==3DbKH^YXkE>h+X>wpd64aHe zbe>_7i`8V-RCfX&hL13KjnE6xxYscnl%K>8q;Xl4k`-GiQut`EN9JcbTXdc5C4%cc z+R!=2j#RuV7i*5uy2I=Le-ilrH4^_L?7#M};Q!wN{{KF-+raxjjP?!G1o$(wKScWo z+WXKRKzr>A#I+wzpz87u25|Z$d;jU_#EA#rvHXs?APYNTt=`;#6S6R&oexa>-GCocJSm$xHfM*$|BNj|PC(TnSsa)S1rW9-%p`GDO;2{#`w%Ep5 ziVjucxAIgbIVC>ssf-a*8Fd%c%2PSG(K-Q>ot?^4>A1QgX9ASGz9qDSXE(8J-&NYA z4_E{cSLKCaP{LC-)G2^1>8HfNHl0zUAtg&4$Itszv}hmo@m&XXT1*7u)7OKaVoQ;S zQ_#2SL=1kj65w$*{^n4wgeSySArT)f;Gv-Ul^Vm(+P&5q!N7-Mr^Nn<91=b}3qUAk z+WAUfOJh0(({uK$%&}q>Ee)et%SKUqbQhUZfX=rm1au&FG6Lsb586t(`3bD^)S>+H-3w9dGAbHji7b>R_%vIc^;8p1Uyusa{>uZ~r zwg7o6*XKw~lkJ8hCr_eSi1Nchz~`Z>#KL$=6!DV|ea;}sPd4`1KEco@dz~-GW+d+t z1-fMpcYOo9IDPBCw62`gXB z@+VxSfRQEKHq`^G-k^$sPJqCekySIOAQ8CER7qBa{Bsa}#{(nB&D%uJf?Faa`P#EAr*ahx~U4YL0>Y0rDKMTf!%|?Tot|Kw5r%B+el6_nu(S>&H%7gsqs6D6tMjR} zJ*HoRFEhVTVIIj7v~8afI$u998;97>{J=y2xeJ&a0%z0?;!q6x#8HXmMD-9}c&Gx= zCl^H59S2pey?s7v6>LVRMYy@oP&S|<(^-bhLz@*~a~+HJ7B~*KE$AADCPS$X>}$$H z%EJ{4(_c*fx(P=CDv`NP!}L5V&5jD|WM5+r7apy^ZulrP=J9P$ahd7$%!b9EVs(3{ zlW1TZ;B=q|^ZgB4$YJ64Y&vv1!#<{xWt7U2lx{kSD%vIQ(Q6WdCiIF~*8dNS)q9NZ z_AoPXnp4)I*!(+YDxyk3MZFn>9x8htvJCm2-HAZ52SeNa)uiz+b|J4L6T8b6B{h!E zGNV=NkSpd{PvlDsMtrnJ3+!td$n7Ki|0VeN7he$J|G$R1e=mdY|D|aE4p{$FXupB> zG}?oR`=@i?TjBpvfVTEI3Dm8n=qAhiYnrPGMz99To*^J2Q6KkM7gsj-^FzBZHINZt@E2_TXBs5!p}~&PQluq+4&+CsUqmuxEA!2 zK=acRK{vi&(4Ee{3-f>G;FPW)&^k>9q|lC?VZFXGfr&|%1bhXZF%6)l%r4Mxz$U`+ z|C@mSe@e#x)Bb-Cz}NpQu>a?f|NmLEUq|~W+6U1tqx~9e06zm809|uf*#Snz_(^Qv zgg4#FX0@XACl$hmuyR`WRXNh3Yb(=m5ZY>YcEoNM+=h_Xue|R_Dh;GyP77kEbo^AI z?VFn;>4T~K6|*-yrd$NjD{8uq(}dssOdvIXWu_;!HK3genBDv)2V^Z#3?v}~Z26U|Y+0{n>CwrrdH zsnVCaxh%3^LX|j_F5@qbB?8q7G(}Znlm>JHK$BQUxCxbuKbTkD!wNO!>x^2K#91jE zxiMZst%yf@r4Bq>ghp{OTV523F)hkuKPOeej4AE9(0n!PaAN1IP=Ta6#8E33Xj6zE8xzvl41Pp9?HRd%3I*r(XwwY|JxSCOa}++9Z>7rtt?ySNe!ikd}SNQFH-=`cntuhIC?>^*RkLdrGov8?*fZ0)Zu<6|0@7wGr*RlX(@OC5Ttpgkz?@e zn%Wv|y^LMSjBfAB{q!F-waAG$d=@;rE}yl%J^g3NE#Id9Ai4eA^dEFdP>-N{BDn2A z3SwZ0NK^r!%BZ-A`gFW-eNe&uDZ%~V*~=Ie_KvkB`R|lM`$jK&wJ2R~@33nOwa-v} z8ZolI*~TK~nmtr_BBMXWPsao`nGn^6&|(K^1^;v8vJyOrr6{Idmx%gx*Q_~Iq`L-f zV1&)M6E5 z%+3KiD=qd0-#C%-v7{V`_=_3zaaaX`t6-$aNoY@Mn;;Ms35fZ}NlvKCs=Tg^7XF>w zyso4nK@R9*fYo4gV;`z~@8s0~UwA>S)k=;Q@-#qznaKfOp|3QU@058qrrnS>IUqk1Qwf27GpLfq*IR z@+4ew3n4PRNM|r8TT6kx=)#q>_twqUg}sl+mQ46h+=SZz3eJI-H+i4hj4VGxh7`^N z*}7hl=A_IvI(MLR^Ulxvjx@Mz)m`-_<*d-~5SeI_&&e|)DV{7g0FC6nuMA14Y?U%Z z!jQymcnRG-JarU$>#F41W~8KYlA{n8A}1|xLXL7){Gc%t9{W73pzkye4A-OG6k$m~ z9+c_&Y_ZX!FW2sET{e08BW;~WpT!LkG?a_|6q5yR9G2Rzb}~pXdkm;f6pC;@b~2c0 zPy%A?N7I*vR2?W#LTsp@q?n$9Iy4wl-KLGn?h4rsBj|?DuQCe78BkRY)f=0znU~p(MJLuM7iG;B&nysj* zBr6ZA1uSNjEP}>DTOY>8v2tuv!q*~MKlU6kO^3Qt;qdLcGBjl`T}3i*S~%PgY+EeI zM}dWUoy-ASMbyw*Ur%s6M`#5}pp*F8Shxo3E{+pIajsf8E!ZZhOyYma|0n9!`6rKd0%geakq10T2IrtZ~D0- ze42jBD23UyNOgh%p@QS%%Vk}Gc@FSksY*ezK62l2Q4`&<5mR{obAjVuL2^XjaGT-) zv_(}*#O#p4&w$gY4JGPml9I)`qibhRsY0Dh_gPD1>qOOOq|U%HV8TYwm|#hMn%jl#pbPoI>4z9V~0F|NklW z`}-;i0eta=wOX5E{Ly{@?e%D{K>HB<{&fCNPYdqHK|KB*pT+fNl0fjnufDLCM5!h8 zA@3G>-;p564iMLR#pFAksMLBjy|XD%x*|0p^v+pu3%jZ{YB>vG@q_e~_Yhjqa8T?s zhfSXKc8FarsVMe2SqgAxM-APg3de=ij^e3#;B@w=7OhfRlp3gRn*G#by6I>T+0Q9_ zJH6pKkJyUWRLrJu;YjMLIZ(%A8Dc!=GLg}x;y}})oU^-G#0Yc5I#}>#S@kZ3zQS4u z4CF7P&Vx8o$!^rU0mpTOgD6*z0(Vuw9>F35CNSSUCK!srYnq4!>|&QoeqP74+x`KU z`wfh8U3@w)yhu@wK`o?dx>S6n@?mR*w{vo2`zBH`<3ZLQ0;pk~4eQllFS_jM zQ#i}DmWiANwP>DdJ!d$w9O<3NXy`kc9F5&eDE$-+P5G)ziLB7{;MEvyyE9WsvPY_j zfV$#%9;kCF=rRPdp5NTrQK<&|zH1;%c7j2k@FgiF}^#Seq z)iVLLOL)N0;>qnOf=X4id8Pmz(>}uYUnB}EUQ!X~m)5C1y{H)|^7`qxxxIOb|8B0E zr?kA?DcFk*;!hb(#2EyUG*`W=2;%zGf482 zjqtWBfH&abO=ZE#!&}}4WibbDu(=U;{Z7St#`B>0atbXC9^T|5Ll5u3$CZb7t<}PQ zboD&~bbxiyUbTvrb3u67x__#1WLy>VWQPX}fIU8J(_P2M3J=E0Vm>$H4Y?`)|NX%6 zpN`@GKLY;$q{RPU2mXH!`2QaO@Bb>=0a_dF1{z)S?@uWHuP}f1yr0ZSF$@&qn16Fa zXm2rBw5+@pkNd`S+}y}W7f(U{>ZVf;`38!d%6iJhzaUE$>79_l@x>GCrakAk*z;5N>O-6X9 zO?qJstRlQi*`Rz*_@8TOS`OX;lYp4XRe(2l@NRBY@SksCo()Wj3=+C1TpD7ezcKr{ zg@8!+!DGRHT3P0T{~raG|4B5p{~!7PKMnlkmD& z#8ho}!d@M!&W8&!2!t&CB|G zr)+&C0aD*fos1#>M^yvv+Ta{F@UVXYa5;C;V6TG)u?^LR?isFcHG&R|Ax+b>+E{5_ zW*(vD7PfM3yG9o^(Aq#Q#~m<;D8v<@q{0e|?Die2?qVy!C8~DD3~(N*?hv$e2JBdn9KI?T1 zA1~9T_}(V+_q#o-gIs1}CE_h?#bG=4a(EYB;SGCIpr~2b=yagtG)QFO;h+U}mIKRT z*WS>c2QD|;w7Vt5Tl7pSBjQnqT@y9?u=F*}mUtq{qKis?Fa+9dZ1F4i{3Y(@H;KQb z)uMMkW=+R_z(-Mazi~hxSZ?@I;w^A@Bk1_(tMBu9BZxz@0VC|Pw_0|SO0zF=5hVErl8C@7c_=dn zB4Fj9)q_`>$AY7*92g(Z!vG-wUYsS~FHTj-I-ELu2{Sx#8w?@N!@A^AfqTya(@pZi zGn@ZfYv3Ke9*arT$D0cA3r55`UpLRuQv9e`ciN885uyQ;+oBQ&r)^=rIRa9T7Ura` z6)W`iv;$ejMG(m}2syuay&q*6)a%{^%ZlbSfQ;v2{<*+voT$>`WCkFIV{)WI9cvVt z&%^*QFaQ5bf#Y{2{s+XbeH_^TJ=p*6Cb0fZV126h{~>wKBl!M<31FQ~)l}6s=JFd+ zZ5hSK01REAN&`fvAjA!GyMFl%JZeWk>4LjlRdV|JElk!z?3^-d zK%Hv17GMeLBKrrlz!1;{itPee;bVif9jA$#n7#0nriGr;4QA$lnqY`ZVo`{KMhE5r z)B`ONg(HD&y>3nL{v)meQt&Z^k+C&0g6)L3j1bq;Pd#(%FR=pyn`%Y; z1omqP=cQshVPSJnAf9I~5Zc2@25caJM1%%`Qo(f`*8;)@bznlmrX0$5k70@vsz0U) z04IW%fRi}P6xR{rCBV<;gGHE=b1mmt({_UP!4fmQ9yDb>!`d+K2@O ztzBB7192z#6e9Iyyj$$9UJK2dq7-=sH^%gd#9(Y0myHHgo!wyXam+8oQVVnV?rIR; zIB|g%e;1E$Own0k*BCc}Fg{7K6!Ll6HJY$PWv*qyMgkn{u$F0dAS4cFx$7*iFfU0q zKApzEJS{Hkuwy!8A(4km-(v_&DNhqdB@$T>jV|8<0-HgE!VwAr86LU6WQ9EfF(>^R zBnm;xhG}~wr!CJ5sCbbN>s)R38h2!DP_+7 zTi5R@t#U4N{}x~SQK=IS7N>8zt)R2SE#4h$E3<^TZO}5j47)0pahQ{FAf|rbYMM}( zq5U=?K}ZhNtu}3VB3(G-WU-gq!X_&&X*#AyR%xif$JoaPdIPtm63Y*2*a$Cx{?oE- zRDe`^dAAF7h))@s8c-XEcH993ZG>dVqm{H>psk?Vt`peJWj-Tz*o9VVm5pNK z=&)lpC#?@i&g|G5YJjRMFt|V3fX0p^R z9Pm8AL^3kM_6(~_jAb;BoCbxQlAE2!sfH~jM1Y+jfpsWt0s{~r=U3X1>$ literal 0 HcmV?d00001 diff --git a/doc/err.txt b/doc/err.txt new file mode 100644 index 0000000..0dbdfb2 --- /dev/null +++ b/doc/err.txt @@ -0,0 +1,10 @@ +line 2 column 1 - Warning: missing declaration +line 2 column 1 - Warning: inserting missing 'title' element +line 182 column 1 - Warning: trimming empty

+Info: Document content looks like HTML 3.2 +3 warnings, 0 errors were found! + +To learn more about HTML Tidy see http://tidy.sourceforge.net +Please send bug reports to html-tidy@w3.org +HTML and CSS specifications are available from http://www.w3.org/ +Lobby your company to join W3C, see http://www.w3.org/Consortium diff --git a/doc/utf8cpp.html b/doc/utf8cpp.html new file mode 100644 index 0000000..7354353 --- /dev/null +++ b/doc/utf8cpp.html @@ -0,0 +1,692 @@ + + + + + + + +

Introduction

+

Many C++ developers miss an easy and portable way of handling +Unicode encoded strings. C++ Standard is currently Unicode +agnostic, and while some work is being done to introduce Unicode to +the next incarnation called C++0x, for the moment nothing of the +sort is available. In the meantime, developers use 3rd party +libraries like ICU, OS specific capabilities, or simply roll out +their own solutions.

+

In order to easily handle UTF-8 encoded Unicode strings, I have +come up with a set of template functions. For anybody used to work +with STL algorithms, they should be easy and natural to use. The +code is freely available for any purpose - check out the license at +the beginning of the utf8.h file. Be aware, though, that while I +did some testing, this library has not been used in production yet. +If you run into bugs or performance issues, please let me know and +I'll do my best to address them.

+

The purpose of this article is not to offer an introduction to +Unicode in general, and UTF-8 in particular. If you are not +familiar with Unicode, be sure to check out Unicode Home Page or some other +source of information for Unicode. Also, it is not my aim to +advocate the use of UTF-8 encoded strings in C++ programs; if you +want to handle UTF-8 encoded strings from C++, I am sure you have +good reasons for it.

+

Examples of use

+

To illustrate the use of this utf8 library, we shall open a file +containing a line of UTF-8 encoded text, read the line into +std::string, convert the text to UTF-16, and write it +to another file:

+
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+using namespace std;
+
+int main()
+{
+    // Open the file with a utf-8 encoded line of text in it
+    ifstream fs8("utf8.txt");
+    if (!fs8.is_open()) {
+        cout << "Could not open utf8.txt" << endl;
+        return 0;
+    }
+
+    // is there a utf8 marker? if yes, skip it.
+    fs8.seekg(0, ios::end);
+    ifstream::pos_type file_length = fs8.tellg();
+    fs8.seekg(0, ios::beg);
+    if (file_length > 3) {
+        char bom[3];
+        fs8.read(bom, 3);
+        if (!utf8::is_bom(bom))
+            fs8.seekg(0, ios::beg);        
+    }
+
+    // Read the line from the file
+    string text8;
+    getline(fs8, text8);
+
+    // Make sure it is valid utf-8
+    if (!utf8::is_valid(text8.begin(), text8.end())) {
+        cout << "Invalid utf-8 text";
+        return 0;
+    }
+
+    // Convert the text to utf-16
+    vector<unsigned short> text16;
+    text16.push_back(0xfeff); // bom
+    utf8::utf8to16(text8.begin(), text8.end(), back_inserter(text16));
+
+    // Create  the file for writing the utf-16 string
+    ofstream fs16("utf16.txt", ios_base::out | ios_base::binary);
+    if (!fs16.is_open()) {
+        cout << "Could not open utf16.txt" << endl;
+        return 0;
+    }
+    
+    // Write the utf16 text to the file
+    fs16.write(reinterpret_cast<const char*>(&text16[0]), text16.size() * sizeof (unsigned short));
+}
+
+

In the previous code sample, we have seen the use of 3 functions +from utf8 namespace: first we used is_bom +function to detect UTF-8 byte order mark at the beginning of the +file, then is_valid to make sure that the text we +loaded is valid UTF-8, and finally utf8to16 to convert +the text to UTF-16 encoding. Note that the use of +is_valid was optional in this case; +utf8to16 throws an exception in case of invalid UTF-8 +text.

+

Reference

+

Functions From utf8 Namespace

+

utf8::append

+

Encodes a 32 bit code point as a UTF-8 sequence of octets and +appends the sequence to a UTF-8 string.

+template <typename octet_iterator> octet_iterator +append(uint32_t cp, octet_iterator result); +

cp: A 32 bit integer representing a code point to +append to the sequence.
+result: An output iterator to the place in the +sequence where to append the code point.
+Return value: An iterator pointing to the place after the +newly appended sequence.

+

Example of use:

+
+unsigned char u[5] = {0,0,0,0,0};
+
+unsigned char* end = append(0x0448, u);
+
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

Note that append does not allocate any memory - it +is the burden of the caller to make sure there is enough memory +allocated for the operation. To make things more interesting, +append can add anywhere between 1 and 4 octets to the +sequence. In practice, you would most often want to use +std::back_inserter to ensure that the necessary memory +is allocated.

+

In case of an invalid code point, a +utf8::invalid_code_point exception is thrown.

+

utf8::next

+

Given the iterator to the beginning of the UTF-8 sequence, it +returns the code point and moves the iterator to the next +position.

+template <typename octet_iterator> uint32_t +next(octet_iterator& it, octet_iterator end); +

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the beginning of the next +code point.
+end: end of the UTF-8 sequence to be processed. If +it gets equal to end during the +extraction of a code point, an utf8::not_enough_room +exception is thrown.
+Return value: the 32 bit representation of the processed +UTF-8 code point.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+int cp = next(w, twochars + 6);
+
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

This function is typically used to iterate through a UTF-8 +encoded string.

+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown.

+

utf8::previous

+

Given a reference to an iterator pointing to an octet in a UTF-8 +seqence, it decreases the iterator until it hits the beginning of +the previous UTF-8 encoded code point and returns the 32 bits +representation of the code point.

+template <typename octet_iterator> uint32_t +previous(octet_iterator& it, octet_iterator pass_start); +

it: a reference pointing to an octet within a UTF-8 +encoded string. After the function returns, it is decremented to +point to the beginning of the previous code point.
+pass_start: an iterator to the point in the sequence +where the search for the beginning of a code point is aborted if no +result was reached. It is a safety measure to prevent passing the +beginning of the string in the search for a UTF-8 lead octet.
+Return value: the 32 bit representation of the previous code +point.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars + 3;
+
+int cp = previous (w, twochars - 1);
+
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

The primary purpose of this function is to iterate backwards +through a UTF-8 encoded string. Therefore, it will +typically point to the beginning of a code point, and +pass_start will point to the octet just before the +beginning of the string to ensure we don't go backwards too far. +it is decreased until it points to a lead UTF-8 octet, +and then the UTF-8 sequence beginning with that octet is decoded to +a 32 bit representation and returned.

+

In case pass_end is reached before a UTF-8 lead +octet is hit, or if an invalid UTF-8 sequence is started by the +lead octet, an invalid_utf8 exception is thrown

+

utf8::advance

+

Advances an iterator by the specified number of code points +within an UTF-8 sequence.

+template <typename octet_iterator, typename +distance_type> void advance (octet_iterator& it, +distance_type n, octet_iterator end); +

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the nth following code +point.
+n: a positive integer that shows how many code points +we want to advance.
+end: end of the UTF-8 sequence to be processed. If +it gets equal to end during the +extraction of a code point, an utf8::not_enough_room +exception is thrown.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+advance (w, 2, twochars + 6);
+
+assert (w == twochars + 5);
+
+

This function works only "forward". In case of a negative +n, there is no effect.

+

In case of an invalid code point, a +utf8::invalid_code_point exception is thrown.

+

utf8::distance

+

Given the iterators to two UTF-8 encoded code points in a +seqence, returns the number of code points between them.

+template <typename octet_iterator> typename +std::iterator_traits<octet_iterator>::difference_type +distance (octet_iterator first, octet_iterator last); +

first: an iterator to a beginning of a UTF-8 +encoded code point.
+last: an iterator to a "post-end" of the last UTF-8 +encoded code point in the sequence we are trying to determine the +length. It can be the beginning of a new code point, or not.
+Return value the distance between the iterators, in code +points.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+
+size_t dist = utf8::distance(twochars, twochars + 5);
+
+assert (dist == 2);
+
+

This function is used to find the length (in code points) of a +UTF-8 encoded string. The reason it is called distance, +rather than, say, length is mainly because developers are +used that length is an O(1) function. Computing the length +of an UTF-8 string is a linear operation, and it looked better to +model it after std::distance algorithm.

+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

+

utf8::utf16to8

+

Converts a UTF-16 encoded string to UTF-8.

+template <typename u16bit_iterator, typename +octet_iterator> void utf16to8 (u16bit_iterator start, +u16bit_iterator end, octet_iterator result); +

start: an iterator pointing to the beginning of the +UTF-16 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-16 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

+

Example of use:

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+
+utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+
+assert (utf8result.size() == 10);    
+
+

In case of invalid UTF-16 sequence, a +utf8::invalid_utf16 exception is thrown.

+

utf8::utf8to16

+

Converts an UTF-8 encoded string to UTF-16

+template <typename u16bit_iterator, typename +octet_iterator> void utf8to16 (octet_iterator start, +octet_iterator end, u16bit_iterator result); +

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert. < br /> end: an +iterator pointing to pass-the-end of the UTF-8 encoded string to +convert.
+result: an output iterator to the place in the UTF-16 +string where to append the result of conversion.

+

Example of use:

+
+unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88,
+    0xf0, 0x9d, 0x84, 0x9e};
+vector <unsigned short> utf16result;
+
+utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

+

utf8::utf32to8

+

Converts a UTF-32 encoded string to UTF-8.

+template <typename octet_iterator, typename +u32bit_iterator> void utf32to8 (u32bit_iterator start, +u32bit_iterator end, octet_iterator result); +

start: an iterator pointing to the beginning of the +UTF-32 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-32 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

+

Example of use:

+
+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+
+assert (utf8result.size() == 9);
+
+

In case of invalid UTF-32 string, a +utf8::invalid_code_point exception is thrown.

+

utf8::utf8to32

+

Converts a UTF-8 encoded string to UTF-32.

+template <typename octet_iterator, typename +u32bit_iterator> void utf8to32 (octet_iterator start, +octet_iterator end, u32bit_iterator result); +

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert.
+end: an iterator pointing to pass-the-end of the UTF-8 +encoded string to convert.
+result: an output iterator to the place in the UTF-32 +string where to append the result of conversion.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+vector<int> utf32result;
+
+utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+
+assert (utf32result.size() == 2);
+
+

In case of an invalid UTF-8 seqence, a +utf8::invalid_utf8 exception is thrown. If +last does not point to the past-of-end of a UTF-8 +seqence, a utf8::not_enough_room exception is +thrown.

+

utf8::find_invalid

+

Detects an invalid sequence within a UTF-8 string.

+template <typename octet_iterator> octet_iterator +find_invalid(octet_iterator start, octet_iterator end); +

start: an iterator pointing to the beginning of the +UTF-8 string to test for validity.
+end: an iterator pointing to pass-the-end of the UTF-8 +string to test for validity.
+Return value: an iterator pointing to the first invalid +octet in the UTF-8 string. In case none were found, equals +end.

+

Example of use:

+
+unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa};
+
+unsigned char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
+
+assert (invalid == utf_invalid + 5);
+
+

This function is typically used to make sure a UTF-8 string is +valid before processing it with other functions. It is especially +important to call it if before doing any of the unchecked +operations on it.

+

utf8::is_valid

+

Checks whether a sequence of octets is a valid UTF-8 string.

+template <typename octet_iterator> bool +is_valid(octet_iterator start, octet_iterator end); +

start: an iterator pointing to the beginning of the +UTF-8 string to test for validity.
+end: an iterator pointing to pass-the-end of the UTF-8 +string to test for validity.
+Return value: true if the sequence is a valid +UTF-8 string; false if not.

+Example of use: +
+unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa};
+
+bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
+
+assert (bvalid == false);
+
+

is_valid is a shorthand for +find_invalid(start, end) == end;. You may want to use +it to make sure that a byte seqence is a valid UTF-8 string without +the need to know where it fails if it is not valid.

+

utf8::is_bom

+

Checks whether a sequence of three octets is a UTF-8 byte order +mark (BOM)

+template <typename octet_iterator> bool is_bom +(octet_iterator it); +

it Beginning of the 3-octet sequence to check
+Return value: true if the sequence is UTF-8 +byte order mark; false if not.

+

Example of use:

+
+unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
+
+bool bbom = is_bom(byte_order_mark);
+
+assert (bbom == true);
+
+

The typical use of this function is to check the first three +bytes of a file. If they form the UTF-8 BOM, we want to skip them +before processing the actual UTF-8 encoded text.

+

Functions From utf8::unchecked Namespace

+

utf8::unchecked::append

+

Encodes a 32 bit code point as a UTF-8 sequence of octets and +appends the sequence to a UTF-8 string.

+template <typename octet_iterator> octet_iterator +append(uint32_t cp, octet_iterator result); +

cp: A 32 bit integer representing a code point to +append to the sequence.
+result: An output iterator to the place in the +sequence where to append the code point.
+Return value: An iterator pointing to the place after the +newly appended sequence.

+

Example of use:

+
+unsigned char u[5] = {0,0,0,0,0};
+
+unsigned char* end = unchecked::append(0x0448, u);
+
+assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
+
+

This is a quicker but less safe version of +utf8::append. It does not check for validity of the +supplied code point, and may produce an invalid UTF-8 sequence.

+

utf8::unchecked::next

+

Given the iterator to the beginning of a UTF-8 sequence, it +returns the code point and moves the iterator to the next +position.

+template <typename octet_iterator> uint32_t +next(octet_iterator& it); +

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the beginning of the next +code point.
+Return value: the 32 bit representation of the processed +UTF-8 code point.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+int cp = unchecked::next(w);
+
+assert (cp == 0x65e5);
+assert (w == twochars + 3);
+
+

This is a quicker but less safe version of +utf8::next. It does not check for validity of the +supplied UTF-8 sequence.

+

utf8::unchecked::previous

+

Given a reference to an iterator pointing to an octet in a UTF-8 +seqence, it decreases the iterator until it hits the beginning of +the previous UTF-8 encoded code point and returns the 32 bits +representation of the code point.

+template <typename octet_iterator> uint32_t +previous(octet_iterator& it); +

it: a reference pointing to an octet within a UTF-8 +encoded string. After the function returns, it is decremented to +point to the beginning of the previous code point.
+Return value: the 32 bit representation of the previous code +point.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars + 3;
+
+int cp = unchecked::previous (w);
+
+assert (cp == 0x65e5);
+assert (w == twochars);
+
+

This is a quicker but less safe version of +utf8::previous. It does not check for validity of the +supplied UTF-8 sequence and offers no boundary checking.

+

utf8::unchecked::advance

+

Advances an iterator by the specified number of code points +within an UTF-8 sequence.

+template <typename octet_iterator, typename +distance_type> void advance (octet_iterator& it, +distance_type n); +

it: a reference to an iterator pointing to the +beginning of an UTF-8 encoded code point. After the function +returns, it is incremented to point to the nth following code +point.
+n: a positive integer that shows how many code points +we want to advance.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+unsigned char* w = twochars;
+
+unchecked::advance (w, 2);
+
+assert (w == twochars + 5);
+
+

This function works only "forward". In case of a negative +n, there is no effect.

+

This is a quicker but less safe version of +utf8::advance. It does not check for validity of the +supplied UTF-8 sequence and offers no boundary checking.

+

utf8::unchecked::distance

+

Given the iterators to two UTF-8 encoded code points in a +seqence, returns the number of code points between them.

+template <typename octet_iterator> typename +std::iterator_traits<octet_iterator>::difference_type +distance (octet_iterator first, octet_iterator last); +

first: an iterator to a beginning of a UTF-8 +encoded code point.
+last: an iterator to a "post-end" of the last UTF-8 +encoded code point in the sequence we are trying to determine the +length. It can be the beginning of a new code point, or not.
+Return value the distance between the iterators, in code +points.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+
+size_t dist = utf8::unchecked::distance(twochars, twochars + 5);
+
+assert (dist == 2);
+
+

This is a quicker but less safe version of +utf8::distance. It does not check for validity of the +supplied UTF-8 sequence.

+

utf8::unchecked::utf16to8

+

Converts a UTF-16 encoded string to UTF-8.

+template <typename u16bit_iterator, typename +octet_iterator> void utf16to8 (u16bit_iterator start, +u16bit_iterator end, octet_iterator result); +

start: an iterator pointing to the beginning of the +UTF-16 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-16 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

+

Example of use:

+
+unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
+vector<unsigned char> utf8result;
+
+unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
+
+assert (utf8result.size() == 10);    
+
+

This is a quicker but less safe version of +utf8::utf16to8. It does not check for validity of the +supplied UTF-16 sequence.

+

utf8::unchecked::utf8to16

+

Converts an UTF-8 encoded string to UTF-16

+template <typename u16bit_iterator, typename +octet_iterator> void utf8to16 (octet_iterator start, +octet_iterator end, u16bit_iterator result); +

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert. < br /> end: an +iterator pointing to pass-the-end of the UTF-8 encoded string to +convert.
+result: an output iterator to the place in the UTF-16 +string where to append the result of conversion.

+

Example of use:

+
+unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88,
+    0xf0, 0x9d, 0x84, 0x9e};
+vector <unsigned short> utf16result;
+
+unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
+
+assert (utf16result.size() == 4);
+assert (utf16result[2] == 0xd834);
+assert (utf16result[3] == 0xdd1e);
+
+

This is a quicker but less safe version of +utf8::utf8to16. It does not check for validity of the +supplied UTF-8 sequence.

+

utf8::unchecked::utf32to8

+

Converts a UTF-32 encoded string to UTF-8.

+template <typename octet_iterator, typename +u32bit_iterator> void utf32to8 (u32bit_iterator start, +u32bit_iterator end, octet_iterator result); +

start: an iterator pointing to the beginning of the +UTF-32 encoded string to convert.
+end: an iterator pointing to pass-the-end of the +UTF-32 encoded string to convert.
+result: an output iterator to the place in the UTF-8 +string where to append the result of conversion.

+

Example of use:

+
+int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
+vector<unsigned char> utf8result;
+
+utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
+
+assert (utf8result.size() == 9);
+
+

This is a quicker but less safe version of +utf8::utf32to8. It does not check for validity of the +supplied UTF-32 sequence.

+

utf8::unchecked::utf8to32

+

Converts a UTF-8 encoded string to UTF-32.

+template <typename octet_iterator, typename +u32bit_iterator> void utf8to32 (octet_iterator start, +octet_iterator end, u32bit_iterator result); +

start: an iterator pointing to the beginning of the +UTF-8 encoded string to convert.
+end: an iterator pointing to pass-the-end of the UTF-8 +encoded string to convert.
+result: an output iterator to the place in the UTF-32 +string where to append the result of conversion.

+

Example of use:

+
+unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0};
+vector<int> utf32result;
+
+unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
+
+assert (utf32result.size() == 2);
+
+

This is a quicker but less safe version of +utf8::utf8to32. It does not check for validity of the +supplied UTF-8 sequence.

+

Points of interest

+

Design goals and decisions

+

The library was designed to be:

+
    +
  1. Generic: for better or worse, there are many C++ string classes +out there, and the library should work with as many of them as +possible.
    2. +
  2. Portable: the library should be portable both accross different +platforms and compilers. The only non-portable code is a small +section that declares unsigned integers of different sizes: three +typedefs. They can be changed by the users of the library if they +don't match their platform. The default setting should work for +Windows (both 32 and 64 bit), and most 32 bit and 64 bit Unix +derivatives.
    3. +
  3. Lightweight: follow the "pay only for what you use" +guidline.
    4. +
  4. Unintrusive: avoid forcing any particular design or even +programming style on the user. This is a library, not a +framework.
    5. +
+

Alternatives

+

In case you want to look into other means of working with UTF-8 +strings from C++, here is the list of solutions I am aware of:

+
    +
  1. ICU Library. It is +very powerful, complete, feature-rich, mature, and widely used. +Also big, intrusive, non-generic, and doesn't play well with the +Standard Library. I definitelly recommend looking at ICU even if +you don't plan to use it.
    2. +
  2. Glib::ustring. +A class specifically made to work with UTF-8 strings, and also feel +like std::string. If you prefer to have yet another +string class in your code, it may be worth a look. Be aware of the +licensing issues, though.
    3. +
  3. Platform dependent solutions: Windows and POSIX have functions +to convert strings from one encoding to another. That is only a +subset of what my library offers, but if that is all you need it +may be good enough, especially given the fact that these functions +are mature and tested in production.
    4. +
+

Conclusion

+

Until Unicode becomes officially recognized by the C++ Standard +Library, we need to use other means to work with UTF-8 strings. +Template functions I describe in this article may be a good step in +this direction.

+

References

+
    +
  1. The Unicode +Consortium.
    2. +
  2. ICU Library.
    3. +
  3. UTF-8 at +Wikipedia
    4. +
+ + diff --git a/source/utf8.h b/source/utf8.h new file mode 100644 index 0000000..77f3f35 --- /dev/null +++ b/source/utf8.h @@ -0,0 +1,504 @@ +// Copyright (c) 2006 Nemanja Trifunovic + +/* +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + + +#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 +#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 + +#include +#include + +namespace utf8 +{ + // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers + // You may need to change them to match your system. + // These typedefs have the same names as ones from cstdint, or boost/cstdint + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + + // Exceptions that may be thrown from the library functions. + class invalid_code_point : public std::exception { + uint32_t cp; + public: + invalid_code_point(uint32_t cp) : cp(cp) {} + const char* what() { return "Invalid code point"; } + uint32_t code_point() const {return cp;} + }; + + class invalid_utf8 : public std::exception { + uint8_t u8; + public: + invalid_utf8 (uint8_t u) : u8(u) {} + const char* what() { return "Invalid UTF-8"; } + uint8_t utf8_octet() const {return u8;} + }; + + class invalid_utf16 : public std::exception { + uint16_t u16; + public: + invalid_utf16 (uint16_t u) : u16(u) {} + const char* what() { return "Invalid UTF-16"; } + uint16_t utf16_word() const {return u16;} + }; + + class not_enough_room : public std::exception { + public: + const char* what() { return "Not enough space"; } + }; + + + + + // Unicode constants + // Leading (high) surrogates: 0xd800 - 0xdbff + // Trailing (low) surrogates: 0xdc00 - 0xdfff + const uint32_t LEAD_SURROGATE_MIN = 0xd800; + const uint32_t LEAD_SURROGATE_MAX = 0xdbff; + const uint32_t TRAIL_SURROGATE_MIN = 0xdc00; + const uint32_t TRAIL_SURROGATE_MAX = 0xdfff; + const uint32_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10); + const uint32_t SURROGATE_OFFSET = 0x10000 - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN; + + // Maximum valid value for a Unicode code point + const uint32_t CODE_POINT_MAX = 0x0010ffff; + + // Byte order mark + const uint8_t bom[] = {0xef, 0xbb, 0xbf}; + + /// Helper functions - not intended to be directly called by the library users + template + inline uint8_t mask8(octet_type oc) + { + return static_cast(0xff & oc); + } + template + inline uint16_t mask16(u16_type oc) + { + return static_cast(0xffff & oc); + } + template + inline bool is_trail(octet_type oc) + { + return ((mask8(oc) >> 6) == 0x2); + } + + template + inline bool is_surrogate(u16 cp) + { + return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); + } + + + + /// The library API - functions intended to be called by the users + template + octet_iterator find_invalid(octet_iterator start, octet_iterator end) + { + octet_iterator result = start; + while (result != end) { + if (mask8(*result) > 0xf4) + break; + if (mask8(*result) < 0x80) + ; + else if ((mask8(*result) >> 5) == 0x6) { + uint8_t lead = mask8(*result); + if (++result == end) + return (--result); + if (!is_trail(*result)) + return result; + switch (lead) { + case 0xe0: + if ((mask8(*result)) < 0xa0) + return result; + break; + case 0xed: + if ((mask8(*result)) > 0x9F) + return result; + break; + case 0xf0: + if ((mask8(*result)) < 0x90) + return result; + break; + } + } + else if ((mask8(*result) >> 4) == 0xe) { + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + } + + else if ((mask8(*result) >> 3) == 0x1e) { + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + if (++result == end) + break; + if (!is_trail(*result)) + break; + } + else + break; + ++result; + } + return result; + } + + template + bool is_valid(octet_iterator start, octet_iterator end) + { + return (find_invalid(start, end) == end); + } + + template + bool is_bom (octet_iterator it) + { + return ( + (mask8(*it++)) == bom[0] && + (mask8(*it++)) == bom[1] && + (mask8(*it)) == bom[2] + ); + } + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + if (is_surrogate(cp)) + throw invalid_code_point(cp); + + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp <= CODE_POINT_MAX) { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast((cp >> 12) | 0x80); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else + throw invalid_code_point(cp); + + return result; + } + + template + uint32_t next(octet_iterator& it, octet_iterator end) + { + uint32_t cp = mask8(*it); + if (cp < 0x80) + ; + else if ((mask8(*it) >> 5) == 0x6) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else if ((mask8(*it) >> 4) == 0xe) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); + if (++it != end) { + if (is_trail(*it)) { + cp += (*it) & 0x3f; + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else if ((mask8(*it) >> 3) == 0x1e) { + if (++it != end) { + if (is_trail(*it)) { + cp = ((cp << 18) & 0x1fffff) + (mask8(*it) << 12) & 0x3ffff; + if (++it != end) { + if (is_trail(*it)) { + cp += (mask8(*it) << 6) & 0xfff; + if (++it != end) { + if (is_trail(*it)) { + cp += (*it) & 0x3f; + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + else + throw invalid_utf8 (*it); + } + else + throw not_enough_room(); + } + ++it; + if (cp > CODE_POINT_MAX || is_surrogate(cp)) + throw invalid_code_point(cp); + + return cp; + } + + + template + uint32_t previous(octet_iterator& it, octet_iterator pass_start) + { + octet_iterator end = it; + while (is_trail(*(--it))) + if (it == pass_start) + throw invalid_utf8(*it); // error - no lead byte in the sequence + octet_iterator temp = it; + return next(temp, end); + } + + template + void advance (octet_iterator& it, distance_type n, octet_iterator end) + { + for (distance_type i = 0; i < n; ++i) + next(it, end); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + next(first, last); + return dist; + } + + template + void utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = mask16(*start++); + // Take care of surrogate pairs first + if (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX) { + if (start != end) { + uint32_t trail_surrogate = mask16(*start++); + if (trail_surrogate >= TRAIL_SURROGATE_MIN && trail_surrogate <= TRAIL_SURROGATE_MAX) + cp = (cp << 10) + trail_surrogate + SURROGATE_OFFSET; + else + throw invalid_utf16(static_cast(trail_surrogate)); + } + else + throw invalid_utf16(static_cast(*start)); + + } + *result = append(cp, result); + } + } + + template + void utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = next(start, end); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + } + + template + void utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + *result = append(*(start++), result); + } + + template + void utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = next(start, end); + } + + namespace unchecked + { + template + octet_iterator append(uint32_t cp, octet_iterator result) + { + if (cp < 0x80) // one octet + *(result++) = static_cast(cp); + else if (cp < 0x800) { // two octets + *(result++) = static_cast((cp >> 6) | 0xc0); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else if (cp < 0x10000) { // three octets + *(result++) = static_cast((cp >> 12) | 0xe0); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + else { // four octets + *(result++) = static_cast((cp >> 18) | 0xf0); + *(result++) = static_cast((cp >> 12) | 0x80); + *(result++) = static_cast((cp >> 6) | 0x80); + *(result++) = static_cast((cp & 0x3f) | 0x80); + } + return result; + } + template + uint32_t next(octet_iterator& it) + { + uint32_t cp = mask8(*it); + if (cp < 0x80) + ; + else if ((mask8(*it) >> 5) == 0x6) { + it++; + cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); + } + else if ((mask8(*it) >> 4) == 0xe) { + ++it; + cp = ((cp << 12) & 0xffff) + ((mask8(*it) << 6) & 0xfff); + ++it; + cp += (*it) & 0x3f; + } + else if (((*it) >> 3) == 0x1e) { + ++it; + cp = ((cp << 18) & 0x1fffff) + (mask8(*it) << 12) & 0x3ffff; + ++it; + cp += (mask8(*it) << 6) & 0xfff; + ++it; + cp += (*it) & 0x3f; + } + ++it; + return cp; + } + + template + uint32_t previous(octet_iterator& it) + { + while (is_trail(*(--it))) ; + octet_iterator temp = it; + return next(temp); + } + + template + void advance (octet_iterator& it, distance_type n) + { + for (distance_type i = 0; i < n; ++i) + next(it); + } + + template + uint32_t get(octet_iterator it) + { + return next(it); + } + + template + typename std::iterator_traits::difference_type + distance (octet_iterator first, octet_iterator last) + { + typename std::iterator_traits::difference_type dist; + for (dist = 0; first < last; ++dist) + next(first); + return dist; + } + + template + void utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) + { + while (start != end) { + uint32_t cp = mask16(*start++); + // Take care of surrogate pairs first + if (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX) { + uint32_t trail_surrogate = mask16(*start++); + cp = (cp << 10) + trail_surrogate + SURROGATE_OFFSET; + } + *result = append(cp, result); + } + } + + template + void utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) + { + while (start != end) { + uint32_t cp = next(start); + if (cp > 0xffff) { //make a surrogate pair + *result++ = static_cast((cp >> 10) + LEAD_OFFSET); + *result++ = static_cast((cp & 0x3ff) + TRAIL_SURROGATE_MIN); + } + else + *result++ = static_cast(cp); + } + } + + template + void utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) + { + while (start != end) + *result = append(*(start++), result); + } + + template + void utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) + { + while (start < end) + (*result++) = next(start); + } + + } // namespace utf8::unchecked +} // namespace utf8 + +#endif // header guard diff --git a/test_data/negative/utf8_invalid.txt b/test_data/negative/utf8_invalid.txt new file mode 100755 index 0000000..cc03e7b --- /dev/null +++ b/test_data/negative/utf8_invalid.txt @@ -0,0 +1,271 @@ +UTF-8 decoder capability and stress test +---------------------------------------- + +Markus Kuhn - 2003-02-19 + +This test file can help you examine, how your UTF-8 decoder handles +various types of correct, malformed, or otherwise interesting UTF-8 +sequences. This file is not meant to be a conformance test. It does +not prescribes any particular outcome and therefore there is no way to +"pass" or "fail" this test file, even though the texts suggests a +preferable decoder behaviour at some places. The aim is instead to +help you think about and test the behaviour of your UTF-8 on a +systematic collection of unusual inputs. Experience so far suggests +that most first-time authors of UTF-8 decoders find at least one +serious problem in their decoder by using this file. + +The test lines below cover boundary conditions, malformed UTF-8 +sequences as well as correctly encoded UTF-8 sequences of Unicode code +points that should never occur in a correct UTF-8 file. + +According to ISO 10646-1:2000, sections D.7 and 2.3c, a device +receiving UTF-8 shall interpret a "malformed sequence in the same way +that it interprets a character that is outside the adopted subset" and +"characters that are not within the adopted subset shall be indicated +to the user" by a receiving device. A quite commonly used approach in +UTF-8 decoders is to replace any malformed UTF-8 sequence by a +replacement character (U+FFFD), which looks a bit like an inverted +question mark, or a similar symbol. It might be a good idea to +visually distinguish a malformed UTF-8 sequence from a correctly +encoded Unicode character that is just not available in the current +font but otherwise fully legal, even though ISO 10646-1 doesn't +mandate this. In any case, just ignoring malformed sequences or +unavailable characters does not conform to ISO 10646, will make +debugging more difficult, and can lead to user confusion. + +Please check, whether a malformed UTF-8 sequence is (1) represented at +all, (2) represented by exactly one single replacement character (or +equivalent signal), and (3) the following quotation mark after an +illegal UTF-8 sequence is correctly displayed, i.e. proper +resynchronization takes place immageately after any malformed +sequence. This file says "THE END" in the last line, so if you don't +see that, your decoder crashed somehow before, which should always be +cause for concern. + +All lines in this file are exactly 79 characters long (plus the line +feed). In addition, all lines end with "|", except for the two test +lines 2.1.1 and 2.2.1, which contain non-printable ASCII controls +U+0000 and U+007F. If you display this file with a fixed-width font, +these "|" characters should all line up in column 79 (right margin). +This allows you to test quickly, whether your UTF-8 decoder finds the +correct number of characters in every line, that is whether each +malformed sequences is replaced by a single replacement character. + +Note that as an alternative to the notion of malformed sequence used +here, it is also a perfectly acceptable (and in some situations even +preferable) solution to represent each individual byte of a malformed +sequence by a replacement character. If you follow this strategy in +your decoder, then please ignore the "|" column. + + +Here come the tests: | + | +1 Some correct UTF-8 text | + | +You should see the Greek word 'kosme': "κόσμε" | + | +2 Boundary condition test cases | + | +2.1 First possible sequence of a certain length | + | +2.1.1 1 byte (U-00000000): "" +2.1.2 2 bytes (U-00000080): "€" | +2.1.3 3 bytes (U-00000800): "ࠀ" | +2.1.4 4 bytes (U-00010000): "𐀀" | +2.1.5 5 bytes (U-00200000): "?" | +2.1.6 6 bytes (U-04000000): "?" | + | +2.2 Last possible sequence of a certain length | + | +2.2.1 1 byte (U-0000007F): "" +2.2.2 2 bytes (U-000007FF): "߿" | +2.2.3 3 bytes (U-0000FFFF): "?" | +2.2.4 4 bytes (U-001FFFFF): "?" | +2.2.5 5 bytes (U-03FFFFFF): "?" | +2.2.6 6 bytes (U-7FFFFFFF): "?" | + | +2.3 Other boundary conditions | + | +2.3.1 U-0000D7FF = ed 9f bf = "퟿" | +2.3.2 U-0000E000 = ee 80 80 = "" | +2.3.3 U-0000FFFD = ef bf bd = "�" | +2.3.4 U-0010FFFF = f4 8f bf bf = "􏿿" | +2.3.5 U-00110000 = f4 90 80 80 = "?" | + | +3 Malformed sequences | + | +3.1 Unexpected continuation bytes | + | +Each unexpected continuation byte should be separately signalled as a | +malformed sequence of its own. | + | +3.1.1 First continuation byte 0x80: "?" | +3.1.2 Last continuation byte 0xbf: "?" | + | +3.1.3 2 continuation bytes: "??" | +3.1.4 3 continuation bytes: "???" | +3.1.5 4 continuation bytes: "????" | +3.1.6 5 continuation bytes: "?????" | +3.1.7 6 continuation bytes: "??????" | +3.1.8 7 continuation bytes: "???????" | + | +3.1.9 Sequence of all 64 possible continuation bytes (0x80-0xbf): | + | + "???????????????? | + ???????????????? | + ???????????????? | + ????????????????" | + | +3.2 Lonely start characters | + | +3.2.1 All 32 first bytes of 2-byte sequences (0xc0-0xdf), | + each followed by a space character: | + | + "?? àĠŠƠǠȠɠʠˠ̠͠ΠϠ | + РѠҠӠԠՠ֠נؠ٠ڠ۠ܠݠޠߠ" | + | +3.2.2 All 16 first bytes of 3-byte sequences (0xe0-0xef), | + each followed by a space character: | + | + "ࠡ ⠣ 䠥 栧 蠩 ꠫ 젭  " | + | +3.2.3 All 8 first bytes of 4-byte sequences (0xf0-0xf7), | + each followed by a space character: | + | + "𠱠򠳠??" | + | +3.2.4 All 4 first bytes of 5-byte sequences (0xf8-0xfb), | + each followed by a space character: | + | + "? ? | + | +3.2.5 All 2 first bytes of 6-byte sequences (0xfc-0xfd), | + each followed by a space character: | + | + "? | + | +3.3 Sequences with last continuation byte missing | + | +All bytes of an incomplete sequence should be signalled as a single | +malformed sequence, i.e., you should see only a single replacement | +character in each of the next 10 tests. (Characters as in section 2) | + | +3.3.1 2-byte sequence with last byte missing (U+0000): "? | +3.3.2 3-byte sequence with last byte missing (U+0000): "? | +3.3.3 4-byte sequence with last byte missing (U+0000): "? | +3.3.4 5-byte sequence with last byte missing (U+0000): "? | +3.3.5 6-byte sequence with last byte missing (U+0000): "? | +3.3.6 2-byte sequence with last byte missing (U-000007FF): "ߢ | +3.3.7 3-byte sequence with last byte missing (U-0000FFFF): "￢ | +3.3.8 4-byte sequence with last byte missing (U-001FFFFF): "? | +3.3.9 5-byte sequence with last byte missing (U-03FFFFFF): "? | +3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF): "? | + | +3.4 Concatenation of incomplete sequences | + | +All the 10 sequences of 3.3 concatenated, you should see 10 malformed | +sequences being signalled: | + | + "???????????????" | + | +3.5 Impossible bytes | + | +The following two bytes cannot appear in a correct UTF-8 string | + | +3.5.1 fe = "? | +3.5.2 ff = "? | +3.5.3 fe fe ff ff = "? | + | +4 Overlong sequences | + | +The following sequences are not malformed according to the letter of | +the Unicode 2.0 standard. However, they are longer then necessary and | +a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8 | +decoder" should reject them just like malformed sequences for two | +reasons: (1) It helps to debug applications if overlong sequences are | +not treated as valid representations of characters, because this helps | +to spot problems more quickly. (2) Overlong sequences provide | +alternative representations of characters, that could maliciously be | +used to bypass filters that check only for ASCII characters. For | +instance, a 2-byte encoded line feed (LF) would not be caught by a | +line counter that counts only 0x0a bytes, but it would still be | +processed as a line feed by an unsafe UTF-8 decoder later in the | +pipeline. From a security point of view, ASCII compatibility of UTF-8 | +sequences means also, that ASCII characters are *only* allowed to be | +represented by ASCII bytes in the range 0x00-0x7f. To ensure this | +aspect of ASCII compatibility, use only "safe UTF-8 decoders" that | +reject overlong UTF-8 sequences for which a shorter encoding exists. | + | +4.1 Examples of an overlong ASCII character | + | +With a safe UTF-8 decoder, all of the following five overlong | +representations of the ASCII character slash ("/") should be rejected | +like a malformed UTF-8 sequence, for instance by substituting it with | +a replacement character. If you see a slash below, you do not have a | +safe UTF-8 decoder! | + | +4.1.1 U+002F = c0 af = "?" | +4.1.2 U+002F = e0 80 af = "?" | +4.1.3 U+002F = f0 80 80 af = "?" | +4.1.4 U+002F = f8 80 80 80 af = "?" | +4.1.5 U+002F = fc 80 80 80 80 af = "?" | + | +4.2 Maximum overlong sequences | + | +Below you see the highest Unicode value that is still resulting in an | +overlong sequence if represented with the given number of bytes. This | +is a boundary test for safe UTF-8 decoders. All five characters should | +be rejected like malformed UTF-8 sequences. | + | +4.2.1 U-0000007F = c1 bf = "?" | +4.2.2 U-000007FF = e0 9f bf = "?" | +4.2.3 U-0000FFFF = f0 8f bf bf = "?" | +4.2.4 U-001FFFFF = f8 87 bf bf bf = "?" | +4.2.5 U-03FFFFFF = fc 83 bf bf bf bf = "?" | + | +4.3 Overlong representation of the NUL character | + | +The following five sequences should also be rejected like malformed | +UTF-8 sequences and should not be treated like the ASCII NUL | +character. | + | +4.3.1 U+0000 = c0 80 = "?" | +4.3.2 U+0000 = e0 80 80 = "?" | +4.3.3 U+0000 = f0 80 80 80 = "?" | +4.3.4 U+0000 = f8 80 80 80 80 = "?" | +4.3.5 U+0000 = fc 80 80 80 80 80 = "?" | + | +5 Illegal code positions | + | +The following UTF-8 sequences should be rejected like malformed | +sequences, because they never represent valid ISO 10646 characters and | +a UTF-8 decoder that accepts them might introduce security problems | +comparable to overlong UTF-8 sequences. | + | +5.1 Single UTF-16 surrogates | + | +5.1.1 U+D800 = ed a0 80 = "" | +5.1.2 U+DB7F = ed ad bf = "" | +5.1.3 U+DB80 = ed ae 80 = "" | +5.1.4 U+DBFF = ed af bf = "" | +5.1.5 U+DC00 = ed b0 80 = "" | +5.1.6 U+DF80 = ed be 80 = "" | +5.1.7 U+DFFF = ed bf bf = "" | + | +5.2 Paired UTF-16 surrogates | + | +5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80 = "𐀀" | +5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf = "𐏿" | +5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80 = "󯰀" | +5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf = "󯿿" | +5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80 = "󰀀" | +5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf = "󰏿" | +5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80 = "􏰀" | +5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf = "􏿿" | + | +5.3 Other illegal code positions | + | +5.3.1 U+FFFE = ef bf be = "?" | +5.3.2 U+FFFF = ef bf bf = "?" | + | +THE END | diff --git a/test_drivers/smoke_test/test.cpp b/test_drivers/smoke_test/test.cpp new file mode 100644 index 0000000..0281656 --- /dev/null +++ b/test_drivers/smoke_test/test.cpp @@ -0,0 +1,144 @@ + +#include +#include +#include +#include "utf8.h" +using namespace utf8; +using namespace std; + +int main() +{ + //append + unsigned char u[5] = {0,0,0,0,0}; + + unsigned char* end = append(0x0448, u); + assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0); + + end = append(0x65e5, u); + assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0); + + end = append(0x10346, u); + assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0); + + //next + unsigned char twochars[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0x0}; + unsigned char* w = twochars; + int cp = next(w, twochars + 6); + assert (cp == 0x65e5); + assert (w == twochars + 3); + + //previous + cp = previous (w, twochars - 1); + assert (cp == 0x65e5); + assert (w == twochars); + + // advance + w = twochars; + advance (w, 2, twochars + 6); + assert (w == twochars + 5); + + // distance + size_t dist = utf8::distance(twochars, twochars + 5); + assert (dist == 2); + + // utf32to8 + int utf32string[] = {0x448, 0x65E5, 0x10346, 0}; + vector utf8result; + utf32to8(utf32string, utf32string + 3, back_inserter(utf8result)); + assert (utf8result.size() == 9); + + //utf8to32 + vector utf32result; + utf8to32(twochars, twochars + 5, back_inserter(utf32result)); + assert (utf32result.size() == 2); + + //utf16to8 + unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e}; + utf8result.clear(); + utf16to8(utf16string, utf16string + 5, back_inserter(utf8result)); + assert (utf8result.size() == 10); + + //utf8to16 + unsigned char utf8_with_surrogates[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, + 0xf0, 0x9d, 0x84, 0x9e}; + vector utf16result; + utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result)); + assert (utf16result.size() == 4); + assert (utf16result[2] == 0xd834); + assert (utf16result[3] == 0xdd1e); + + //find_invalid + unsigned char utf_invalid[] = {0xE6, 0x97, 0xA5, 0xd1, 0x88, 0xfa}; + unsigned char* invalid = find_invalid(utf_invalid, utf_invalid + 6); + assert (invalid == utf_invalid + 5); + + //is_valid + bool bvalid = is_valid(utf_invalid, utf_invalid + 6); + assert (bvalid == false); + bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9); + assert (bvalid == true); + + //is_bom + unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf}; + bool bbom = is_bom(byte_order_mark); + assert (bbom == true); + + ////////////////////////////////////////////////////////// + //// Unchecked variants + ////////////////////////////////////////////////////////// + + //append + memset(u, 0, 5); + end = unchecked::append(0x0448, u); + assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0); + + end = unchecked::append(0x65e5, u); + assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0); + + end = unchecked::append(0x10346, u); + assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0); + + //next + w = twochars; + cp = unchecked::next(w); + assert (cp == 0x65e5); + assert (w == twochars + 3); + + //previous + cp = unchecked::previous (w); + assert (cp == 0x65e5); + assert (w == twochars); + + // advance + w = twochars; + unchecked::advance (w, 2); + assert (w == twochars + 5); + + // distance + dist = unchecked::distance(twochars, twochars + 5); + assert (dist == 2); + + // utf32to8 + utf8result.clear(); + unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result)); + assert (utf8result.size() == 9); + + //utf8to32 + utf32result.clear(); + unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result)); + assert (utf32result.size() == 2); + + //utf16to8 + utf8result.clear(); + unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result)); + assert (utf8result.size() == 10); + + //utf8to16 + utf16result.clear(); + unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result)); + assert (utf16result.size() == 4); + assert (utf16result[2] == 0xd834); + assert (utf16result[3] == 0xdd1e); +} + +