2019-06-23 05:21:30 +03:00
|
|
|
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
|
|
|
|
// Use of this source code is governed by an MIT license
|
|
|
|
// that can be found in the LICENSE file.
|
|
|
|
|
2019-06-22 21:20:28 +03:00
|
|
|
module builtin
|
|
|
|
|
2019-09-15 15:36:05 +03:00
|
|
|
pub fn utf8_char_len(b byte) int {
|
|
|
|
return (( 0xe5000000 >> (( b >> 3 ) & 0x1e )) & 3 ) + 1
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// Convert utf32 to utf8
|
|
|
|
// utf32 == Codepoint
|
2019-06-27 14:14:59 +03:00
|
|
|
pub fn utf32_to_str(code u32) string {
|
2019-06-27 03:33:49 +03:00
|
|
|
icode := int(code) //Prevents doing casts everywhere
|
|
|
|
mut buffer := malloc(5)
|
|
|
|
if icode <= 127 /* 0x7F */ {
|
|
|
|
buffer[0] = icode
|
|
|
|
return tos(buffer, 1)
|
|
|
|
}
|
|
|
|
if (icode <= 2047 /* 0x7FF */) {
|
|
|
|
buffer[0] = 192 /*0xC0*/ | (icode >> 6) /* 110xxxxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 2)
|
|
|
|
}
|
|
|
|
if (icode <= 65535 /* 0xFFFF */) {
|
|
|
|
buffer[0] = 224 /*0xE0*/ | (icode >> 12) /* 1110xxxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | ((icode >> 6) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[2] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 3)
|
|
|
|
}
|
|
|
|
if (icode <= 1114111 /* 0x10FFFF */) {
|
|
|
|
buffer[0] = 240 /*0xF0*/ | (icode >> 18) /* 11110xxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | ((icode >> 12) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[2] = 128 /*0x80*/ | ((icode >> 6) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[3] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 4)
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
return ''
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO copypasta
|
2019-06-27 14:14:59 +03:00
|
|
|
pub fn utf32_to_str_no_malloc(code u32, buf voidptr) string {
|
2019-06-27 03:33:49 +03:00
|
|
|
icode := int(code) //Prevents doing casts everywhere
|
|
|
|
mut buffer := byteptr(buf)
|
|
|
|
if icode <= 127 /* 0x7F */ {
|
|
|
|
buffer[0] = icode
|
|
|
|
return tos(buffer, 1)
|
|
|
|
}
|
|
|
|
if (icode <= 2047 /* 0x7FF */) {
|
|
|
|
buffer[0] = 192 /*0xC0*/ | (icode >> 6) /* 110xxxxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 2)
|
|
|
|
}
|
|
|
|
if (icode <= 65535 /* 0xFFFF */) {
|
|
|
|
buffer[0] = 224 /*0xE0*/ | (icode >> 12) /* 1110xxxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | ((icode >> 6) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[2] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 3)
|
|
|
|
}
|
|
|
|
if (icode <= 1114111 /* 0x10FFFF */) {
|
|
|
|
buffer[0] = 240 /*0xF0*/ | (icode >> 18) /* 11110xxx */
|
|
|
|
buffer[1] = 128 /*0x80*/ | ((icode >> 12) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[2] = 128 /*0x80*/ | ((icode >> 6) & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
buffer[3] = 128 /*0x80*/ | (icode & 63 /*0x3F*/) /* 10xxxxxx */
|
|
|
|
return tos(buffer, 4)
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
return ''
|
|
|
|
}
|
|
|
|
|
|
|
|
// Convert utf8 to utf32
|
2019-06-27 14:14:59 +03:00
|
|
|
pub fn (_rune string) utf32_code() int {
|
2019-06-22 21:20:28 +03:00
|
|
|
if _rune.len == 0 {
|
|
|
|
return 0
|
|
|
|
}
|
|
|
|
// save ASC symbol as is
|
|
|
|
if _rune.len == 1 {
|
|
|
|
return int(_rune[0])
|
|
|
|
}
|
2019-06-27 03:33:49 +03:00
|
|
|
mut b := byte(int(_rune[0]))
|
2019-06-22 21:20:28 +03:00
|
|
|
// TODO should be
|
|
|
|
// res := int( rune[0] << rune.len)
|
2019-06-27 03:33:49 +03:00
|
|
|
b = b << _rune.len
|
|
|
|
mut res := int(b)
|
2019-06-22 21:20:28 +03:00
|
|
|
mut shift := 6 - _rune.len
|
|
|
|
for i := 1; i < _rune.len; i++ {
|
|
|
|
c := int(_rune[i])
|
2019-06-27 03:33:49 +03:00
|
|
|
res = res << shift
|
2019-11-13 21:14:37 +03:00
|
|
|
res |= c & 63 // 0x3f
|
2019-06-22 21:20:28 +03:00
|
|
|
shift = 6
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
2019-07-24 13:16:45 +03:00
|
|
|
const (
|
|
|
|
CP_UTF8 = 65001
|
|
|
|
)
|
|
|
|
|
|
|
|
pub fn (_str string) to_wide() &u16 {
|
2019-11-16 02:30:50 +03:00
|
|
|
$if windows {
|
2019-12-07 16:58:43 +03:00
|
|
|
num_chars := (C.MultiByteToWideChar(CP_UTF8, 0, _str.str, _str.len, 0, 0))
|
2019-11-16 02:30:50 +03:00
|
|
|
mut wstr := &u16(malloc((num_chars + 1) * 2)) // sizeof(wchar_t)
|
|
|
|
if !isnil(wstr) {
|
|
|
|
C.MultiByteToWideChar(CP_UTF8, 0, _str.str, _str.len, wstr, num_chars)
|
|
|
|
C.memset(&byte(wstr) + num_chars * 2, 0, 2)
|
|
|
|
}
|
|
|
|
return wstr
|
|
|
|
} $else {
|
|
|
|
return 0
|
|
|
|
}
|
2019-07-24 13:16:45 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn string_from_wide(_wstr &u16) string {
|
|
|
|
$if windows {
|
2019-12-07 16:58:43 +03:00
|
|
|
wstr_len := C.wcslen(_wstr)
|
2019-07-24 13:16:45 +03:00
|
|
|
return string_from_wide2(_wstr, wstr_len)
|
|
|
|
} $else {
|
|
|
|
return ''
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn string_from_wide2(_wstr &u16, len int) string {
|
|
|
|
$if windows {
|
2019-12-07 16:58:43 +03:00
|
|
|
num_chars := C.WideCharToMultiByte(CP_UTF8, 0, _wstr, len, 0, 0, 0, 0))
|
|
|
|
mut str_to := malloc(num_chars + 1)
|
2019-11-16 02:30:50 +03:00
|
|
|
if !isnil(str_to) {
|
2019-07-24 13:16:45 +03:00
|
|
|
C.WideCharToMultiByte(CP_UTF8, 0, _wstr, len, str_to, num_chars, 0, 0)
|
2019-12-07 16:58:43 +03:00
|
|
|
C.memset(str_to + num_chars, 0, 1)
|
2019-07-24 13:16:45 +03:00
|
|
|
}
|
|
|
|
return tos2(str_to)
|
|
|
|
} $else {
|
|
|
|
return ''
|
|
|
|
}
|
|
|
|
}
|
2019-09-15 04:41:24 +03:00
|
|
|
|
|
|
|
// Calculate length to read from the first byte
|
|
|
|
fn utf8_len(c byte) int {
|
|
|
|
mut b := 0
|
|
|
|
mut x := c
|
|
|
|
|
|
|
|
if ((x & 240) != 0) { //0xF0
|
|
|
|
x >>= 4
|
|
|
|
} else {
|
|
|
|
b += 4
|
|
|
|
}
|
|
|
|
if ((x & 12) != 0) { //0x0C
|
|
|
|
x >>= 2
|
|
|
|
} else {
|
|
|
|
b += 2
|
|
|
|
}
|
|
|
|
if ((x & 2) == 0) { //0x02
|
|
|
|
b++
|
|
|
|
}
|
|
|
|
return b
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reads an utf8 character from standard input
|
|
|
|
pub fn utf8_getchar() int {
|
2019-12-07 15:31:56 +03:00
|
|
|
c := C.getchar()
|
2019-09-15 04:41:24 +03:00
|
|
|
len := utf8_len(~c)
|
|
|
|
if c < 0 {
|
|
|
|
return 0
|
2019-09-15 15:36:05 +03:00
|
|
|
} else if len == 0 {
|
2019-09-15 04:41:24 +03:00
|
|
|
return c
|
2019-09-15 15:36:05 +03:00
|
|
|
} else if len == 1 {
|
2019-09-15 04:41:24 +03:00
|
|
|
return -1
|
|
|
|
} else {
|
2019-12-07 15:31:56 +03:00
|
|
|
mut uc := c & ((1 << (7 - len)) - 1)
|
2019-09-15 04:41:24 +03:00
|
|
|
for i := 0; i + 1 < len; i++ {
|
2019-12-07 15:31:56 +03:00
|
|
|
c2 := C.getchar()
|
2019-09-15 04:41:24 +03:00
|
|
|
if c2 != -1 && (c2 >> 6) == 2 {
|
|
|
|
uc <<= 6
|
2019-12-07 15:31:56 +03:00
|
|
|
uc |= (c2 & 63)
|
2019-09-15 15:36:05 +03:00
|
|
|
} else if c2 == -1 {
|
2019-09-15 04:41:24 +03:00
|
|
|
return 0
|
|
|
|
} else {
|
|
|
|
return -1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return uc
|
|
|
|
}
|
|
|
|
}
|