yzyCli/u3d/Lua/tolua/misc/utf8.lua

-- local utf8 = {}

-- --byte index of the next char after the char at byte index i, followed by a valid flag for the char at byte index i.
-- --nil if not found. invalid characters are iterated as 1-byte chars.
-- function utf8.next_raw(s, i)
-- 	if not i then
-- 		if #s == 0 then return nil end
-- 		return 1, true --fake flag (doesn't matter since this flag is not to be taken as full validation)
-- 	end
-- 	if i > #s then return end
-- 	local c = s:byte(i)
-- 	if c >= 0x00 and c <= 0x7F then
-- 		i = i + 1
-- 	elseif c >= 0xC2 and c <= 0xDF then
-- 		i = i + 2
-- 	elseif c >= 0xE0 and c <= 0xEF then
-- 		i = i + 3
-- 	elseif c >= 0xF0 and c <= 0xF4 then
-- 		i = i + 4
-- 	else --invalid
-- 		return i + 1, false
-- 	end
-- 	if i > #s then return end
-- 	return i, true
-- end

-- --next() is the generic iterator and can be replaced for different semantics. next_raw() must preserve its semantics.
-- utf8.next = utf8.next_raw

-- --iterate chars, returning the byte index where each char starts
-- function utf8.byte_indices(s, previ)
-- 	return utf8.next, s, previ
-- end

-- --number of chars in string
-- function utf8.len(s)
-- 	assert(s, "bad argument #1 to 'len' (string expected, got nil)")
-- 	local len = 0
-- 	for _ in utf8.byte_indices(s) do
-- 		len = len + 1
-- 	end
-- 	return len
-- end

-- --byte index given char index. nil if the index is outside the string.
-- function utf8.byte_index(s, target_ci)
-- 	if target_ci < 1 then return end
-- 	local ci = 0
-- 	for i in utf8.byte_indices(s) do
-- 		ci = ci + 1
-- 		if ci == target_ci then
-- 			return i
-- 		end
-- 	end
-- 	assert(target_ci > ci, "invalid index")
-- end

-- --char index given byte index. nil if the index is outside the string.
-- function utf8.char_index(s, target_i)
-- 	if target_i < 1 or target_i > #s then return end
-- 	local ci = 0
-- 	for i in utf8.byte_indices(s) do
-- 		ci = ci + 1
-- 		if i == target_i then
-- 			return ci
-- 		end
-- 	end
-- 	LogError("invalid index")
-- end

-- --byte index of the prev. char before the char at byte index i, which defaults to #s + 1.
-- --nil if the index is outside the 2..#s+1 range.
-- --NOTE: unlike next(), this is a O(N) operation!
-- function utf8.prev(s, nexti)
-- 	nexti = nexti or #s + 1
-- 	if nexti <= 1 or nexti > #s + 1 then return end
-- 	local lasti, lastvalid = utf8.next(s)
-- 	for i, valid in utf8.byte_indices(s) do
-- 		if i == nexti then
-- 			return lasti, lastvalid
-- 		end
-- 		lasti, lastvalid = i, valid
-- 	end
-- 	if nexti == #s + 1 then
-- 		return lasti, lastvalid
-- 	end
-- 	LogError("invalid index")
-- end

-- --iterate chars in reverse order, returning the byte index where each char starts.
-- function utf8.byte_indices_reverse(s, nexti)
-- 	if #s < 200 then
-- 		--using prev() is a O(N^2/2) operation, ok for small strings (200 chars need 40,000 iterations)
-- 		return utf8.prev, s, nexti
-- 	else
-- 		--store byte indices in a table and iterate them in reverse.
-- 		--this is 40x slower than byte_indices() but still fast at 2mil chars/second (but eats RAM and makes garbage).
-- 		local t = {}
-- 		for i in utf8.byte_indices(s) do
-- 			if nexti and i >= nexti then break end
-- 			table.insert(t, i)
-- 		end
-- 		local i = #t + 1
-- 		return function()
-- 			i = i - 1
-- 			return t[i]
-- 		end
-- 	end
-- end

-- --sub based on char indices, which, unlike with standard string.sub(), can't be negative.
-- --start_ci can be 1..inf and end_ci can be 0..inf. end_ci can be nil meaning last char.
-- --if start_ci is out of range or end_ci < start_ci, the empty string is returned.
-- --if end_ci is out of range, it is considered to be the last position in the string.
-- function utf8.sub(s, start_ci, end_ci)
-- 	--assert for positive indices because we might implement negative indices in the future.
-- 	assert(start_ci >= 1)
-- 	assert(not end_ci or end_ci >= 0)
-- 	local ci = 0
-- 	local start_i, end_i
-- 	for i in utf8.byte_indices(s) do
-- 		ci = ci + 1
-- 		if ci == start_ci then
-- 			start_i = i
-- 		end
-- 		if ci == end_ci then
-- 			end_i = i
-- 		end
-- 	end
-- 	if not start_i then
-- 		assert(start_ci > ci, 'invalid index')
-- 		return ''
-- 	end
-- 	if end_ci and not end_i then
-- 		if end_ci < start_ci then
-- 			return ''
-- 		end
-- 		assert(end_ci > ci, 'invalid index')
-- 	end
-- 	return s:sub(start_i, end_i and end_i - 1)
-- end

-- --check if a string contains a substring at byte index i without making garbage.
-- --nil if the index is out of range. true if searching for the empty string.
-- function utf8.contains(s, i, sub)
-- 	if i < 1 or i > #s then return nil end
-- 	for si = 1, #sub do
-- 		if s:byte(i + si - 1) ~= sub:byte(si) then
-- 			return false
-- 		end
-- 	end
-- 	return true
-- end

-- --count the number of occurences of a substring in a string. the substring cannot be the empty string.
-- function utf8.count(s, sub)
-- 	assert(#sub > 0)
-- 	local count = 0
-- 	local i = 1
-- 	while i do
-- 		if utf8.contains(s, i, sub) then
-- 			count = count + 1
-- 			i = i + #sub
-- 			if i > #s then break end
-- 		else
-- 			i = utf8.next(s, i)
-- 		end
-- 	end
-- 	return count
-- end

-- --utf8 validation and sanitization

-- --check if there's a valid utf8 codepoint at byte index i. valid ranges for each utf8 byte are:
-- -- byte  1          2           3          4
-- --------------------------------------------
-- -- 00 - 7F
-- -- C2 - DF    80 - BF
-- -- E0         A0 - BF     80 - BF
-- -- E1 - EC    80 - BF     80 - BF
-- -- ED         80 - 9F     80 - BF
-- -- EE - EF    80 - BF     80 - BF
-- -- F0         90 - BF     80 - BF    80 - BF
-- -- F1 - F3    80 - BF     80 - BF    80 - BF
-- -- F4         80 - 8F     80 - BF    80 - BF
-- function utf8.isvalid(s, i)
-- 	local c = s:byte(i)
-- 	if not c then
-- 		return false
-- 	elseif c >= 0x00 and c <= 0x7F then
-- 		return true
-- 	elseif c >= 0xC2 and c <= 0xDF then
-- 		local c2 = s:byte(i + 1)
-- 		return c2 and c2 >= 0x80 and c2 <= 0xBF
-- 	elseif c >= 0xE0 and c <= 0xEF then
-- 		local c2 = s:byte(i + 1)
-- 		local c3 = s:byte(i + 2)
-- 		if c == 0xE0 then
-- 			return c2 and c3 and
-- 				c2 >= 0xA0 and c2 <= 0xBF and
-- 				c3 >= 0x80 and c3 <= 0xBF
-- 		elseif c >= 0xE1 and c <= 0xEC then
-- 			return c2 and c3 and
-- 				c2 >= 0x80 and c2 <= 0xBF and
-- 				c3 >= 0x80 and c3 <= 0xBF
-- 		elseif c == 0xED then
-- 			return c2 and c3 and
-- 				c2 >= 0x80 and c2 <= 0x9F and
-- 				c3 >= 0x80 and c3 <= 0xBF
-- 		elseif c >= 0xEE and c <= 0xEF then
-- 			if c == 0xEF and c2 == 0xBF and (c3 == 0xBE or c3 == 0xBF) then
-- 				return false --uFFFE and uFFFF non-characters
-- 			end
-- 			return c2 and c3 and
-- 				c2 >= 0x80 and c2 <= 0xBF and
-- 				c3 >= 0x80 and c3 <= 0xBF
-- 		end
-- 	elseif c >= 0xF0 and c <= 0xF4 then
-- 		local c2 = s:byte(i + 1)
-- 		local c3 = s:byte(i + 2)
-- 		local c4 = s:byte(i + 3)
-- 		if c == 0xF0 then
-- 			return c2 and c3 and c4 and
-- 				c2 >= 0x90 and c2 <= 0xBF and
-- 				c3 >= 0x80 and c3 <= 0xBF and
-- 				c4 >= 0x80 and c4 <= 0xBF
-- 		elseif c >= 0xF1 and c <= 0xF3 then
-- 			return c2 and c3 and c4 and
-- 				c2 >= 0x80 and c2 <= 0xBF and
-- 				c3 >= 0x80 and c3 <= 0xBF and
-- 				c4 >= 0x80 and c4 <= 0xBF
-- 		elseif c == 0xF4 then
-- 			return c2 and c3 and c4 and
-- 				c2 >= 0x80 and c2 <= 0x8F and
-- 				c3 >= 0x80 and c3 <= 0xBF and
-- 				c4 >= 0x80 and c4 <= 0xBF
-- 		end
-- 	end
-- 	return false
-- end

-- --byte index of the next valid utf8 char after the char at byte index i.
-- --nil if indices go out of range. invalid characters are skipped.
-- function utf8.next_valid(s, i)
-- 	local valid
-- 	i, valid = utf8.next_raw(s, i)
-- 	while i and (not valid or not utf8.isvalid(s, i)) do
-- 		i, valid = utf8.next(s, i)
-- 	end
-- 	return i
-- end

-- --iterate valid chars, returning the byte index where each char starts
-- function utf8.valid_byte_indices(s)
-- 	return utf8.next_valid, s
-- end

-- --assert that a string only contains valid utf8 characters
-- function utf8.validate(s)
-- 	for i, valid in utf8.byte_indices(s) do
-- 		if not valid or not utf8.isvalid(s, i) then
-- 			LogError(string.format('invalid utf8 char at #%d', i))
-- 		end
-- 	end
-- end

-- local function table_lookup(s, i, j, t)
-- 	return t[s:sub(i, j)]
-- end

-- --replace characters in string based on a function f(s, i, j, ...) -> replacement_string | nil
-- function utf8.replace(s, f, ...)
-- 	if type(f) == 'table' then
-- 		return utf8.replace(s, table_lookup, f)
-- 	end
-- 	if s == '' then
-- 		return s
-- 	end
-- 	local t = {}
-- 	local lasti = 1
-- 	for i in utf8.byte_indices(s) do
-- 		local nexti = utf8.next(s, i) or #s + 1
-- 		local repl = f(s, i, nexti - 1, ...)
-- 		if repl then
-- 			table.insert(t, s:sub(lasti, i - 1))
-- 			table.insert(t, repl)
-- 			lasti = nexti
-- 		end
-- 	end
-- 	table.insert(t, s:sub(lasti))
-- 	return table.concat(t)
-- end

-- local function replace_invalid(s, i, j, repl_char)
-- 	if not utf8.isvalid(s, i) then
-- 		return repl_char
-- 	end
-- end

-- --replace invalid utf8 chars with a replacement char
-- function utf8.sanitize(s, repl_char)
-- 	repl_char = repl_char or '�' --\uFFFD
-- 	return utf8.replace(s, replace_invalid, repl_char)
-- end

-- return utf8