def tidy_bytes(string, force = false)
if force
return string.unpack("C*").map do |b|
tidy_byte(b)
end.flatten.compact.pack("C*").unpack("U*").pack("U*")
end
bytes = string.unpack("C*")
conts_expected = 0
last_lead = 0
bytes.each_index do |i|
byte = bytes[i]
is_cont = byte > 127 && byte < 192
is_lead = byte > 191 && byte < 245
is_unused = byte > 240
is_restricted = byte > 244
if is_unused || is_restricted
bytes[i] = tidy_byte(byte)
elsif is_cont
conts_expected == 0 ? bytes[i] = tidy_byte(byte) : conts_expected -= 1
else
if conts_expected > 0
(1..(i - last_lead)).each {|j| bytes[i - j] = tidy_byte(bytes[i - j])}
conts_expected = 0
end
if is_lead
if i == bytes.length - 1
bytes[i] = tidy_byte(bytes.last)
else
conts_expected = byte < 224 ? 1 : byte < 240 ? 2 : 3
last_lead = i
end
end
end
end
bytes.empty? ? "" : bytes.flatten.compact.pack("C*").unpack("U*").pack("U*")
end