# Originally we tried using `Sys.setlocale` but that isn't guaranteed to work
# e.g. failed on windows

line 91 appears to contain an embedded nul

Warning in readLines("unitizer/helper/UTF-8-test.txt", encoding = "UTF-8") :
  line 91 appears to contain an embedded nul

test.start <- grep("^Here come the tests:", utf8.kuhn)

nchar.base <- nchar(utf8.test, allowNA = TRUE)

# suppressWarnings(utf8.test)  # Solaris problems

base.vetr.diff <- !is.na(nchar.base) & nchar.vetr != nchar.base

# Not all lines match between nchar and vetr, differences seem to be that the
# invalid encoding starting \xf4\x90 is resolved to 1 character by base (test
# 2.3.5), and that base resolves things in the U+D800 - U+DFFF range as 1
# character (5.1-5.2); perhaps base implements the more forgiving versions of
# UTF8?

# Confirm offsets are what they should be

# Code Points        | Byte 1 | Byte 2 | Byte 3 | Byte 4
# Check all the critical cases where we transition from legal to illegal

vetr:::strsub(lat.1.1, 3L, mark = FALSE)

vetr:::strsub(lat.1.2, 3L, mark = FALSE)