Fix: Use utf-16 encoding in ContentReader::value_at

Author: Schottkyc137

vhdl_lang/src/data/contents.rs

@@ -6,6 +6,7 @@
 
 use super::latin_1::{char_to_latin1, Latin1String, Utf8ToLatin1Error};
 use super::source::{Position, Range};
+use itertools::Itertools;
 use std::fs::File;
 use std::io;
 use std::io::prelude::Read;
@@ -353,10 +354,9 @@ impl<'a> ContentReader<'a> {
 
     pub fn value_at(&self, line: usize, start: usize, stop: usize) -> Option<Latin1String> {
         let line = self.contents.get_line(line)?;
-        if stop > line.len() {
-            return None;
-        }
-        Latin1String::from_utf8(&line[start..stop]).ok()
+        let utf16_view: Box<[u16]> = line.encode_utf16().get(start..stop).collect();
+        let str_view = String::from_utf16(&utf16_view).ok()?;
+        Latin1String::from_utf8(&str_view).ok()
     }
 }
 

vhdl_lang/src/data/source.rs

@@ -185,6 +185,7 @@ pub struct Position {
     /// Line (zero-based).
     pub line: u32,
     /// Column (zero-based).
+    /// The character offset is utf-16 encoded
     pub character: u32,
 }
 

vhdl_lang/src/syntax/tokens/tokenizer.rs

@@ -2651,6 +2651,34 @@ my_other_ident",
         );
     }
 
+    #[test]
+    fn non_ascii_before_bit_string() {
+        let code = Code::new("€X\"ABC\"");
+        let (tokens, _) = code.tokenize_result();
+        assert_eq!(
+            tokens,
+            vec![
+                Err(Diagnostic::syntax_error(
+                    code.s1("€"),
+                    "Found invalid latin-1 character '€'",
+                )),
+                Ok(Token {
+                    kind: BitString,
+                    value: Value::BitString(
+                        Latin1String::from_utf8_unchecked("X\"ABC\""),
+                        ast::BitString {
+                            length: None,
+                            base: BaseSpecifier::X,
+                            value: Latin1String::from_utf8_unchecked("ABC"),
+                        }
+                    ),
+                    pos: code.s1("X\"ABC\"").pos(),
+                    comments: None,
+                })
+            ],
+        );
+    }
+
     #[test]
     fn tokenize_based_integer() {
         assert_eq!(