feat: UTF-8 string encoding and decoding

packages/typed-binary/src/io/bufferReader.ts

@@ -3,6 +3,15 @@ import type { ISerialInput } from './types';
 import { unwrapBuffer } from './unwrapBuffer';
 
 export class BufferReader extends BufferIOBase implements ISerialInput {
+  private _cachedTextDecoder: TextDecoder | undefined;
+
+  private get _textDecoder() {
+    if (!this._cachedTextDecoder) {
+      this._cachedTextDecoder = new TextDecoder(undefined, { fatal: true });
+    }
+    return this._cachedTextDecoder;
+  }
+
   private copyInputToHelper(bytes: number) {
     for (let i = 0; i < bytes; ++i) {
       this.helperByteView[this.switchEndianness ? bytes - 1 - i : i] =
@@ -37,15 +46,21 @@ export class BufferReader extends BufferIOBase implements ISerialInput {
   }
 
   readString() {
-    let contents = '';
-
-    let char = String.fromCharCode(this.uint8View[this.byteOffset++]);
-    while (char !== '\0') {
-      contents += char;
-      char = String.fromCharCode(this.uint8View[this.byteOffset++]);
+    // Looking for the 'NULL' byte.
+    let end = this.byteOffset;
+    while (end < this.uint8View.byteLength) {
+      if (this.uint8View[end++] === 0) {
+        break;
+      }
     }
 
-    return contents;
+    const result = this._textDecoder.decode(
+      this.uint8View.subarray(this.byteOffset, end - 1),
+    );
+
+    this.byteOffset = end;
+
+    return result;
   }
 
   readSlice(

packages/typed-binary/src/io/bufferWriter.ts

@@ -3,6 +3,15 @@ import type { ISerialOutput } from './types';
 import { unwrapBuffer } from './unwrapBuffer';
 
 export class BufferWriter extends BufferIOBase implements ISerialOutput {
+  private _cachedTextEncoder: TextEncoder | undefined;
+
+  private get _textEncoder() {
+    if (!this._cachedTextEncoder) {
+      this._cachedTextEncoder = new TextEncoder();
+    }
+    return this._cachedTextEncoder;
+  }
+
   private copyHelperToOutput(bytes: number) {
     for (let i = 0; i < bytes; ++i)
       this.uint8View[this.byteOffset++] =
@@ -36,9 +45,11 @@ export class BufferWriter extends BufferIOBase implements ISerialOutput {
   }
 
   writeString(value: string) {
-    for (let i = 0; i < value.length; ++i) {
-      this.uint8View[this.byteOffset++] = value.charCodeAt(i);
-    }
+    const result = this._textEncoder.encodeInto(
+      value,
+      this.uint8View.subarray(this.byteOffset),
+    );
+    this.byteOffset += result.written;
 
     // Extra null character
     this.uint8View[this.byteOffset++] = 0;

packages/typed-binary/src/structure/baseTypes.ts

@@ -39,6 +39,15 @@ export class BoolSchema extends Schema<boolean> {
 ////
 
 export class StringSchema extends Schema<string> {
+  private static _cachedEncoder: TextEncoder | undefined;
+
+  private static get _encoder() {
+    if (!StringSchema._cachedEncoder) {
+      StringSchema._cachedEncoder = new TextEncoder();
+    }
+    return StringSchema._cachedEncoder;
+  }
+
   read(input: ISerialInput): string {
     return input.readString();
   }
@@ -55,7 +64,8 @@ export class StringSchema extends Schema<string> {
       // A string cannot be bound
       return measurer.unbounded;
     }
-    return measurer.add(value.length + 1);
+    const encoded = StringSchema._encoder.encode(value);
+    return measurer.add(encoded.byteLength + 1);
   }
 }
 

packages/typed-binary/src/test/object.test.ts

@@ -9,6 +9,7 @@ import {
   object,
   optional,
   string,
+  u32,
 } from '../describe';
 import { type ISchema, MaxValue, type ObjectSchema } from '../structure';
 import type { Parsed } from '../utilityTypes';
@@ -28,11 +29,13 @@ describe('ObjectSchema', () => {
     const description = object({
       value: i32,
       label: string,
+      extra: u32,
     });
 
     const value = {
       value: 70,
       label: 'Banana',
+      extra: 43,
     };
 
     const { output, input } = makeIO(64);

packages/typed-binary/src/test/string.test.ts

@@ -28,4 +28,16 @@ describe('StringSchema', () => {
     const decoded = encodeAndDecode(string, value);
     expect(decoded).to.equal(value);
   });
+
+  it('should encode an emoji', () => {
+    const value = '⛓️‍💥';
+    const decoded = encodeAndDecode(string, value);
+    expect(decoded).to.equal(value);
+  });
+
+  it('should encode a unicode string', () => {
+    const value = 'A wonderful 🌞 sunny day! 🌲🌲🌲 Forest trip.';
+    const decoded = encodeAndDecode(string, value);
+    expect(decoded).to.equal(value);
+  });
 });