Skip to content

Commit

Permalink
Initial implementation of TypedArray support. (#11)
Browse files Browse the repository at this point in the history
  • Loading branch information
iwoplaza authored Sep 11, 2024
1 parent 5fe4168 commit 9eb48a3
Show file tree
Hide file tree
Showing 14 changed files with 470 additions and 11 deletions.
1 change: 1 addition & 0 deletions apps/typed-binary-docs/astro.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export default defineConfig({
{ label: 'Objects', slug: 'guides/objects' },
{ label: 'Arrays and Tuples', slug: 'guides/arrays-and-tuples' },
{ label: 'Optionals', slug: 'guides/optionals' },
{ label: 'Typed Arrays', slug: 'guides/typed-arrays' },
{ label: 'Recursive Types', slug: 'guides/recursive-types' },
{
label: 'Custom Schema Types',
Expand Down
37 changes: 37 additions & 0 deletions apps/typed-binary-docs/src/content/docs/guides/typed-arrays.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
---
title: Typed Arrays
description: A guide on how typed arrays can be represented in Typed Binary
---

Sometimes binary is the format that we want to work with directly. [Typed array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Typed_arrays) schemas
allow that data to be nested in plain objects or arrays.

```ts
const Image = object({
size: tupleOf([u32, u32]),
bitmap: u8Array(256 * 256),
});

// {
// size: [number, number];
// bitmap: Uint8Array;
// }
const image = Image.read(...);

image.size // [number, number]
image.bitmap // Uint8Array
```

Below is the list of available typed array schemas.

Schema constructor | Encoded as | JavaScript value
---|---|---
`u8Array` | consecutive 8-bit unsigned integers | `Uint8Array`
`u8ClampedArray` | consecutive 8-bit unsigned integers | `Uint8ClampedArray`
`u16Array` | consecutive 16-bit unsigned integers | `Uint16Array`
`u32Array` | consecutive 32-bit unsigned integers | `Uint32Array`
`i8Array` | consecutive 8-bit signed integers | `Int8Array`
`i16Array` | consecutive 16-bit signed integers | `Int16Array`
`i32Array` | consecutive 32-bit signed integers | `Int32Array`
`f32Array` | consecutive 32-bit floats | `Float32Array`
`f64Array` | consecutive 64-bit floats | `Float64Array`
27 changes: 27 additions & 0 deletions cypress/e2e/bufferReader.cy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { BufferReader } from 'typed-binary';

describe('BufferReader', () => {
it('reads a uint32 from an ArrayBuffer', () => {
const buffer = new ArrayBuffer(64);
const u32Array = new Uint32Array(buffer);
u32Array[0] = 256;

const reader = new BufferReader(buffer);
expect(reader.readUint32()).to.equal(256);
});

it('reads an int32 array from an ArrayBuffer', () => {
const buffer = new ArrayBuffer(3 * 4);
const i32View = new Int32Array(buffer);

i32View[0] = 1;
i32View[1] = 2;
i32View[2] = 3;

const i32Array = new Int32Array(3);
const reader = new BufferReader(buffer);
reader.readSlice(i32Array, 0, i32Array.byteLength);

expect([...i32Array]).to.deep.eq([1, 2, 3]);
});
});
34 changes: 30 additions & 4 deletions cypress/e2e/bufferWriter.cy.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,39 @@
import { BufferReader, BufferWriter } from 'typed-binary';
import { BufferWriter } from 'typed-binary';

describe('BufferWriter', () => {
it('writes and reads from an ArrayBuffer', () => {
it('writes a uint32 from an ArrayBuffer', () => {
const buffer = new ArrayBuffer(64);
const writer = new BufferWriter(buffer);

writer.writeUint32(256);

const reader = new BufferReader(buffer);
expect(reader.readUint32()).to.equal(256);
const u32View = new Uint32Array(buffer);
expect(u32View[0]).to.equal(256);
});

it('writes an int32 array to an ArrayBuffer', () => {
const buffer = new ArrayBuffer(64);
const writer = new BufferWriter(buffer);

const i32Array = new Int32Array([1, 2, 3]);
writer.writeSlice(i32Array);

const i32View = new Int32Array(buffer);
expect(i32View[0]).to.equal(1);
expect(i32View[1]).to.equal(2);
expect(i32View[2]).to.equal(3);
});

it('writes a uint32 array to an ArrayBuffer', () => {
const buffer = new ArrayBuffer(64);
const writer = new BufferWriter(buffer);

const u32Array = new Uint32Array([1, 2, 3]);
writer.writeSlice(u32Array);

const u32View = new Uint32Array(buffer);
expect(u32View[0]).to.equal(1);
expect(u32View[1]).to.equal(2);
expect(u32View[2]).to.equal(3);
});
});
28 changes: 28 additions & 0 deletions packages/typed-binary/src/describe/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { DynamicArraySchema } from '../structure/dynamicArray';
import { KeyedSchema } from '../structure/keyed';
import { type AnyObjectSchema, GenericObjectSchema } from '../structure/object';
import { OptionalSchema } from '../structure/optional';
import { TypedArraySchema } from '../structure/typedArray';
import type {
AnySchema,
AnySchemaWithProperties,
Expand Down Expand Up @@ -56,6 +57,33 @@ export const tupleOf = <TSchema extends [AnySchema, ...AnySchema[]]>(
schemas: TSchema,
) => new TupleSchema(schemas);

export const u8Array = (length: number) =>
new TypedArraySchema(length, Uint8Array);

export const u8ClampedArray = (length: number) =>
new TypedArraySchema(length, Uint8ClampedArray);

export const u16Array = (length: number) =>
new TypedArraySchema(length, Uint16Array);

export const u32Array = (length: number) =>
new TypedArraySchema(length, Uint32Array);

export const i8Array = (length: number) =>
new TypedArraySchema(length, Int8Array);

export const i16Array = (length: number) =>
new TypedArraySchema(length, Int16Array);

export const i32Array = (length: number) =>
new TypedArraySchema(length, Int32Array);

export const f32Array = (length: number) =>
new TypedArraySchema(length, Float32Array);

export const f64Array = (length: number) =>
new TypedArraySchema(length, Float64Array);

export const optional = <TSchema extends AnySchema>(innerType: TSchema) =>
new OptionalSchema(innerType);

Expand Down
12 changes: 5 additions & 7 deletions packages/typed-binary/src/io/bufferIOBase.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { getSystemEndianness } from '../util';
import type { Endianness } from './types';
import { unwrapBuffer } from './unwrapBuffer';

export type BufferIOOptions = {
/**
Expand Down Expand Up @@ -33,14 +34,11 @@ export class BufferIOBase {
this.endianness = endianness === 'system' ? systemEndianness : endianness;
this.switchEndianness = this.endianness !== systemEndianness;

let innerBuffer = buffer;
if (typeof Buffer !== 'undefined' && innerBuffer instanceof Buffer) {
// Getting rid of the outer shell, which causes the Uint8Array line to create a copy, instead of a view.
this.byteOffset += innerBuffer.byteOffset;
innerBuffer = innerBuffer.buffer;
}
// Getting rid of the outer shell, which causes the Uint8Array line to create a copy, instead of a view.
const unwrapped = unwrapBuffer(buffer);
this.byteOffset += unwrapped.byteOffset;

this.uint8View = new Uint8Array(innerBuffer, 0);
this.uint8View = new Uint8Array(unwrapped.buffer, 0);

const helperBuffer = new ArrayBuffer(4);
this.helperInt32View = new Int32Array(helperBuffer);
Expand Down
17 changes: 17 additions & 0 deletions packages/typed-binary/src/io/bufferReader.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { BufferIOBase } from './bufferIOBase';
import type { ISerialInput } from './types';
import { unwrapBuffer } from './unwrapBuffer';

export class BufferReader extends BufferIOBase implements ISerialInput {
private copyInputToHelper(bytes: number) {
Expand Down Expand Up @@ -46,4 +47,20 @@ export class BufferReader extends BufferIOBase implements ISerialInput {

return contents;
}

readSlice(
bufferView: ArrayLike<number> & ArrayBufferView,
offset: number,
byteLength: number,
): void {
const unwrapped = unwrapBuffer(bufferView);
const destU8 = new Uint8Array(
unwrapped.buffer,
unwrapped.byteOffset + offset,
);

for (let i = 0; i < byteLength; ++i) {
destU8[i] = this.uint8View[this.byteOffset++];
}
}
}
124 changes: 124 additions & 0 deletions packages/typed-binary/src/io/bufferReaderWriter.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,130 @@ import { getSystemEndianness } from '../util';
import { BufferReader } from './bufferReader';
import { BufferWriter } from './bufferWriter';

describe('BufferWriter', () => {
it('should encode a Uint8Array', () => {
const uint8s = new Uint8Array([0, 1, 1, 2, 3, 5, 8, 255]);

const buffer = Buffer.alloc(8);
const writer = new BufferWriter(buffer);
writer.writeSlice(uint8s);

const reader = new BufferReader(buffer);
expect(reader.readByte()).to.eq(0);
expect(reader.readByte()).to.eq(1);
expect(reader.readByte()).to.eq(1);
expect(reader.readByte()).to.eq(2);
expect(reader.readByte()).to.eq(3);
expect(reader.readByte()).to.eq(5);
expect(reader.readByte()).to.eq(8);
expect(reader.readByte()).to.eq(255);
});

it('should encode a Int32Array', () => {
const int32s = new Int32Array([
0,
1,
1,
2,
3,
5,
-2_147_483_648, // min signed 32-bit integer value
2_147_483_647, // max signed 32-bit integer value
]);

const buffer = Buffer.alloc(8 * 4);
const writer = new BufferWriter(buffer);
writer.writeSlice(int32s);

const reader = new BufferReader(buffer);
expect(reader.readInt32()).to.eq(0);
expect(reader.readInt32()).to.eq(1);
expect(reader.readInt32()).to.eq(1);
expect(reader.readInt32()).to.eq(2);
expect(reader.readInt32()).to.eq(3);
expect(reader.readInt32()).to.eq(5);
expect(reader.readInt32()).to.eq(-2_147_483_648);
expect(reader.readInt32()).to.eq(2_147_483_647);
});

it('should encode a Uint32Array', () => {
const uint32s = new Uint32Array([
0,
1,
1,
2,
3,
5,
8,
4_294_967_295, // max unsigned 32-bit integer value
]);

const buffer = Buffer.alloc(8 * 4);
const writer = new BufferWriter(buffer);
writer.writeSlice(uint32s);

const reader = new BufferReader(buffer);
expect(reader.readUint32()).to.eq(0);
expect(reader.readUint32()).to.eq(1);
expect(reader.readUint32()).to.eq(1);
expect(reader.readUint32()).to.eq(2);
expect(reader.readUint32()).to.eq(3);
expect(reader.readUint32()).to.eq(5);
expect(reader.readUint32()).to.eq(8);
expect(reader.readUint32()).to.eq(4_294_967_295);
});
});

describe('BufferReader', () => {
it('should decode a Uint8Array', () => {
const buffer = Buffer.alloc(4);
const writer = new BufferWriter(buffer);
writer.writeByte(0);
writer.writeByte(15);
writer.writeByte(64);
writer.writeByte(255);

const destBuffer = new ArrayBuffer(4);
const destU8 = new Uint8Array(destBuffer);
const reader = new BufferReader(buffer);
reader.readSlice(destU8, 0, 4);

expect([...destU8]).toEqual([0, 15, 64, 255]);
});

it('should decode a Uint32Array', () => {
const buffer = Buffer.alloc(4 * 4);
const writer = new BufferWriter(buffer);
writer.writeUint32(0);
writer.writeUint32(15);
writer.writeUint32(255);
writer.writeUint32(4_294_967_295);

const destBuffer = new ArrayBuffer(4 * 4);
const destU32 = new Uint32Array(destBuffer);
const reader = new BufferReader(buffer);
reader.readSlice(destU32, 0, destBuffer.byteLength);

expect([...destU32]).toEqual([0, 15, 255, 4_294_967_295]);
});

it('should decode a Int32Array', () => {
const buffer = Buffer.alloc(4 * 4);
const writer = new BufferWriter(buffer);
writer.writeInt32(0);
writer.writeInt32(15);
writer.writeInt32(-2_147_483_648);
writer.writeInt32(2_147_483_647);

const destBuffer = new ArrayBuffer(4 * 4);
const destI32 = new Int32Array(destBuffer);
const reader = new BufferReader(buffer);
reader.readSlice(destI32, 0, destBuffer.byteLength);

expect([...destI32]).toEqual([0, 15, -2_147_483_648, 2_147_483_647]);
});
});

describe('BufferWriter/BufferReader', () => {
it('parses options correctly', () => {
const buffer = Buffer.alloc(16);
Expand Down
9 changes: 9 additions & 0 deletions packages/typed-binary/src/io/bufferWriter.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { BufferIOBase } from './bufferIOBase';
import type { ISerialOutput } from './types';
import { unwrapBuffer } from './unwrapBuffer';

export class BufferWriter extends BufferIOBase implements ISerialOutput {
private copyHelperToOutput(bytes: number) {
Expand Down Expand Up @@ -42,4 +43,12 @@ export class BufferWriter extends BufferIOBase implements ISerialOutput {
// Extra null character
this.uint8View[this.byteOffset++] = 0;
}

writeSlice(bufferView: ArrayLike<number> & ArrayBufferView): void {
const unwrapped = unwrapBuffer(bufferView);
const srcU8 = new Uint8Array(unwrapped.buffer, unwrapped.byteOffset);
for (const srcByte of srcU8) {
this.uint8View[this.byteOffset++] = srcByte;
}
}
}
4 changes: 4 additions & 0 deletions packages/typed-binary/src/io/types.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
export type Endianness = 'big' | 'little';

export type BufferView = ArrayLike<number> & ArrayBufferView;

export interface ISerialInput {
readBool(): boolean;
readByte(): number;
readInt32(): number;
readUint32(): number;
readFloat32(): number;
readString(): string;
readSlice(bufferView: BufferView, offset: number, byteLength: number): void;
seekTo(offset: number): void;
skipBytes(bytes: number): void;
readonly endianness: Endianness;
Expand All @@ -20,6 +23,7 @@ export interface ISerialOutput {
writeUint32(value: number): void;
writeFloat32(value: number): void;
writeString(value: string): void;
writeSlice(bufferView: BufferView): void;
seekTo(offset: number): void;
skipBytes(bytes: number): void;
readonly endianness: Endianness;
Expand Down
Loading

0 comments on commit 9eb48a3

Please sign in to comment.