diff --git a/.changeset/real-pens-smile.md b/.changeset/real-pens-smile.md new file mode 100644 index 00000000000..34e746e1626 --- /dev/null +++ b/.changeset/real-pens-smile.md @@ -0,0 +1,5 @@ +--- +"@smithy/core": minor +--- + +support BigInt in cbor diff --git a/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts b/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts index ca6bb407f5f..4b4271a79ca 100644 --- a/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts +++ b/packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts @@ -155,7 +155,7 @@ describe(SmithyRpcV2CborProtocol.name, () => { { name: "dummy", input: testCase.schema, - output: void 0, + output: "unit", traits: {}, }, testCase.input, @@ -257,7 +257,7 @@ describe(SmithyRpcV2CborProtocol.name, () => { const output = await protocol.deserializeResponse( { name: "dummy", - input: void 0, + input: "unit", output: testCase.schema, traits: {}, }, diff --git a/packages/core/src/submodules/cbor/byte-printer.ts b/packages/core/src/submodules/cbor/byte-printer.ts new file mode 100644 index 00000000000..7c32fc2be6e --- /dev/null +++ b/packages/core/src/submodules/cbor/byte-printer.ts @@ -0,0 +1,7 @@ +/** + * Prints bytes as binary string with numbers. + * @param bytes + */ +export function printBytes(bytes: Uint8Array) { + return [...bytes].map((n) => ("0".repeat(8) + n.toString(2)).slice(-8) + ` (${n})`); +} diff --git a/packages/core/src/submodules/cbor/cbor-decode.ts b/packages/core/src/submodules/cbor/cbor-decode.ts index 1931e23c064..7f7c3d317d8 100644 --- a/packages/core/src/submodules/cbor/cbor-decode.ts +++ b/packages/core/src/submodules/cbor/cbor-decode.ts @@ -1,3 +1,4 @@ +import { NumericValue } from "@smithy/core/serde"; import { toUtf8 } from "@smithy/util-utf8"; import { @@ -119,11 +120,33 @@ export function decode(at: Uint32, to: Uint32): CborValueType { _offset = offset; return castBigInt(negativeInt); } else { - const value = decode(at + offset, to); - const valueOffset = _offset; + /* major === majorTag */ + if (minor === 2 || minor === 3) { + const length = decodeCount(at + offset, to); + + let b = BigInt(0); + const start = at + offset + _offset; + for (let i = start; i < start + length; ++i) { + b = (b << BigInt(8)) | BigInt(payload[i]); + } + + _offset = offset + length; + return minor === 3 ? -b - BigInt(1) : b; + } else if (minor === 4) { + const decimalFraction = decode(at + offset, to); + const [exponent, mantissa] = decimalFraction; + const s = mantissa.toString(); + const numericString = exponent === 0 ? s : s.slice(0, s.length + exponent) + "." + s.slice(exponent); + + return new NumericValue(numericString, "bigDecimal"); + } else { + const value = decode(at + offset, to); + const valueOffset = _offset; + + _offset = offset + valueOffset; - _offset = offset + valueOffset; - return tag({ tag: castBigInt(unsignedInt), value }); + return tag({ tag: castBigInt(unsignedInt), value }); + } } case majorUtf8String: case majorMap: diff --git a/packages/core/src/submodules/cbor/cbor-encode.ts b/packages/core/src/submodules/cbor/cbor-encode.ts index 6ee27f65b65..168fe6b2cb0 100644 --- a/packages/core/src/submodules/cbor/cbor-encode.ts +++ b/packages/core/src/submodules/cbor/cbor-encode.ts @@ -1,6 +1,8 @@ +import { NumericValue } from "@smithy/core/serde"; import { fromUtf8 } from "@smithy/util-utf8"; import { + alloc, CborMajorType, extendedFloat16, extendedFloat32, @@ -19,7 +21,6 @@ import { tagSymbol, Uint64, } from "./cbor-types"; -import { alloc } from "./cbor-types"; const USE_BUFFER = typeof Buffer !== "undefined"; @@ -152,10 +153,30 @@ export function encode(_input: any): void { data[cursor++] = (major << 5) | extendedFloat32; dataView.setUint32(cursor, n); cursor += 4; - } else { + } else if (value < BigInt("18446744073709551616")) { data[cursor++] = (major << 5) | extendedFloat64; dataView.setBigUint64(cursor, value); cursor += 8; + } else { + // refer to https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums + const binaryBigInt = value.toString(2); + const bigIntBytes = new Uint8Array(Math.ceil(binaryBigInt.length / 8)); + let b = value; + let i = 0; + while (bigIntBytes.byteLength - ++i >= 0) { + bigIntBytes[bigIntBytes.byteLength - i] = Number(b & BigInt(255)); + b >>= BigInt(8); + } + ensureSpace(bigIntBytes.byteLength * 2); + data[cursor++] = nonNegative ? 0b110_00010 : 0b110_00011; + + if (USE_BUFFER) { + encodeHeader(majorUnstructuredByteString, Buffer.byteLength(bigIntBytes)); + } else { + encodeHeader(majorUnstructuredByteString, bigIntBytes.byteLength); + } + data.set(bigIntBytes, cursor); + cursor += bigIntBytes.byteLength; } continue; } else if (input === null) { @@ -181,6 +202,18 @@ export function encode(_input: any): void { cursor += input.byteLength; continue; } else if (typeof input === "object") { + if (input instanceof NumericValue) { + const decimalIndex = input.string.indexOf("."); + const exponent = decimalIndex === -1 ? 0 : decimalIndex - input.string.length + 1; + const mantissa = BigInt(input.string.replace(".", "")); + + data[cursor++] = 0b110_00100; // major 6, tag 4. + + encodeStack.push(mantissa); + encodeStack.push(exponent); + encodeHeader(majorList, 2); + continue; + } if (input[tagSymbol]) { if ("tag" in input && "value" in input) { encodeStack.push(input.value); diff --git a/packages/core/src/submodules/cbor/cbor.spec.ts b/packages/core/src/submodules/cbor/cbor.spec.ts index 2c736242ea6..b2346892aeb 100644 --- a/packages/core/src/submodules/cbor/cbor.spec.ts +++ b/packages/core/src/submodules/cbor/cbor.spec.ts @@ -1,3 +1,4 @@ +import { NumericValue } from "@smithy/core/serde"; import * as fs from "fs"; // @ts-ignore import JSONbig from "json-bigint"; @@ -88,12 +89,12 @@ describe("cbor", () => { { name: "negative float", data: -3015135.135135135, - cbor: allocByteArray([0b111_11011, +193, +71, +0, +239, +145, +76, +27, +173]), + cbor: allocByteArray([0b111_11011, 193, 71, 0, 239, 145, 76, 27, 173]), }, { name: "positive float", data: 3015135.135135135, - cbor: allocByteArray([0b111_11011, +65, +71, +0, +239, +145, +76, +27, +173]), + cbor: allocByteArray([0b111_11011, 65, 71, 0, 239, 145, 76, 27, 173]), }, { name: "various numbers", @@ -214,6 +215,18 @@ describe("cbor", () => { 65, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32, 66, ]), }, + { + name: "object containing big numbers", + data: { + map: { + items: [BigInt(1e80)], + }, + }, + cbor: allocByteArray([ + 161, 99, 109, 97, 112, 161, 101, 105, 116, 101, 109, 115, 129, 194, 88, 34, 3, 95, 157, 234, 62, 31, 107, 224, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]), + }, ]; const toBytes = (hex: string) => { @@ -226,6 +239,72 @@ describe("cbor", () => { }; describe("locally curated scenarios", () => { + it("should round-trip bigInteger to major 6 with tag 2", () => { + const bigInt = BigInt("1267650600228229401496703205376"); + const serialized = cbor.serialize(bigInt); + + const major = serialized[0] >> 5; + expect(major).toEqual(0b110); // 6 + + const tag = serialized[0] & 0b11111; + expect(tag).toEqual(0b010); // 2 + + const byteStringCount = serialized[1]; + expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes + + const byteString = serialized.slice(2); + expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])); + + const deserialized = cbor.deserialize(serialized); + expect(deserialized).toEqual(bigInt); + }); + + it("should round-trip negative bigInteger to major 6 with tag 3", () => { + const bigInt = BigInt("-1267650600228229401496703205377"); + const serialized = cbor.serialize(bigInt); + + const major = serialized[0] >> 5; + expect(major).toEqual(0b110); // 6 + + const tag = serialized[0] & 0b11111; + expect(tag).toEqual(0b011); // 3 + + const byteStringCount = serialized[1]; + expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes + + const byteString = serialized.slice(2); + expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])); + + const deserialized = cbor.deserialize(serialized); + expect(deserialized).toEqual(bigInt); + }); + + it("should round-trip NumericValue to major 6 with tag 4", () => { + for (const bigDecimal of [ + "10000000000000000000000054.321", + "1000000000000000000000000000000000054.134134321", + "100000000000000000000000000000000000054.0000000000000001", + "100000000000000000000000000000000000054.00510351095130000", + "-10000000000000000000000054.321", + "-1000000000000000000000000000000000054.134134321", + "-100000000000000000000000000000000000054.0000000000000001", + "-100000000000000000000000000000000000054.00510351095130000", + ]) { + const nv = new NumericValue(bigDecimal, "bigDecimal"); + const serialized = cbor.serialize(nv); + + const major = serialized[0] >> 5; + expect(major).toEqual(0b110); // 6 + + const tag = serialized[0] & 0b11111; + expect(tag).toEqual(0b0100); // 4 + + const deserialized = cbor.deserialize(serialized); + expect(deserialized).toEqual(nv); + expect(deserialized.string).toEqual(nv.string); + } + }); + it("should throw an error if serializing a tag with missing properties", () => { expect(() => cbor.serialize({ diff --git a/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts b/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts index 3a815c00414..da32f45f262 100644 --- a/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts +++ b/packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts @@ -5,6 +5,7 @@ import { CodecSettings, HandlerExecutionContext, HttpResponse as IHttpResponse, + MetadataBearer, OperationSchema, ResponseMetadata, ShapeDeserializer, @@ -64,7 +65,7 @@ describe(HttpBindingProtocol.name, () => { }); const protocol = new StringRestProtocol(); - const output = await protocol.deserializeResponse( + const output = (await protocol.deserializeResponse( op( "", "", @@ -87,7 +88,7 @@ describe(HttpBindingProtocol.name, () => { ), {} as any, response - ); + )) as Partial; delete output.$metadata; expect(output).toEqual({ timestampList: [new Date("2019-12-16T23:48:18.000Z"), new Date("2019-12-16T23:48:18.000Z")], @@ -104,7 +105,7 @@ describe(HttpBindingProtocol.name, () => { }); const protocol = new StringRestProtocol(); - const output = await protocol.deserializeResponse( + const output = (await protocol.deserializeResponse( op( "", "", @@ -127,7 +128,7 @@ describe(HttpBindingProtocol.name, () => { ), {} as any, response - ); + )) as Partial; delete output.$metadata; expect(output).toEqual({ httpPrefixHeaders: { diff --git a/packages/core/src/submodules/serde/parse-utils.spec.ts b/packages/core/src/submodules/serde/parse-utils.spec.ts index 455af97344c..4b904c99692 100644 --- a/packages/core/src/submodules/serde/parse-utils.spec.ts +++ b/packages/core/src/submodules/serde/parse-utils.spec.ts @@ -22,6 +22,8 @@ import { } from "./parse-utils"; import { expectBoolean, expectNumber, expectString } from "./parse-utils"; +logger.warn = () => {}; + describe("parseBoolean", () => { it('Returns true for "true"', () => { expect(parseBoolean("true")).toEqual(true); diff --git a/packages/core/src/submodules/serde/value/NumericValue.spec.ts b/packages/core/src/submodules/serde/value/NumericValue.spec.ts index b5fb7090842..26cbacc4dbd 100644 --- a/packages/core/src/submodules/serde/value/NumericValue.spec.ts +++ b/packages/core/src/submodules/serde/value/NumericValue.spec.ts @@ -9,4 +9,11 @@ describe(NumericValue.name, () => { expect(num.string).toEqual("1.0"); expect(num.type).toEqual("bigDecimal"); }); + + it("allows only numeric digits and at most one decimal point", () => { + expect(() => nv("a")).toThrow(); + expect(() => nv("1.0.1")).toThrow(); + expect(() => nv("-10.1")).not.toThrow(); + expect(() => nv("-.101")).not.toThrow(); + }); }); diff --git a/packages/core/src/submodules/serde/value/NumericValue.ts b/packages/core/src/submodules/serde/value/NumericValue.ts index 11b54144114..4ce3185b667 100644 --- a/packages/core/src/submodules/serde/value/NumericValue.ts +++ b/packages/core/src/submodules/serde/value/NumericValue.ts @@ -24,7 +24,41 @@ export class NumericValue { public constructor( public readonly string: string, public readonly type: NumericType - ) {} + ) { + let dot = 0; + for (let i = 0; i < string.length; ++i) { + const char = string.charCodeAt(i); + if (i === 0 && char === 45) { + // negation prefix "-" + continue; + } + if (char === 46) { + // decimal point "." + if (dot) { + throw new Error("@smithy/core/serde - NumericValue must contain at most one decimal point."); + } + dot = 1; + continue; + } + if (char < 48 || char > 57) { + // not in 0 through 9 + throw new Error( + `@smithy/core/serde - NumericValue must only contain [0-9], at most one decimal point ".", and an optional negation prefix "-".` + ); + } + } + } + + public [Symbol.hasInstance](object: unknown) { + if (!object || typeof object !== "object") { + return false; + } + const _nv = object as NumericValue; + if (typeof _nv.string === "string" && typeof _nv.type === "string" && _nv.constructor?.name === "NumericValue") { + return true; + } + return false; + } } /** diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index 4e46707824a..ffbea510c49 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -5,5 +5,6 @@ export default defineConfig({ exclude: ["**/*.{integ,e2e,browser}.spec.ts"], include: ["**/*.spec.ts"], environment: "node", + hideSkippedTests: true, }, });