From 1d5c2de7e246d213bf356fc5575f58af9996ec6c Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 10 Feb 2025 11:13:25 -0500 Subject: [PATCH 01/13] fix(NODE-6735): prevent serializer and stringify from creating invalid float32 vectors --- src/binary.ts | 6 + test/node/bson_binary_vector.spec.test.ts | 146 ++++++++++++------ .../specs/bson-binary-vector/float32.json | 11 +- test/node/specs/bson-binary-vector/int8.json | 4 +- .../specs/bson-binary-vector/packed_bit.json | 23 +-- 5 files changed, 123 insertions(+), 67 deletions(-) diff --git a/src/binary.ts b/src/binary.ts index 1fe098058..85b689486 100644 --- a/src/binary.ts +++ b/src/binary.ts @@ -517,6 +517,12 @@ export function validateBinaryVector(vector: Binary): void { throw new BSONError('Invalid Vector: padding must be zero for int8 and float32 vectors'); } + if (datatype === Binary.VECTOR_TYPE.Float32) { + if (size !== 0 && size - 2 !== 0 && (size - 2) % 4 !== 0) { + throw new BSONError('Invalid Vector: Float32 vector must contain a multiple of 4 bytes'); + } + } + if (datatype === Binary.VECTOR_TYPE.PackedBit && padding !== 0 && size === 2) { throw new BSONError( 'Invalid Vector: padding must be zero for packed bit vectors that are empty' diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 87f573abe..424c1b0d0 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -1,6 +1,7 @@ +import * as util from 'util'; import * as fs from 'fs'; import * as path from 'path'; -import { BSON, BSONError, Binary } from '../register-bson'; +import { BSON, BSONError, Binary, EJSON } from '../register-bson'; import { expect } from 'chai'; const { toHex, fromHex } = BSON.onDemand.ByteUtils; @@ -8,7 +9,7 @@ const { toHex, fromHex } = BSON.onDemand.ByteUtils; type VectorHexType = '0x03' | '0x27' | '0x10'; type VectorTest = { description: string; - vector: (number | string)[]; + vector?: (number | string)[]; valid: boolean; dtype_hex: VectorHexType; padding?: number; @@ -87,6 +88,10 @@ const invalidTestExpectedError = new Map() 'Invalid Vector: padding must be a value between 0 and 7' ) .set('Negative padding PACKED_BIT', 'Invalid Vector: padding must be a value between 0 and 7') + .set( + 'Insufficient vector data FLOAT32', + 'Invalid Vector: Float32 vector must contain a multiple of 4 bytes' + ) // skipped .set('Overflow Vector PACKED_BIT', false) .set('Underflow Vector PACKED_BIT', false) @@ -97,6 +102,84 @@ const invalidTestExpectedError = new Map() .set('Vector with float values PACKED_BIT', false) .set('Vector with float values PACKED_BIT', false); +function testVectorBuilding(test: VectorTest, expectedErrorMessage: string) { + describe('creating an instance of a Binary class using parameters', () => { + it(`bson: ${test.description}`, function () { + let thrownError: Error | undefined; + try { + const bin = make(test.vector!, test.dtype_hex, test.padding); + BSON.serialize({ bin }); + } catch (error) { + thrownError = error; + } + + if (thrownError?.message.startsWith('unsupported_error')) { + expect( + expectedErrorMessage, + 'We expect a certain error message but got an unsupported error' + ).to.be.false; + this.skip(); + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + + it(`ejson: ${test.description}`, function () { + let thrownError: Error | undefined; + try { + const bin = make(test.vector!, test.dtype_hex, test.padding); + BSON.EJSON.stringify({ bin }); + } catch (error) { + thrownError = error; + } + + if (thrownError?.message.startsWith('unsupported_error')) { + expect( + expectedErrorMessage, + 'We expect a certain error message but got an unsupported error' + ).to.be.false; + this.skip(); + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + }); +} + +function testVectorReserializing(test: VectorTest, expectedErrorMessage: string) { + describe('creating an instance of a Binary class using canonical_bson', () => { + it(`bson deserialize: ${test.description}`, function () { + let thrownError: Error | undefined; + const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); + + try { + BSON.serialize(bin); + } catch (error) { + thrownError = error; + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + + it(`ejson stringify: ${test.description}`, function () { + let thrownError: Error | undefined; + const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); + + try { + EJSON.stringify(bin); + } catch (error) { + thrownError = error; + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + }); +} + describe('BSON Binary Vector spec tests', () => { const tests: Record = Object.create(null); @@ -121,7 +204,7 @@ describe('BSON Binary Vector spec tests', () => { */ for (const test of valid) { it(`encode ${test.description}`, function () { - const bin = make(test.vector, test.dtype_hex, test.padding); + const bin = make(test.vector!, test.dtype_hex, test.padding); const buffer = BSON.serialize({ [suite.test_key]: bin }); expect(toHex(buffer)).to.equal(test.canonical_bson!.toLowerCase()); @@ -147,47 +230,22 @@ describe('BSON Binary Vector spec tests', () => { for (const test of invalid) { const expectedErrorMessage = invalidTestExpectedError.get(test.description); - it(`bson: ${test.description}`, function () { - let thrownError: Error | undefined; - try { - const bin = make(test.vector, test.dtype_hex, test.padding); - BSON.serialize({ bin }); - } catch (error) { - thrownError = error; - } - - if (thrownError?.message.startsWith('unsupported_error')) { - expect( - expectedErrorMessage, - 'We expect a certain error message but got an unsupported error' - ).to.be.false; - this.skip(); - } - - expect(thrownError).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); - }); - - it(`extended json: ${test.description}`, function () { - let thrownError: Error | undefined; - try { - const bin = make(test.vector, test.dtype_hex, test.padding); - BSON.EJSON.stringify({ bin }); - } catch (error) { - thrownError = error; - } - - if (thrownError?.message.startsWith('unsupported_error')) { - expect( - expectedErrorMessage, - 'We expect a certain error message but got an unsupported error' - ).to.be.false; - this.skip(); - } - - expect(thrownError).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); - }); + if (test.vector != null && test.canonical_bson != null) { + describe('both manual vector building and re-serializing', () => { + testVectorBuilding(test, expectedErrorMessage); + testVectorReserializing(test, expectedErrorMessage); + }); + } else if (test.canonical_bson != null) { + describe('vector re-serializing', () => { + testVectorReserializing(test, expectedErrorMessage); + }); + } else if (test.vector != null) { + describe('manual vector building', () => { + testVectorBuilding(test, expectedErrorMessage); + }); + } else { + throw new Error('not testing anything for: ' + util.inspect(test)); + } } }); }); diff --git a/test/node/specs/bson-binary-vector/float32.json b/test/node/specs/bson-binary-vector/float32.json index 872c43532..4ec68eae0 100644 --- a/test/node/specs/bson-binary-vector/float32.json +++ b/test/node/specs/bson-binary-vector/float32.json @@ -44,8 +44,15 @@ "vector": [127.0, 7.0], "dtype_hex": "0x27", "dtype_alias": "FLOAT32", - "padding": 3 + "padding": 3, + "canonical_bson": "1C00000005766563746F72000A0000000927030000FE420000E04000" + }, + { + "description": "Insufficient vector data FLOAT32", + "valid": false, + "dtype_hex": "0x27", + "dtype_alias": "FLOAT32", + "canonical_bson": "1700000005766563746F7200050000000927002A2A2A00" } ] } - diff --git a/test/node/specs/bson-binary-vector/int8.json b/test/node/specs/bson-binary-vector/int8.json index 7529721e5..29524fb61 100644 --- a/test/node/specs/bson-binary-vector/int8.json +++ b/test/node/specs/bson-binary-vector/int8.json @@ -42,7 +42,8 @@ "vector": [127, 7], "dtype_hex": "0x03", "dtype_alias": "INT8", - "padding": 3 + "padding": 3, + "canonical_bson": "1600000005766563746F7200040000000903037F0700" }, { "description": "INT8 with float inputs", @@ -54,4 +55,3 @@ } ] } - diff --git a/test/node/specs/bson-binary-vector/packed_bit.json b/test/node/specs/bson-binary-vector/packed_bit.json index 035776e87..a220e7e31 100644 --- a/test/node/specs/bson-binary-vector/packed_bit.json +++ b/test/node/specs/bson-binary-vector/packed_bit.json @@ -8,7 +8,8 @@ "vector": [], "dtype_hex": "0x10", "dtype_alias": "PACKED_BIT", - "padding": 1 + "padding": 1, + "canonical_bson": "1400000005766563746F72000200000009100100" }, { "description": "Simple Vector PACKED_BIT", @@ -61,21 +62,14 @@ "dtype_alias": "PACKED_BIT", "padding": 0 }, - { - "description": "Padding specified with no vector data PACKED_BIT", - "valid": false, - "vector": [], - "dtype_hex": "0x10", - "dtype_alias": "PACKED_BIT", - "padding": 1 - }, { "description": "Exceeding maximum padding PACKED_BIT", "valid": false, "vector": [1], "dtype_hex": "0x10", "dtype_alias": "PACKED_BIT", - "padding": 8 + "padding": 8, + "canonical_bson": "1500000005766563746F7200030000000910080100" }, { "description": "Negative padding PACKED_BIT", @@ -84,15 +78,6 @@ "dtype_hex": "0x10", "dtype_alias": "PACKED_BIT", "padding": -1 - }, - { - "description": "Vector with float values PACKED_BIT", - "valid": false, - "vector": [127.5], - "dtype_hex": "0x10", - "dtype_alias": "PACKED_BIT", - "padding": 0 } ] } - From 595ea7ae1551d8aa50ffa081acc8f4684d3bb42a Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Mon, 24 Mar 2025 12:30:41 -0400 Subject: [PATCH 02/13] chore: update tests to ejson --- test/node/bson_binary_vector.spec.test.ts | 12 +++--------- test/node/specs/bson-binary-vector/float32.json | 11 +++++++++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 424c1b0d0..f1fa8daef 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -18,15 +18,11 @@ type VectorTest = { type VectorSuite = { description: string; test_key: string; tests: VectorTest[] }; function fixFloats(f: string | number): number { + // Should be nothing to "fix" but validates we didn't get + // an unexpected type so we don't silently fail on it during the test if (typeof f === 'number') { return f; } - if (f === 'inf') { - return Infinity; - } - if (f === '-inf') { - return -Infinity; - } throw new Error(`test format error: unknown float value: ${f}`); } @@ -98,8 +94,6 @@ const invalidTestExpectedError = new Map() .set('Overflow Vector INT8', false) .set('Underflow Vector INT8', false) .set('INT8 with float inputs', false) - // duplicate test! but also skipped. - .set('Vector with float values PACKED_BIT', false) .set('Vector with float values PACKED_BIT', false); function testVectorBuilding(test: VectorTest, expectedErrorMessage: string) { @@ -184,7 +178,7 @@ describe('BSON Binary Vector spec tests', () => { const tests: Record = Object.create(null); for (const file of fs.readdirSync(path.join(__dirname, 'specs/bson-binary-vector'))) { - tests[path.basename(file, '.json')] = JSON.parse( + tests[path.basename(file, '.json')] = EJSON.parse( fs.readFileSync(path.join(__dirname, 'specs/bson-binary-vector', file), 'utf8') ); } diff --git a/test/node/specs/bson-binary-vector/float32.json b/test/node/specs/bson-binary-vector/float32.json index 4ec68eae0..72dafce10 100644 --- a/test/node/specs/bson-binary-vector/float32.json +++ b/test/node/specs/bson-binary-vector/float32.json @@ -32,7 +32,7 @@ { "description": "Infinity Vector FLOAT32", "valid": true, - "vector": ["-inf", 0.0, "inf"], + "vector": [{"$numberDouble": "-Infinity"}, 0.0, {"$numberDouble": "Infinity"} ], "dtype_hex": "0x27", "dtype_alias": "FLOAT32", "padding": 0, @@ -48,11 +48,18 @@ "canonical_bson": "1C00000005766563746F72000A0000000927030000FE420000E04000" }, { - "description": "Insufficient vector data FLOAT32", + "description": "Insufficient vector data with 3 bytes FLOAT32", "valid": false, "dtype_hex": "0x27", "dtype_alias": "FLOAT32", "canonical_bson": "1700000005766563746F7200050000000927002A2A2A00" + }, + { + "description": "Insufficient vector data with 5 bytes FLOAT32", + "valid": false, + "dtype_hex": "0x27", + "dtype_alias": "FLOAT32", + "canonical_bson": "1900000005766563746F7200070000000927002A2A2A2A2A00" } ] } From aaab7a3abfdbab0a74981e07c336c0191063ff36 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 25 Mar 2025 17:51:05 -0400 Subject: [PATCH 03/13] refactor: improve BSON Binary Vector tests titles --- test/node/bson_binary_vector.spec.test.ts | 125 ++++++++++++---------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index f1fa8daef..df5fd36e5 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -9,7 +9,7 @@ const { toHex, fromHex } = BSON.onDemand.ByteUtils; type VectorHexType = '0x03' | '0x27' | '0x10'; type VectorTest = { description: string; - vector?: (number | string)[]; + vector?: number[]; valid: boolean; dtype_hex: VectorHexType; padding?: number; @@ -46,7 +46,22 @@ function fixBits(f: number | string): number { return f; } -function make(vector: (number | string)[], dtype_hex: VectorHexType, padding?: number): Binary { +function dtypeToHelper(dtype_hex: string) { + switch (dtype_hex) { + case '0x10' /* packed_bit */: + return 'fromPackedBits'; + case '0x03' /* int8 */: + return 'fromInt8Array'; + break; + case '0x27' /* float32 */: + return 'fromFloat32Array'; + break; + default: + throw new Error(`Unknown dtype_hex: ${dtype_hex}`); + } +} + +function make(vector: number[], dtype_hex: VectorHexType, padding?: number): Binary { let binary: Binary; switch (dtype_hex) { case '0x10' /* packed_bit */: @@ -88,17 +103,18 @@ const invalidTestExpectedError = new Map() 'Insufficient vector data FLOAT32', 'Invalid Vector: Float32 vector must contain a multiple of 4 bytes' ) - // skipped - .set('Overflow Vector PACKED_BIT', false) - .set('Underflow Vector PACKED_BIT', false) - .set('Overflow Vector INT8', false) - .set('Underflow Vector INT8', false) - .set('INT8 with float inputs', false) - .set('Vector with float values PACKED_BIT', false); - -function testVectorBuilding(test: VectorTest, expectedErrorMessage: string) { - describe('creating an instance of a Binary class using parameters', () => { - it(`bson: ${test.description}`, function () { + // These are not possible given the constraints of the input types allowed: + // our helpers will throw an "unsupported_error" for these + .set('Overflow Vector PACKED_BIT', 'unsupported_error') + .set('Underflow Vector PACKED_BIT', 'unsupported_error') + .set('Overflow Vector INT8', 'unsupported_error') + .set('Underflow Vector INT8', 'unsupported_error') + .set('INT8 with float inputs', 'unsupported_error') + .set('Vector with float values PACKED_BIT', 'unsupported_error'); + +function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: string) { + describe('when creating a BSON Vector given invalid input values', () => { + it(`either BSON.serialize() or Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { let thrownError: Error | undefined; try { const bin = make(test.vector!, test.dtype_hex, test.padding); @@ -111,15 +127,14 @@ function testVectorBuilding(test: VectorTest, expectedErrorMessage: string) { expect( expectedErrorMessage, 'We expect a certain error message but got an unsupported error' - ).to.be.false; - this.skip(); + ).to.equal('unsupported_error'); + } else { + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); } - - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); - it(`ejson: ${test.description}`, function () { + it(`either EJSON.stringify() or Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { let thrownError: Error | undefined; try { const bin = make(test.vector!, test.dtype_hex, test.padding); @@ -132,19 +147,18 @@ function testVectorBuilding(test: VectorTest, expectedErrorMessage: string) { expect( expectedErrorMessage, 'We expect a certain error message but got an unsupported error' - ).to.be.false; - this.skip(); + ).to.equal('unsupported_error'); + } else { + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); } - - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); }); } -function testVectorReserializing(test: VectorTest, expectedErrorMessage: string) { - describe('creating an instance of a Binary class using canonical_bson', () => { - it(`bson deserialize: ${test.description}`, function () { +function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: string) { + describe('when creating a Binary Vector instance from invalid bytes', () => { + it(`BSON.serialize() throw a BSONError`, function () { let thrownError: Error | undefined; const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); @@ -158,7 +172,7 @@ function testVectorReserializing(test: VectorTest, expectedErrorMessage: string) expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); - it(`ejson stringify: ${test.description}`, function () { + it(`EJSON.stringify() throw a BSONError`, function () { let thrownError: Error | undefined; const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); @@ -174,7 +188,7 @@ function testVectorReserializing(test: VectorTest, expectedErrorMessage: string) }); } -describe('BSON Binary Vector spec tests', () => { +describe.only('BSON Binary Vector spec tests', () => { const tests: Record = Object.create(null); for (const file of fs.readdirSync(path.join(__dirname, 'specs/bson-binary-vector'))) { @@ -197,20 +211,22 @@ describe('BSON Binary Vector spec tests', () => { * > MUST assert that the input float array is the same after encoding and decoding. */ for (const test of valid) { - it(`encode ${test.description}`, function () { - const bin = make(test.vector!, test.dtype_hex, test.padding); + describe(test.description, () => { + it(`calling Binary.${dtypeToHelper(test.dtype_hex)}() with input numbers and serializing it does not throw`, function () { + const bin = make(test.vector!, test.dtype_hex, test.padding); - const buffer = BSON.serialize({ [suite.test_key]: bin }); - expect(toHex(buffer)).to.equal(test.canonical_bson!.toLowerCase()); - }); + const buffer = BSON.serialize({ [suite.test_key]: bin }); + expect(toHex(buffer)).to.equal(test.canonical_bson!.toLowerCase()); + }); - it(`decode ${test.description}`, function () { - const canonical_bson = fromHex(test.canonical_bson!.toLowerCase()); - const doc = BSON.deserialize(canonical_bson); + it(`creating a Binary instance from BSON bytes does not throw`, function () { + const canonical_bson = fromHex(test.canonical_bson!.toLowerCase()); + const doc = BSON.deserialize(canonical_bson); - expect(doc[suite.test_key].sub_type).to.equal(0x09); - expect(doc[suite.test_key].buffer[0]).to.equal(+test.dtype_hex); - expect(doc[suite.test_key].buffer[1]).to.equal(test.padding); + expect(doc[suite.test_key].sub_type).to.equal(0x09); + expect(doc[suite.test_key].buffer[0]).to.equal(+test.dtype_hex); + expect(doc[suite.test_key].buffer[1]).to.equal(test.padding); + }); }); } }); @@ -224,22 +240,19 @@ describe('BSON Binary Vector spec tests', () => { for (const test of invalid) { const expectedErrorMessage = invalidTestExpectedError.get(test.description); - if (test.vector != null && test.canonical_bson != null) { - describe('both manual vector building and re-serializing', () => { - testVectorBuilding(test, expectedErrorMessage); - testVectorReserializing(test, expectedErrorMessage); - }); - } else if (test.canonical_bson != null) { - describe('vector re-serializing', () => { - testVectorReserializing(test, expectedErrorMessage); - }); - } else if (test.vector != null) { - describe('manual vector building', () => { - testVectorBuilding(test, expectedErrorMessage); - }); - } else { - throw new Error('not testing anything for: ' + util.inspect(test)); - } + describe(test.description, () => { + if (test.canonical_bson != null) { + testVectorInvalidBSONBytes(test, expectedErrorMessage); + } + + if (test.vector != null) { + testVectorInvalidInputValues(test, expectedErrorMessage); + } + + if (test.vector == null && test.canonical_bson == null) { + throw new Error('not testing anything for: ' + util.inspect(test)); + } + }); } }); }); From 4142fd915e624cbba972d38bcc1a35317b922d98 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 25 Mar 2025 17:54:03 -0400 Subject: [PATCH 04/13] rm only --- test/node/bson_binary_vector.spec.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index df5fd36e5..f719dd10c 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -188,7 +188,7 @@ function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: stri }); } -describe.only('BSON Binary Vector spec tests', () => { +describe('BSON Binary Vector spec tests', () => { const tests: Record = Object.create(null); for (const file of fs.readdirSync(path.join(__dirname, 'specs/bson-binary-vector'))) { From abde5f730612e0fc508bef2a131e0e6efcb23d14 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 25 Mar 2025 18:16:54 -0400 Subject: [PATCH 05/13] fix: add validation for binary vector in Binary toXArray methods --- src/binary.ts | 6 ++++++ test/node/bson_binary_vector.spec.test.ts | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/binary.ts b/src/binary.ts index 85b689486..0a3c0a319 100644 --- a/src/binary.ts +++ b/src/binary.ts @@ -341,6 +341,8 @@ export class Binary extends BSONValue { throw new BSONError('Binary datatype field is not Int8'); } + validateBinaryVector(this); + return new Int8Array( this.buffer.buffer.slice(this.buffer.byteOffset + 2, this.buffer.byteOffset + this.position) ); @@ -361,6 +363,8 @@ export class Binary extends BSONValue { throw new BSONError('Binary datatype field is not Float32'); } + validateBinaryVector(this); + const floatBytes = new Uint8Array( this.buffer.buffer.slice(this.buffer.byteOffset + 2, this.buffer.byteOffset + this.position) ); @@ -387,6 +391,8 @@ export class Binary extends BSONValue { throw new BSONError('Binary datatype field is not packed bit'); } + validateBinaryVector(this); + return new Uint8Array( this.buffer.buffer.slice(this.buffer.byteOffset + 2, this.buffer.byteOffset + this.position) ); diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index f719dd10c..4f71fe150 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -172,6 +172,21 @@ function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: stri expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); + const toHelper = dtypeToHelper(test.dtype_hex).replace('from', 'to'); + it(`Binary.${toHelper}() throw a BSONError`, function () { + let thrownError: Error | undefined; + const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); + + try { + bin.vector[toHelper](); + } catch (error) { + thrownError = error; + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + it(`EJSON.stringify() throw a BSONError`, function () { let thrownError: Error | undefined; const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); From 266f9d09813bc5914c3babfe51621274b88094c4 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 26 Mar 2025 12:51:17 -0400 Subject: [PATCH 06/13] refactor: remove unreachable break statements in dtypeToHelper function --- test/node/bson_binary_vector.spec.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 4f71fe150..7e9250883 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -52,10 +52,8 @@ function dtypeToHelper(dtype_hex: string) { return 'fromPackedBits'; case '0x03' /* int8 */: return 'fromInt8Array'; - break; case '0x27' /* float32 */: return 'fromFloat32Array'; - break; default: throw new Error(`Unknown dtype_hex: ${dtype_hex}`); } From 3b8e801282e154042d98ca6ba248273e71c1f231 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 26 Mar 2025 13:08:01 -0400 Subject: [PATCH 07/13] chore: add toBits validation --- src/binary.ts | 2 ++ test/node/bson_binary_vector.spec.test.ts | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/binary.ts b/src/binary.ts index 0a3c0a319..bcfbd0cbd 100644 --- a/src/binary.ts +++ b/src/binary.ts @@ -415,6 +415,8 @@ export class Binary extends BSONValue { throw new BSONError('Binary datatype field is not packed bit'); } + validateBinaryVector(this); + const byteCount = this.length() - 2; const bitCount = byteCount * 8 - this.buffer[1]; const bits = new Int8Array(bitCount); diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 7e9250883..4ee2bfebc 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -185,6 +185,22 @@ function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: stri expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); + if (toHelper === 'toPackedBits') { + it(`Binary.toBits() throw a BSONError`, function () { + let thrownError: Error | undefined; + const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); + + try { + bin.vector.toBits(); + } catch (error) { + thrownError = error; + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + } + it(`EJSON.stringify() throw a BSONError`, function () { let thrownError: Error | undefined; const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex')); From bb4a895a40c5f25b34118f77ff505fc25a2fcc16 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 1 Apr 2025 16:49:14 -0400 Subject: [PATCH 08/13] chore: add more validation, split out helper and serialize/stringify testing --- src/binary.ts | 12 +++- test/node/bson_binary_vector.spec.test.ts | 67 +++++++++++++++++++---- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/src/binary.ts b/src/binary.ts index bcfbd0cbd..f7cd61b1f 100644 --- a/src/binary.ts +++ b/src/binary.ts @@ -442,7 +442,9 @@ export class Binary extends BSONValue { buffer[1] = 0; const intBytes = new Uint8Array(array.buffer, array.byteOffset, array.byteLength); buffer.set(intBytes, 2); - return new this(buffer, this.SUBTYPE_VECTOR); + const bin = new this(buffer, this.SUBTYPE_VECTOR); + validateBinaryVector(bin); + return bin; } /** Constructs a Binary representing an Float32 Vector. */ @@ -456,7 +458,9 @@ export class Binary extends BSONValue { if (NumberUtils.isBigEndian) ByteUtils.swap32(new Uint8Array(binaryBytes.buffer, 2)); - return new this(binaryBytes, this.SUBTYPE_VECTOR); + const bin = new this(binaryBytes, this.SUBTYPE_VECTOR); + validateBinaryVector(bin); + return bin; } /** @@ -469,7 +473,9 @@ export class Binary extends BSONValue { buffer[0] = Binary.VECTOR_TYPE.PackedBit; buffer[1] = padding; buffer.set(array, 2); - return new this(buffer, this.SUBTYPE_VECTOR); + const bin = new this(buffer, this.SUBTYPE_VECTOR); + validateBinaryVector(bin); + return bin; } /** diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 4ee2bfebc..6a3c20888 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -110,33 +110,71 @@ const invalidTestExpectedError = new Map() .set('INT8 with float inputs', 'unsupported_error') .set('Vector with float values PACKED_BIT', 'unsupported_error'); +const invalidTestsWhereHelpersDoNotThrow = new Set() + .add('FLOAT32 with padding') + .add('INT8 with padding'); + function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: string) { describe('when creating a BSON Vector given invalid input values', () => { - it(`either BSON.serialize() or Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { + it(`BSON.serialize() throws a BSONError`, function () { let thrownError: Error | undefined; + + let bin; + try { + bin = make(test.vector!, test.dtype_hex, test.padding); + } catch (error) { + thrownError = error; + } + + if (thrownError?.message.startsWith('unsupported_error')) { + expect( + expectedErrorMessage, + 'We expect a certain error message but got an unsupported error' + ).to.equal('unsupported_error'); + return; + } + try { - const bin = make(test.vector!, test.dtype_hex, test.padding); BSON.serialize({ bin }); } catch (error) { thrownError = error; } + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + }); + + it(`Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { + let thrownError: Error | undefined; + try { + make(test.vector!, test.dtype_hex, test.padding); + } catch (error) { + thrownError = error; + } + + if (invalidTestsWhereHelpersDoNotThrow.has(test.description)) { + expect(thrownError).to.not.exist; + return; + } + if (thrownError?.message.startsWith('unsupported_error')) { expect( expectedErrorMessage, 'We expect a certain error message but got an unsupported error' ).to.equal('unsupported_error'); - } else { - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + return; } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); - it(`either EJSON.stringify() or Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { + it(`EJSON.stringify() throws a BSONError`, function () { let thrownError: Error | undefined; + + let bin; try { - const bin = make(test.vector!, test.dtype_hex, test.padding); - BSON.EJSON.stringify({ bin }); + bin = make(test.vector!, test.dtype_hex, test.padding); } catch (error) { thrownError = error; } @@ -146,10 +184,17 @@ function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: st expectedErrorMessage, 'We expect a certain error message but got an unsupported error' ).to.equal('unsupported_error'); - } else { - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); + return; } + + try { + BSON.EJSON.stringify({ bin }); + } catch (error) { + thrownError = error; + } + + expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); + expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); }); }); } From 3cd365500e9e53cae1a340992fde0b8322296a6c Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 1 Apr 2025 17:39:59 -0400 Subject: [PATCH 09/13] chore: separate serialize and fromX helpers --- test/node/bson_binary_vector.spec.test.ts | 135 ++++++++++------------ 1 file changed, 58 insertions(+), 77 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 6a3c20888..76786e4a8 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -1,6 +1,7 @@ import * as util from 'util'; import * as fs from 'fs'; import * as path from 'path'; +import * as assert from 'node:assert/strict'; import { BSON, BSONError, Binary, EJSON } from '../register-bson'; import { expect } from 'chai'; @@ -114,88 +115,68 @@ const invalidTestsWhereHelpersDoNotThrow = new Set() .add('FLOAT32 with padding') .add('INT8 with padding'); +function catchError( + fn: () => T +): { status: 'returned'; result: T } | { status: 'thrown'; result: Error } { + try { + return { status: 'returned', result: fn() }; + } catch (error) { + return { status: 'thrown', result: error }; + } +} + function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: string) { describe('when creating a BSON Vector given invalid input values', () => { - it(`BSON.serialize() throws a BSONError`, function () { - let thrownError: Error | undefined; - - let bin; - try { - bin = make(test.vector!, test.dtype_hex, test.padding); - } catch (error) { - thrownError = error; - } - - if (thrownError?.message.startsWith('unsupported_error')) { - expect( - expectedErrorMessage, - 'We expect a certain error message but got an unsupported error' - ).to.equal('unsupported_error'); - return; - } - - try { - BSON.serialize({ bin }); - } catch (error) { - thrownError = error; - } - - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); - }); - - it(`Binary.${dtypeToHelper(test.dtype_hex)}() throws a BSONError`, function () { - let thrownError: Error | undefined; - try { - make(test.vector!, test.dtype_hex, test.padding); - } catch (error) { - thrownError = error; - } - - if (invalidTestsWhereHelpersDoNotThrow.has(test.description)) { - expect(thrownError).to.not.exist; - return; - } - - if (thrownError?.message.startsWith('unsupported_error')) { - expect( - expectedErrorMessage, - 'We expect a certain error message but got an unsupported error' - ).to.equal('unsupported_error'); - return; - } - - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); - }); - - it(`EJSON.stringify() throws a BSONError`, function () { - let thrownError: Error | undefined; - - let bin; - try { - bin = make(test.vector!, test.dtype_hex, test.padding); - } catch (error) { - thrownError = error; - } + const binaryCreation = catchError(make.bind(null, test.vector!, test.dtype_hex, test.padding)); + const bsonBytesCreation = + binaryCreation.status !== 'thrown' + ? catchError(BSON.serialize.bind(null, { bin: binaryCreation.result })) + : undefined; + const ejsonStringCreation = + binaryCreation.status !== 'thrown' + ? catchError(BSON.EJSON.stringify.bind(null, { bin: binaryCreation.result })) + : undefined; + + const binaryHelperValidations = [ + 'Padding specified with no vector data PACKED_BIT', + 'Exceeding maximum padding PACKED_BIT', + 'Negative padding PACKED_BIT', + ...Array.from(invalidTestExpectedError.entries()) + .filter(([, v]) => v === 'unsupported_error') + .map(([k]) => k) + ]; + + const errorType = expectedErrorMessage === 'unsupported_error' ? Error : BSONError; + const errorName = expectedErrorMessage === 'unsupported_error' ? 'Error' : 'BSONError'; + + const check = outcome => { + expect(outcome).to.exist; + expect(outcome.status).to.equal('thrown'); + expect(outcome.result).to.be.instanceOf(errorType); + expect(outcome.result) + .to.have.property('message') + .that.matches(new RegExp(expectedErrorMessage)); + }; + + if (binaryHelperValidations.includes(test.description)) { + it(`Binary.${dtypeToHelper(test.dtype_hex)}() throws a ${errorName}`, function () { + check(binaryCreation); + }); + } else { + expect(errorName).to.equal('BSONError'); // unsupported_error are only when making vectors - if (thrownError?.message.startsWith('unsupported_error')) { - expect( - expectedErrorMessage, - 'We expect a certain error message but got an unsupported error' - ).to.equal('unsupported_error'); - return; - } + it(`Binary.${dtypeToHelper(test.dtype_hex)}() not throw`, function () { + expect(binaryCreation).to.have.property('status', 'returned'); + }); - try { - BSON.EJSON.stringify({ bin }); - } catch (error) { - thrownError = error; - } + it(`BSON.serialize() throws a BSONError`, function () { + check(bsonBytesCreation); + }); - expect(thrownError, thrownError?.stack).to.be.instanceOf(BSONError); - expect(thrownError?.message).to.match(new RegExp(expectedErrorMessage)); - }); + it(`EJSON.stringify() throws a BSONError`, function () { + check(ejsonStringCreation); + }); + } }); } From b545971fe9b373ac8aaa61768e89ba8fe04736e3 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 1 Apr 2025 17:47:27 -0400 Subject: [PATCH 10/13] chore: does not throw --- test/node/bson_binary_vector.spec.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 76786e4a8..a1e05728d 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -165,7 +165,7 @@ function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: st } else { expect(errorName).to.equal('BSONError'); // unsupported_error are only when making vectors - it(`Binary.${dtypeToHelper(test.dtype_hex)}() not throw`, function () { + it(`Binary.${dtypeToHelper(test.dtype_hex)}() does not throw`, function () { expect(binaryCreation).to.have.property('status', 'returned'); }); From 5ddd37ed3d427a1efa0761f7e4349bf4be35b219 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Tue, 1 Apr 2025 17:48:33 -0400 Subject: [PATCH 11/13] lint --- test/node/bson_binary_vector.spec.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index a1e05728d..3e2876014 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -1,7 +1,6 @@ import * as util from 'util'; import * as fs from 'fs'; import * as path from 'path'; -import * as assert from 'node:assert/strict'; import { BSON, BSONError, Binary, EJSON } from '../register-bson'; import { expect } from 'chai'; @@ -111,10 +110,6 @@ const invalidTestExpectedError = new Map() .set('INT8 with float inputs', 'unsupported_error') .set('Vector with float values PACKED_BIT', 'unsupported_error'); -const invalidTestsWhereHelpersDoNotThrow = new Set() - .add('FLOAT32 with padding') - .add('INT8 with padding'); - function catchError( fn: () => T ): { status: 'returned'; result: T } | { status: 'thrown'; result: Error } { From 16b3b313b839ad61c1c544309ac770bd40f15fb5 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Wed, 2 Apr 2025 16:54:10 -0400 Subject: [PATCH 12/13] chore: nesting, regex on error --- test/node/bson_binary_vector.spec.test.ts | 74 +++++++++++------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 3e2876014..82e3ce047 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -121,45 +121,45 @@ function catchError( } function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: string) { - describe('when creating a BSON Vector given invalid input values', () => { - const binaryCreation = catchError(make.bind(null, test.vector!, test.dtype_hex, test.padding)); - const bsonBytesCreation = - binaryCreation.status !== 'thrown' - ? catchError(BSON.serialize.bind(null, { bin: binaryCreation.result })) - : undefined; - const ejsonStringCreation = - binaryCreation.status !== 'thrown' - ? catchError(BSON.EJSON.stringify.bind(null, { bin: binaryCreation.result })) - : undefined; - - const binaryHelperValidations = [ - 'Padding specified with no vector data PACKED_BIT', - 'Exceeding maximum padding PACKED_BIT', - 'Negative padding PACKED_BIT', - ...Array.from(invalidTestExpectedError.entries()) - .filter(([, v]) => v === 'unsupported_error') - .map(([k]) => k) - ]; - - const errorType = expectedErrorMessage === 'unsupported_error' ? Error : BSONError; - const errorName = expectedErrorMessage === 'unsupported_error' ? 'Error' : 'BSONError'; - - const check = outcome => { - expect(outcome).to.exist; - expect(outcome.status).to.equal('thrown'); - expect(outcome.result).to.be.instanceOf(errorType); - expect(outcome.result) - .to.have.property('message') - .that.matches(new RegExp(expectedErrorMessage)); - }; - - if (binaryHelperValidations.includes(test.description)) { + const binaryCreation = catchError(make.bind(null, test.vector!, test.dtype_hex, test.padding)); + const bsonBytesCreation = + binaryCreation.status !== 'thrown' + ? catchError(BSON.serialize.bind(null, { bin: binaryCreation.result })) + : undefined; + const ejsonStringCreation = + binaryCreation.status !== 'thrown' + ? catchError(BSON.EJSON.stringify.bind(null, { bin: binaryCreation.result })) + : undefined; + + const binaryHelperValidations = [ + 'Padding specified with no vector data PACKED_BIT', + 'Exceeding maximum padding PACKED_BIT', + 'Negative padding PACKED_BIT', + ...Array.from(invalidTestExpectedError.entries()) + .filter(([, v]) => v === 'unsupported_error') + .map(([k]) => k) + ]; + + const errorType = expectedErrorMessage === 'unsupported_error' ? Error : BSONError; + const errorName = expectedErrorMessage === 'unsupported_error' ? 'Error' : 'BSONError'; + + const check = outcome => { + expect(outcome).to.exist; + expect(outcome.status).to.equal('thrown'); + expect(outcome.result).to.be.instanceOf(errorType); + expect(outcome.result).to.match(new RegExp(expectedErrorMessage)); + }; + + if (binaryHelperValidations.includes(test.description)) { + describe('when creating a BSON Vector given invalid input values', () => { it(`Binary.${dtypeToHelper(test.dtype_hex)}() throws a ${errorName}`, function () { check(binaryCreation); }); - } else { - expect(errorName).to.equal('BSONError'); // unsupported_error are only when making vectors + }); + } else { + expect(errorName).to.equal('BSONError'); // unsupported_error are only when making vectors + describe('when encoding a BSON Vector given invalid input values', () => { it(`Binary.${dtypeToHelper(test.dtype_hex)}() does not throw`, function () { expect(binaryCreation).to.have.property('status', 'returned'); }); @@ -171,8 +171,8 @@ function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: st it(`EJSON.stringify() throws a BSONError`, function () { check(ejsonStringCreation); }); - } - }); + }); + } } function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: string) { From 7ec9f1922ddc7ec2074196ac4b5e6542263a71f5 Mon Sep 17 00:00:00 2001 From: Neal Beeken Date: Thu, 3 Apr 2025 13:34:16 -0400 Subject: [PATCH 13/13] chore: title --- test/node/bson_binary_vector.spec.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/node/bson_binary_vector.spec.test.ts b/test/node/bson_binary_vector.spec.test.ts index 82e3ce047..50475d94d 100644 --- a/test/node/bson_binary_vector.spec.test.ts +++ b/test/node/bson_binary_vector.spec.test.ts @@ -176,7 +176,7 @@ function testVectorInvalidInputValues(test: VectorTest, expectedErrorMessage: st } function testVectorInvalidBSONBytes(test: VectorTest, expectedErrorMessage: string) { - describe('when creating a Binary Vector instance from invalid bytes', () => { + describe('when encoding a Binary Vector made from invalid bytes', () => { it(`BSON.serialize() throw a BSONError`, function () { let thrownError: Error | undefined; const bin = BSON.deserialize(Buffer.from(test.canonical_bson!, 'hex'));