From 1176dd5565b76200b70dc85a78925419263101b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20Cvetkovi=C4=87?= Date: Wed, 26 Aug 2026 00:07:59 +0100 Subject: [PATCH] fix: Support decoding N-dimensional arrays N-dimensional arrays are a draft 3 construct that this library rejected as invalid, so valid draft 3 data could not be read. `[$type#[Nx Ny ...]` now decodes to nested lists, keeping the innermost axis as the typed list. The nesting is built from views rather than copies, so an N-dimensional array is still one contiguous allocation once decoded. Dimension arrays are accepted in both optimized and non-optimized form. The column-major form, `[$type#[[Nx Ny ...]]` as MATLAB and FORTRAN write it, is reordered into row-major order so that it reads the same way as an array of the same shape written row-major. Elements are permuted whole, whatever their width, so this works for every strong type. Encoding N-dimensional arrays is not supported, so a decoded array is written back as nested arrays; the values are unchanged. Also throws a FormatException rather than a RangeError when a string or buffer runs past the end of the input, which a payload shorter than its dimensions makes easy to hit. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 14 +++ README.md | 6 +- lib/src/decoder/sink.dart | 206 ++++++++++++++++++++++++++++++++---- lib/src/marker.dart | 3 + pubspec.yaml | 2 +- test/bjdata_test.dart | 213 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 422 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eb0f75..6de334f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## 0.9.3 +- Support decoding N-dimensional arrays, which are a draft 3 construct that was previously + rejected as invalid + - `[$type#[Nx Ny ...]` decodes to nested lists, with the innermost axis kept as the typed + list. The nested lists are views onto one buffer rather than copies, so the payload is + still a single contiguous allocation + - The column-major form, `[$type#[[Nx Ny ...]]` as written by MATLAB and FORTRAN, is + reordered into row-major order so that it reads the same way + - Dimension arrays are accepted in both optimized and non-optimized form + - Encoding N-dimensional arrays is not supported, so a decoded array is written back as + nested arrays; the values are unchanged +- Throw a `FormatException` instead of a `RangeError` when a string or buffer runs past the + end of the input + ## 0.9.2 - Update description to improve package score diff --git a/README.md b/README.md index 7022962..74e08d1 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,10 @@ echo -n "[1, 2, 3]" | bjdata print ## Types ### Decoding BJData to Dart -- Multi dimensional arrays are not yet supported. +- N-dimensional arrays (`#[Nx Ny ...]`) decode to nested lists, with the innermost axis + kept as the typed list. Both row-major and column-major (`#[[Nx Ny ...]]`) orderings are + read; a column-major payload is reordered so that it reads the same way. Encoding + N-dimensional arrays is not yet supported, so they are written back as nested arrays. | BJData Type | Marker | Dart | |------------------|--------|--------------------------------| @@ -100,6 +103,7 @@ echo -n "[1, 2, 3]" | bjdata print | `array[float32]` | `[$d` | `Float32List` | | `array[float64]` | `[$D` | `Float64List` | | `object` | `{}` | `Map` | +| `array[T]` N-D | `#[` | Nested `List` of `T` | \* Warning: `int` in Dart is a signed 64-bit integer. `uint64`/`M` values are decoded as `int64` diff --git a/lib/src/decoder/sink.dart b/lib/src/decoder/sink.dart index da0b9e9..3915639 100644 --- a/lib/src/decoder/sink.dart +++ b/lib/src/decoder/sink.dart @@ -64,6 +64,9 @@ class BjdataReader { } Uint8List _readUint8ListView(int length) { + if (_offset + length > _bytes.lengthInBytes) { + throw FormatException("Unexpected end of input", _bytes, _bytes.lengthInBytes); + } final view = Uint8List.sublistView(_bytes, _offset, _offset + length); _offset += length; return view; @@ -187,20 +190,21 @@ class BjdataReader { }; } + int _readIntForMarker(BjdataMarker marker, String what, int offsetBefore) => switch (marker) { + BjdataMarker.uint8 => _readUint8(), + BjdataMarker.int8 => _readInt8(), + BjdataMarker.uint16 => _readUint16(), + BjdataMarker.int16 => _readInt16(), + BjdataMarker.uint32 => _readUint32(), + BjdataMarker.int32 => _readInt32(), + BjdataMarker.uint64 => _readUint64(), + BjdataMarker.int64 => _readInt64(), + _ => throw FormatException('Unexpected non-$what type marker: $marker', _bytes, offsetBefore), + }; + int _readLength() { final offsetBefore = _offset; - final marker = _readMarker(); - final length = switch (marker) { - BjdataMarker.uint8 => _readUint8(), - BjdataMarker.int8 => _readInt8(), - BjdataMarker.uint16 => _readUint16(), - BjdataMarker.int16 => _readInt16(), - BjdataMarker.uint32 => _readUint32(), - BjdataMarker.int32 => _readInt32(), - BjdataMarker.uint64 => _readUint64(), - BjdataMarker.int64 => _readInt64(), - _ => throw FormatException('Unexpected non-length type marker: $marker', _bytes, offsetBefore), - }; + final length = _readIntForMarker(_readMarker(), 'length', offsetBefore); if (length < 0) throw FormatException('Negative length: $length', _bytes, offsetBefore); return length; } @@ -279,10 +283,68 @@ class BjdataReader { BigInt _readHuge() => BigInt.parse(_readString()); String _readString() => utf8.decode(_readUint8ListView(_readLength())); - (BjdataMarker?, int?) _readStrongTypeAndCount() { + /// The element count of a container, and the dimensions it was given as. + /// + /// A count is either a single integer, a dimension array (`#[Nx Ny ...]`) for + /// an N-dimensional array serialized in row-major order, or a dimension array + /// wrapped in a single element array (`#[[Nx Ny ...]]`) for one serialized in + /// column-major order, as MATLAB and FORTRAN write it. + ({int count, List? dimensions, bool columnMajor}) _readCount() { + final offsetBefore = _offset; + if (!_peekMarkerConsumeIf(BjdataMarker.arrayOpen)) { + return (count: _readLength(), dimensions: null, columnMajor: false); + } + + // A second '[' wraps the dimension array, which marks column-major order. + final columnMajor = _peekMarker() == BjdataMarker.arrayOpen; + if (columnMajor) _offset++; + + final dimensions = _readDimensions(offsetBefore); + + if (columnMajor && !_peekMarkerConsumeIf(BjdataMarker.arrayClose)) { + throw FormatException('Expected end of the wrapped dimension array', _bytes, _offset); + } + return ( + count: dimensions.fold(1, (a, b) => a * b), + dimensions: dimensions, + columnMajor: columnMajor, + ); + } + + /// Reads a dimension array, with its leading `[` already consumed. + List _readDimensions(int offsetBefore) { BjdataMarker? strongType; int? count; + if (_peekMarkerConsumeIf(BjdataMarker.strongType)) { + strongType = _readMarker(); + if (!strongType.isIntegerType) { + throw FormatException('Dimensions must be of an integer type: $strongType', _bytes, _offset - 1); + } + if (!_peekMarkerConsumeIf(BjdataMarker.count)) { + throw FormatException('Expected count marker to follow strong type', _bytes, _offset); + } + count = _readLength(); + } else if (_peekMarkerConsumeIf(BjdataMarker.count)) { + count = _readLength(); + } + + final dimensions = []; + for (var i = 0; count != null ? i < count : true; i++) { + if (count == null && _peekMarkerConsumeIf(BjdataMarker.arrayClose)) break; + final before = _offset; + final dimension = _readIntForMarker(strongType ?? _readMarker(), 'dimension', before); + if (dimension < 0) throw FormatException('Negative dimension: $dimension', _bytes, before); + dimensions.add(dimension); + } + if (dimensions.isEmpty) throw FormatException('Empty dimension array', _bytes, offsetBefore); + return dimensions; + } + + ({BjdataMarker? strongType, int? count, List? dimensions, bool columnMajor}) _readStrongTypeAndCount() { + BjdataMarker? strongType; + const int? count = null; + if (_peekMarkerConsumeIf(BjdataMarker.strongType)) { // Read strong type strongType = _readMarker(); @@ -302,21 +364,23 @@ class BjdataReader { } // Read count - count = _readLength(); + final (:count, :dimensions, :columnMajor) = _readCount(); + return (strongType: strongType, count: count, dimensions: dimensions, columnMajor: columnMajor); } else if (_peekMarkerConsumeIf(BjdataMarker.count)) { // Read count - count = _readLength(); + final (:count, :dimensions, :columnMajor) = _readCount(); + return (strongType: strongType, count: count, dimensions: dimensions, columnMajor: columnMajor); } - return (strongType, count); + return (strongType: strongType, count: count, dimensions: null, columnMajor: false); } Object _readArray() { final offsetBefore = _offset - 1; - final (strongType, count) = _readStrongTypeAndCount(); + final (:strongType, :count, :dimensions, :columnMajor) = _readStrongTypeAndCount(); if (strongType != null && strongType.isValidStrongType && strongType != BjdataMarker.char) { - return switch (strongType) { + final flat = switch (strongType) { BjdataMarker.byte => _readByteDataCopy(count!), BjdataMarker.uint8 => _readUint8ListCopy(count!), BjdataMarker.int8 => _readInt8ListCopy(count!), @@ -331,6 +395,8 @@ class BjdataReader { BjdataMarker.float64 => _readFloat64ListCopy(count!), _ => throw FormatException('Invalid strong type: $strongType', _bytes, offsetBefore), }; + if (dimensions == null) return flat; + return _reshape(columnMajor ? _toRowMajor(flat, dimensions) : flat, dimensions, 0); } final list = []; @@ -340,11 +406,111 @@ class BjdataReader { final value = _readValueForMarker(strongType ?? _readMarker()); list.add(_reviver == null ? value : _reviver!(i, value)); } - return list; + if (dimensions == null) return list; + return _reshape(columnMajor ? _toRowMajor(list, dimensions) : list, dimensions, 0); + } + + /// Nests [flat] according to [dimensions], one list per axis. + /// + /// Slices are views onto [flat] wherever it is a typed list, so an + /// N-dimensional array still holds a single contiguous buffer once decoded. + Object _reshape(Object flat, List dimensions, int axis) { + if (axis == dimensions.length - 1) return flat; + + var stride = 1; + for (var i = axis + 1; i < dimensions.length; i++) { + stride *= dimensions[i]; + } + return [ + for (var i = 0; i < dimensions[axis]; i++) + _reshape(_slice(flat, i * stride, (i + 1) * stride), dimensions, axis + 1), + ]; + } + + /// A view of [list] from [start] to [end], keeping its type. + Object _slice(Object list, int start, int end) => switch (list) { + ByteData l => ByteData.sublistView(l, start, end), + Uint8List l => Uint8List.sublistView(l, start, end), + Int8List l => Int8List.sublistView(l, start, end), + Uint16List l => Uint16List.sublistView(l, start, end), + Int16List l => Int16List.sublistView(l, start, end), + Uint32List l => Uint32List.sublistView(l, start, end), + Int32List l => Int32List.sublistView(l, start, end), + Float32List l => Float32List.sublistView(l, start, end), + Float64List l => Float64List.sublistView(l, start, end), + // Not reachable on the web, where 64 bit lists decode as List. + Uint64List l => Uint64List.sublistView(l, start, end), + Int64List l => Int64List.sublistView(l, start, end), + List l => l.sublist(start, end), + _ => throw FormatException('Cannot reshape ${list.runtimeType}', _bytes, _offset), + }; + + /// Reorders column-major [flat] into row-major order. + /// + /// The elements are permuted rather than reinterpreted, so the result reads + /// the same way a row-major array of the same shape would. + Object _toRowMajor(Object flat, List dimensions) { + final rank = dimensions.length; + if (rank < 2) return flat; + + // In column-major order the first axis varies fastest. + final strides = List.filled(rank, 1); + for (var axis = 1; axis < rank; axis++) { + strides[axis] = strides[axis - 1] * dimensions[axis - 1]; + } + + final total = dimensions.fold(1, (a, b) => a * b); + final source = List.filled(total, 0); + final indices = List.filled(rank, 0); + for (var i = 0; i < total; i++) { + var offset = 0; + for (var axis = 0; axis < rank; axis++) { + offset += indices[axis] * strides[axis]; + } + source[i] = offset; + // Step through the output in row-major order, last axis fastest. + for (var axis = rank - 1; axis >= 0; axis--) { + if (++indices[axis] < dimensions[axis]) break; + indices[axis] = 0; + } + } + + if (flat is List) return [for (final index in source) flat[index]]; + + // Permute whole elements, whatever their width, and reinterpret the result + // as the same kind of list. + final data = flat as TypedData; + final size = data.elementSizeInBytes; + final bytes = Uint8List.sublistView(data); + final reordered = Uint8List(bytes.length); + for (var i = 0; i < total; i++) { + reordered.setRange(i * size, (i + 1) * size, bytes, source[i] * size); + } + return _sameTypeAs(data, reordered); } + /// Reinterprets [bytes] as the same kind of list as [like]. + Object _sameTypeAs(TypedData like, Uint8List bytes) => switch (like) { + ByteData() => ByteData.sublistView(bytes), + Uint8List() => bytes, + Int8List() => Int8List.sublistView(bytes), + Uint16List() => Uint16List.sublistView(bytes), + Int16List() => Int16List.sublistView(bytes), + Uint32List() => Uint32List.sublistView(bytes), + Int32List() => Int32List.sublistView(bytes), + Float32List() => Float32List.sublistView(bytes), + Float64List() => Float64List.sublistView(bytes), + Uint64List() => Uint64List.sublistView(bytes), + Int64List() => Int64List.sublistView(bytes), + _ => throw FormatException('Cannot reorder ${like.runtimeType}', _bytes, _offset), + }; + Map? _readMap() { - final (strongType, count) = _readStrongTypeAndCount(); + final offsetBefore = _offset; + final (:strongType, :count, :dimensions, columnMajor: _) = _readStrongTypeAndCount(); + if (dimensions != null) { + throw FormatException('An object cannot be counted by a dimension array', _bytes, offsetBefore); + } final map = {}; for (var i = 0; count != null ? i < count : true; i++) { diff --git a/lib/src/marker.dart b/lib/src/marker.dart index ee31092..7bea3c1 100644 --- a/lib/src/marker.dart +++ b/lib/src/marker.dart @@ -69,4 +69,7 @@ enum BjdataMarker { /// Whether this marker is a valid strong type (for typed containers). bool get isValidStrongType => index >= BjdataMarker.uint8.index && index <= BjdataMarker.byte.index; + + /// Whether this marker is an integer type. + bool get isIntegerType => index >= BjdataMarker.uint8.index && index <= BjdataMarker.int64.index; } diff --git a/pubspec.yaml b/pubspec.yaml index d3f5a60..12e5ae7 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: bjdata description: A library for encoding and decoding BJData, a binary JSON-compatible data format. -version: 0.9.2 +version: 0.9.3 homepage: https://github.com/nebkat/dart-bjdata repository: https://github.com/nebkat/dart-bjdata issue_tracker: https://github.com/nebkat/dart-bjdata/issues diff --git a/test/bjdata_test.dart b/test/bjdata_test.dart index 3a79fb7..a4f2f72 100644 --- a/test/bjdata_test.dart +++ b/test/bjdata_test.dart @@ -225,6 +225,219 @@ void main() { } }); + group('n-dimensional array', () { + // The 2x3x4 uint8 array from the specification's worked example. + const expected = [ + [ + [1, 9, 6, 0], + [2, 9, 3, 1], + [8, 0, 9, 6], + ], + [ + [6, 4, 2, 7], + [8, 5, 1, 2], + [3, 3, 2, 6], + ], + ]; + const rowMajorData = [1, 9, 6, 0, 2, 9, 3, 1, 8, 0, 9, 6, 6, 4, 2, 7, 8, 5, 1, 2, 3, 3, 2, 6]; + const columnMajorData = [1, 6, 2, 8, 8, 3, 9, 4, 9, 5, 0, 3, 6, 2, 3, 1, 9, 2, 0, 7, 1, 2, 6, 6]; + + /// `[$U#` followed by [count], the dimension array and the payload. + List uint8Array(List count, List data) => [ + M.arrayOpen.i, M.strongType.i, M.uint8.i, M.count.i, // + ...count, + ...data, + ]; + + /// An optimized dimension array, `[$U#` with no closing bracket. + List optimizedDims(List dimensions) => [ + M.arrayOpen.i, M.strongType.i, M.uint8.i, M.count.i, M.uint8.i, dimensions.length, // + ...dimensions, + ]; + + /// A plain dimension array, each entry with its own marker. + List plainDims(List dimensions) => [ + M.arrayOpen.i, // + for (final dimension in dimensions) ...[M.uint8.i, dimension], + M.arrayClose.i, + ]; + + test('decodes the specification example in row-major order', () { + expect(bjdataDecode(uint8Array(optimizedDims([2, 3, 4]), rowMajorData)), expected); + }); + + test('decodes the specification example in column-major order', () { + // The dimension array wrapped in a single element array. + final count = [ + M.arrayOpen.i, + ...optimizedDims([2, 3, 4]), + M.arrayClose.i + ]; + expect(bjdataDecode(uint8Array(count, columnMajorData)), expected); + }); + + test('accepts optimized and non-optimized dimension arrays alike', () { + expect(bjdataDecode(uint8Array(plainDims([2, 3, 4]), rowMajorData)), expected); + expect( + bjdataDecode(uint8Array(plainDims([2, 3, 4]), rowMajorData)), + bjdataDecode(uint8Array(optimizedDims([2, 3, 4]), rowMajorData)), + ); + }); + + test('accepts a non-optimized wrapper around the dimension array', () { + final count = [ + M.arrayOpen.i, + ...plainDims([2, 3, 4]), + M.arrayClose.i + ]; + expect(bjdataDecode(uint8Array(count, columnMajorData)), expected); + }); + + test('keeps the innermost axis as a typed list', () { + final decoded = bjdataDecode(uint8Array(optimizedDims([2, 3, 4]), rowMajorData)); + expect(decoded, isA()); + expect(decoded[0], isA()); + expect(decoded[0][0], isA()); + expect(decoded[0][0], [1, 9, 6, 0]); + }); + + test('slices the payload rather than copying it', () { + final decoded = bjdataDecode(uint8Array(optimizedDims([2, 3, 4]), rowMajorData)); + final first = decoded[0][0] as Uint8List; + final last = decoded[1][2] as Uint8List; + // One buffer holding all 24 elements, viewed at different offsets. + expect(first.offsetInBytes, 0); + expect(last.offsetInBytes, 20); + expect(first.buffer.lengthInBytes, 24); + }); + + test('a single dimension behaves like a plain count', () { + final decoded = bjdataDecode(uint8Array(optimizedDims([4]), [1, 2, 3, 4])); + expect(decoded, isA()); + expect(decoded, [1, 2, 3, 4]); + }); + + test('handles every strong type', () { + final types = { + M.byte: 1, + M.int8: 1, + M.uint16: 2, + M.int16: 2, + M.uint32: 4, + M.int32: 4, + M.float32: 4, + M.float64: 8, + M.float16: 2, + if (1 is! double) M.uint64: 8, + if (1 is! double) M.int64: 8, + }; + types.forEach((marker, size) { + final encoded = [ + M.arrayOpen.i, M.strongType.i, marker.i, M.count.i, // + ...optimizedDims([2, 2]), + ...List.filled(4 * size, 0), + ]; + final decoded = bjdataDecode(encoded); + // Two rows, each still a typed view onto the one payload. + expect((decoded as List).length, 2, reason: '$marker'); + expect(decoded[0], isA(), reason: '$marker'); + }); + }); + + test('reorders column-major payloads of wider types', () { + // A 2x2 int16 array, stored column-major as 1 3 2 4. + final data = []; + for (final value in [1, 3, 2, 4]) { + data.addAll([value, 0]); + } + final encoded = [ + M.arrayOpen.i, M.strongType.i, M.int16.i, M.count.i, // + M.arrayOpen.i, ...optimizedDims([2, 2]), M.arrayClose.i, + ...data, + ]; + expect(bjdataDecode(encoded), [ + [1, 2], + [3, 4], + ]); + }); + + test('reshapes containers that are not strongly typed', () { + // char is excluded from the packed path, so it exercises the element loop. + final chars = [ + M.arrayOpen.i, M.strongType.i, M.char.i, M.count.i, // + ...plainDims([2, 2]), + 0x61, 0x62, 0x63, 0x64, + ]; + expect(bjdataDecode(chars), [ + ['a', 'b'], + ['c', 'd'], + ]); + + final mixed = [ + M.arrayOpen.i, M.count.i, ...plainDims([2, 2]), // + M.uint8.i, 1, M.true_.i, M.null_.i, M.string.i, M.uint8.i, 1, 0x78, + ]; + expect(bjdataDecode(mixed), [ + [1, true], + [null, 'x'], + ]); + }); + + test('rejects malformed dimensions', () { + final entries = >{ + 'an empty dimension array': [ + M.arrayOpen.i, + M.strongType.i, + M.uint8.i, + M.count.i, + M.arrayOpen.i, + M.arrayClose.i, + ], + 'a negative dimension': [ + M.arrayOpen.i, + M.strongType.i, + M.uint8.i, + M.count.i, + M.arrayOpen.i, + M.int8.i, + 0xFF, + M.arrayClose.i, + ], + 'a non-integer dimension type': [ + M.arrayOpen.i, M.strongType.i, M.uint8.i, M.count.i, // + M.arrayOpen.i, M.strongType.i, M.float64.i, M.count.i, M.uint8.i, 1, ...List.filled(8, 0), + ], + 'an unclosed column-major wrapper': [ + M.arrayOpen.i, M.strongType.i, M.uint8.i, M.count.i, // + M.arrayOpen.i, ...plainDims([2]), M.uint8.i, 1, 2, + ], + 'an object counted by dimensions': [ + M.objectOpen.i, M.strongType.i, M.uint8.i, M.count.i, ...plainDims([1]), // + M.uint8.i, 1, 0x61, 1, + ], + 'a payload shorter than the dimensions': [ + M.arrayOpen.i, + M.strongType.i, + M.uint8.i, + M.count.i, + ...plainDims([2, 3]), + 1, + 2, + ], + }; + entries.forEach((reason, entry) { + expect(() => bjdataDecode(entry), throwsA(isA()), reason: reason); + }); + }); + + test('re-encodes as nested arrays', () { + // Encoding N-dimensional containers is not supported, so a decoded array + // is written back as an array of arrays. The values are unchanged. + final decoded = bjdataDecode(uint8Array(optimizedDims([2, 3, 4]), rowMajorData)); + expect(bjdataDecode(bjdataEncode(decoded)), expected); + }); + }); + group('invalid', () { test('type', () { List emptyStrongTypeOf(M m) => [