From e4cc525f11af07033c2eaf06acc11d86b383233f Mon Sep 17 00:00:00 2001 From: olabusayoT <50379531+olabusayoT@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:39:44 -0400 Subject: [PATCH] Parse Error on Out of Range Binary Integers - currently we don't check the minimum lengths for signed/unsigned binary ints and this causes issues when you try to set a binary int to 0 bits (RSDE), so per the DFDL workgroup we add in these checks. - add width check to unparse code and associated test - fix bug where we weren't returning after UnparseError - add SDEs for checks for non-expression fixed lengths - currently we pass in isSigned boolean from ElementBaseGrammarMixin, but we can use the new PrimType.PrimNumeric.isSigned member instead for simplicity. We also added the minWidth member to simplify checking the minimum width of PrimNumeric types - add tunable allowSignedIntegerLength1Bit and update tests to use - add tests for parse and unparse - update failing tests - make it a PE if nBits == 0 for PackedBinaryIntegerBaseParser - add checkMinWidth and checkMaxWidth into BinaryNumberCheckWidth trait - add tests for 0 bit length and multiple of 4 check - add runtime multiple of 4 length check Deprecation/Compatibility Although still supported via the tunable allowSignedIntegerLength1Bit, it is recommended that schemas be updated to not depend on 1-bit signed binary integers. allowSignedIntegerLength1Bit=true may be deprecated and removed in the future. Binary integers with 0 bit length are no longer supported. DAFFODIL-2297 --- .../codegen/c/DaffodilCCodeGenerator.scala | 2 +- ...inaryIntegerKnownLengthCodeGenerator.scala | 3 +- .../daffodil/core/dsom/ElementBase.scala | 32 +- .../grammar/ElementBaseGrammarMixin.scala | 47 +- .../primitives/PrimitivesBinaryNumber.scala | 16 +- .../primitives/PrimitivesIBM4690Packed.scala | 12 +- .../primitives/PrimitivesLengthKind.scala | 11 +- .../grammar/primitives/PrimitivesPacked.scala | 6 - .../org/apache/daffodil/xsd/dafext.xsd | 9 + .../runtime1/BinaryNumberUnparsers.scala | 51 +- .../runtime1/dpath/DFDLXFunctions.scala | 4 +- .../daffodil/runtime1/dpath/NodeInfo.scala | 59 +- .../IBM4690PackedDecimalParsers.scala | 9 +- .../processors/PackedBinaryTraits.scala | 58 +- .../processors/PackedDecimalParsers.scala | 9 +- .../parsers/BinaryNumberParsers.scala | 89 ++- .../daffodil/extensions/enum/enums.tdml | 2 +- .../validation_errors/Validation.tdml | 7 +- .../ContentFramingProps.tdml | 2 +- .../section12/lengthKind/ExplicitTests.tdml | 649 +++++++++++++++++- .../length_properties/LengthProperties.tdml | 20 +- .../daffodil/section13/packed/packed.tdml | 52 +- .../lengthKind/TestLengthKindExplicit.scala | 104 +++ .../section13/packed/TestPacked.scala | 10 + 24 files changed, 1107 insertions(+), 156 deletions(-) diff --git a/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/DaffodilCCodeGenerator.scala b/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/DaffodilCCodeGenerator.scala index 959d97e337..92348ee819 100644 --- a/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/DaffodilCCodeGenerator.scala +++ b/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/DaffodilCCodeGenerator.scala @@ -278,7 +278,7 @@ object DaffodilCCodeGenerator case g: BinaryDouble => binaryFloatGenerateCode(g.e, lengthInBits = 64, cgState) case g: BinaryFloat => binaryFloatGenerateCode(g.e, lengthInBits = 32, cgState) case g: BinaryIntegerKnownLength => - binaryIntegerKnownLengthGenerateCode(g.e, g.lengthInBits, g.signed, cgState) + binaryIntegerKnownLengthGenerateCode(g.e, g.lengthInBits, cgState) case g: CaptureContentLengthEnd => noop(g) case g: CaptureContentLengthStart => noop(g) case g: CaptureValueLengthEnd => noop(g) diff --git a/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/generators/BinaryIntegerKnownLengthCodeGenerator.scala b/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/generators/BinaryIntegerKnownLengthCodeGenerator.scala index 4a835c9ac4..1d2476cb3d 100644 --- a/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/generators/BinaryIntegerKnownLengthCodeGenerator.scala +++ b/daffodil-codegen-c/src/main/scala/org/apache/daffodil/codegen/c/generators/BinaryIntegerKnownLengthCodeGenerator.scala @@ -18,6 +18,7 @@ package org.apache.daffodil.codegen.c.generators import org.apache.daffodil.core.dsom.ElementBase +import org.apache.daffodil.runtime1.dpath.NodeInfo.PrimType.PrimNumeric trait BinaryIntegerKnownLengthCodeGenerator extends BinaryValueCodeGenerator { @@ -25,7 +26,6 @@ trait BinaryIntegerKnownLengthCodeGenerator extends BinaryValueCodeGenerator { def binaryIntegerKnownLengthGenerateCode( e: ElementBase, lengthInBits: Long, - signed: Boolean, cgState: CodeGeneratorState ): Unit = { val cLengthInBits = lengthInBits match { @@ -35,6 +35,7 @@ trait BinaryIntegerKnownLengthCodeGenerator extends BinaryValueCodeGenerator { case n if n <= 64 => 64 case _ => e.SDE("Binary integer lengths longer than 64 bits are not supported.") } + val signed = e.primType.asInstanceOf[PrimNumeric].isSigned val primType = if (signed) s"int$cLengthInBits" else s"uint$cLengthInBits" val addField = valueAddField(e, lengthInBits, primType, _, cgState) val validateFixed = valueValidateFixed(e, _, cgState) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ElementBase.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ElementBase.scala index 3992415bcd..825f69da83 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ElementBase.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ElementBase.scala @@ -774,11 +774,37 @@ trait ElementBase if ( result.isDefined && repElement.isSimpleType && representation == Representation.Binary ) { + val nBits = result.get primType match { case primNumeric: NodeInfo.PrimType.PrimNumeric => - if (primNumeric.width.isDefined) { - val nBits = result.get - val width = primNumeric.width.get + if (primNumeric.minWidth.isDefined) { + val minWidth = primNumeric.minWidth.get + if (nBits < minWidth) { + val isSigned = primNumeric.isSigned + val signedStr = if (isSigned) "a signed" else "an unsigned" + val outOfRangeFmtStr = + "Minimum length for %s binary integer is %d bit(s), number of bits %d out of range. " + + "An unsigned integer with length 1 bit could be used instead." + if (isSigned && tunable.allowSignedIntegerLength1Bit && nBits == 1) { + SDW( + WarnID.SignedBinaryIntegerLength1Bit, + outOfRangeFmtStr, + signedStr, + minWidth, + nBits + ) + } else { + SDE( + outOfRangeFmtStr, + signedStr, + minWidth, + nBits + ) + } + } + } + if (primNumeric.maxWidth.isDefined) { + val width = primNumeric.maxWidth.get if (nBits > width) { SDE( "Number of bits %d out of range for binary %s, must be between 1 and %d bits.", diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index f9982d71a4..12b53c352a 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -786,7 +786,7 @@ trait ElementBaseGrammarMixin ) { ConvertZonedCombinator( this, - new IBM4690PackedIntegerKnownLength(this, false, binaryNumberKnownLengthInBits), + new IBM4690PackedIntegerKnownLength(this, binaryNumberKnownLengthInBits), textConverter ) } @@ -796,7 +796,7 @@ trait ElementBaseGrammarMixin ) { ConvertZonedCombinator( this, - new IBM4690PackedIntegerRuntimeLength(this, false), + new IBM4690PackedIntegerRuntimeLength(this), textConverter ) } @@ -806,7 +806,7 @@ trait ElementBaseGrammarMixin ) { ConvertZonedCombinator( this, - new IBM4690PackedIntegerDelimitedEndOfData(this, false), + new IBM4690PackedIntegerDelimitedEndOfData(this), textConverter ) } @@ -816,7 +816,7 @@ trait ElementBaseGrammarMixin ) { ConvertZonedCombinator( this, - new IBM4690PackedIntegerPrefixedLength(this, false), + new IBM4690PackedIntegerPrefixedLength(this), textConverter ) } @@ -827,7 +827,6 @@ trait ElementBaseGrammarMixin this, new PackedIntegerKnownLength( this, - false, packedSignCodes, binaryNumberKnownLengthInBits ), @@ -838,7 +837,7 @@ trait ElementBaseGrammarMixin prod("packedRuntimeLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { ConvertZonedCombinator( this, - new PackedIntegerRuntimeLength(this, false, packedSignCodes), + new PackedIntegerRuntimeLength(this, packedSignCodes), textConverter ) } @@ -846,7 +845,7 @@ trait ElementBaseGrammarMixin prod("packedDelimitedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { ConvertZonedCombinator( this, - new PackedIntegerDelimitedEndOfData(this, false, packedSignCodes), + new PackedIntegerDelimitedEndOfData(this, packedSignCodes), textConverter ) } @@ -854,7 +853,7 @@ trait ElementBaseGrammarMixin prod("packedPrefixedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { ConvertZonedCombinator( this, - new PackedIntegerPrefixedLength(this, false, packedSignCodes), + new PackedIntegerPrefixedLength(this, packedSignCodes), textConverter ) } @@ -897,7 +896,7 @@ trait ElementBaseGrammarMixin private lazy val packedSignCodes = PackedSignCodes(binaryPackedSignCodes, binaryNumberCheckPolicy) - private def binaryIntegerValue(isSigned: Boolean) = { + private lazy val binaryIntegerValue = { // // Is it a single byte or smaller // @@ -910,10 +909,10 @@ trait ElementBaseGrammarMixin } (binaryNumberRep, lengthKind, binaryNumberKnownLengthInBits) match { case (BinaryNumberRep.Binary, LengthKind.Prefixed, _) => - new BinaryIntegerPrefixedLength(this, isSigned) - case (BinaryNumberRep.Binary, _, -1) => new BinaryIntegerRuntimeLength(this, isSigned) + new BinaryIntegerPrefixedLength(this) + case (BinaryNumberRep.Binary, _, -1) => new BinaryIntegerRuntimeLength(this) case (BinaryNumberRep.Binary, _, _) => - new BinaryIntegerKnownLength(this, isSigned, binaryNumberKnownLengthInBits) + new BinaryIntegerKnownLength(this, binaryNumberKnownLengthInBits) case (_, LengthKind.Implicit, _) => SDE("lengthKind='implicit' is not allowed with packed binary formats") case (_, _, _) @@ -923,26 +922,25 @@ trait ElementBaseGrammarMixin binaryNumberKnownLengthInBits ) case (BinaryNumberRep.Packed, LengthKind.Delimited, -1) => - new PackedIntegerDelimitedEndOfData(this, isSigned, packedSignCodes) + new PackedIntegerDelimitedEndOfData(this, packedSignCodes) case (BinaryNumberRep.Packed, LengthKind.Prefixed, -1) => - new PackedIntegerPrefixedLength(this, isSigned, packedSignCodes) + new PackedIntegerPrefixedLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, -1) => - new PackedIntegerRuntimeLength(this, isSigned, packedSignCodes) + new PackedIntegerRuntimeLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, _) => new PackedIntegerKnownLength( this, - isSigned, packedSignCodes, binaryNumberKnownLengthInBits ) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Delimited, -1) => - new IBM4690PackedIntegerDelimitedEndOfData(this, isSigned) + new IBM4690PackedIntegerDelimitedEndOfData(this) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Prefixed, -1) => - new IBM4690PackedIntegerPrefixedLength(this, isSigned) + new IBM4690PackedIntegerPrefixedLength(this) case (BinaryNumberRep.Ibm4690Packed, _, -1) => - new IBM4690PackedIntegerRuntimeLength(this, isSigned) + new IBM4690PackedIntegerRuntimeLength(this) case (BinaryNumberRep.Ibm4690Packed, _, _) => - new IBM4690PackedIntegerKnownLength(this, isSigned, binaryNumberKnownLengthInBits) + new IBM4690PackedIntegerKnownLength(this, binaryNumberKnownLengthInBits) case (BinaryNumberRep.Bcd, _, _) => primType match { case PrimType.Long | PrimType.Int | PrimType.Short | PrimType.Byte => @@ -974,13 +972,8 @@ trait ElementBaseGrammarMixin // This is in the spirit of that section. val res: Gram = primType match { - case PrimType.Byte | PrimType.Short | PrimType.Int | PrimType.Long | PrimType.Integer => { - binaryIntegerValue(true) - } - - case PrimType.UnsignedByte | PrimType.UnsignedShort | PrimType.UnsignedInt | - PrimType.UnsignedLong | PrimType.NonNegativeInteger => { - binaryIntegerValue(false) + case n: PrimType.PrimNumeric if n.isInteger => { + binaryIntegerValue } case PrimType.Double | PrimType.Float => { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryNumber.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryNumber.scala index 2fbcad59e4..8c2b4b5940 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryNumber.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBinaryNumber.scala @@ -40,37 +40,33 @@ import org.apache.daffodil.unparsers.runtime1.BinaryIntegerKnownLengthUnparser import org.apache.daffodil.unparsers.runtime1.BinaryIntegerPrefixedLengthUnparser import org.apache.daffodil.unparsers.runtime1.BinaryIntegerRuntimeLengthUnparser -class BinaryIntegerRuntimeLength(val e: ElementBase, signed: Boolean) - extends Terminal(e, true) { +class BinaryIntegerRuntimeLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new BinaryIntegerRuntimeLengthParser( e.elementRuntimeData, - signed, e.lengthEv, e.lengthUnits ) override lazy val unparser: Unparser = new BinaryIntegerRuntimeLengthUnparser( e.elementRuntimeData, - signed, e.lengthEv, e.lengthUnits ) } -class BinaryIntegerKnownLength(val e: ElementBase, val signed: Boolean, val lengthInBits: Long) +class BinaryIntegerKnownLength(val e: ElementBase, val lengthInBits: Long) extends Terminal(e, true) { override lazy val parser = { - new BinaryIntegerKnownLengthParser(e.elementRuntimeData, signed, lengthInBits.toInt) + new BinaryIntegerKnownLengthParser(e.elementRuntimeData, lengthInBits.toInt) } override lazy val unparser: Unparser = - new BinaryIntegerKnownLengthUnparser(e.elementRuntimeData, signed, lengthInBits.toInt) + new BinaryIntegerKnownLengthUnparser(e.elementRuntimeData, lengthInBits.toInt) } -class BinaryIntegerPrefixedLength(val e: ElementBase, signed: Boolean) - extends Terminal(e, true) { +class BinaryIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { private lazy val erd = e.elementRuntimeData private lazy val plerd = e.prefixedLengthElementDecl.elementRuntimeData @@ -81,7 +77,6 @@ class BinaryIntegerPrefixedLength(val e: ElementBase, signed: Boolean) erd, e.prefixedLengthBody.parser, plerd, - signed, e.lengthUnits, pladj ) @@ -101,7 +96,6 @@ class BinaryIntegerPrefixedLength(val e: ElementBase, signed: Boolean) e.prefixedLengthBody.unparser, plerd, maybeNBits, - signed, e.lengthUnits, pladj ) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala index 152af02cba..a5380474f5 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala @@ -33,11 +33,9 @@ import org.apache.daffodil.unparsers.runtime1.IBM4690PackedIntegerKnownLengthUnp import org.apache.daffodil.unparsers.runtime1.IBM4690PackedIntegerPrefixedLengthUnparser import org.apache.daffodil.unparsers.runtime1.IBM4690PackedIntegerRuntimeLengthUnparser -class IBM4690PackedIntegerRuntimeLength(val e: ElementBase, signed: Boolean) - extends Terminal(e, true) { +class IBM4690PackedIntegerRuntimeLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new IBM4690PackedIntegerRuntimeLengthParser( e.elementRuntimeData, - signed, e.lengthEv, e.lengthUnits ) @@ -49,23 +47,21 @@ class IBM4690PackedIntegerRuntimeLength(val e: ElementBase, signed: Boolean) ) } -class IBM4690PackedIntegerKnownLength(val e: ElementBase, signed: Boolean, lengthInBits: Long) +class IBM4690PackedIntegerKnownLength(val e: ElementBase, lengthInBits: Long) extends Terminal(e, true) { override lazy val parser = - new IBM4690PackedIntegerKnownLengthParser(e.elementRuntimeData, signed, lengthInBits.toInt) + new IBM4690PackedIntegerKnownLengthParser(e.elementRuntimeData, lengthInBits.toInt) override lazy val unparser: Unparser = new IBM4690PackedIntegerKnownLengthUnparser(e.elementRuntimeData, lengthInBits.toInt) } -class IBM4690PackedIntegerPrefixedLength(val e: ElementBase, signed: Boolean) - extends Terminal(e, true) { +class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new IBM4690PackedIntegerPrefixedLengthParser( e.elementRuntimeData, e.prefixedLengthBody.parser, e.prefixedLengthElementDecl.elementRuntimeData, - signed, e.lengthUnits, e.prefixedLengthAdjustmentInUnits ) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala index 9459267ce0..21baad5654 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala @@ -205,7 +205,6 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { abstract class PackedIntegerDelimited( e: ElementBase, - signed: Boolean, packedSignCodes: PackedSignCodes ) extends StringDelimited(e) { @@ -223,9 +222,8 @@ abstract class PackedIntegerDelimited( case class PackedIntegerDelimitedEndOfData( e: ElementBase, - signed: Boolean, packedSignCodes: PackedSignCodes -) extends PackedIntegerDelimited(e, signed, packedSignCodes) { +) extends PackedIntegerDelimited(e, packedSignCodes) { val isDelimRequired: Boolean = false } @@ -289,8 +287,7 @@ case class BCDDecimalDelimitedEndOfData(e: ElementBase) extends BCDDecimalDelimi val isDelimRequired: Boolean = false } -abstract class IBM4690PackedIntegerDelimited(e: ElementBase, signed: Boolean) - extends StringDelimited(e) { +abstract class IBM4690PackedIntegerDelimited(e: ElementBase) extends StringDelimited(e) { override lazy val parser: DaffodilParser = new IBM4690PackedIntegerDelimitedParser( e.elementRuntimeData, @@ -304,8 +301,8 @@ abstract class IBM4690PackedIntegerDelimited(e: ElementBase, signed: Boolean) ) } -case class IBM4690PackedIntegerDelimitedEndOfData(e: ElementBase, signed: Boolean) - extends IBM4690PackedIntegerDelimited(e, signed) { +case class IBM4690PackedIntegerDelimitedEndOfData(e: ElementBase) + extends IBM4690PackedIntegerDelimited(e) { val isDelimRequired: Boolean = false } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala index fef8199c69..c0f7a73ecd 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala @@ -36,12 +36,10 @@ import org.apache.daffodil.unparsers.runtime1.PackedIntegerRuntimeLengthUnparser class PackedIntegerRuntimeLength( val e: ElementBase, - signed: Boolean, packedSignCodes: PackedSignCodes ) extends Terminal(e, true) { override lazy val parser = new PackedIntegerRuntimeLengthParser( e.elementRuntimeData, - signed, packedSignCodes, e.lengthEv, e.lengthUnits @@ -57,14 +55,12 @@ class PackedIntegerRuntimeLength( class PackedIntegerKnownLength( val e: ElementBase, - signed: Boolean, packedSignCodes: PackedSignCodes, lengthInBits: Long ) extends Terminal(e, true) { override lazy val parser = new PackedIntegerKnownLengthParser( e.elementRuntimeData, - signed, packedSignCodes, lengthInBits.toInt ) @@ -78,7 +74,6 @@ class PackedIntegerKnownLength( class PackedIntegerPrefixedLength( val e: ElementBase, - signed: Boolean, packedSignCodes: PackedSignCodes ) extends Terminal(e, true) { @@ -86,7 +81,6 @@ class PackedIntegerPrefixedLength( e.elementRuntimeData, e.prefixedLengthBody.parser, e.prefixedLengthElementDecl.elementRuntimeData, - signed, packedSignCodes, e.lengthUnits, e.prefixedLengthAdjustmentInUnits diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd index d0e2f4a087..69ea3fdbb5 100644 --- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd +++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd @@ -137,6 +137,14 @@ + + + + When processing signed binary integers, which should have a length of at least 2 bits, issue + a warning if the length is less than 2 bits by default, otherwise (if false) issue a SDE or Processing Error. + + + @@ -734,6 +742,7 @@ + diff --git a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/unparsers/runtime1/BinaryNumberUnparsers.scala b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/unparsers/runtime1/BinaryNumberUnparsers.scala index 2c5745fba0..b4f99d9ab2 100644 --- a/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/unparsers/runtime1/BinaryNumberUnparsers.scala +++ b/daffodil-runtime1-unparser/src/main/scala/org/apache/daffodil/unparsers/runtime1/BinaryNumberUnparsers.scala @@ -24,13 +24,16 @@ import org.apache.daffodil.io.FormatInfo import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo +import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo.Yes import org.apache.daffodil.lib.util.Maybe._ import org.apache.daffodil.lib.util.MaybeInt import org.apache.daffodil.lib.util.Numbers._ +import org.apache.daffodil.runtime1.dpath.NodeInfo import org.apache.daffodil.runtime1.processors.ElementRuntimeData import org.apache.daffodil.runtime1.processors.Evaluatable import org.apache.daffodil.runtime1.processors.ParseOrUnparseState import org.apache.daffodil.runtime1.processors.Processor +import org.apache.daffodil.runtime1.processors.parsers.BinaryNumberCheckWidth import org.apache.daffodil.runtime1.processors.parsers.HasKnownLengthInBits import org.apache.daffodil.runtime1.processors.parsers.HasRuntimeExplicitLength import org.apache.daffodil.runtime1.processors.unparsers._ @@ -56,12 +59,7 @@ abstract class BinaryNumberBaseUnparser(override val context: ElementRuntimeData val nBits = getBitLength(state) val value = getNumberToPut(state) val dos = state.dataOutputStream - val res = - if (nBits > 0) { - putNumber(dos, value, nBits, state) - } else { - true - } + val res = putNumber(dos, value, nBits, state) if (!res) { Assert.invariant(dos.maybeRelBitLimit0b.isDefined) @@ -78,8 +76,11 @@ abstract class BinaryNumberBaseUnparser(override val context: ElementRuntimeData } -abstract class BinaryIntegerBaseUnparser(e: ElementRuntimeData, signed: Boolean) - extends BinaryNumberBaseUnparser(e) { +abstract class BinaryIntegerBaseUnparser(e: ElementRuntimeData) + extends BinaryNumberBaseUnparser(e) + with BinaryNumberCheckWidth { + + private val primNumeric = e.optPrimType.get.asInstanceOf[NodeInfo.PrimType.PrimNumeric] override def putNumber( dos: DataOutputStream, @@ -87,8 +88,18 @@ abstract class BinaryIntegerBaseUnparser(e: ElementRuntimeData, signed: Boolean) nBits: Int, finfo: FormatInfo ): Boolean = { + val state = finfo.asInstanceOf[UState] + if (primNumeric.minWidth.isDefined) { + val minWidth = primNumeric.minWidth.get + val isSigned = primNumeric.isSigned + checkMinWidth(state, isSigned, nBits, minWidth) + } + if (primNumeric.maxWidth.isDefined) { + val maxWidth = primNumeric.maxWidth.get + checkMaxWidth(state, nBits, maxWidth) + } if (nBits > 64) { - dos.putBigInt(asBigInt(value), nBits, signed, finfo) + dos.putBigInt(asBigInt(value), nBits, primNumeric.isSigned, finfo) } else { dos.putLong(asLong(value), nBits, finfo) } @@ -97,9 +108,8 @@ abstract class BinaryIntegerBaseUnparser(e: ElementRuntimeData, signed: Boolean) class BinaryIntegerKnownLengthUnparser( e: ElementRuntimeData, - signed: Boolean, override val lengthInBits: Int -) extends BinaryIntegerBaseUnparser(e, signed) +) extends BinaryIntegerBaseUnparser(e) with HasKnownLengthInBits { override lazy val runtimeDependencies = Vector() @@ -108,10 +118,9 @@ class BinaryIntegerKnownLengthUnparser( class BinaryIntegerRuntimeLengthUnparser( val e: ElementRuntimeData, - signed: Boolean, val lengthEv: Evaluatable[JLong], val lengthUnits: LengthUnits -) extends BinaryIntegerBaseUnparser(e, signed) +) extends BinaryIntegerBaseUnparser(e) with HasRuntimeExplicitLength { override val runtimeDependencies = Vector(lengthEv) @@ -122,12 +131,13 @@ class BinaryIntegerPrefixedLengthUnparser( override val prefixedLengthUnparser: Unparser, override val prefixedLengthERD: ElementRuntimeData, maybeNBits: MaybeInt, - signed: Boolean, override val lengthUnits: LengthUnits, override val prefixedLengthAdjustmentInUnits: Long -) extends BinaryIntegerBaseUnparser(e: ElementRuntimeData, signed: Boolean) +) extends BinaryIntegerBaseUnparser(e: ElementRuntimeData) with KnownPrefixedLengthUnparserMixin { + private val primNumeric = e.optPrimType.get.asInstanceOf[NodeInfo.PrimType.PrimNumeric] + override def childProcessors: Vector[Processor] = Vector(prefixedLengthUnparser) override lazy val runtimeDependencies = Vector() @@ -140,7 +150,7 @@ class BinaryIntegerPrefixedLengthUnparser( // bytes needed to represent the number val value = getNumberToPut(s.asInstanceOf[UState]) val len = Math.max(asBigInt(value).bitLength, 1) - val signedLen = if (signed) len + 1 else len + val signedLen = if (primNumeric.isSigned) len + 1 else len (signedLen + 7) & ~0x7 // round up to nearest multilpe of 8 } } @@ -241,7 +251,8 @@ abstract class BinaryDecimalUnparserBase( e: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int -) extends BinaryNumberBaseUnparser(e) { +) extends BinaryNumberBaseUnparser(e) + with BinaryNumberCheckWidth { override def getNumberToPut(state: UState): JNumber = { val node = state.currentInfosetNode.asSimple @@ -268,6 +279,10 @@ abstract class BinaryDecimalUnparserBase( nBits: Int, finfo: FormatInfo ): Boolean = { - dos.putBigInt(asBigInt(value), nBits, signed == YesNo.Yes, finfo) + val state = finfo.asInstanceOf[UState] + val isSigned: Boolean = signed == Yes + val minWidth: Int = if (isSigned) 2 else 1 + checkMinWidth(state, isSigned, nBits, minWidth) + dos.putBigInt(asBigInt(value), nBits, isSigned, finfo) } } diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLXFunctions.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLXFunctions.scala index 5cec60fff4..106d4740e2 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLXFunctions.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/DFDLXFunctions.scala @@ -54,7 +54,7 @@ case class DFDLXLeftShift(recipes: List[CompiledDPath], argType: NodeInfo.Kind) val shiftLong = arg2.getLong val shift = shiftLong.toInt - val width = argType.asInstanceOf[PrimNumeric].width.get + val width = argType.asInstanceOf[PrimNumeric].maxWidth.get Assert.invariant(shift >= 0) if (shift >= width) dstate.SDE( @@ -101,7 +101,7 @@ case class DFDLXRightShift(recipes: List[CompiledDPath], argType: NodeInfo.Kind) ): DataValuePrimitive = { val shiftLong = arg2.getLong val shift = shiftLong.toInt - val width = argType.asInstanceOf[PrimNumeric].width.get + val width = argType.asInstanceOf[PrimNumeric].maxWidth.get Assert.invariant(shift >= 0) if (shift >= width) dstate.SDE( diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/NodeInfo.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/NodeInfo.scala index cb94dec31f..5ce59bf1ac 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/NodeInfo.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/dpath/NodeInfo.scala @@ -534,7 +534,9 @@ object NodeInfo extends Enum { } trait PrimNumeric { self: Numeric.Kind => - def width: MaybeInt + def isSigned: Boolean + def minWidth: MaybeInt + def maxWidth: MaybeInt def isValid(n: Number): Boolean protected def fromNumberNoCheck(n: Number): DataValueNumber def fromNumber(n: Number): DataValueNumber = { @@ -648,7 +650,9 @@ object NodeInfo extends Enum { // toString would have a precision different than Float.MaxValue.toString override val minStr = "-" + JFloat.MAX_VALUE.toString override val maxStr = JFloat.MAX_VALUE.toString - override val width: MaybeInt = MaybeInt(32) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(32) + override val maxWidth: MaybeInt = MaybeInt(32) } protected sealed trait DoubleKind extends SignedNumeric.Kind @@ -667,7 +671,9 @@ object NodeInfo extends Enum { override val max = JDouble.MAX_VALUE override val minStr = "-" + JDouble.MAX_VALUE.toString override val maxStr = JDouble.MAX_VALUE.toString - override val width: MaybeInt = MaybeInt(64) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(64) + override val maxWidth: MaybeInt = MaybeInt(64) } protected sealed trait DecimalKind extends SignedNumeric.Kind @@ -686,8 +692,9 @@ object NodeInfo extends Enum { case _ => true } } - - override val width: MaybeInt = MaybeInt.Nope + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt.Nope + override val maxWidth: MaybeInt = MaybeInt.Nope override def isInteger = false } @@ -708,8 +715,9 @@ object NodeInfo extends Enum { case _ => true } } - - override val width: MaybeInt = MaybeInt.Nope + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(2) + override val maxWidth: MaybeInt = MaybeInt.Nope override def isInteger = true } @@ -725,7 +733,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueLong = n.longValue override val min = JLong.MIN_VALUE override val max = JLong.MAX_VALUE - override val width: MaybeInt = MaybeInt(64) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(2) + override val maxWidth: MaybeInt = MaybeInt(64) } protected sealed trait IntKind extends Long.Kind @@ -739,7 +749,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueInt = n.intValue override val min = JInt.MIN_VALUE.toLong override val max = JInt.MAX_VALUE.toLong - override val width: MaybeInt = MaybeInt(32) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(2) + override val maxWidth: MaybeInt = MaybeInt(32) } protected sealed trait ShortKind extends Int.Kind @@ -753,7 +765,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueShort = n.shortValue override val min = JShort.MIN_VALUE.toLong override val max = JShort.MAX_VALUE.toLong - override val width: MaybeInt = MaybeInt(16) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(2) + override val maxWidth: MaybeInt = MaybeInt(16) } protected sealed trait ByteKind extends Short.Kind @@ -767,7 +781,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueByte = n.byteValue override val min = JByte.MIN_VALUE.toLong override val max = JByte.MAX_VALUE.toLong - override val width: MaybeInt = MaybeInt(8) + override val isSigned: Boolean = true + override val minWidth: MaybeInt = MaybeInt(2) + override val maxWidth: MaybeInt = MaybeInt(8) } protected sealed trait NonNegativeIntegerKind extends Integer.Kind @@ -786,8 +802,9 @@ object NodeInfo extends Enum { case f: JFloat if f.isInfinite || f.isNaN => false case _ => n.longValue >= 0 } - - override val width: MaybeInt = MaybeInt.Nope + override val isSigned: Boolean = false + override val minWidth: MaybeInt = MaybeInt(1) + override val maxWidth: MaybeInt = MaybeInt.Nope override def isInteger = true } @@ -814,7 +831,9 @@ object NodeInfo extends Enum { } val max = new JBigInt(1, scala.Array.fill(8)(0xff.toByte)) val maxBD = new JBigDecimal(max) - override val width: MaybeInt = MaybeInt(64) + override val isSigned: Boolean = false + override val minWidth: MaybeInt = MaybeInt(1) + override val maxWidth: MaybeInt = MaybeInt(64) override def isInteger = true } @@ -834,7 +853,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueLong = n.longValue override val min = 0L override val max = 0xffffffffL - override val width: MaybeInt = MaybeInt(32) + override val isSigned: Boolean = false + override val minWidth: MaybeInt = MaybeInt(1) + override val maxWidth: MaybeInt = MaybeInt(32) } protected sealed trait UnsignedShortKind extends UnsignedInt.Kind @@ -848,7 +869,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueInt = n.intValue override val min = 0L override val max = 0xffffL - override val width: MaybeInt = MaybeInt(16) + override val isSigned: Boolean = false + override val minWidth: MaybeInt = MaybeInt(1) + override val maxWidth: MaybeInt = MaybeInt(16) } protected sealed trait UnsignedByteKind extends UnsignedShort.Kind @@ -862,7 +885,9 @@ object NodeInfo extends Enum { protected override def fromNumberNoCheck(n: Number): DataValueShort = n.shortValue override val min = 0L override val max = 0xffL - override val width: MaybeInt = MaybeInt(8) + override val isSigned: Boolean = false + override val minWidth: MaybeInt = MaybeInt(1) + override val maxWidth: MaybeInt = MaybeInt(8) } protected sealed trait StringKind extends AnyAtomic.Kind diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala index 36067c404f..dc2799262f 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala @@ -81,10 +81,9 @@ class IBM4690PackedDecimalPrefixedLengthParser( class IBM4690PackedIntegerRuntimeLengthParser( val e: ElementRuntimeData, - signed: Boolean, val lengthEv: Evaluatable[JLong], val lengthUnits: LengthUnits -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with HasRuntimeExplicitLength { override def toBigInteger(num: Array[Byte]): JBigInteger = @@ -96,9 +95,8 @@ class IBM4690PackedIntegerRuntimeLengthParser( class IBM4690PackedIntegerKnownLengthParser( e: ElementRuntimeData, - signed: Boolean, val lengthInBits: Int -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with HasKnownLengthInBits { override def toBigInteger(num: Array[Byte]): JBigInteger = @@ -112,10 +110,9 @@ class IBM4690PackedIntegerPrefixedLengthParser( e: ElementRuntimeData, override val prefixedLengthParser: Parser, override val prefixedLengthERD: ElementRuntimeData, - signed: Boolean, override val lengthUnits: LengthUnits, override val prefixedLengthAdjustmentInUnits: Long -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with PrefixedLengthParserMixin { override def toBigInteger(num: Array[Byte]): JBigInteger = diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala index 5030e0a687..ca7bedf415 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedBinaryTraits.scala @@ -26,6 +26,7 @@ import org.apache.daffodil.lib.equality.TypeEqual import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.util.Maybe import org.apache.daffodil.lib.util.MaybeChar +import org.apache.daffodil.runtime1.dpath.NodeInfo import org.apache.daffodil.runtime1.processors.ElementRuntimeData import org.apache.daffodil.runtime1.processors.FieldDFAParseEv import org.apache.daffodil.runtime1.processors.ParseOrUnparseState @@ -40,18 +41,52 @@ trait PackedBinaryConversion { def toBigDecimal(num: Array[Byte], scale: Int): JBigDecimal } +trait PackedBinaryLengthCheck { + def PE(state: PState, str: String, args: Any*): Unit + def checkLengthNotEqualToZero(nBits: Int, start: PState, packedType: String): Boolean = { + if (nBits == 0) { + PE( + start, + s"Number of bits %d out of range for a packed $packedType.", + nBits + ) + false + } else { + true + } + } + + def checkLengthIsMultipleOf4(nBits: Int, start: PState): Boolean = { + if ((nBits % 4) != 0) { + PE( + start, + "The given length (%s bits) must be a multiple of 4 when using packed binary formats", + nBits + ) + false + } else { + true + } + } +} + abstract class PackedBinaryDecimalBaseParser( override val context: ElementRuntimeData, binaryDecimalVirtualPoint: Int ) extends PrimParser - with PackedBinaryConversion { + with PackedBinaryConversion + with PackedBinaryLengthCheck { override lazy val runtimeDependencies = Vector() protected def getBitLength(s: ParseOrUnparseState): Int def parse(start: PState): Unit = { val nBits = getBitLength(start) - if (nBits == 0) return // zero length is used for outputValueCalc often. + val lengthEqualsZero = !checkLengthNotEqualToZero(nBits, start, packedType = "decimal") + if (lengthEqualsZero) return + val lengthNotMultipleOf4 = !checkLengthIsMultipleOf4(nBits, start) + if (lengthNotMultipleOf4) return + val dis = start.dataInputStream if (!dis.isDefinedForLength(nBits)) { @@ -69,17 +104,28 @@ abstract class PackedBinaryDecimalBaseParser( } abstract class PackedBinaryIntegerBaseParser( - override val context: ElementRuntimeData, - signed: Boolean = false + override val context: ElementRuntimeData ) extends PrimParser - with PackedBinaryConversion { + with PackedBinaryConversion + with PackedBinaryLengthCheck { override lazy val runtimeDependencies = Vector() + val signed = { + context.optPrimType.get match { + case n: NodeInfo.PrimType.PrimNumeric => n.isSigned + // context.optPrimType can be of type date/time via ConvertZonedCombinator + case _ => false + } + } protected def getBitLength(s: ParseOrUnparseState): Int def parse(start: PState): Unit = { val nBits = getBitLength(start) - if (nBits == 0) return // zero length is used for outputValueCalc often. + val lengthEqualsZero = !checkLengthNotEqualToZero(nBits, start, packedType = "integer") + if (lengthEqualsZero) return + val lengthNotMultipleOf4 = !checkLengthIsMultipleOf4(nBits, start) + if (lengthNotMultipleOf4) return + val dis = start.dataInputStream if (!dis.isDefinedForLength(nBits)) { diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala index 248322f3e1..6f130de9f6 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala @@ -84,11 +84,10 @@ class PackedDecimalPrefixedLengthParser( class PackedIntegerRuntimeLengthParser( val e: ElementRuntimeData, - signed: Boolean, packedSignCodes: PackedSignCodes, val lengthEv: Evaluatable[JLong], val lengthUnits: LengthUnits -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with HasRuntimeExplicitLength { override def toBigInteger(num: Array[Byte]): JBigInteger = @@ -100,10 +99,9 @@ class PackedIntegerRuntimeLengthParser( class PackedIntegerKnownLengthParser( e: ElementRuntimeData, - signed: Boolean, packedSignCodes: PackedSignCodes, val lengthInBits: Int -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with HasKnownLengthInBits { override def toBigInteger(num: Array[Byte]): JBigInteger = @@ -117,11 +115,10 @@ class PackedIntegerPrefixedLengthParser( e: ElementRuntimeData, override val prefixedLengthParser: Parser, override val prefixedLengthERD: ElementRuntimeData, - signed: Boolean, packedSignCodes: PackedSignCodes, override val lengthUnits: LengthUnits, override val prefixedLengthAdjustmentInUnits: Long -) extends PackedBinaryIntegerBaseParser(e, signed) +) extends PackedBinaryIntegerBaseParser(e) with PrefixedLengthParserMixin { override def toBigInteger(num: Array[Byte]): JBigInteger = diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala index a373be2fbc..fac7a9078f 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberParsers.scala @@ -22,12 +22,14 @@ import java.math.{ BigDecimal => JBigDecimal, BigInteger => JBigInt } import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo +import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo.Yes import org.apache.daffodil.runtime1.dpath.InvalidPrimitiveDataException import org.apache.daffodil.runtime1.dpath.NodeInfo import org.apache.daffodil.runtime1.processors.ElementRuntimeData import org.apache.daffodil.runtime1.processors.Evaluatable import org.apache.daffodil.runtime1.processors.ParseOrUnparseState import org.apache.daffodil.runtime1.processors.Processor +import org.apache.daffodil.runtime1.processors.unparsers.UState class BinaryFloatParser(override val context: ElementRuntimeData) extends PrimParser { override lazy val runtimeDependencies = Vector() @@ -100,13 +102,18 @@ abstract class BinaryDecimalParserBase( override val context: ElementRuntimeData, signed: YesNo, binaryDecimalVirtualPoint: Int -) extends PrimParser { +) extends PrimParser + with BinaryNumberCheckWidth { override lazy val runtimeDependencies = Vector() protected def getBitLength(s: ParseOrUnparseState): Int def parse(start: PState): Unit = { val nBits = getBitLength(start) + val isSigned = signed == Yes + val minWidth = if (isSigned) 2 else 1 + val res = checkMinWidth(start, isSigned, nBits, minWidth) + if (!res) return val dis = start.dataInputStream if (!dis.isDefinedForLength(nBits)) { PENotEnoughBits(start, nBits, dis) @@ -124,27 +131,24 @@ abstract class BinaryDecimalParserBase( class BinaryIntegerRuntimeLengthParser( val e: ElementRuntimeData, - signed: Boolean, val lengthEv: Evaluatable[JLong], val lengthUnits: LengthUnits -) extends BinaryIntegerBaseParser(e, signed) +) extends BinaryIntegerBaseParser(e) with HasRuntimeExplicitLength {} class BinaryIntegerKnownLengthParser( e: ElementRuntimeData, - signed: Boolean, val lengthInBits: Int -) extends BinaryIntegerBaseParser(e, signed) +) extends BinaryIntegerBaseParser(e) with HasKnownLengthInBits {} class BinaryIntegerPrefixedLengthParser( e: ElementRuntimeData, override val prefixedLengthParser: Parser, override val prefixedLengthERD: ElementRuntimeData, - signed: Boolean, override val lengthUnits: LengthUnits, override val prefixedLengthAdjustmentInUnits: Long -) extends BinaryIntegerBaseParser(e, signed) +) extends BinaryIntegerBaseParser(e) with PrefixedLengthParserMixin { override def childProcessors: Vector[Processor] = Vector(prefixedLengthParser) @@ -155,9 +159,9 @@ class BinaryIntegerPrefixedLengthParser( } abstract class BinaryIntegerBaseParser( - override val context: ElementRuntimeData, - signed: Boolean -) extends PrimParser { + override val context: ElementRuntimeData +) extends PrimParser + with BinaryNumberCheckWidth { override lazy val runtimeDependencies = Vector() protected def getBitLength(s: ParseOrUnparseState): Int @@ -166,16 +170,16 @@ abstract class BinaryIntegerBaseParser( def parse(start: PState): Unit = { val nBits = getBitLength(start) - if (nBits == 0) return // zero length is used for outputValueCalc often. - if (primNumeric.width.isDefined) { - val width = primNumeric.width.get - if (nBits > width) - PE( - start, - "Number of bits %d out of range, must be between 1 and %d bits.", - nBits, - width - ) + if (primNumeric.minWidth.isDefined) { + val minWidth = primNumeric.minWidth.get + val isSigned: Boolean = primNumeric.isSigned + val res = checkMinWidth(start, isSigned, nBits, minWidth) + if (!res) return + } + if (primNumeric.maxWidth.isDefined) { + val maxWidth = primNumeric.maxWidth.get + val res = checkMaxWidth(start, nBits, maxWidth) + if (!res) return } val dis = start.dataInputStream if (!dis.isDefinedForLength(nBits)) { @@ -184,7 +188,7 @@ abstract class BinaryIntegerBaseParser( } val num: JNumber = - if (signed) { + if (primNumeric.isSigned) { if (nBits > 64) { dis.getSignedBigInt(nBits, start) } else { dis.getSignedLong(nBits, start) } } else { @@ -205,3 +209,46 @@ abstract class BinaryIntegerBaseParser( start.simpleElement.overwriteDataValue(res) } } + +trait BinaryNumberCheckWidth { + def checkMinWidth( + state: ParseOrUnparseState, + isSigned: Boolean, + nBits: Int, + minWidth: Int + ): Boolean = { + if ( + nBits < minWidth && !(isSigned && state.tunable.allowSignedIntegerLength1Bit && nBits == 1) + ) { + val signedStr = if (isSigned) "a signed" else "an unsigned" + val outOfRangeStr = + s"Minimum length for $signedStr binary number is $minWidth bit(s), number of bits $nBits out of range. " + + "An unsigned number with length 1 bit could be used instead." + val procErr = state.toProcessingError(outOfRangeStr) + state match { + case s: PState => + s.setFailed(procErr) + return false + case s: UState => + s.toss(procErr) + } + } + true + } + + def checkMaxWidth(state: ParseOrUnparseState, nBits: Int, maxWidth: Int): Boolean = { + if (nBits > maxWidth) { + val procErr = state.toProcessingError( + s"Number of bits $nBits out of range, must be between 1 and $maxWidth bits." + ) + state match { + case s: PState => + s.setFailed(procErr) + return false + case s: UState => + s.toss(procErr) + } + } + true + } +} diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/enum/enums.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/enum/enums.tdml index 7923ae7c44..4205964498 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/enum/enums.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/enum/enums.tdml @@ -134,7 +134,7 @@ - + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section02/validation_errors/Validation.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section02/validation_errors/Validation.tdml index 88bb887021..1467b4e701 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section02/validation_errors/Validation.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section02/validation_errors/Validation.tdml @@ -2521,7 +2521,7 @@ - + @@ -2537,7 +2537,10 @@ - a + + 1 + a + Runtime Schema Definition Error dfdl:bitOrder diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section11/content_framing_properties/ContentFramingProps.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section11/content_framing_properties/ContentFramingProps.tdml index 20f7cb22ec..fe747f4a97 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section11/content_framing_properties/ContentFramingProps.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section11/content_framing_properties/ContentFramingProps.tdml @@ -1319,7 +1319,7 @@ - + + + + + 64 + 32 + + + + + 64 out of range + between 1 and 32 + + + @@ -1490,7 +1506,7 @@ - + @@ -1521,4 +1537,635 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + ff + + + + Schema Definition Error + unsigned binary integer + 1 bit(s) + 0 out of range + + + + + + + ff + + + + Schema Definition Error + signed binary integer + 2 bit(s) + 1 out of range + + + + + + + 0 + + + + Schema Definition Error + unsigned binary integer + 1 bit(s) + 0 out of range + + + + + + + 0 + + + + Schema Definition Error + signed binary integer + 2 bit(s) + 1 out of range + + + + + + + + + -1 + + + + + Unparse Error + signed binary number + 2 bit(s) + 1 out of range + + + + + + + + 0 + 65535 + + + + + Unparse Error + unsigned binary number + 1 bit(s) + 0 out of range + + + + + + + 1 + -1 + + + + + 0180 + + + + + + 01 + 1 + + + + + 1 + 1 + + + + + + + + 1 + + + + 1 + + + + Schema Definition Warning + signed binary integer + 2 bit(s) + 1 out of range + + + + + + 00123C + + + Parse Error + 0 out of range + + + + + + 001088 + + + Parse Error + 0 out of range + + + + + + 00123C + + + Parse Error + 0 out of range + + + + + + 001088 + + + Parse Error + 0 out of range + + + + + + + 1 + + + + + 1 + + + + + + + + 11 + + + + + -1 + + + + + + + + 11 + + + + + -1 + + + + + + + + 1 + + + + + 1 + + + + + + + + + 2 + -1 + + + + + 02 + 11 + + + + + + 02123C + + + + + 2 + 123 + + + + + + + + 021088 + + + + + 2 + 1088 + + + + + + + + 02 + 11 + + + + + 2 + -1 + + + + + + + + + + 1 + 1 + + + + + 01 + 1 + + + + + + + 4668 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ff + + + + Parse Error + unsigned binary number + 1 bit(s) + 0 out of range + + + + + + + ff + + + + Parse Error + signed binary number + 2 bit(s) + 1 out of range + + + + + + + 1 + + + + Unparse Error + unsigned binary number + 1 bit(s) + 0 out of range + + + + + + + 1 + + + + Unparse Error + signed binary number + 2 bit(s) + 1 out of range + + + + + + 1 + + + + 1 + + + + + + + + 1 + + + + 1 + + + + + + 00 + 1 + + + Parse Error + signed binary number + 2 bit(s) + 0 out of range + + + + + + + + 0 + 1 + + + + + Unparse Error + signed binary number + 2 bit(s) + 0 out of range + + + + + + 00 + 1 + + + Parse Error + signed binary number + 1 bit(s) + 0 out of range + + + + + + + + 0 + 1 + + + + + Unparse Error + signed binary number + 1 bit(s) + 0 out of range + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/length_properties/LengthProperties.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/length_properties/LengthProperties.tdml index 866691abc0..52c8ed0f08 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/length_properties/LengthProperties.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/length_properties/LengthProperties.tdml @@ -913,7 +913,7 @@ - + @@ -922,15 +922,15 @@ - + - - + + - + @@ -1118,7 +1118,7 @@ - + @@ -1190,7 +1190,7 @@ - + @@ -1245,10 +1245,10 @@ root="s4" model="bitSchema" description="Section 12 Length Properties - DFDL-12-161R"> 00101010 - 1 + 01 00000000 00000000 00000000 00000001 - 0 - 1 + 00 + 01 diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml index b8e643fa02..572476206b 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml @@ -124,11 +124,22 @@ + + + + + + + + + + + @@ -416,6 +427,45 @@ + + + + 0010101 + + + Parse Error + bits 0 out of range + + + + + + + 00 + 0010101 + + + Parse Error + bits 0 out of range + + + + + + + 07 + 0010101 + + + Parse Error + The given length + must be a multiple of 4 when using packed binary formats + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala index f8a62cb6de..c787fc2267 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindExplicit.scala @@ -164,6 +164,9 @@ class TestLengthKindExplicit { @Test def test_invalidIntBitLengthExpr(): Unit = { runner.runOneTest("invalidIntBitLengthExpr") } + @Test def test_unparseInvalidIntBitLengthExpr(): Unit = { + runner.runOneTest("unparseInvalidIntBitLengthExpr") + } @Test def test_invalidShortBitLengthExpr(): Unit = { runner.runOneTest("invalidShortBitLengthExpr") @@ -180,4 +183,105 @@ class TestLengthKindExplicit { @Test def test_insufficientBitsByte(): Unit = { runner.runOneTest("insufficientBitsByte") } + + // DFDL-2297 + @Test def test_outOfRangeLengthBinaryInteger1(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger1") + } + @Test def test_outOfRangeLengthBinaryInteger2(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger2") + } + @Test def test_outOfRangeLengthBinaryInteger3(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger3") + } + @Test def test_outOfRangeLengthBinaryInteger4(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger4") + } + @Test def test_outOfRangeLengthBinaryInteger5(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger5") + } + @Test def test_outOfRangeLengthBinaryInteger6(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger6") + } + @Test def test_outOfRangeLengthBinaryInteger7(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger7") + } + @Test def test_outOfRangeLengthBinaryInteger8(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger8") + } + @Test def test_outOfRangeLengthBinaryInteger9(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger9") + } + @Test def test_outOfRangeLengthBinaryInteger10(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger10") + } + @Test def test_outOfRangeLengthBinaryInteger11(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger11") + } + @Test def test_outOfRangeLengthBinaryInteger12(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger12") + } + @Test def test_outOfRangeLengthBinaryInteger13(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryInteger13") + } + @Test def test_inRangeLengthBinaryInteger1(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger1") + } + @Test def test_inRangeLengthBinaryInteger2(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger2") + } + @Test def test_inRangeLengthBinaryInteger3(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger3") + } + @Test def test_inRangeLengthBinaryInteger4(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger4") + } + @Test def test_inRangeLengthBinaryInteger5(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger5") + } + @Test def test_inRangeLengthBinaryInteger6(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger6") + } + @Test def test_inRangeLengthBinaryInteger7(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger7") + } + @Test def test_inRangeLengthBinaryInteger8(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger8") + } + @Test def test_inRangeLengthBinaryInteger9(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger9") + } + @Test def test_inRangeLengthBinaryInteger10(): Unit = { + runner.runOneTest("inRangeLengthBinaryInteger10") + } + @Test def test_outOfRangeLengthBinaryDecimal1(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal1") + } + @Test def test_outOfRangeLengthBinaryDecimal2(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal2") + } + @Test def test_outOfRangeLengthBinaryDecimal3(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal3") + } + @Test def test_outOfRangeLengthBinaryDecimal4(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal4") + } + @Test def test_outOfRangeLengthBinaryDecimal5(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal5") + } + @Test def test_outOfRangeLengthBinaryDecimal6(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal6") + } + @Test def test_outOfRangeLengthBinaryDecimal7(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal7") + } + @Test def test_outOfRangeLengthBinaryDecimal8(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal8") + } + @Test def test_outOfRangeLengthBinaryDecimal9(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal9") + } + @Test def test_outOfRangeLengthBinaryDecimal10(): Unit = { + runner.runOneTest("outOfRangeLengthBinaryDecimal10") + } } diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala index cab7f1be8d..029ab04ea6 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala @@ -51,6 +51,16 @@ class TestPacked { @Test def testPackedCharset09(): Unit = { runner.runOneTest("packedCharset09") } @Test def testPackedCharset10(): Unit = { runner.runOneTest("packedCharset10") } + @Test def testZeroLengthPackedCharset(): Unit = { + runner.runOneTest("zeroLengthPackedCharset") + } + @Test def testRuntimeLengthPackedCharset1(): Unit = { + runner.runOneTest("runtimeLengthPackedCharset1") + } + @Test def testRuntimeLengthPackedCharset2(): Unit = { + runner.runOneTest("runtimeLengthPackedCharset2") + } + @Test def testBCDCharset01(): Unit = { runner.runOneTest("bcdCharset01") } @Test def testBCDCharset02(): Unit = { runner.runOneTest("bcdCharset02") } @Test def testBCDCharset03(): Unit = { runner.runOneTest("bcdCharset03") }