Skip to content

header checksum and Schema Extension frame added separately #341

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 19, 2025
+42 −32
Merged

header checksum and Schema Extension frame added separately #341

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e131ec3
Added logic for extracting and matching header checksum
Krmjn09 Jun 18, 2025
d4c8b3c
SchemaExtension Frame Reader
Krmjn09 Jun 18, 2025
2a0e27a
Adding suggested changes
Krmjn09 Jun 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 5 additions & 10 deletions demo/node/rntuple_test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,12 @@ else {
console.log(`OK: Field ${i}: ${field.fieldName} (${field.typeName})`);
if (i === 0) {
if (field.fieldName !== 'Category' || field.typeName !== 'std::int32_t')
console.error(`FAILURE: First field should be 'Category (std::int32_t)' but got '${field.fieldName} (${field.typeName})'`);

console.error(`FAILURE: First field should be 'Category (std::int32_t)' but got '${field.fieldName} (${field.typeName})'`);
} else if (i === rntuple.builder.fieldDescriptors.length - 1){
if (field.fieldName !== 'Nation' || field.typeName !== 'std::string')
console.error(`FAILURE: Last field should be 'Nation (std::string)' but got '${field.fieldName} (${field.typeName})'`);

console.error(`FAILURE: Last field should be 'Nation (std::string)' but got '${field.fieldName} (${field.typeName})'`);
}
}

}

// Column Check
Expand All @@ -68,12 +65,10 @@ else {
console.log(`OK: Column ${i} fieldId: ${column.fieldId} `);
if (i === 0) {
if (column.fieldId !== 0)
console.error(`FAILURE: First column should be for fieldId 0 (Category)`);
console.error('FAILURE: First column should be for fieldId 0 (Category)');
} else if (i === rntuple.builder.columnDescriptors.length - 1){
if (column.fieldId !== 10)
console.error(`FAILURE: Last column should be for fieldId 10 (Nation)`);
console.error('FAILURE: Last column should be for fieldId 10 (Nation)');
}
}
}


}
59 changes: 37 additions & 22 deletions modules/rntuple.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,20 @@ class RNTupleDescriptorBuilder {
deserializeHeader(header_blob) {
if (!header_blob) return;

const reader = new RBufferReader(header_blob);
const reader = new RBufferReader(header_blob),

payloadStart = reader.offset,
// Read the envelope metadata
this._readEnvelopeMetadata(reader);
{ envelopeLength } = this._readEnvelopeMetadata(reader),

// TODO: Validate the envelope checksum at the end of deserialization
// const payloadStart = reader.offset;
// Seek to end of envelope to get checksum
checksumPos = payloadStart + envelopeLength - 8,
currentPos = reader.offset;

reader.seek(checksumPos);
this.headerEnvelopeChecksum = reader.readU64();

reader.seek(currentPos);

// Read feature flags list (may span multiple 64-bit words)
this._readFeatureFlags(reader);
Expand All @@ -125,15 +133,8 @@ deserializeHeader(header_blob) {
this.description = reader.readString();
this.writer = reader.readString();

// List frame: list of field record frames
this._readFieldDescriptors(reader);

// List frame: list of column record frames
this._readColumnDescriptors(reader);
// Read alias column descriptors
this._readAliasColumn(reader);
// Read Extra Type Information
this._readExtraTypeInformation(reader);
// 4 list frames inside the header envelope
this._readSchemaDescription(reader);
}

deserializeFooter(footer_blob) {
Expand All @@ -148,7 +149,9 @@ deserializeFooter(footer_blob) {
// Feature flag(32 bits)
this._readFeatureFlags(reader);
// Header checksum (64-bit xxhash3)
this.headerChecksum = reader.readU64();
const headerChecksumFromFooter = reader.readU64();
if (headerChecksumFromFooter !== this.headerEnvelopeChecksum)
throw new Error('RNTuple corrupted: header checksum does not match footer checksum.');

const schemaExtensionSize = reader.readS64();

Expand All @@ -157,10 +160,7 @@ deserializeFooter(footer_blob) {
throw new Error('Schema extension frame is not a record frame, which is unexpected.');

// Schema extension record frame (4 list frames inside)
this._readFieldDescriptors(reader);
this._readColumnDescriptors(reader);
this._readAliasColumn(reader);
this._readExtraTypeInformation(reader);
this._readSchemaDescription(reader);

// Cluster Group record frame
this._readClusterGroups(reader);
Expand All @@ -180,6 +180,21 @@ _readEnvelopeMetadata(reader) {
return { envelopeType, envelopeLength };
}

_readSchemaDescription(reader) {
// Reading new descriptor arrays from the input
const newFields = this._readFieldDescriptors(reader),
newColumns = this._readColumnDescriptors(reader),
newAliases = this._readAliasColumn(reader),
newExtra = this._readExtraTypeInformation(reader);

// Merging these new arrays into existing arrays
this.fieldDescriptors = (this.fieldDescriptors || []).concat(newFields);
this.columnDescriptors = (this.columnDescriptors || []).concat(newColumns);
this.aliasColumns = (this.aliasColumns || []).concat(newAliases);
this.extraTypeInfo = (this.extraTypeInfo || []).concat(newExtra);
}


_readFeatureFlags(reader) {
this.featureFlags = [];
while (true) {
Expand Down Expand Up @@ -241,7 +256,7 @@ fieldListIsList = fieldListSize < 0;
checksum
});
}
this.fieldDescriptors = fieldDescriptors;
return fieldDescriptors;
}

_readColumnDescriptors(reader) {
Expand Down Expand Up @@ -287,7 +302,7 @@ _readColumnDescriptors(reader) {

columnDescriptors.push(column);
}
this.columnDescriptors = columnDescriptors;
return columnDescriptors;
}
_readAliasColumn(reader){
const aliasColumnListSize = reader.readS64(),
Expand All @@ -307,7 +322,7 @@ _readAliasColumn(reader){
fieldId
});
}
this.aliasColumns = aliasColumns;
return aliasColumns;
}
_readExtraTypeInformation(reader) {
const extraTypeInfoListSize = reader.readS64(),
Expand All @@ -330,7 +345,7 @@ _readExtraTypeInformation(reader) {
typeVersion
});
}
this.extraTypeInfo = extraTypeInfo;
return extraTypeInfo;
}
_readClusterGroups(reader) {
const clusterGroupListSize = reader.readS64(),
Expand Down