You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Using reduceMean and add, mul, div, sqrt to emulation when axes are inconsecutive.
function layerNormalization(builder, input, options) {
// The reduction of the mean and variance values happens over the spatial
// dimensions across all the input features (i.e. all channels) of the input
// tensor.
const reduceOptions = {axes: [1, 2, 3], keepDimensions: true};
const mean = builder.reduceMean(input, reduceOptions);
const variance = builder.reduceMean(
builder.pow(builder.sub(input, mean), builder.constant(input.dataType, 2)),
reduceOptions);
// The scale and bias tensors are of the shape of the input
// specified by the values in the axes parameter (i.e. [1,2,3]).
return builder.add(
builder.mul(
options.scale,
builder.div(
builder.sub(input, mean),
builder.sqrt(builder.add(variance, options.epsilon)))),
options.bias);
}
The text was updated successfully, but these errors were encountered:
lisa0314
changed the title
ONNX layer norm only supports last consecutive dimensions as axes
Support inconsecutive dimensions as axes for layernormalization
Jan 8, 2025
lisa0314
changed the title
Support inconsecutive dimensions as axes for layernormalization
[LayerNorm] Support inconsecutive dimensions as axes
Jan 13, 2025
Using reduceMean and add, mul, div, sqrt to emulation when axes are inconsecutive.
function layerNormalization(builder, input, options) {
// The reduction of the mean and variance values happens over the spatial
// dimensions across all the input features (i.e. all channels) of the input
// tensor.
const reduceOptions = {axes: [1, 2, 3], keepDimensions: true};
const mean = builder.reduceMean(input, reduceOptions);
const variance = builder.reduceMean(
builder.pow(builder.sub(input, mean), builder.constant(input.dataType, 2)),
reduceOptions);
// The scale and bias tensors are of the shape of the input
// specified by the values in the axes parameter (i.e. [1,2,3]).
return builder.add(
builder.mul(
options.scale,
builder.div(
builder.sub(input, mean),
builder.sqrt(builder.add(variance, options.epsilon)))),
options.bias);
}
The text was updated successfully, but these errors were encountered: