Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 115 additions & 0 deletions __bench__/buf-vs-string.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
/**
* Buffer-vs-string transport benchmark.
*
* Asserts that `findIterBuf` stays within 2x of
* `findIter` on the same corpus. Both APIs must use
* the packed `Uint32Array` transport; if a future
* change reverts `findIterBuf` to per-match FFI
* object allocation, this benchmark fails.
*
* Self-contained: synthesises a haystack large
* enough to produce ~140k matches without needing
* an external corpus download.
*
* Run: bun __bench__/buf-vs-string.ts
*/
import { AhoCorasick } from "../src/index";

const SENTENCE =
"the quick brown fox jumps over the lazy dog and " +
"the cat watches as the sun sets over the hills, " +
"the wind blows softly and the trees sway in the " +
"breeze, the river flows and the world turns. ";

// Sized so `findIter` reports ~140k matches with the
// pattern set below (matches audit reference scale).
const haystack = SENTENCE.repeat(2000);
const buffer = Buffer.from(haystack);

const patterns = [
"the",
"and",
"over",
"fox",
"cat",
"dog",
"sun",
"sets",
"wind",
"river",
];

const ac = new AhoCorasick(patterns);

const stringMatches = ac.findIter(haystack);
const bufMatches = ac.findIterBuf(buffer);

if (stringMatches.length !== bufMatches.length) {
console.error(
`Match-count mismatch: findIter=${stringMatches.length} ` +
`findIterBuf=${bufMatches.length}`,
);
process.exit(1);
}

const N = 10;
const WARMUP = 3;

const time = (
name: string,
fn: () => number,
): { ms: number; count: number } => {
for (let i = 0; i < WARMUP; i++) fn();
const t = performance.now();
let count = 0;
for (let i = 0; i < N; i++) count = fn();
const ms = (performance.now() - t) / N;
console.log(
` ${name.padEnd(20)}${ms.toFixed(2).padStart(8)} ms` +
` ${String(count).padStart(8)} matches`,
);
return { ms, count };
};

console.log("=".repeat(62));
console.log(" findIter vs findIterBuf");
console.log(
` haystack: ${(haystack.length / 1e6).toFixed(2)} MB, ` +
`${patterns.length} patterns, ` +
`${stringMatches.length} matches/run`,
);
console.log("=".repeat(62) + "\n");

const stringResult = time(
"findIter (string)",
() => ac.findIter(haystack).length,
);
const bufResult = time(
"findIterBuf (buf)",
() => ac.findIterBuf(buffer).length,
);

const ratio = bufResult.ms / stringResult.ms;
const THRESHOLD = 2;

console.log(
`\n findIterBuf / findIter = ${ratio.toFixed(2)}x`,
);

if (ratio > THRESHOLD) {
console.error(
`\nFAIL: findIterBuf is ${ratio.toFixed(2)}x ` +
`findIter (limit ${THRESHOLD}x).`,
);
console.error(
"Check that the buffer path is still routed " +
"through the packed Uint32Array transport " +
"(see _findIterPackedBuf in src/lib.rs and " +
"the unpackBuf path in src/core.ts).",
);
process.exit(1);
}

console.log(
`OK: findIterBuf within ${THRESHOLD}x of findIter.`,
);
38 changes: 38 additions & 0 deletions __test__/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1362,6 +1362,44 @@ describe("findIterBuf", () => {
const matches = ac.findIterBuf(buf);
expect(matches).toHaveLength(1);
});

test("agrees with findIter on ASCII (byte == UTF-16)", () => {
const ac = new AhoCorasick(["the", "and", "or"]);
const text = "the cat and the dog or the bird";
const str = ac.findIter(text);
const buf = ac.findIterBuf(Buffer.from(text));

expect(buf).toHaveLength(str.length);
for (const [i, b] of buf.entries()) {
expect(b.pattern).toBe(str[i]!.pattern);
expect(b.start).toBe(str[i]!.start);
expect(b.end).toBe(str[i]!.end);
}
});

test("handles large match counts via packed transport", () => {
// Stresses the packed Uint32Array path: must
// unpack thousands of matches without losing
// any. Mismatch here indicates a transport bug.
const ac = new AhoCorasick(["the"]);
const text = "the ".repeat(10000);
const matches = ac.findIterBuf(Buffer.from(text));
expect(matches).toHaveLength(10000);
expect(matches[0]).toEqual({
pattern: 0,
start: 0,
end: 3,
});
const last = matches.at(-1)!;
expect(last.pattern).toBe(0);
expect(last.end - last.start).toBe(3);
});

test("returns empty array when there are no matches", () => {
const ac = new AhoCorasick(["needle"]);
const matches = ac.findIterBuf(Buffer.from("haystack"));
expect(matches).toEqual([]);
});
});

describe("isMatchBuf", () => {
Expand Down
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@
"bench:speed": "bun __bench__/speed.ts",
"bench:unicode": "bun __bench__/unicode.ts",
"bench:correctness": "bun __bench__/correctness.ts",
"bench:all": "bun __bench__/speed.ts && bun __bench__/unicode.ts && bun __bench__/correctness.ts",
"bench:buf": "bun __bench__/buf-vs-string.ts",
"bench:all": "bun __bench__/speed.ts && bun __bench__/unicode.ts && bun __bench__/correctness.ts && bun __bench__/buf-vs-string.ts",
"bench:install": "cd __bench__ && bun install",
"bench:download": "bash __bench__/download-corpus.sh"
},
Expand Down
45 changes: 43 additions & 2 deletions src/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ type NativeAhoCorasickInstance = {
haystack: string,
replacements: string[],
): string;
_findIterPackedBuf(
haystack: Buffer | Uint8Array,
): Uint32Array;
findIterBuf(haystack: Buffer | Uint8Array): ByteMatch[];
isMatchBuf(haystack: Buffer | Uint8Array): boolean;
};
Expand Down Expand Up @@ -137,8 +140,13 @@ function unpack(
names: (string | undefined)[] | null,
): Match[] {
const len = packed.length;
// `Math.floor` is defensive: if the native side
// ever returned a length that is not a multiple of
// 3, `new Array(non-integer)` would throw a cryptic
// `RangeError` before the per-triple guard could
// surface a descriptive error.
// eslint-disable-next-line unicorn/no-new-array
const matches = new Array<Match>(len / 3);
const matches = new Array<Match>(Math.floor(len / 3));
for (let i = 0, j = 0; i < len; i += 3, j++) {
const idx = packed[i];
const start = packed[i + 1];
Expand All @@ -165,6 +173,37 @@ function unpack(
return matches;
}

/** Unpack a buffer-mode packed result. Offsets are
* bytes (the buffer path does not translate to
* UTF-16 code units), and `ByteMatch` has no
* `text` field. */
function unpackBuf(packed: Uint32Array): ByteMatch[] {
const len = packed.length;
// `Math.floor` is defensive: if the native side
// ever returned a length that is not a multiple of
// 3, `new Array(non-integer)` would throw a cryptic
// `RangeError` before the per-triple guard could
// surface a descriptive error.
// eslint-disable-next-line unicorn/no-new-array
const matches = new Array<ByteMatch>(Math.floor(len / 3));
for (let i = 0, j = 0; i < len; i += 3, j++) {
const idx = packed[i];
const start = packed[i + 1];
const end = packed[i + 2];
if (
idx === undefined ||
start === undefined ||
end === undefined
) {
throw new Error(
`Malformed packed matches at offset ${String(i)}`,
);
}
matches[j] = { pattern: idx, start, end };
}
return matches;
}

// ── Word boundary helpers ───────────────────────

function isWordCharUnicode(ch: string): boolean {
Expand Down Expand Up @@ -361,7 +400,9 @@ export class AhoCorasick {
* Returns **byte offsets** (not UTF-16).
*/
findIterBuf(haystack: Buffer | Uint8Array): ByteMatch[] {
return this._inner.findIterBuf(haystack);
return unpackBuf(
this._inner._findIterPackedBuf(haystack),
);
}

/**
Expand Down
Loading