HyperLogLog
in reviewThis lesson is complete and being validated before release - details may still change.
in reviewThis lesson is complete and being validated before release - details may still change.
hyperloglog.ts
import type { Tracer } from "../trace";import { MixHash } from "./mix-hash";/** The fewest and most index bits a sketch may use: 4 to 64 registers. */export const HLL_MIN_PRECISION = 2;export const HLL_MAX_PRECISION = 6;/** Keys are small non-negative integers so a stage can label them. */export const HLL_MAX_KEY = 999;/** Items one run adds at most, duplicates included, so a stage stays readable. */export const HLL_MAX_ITEMS = 48;export interface HllOptions { /** Index bits p: the hash's top p bits pick one of m = 2^p registers (default 4). */ readonly precision?: number; /** Seeds the hash, so the registers follow from the seed and the items alone (default 1). */ readonly seed?: number;}export interface HllReport { /** The true distinct count of the run's items. */ readonly distinct: number; /** (estimate - distinct) / distinct, 0 when nothing distinct was added. */ readonly error: number; readonly estimate: number; /** The standard error the sketch promises, 1.04 / sqrt(m). */ readonly expectedError: number; /** Which estimator the count used: linear counting on the empty registers, or the raw harmonic mean. */ readonly method: "linear" | "raw"; readonly precision: number; /** The registers after the run, bucket order. */ readonly registers: readonly number[]; /** Registers still at 0. */ readonly zeros: number;}/** What one count emits and the report reads. */interface HllEstimate { readonly alpha: number; readonly estimate: number; readonly method: "linear" | "raw"; readonly raw: number; readonly zeros: number;}/** * A cardinality sketch: m small registers that together estimate how * many distinct keys passed by, without remembering any of them. Each * key is hashed to 32 bits. The top p bits pick a register; the rest is * a run of coin flips, and its rank is the position of the first 1 bit. * A rank of r is a 1-in-2^r event, so the largest rank a register has * seen hints at how many distinct keys fell into it: about 2^rank. * Repeats hash to the same register with the same rank, so they never * move it, which is why duplicates do not inflate the count. The * estimate is a harmonic mean of the registers' 2^rank guesses, scaled * by a constant alpha that corrects the mean's bias. While the sketch is * nearly empty the registers at 0 are the better evidence, and linear * counting (m ln(m / zeros)) takes over. * * Time: O(1) per add, O(m) per count. Space: m registers, each a few * bits, for a count of any size. */export class HyperLogLog { readonly precision: number; readonly seed: number; private readonly m: number; private readonly registerCount: number; private readonly mix: MixHash; private readonly regs: number[]; private readonly registerArray: number[]; constructor( readonly options: HllOptions = {}, private readonly tracer?: Tracer, ) { const precision = options.precision ?? 4; if (!Number.isInteger(precision) || precision < HLL_MIN_PRECISION || precision > HLL_MAX_PRECISION) { throw new RangeError(`precision must be an integer in [${HLL_MIN_PRECISION}, ${HLL_MAX_PRECISION}]`); } this.precision = precision; this.seed = options.seed ?? 1; this.m = 2 ** precision; this.registerCount = 2 ** precision; this.mix = new MixHash(this.seed); this.regs = Array.from({ length: this.m }, () => 0); this.registerArray = Array.from({ length: this.registerCount }, () => 0); } /** Registers still at 0: the evidence linear counting reads. */ get zeros(): number { return this.regs.filter((register) => register === 0).length; return this.registerArray.filter((register) => register === 0).length; } /** The registers in bucket order, as a copy. */ registers(): number[] { return [...this.regs]; return [...this.registerArray]; } /** Hashes a key, picks its register from the top p bits, and raises the register to the rank of the rest if it is higher. */ add(key: number): void { if (!Number.isInteger(key) || key < 0 || key > HLL_MAX_KEY) { throw new RangeError(`key must be an integer in [0, ${HLL_MAX_KEY}]`); } const p = this.precision; const indexBits = this.precision; const hash = this.mix.hash(key); const bucket = hash >>> (32 - p); const bucket = hash >>> (32 - indexBits); // Shift the index bits out so the leading-zero count reads the remaining 32 - p bits only. const rest = (hash << p) >>> 0; const rest = (hash << indexBits) >>> 0; // An all-zero rest is one past the longest run the 32 - p bits can show. const rank = rest === 0 ? 33 - p : Math.clz32(rest) + 1; const rank = rest === 0 ? 33 - indexBits : Math.clz32(rest) + 1; // ▸ hash the item: the top p bits pick a register, the rest is a run of coin flips whose rank is the position of its first 1 this.tracer?.emit("hash", { bucket, hash, key, rank }); const register = this.regs[bucket]!; const register = this.registerArray[bucket]!; if (rank > register) { this.regs[bucket] = rank; this.registerArray[bucket] = rank; // ▸ the rank beats what the register holds: the register remembers the new longest run this.tracer?.emit("raise", { bucket, from: register, key, to: rank }); } else { // ▸ the rank is no higher than the register: nothing changes, which is why a repeat never inflates the count this.tracer?.emit("keep", { bucket, key, rank, register }); } } /** The distinct-count estimate from the registers as they stand. */ count(): number { return this.estimate().estimate; } /** Adds every item (repeats allowed), counts, and compares the estimate with the true distinct count. */ run(items: readonly number[]): HllReport { if (items.length > HLL_MAX_ITEMS) throw new RangeError(`a run adds at most ${HLL_MAX_ITEMS} items`); for (const key of items) this.add(key); return this.report(new Set(items).size); } /** Counts and scores the estimate against a distinct count the caller knows. */ report(distinct: number): HllReport { if (!Number.isInteger(distinct) || distinct < 0) throw new RangeError("distinct must be a non-negative integer"); const { estimate, method, zeros } = this.estimate(); const error = distinct === 0 ? 0 : (estimate - distinct) / distinct; // ▸ the estimate beside the true distinct count: the relative error is the sketch's score this.tracer?.emit("verdict", { distinct, error, estimate }); return { distinct, error, estimate, expectedError: 1.04 / Math.sqrt(this.m), expectedError: 1.04 / Math.sqrt(this.registerCount), method, precision: this.precision, registers: this.registers(), zeros, }; } /** * The estimator both `count` and `report` share. Alpha corrects the * harmonic mean's bias; the published constants cover 16, 32, and 64 * registers, the closed form the rest. Small counts fall to linear * counting while registers remain empty. */ private estimate(): HllEstimate { const m = this.m; const registerCount = this.registerCount; const alpha = m === 16 ? 0.673 : m === 32 ? 0.697 : m === 64 ? 0.709 : 0.7213 / (1 + 1.079 / m); const alpha = registerCount === 16 ? 0.673 : registerCount === 32 ? 0.697 : registerCount === 64 ? 0.709 : 0.7213 / (1 + 1.079 / registerCount); let sum = 0; for (const register of this.regs) sum += 2 ** -register; for (const register of this.registerArray) sum += 2 ** -register; const raw = (alpha * m * m) / sum; const raw = (alpha * registerCount * registerCount) / sum; const zeros = this.zeros; const linear = raw <= 2.5 * m && zeros > 0; const linear = raw <= 2.5 * registerCount && zeros > 0; const method: "linear" | "raw" = linear ? "linear" : "raw"; const estimate = linear ? m * Math.log(m / zeros) : raw; const estimate = linear ? registerCount * Math.log(registerCount / zeros) : raw; // ▸ a harmonic mean of the registers' 2 to the rank guesses, scaled by alpha; while registers remain empty, linear counting reads them instead this.tracer?.emit("estimate", { alpha, estimate, method, raw, zeros }); return { alpha, estimate, method, raw, zeros }; }}24 distinct keys estimated as 18.61 by linear counting, an error of -22%, from registers that remembered no key at all.