diff --git a/packages/compiler/src/codegen/codegen.ts b/packages/compiler/src/codegen/codegen.ts index 05d8d01c..3b8f59d6 100644 --- a/packages/compiler/src/codegen/codegen.ts +++ b/packages/compiler/src/codegen/codegen.ts @@ -49,6 +49,10 @@ import type { SymbolId, TypeId, } from "../semantics/ids.js"; +import { + markCompilerPerfPhaseDuration, + startCompilerPerfPhase, +} from "../perf.js"; import { DiagnosticEmitter } from "../diagnostics/index.js"; import { createCodegenModule } from "./wasm-module.js"; import { createProgramHelperRegistry } from "./program-helpers.js"; @@ -94,6 +98,7 @@ export const codegenProgram = ({ options = {}, optimization, }: CodegenProgramParams): CodegenResult => { + const codegenStartedAt = startCompilerPerfPhase(); const modules = Array.from(program.modules.values()); const mod = createCodegenModule(); const mergedOptions = normalizeCodegenOptions(options); @@ -220,18 +225,28 @@ export const codegenProgram = ({ entryCtx.programHelpers.ensureEffectHelpers(entryCtx); } - const outputModule = mergedOptions.optimize - ? binaryen.readBinary(emitWasmBytes(mod)) - : mod; + markCompilerPerfPhaseDuration("codegen", codegenStartedAt); + + let outputModule = mod; if (mergedOptions.optimize) { + const prepareStartedAt = startCompilerPerfPhase(); + outputModule = binaryen.readBinary(emitWasmBytes(mod)); + markCompilerPerfPhaseDuration("binaryen.prepareOptimizeModule", prepareStartedAt); outputModule.setFeatures(VOYD_BINARYEN_FEATURES); + const optimizeStartedAt = startCompilerPerfPhase(); optimizeBinaryenModule({ module: outputModule, profile: mergedOptions.optimizationProfile, }); + markCompilerPerfPhaseDuration("binaryen.optimize", optimizeStartedAt); } - const wasm = mergedOptions.validate ? emitWasmBytes(outputModule) : undefined; + let wasm: Uint8Array | undefined; + if (mergedOptions.validate) { + const validateEmitStartedAt = startCompilerPerfPhase(); + wasm = emitWasmBytes(outputModule); + markCompilerPerfPhaseDuration("binaryen.emitValidatedWasm", validateEmitStartedAt); + } if (wasm) { if (!WebAssembly.validate(wasm as BufferSource)) { outputModule.validate(); diff --git a/packages/compiler/src/optimize/pipeline.ts b/packages/compiler/src/optimize/pipeline.ts index 86c23428..6204749b 100644 --- a/packages/compiler/src/optimize/pipeline.ts +++ b/packages/compiler/src/optimize/pipeline.ts @@ -27,6 +27,11 @@ import type { } from "../semantics/ids.js"; import type { SemanticsPipelineResult } from "../semantics/pipeline.js"; import type { CodegenOptions } from "../codegen/context.js"; +import { + incrementCompilerPerfCounter, + markCompilerPerfPhaseDuration, + startCompilerPerfPhase, +} from "../perf.js"; import { type ProgramOptimizationContext, type ProgramOptimizationPass, @@ -6043,7 +6048,21 @@ export const optimizeProgram = ({ void options; OPTIMIZATION_PASSES.forEach((pass) => { + const passStartedAt = startCompilerPerfPhase(); const result = pass.run(context); + markCompilerPerfPhaseDuration( + `optimize.pass.${pass.name}`, + passStartedAt, + ); + if (result.changed) { + incrementCompilerPerfCounter(`optimize.pass.${pass.name}.changed`); + } + if (result.invalidates?.length) { + incrementCompilerPerfCounter( + `optimize.pass.${pass.name}.invalidations`, + result.invalidates.length, + ); + } if (result.invalidates?.length) { context.invalidateAnalyses(result.invalidates); } diff --git a/packages/compiler/src/perf.ts b/packages/compiler/src/perf.ts index b81d41e3..d0a22643 100644 --- a/packages/compiler/src/perf.ts +++ b/packages/compiler/src/perf.ts @@ -1,4 +1,5 @@ type CompilerPerfCounterSnapshot = Map; +type CompilerPerfPhaseSnapshot = Map; type CompilerPerfSummary = { entryPath: string; @@ -6,6 +7,16 @@ type CompilerPerfSummary = { phasesMs: Readonly>; counters: Readonly>; diagnostics: number; + overlapped?: boolean; +}; + +export type CompilerPerfSession = { + entryPath: string; + enabled: boolean; + startedAt: number; + countersBefore?: CompilerPerfCounterSnapshot; + phasesBefore?: CompilerPerfPhaseSnapshot; + overlapped?: boolean; }; const COMPILER_PERF_ENV = "VOYD_COMPILER_PERF"; @@ -25,6 +36,8 @@ const PERF_ENABLED = (() => { })(); const counters = new Map(); +const phaseDurationsMs = new Map(); +const activeSessions = new Set(); const roundMs = (value: number): number => Math.round(value * 1000) / 1000; @@ -50,9 +63,35 @@ export const incrementCompilerPerfCounter = ( counters.set(name, (counters.get(name) ?? 0) + amount); }; +export const startCompilerPerfPhase = (): number => + PERF_ENABLED ? performance.now() : 0; + +export const addCompilerPerfPhaseDuration = ( + name: string, + durationMs: number, +): void => { + if (!PERF_ENABLED || durationMs <= 0) { + return; + } + phaseDurationsMs.set(name, (phaseDurationsMs.get(name) ?? 0) + durationMs); +}; + +export const markCompilerPerfPhaseDuration = ( + name: string, + startedAt: number, +): void => { + if (!PERF_ENABLED) { + return; + } + addCompilerPerfPhaseDuration(name, performance.now() - startedAt); +}; + export const snapshotCompilerPerfCounters = (): CompilerPerfCounterSnapshot => PERF_ENABLED ? new Map(counters) : new Map(); +export const snapshotCompilerPerfPhases = (): CompilerPerfPhaseSnapshot => + PERF_ENABLED ? new Map(phaseDurationsMs) : new Map(); + export const diffCompilerPerfCounters = ({ before, after, @@ -75,6 +114,28 @@ export const diffCompilerPerfCounters = ({ return toSortedRecord(delta); }; +export const diffCompilerPerfPhases = ({ + before, + after, +}: { + before: ReadonlyMap; + after: ReadonlyMap; +}): Record => { + if (!PERF_ENABLED) { + return {}; + } + + const keys = new Set([...before.keys(), ...after.keys()]); + const delta = new Map(); + keys.forEach((key) => { + const diff = (after.get(key) ?? 0) - (before.get(key) ?? 0); + if (diff !== 0) { + delta.set(key, diff); + } + }); + return toSortedRecord(delta); +}; + export const normalizeCompilerPerfPhases = ( phasesMs: Readonly>, ): Record => @@ -90,6 +151,7 @@ export const logCompilerPerfSummary = ({ phasesMs, counters, diagnostics, + overlapped, }: CompilerPerfSummary): void => { if (!PERF_ENABLED) { return; @@ -101,7 +163,73 @@ export const logCompilerPerfSummary = ({ diagnostics, phasesMs: normalizeCompilerPerfPhases(phasesMs), counters, + ...(overlapped ? { overlapped: true } : {}), }; console.error(`[voyd:compiler:perf] ${JSON.stringify(summary)}`); }; + +export const startCompilerPerfSession = ({ + entryPath, +}: { + entryPath: string; +}): CompilerPerfSession => { + if (!PERF_ENABLED) { + return { + entryPath, + enabled: false, + startedAt: 0, + }; + } + + const session: CompilerPerfSession = { + entryPath, + enabled: true, + startedAt: performance.now(), + countersBefore: snapshotCompilerPerfCounters(), + phasesBefore: snapshotCompilerPerfPhases(), + }; + if (activeSessions.size > 0) { + session.overlapped = true; + activeSessions.forEach((activeSession) => { + activeSession.overlapped = true; + }); + } + activeSessions.add(session); + return session; +}; + +export const completeCompilerPerfSession = ({ + session, + success, + diagnostics, +}: { + session: CompilerPerfSession; + success: boolean; + diagnostics: number; +}): void => { + if (!session.enabled || !session.countersBefore || !session.phasesBefore) { + return; + } + activeSessions.delete(session); + + const phases = diffCompilerPerfPhases({ + before: session.phasesBefore, + after: snapshotCompilerPerfPhases(), + }); + + logCompilerPerfSummary({ + entryPath: session.entryPath, + success, + diagnostics, + phasesMs: { + ...phases, + total: performance.now() - session.startedAt, + }, + counters: diffCompilerPerfCounters({ + before: session.countersBefore, + after: snapshotCompilerPerfCounters(), + }), + overlapped: session.overlapped, + }); +}; diff --git a/packages/compiler/src/pipeline-shared.ts b/packages/compiler/src/pipeline-shared.ts index 4f50e8e9..98d8879e 100644 --- a/packages/compiler/src/pipeline-shared.ts +++ b/packages/compiler/src/pipeline-shared.ts @@ -22,10 +22,10 @@ import type { SourceSpan, SymbolId } from "./semantics/ids.js"; import { getSymbolTable } from "./semantics/_internal/symbol-table.js"; import { formatEffectRow } from "./semantics/effects/format.js"; import { - diffCompilerPerfCounters, - isCompilerPerfEnabled, - logCompilerPerfSummary, - snapshotCompilerPerfCounters, + completeCompilerPerfSession, + markCompilerPerfPhaseDuration, + startCompilerPerfPhase, + startCompilerPerfSession, } from "./perf.js"; export type LoadModulesOptions = { @@ -404,7 +404,9 @@ export const emitProgram = async ({ module: binaryen.Module; diagnostics: Diagnostic[]; }> => { + const lowerStartedAt = startCompilerPerfPhase(); const { orderedModules, entry } = lowerProgram({ graph, semantics }); + markCompilerPerfPhaseDuration("lowerProgram", lowerStartedAt); const targetModuleId = entryModuleId ?? entry; const modules = orderedModules .map((id) => semantics.get(id)) @@ -413,15 +415,22 @@ export const emitProgram = async ({ throw new Error("No semantics available for codegen"); } + const codegenLoadStartedAt = startCompilerPerfPhase(); const codegen = await lazyCodegen(); + markCompilerPerfPhaseDuration("loadCodegen", codegenLoadStartedAt); + const monomorphizeStartedAt = startCompilerPerfPhase(); const monomorphized = linkSemantics !== false ? monomorphizeProgram({ modules, semantics }) : { instances: [], moduleTyping: new Map() }; + markCompilerPerfPhaseDuration("monomorphizeProgram", monomorphizeStartedAt); + const viewStartedAt = startCompilerPerfPhase(); const program = buildProgramCodegenView(modules, { instances: monomorphized.instances, moduleTyping: monomorphized.moduleTyping, }); + markCompilerPerfPhaseDuration("buildProgramCodegenView", viewStartedAt); + const optimizeStartedAt = startCompilerPerfPhase(); const optimized = codegenOptions?.optimize ? optimizeProgram({ program, @@ -430,13 +439,18 @@ export const emitProgram = async ({ options: codegenOptions, }) : undefined; + markCompilerPerfPhaseDuration("optimizeProgram", optimizeStartedAt); + const codegenStartedAt = startCompilerPerfPhase(); const result = codegen.codegenProgram({ program: optimized?.program ?? program, entryModuleId: targetModuleId, options: codegenOptions, optimization: optimized?.facts, }); + markCompilerPerfPhaseDuration("codegenProgram", codegenStartedAt); + const emitStartedAt = startCompilerPerfPhase(); const wasm = result.wasm ?? emitBinary(result.module); + markCompilerPerfPhaseDuration("emitBinary", emitStartedAt); return { wasm, module: result.module, diagnostics: result.diagnostics }; }; @@ -564,50 +578,27 @@ export const compileProgramWithLoader = async ( ? undefined : preloadCodegen(); void codegenLoadPromise?.catch(() => undefined); - const perfEnabled = isCompilerPerfEnabled(); - const compileStartedAt = perfEnabled ? performance.now() : 0; - const perfCountersBefore = perfEnabled - ? snapshotCompilerPerfCounters() - : undefined; - const phaseDurationsMs: Record = {}; - const markPhaseDuration = (phase: string, startedAt: number) => { - if (!perfEnabled) { - return; - } - phaseDurationsMs[phase] = performance.now() - startedAt; - }; + const perfSession = startCompilerPerfSession({ + entryPath: options.entryPath, + }); const complete = ( result: CompileProgramResult, ): CompileProgramResult => { - if (!perfEnabled || !perfCountersBefore) { - return result; - } - - phaseDurationsMs.total = performance.now() - compileStartedAt; - const perfCountersAfter = snapshotCompilerPerfCounters(); - const diagnostics = result.success ? 0 : result.diagnostics.length; - - logCompilerPerfSummary({ - entryPath: options.entryPath, + completeCompilerPerfSession({ + session: perfSession, success: result.success, - diagnostics, - phasesMs: phaseDurationsMs, - counters: diffCompilerPerfCounters({ - before: perfCountersBefore, - after: perfCountersAfter, - }), + diagnostics: result.success ? 0 : result.diagnostics.length, }); - return result; }; let graph: ModuleGraph; - const loadStartedAt = perfEnabled ? performance.now() : 0; + const loadStartedAt = startCompilerPerfPhase(); try { graph = await loadModuleGraph(options); - markPhaseDuration("loadModuleGraph", loadStartedAt); + markCompilerPerfPhaseDuration("loadModuleGraph", loadStartedAt); } catch (error) { - markPhaseDuration("loadModuleGraph", loadStartedAt); + markCompilerPerfPhaseDuration("loadModuleGraph", loadStartedAt); return complete(compileProgramFailure( diagnosticsFromUnexpectedError({ error, @@ -623,12 +614,12 @@ export const compileProgramWithLoader = async ( : compileProgramSuccess({ graph })); } - const analyzeStartedAt = perfEnabled ? performance.now() : 0; + const analyzeStartedAt = startCompilerPerfPhase(); const { semantics, diagnostics: semanticDiagnostics } = analyzeModules({ graph, includeTests: options.includeTests, }); - markPhaseDuration("analyzeModules", analyzeStartedAt); + markCompilerPerfPhaseDuration("analyzeModules", analyzeStartedAt); const diagnostics = [...graph.diagnostics, ...semanticDiagnostics]; if (hasErrorDiagnostics(diagnostics)) { @@ -637,7 +628,7 @@ export const compileProgramWithLoader = async ( const shouldLinkSemantics = options.linkSemantics !== false; - const emitStartedAt = perfEnabled ? performance.now() : 0; + const emitStartedAt = startCompilerPerfPhase(); try { const wasmResult = await emitProgram({ graph, @@ -646,7 +637,7 @@ export const compileProgramWithLoader = async ( entryModuleId: options.entryModuleId, linkSemantics: shouldLinkSemantics, }); - markPhaseDuration("emitProgram", emitStartedAt); + markCompilerPerfPhaseDuration("emitProgram", emitStartedAt); diagnostics.push(...wasmResult.diagnostics); @@ -660,7 +651,7 @@ export const compileProgramWithLoader = async ( wasm: wasmResult.wasm, })); } catch (error) { - markPhaseDuration("emitProgram", emitStartedAt); + markCompilerPerfPhaseDuration("emitProgram", emitStartedAt); return complete(compileProgramFailure([ ...diagnostics, codegenErrorToDiagnostic(error, { diff --git a/packages/sdk/src/__tests__/sdk-perf.test.ts b/packages/sdk/src/__tests__/sdk-perf.test.ts new file mode 100644 index 00000000..fce8bc40 --- /dev/null +++ b/packages/sdk/src/__tests__/sdk-perf.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const SOURCE = `pub fn main() -> i32 + 42 +`; + +describe("SDK compiler perf instrumentation", () => { + afterEach(() => { + delete process.env.VOYD_COMPILER_PERF; + vi.restoreAllMocks(); + vi.resetModules(); + }); + + it("emits a compiler perf summary from the SDK compile path", async () => { + process.env.VOYD_COMPILER_PERF = "1"; + vi.resetModules(); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const { createSdk } = await import("@voyd-lang/sdk"); + + const result = await createSdk().compile({ + source: SOURCE, + optimize: true, + emitWasmText: true, + }); + + expect(result.success).toBe(true); + expect(result.success ? result.wasmText : undefined).toBeDefined(); + const perfLine = errorSpy.mock.calls + .map(([message]) => String(message)) + .find((message) => message.startsWith("[voyd:compiler:perf] ")); + expect(perfLine).toBeDefined(); + const summary = JSON.parse( + perfLine!.slice("[voyd:compiler:perf] ".length), + ) as { + success: boolean; + phasesMs: Record; + }; + expect(summary.success).toBe(true); + expect(summary.phasesMs.loadModuleGraph).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs.analyzeModules).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs.optimizeProgram).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs.codegen).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs["binaryen.optimize"]).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs["sdk.finalizeCompile"]).toBeGreaterThanOrEqual(0); + expect(summary.phasesMs.total).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/packages/sdk/src/node.ts b/packages/sdk/src/node.ts index 7993d297..5d583e28 100644 --- a/packages/sdk/src/node.ts +++ b/packages/sdk/src/node.ts @@ -244,14 +244,14 @@ const compileSdk = async (options: CompileOptions): Promise => { runtimeDiagnostics, loadModuleGraph, boundaryExports: options.boundaryExports, + finalizeSuccess: (result) => finalizeCompile({ options, result }), }); if (!result.success) { return result; } - const finalized = finalizeCompile({ options, result }); - return createCompileResult(finalized); + return createCompileResult(result); } catch (error) { return { success: false, diff --git a/packages/sdk/src/shared/compile.ts b/packages/sdk/src/shared/compile.ts index b530d15c..facfeeb5 100644 --- a/packages/sdk/src/shared/compile.ts +++ b/packages/sdk/src/shared/compile.ts @@ -9,6 +9,12 @@ import { type LoadModuleGraphFn, type TestScope, } from "@voyd-lang/compiler/pipeline-shared.js"; +import { + completeCompilerPerfSession, + markCompilerPerfPhaseDuration, + startCompilerPerfPhase, + startCompilerPerfSession, +} from "@voyd-lang/compiler/perf.js"; import type { BoundaryExportsOption } from "@voyd-lang/compiler/codegen/context.js"; import type { TestCase } from "./types.js"; import { diagnosticsFromUnknownError } from "./diagnostics.js"; @@ -42,6 +48,7 @@ export const compileWithLoader = async ({ loadModuleGraph, testScope, boundaryExports, + finalizeSuccess, }: { entryPath: string; roots: ModuleRoots; @@ -53,11 +60,48 @@ export const compileWithLoader = async ({ loadModuleGraph: LoadModuleGraphFn; testScope?: TestScope; boundaryExports?: BoundaryExportsOption; + finalizeSuccess?: ( + result: CompileArtifactsSuccess, + ) => CompileArtifactsSuccess | Promise; }): Promise => { const shouldIncludeTests = includeTests || testsOnly; const codegenLoadPromise = preloadCodegen(); void codegenLoadPromise.catch(() => undefined); + const perfSession = startCompilerPerfSession({ entryPath }); + const complete = (result: CompileArtifacts): CompileArtifacts => { + completeCompilerPerfSession({ + session: perfSession, + success: result.success, + diagnostics: result.success ? 0 : result.diagnostics.length, + }); + return result; + }; + const finalize = async ( + result: CompileArtifactsSuccess, + ): Promise => { + if (!finalizeSuccess) { + return result; + } + const finalizeStartedAt = startCompilerPerfPhase(); + try { + return await finalizeSuccess(result); + } finally { + markCompilerPerfPhaseDuration("sdk.finalizeCompile", finalizeStartedAt); + } + }; + const emitProgramWithPerf = async ( + phase: string, + options: Parameters[0], + ): Promise>> => { + const startedAt = startCompilerPerfPhase(); + try { + return await emitProgram(options); + } finally { + markCompilerPerfPhaseDuration(phase, startedAt); + } + }; let graph: Awaited>; + const loadStartedAt = startCompilerPerfPhase(); try { graph = await loadModuleGraph({ entryPath, @@ -65,14 +109,16 @@ export const compileWithLoader = async ({ host, includeTests: shouldIncludeTests, }); + markCompilerPerfPhaseDuration("loadModuleGraph", loadStartedAt); } catch (error) { - return { + markCompilerPerfPhaseDuration("loadModuleGraph", loadStartedAt); + return complete({ success: false, diagnostics: diagnosticsFromUnknownError({ error, fallbackFile: entryPath, }), - }; + }); } const scopedTestScope = testScope ?? "all"; @@ -92,24 +138,26 @@ export const compileWithLoader = async ({ ...runtimeDiagnosticsCodegenOption, boundaryExports: false, } as const; + const analyzeStartedAt = startCompilerPerfPhase(); const { semantics, diagnostics: semanticDiagnostics, tests } = analyzeModules({ graph, includeTests: shouldIncludeTests, testScope: scopedTestScope, }); + markCompilerPerfPhaseDuration("analyzeModules", analyzeStartedAt); const diagnostics = [...graph.diagnostics, ...semanticDiagnostics]; const testCases = shouldIncludeTests ? tests : undefined; if (hasErrorDiagnostics(diagnostics)) { - return { + return complete({ success: false, diagnostics, - }; + }); } try { if (testsOnly) { - const testResult = await emitProgram({ + const testResult = await emitProgramWithPerf("emitProgram", { graph, semantics, codegenOptions: { @@ -120,31 +168,31 @@ export const compileWithLoader = async ({ }); const allDiagnostics = [...diagnostics, ...testResult.diagnostics]; if (hasErrorDiagnostics(allDiagnostics)) { - return { success: false, diagnostics: allDiagnostics }; + return complete({ success: false, diagnostics: allDiagnostics }); } - return { + return complete(await finalize({ success: true, wasm: testResult.wasm, tests: testCases, testsWasm: testResult.wasm, - }; + })); } - const wasmResult = await emitProgram({ + const wasmResult = await emitProgramWithPerf("emitProgram", { graph, semantics, codegenOptions: codegenOption, }); const baseDiagnostics = [...diagnostics, ...wasmResult.diagnostics]; if (hasErrorDiagnostics(baseDiagnostics)) { - return { success: false, diagnostics: baseDiagnostics }; + return complete({ success: false, diagnostics: baseDiagnostics }); } let testsWasm: Uint8Array | undefined; if (shouldIncludeTests && tests.length > 0) { - const testResult = await emitProgram({ + const testResult = await emitProgramWithPerf("emitProgram.tests", { graph, semantics, codegenOptions: { @@ -155,17 +203,17 @@ export const compileWithLoader = async ({ }); const allDiagnostics = [...baseDiagnostics, ...testResult.diagnostics]; if (hasErrorDiagnostics(allDiagnostics)) { - return { success: false, diagnostics: allDiagnostics }; + return complete({ success: false, diagnostics: allDiagnostics }); } testsWasm = testResult.wasm; } - return { + return complete(await finalize({ success: true, wasm: wasmResult.wasm, tests: testCases, testsWasm, - }; + })); } catch (error) { const codegenDiagnostics = error instanceof DiagnosticError @@ -175,9 +223,9 @@ export const compileWithLoader = async ({ moduleId: graph.entry ?? entryPath, }), ]; - return { + return complete({ success: false, diagnostics: [...diagnostics, ...codegenDiagnostics], - }; + }); } }; diff --git a/scripts/bench-v326.ts b/scripts/bench-v326.ts index d0bd051e..560ab83b 100644 --- a/scripts/bench-v326.ts +++ b/scripts/bench-v326.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import path from "node:path"; import { performance } from "node:perf_hooks"; import { gzipSync } from "node:zlib"; -import { createSdk, type CompileResult } from "@voyd-lang/sdk"; +import { createSdk, type CompileResult, type ModuleRoots } from "@voyd-lang/sdk"; import { createVoydHost } from "@voyd-lang/sdk/js-host"; type Scenario = { @@ -13,8 +13,10 @@ type Scenario = { optional?: boolean; source?: string; entryPath?: string; + roots?: ModuleRoots; entryName?: string; - expected?: number; + expected?: unknown; + expectedJsonSha256?: string; }; type ScenarioResult = { @@ -28,6 +30,9 @@ type ScenarioResult = { wasmTextBytes: number; structNewCount: number; structNewDefaultCount: number; + arrayNewCount: number; + refCastCount: number; + callRefCount: number; tupleMakeCount: number; medianMs?: number; samplesMs: number[]; @@ -138,6 +143,8 @@ const optimizeModes = (process.env.VOYD_BENCH_OPTIMIZE_MODES ?? "false,true") throw new Error(`invalid VOYD_BENCH_OPTIMIZE_MODES entry ${value}`); }); const revisionLabel = process.env.VOYD_BENCH_REVISION ?? "worktree"; +const repoRoot = path.join(import.meta.dirname, ".."); +const smokeFixtureRoot = path.join(repoRoot, "apps", "smoke", "fixtures"); const maybeFileScenario = (scenario: Scenario): Scenario[] => scenario.entryPath && !fs.existsSync(scenario.entryPath) ? [] : [scenario]; @@ -201,29 +208,36 @@ const scenarios: Scenario[] = [ }, { name: "representative/vtrace-compute-main", - entryPath: path.join( - import.meta.dirname, - "..", - "apps", - "smoke", - "fixtures", - "vtrace-compute-benchmark.voyd", - ), + entryPath: path.join(smokeFixtureRoot, "vtrace-compute-benchmark.voyd"), entryName: "main", expected: 3_825_271, iterations: representativeIterations, warmups: 1, }, + { + name: "representative/web-framework-route-probe", + entryPath: path.join(smokeFixtureRoot, "web-framework.voyd"), + roots: { + src: smokeFixtureRoot, + pkgDirs: [path.join(repoRoot, "packages")], + }, + entryName: "route_probe", + expected: 405, + iterations: representativeIterations, + warmups: 1, + }, + { + name: "representative/vx-main", + entryPath: path.join(smokeFixtureRoot, "vx.voyd"), + entryName: "main", + expectedJsonSha256: + "29db61d8716594c93e18f6ebe7f72eb2ebc9c3fdef0e8e554066e11011c6507b", + iterations: representativeIterations, + warmups: 1, + }, { name: "representative/scalar-aggregate-particle-step", - entryPath: path.join( - import.meta.dirname, - "..", - "apps", - "smoke", - "fixtures", - "scalar-aggregate-representative.voyd", - ), + entryPath: path.join(smokeFixtureRoot, "scalar-aggregate-representative.voyd"), entryName: "main", expected: 1_100_340_000, iterations: representativeIterations, @@ -267,6 +281,34 @@ const median = (values: readonly number[]): number | undefined => { const countMatches = (source: string, pattern: RegExp): number => source.match(pattern)?.length ?? 0; +const assertScenarioResult = ({ + scenario, + result, + phase, +}: { + scenario: Scenario; + result: unknown; + phase: "warmup" | "iteration"; +}): void => { + if (scenario.expected !== undefined && result !== scenario.expected) { + throw new Error( + `${scenario.name} returned ${String(result)} during ${phase}, expected ${String( + scenario.expected, + )}`, + ); + } + if (scenario.expectedJsonSha256) { + const actualHash = createHash("sha256") + .update(JSON.stringify(result)) + .digest("hex"); + if (actualHash !== scenario.expectedJsonSha256) { + throw new Error( + `${scenario.name} returned JSON hash ${actualHash} during ${phase}, expected ${scenario.expectedJsonSha256}`, + ); + } + } +}; + const runScenario = async ({ scenario, optimize, @@ -280,6 +322,7 @@ const runScenario = async ({ await sdk.compile({ entryPath: scenario.entryPath, source: scenario.source, + roots: scenario.roots, optimize, emitWasmText: true, }), @@ -288,25 +331,21 @@ const runScenario = async ({ const compileMs = performance.now() - compileStartedAt; const samplesMs: number[] = []; - if (scenario.entryName && typeof scenario.expected === "number") { + if ( + scenario.entryName && + (scenario.expected !== undefined || scenario.expectedJsonSha256) + ) { const host = await createVoydHost({ wasm: compiled.wasm }); for (let warmup = 0; warmup < scenario.warmups; warmup += 1) { - const result = await host.run(scenario.entryName); - if (result !== scenario.expected) { - throw new Error( - `${scenario.name} returned ${result} during warmup, expected ${scenario.expected}`, - ); - } + const result = await host.run(scenario.entryName); + assertScenarioResult({ scenario, result, phase: "warmup" }); } for (let iteration = 0; iteration < scenario.iterations; iteration += 1) { const startedAt = performance.now(); - const result = await host.run(scenario.entryName); - if (result !== scenario.expected) { - throw new Error( - `${scenario.name} returned ${result}, expected ${scenario.expected}`, - ); - } - samplesMs.push(performance.now() - startedAt); + const result = await host.run(scenario.entryName); + const elapsedMs = performance.now() - startedAt; + assertScenarioResult({ scenario, result, phase: "iteration" }); + samplesMs.push(elapsedMs); } } @@ -324,6 +363,12 @@ const runScenario = async ({ compiled.wasmText ?? "", /\(struct\.new_default /g, ), + arrayNewCount: countMatches( + compiled.wasmText ?? "", + /\(array\.new(?:_[a-z_]+)?[\s)]/g, + ), + refCastCount: countMatches(compiled.wasmText ?? "", /\(ref\.cast/g), + callRefCount: countMatches(compiled.wasmText ?? "", /\(call_ref\b/g), tupleMakeCount: countMatches(compiled.wasmText ?? "", /\(tuple\.make/g), medianMs: median(samplesMs), samplesMs, @@ -332,7 +377,7 @@ const runScenario = async ({ const printResults = (results: readonly ScenarioResult[]): void => { console.log( - "revision,name,optimize,compileMs,wasmBytes,gzipBytes,wasmTextBytes,structNewCount,structNewDefaultCount,tupleMakeCount,wasmSha256,medianMs,samplesMs", + "revision,name,optimize,compileMs,wasmBytes,gzipBytes,wasmTextBytes,structNewCount,structNewDefaultCount,arrayNewCount,refCastCount,callRefCount,tupleMakeCount,wasmSha256,medianMs,samplesMs", ); results.forEach((result) => { console.log( @@ -346,6 +391,9 @@ const printResults = (results: readonly ScenarioResult[]): void => { result.wasmTextBytes.toString(), result.structNewCount.toString(), result.structNewDefaultCount.toString(), + result.arrayNewCount.toString(), + result.refCastCount.toString(), + result.callRefCount.toString(), result.tupleMakeCount.toString(), result.wasmSha256, result.medianMs === undefined ? "" : result.medianMs.toFixed(3), @@ -382,6 +430,9 @@ const parseResultsCsv = (filePath: string): ScenarioResult[] => { const wasmTextBytesIndex = index("wasmTextBytes"); const structNewCountIndex = index("structNewCount"); const structNewDefaultCountIndex = index("structNewDefaultCount"); + const arrayNewCountIndex = index("arrayNewCount"); + const refCastCountIndex = index("refCastCount"); + const callRefCountIndex = index("callRefCount"); const tupleMakeCountIndex = index("tupleMakeCount"); const wasmSha256Index = index("wasmSha256"); const medianMsIndex = index("medianMs"); @@ -403,6 +454,9 @@ const parseResultsCsv = (filePath: string): ScenarioResult[] => { columns[structNewDefaultCountIndex] ?? "0", 10, ), + arrayNewCount: Number.parseInt(columns[arrayNewCountIndex] ?? "0", 10), + refCastCount: Number.parseInt(columns[refCastCountIndex] ?? "0", 10), + callRefCount: Number.parseInt(columns[callRefCountIndex] ?? "0", 10), tupleMakeCount: Number.parseInt(columns[tupleMakeCountIndex] ?? "0", 10), wasmSha256: columns[wasmSha256Index] ?? "", medianMs: @@ -444,7 +498,7 @@ const printComparison = ({ ); console.log( - "name,optimize,baseRevision,headRevision,compileMsBase,compileMsHead,compileMsDeltaPct,medianMsBase,medianMsHead,medianMsDeltaPct,wasmBytesBase,wasmBytesHead,wasmBytesDeltaPct,gzipBytesBase,gzipBytesHead,gzipBytesDeltaPct,wasmTextBytesBase,wasmTextBytesHead,wasmTextBytesDeltaPct,structNewBase,structNewHead,structNewDelta,structNewDefaultBase,structNewDefaultHead,structNewDefaultDelta,tupleMakeBase,tupleMakeHead,tupleMakeDelta,baseWasmSha256,headWasmSha256", + "name,optimize,baseRevision,headRevision,compileMsBase,compileMsHead,compileMsDeltaPct,medianMsBase,medianMsHead,medianMsDeltaPct,wasmBytesBase,wasmBytesHead,wasmBytesDeltaPct,gzipBytesBase,gzipBytesHead,gzipBytesDeltaPct,wasmTextBytesBase,wasmTextBytesHead,wasmTextBytesDeltaPct,structNewBase,structNewHead,structNewDelta,structNewDefaultBase,structNewDefaultHead,structNewDefaultDelta,arrayNewBase,arrayNewHead,arrayNewDelta,refCastBase,refCastHead,refCastDelta,callRefBase,callRefHead,callRefDelta,tupleMakeBase,tupleMakeHead,tupleMakeDelta,baseWasmSha256,headWasmSha256", ); matched.forEach(({ base, head }) => { console.log( @@ -476,6 +530,15 @@ const printComparison = ({ base.structNewDefaultCount.toString(), head.structNewDefaultCount.toString(), (head.structNewDefaultCount - base.structNewDefaultCount).toString(), + base.arrayNewCount.toString(), + head.arrayNewCount.toString(), + (head.arrayNewCount - base.arrayNewCount).toString(), + base.refCastCount.toString(), + head.refCastCount.toString(), + (head.refCastCount - base.refCastCount).toString(), + base.callRefCount.toString(), + head.callRefCount.toString(), + (head.callRefCount - base.callRefCount).toString(), base.tupleMakeCount.toString(), head.tupleMakeCount.toString(), (head.tupleMakeCount - base.tupleMakeCount).toString(),