diff --git a/chainsafe-swap-or-not-shuffle-1.2.1-local.tgz b/chainsafe-swap-or-not-shuffle-1.2.1-local.tgz new file mode 100644 index 000000000000..1a752674ee7a Binary files /dev/null and b/chainsafe-swap-or-not-shuffle-1.2.1-local.tgz differ diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 2b7df08b80d5..05f3eea37999 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -165,7 +165,7 @@ "xxhash-wasm": "1.0.2" }, "devDependencies": { - "@chainsafe/swap-or-not-shuffle": "^1.2.1", + "@chainsafe/swap-or-not-shuffle": "file:../../chainsafe-swap-or-not-shuffle-1.2.1-local.tgz", "@libp2p/interface-internal": "^3.0.13", "@libp2p/logger": "^6.2.2", "@libp2p/utils": "^7.0.13", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index ee618fd55b3e..d2a232bf3474 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -66,7 +66,7 @@ "@chainsafe/persistent-ts": "^1.0.0", "@chainsafe/pubkey-index-map": "^3.0.0", "@chainsafe/ssz": "^1.4.0", - "@chainsafe/swap-or-not-shuffle": "^1.2.1", + "@chainsafe/swap-or-not-shuffle": "file:../../chainsafe-swap-or-not-shuffle-1.2.1-local.tgz", "@lodestar/config": "workspace:^", "@lodestar/params": "workspace:^", "@lodestar/types": "workspace:^", diff --git a/packages/state-transition/src/epoch/processPtcWindow.ts b/packages/state-transition/src/epoch/processPtcWindow.ts index bdb629b00978..deefe0807ed0 100644 --- a/packages/state-transition/src/epoch/processPtcWindow.ts +++ b/packages/state-transition/src/epoch/processPtcWindow.ts @@ -21,7 +21,7 @@ export function processPtcWindow(state: CachedBeaconStateGloas, cache: EpochTran const newNextPayloadTimelinessCommittees = computePayloadTimelinessCommitteesForEpoch( state, nextEpoch, - nextEpochShuffling.committees, + nextEpochShuffling, state.epochCtx.effectiveBalanceIncrements ); diff --git a/packages/state-transition/src/util/gloas.ts b/packages/state-transition/src/util/gloas.ts index 6f1989acda0b..b93b774c4509 100644 --- a/packages/state-transition/src/util/gloas.ts +++ b/packages/state-transition/src/util/gloas.ts @@ -191,7 +191,7 @@ export function initializePtcWindow(state: CachedBeaconStateFulu): Uint32Array[] ...computePayloadTimelinessCommitteesForEpoch( state, epoch, - shuffling.committees, + shuffling, state.epochCtx.effectiveBalanceIncrements ) ); diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index be01506f6cd4..791ab4c3d62d 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -1,5 +1,7 @@ import {digest} from "@chainsafe/as-sha256"; import { + computePtcIndices, + computePtcIndicesForEpoch, computeProposerIndex as nativeComputeProposerIndex, computeSyncCommitteeIndices as nativeComputeSyncCommitteeIndices, } from "@chainsafe/swap-or-not-shuffle"; @@ -23,6 +25,7 @@ import {assert, bytesToBigInt, bytesToInt, intToBytes} from "@lodestar/utils"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; import {computeEpochAtSlot, computeStartSlotAtEpoch} from "./epoch.js"; +import {EpochShuffling} from "./epochShuffling.js"; /** * Compute proposer indices for an epoch @@ -269,9 +272,12 @@ export function getNextSyncCommitteeIndices( } /** - * Compute PTC for all slots in an epoch eagerly. + * Naive JS version of `computePayloadTimelinessCommitteesForEpoch`. + * Used to verify the optimized Rust-backed version. Not for production use. + * + * SLOW CODE - 🐢 */ -export function computePayloadTimelinessCommitteesForEpoch( +export function naiveComputePayloadTimelinessCommitteesForEpoch( state: BeaconStateAllForks, epoch: number, committees: Uint32Array[][], @@ -293,15 +299,60 @@ export function computePayloadTimelinessCommitteesForEpoch( slotSeedView.setUint32(epochSeed.length + 4, 0, true); const slotSeed = digest(slotSeedInput); - result[i] = computePayloadTimelinessCommitteeForSlot(slotSeed, committees[i], effectiveBalanceIncrements); + result[i] = naiveComputePayloadTimelinessCommitteeForSlot(slotSeed, committees[i], effectiveBalanceIncrements); } return result; } /** - * Compute PTC for a single slot. + * Compute PTC for all slots in an epoch eagerly. */ -export function computePayloadTimelinessCommitteeForSlot( +export function computePayloadTimelinessCommitteesForEpoch( + state: BeaconStateAllForks, + epoch: number, + epochShuffling: EpochShuffling, + effectiveBalanceIncrements: EffectiveBalanceIncrements +): Uint32Array[] { + const epochSeed = getSeed(state, epoch, DOMAIN_PTC_ATTESTER); + const startSlot = epoch * SLOTS_PER_EPOCH; + const {committees, shuffling} = epochShuffling; + + const slotOffsets = new Uint32Array(SLOTS_PER_EPOCH + 1); + for (let i = 0; i < SLOTS_PER_EPOCH; i++) { + let slotLen = 0; + for (const c of committees[i]) slotLen += c.length; + slotOffsets[i + 1] = slotOffsets[i] + slotLen; + } + if (shuffling.length === 0) { + throw Error("Validator indices must not be empty"); + } + + const flat = computePtcIndicesForEpoch( + epochSeed, + startSlot, + SLOTS_PER_EPOCH, + shuffling, + slotOffsets, + effectiveBalanceIncrements, + PTC_SIZE, + MAX_EFFECTIVE_BALANCE_ELECTRA, + EFFECTIVE_BALANCE_INCREMENT + ); + + const result = new Array(SLOTS_PER_EPOCH); + for (let i = 0; i < SLOTS_PER_EPOCH; i++) { + result[i] = flat.subarray(i * PTC_SIZE, (i + 1) * PTC_SIZE); + } + return result; +} + +/** + * Naive JS version of `computePayloadTimelinessCommitteeForSlot`. + * Used to verify the optimized Rust-backed version. Not for production use. + * + * SLOW CODE - 🐢 + */ +export function naiveComputePayloadTimelinessCommitteeForSlot( slotSeed: Uint8Array, slotCommittees: Uint32Array[], effectiveBalanceIncrements: EffectiveBalanceIncrements @@ -317,6 +368,31 @@ export function computePayloadTimelinessCommitteeForSlot( return computePayloadTimelinessCommitteeIndices(effectiveBalanceIncrements, allIndices, slotSeed); } +/** + * Compute PTC for a single slot. + */ +export function computePayloadTimelinessCommitteeForSlot( + slotSeed: Uint8Array, + slotCommittees: Uint32Array[], + effectiveBalanceIncrements: EffectiveBalanceIncrements +): Uint32Array { + const totalLen = slotCommittees.reduce((sum, c) => sum + c.length, 0); + const allIndices = new Uint32Array(totalLen); + let offset = 0; + for (const c of slotCommittees) { + allIndices.set(c, offset); + offset += c.length; + } + return computePtcIndices( + slotSeed, + allIndices, + effectiveBalanceIncrements, + PTC_SIZE, + MAX_EFFECTIVE_BALANCE_ELECTRA, + EFFECTIVE_BALANCE_INCREMENT + ); +} + /** * Optimized version of PTC indices computation. * Avoids BigInt conversions and uses DataView for efficient byte reading. diff --git a/packages/state-transition/test/perf/util/seed.test.ts b/packages/state-transition/test/perf/util/seed.test.ts index cd3aa18677a4..95669bebeda2 100644 --- a/packages/state-transition/test/perf/util/seed.test.ts +++ b/packages/state-transition/test/perf/util/seed.test.ts @@ -1,11 +1,15 @@ import {bench, describe} from "@chainsafe/benchmark"; import {ForkSeq} from "@lodestar/params"; import {fromHex} from "@lodestar/utils"; -import {generatePerfTestCachedStateAltair} from "../../../src/testUtils/util.js"; +import {generatePerfTestCachedStateAltair, generatePerfTestCachedStateElectra} from "../../../src/testUtils/util.js"; import { + computePayloadTimelinessCommitteeForSlot, + computePayloadTimelinessCommitteesForEpoch, computeProposerIndex, computeShuffledIndex, getNextSyncCommitteeIndices, + naiveComputePayloadTimelinessCommitteeForSlot, + naiveComputePayloadTimelinessCommitteesForEpoch, naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, } from "../../../src/util/seed.js"; @@ -89,3 +93,89 @@ describe("computeShuffledIndex", () => { }); } }); + +describe("computePayloadTimelinessCommitteeForSlot - pure TS vs Rust magic (250k-1M validators)", () => { + for (const vc of [250_000, 1_000_000]) { + const seed = new Uint8Array(32).fill(1); + const indices = new Uint32Array(Array.from({length: vc}, (_, i) => i)); + const effectiveBalanceIncrements = new Uint16Array(vc).fill(32); + const slotCommittees = [indices]; // single committee spanning all validators + + bench({ + id: `naive TS - naiveComputePayloadTimelinessCommitteeForSlot - ${vc} validators`, + fn: () => { + naiveComputePayloadTimelinessCommitteeForSlot(seed, slotCommittees, effectiveBalanceIncrements); + }, + }); + + bench({ + id: `(uses native Rust) -computePayloadTimelinessCommitteeForSlot - ${vc} validators`, + fn: () => { + computePayloadTimelinessCommitteeForSlot(seed, slotCommittees, effectiveBalanceIncrements); + }, + }); + } +}); + +describe("computePayloadTimelinessCommitteesForEpoch - pure TS vs Rust magic (250k -1M validators)", () => { + for (const vc of [250_000, 1_000_000]) { + const cachedState = generatePerfTestCachedStateElectra({goBackOneSlot: false, vc}); + const {epochCtx} = cachedState; + const epoch = epochCtx.epoch; + const {effectiveBalanceIncrements} = epochCtx; + + // eslint-disable-next-line no-console + console.log(`[vc=${vc}] effectiveBalanceIncrements[0]=${effectiveBalanceIncrements[0]}`); + + const naiveResult = naiveComputePayloadTimelinessCommitteesForEpoch( + cachedState, + epoch, + epochCtx.currentShuffling.committees, + effectiveBalanceIncrements + ); + const rustResult = computePayloadTimelinessCommitteesForEpoch( + cachedState, + epoch, + epochCtx.currentShuffling, + effectiveBalanceIncrements + ); + for (let i = 0; i < naiveResult.length; i++) { + const naive = naiveResult[i]; + const rust = rustResult[i]; + if (naive.length !== rust.length) { + throw new Error(`PTC length mismatch at slot ${i} (vc=${vc}): naive=${naive.length} rust=${rust.length}`); + } + for (let j = 0; j < naive.length; j++) { + if (naive[j] !== rust[j]) { + throw new Error( + `PTC index mismatch at slot ${i} position ${j} (vc=${vc}): naive=${naive[j]} rust=${rust[j]}` + ); + } + } + } + + bench({ + id: `naive TS - naiveComputePayloadTimelinessCommitteesForEpoch - ${vc} validators`, + fn: () => { + naiveComputePayloadTimelinessCommitteesForEpoch( + cachedState, + epoch, + epochCtx.currentShuffling.committees, + effectiveBalanceIncrements + ); + }, + }); + + bench({ + id: `(uses native Rust) -computePayloadTimelinessCommitteesForEpoch - ${vc} validators`, + fn: () => { + computePayloadTimelinessCommitteesForEpoch( + cachedState, + epoch, + epochCtx.currentShuffling, + effectiveBalanceIncrements + ); + }, + }); + } +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d2dd20db720..7610069b3826 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -383,8 +383,8 @@ importers: version: 1.0.2 devDependencies: '@chainsafe/swap-or-not-shuffle': - specifier: ^1.2.1 - version: 1.2.1 + specifier: file:../../chainsafe-swap-or-not-shuffle-1.2.1-local.tgz + version: file:chainsafe-swap-or-not-shuffle-1.2.1-local.tgz '@libp2p/interface-internal': specifier: ^3.0.13 version: 3.0.13 @@ -987,8 +987,8 @@ importers: specifier: ^1.4.0 version: 1.4.0 '@chainsafe/swap-or-not-shuffle': - specifier: ^1.2.1 - version: 1.2.1 + specifier: file:../../chainsafe-swap-or-not-shuffle-1.2.1-local.tgz + version: file:chainsafe-swap-or-not-shuffle-1.2.1-local.tgz '@lodestar/config': specifier: workspace:^ version: link:../config @@ -1582,60 +1582,42 @@ packages: '@chainsafe/ssz@1.4.0': resolution: {integrity: sha512-DRje073CIU0lvIdNKmW89jAo4UAidoKpgYco5wvQve8WK1DUk/YS/54c5FX/e/iiKiJ1mo44CNCg4xiKWY3/yQ==} - '@chainsafe/swap-or-not-shuffle-darwin-arm64@1.2.1': - resolution: {integrity: sha512-kTewLZe1KqMAJ1gHfagOxo0BI4kTcMOAkGJ7pRLFM5ZkL2P7sh3/4ixnjdbtMkO207vMZEZL3fxJSSs14kg9Kg==} + '@chainsafe/swap-or-not-shuffle-darwin-arm64@0.0.2': + resolution: {integrity: sha512-e2tmpSgGTHFv1g3oEKP/YElDTmxZwuSwVQ+Cf0gTUA8z4YQsJar61293My6JeA7qC5razWjhvcEk13ZbTCqk0w==} engines: {node: '>= 18'} cpu: [arm64] os: [darwin] - '@chainsafe/swap-or-not-shuffle-darwin-x64@1.2.1': - resolution: {integrity: sha512-B/f/peQqOLW5Tqib5CqanAlQgoeib75FNszzHwRBwHdRY5ThOcBbARvOtef39zU5lYjj4j3iWobpD7G70kQyUw==} - engines: {node: '>= 18'} - cpu: [x64] - os: [darwin] - - '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu@1.2.1': - resolution: {integrity: sha512-sDpuUuo3rStvHMQgLxH1UkkUbo8rcjDI70Fq3xzbNMhfjlrP0+sJistlFJtDpWmKsFk/sgje3PzVU9G3KFXzXA==} + '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu@0.0.2': + resolution: {integrity: sha512-nGTIRUXt1QRNiWQXZnA8IWiHnAw6FNkU7RkngkoDzjD8pEhrtWs8tv/pdOxRKEhw21HBvKi7z8J+a7/MtxDLTg==} engines: {node: '>= 18'} cpu: [arm64] os: [linux] libc: [glibc] - '@chainsafe/swap-or-not-shuffle-linux-arm64-musl@1.2.1': - resolution: {integrity: sha512-FFmpDF2dRhhOCT3WBYOoQW9wqhJMfx7iULskBe5cjowVQ15U1TQJ6tC+hPpouUMt3/pKz9tIS9dNWTUuo7kpQg==} + '@chainsafe/swap-or-not-shuffle-linux-arm64-musl@0.0.2': + resolution: {integrity: sha512-sumwkxQ0Mky+W66Jf43cHUybgHQ4FENj2iRBRw3jGWiZ79Vv/DZ1dMA6I4/LVWCsvZmFUIvMvKNthGHXefh2DQ==} engines: {node: '>= 18'} cpu: [arm64] os: [linux] libc: [musl] - '@chainsafe/swap-or-not-shuffle-linux-x64-gnu@1.2.1': - resolution: {integrity: sha512-nGXEnFCqRmCS7ATV7QQ7b7aKHcyET9XEpJylq0sljnkuNRwoQXFUYPMYTn4TMI6g3vgZt/AVK9RdCxp68JKwGQ==} - engines: {node: '>= 18'} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@chainsafe/swap-or-not-shuffle-linux-x64-musl@1.2.1': - resolution: {integrity: sha512-VMQxE/EZjco4damGrJkkCQ2Y2WYgJcTR/ardTnSctM1/xxwUD1wQr7YW2TQsECfK3UZcW2W6gG7iqWfJgMdGSw==} + '@chainsafe/swap-or-not-shuffle-linux-x64-musl@0.0.2': + resolution: {integrity: sha512-ClwDMZd768PIaAUQhbyZpqqip0b6Sgjt8IpP5ACYMJr2AHoiG64POZaiFu+zusT4q3s4/dvqaz86jRQaF4vFkw==} engines: {node: '>= 18'} cpu: [x64] os: [linux] libc: [musl] - '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc@1.2.1': - resolution: {integrity: sha512-qV+ps6KSoR8blc3gMse1BOBUGcCtptnMYoqTMwFCIx9LGaeSo3vKu5XBQDwHIfAW73bZ+PHb5YUqkRgtWBiEUw==} + '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc@0.0.2': + resolution: {integrity: sha512-i9+qj0VSppy2xMChQE38rCmWmmy8SJ0uSaApc0L4KZ+t2aqquYyEUXWGgfdXMZx9MqAUvuigzv34T9qivADFag==} engines: {node: '>= 18'} cpu: [arm64] os: [win32] - '@chainsafe/swap-or-not-shuffle-win32-x64-msvc@1.2.1': - resolution: {integrity: sha512-hsDN4O6PPpF4rEA1tMngMJAOrWTGAu8TAqdEwgdP+U25o8UVlz8W2N9697AMGRnnArN2BgYS0zCeUR9kuZ2XEQ==} - engines: {node: '>= 18'} - cpu: [x64] - os: [win32] - - '@chainsafe/swap-or-not-shuffle@1.2.1': - resolution: {integrity: sha512-H8YdEoXXv2Hw17gDWGOJEya4LHlBbpChJP3jDQRfIk9hhwr0c/zbBemBRmjADowZhArL+ymkO+j5hGaYySjdpw==} + '@chainsafe/swap-or-not-shuffle@file:chainsafe-swap-or-not-shuffle-1.2.1-local.tgz': + resolution: {integrity: sha512-Oi1Ou27Rj+oRBmyYmYe3bVSP6c6q1+jtRX6TRUyx0g6mUjceJ56BSbAAi5rUKYSFsJiA7rRZ7c5KVf9U0fi2XA==, tarball: file:chainsafe-swap-or-not-shuffle-1.2.1-local.tgz} + version: 1.2.1 engines: {node: '>= 18'} '@chainsafe/threads@1.11.3': @@ -8003,40 +7985,28 @@ snapshots: '@chainsafe/as-sha256': 1.2.0 '@chainsafe/persistent-merkle-tree': 1.2.5 - '@chainsafe/swap-or-not-shuffle-darwin-arm64@1.2.1': - optional: true - - '@chainsafe/swap-or-not-shuffle-darwin-x64@1.2.1': - optional: true - - '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu@1.2.1': - optional: true - - '@chainsafe/swap-or-not-shuffle-linux-arm64-musl@1.2.1': + '@chainsafe/swap-or-not-shuffle-darwin-arm64@0.0.2': optional: true - '@chainsafe/swap-or-not-shuffle-linux-x64-gnu@1.2.1': + '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu@0.0.2': optional: true - '@chainsafe/swap-or-not-shuffle-linux-x64-musl@1.2.1': + '@chainsafe/swap-or-not-shuffle-linux-arm64-musl@0.0.2': optional: true - '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc@1.2.1': + '@chainsafe/swap-or-not-shuffle-linux-x64-musl@0.0.2': optional: true - '@chainsafe/swap-or-not-shuffle-win32-x64-msvc@1.2.1': + '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc@0.0.2': optional: true - '@chainsafe/swap-or-not-shuffle@1.2.1': + '@chainsafe/swap-or-not-shuffle@file:chainsafe-swap-or-not-shuffle-1.2.1-local.tgz': optionalDependencies: - '@chainsafe/swap-or-not-shuffle-darwin-arm64': 1.2.1 - '@chainsafe/swap-or-not-shuffle-darwin-x64': 1.2.1 - '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu': 1.2.1 - '@chainsafe/swap-or-not-shuffle-linux-arm64-musl': 1.2.1 - '@chainsafe/swap-or-not-shuffle-linux-x64-gnu': 1.2.1 - '@chainsafe/swap-or-not-shuffle-linux-x64-musl': 1.2.1 - '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc': 1.2.1 - '@chainsafe/swap-or-not-shuffle-win32-x64-msvc': 1.2.1 + '@chainsafe/swap-or-not-shuffle-darwin-arm64': 0.0.2 + '@chainsafe/swap-or-not-shuffle-linux-arm64-gnu': 0.0.2 + '@chainsafe/swap-or-not-shuffle-linux-arm64-musl': 0.0.2 + '@chainsafe/swap-or-not-shuffle-linux-x64-musl': 0.0.2 + '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc': 0.0.2 '@chainsafe/threads@1.11.3': dependencies: