diff --git a/.github/crates-filters.yml b/.github/crates-filters.yml index 5abcff77e1c..ee8afea3d49 100644 --- a/.github/crates-filters.yml +++ b/.github/crates-filters.yml @@ -217,12 +217,6 @@ iota-move-natives-latest: iota-verifier-latest: - "iota-execution/latest/iota-verifier/**" -# Mysticeti Consensus -consensus-config: - - "consensus/config/**" -consensus-core: - - "consensus/core/**" - # Starfish Consensus starfish-config: - "crates/starfish/config/**" diff --git a/.github/workflows/_rust.yml b/.github/workflows/_rust.yml index d99ca070630..15e0965408b 100644 --- a/.github/workflows/_rust.yml +++ b/.github/workflows/_rust.yml @@ -86,7 +86,6 @@ jobs: uses: ./.github/workflows/_rust_tests.yml with: isRust: ${{ inputs.isRust }} - consensusProtocol: "mysticeti" isPgIntegration: ${{ inputs.isPgIntegration }} isMoveExampleUsedByOthers: ${{ inputs.isMoveExampleUsedByOthers }} runSimtest: true diff --git a/.github/workflows/_rust_tests.yml b/.github/workflows/_rust_tests.yml index d4a37ab172c..b0022c5e0c6 100644 --- a/.github/workflows/_rust_tests.yml +++ b/.github/workflows/_rust_tests.yml @@ -24,10 +24,6 @@ on: changedExternalCrates: type: string required: false - consensusProtocol: - type: string - required: false - default: "mysticeti" isNightly: type: boolean default: false @@ -37,7 +33,6 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} env: - CONSENSUS_PROTOCOL: ${{ inputs.consensusProtocol }} CARGO_TERM_COLOR: always RUST_LOG: "error" # Don't emit giant backtraces in the CI logs. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 55a8b4d603f..cad4e99605b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -83,20 +83,6 @@ jobs: testOnlyChangedCrates: false isNightly: true - tests-with-starfish: - if: ${{ inputs.rustTestsOnly || !inputs.simTestsOnly }} - uses: ./.github/workflows/_rust_tests.yml - with: - isRust: true - isPgIntegration: true - isMoveExampleUsedByOthers: true - consensusProtocol: "starfish" - # simtest job below runs a superset of these tests - runSimtest: false - # run all tests by default - testOnlyChangedCrates: false - isNightly: true - deny: if: ${{ !inputs.rustTestsOnly && !inputs.simTestsOnly }} uses: ./.github/workflows/_cargo_deny.yml @@ -125,28 +111,6 @@ jobs: if: ${{ inputs.simTestsOnly || !inputs.rustTestsOnly }} timeout-minutes: 600 runs-on: [self-hosted-x64] - env: - CONSENSUS_PROTOCOL: mysticeti - - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - with: - ref: ${{ env.IOTA_REF }} - - - name: Get latest simtest script from commit that triggered this workflow - run: | - git fetch origin ${{ github.sha }} - git checkout ${{ github.sha }} -- scripts/simtest/simtest-run.sh - - - name: Run simtest - run: scripts/simtest/simtest-run.sh - - simtest-with-starfish: - if: ${{ inputs.simTestsOnly || !inputs.rustTestsOnly }} - timeout-minutes: 600 - runs-on: [self-hosted-x64] - env: - CONSENSUS_PROTOCOL: starfish steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 diff --git a/Cargo.lock b/Cargo.lock index c0ca9935ccf..3611c7ec1d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2566,96 +2566,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "consensus-config" -version = "0.1.0" -dependencies = [ - "fastcrypto", - "insta", - "iota-network-stack", - "iota-sdk-types", - "rand 0.8.5", - "serde", - "tracing", -] - -[[package]] -name = "consensus-core" -version = "0.1.0" -dependencies = [ - "async-trait", - "base64 0.21.7", - "bcs", - "bytes", - "cfg-if", - "consensus-config", - "enum_dispatch", - "fastcrypto", - "futures", - "http 1.4.0", - "iota-common", - "iota-http", - "iota-macros", - "iota-metrics", - "iota-network-stack", - "iota-protocol-config", - "iota-sdk-types", - "iota-tls", - "itertools 0.13.0", - "lru 0.12.4", - "nom", - "parking_lot 0.12.3", - "prometheus", - "prost 0.14.1", - "rand 0.8.5", - "rstest", - "serde", - "strum_macros 0.27.2", - "tap", - "telemetry-subscribers", - "tempfile", - "thiserror 1.0.64", - "tokio", - "tokio-rustls 0.26.2", - "tokio-stream", - "tokio-util 0.7.18", - "tonic 0.14.2", - "tonic-build 0.14.2", - "tonic-prost", - "tonic-rustls", - "tower 0.4.13", - "tower-http 0.5.2", - "tracing", - "typed-store", -] - -[[package]] -name = "consensus-simtests" -version = "0.1.0" -dependencies = [ - "anyhow", - "arc-swap", - "consensus-config", - "consensus-core", - "iota-common", - "iota-config", - "iota-macros", - "iota-metrics", - "iota-network-stack", - "iota-protocol-config", - "iota-simulator", - "parking_lot 0.12.3", - "prometheus", - "rand 0.8.5", - "telemetry-subscribers", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util 0.7.18", - "tracing", - "typed-store", -] - [[package]] name = "console" version = "0.15.8" @@ -6034,7 +5944,6 @@ dependencies = [ "anyhow", "bcs", "clap", - "consensus-config", "csv", "dirs", "fastcrypto", @@ -6067,8 +5976,6 @@ dependencies = [ "bincode", "bytes", "clap", - "consensus-config", - "consensus-core", "count-min-sketch", "criterion", "dashmap", @@ -8371,7 +8278,6 @@ dependencies = [ "better_any", "bincode", "byteorder", - "consensus-config", "coset", "criterion", "derive_more 1.0.0", @@ -12468,7 +12374,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2 0.6.1", - "tokio-macros 2.6.0 (git+https://github.com/iotaledger/tokio-madsim-fork.git?rev=6b1da10b18f9a4b1d42b1432415a0b3092908c77)", + "tokio-macros 2.6.0", "windows-sys 0.61.2", ] @@ -14797,9 +14703,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -14808,7 +14714,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "socket2 0.6.1", - "tokio-macros 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-macros 2.6.1", "tracing", "windows-sys 0.61.2", ] @@ -14816,8 +14722,7 @@ dependencies = [ [[package]] name = "tokio-macros" version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +source = "git+https://github.com/iotaledger/tokio-madsim-fork.git?rev=6b1da10b18f9a4b1d42b1432415a0b3092908c77#6b1da10b18f9a4b1d42b1432415a0b3092908c77" dependencies = [ "proc-macro2", "quote", @@ -14826,8 +14731,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" -source = "git+https://github.com/iotaledger/tokio-madsim-fork.git?rev=6b1da10b18f9a4b1d42b1432415a0b3092908c77#6b1da10b18f9a4b1d42b1432415a0b3092908c77" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 25a53985a86..b0c062c0a5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,9 +62,6 @@ exclude = [ "sdk/move-bytecode-template", ] members = [ - "consensus/config", - "consensus/core", - "consensus/simtests", "crates/bin-version", "crates/iota", "crates/iota-adapter-transactional-tests", @@ -391,9 +388,6 @@ uuid = { version = "1.1.2", features = ["v4", "fast-rng"] } # internal dependencies bin-version = { path = "crates/bin-version" } -consensus-config = { path = "consensus/config" } -consensus-core = { path = "consensus/core" } -consensus-simtests = { path = "consensus/simtests" } iota = { path = "crates/iota" } iota-adapter-transactional-tests = { path = "crates/iota-adapter-transactional-tests" } iota-analytics-indexer = { path = "crates/iota-analytics-indexer" } diff --git a/consensus/.clippy.toml b/consensus/.clippy.toml deleted file mode 100644 index cb2ede8780d..00000000000 --- a/consensus/.clippy.toml +++ /dev/null @@ -1,22 +0,0 @@ -# cyclomatic complexity is not always useful -cognitive-complexity-threshold = 100 -# types are used for safety encoding -type-complexity-threshold = 10000 -# big constructors -too-many-arguments-threshold = 20 - -disallowed-methods = [ - # known to cause blocking issues - { path = "futures::executor::block_on", reason = "use tokio::runtime::runtime::Runtime::block_on instead" }, - # bincode::deserialize_from is easy to shoot your foot with - { path = "bincode::deserialize_from", reason = "use bincode::deserialize instead" }, -] - -disallowed-macros = [ - { path = "anyhow::anyhow", reason = "we prefer to use eyre" }, - # we use tracing with the log feature instead of the log crate. - { path = "log::info", reason = "use tracing::info instead" }, - { path = "log::debug", reason = "use tracing::debug instead" }, - { path = "log::error", reason = "use tracing::error instead" }, - { path = "log::warn", reason = "use tracing::warn instead" }, -] diff --git a/consensus/config/Cargo.toml b/consensus/config/Cargo.toml deleted file mode 100644 index 64f83f3d418..00000000000 --- a/consensus/config/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "consensus-config" -version = "0.1.0" -authors = ["IOTA Foundation "] -edition = "2021" -license = "Apache-2.0" -publish = false - -[lints] -workspace = true - -[dependencies] -# external dependencies -fastcrypto = { workspace = true, features = ["copy_key"] } -rand.workspace = true -serde.workspace = true -tracing.workspace = true - -# internal dependencies -iota-network-stack.workspace = true -iota-sdk-types.workspace = true - -[dev-dependencies] -insta.workspace = true diff --git a/consensus/config/README.md b/consensus/config/README.md deleted file mode 100644 index 385fc555c1d..00000000000 --- a/consensus/config/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This crate contains types for both on-chain and off-chain consensus configs. -It should be kept with minimal dependencies. diff --git a/consensus/config/src/committee.rs b/consensus/config/src/committee.rs deleted file mode 100644 index 5aae41b70e7..00000000000 --- a/consensus/config/src/committee.rs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - fmt::{Display, Formatter}, - ops::{Index, IndexMut}, -}; - -use iota_network_stack::Multiaddr; -use serde::{Deserialize, Serialize}; - -use crate::{AuthorityPublicKey, NetworkPublicKey, ProtocolPublicKey}; - -/// Committee of the consensus protocol is updated each epoch. -pub type Epoch = u64; - -/// Voting power of an authority, roughly proportional to the actual amount of -/// IOTA staked by the authority. -/// Total stake / voting power of all authorities should sum to 10,000. -pub type Stake = u64; - -/// Committee is the set of authorities that participate in the consensus -/// protocol for this epoch. Its configuration is stored and computed on chain. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Committee { - /// The epoch number of this committee - epoch: Epoch, - /// Total stake in the committee. - total_stake: Stake, - /// The quorum threshold (2f+1). - quorum_threshold: Stake, - /// The validity threshold (f+1). - validity_threshold: Stake, - /// Protocol and network info of each authority. - authorities: Vec, -} - -impl Committee { - pub fn new(epoch: Epoch, authorities: Vec) -> Self { - assert!(!authorities.is_empty(), "Committee cannot be empty!"); - assert!( - authorities.len() < u32::MAX as usize, - "Too many authorities ({})!", - authorities.len() - ); - - let total_stake = authorities.iter().map(|a| a.stake).sum::(); - assert_ne!(total_stake, 0, "Total stake cannot be zero!"); - let quorum_threshold = 2 * total_stake / 3 + 1; - let validity_threshold = total_stake.div_ceil(3); - Self { - epoch, - total_stake, - quorum_threshold, - validity_threshold, - authorities, - } - } - - // ----------------------------------------------------------------------- - // Accessors to Committee fields. - - pub fn epoch(&self) -> Epoch { - self.epoch - } - - pub fn total_stake(&self) -> Stake { - self.total_stake - } - - pub fn quorum_threshold(&self) -> Stake { - self.quorum_threshold - } - - pub fn validity_threshold(&self) -> Stake { - self.validity_threshold - } - - pub fn stake(&self, authority_index: AuthorityIndex) -> Stake { - self.authorities[authority_index].stake - } - - pub fn authority(&self, authority_index: AuthorityIndex) -> &Authority { - &self.authorities[authority_index] - } - - pub fn authorities(&self) -> impl Iterator { - self.authorities - .iter() - .enumerate() - .map(|(i, a)| (AuthorityIndex(i as u32), a)) - } - - // ----------------------------------------------------------------------- - // Helpers for Committee properties. - - /// Returns true if the provided stake has reached quorum (2f+1). - pub fn reached_quorum(&self, stake: Stake) -> bool { - stake >= self.quorum_threshold() - } - - /// Returns true if the provided stake has reached validity (f+1). - pub fn reached_validity(&self, stake: Stake) -> bool { - stake >= self.validity_threshold() - } - - /// Coverts an index to an AuthorityIndex, if valid. - /// Returns None if index is out of bound. - pub fn to_authority_index(&self, index: usize) -> Option { - if index < self.authorities.len() { - Some(AuthorityIndex(index as u32)) - } else { - None - } - } - - /// Returns true if the provided index is valid. - pub fn is_valid_index(&self, index: AuthorityIndex) -> bool { - index.value() < self.size() - } - - /// Returns number of authorities in the committee. - pub fn size(&self) -> usize { - self.authorities.len() - } -} - -/// Represents one authority in the committee. -/// -/// NOTE: this is intentionally un-cloneable, to encourage only copying relevant -/// fields. AuthorityIndex should be used to reference an authority instead. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Authority { - /// Voting power of the authority in the committee. - pub stake: Stake, - /// Network address for communicating with the authority. - pub address: Multiaddr, - /// The authority's hostname, for metrics and logging. - pub hostname: String, - /// The public key bytes corresponding to the private key that the validator - /// holds to sign transactions. - pub authority_key: AuthorityPublicKey, - /// The public key bytes corresponding to the private key that the validator - /// holds to sign consensus blocks. - pub protocol_key: ProtocolPublicKey, - /// The public key bytes corresponding to the private key that the validator - /// uses to establish TLS connections. - pub network_key: NetworkPublicKey, -} - -/// Each authority is uniquely identified by its AuthorityIndex in the -/// Committee. AuthorityIndex is between 0 (inclusive) and the total number of -/// authorities (exclusive). -/// -/// NOTE: for safety, invalid AuthorityIndex should be impossible to create. So -/// AuthorityIndex should not be created or incremented outside of this file. -/// AuthorityIndex received from peers should be validated before use. -#[derive( - Eq, PartialEq, Ord, PartialOrd, Clone, Copy, Debug, Default, Hash, Serialize, Deserialize, -)] -pub struct AuthorityIndex(u32); - -impl AuthorityIndex { - // Minimum committee size is 1, so 0 index is always valid. - pub const ZERO: Self = Self(0); - - // Only for scanning rows in the database. Invalid elsewhere. - pub const MIN: Self = Self::ZERO; - pub const MAX: Self = Self(u32::MAX); - - pub fn value(&self) -> usize { - self.0 as usize - } -} - -impl AuthorityIndex { - pub fn new_for_test(index: u32) -> Self { - Self(index) - } -} - -impl Display for AuthorityIndex { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "[{}]", self.value()) - } -} - -impl Index for [T; N] { - type Output = T; - - fn index(&self, index: AuthorityIndex) -> &Self::Output { - self.get(index.value()).unwrap() - } -} - -impl Index for Vec { - type Output = T; - - fn index(&self, index: AuthorityIndex) -> &Self::Output { - self.get(index.value()).unwrap() - } -} - -impl IndexMut for [T; N] { - fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output { - self.get_mut(index.value()).unwrap() - } -} - -impl IndexMut for Vec { - fn index_mut(&mut self, index: AuthorityIndex) -> &mut Self::Output { - self.get_mut(index.value()).unwrap() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::local_committee_and_keys; - - #[test] - fn committee_basic() { - // GIVEN - let epoch = 100; - let num_of_authorities = 9; - let authority_stakes = (1..=9).map(|s| s as Stake).collect(); - let (committee, _) = local_committee_and_keys(epoch, authority_stakes); - - // THEN make sure the output Committee fields are populated correctly. - assert_eq!(committee.size(), num_of_authorities); - for (i, authority) in committee.authorities() { - assert_eq!((i.value() + 1) as Stake, authority.stake); - } - - // AND ensure thresholds are calculated correctly. - assert_eq!(committee.total_stake(), 45); - assert_eq!(committee.quorum_threshold(), 31); - assert_eq!(committee.validity_threshold(), 15); - } -} diff --git a/consensus/config/src/crypto.rs b/consensus/config/src/crypto.rs deleted file mode 100644 index 69f610b8a0c..00000000000 --- a/consensus/config/src/crypto.rs +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Here we select the cryptographic types that are used by default in the code -//! base. The whole code base should only: -//! - refer to those aliases and not use the individual scheme implementations -//! - not use the schemes in a way that break genericity (e.g. using their -//! Struct impl functions) -//! - swap one of those aliases to point to another type if necessary -//! -//! Beware: if you change those aliases to point to another scheme -//! implementation, you will have to change all four aliases to point to -//! concrete types that work with each other. Failure to do so will result in a -//! ton of compilation errors, and worse: it will not make sense! - -use fastcrypto::{ - bls12381, ed25519, - error::FastCryptoError, - hash::{Blake2b256, HashFunction}, - traits::{KeyPair as _, Signer as _, ToFromBytes as _, VerifyingKey as _}, -}; -use iota_sdk_types::crypto::INTENT_PREFIX_LENGTH; -use serde::{Deserialize, Serialize}; -use tracing::instrument; - -/// Network key is used for TLS and as the network identity of the authority. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct NetworkPublicKey(ed25519::Ed25519PublicKey); -pub struct NetworkPrivateKey(ed25519::Ed25519PrivateKey); -pub struct NetworkKeyPair(ed25519::Ed25519KeyPair); - -impl NetworkPublicKey { - pub fn new(key: ed25519::Ed25519PublicKey) -> Self { - Self(key) - } - - pub fn into_inner(self) -> ed25519::Ed25519PublicKey { - self.0 - } - - pub fn to_bytes(&self) -> [u8; 32] { - self.0.0.to_bytes() - } -} - -impl NetworkPrivateKey { - pub fn into_inner(self) -> ed25519::Ed25519PrivateKey { - self.0 - } -} - -impl NetworkKeyPair { - pub fn new(keypair: ed25519::Ed25519KeyPair) -> Self { - Self(keypair) - } - - pub fn generate(rng: &mut R) -> Self { - Self(ed25519::Ed25519KeyPair::generate(rng)) - } - - pub fn public(&self) -> NetworkPublicKey { - NetworkPublicKey(self.0.public().clone()) - } - - pub fn private_key(self) -> NetworkPrivateKey { - NetworkPrivateKey(self.0.copy().private()) - } - - pub fn private_key_bytes(self) -> [u8; 32] { - self.0.private().0.to_bytes() - } -} - -impl Clone for NetworkKeyPair { - fn clone(&self) -> Self { - Self(self.0.copy()) - } -} - -/// Protocol key is used for signing blocks and verifying block signatures. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct ProtocolPublicKey(ed25519::Ed25519PublicKey); -pub struct ProtocolKeyPair(ed25519::Ed25519KeyPair); -pub struct ProtocolKeySignature(ed25519::Ed25519Signature); - -impl ProtocolPublicKey { - pub fn new(key: ed25519::Ed25519PublicKey) -> Self { - Self(key) - } - - #[instrument(level = "trace", skip_all)] - pub fn verify( - &self, - message: &[u8], - signature: &ProtocolKeySignature, - ) -> Result<(), FastCryptoError> { - self.0.verify(message, &signature.0) - } - - pub fn to_bytes(&self) -> &[u8] { - self.0.as_bytes() - } -} - -impl ProtocolKeyPair { - pub fn new(keypair: ed25519::Ed25519KeyPair) -> Self { - Self(keypair) - } - - pub fn generate(rng: &mut R) -> Self { - Self(ed25519::Ed25519KeyPair::generate(rng)) - } - - pub fn public(&self) -> ProtocolPublicKey { - ProtocolPublicKey(self.0.public().clone()) - } - - #[instrument(level = "trace", skip_all)] - pub fn sign(&self, message: &[u8]) -> ProtocolKeySignature { - ProtocolKeySignature(self.0.sign(message)) - } -} - -impl Clone for ProtocolKeyPair { - fn clone(&self) -> Self { - Self(self.0.copy()) - } -} - -impl ProtocolKeySignature { - pub fn from_bytes(bytes: &[u8]) -> Result { - Ok(Self(ed25519::Ed25519Signature::from_bytes(bytes)?)) - } - - pub fn to_bytes(&self) -> &[u8] { - self.0.as_bytes() - } -} - -/// Authority key represents the identity of an authority. It is only used for -/// identity sanity checks and not used for verification. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct AuthorityPublicKey(bls12381::min_sig::BLS12381PublicKey); -pub struct AuthorityKeyPair(bls12381::min_sig::BLS12381KeyPair); - -impl AuthorityPublicKey { - pub fn new(key: bls12381::min_sig::BLS12381PublicKey) -> Self { - Self(key) - } - - pub fn inner(&self) -> &bls12381::min_sig::BLS12381PublicKey { - &self.0 - } - - pub fn to_bytes(&self) -> &[u8] { - self.0.as_bytes() - } -} - -impl AuthorityKeyPair { - pub fn new(keypair: bls12381::min_sig::BLS12381KeyPair) -> Self { - Self(keypair) - } - - pub fn generate(rng: &mut R) -> Self { - Self(bls12381::min_sig::BLS12381KeyPair::generate(rng)) - } - - pub fn public(&self) -> AuthorityPublicKey { - AuthorityPublicKey(self.0.public().clone()) - } -} - -/// Defines algorithm and format of block and commit digests. -pub type DefaultHashFunction = Blake2b256; -pub const DIGEST_LENGTH: usize = DefaultHashFunction::OUTPUT_SIZE; -pub const INTENT_MESSAGE_LENGTH: usize = INTENT_PREFIX_LENGTH + DIGEST_LENGTH; diff --git a/consensus/config/src/lib.rs b/consensus/config/src/lib.rs deleted file mode 100644 index 98be0f98e42..00000000000 --- a/consensus/config/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -mod committee; -mod crypto; -mod parameters; -mod test_committee; - -pub use committee::*; -pub use crypto::*; -pub use parameters::*; -pub use test_committee::*; diff --git a/consensus/config/src/parameters.rs b/consensus/config/src/parameters.rs deleted file mode 100644 index 3a2e8835b96..00000000000 --- a/consensus/config/src/parameters.rs +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{path::PathBuf, time::Duration}; - -use serde::{Deserialize, Serialize}; - -/// Operational configurations of a consensus authority. -/// -/// All fields should tolerate inconsistencies among authorities, without -/// affecting safety of the protocol. Otherwise, they need to be part of IOTA -/// protocol config or epoch state on-chain. -/// -/// NOTE: fields with default values are specified in the serde default -/// functions. Most operators should not need to specify any field, except -/// db_path. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Parameters { - /// Path to consensus DB for this epoch. Required when initializing - /// consensus. This is calculated based on user configuration for base - /// directory. - #[serde(skip)] - pub db_path: PathBuf, - - /// Time to wait for parent round leader before sealing a block, from when - /// parent round has a quorum. - #[serde(default = "Parameters::default_leader_timeout")] - pub leader_timeout: Duration, - - /// Minimum delay between rounds, to avoid generating too many rounds when - /// latency is low. This is especially necessary for tests running - /// locally. If setting a non-default value, it should be set low enough - /// to avoid reducing round rate and increasing latency in realistic and - /// distributed configurations. - #[serde(default = "Parameters::default_min_round_delay")] - pub min_round_delay: Duration, - - /// Maximum forward time drift (how far in future) allowed for received - /// blocks. - #[serde(default = "Parameters::default_max_forward_time_drift")] - pub max_forward_time_drift: Duration, - - /// Number of blocks to fetch per commit sync request. - #[serde(default = "Parameters::default_max_blocks_per_fetch")] - pub max_blocks_per_fetch: usize, - - /// Number of blocks to fetch by the periodic or live sync mechanism after a - /// single request - #[serde(default = "Parameters::default_max_blocks_per_sync")] - pub max_blocks_per_sync: usize, - - /// Time to wait during node start up until the node has synced the last - /// proposed block via the network peers. When set to `0` the sync - /// mechanism is disabled. This property is meant to be used for amnesia - /// recovery. - #[serde(default = "Parameters::default_sync_last_known_own_block_timeout")] - pub sync_last_known_own_block_timeout: Duration, - - /// Interval in milliseconds to probe highest received rounds of peers. - #[serde(default = "Parameters::default_round_prober_interval_ms")] - pub round_prober_interval_ms: u64, - - /// Timeout in milliseconds for a round prober request. - #[serde(default = "Parameters::default_round_prober_request_timeout_ms")] - pub round_prober_request_timeout_ms: u64, - - /// Proposing new block is stopped when the propagation delay is greater - /// than this threshold. Propagation delay is the difference between the - /// round of the last proposed block and the highest round from this - /// authority that is received by all validators in a quorum. - #[serde(default = "Parameters::default_propagation_delay_stop_proposal_threshold")] - pub propagation_delay_stop_proposal_threshold: u32, - - /// The number of rounds of blocks to be kept in the Dag state cache per - /// authority. The larger the number the more the blocks that will be - /// kept in memory allowing minimising any potential disk access. - /// Value should be at minimum 50 rounds to ensure node performance, but - /// being too large can be expensive in memory usage. - #[serde(default = "Parameters::default_dag_state_cached_rounds")] - pub dag_state_cached_rounds: u32, - - // Number of authorities commit syncer fetches in parallel. - // Both commits in a range and blocks referenced by the commits are fetched per authority. - #[serde(default = "Parameters::default_commit_sync_parallel_fetches")] - pub commit_sync_parallel_fetches: usize, - - // Number of commits to fetch in a batch, also the maximum number of commits returned per - // fetch. If this value is set too small, fetching becomes inefficient. - // If this value is set too large, it can result in load imbalance and stragglers. - #[serde(default = "Parameters::default_commit_sync_batch_size")] - pub commit_sync_batch_size: u32, - - // This affects the maximum number of commit batches being fetched, and those fetched but not - // processed as consensus output, before throttling of outgoing commit fetches starts. - #[serde(default = "Parameters::default_commit_sync_batches_ahead")] - pub commit_sync_batches_ahead: usize, - - /// Tonic network settings. - #[serde(default = "TonicParameters::default")] - pub tonic: TonicParameters, -} - -impl Parameters { - pub(crate) fn default_leader_timeout() -> Duration { - Duration::from_millis(250) - } - - pub(crate) fn default_min_round_delay() -> Duration { - if cfg!(msim) || std::env::var("__TEST_ONLY_CONSENSUS_USE_LONG_MIN_ROUND_DELAY").is_ok() { - // Checkpoint building and execution cannot keep up with high commit rate in - // simtests, leading to long reconfiguration delays. This is because - // simtest is single threaded, and spending too much time in - // consensus can lead to starvation elsewhere. - Duration::from_millis(400) - } else if cfg!(test) { - // Avoid excessive CPU, data and logs in tests. - Duration::from_millis(250) - } else { - Duration::from_millis(50) - } - } - - pub(crate) fn default_max_forward_time_drift() -> Duration { - Duration::from_millis(500) - } - - pub(crate) fn default_max_blocks_per_fetch() -> usize { - if cfg!(msim) { - // Exercise hitting blocks per fetch limit. - 10 - } else { - 1000 - } - } - - pub(crate) fn default_max_blocks_per_sync() -> usize { - if cfg!(msim) { 4 } else { 32 } - } - - pub(crate) fn default_sync_last_known_own_block_timeout() -> Duration { - if cfg!(msim) { - Duration::from_millis(500) - } else { - // Here we prioritise liveness over the complete de-risking of block - // equivocation. 5 seconds in the majority of cases should be good - // enough for this given a healthy network. - Duration::from_secs(5) - } - } - - pub(crate) fn default_round_prober_interval_ms() -> u64 { - if cfg!(msim) { 1000 } else { 5000 } - } - - pub(crate) fn default_round_prober_request_timeout_ms() -> u64 { - if cfg!(msim) { 800 } else { 4000 } - } - - pub(crate) fn default_propagation_delay_stop_proposal_threshold() -> u32 { - // Propagation delay is usually 0 round in production. - if cfg!(msim) { 2 } else { 5 } - } - - pub(crate) fn default_dag_state_cached_rounds() -> u32 { - if cfg!(msim) { - // Exercise reading blocks from store. - 5 - } else { - 500 - } - } - - pub(crate) fn default_commit_sync_parallel_fetches() -> usize { - 8 - } - - pub(crate) fn default_commit_sync_batch_size() -> u32 { - if cfg!(msim) { - // Exercise commit sync. - 5 - } else { - 100 - } - } - - pub(crate) fn default_commit_sync_batches_ahead() -> usize { - // This is set to be a multiple of default commit_sync_parallel_fetches to allow - // fetching ahead, while keeping the total number of inflight fetches - // and unprocessed fetched commits limited. - 32 - } -} - -impl Default for Parameters { - fn default() -> Self { - Self { - db_path: PathBuf::default(), - leader_timeout: Parameters::default_leader_timeout(), - min_round_delay: Parameters::default_min_round_delay(), - max_forward_time_drift: Parameters::default_max_forward_time_drift(), - max_blocks_per_fetch: Parameters::default_max_blocks_per_fetch(), - max_blocks_per_sync: Parameters::default_max_blocks_per_sync(), - sync_last_known_own_block_timeout: - Parameters::default_sync_last_known_own_block_timeout(), - round_prober_interval_ms: Parameters::default_round_prober_interval_ms(), - round_prober_request_timeout_ms: Parameters::default_round_prober_request_timeout_ms(), - propagation_delay_stop_proposal_threshold: - Parameters::default_propagation_delay_stop_proposal_threshold(), - dag_state_cached_rounds: Parameters::default_dag_state_cached_rounds(), - commit_sync_parallel_fetches: Parameters::default_commit_sync_parallel_fetches(), - commit_sync_batch_size: Parameters::default_commit_sync_batch_size(), - commit_sync_batches_ahead: Parameters::default_commit_sync_batches_ahead(), - tonic: TonicParameters::default(), - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct TonicParameters { - /// Keepalive interval and timeouts for both client and server. - /// - /// If unspecified, this will default to 5s. - #[serde(default = "TonicParameters::default_keepalive_interval")] - pub keepalive_interval: Duration, - - /// Size of various per-connection buffers. - /// - /// If unspecified, this will default to 32MiB. - #[serde(default = "TonicParameters::default_connection_buffer_size")] - pub connection_buffer_size: usize, - - /// Messages over this size threshold will increment a counter. - /// - /// If unspecified, this will default to 16MiB. - #[serde(default = "TonicParameters::default_excessive_message_size")] - pub excessive_message_size: usize, - - /// Hard message size limit for both requests and responses. - /// This value is higher than strictly necessary, to allow overheads. - /// Message size targets and soft limits are computed based on this value. - /// - /// If unspecified, this will default to 1GiB. - #[serde(default = "TonicParameters::default_message_size_limit")] - pub message_size_limit: usize, -} - -impl TonicParameters { - fn default_keepalive_interval() -> Duration { - Duration::from_secs(5) - } - - fn default_connection_buffer_size() -> usize { - 32 << 20 - } - - fn default_excessive_message_size() -> usize { - 16 << 20 - } - - fn default_message_size_limit() -> usize { - 64 << 20 - } -} - -impl Default for TonicParameters { - fn default() -> Self { - Self { - keepalive_interval: TonicParameters::default_keepalive_interval(), - connection_buffer_size: TonicParameters::default_connection_buffer_size(), - excessive_message_size: TonicParameters::default_excessive_message_size(), - message_size_limit: TonicParameters::default_message_size_limit(), - } - } -} diff --git a/consensus/config/src/test_committee.rs b/consensus/config/src/test_committee.rs deleted file mode 100644 index 231cb2fbde6..00000000000 --- a/consensus/config/src/test_committee.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::net::{TcpListener, TcpStream}; - -use iota_network_stack::Multiaddr; -use rand::{SeedableRng as _, rngs::StdRng}; - -use crate::{ - Authority, AuthorityKeyPair, Committee, Epoch, NetworkKeyPair, ProtocolKeyPair, Stake, -}; - -/// Creates a committee for local testing, and the corresponding key pairs for -/// the authorities. -pub fn local_committee_and_keys( - epoch: Epoch, - authorities_stake: Vec, -) -> (Committee, Vec<(NetworkKeyPair, ProtocolKeyPair)>) { - let mut authorities = vec![]; - let mut key_pairs = vec![]; - let mut rng = StdRng::from_seed([0; 32]); - for (i, stake) in authorities_stake.into_iter().enumerate() { - let authority_keypair = AuthorityKeyPair::generate(&mut rng); - let protocol_keypair = ProtocolKeyPair::generate(&mut rng); - let network_keypair = NetworkKeyPair::generate(&mut rng); - authorities.push(Authority { - stake, - address: get_available_local_address(), - hostname: format!("test_host_{i}").to_string(), - authority_key: authority_keypair.public(), - protocol_key: protocol_keypair.public(), - network_key: network_keypair.public(), - }); - key_pairs.push((network_keypair, protocol_keypair)); - } - - let committee = Committee::new(epoch, authorities); - (committee, key_pairs) -} - -/// Returns a local address with an ephemeral port. -fn get_available_local_address() -> Multiaddr { - let host = "127.0.0.1"; - let port = get_available_port(host); - format!("/ip4/{host}/udp/{port}").parse().unwrap() -} - -/// Returns an ephemeral, available port. On unix systems, the port returned -/// will be in the TIME_WAIT state ensuring that the OS won't hand out this port -/// for some grace period. Callers should be able to bind to this port given -/// they use SO_REUSEADDR. -fn get_available_port(host: &str) -> u16 { - const MAX_PORT_RETRIES: u32 = 1000; - - for _ in 0..MAX_PORT_RETRIES { - if let Ok(port) = get_ephemeral_port(host) { - return port; - } - } - - panic!("Error: could not find an available port"); -} - -fn get_ephemeral_port(host: &str) -> std::io::Result { - // Request a random available port from the OS - let listener = TcpListener::bind((host, 0))?; - let addr = listener.local_addr()?; - - // Create and accept a connection (which we'll promptly drop) in order to force - // the port into the TIME_WAIT state, ensuring that the port will be - // reserved from some limited amount of time (roughly 60s on some Linux - // systems) - let _sender = TcpStream::connect(addr)?; - let _incoming = listener.accept()?; - - Ok(addr.port()) -} diff --git a/consensus/config/tests/committee_test.rs b/consensus/config/tests/committee_test.rs deleted file mode 100644 index a58eac32dcb..00000000000 --- a/consensus/config/tests/committee_test.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use consensus_config::{ - Authority, AuthorityKeyPair, Committee, NetworkKeyPair, ProtocolKeyPair, Stake, -}; -use insta::assert_yaml_snapshot; -use iota_network_stack::Multiaddr; -use rand::{SeedableRng as _, rngs::StdRng}; - -// Committee is not sent over network or stored on disk itself, but some of its -// fields are. So this test can still be useful to detect accidental format -// changes. -#[test] -fn committee_snapshot_matches() { - let epoch = 100; - - let mut authorities: Vec<_> = vec![]; - let mut rng = StdRng::from_seed([9; 32]); - let num_of_authorities = 10; - for i in 1..=num_of_authorities { - let authority_keypair = AuthorityKeyPair::generate(&mut rng); - let protocol_keypair = ProtocolKeyPair::generate(&mut rng); - let network_keypair = NetworkKeyPair::generate(&mut rng); - authorities.push(Authority { - stake: i as Stake, - address: Multiaddr::empty(), - hostname: "test_host".to_string(), - authority_key: authority_keypair.public(), - protocol_key: protocol_keypair.public(), - network_key: network_keypair.public(), - }); - } - - let committee = Committee::new(epoch, authorities); - - assert_yaml_snapshot!("committee", committee) -} diff --git a/consensus/config/tests/parameters_test.rs b/consensus/config/tests/parameters_test.rs deleted file mode 100644 index 205b922cbc4..00000000000 --- a/consensus/config/tests/parameters_test.rs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -#[test] -#[cfg(not(msim))] -fn parameters_snapshot_matches() { - let parameters = consensus_config::Parameters::default(); - insta::assert_yaml_snapshot!("parameters", parameters) -} diff --git a/consensus/config/tests/snapshots/committee_test__committee.snap b/consensus/config/tests/snapshots/committee_test__committee.snap deleted file mode 100644 index 857238eedb1..00000000000 --- a/consensus/config/tests/snapshots/committee_test__committee.snap +++ /dev/null @@ -1,70 +0,0 @@ ---- -source: consensus/config/tests/committee_test.rs -expression: committee ---- -epoch: 100 -total_stake: 55 -quorum_threshold: 37 -validity_threshold: 19 -authorities: - - stake: 1 - address: "" - hostname: test_host - authority_key: mDymMjVAbD9dvp4JFE6AGTWhRmWUUMgcu3ngSzTdLDZkYDUMOlU6+7OL/xAYtp1rF6QP3LJAyEM6F2HOe7CRxHCyG4aJZHaHMO/NvjqZk8BkUerc6AxFlkqKoIZXxz9O - protocol_key: FqZOnpC9+yY05TuGV/eF7FcU4qUyUwzlqtPDsqJ/4Cc= - network_key: VnXFtZxaTby9AquydHFr9dy8+GlVoPcsm8icljGTbWQ= - - stake: 2 - address: "" - hostname: test_host - authority_key: hLIJiig6cqajwuIlnxjL+bxhZxCgEWUGdN9VyJSqMAvuN6tDm7qTcBplvDICmgSlGJwHZ5IzyOH6N+Wj1Fa1odxsM86eLf9g+MjzT/h869sqV9EpyXtpYbufPYCW0fkT - protocol_key: +yTI/ZSBZa9CmHP/Qhjtf2bgTR0lC+9NUP8BgQq0OPw= - network_key: BpFcuO/0ux5y/AjZd5uiIM1RLa9lDLCzrixWLD26s9A= - - stake: 3 - address: "" - hostname: test_host - authority_key: tigsWqTfTjWYKyWSadMs67Nt7xcittDMp05ZfITIop7tjqxeuTUv9+bhVs+vgLqoBzEFh1v1QuCpTSdXmBJyaYksY60POX75wGk+1W4Al95VQcze57f2/0xbPCL2EW5j - protocol_key: zYiyu2/Pht26Hn499pLepEsUzAROWMNdmYatcGai/LU= - network_key: 7DYZ4De7EKRhhH0GBMTKlBqUeXnFMgkzLKjekMAuesw= - - stake: 4 - address: "" - hostname: test_host - authority_key: iM3E2wCApVtOKv1I/jSpNXO1kPRCnHo0c6H/qgyxvJngmgW1KYnWtycts8drNLIUGLtinT83V8D3Xnc3whUAqiKYrJ5vu2knQXqFBCmgrqo7N0xZ0T/uuKVWjoNqyHHl - protocol_key: B2I7VErvTa/POXtLemf+iDEt/966mNqq3F+DdHcvi8U= - network_key: d20nQz3k9ycSy5wXirVQ0FgTygI/v59O0beWvR/iRCc= - - stake: 5 - address: "" - hostname: test_host - authority_key: rrLWOAYMh3RQTJGxr7UdJ3S7VHAjrZbFqsZTySFOO4lu+7I7ylpUMwH1fXs4GNNkEjkRQxV4YA8jTBCvVRldDmSJZB9BhFGrghhgW9NgnsbLCPXRML8fJafXFWtihJrV - protocol_key: FWIIqmDHCuPzJQUOSw0kK0XQwBNp+xgjt9O00469mNk= - network_key: +0ebVcUY7X8QIafWGbUmzlhy/Vc2LtNRqqLzk4NgLoU= - - stake: 6 - address: "" - hostname: test_host - authority_key: jDvXpT0oTY7w9BWERszWiLBu9Aajj9WbbPkfmE9M+ijxGCooLK14P1lsqOZ67XMoAIfh8Z5t/hNanV5fH+3PtJT6o8APO5pCoTcLi5SoScRSQKMCevHqHAizI57wG4cl - protocol_key: YxoEz+3fDnLfrOkpqVkRxfj/HqlqsoQqt9bzSLjxJEc= - network_key: ebKuTwKcQhwyVqO2d64DdsWDPMnAQc23fBI1CXa05vo= - - stake: 7 - address: "" - hostname: test_host - authority_key: tRwqYQatDzTkoQrm5CakqfAAri5UuWkeli6WQm8NMkf6zw10x2iWweAgBsLWtgp4EB6YGVNpvNb3uI/wYAfbTxn7NUOlxpV3E3dDRJBKDbK8yvCvi9ayJxFIdki8QAD3 - protocol_key: 9ceGm+Ew/BvsSs4/PxU6btwNQy2ThQlh/7TbueVHkL8= - network_key: Ov4bLcrJ+q/p14z4MDX8Qyw6farqkOaXAsyYJCZMa0Q= - - stake: 8 - address: "" - hostname: test_host - authority_key: h0zh/ZVohjK4SQvYaZ5EeFO8v0m5AQbtFkka3jST4LN5pyN4ve8U1HFE7IkiJ/uYD9ZrKp6PKDQvmERy1H0c2YsC7y4Vtol5BiUhnVWpG6nbHSphdQAc2n1K3g14aTH+ - protocol_key: 7sr/v/qM67DaaDRN6sDXqxHb0ju78jkwX38oHvnbGls= - network_key: ryJ4Fkd0vnxNmuueLXsQGOPfQw4ZhflqlrycBLrd0zs= - - stake: 9 - address: "" - hostname: test_host - authority_key: hMl/A0iNbz42T0gSXtDva81lf30AcQ8K3QP5NvrakR9zx+lWtVm097YG1fmEW3C4AALagcAV0Y9vYQNt8JoHbUbwYiJEUkGyqq4QXLl7c/rLJa3FwqqWdLpfpHe/4Y9D - protocol_key: C/90lYealIRsJ6ut8kDQ2g9O4jgaKHHFKscQicDTo1w= - network_key: OIM99Vk76Bq9rZQPgj9EOcq+WyoS95oio2orwG/r7b4= - - stake: 10 - address: "" - hostname: test_host - authority_key: tC8uWyTrohwv5Gp+pEytkyFtTo7JKqgrOGlltChHjqGFN+t9jhIqo0P/opo5xly9DeFNmwlRMLmHdyvtTwRNvuYV06NgrXHnoLRoRcKC0XP93I10DHus5GdB1roOKoYf - protocol_key: Edh6VmJUkuP4TiU3owC8Qlg13OytuOpAdfHTUyLCGN8= - network_key: OKXew91B23hZ7ihzvjYptdWUWnQ8qTnvxzWxSIzR84c= - diff --git a/consensus/config/tests/snapshots/parameters_test__parameters.snap b/consensus/config/tests/snapshots/parameters_test__parameters.snap deleted file mode 100644 index 9e5203df0e5..00000000000 --- a/consensus/config/tests/snapshots/parameters_test__parameters.snap +++ /dev/null @@ -1,32 +0,0 @@ ---- -source: consensus/config/tests/parameters_test.rs -expression: parameters ---- -leader_timeout: - secs: 0 - nanos: 250000000 -min_round_delay: - secs: 0 - nanos: 50000000 -max_forward_time_drift: - secs: 0 - nanos: 500000000 -max_blocks_per_fetch: 1000 -max_blocks_per_sync: 32 -sync_last_known_own_block_timeout: - secs: 5 - nanos: 0 -round_prober_interval_ms: 5000 -round_prober_request_timeout_ms: 4000 -propagation_delay_stop_proposal_threshold: 5 -dag_state_cached_rounds: 500 -commit_sync_parallel_fetches: 8 -commit_sync_batch_size: 100 -commit_sync_batches_ahead: 32 -tonic: - keepalive_interval: - secs: 5 - nanos: 0 - connection_buffer_size: 33554432 - excessive_message_size: 16777216 - message_size_limit: 67108864 diff --git a/consensus/core/Cargo.toml b/consensus/core/Cargo.toml deleted file mode 100644 index d2593996549..00000000000 --- a/consensus/core/Cargo.toml +++ /dev/null @@ -1,66 +0,0 @@ -[package] -name = "consensus-core" -version = "0.1.0" -authors = ["IOTA Foundation "] -edition = "2021" -license = "Apache-2.0" -publish = false - -[lints] -workspace = true - -[dependencies] -# external dependencies -async-trait.workspace = true -base64.workspace = true -bcs.workspace = true -bytes.workspace = true -cfg-if.workspace = true -enum_dispatch.workspace = true -fastcrypto.workspace = true -futures.workspace = true -http.workspace = true -itertools.workspace = true -lru.workspace = true -nom.workspace = true -parking_lot.workspace = true -prometheus.workspace = true -prost.workspace = true -rand.workspace = true -serde.workspace = true -strum_macros.workspace = true -tap.workspace = true -thiserror.workspace = true -tokio.workspace = true -tokio-rustls.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true -tonic.workspace = true -tonic-prost.workspace = true -tonic-rustls.workspace = true -tower.workspace = true -tower-http.workspace = true -tracing.workspace = true - -# internal dependencies -consensus-config.workspace = true -iota-common.workspace = true -iota-http.workspace = true -iota-macros.workspace = true -iota-metrics.workspace = true -iota-network-stack.workspace = true -iota-protocol-config.workspace = true -iota-sdk-types.workspace = true -iota-tls.workspace = true -typed-store.workspace = true - -[dev-dependencies] -# external dependencies -rstest.workspace = true -tempfile.workspace = true - -# internal dependencies -telemetry-subscribers.workspace = true - -[build-dependencies] -tonic-build.workspace = true diff --git a/consensus/core/build.rs b/consensus/core/build.rs deleted file mode 100644 index f8fe0ed6836..00000000000 --- a/consensus/core/build.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - env, - path::{Path, PathBuf}, -}; - -type Result = std::result::Result>; - -// Build script to generate RPC stubs. -fn main() -> Result<()> { - let out_dir = PathBuf::from(env::var("OUT_DIR")?); - build_tonic_services(&out_dir); - - println!("cargo:rerun-if-changed=build.rs"); - - Ok(()) -} - -fn build_tonic_services(out_dir: &Path) { - let codec_path = "tonic_prost::ProstCodec"; - - let service = tonic_build::manual::Service::builder() - .name("ConsensusService") - .package("consensus") - .comment("Consensus authority interface") - .method( - tonic_build::manual::Method::builder() - .name("send_block") - .route_name("SendBlock") - .input_type("crate::network::tonic_network::SendBlockRequest") - .output_type("crate::network::tonic_network::SendBlockResponse") - .codec_path(codec_path) - .build(), - ) - .method( - tonic_build::manual::Method::builder() - .name("subscribe_blocks") - .route_name("SubscribeBlocks") - .input_type("crate::network::tonic_network::SubscribeBlocksRequest") - .output_type("crate::network::tonic_network::SubscribeBlocksResponse") - .codec_path(codec_path) - .server_streaming() - .client_streaming() - .build(), - ) - .method( - tonic_build::manual::Method::builder() - .name("fetch_blocks") - .route_name("FetchBlocks") - .input_type("crate::network::tonic_network::FetchBlocksRequest") - .output_type("crate::network::tonic_network::FetchBlocksResponse") - .codec_path(codec_path) - .server_streaming() - .build(), - ) - .method( - tonic_build::manual::Method::builder() - .name("fetch_commits") - .route_name("FetchCommits") - .input_type("crate::network::tonic_network::FetchCommitsRequest") - .output_type("crate::network::tonic_network::FetchCommitsResponse") - .codec_path(codec_path) - .build(), - ) - .method( - tonic_build::manual::Method::builder() - .name("fetch_latest_blocks") - .route_name("FetchLatestBlocks") - .input_type("crate::network::tonic_network::FetchLatestBlocksRequest") - .output_type("crate::network::tonic_network::FetchLatestBlocksResponse") - .codec_path(codec_path) - .server_streaming() - .build(), - ) - .method( - tonic_build::manual::Method::builder() - .name("get_latest_rounds") - .route_name("GetLatestRounds") - .input_type("crate::network::tonic_network::GetLatestRoundsRequest") - .output_type("crate::network::tonic_network::GetLatestRoundsResponse") - .codec_path(codec_path) - .build(), - ) - .build(); - - tonic_build::manual::Builder::new() - .out_dir(out_dir) - .compile(&[service]); -} diff --git a/consensus/core/src/ancestor.rs b/consensus/core/src/ancestor.rs deleted file mode 100644 index 8ed85f8e620..00000000000 --- a/consensus/core/src/ancestor.rs +++ /dev/null @@ -1,599 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use consensus_config::AuthorityIndex; -use tracing::info; - -use crate::{context::Context, leader_scoring::ReputationScores, round_prober::QuorumRound}; - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(crate) enum AncestorState { - Include, - Exclude(u64), -} - -#[derive(Clone)] -struct AncestorInfo { - state: AncestorState, - // This will be set to the count of either the quorum round update count or - // the score update count for which the EXCLUDE or INCLUDE state are locked - // in respectively. - lock_expiry_count: u32, -} - -impl AncestorInfo { - fn new() -> Self { - Self { - state: AncestorState::Include, - lock_expiry_count: 0, - } - } - - fn is_locked( - &self, - propagation_score_update_count: u32, - quorum_round_update_count: u32, - ) -> bool { - match self.state { - AncestorState::Include => self.lock_expiry_count > propagation_score_update_count, - AncestorState::Exclude(_) => self.lock_expiry_count > quorum_round_update_count, - } - } - - fn set_lock(&mut self, future_count: u32) { - self.lock_expiry_count = future_count; - } -} - -pub(crate) struct AncestorStateManager { - context: Arc, - state_map: Vec, - propagation_score_update_count: u32, - quorum_round_update_count: u32, - pub(crate) received_quorum_round_per_authority: Vec, - pub(crate) accepted_quorum_round_per_authority: Vec, - // This is the reputation scores that we use for leader election but we are - // using it here as a signal for high quality block propagation as well. - pub(crate) propagation_scores: ReputationScores, -} - -impl AncestorStateManager { - // Number of quorum round updates for which an ancestor is locked in the EXCLUDE - // state Chose 10 updates as that should be ~50 seconds of waiting with the - // current round prober interval of 5s - #[cfg(not(test))] - const STATE_LOCK_QUORUM_ROUND_UPDATES: u32 = 10; - #[cfg(test)] - const STATE_LOCK_QUORUM_ROUND_UPDATES: u32 = 1; - - // Number of propagation score updates for which an ancestor is locked in the - // INCLUDE state Chose 2 leader schedule updates (~300 commits per schedule) - // which should be ~30-90 seconds depending on the round rate for the - // authority to improve scores. - #[cfg(not(test))] - const STATE_LOCK_SCORE_UPDATES: u32 = 2; - #[cfg(test)] - const STATE_LOCK_SCORE_UPDATES: u32 = 1; - - // Exclusion threshold is based on propagation (reputation) scores - const EXCLUSION_THRESHOLD_PERCENTAGE: u64 = 10; - - pub(crate) fn new(context: Arc) -> Self { - let state_map = vec![AncestorInfo::new(); context.committee.size()]; - - let received_quorum_round_per_authority = vec![(0, 0); context.committee.size()]; - let accepted_quorum_round_per_authority = vec![(0, 0); context.committee.size()]; - Self { - context, - state_map, - propagation_score_update_count: 0, - quorum_round_update_count: 0, - propagation_scores: ReputationScores::default(), - received_quorum_round_per_authority, - accepted_quorum_round_per_authority, - } - } - - pub(crate) fn set_quorum_rounds_per_authority( - &mut self, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, - ) { - self.received_quorum_round_per_authority = received_quorum_rounds; - self.accepted_quorum_round_per_authority = accepted_quorum_rounds; - self.quorum_round_update_count += 1; - } - - pub(crate) fn set_propagation_scores(&mut self, scores: ReputationScores) { - self.propagation_scores = scores; - self.propagation_score_update_count += 1; - } - - pub(crate) fn get_ancestor_states(&self) -> Vec { - self.state_map.iter().map(|info| info.state).collect() - } - - /// Updates the state of all ancestors based on the latest scores and quorum - /// rounds - pub(crate) fn update_all_ancestors_state(&mut self) { - let propagation_scores_by_authority = self - .propagation_scores - .scores_per_authority - .clone() - .into_iter() - .enumerate() - .map(|(idx, score)| { - ( - self.context - .committee - .to_authority_index(idx) - .expect("Index should be valid"), - score, - ) - }) - .collect::>(); - - // If round prober has not run yet and we don't have network quorum round, - // it is okay because network_high_quorum_round will be zero and we will - // include all ancestors until we get more information. - let network_high_quorum_round = self.calculate_network_high_quorum_round(); - - // If propagation scores are not ready because the first 300 commits have not - // happened, this is okay as we will only start excluding ancestors after that - // point in time. - for (authority_id, score) in propagation_scores_by_authority { - let (_low, authority_high_quorum_round) = if self - .context - .protocol_config - .consensus_round_prober_probe_accepted_rounds() - { - self.accepted_quorum_round_per_authority[authority_id] - } else { - self.received_quorum_round_per_authority[authority_id] - }; - - self.update_state( - authority_id, - score, - authority_high_quorum_round, - network_high_quorum_round, - ); - } - } - - /// Updates the state of the given authority based on current scores and - /// quorum rounds. - fn update_state( - &mut self, - authority_id: AuthorityIndex, - propagation_score: u64, - authority_high_quorum_round: u32, - network_high_quorum_round: u32, - ) { - let block_hostname = &self.context.committee.authority(authority_id).hostname; - let mut ancestor_info = self.state_map[authority_id].clone(); - - if ancestor_info.is_locked( - self.propagation_score_update_count, - self.quorum_round_update_count, - ) { - // If still locked, we won't make any state changes. - return; - } - - let low_score_threshold = - (self.propagation_scores.highest_score() * Self::EXCLUSION_THRESHOLD_PERCENTAGE) / 100; - - match ancestor_info.state { - // Check conditions to switch to EXCLUDE state - // TODO: Consider using received round gaps for exclusion. - AncestorState::Include => { - if propagation_score <= low_score_threshold { - ancestor_info.state = AncestorState::Exclude(propagation_score); - ancestor_info.set_lock( - self.quorum_round_update_count + Self::STATE_LOCK_QUORUM_ROUND_UPDATES, - ); - info!( - "Authority {authority_id} moved to EXCLUDE state with score {propagation_score} <= threshold of {low_score_threshold} and locked for {:?} quorum round updates", - Self::STATE_LOCK_QUORUM_ROUND_UPDATES - ); - self.context - .metrics - .node_metrics - .ancestor_state_change_by_authority - .with_label_values(&[block_hostname.as_str(), "exclude"]) - .inc(); - } - } - // Check conditions to switch back to INCLUDE state - AncestorState::Exclude(_) => { - // It should not be possible for the scores to get over the threshold - // until the node is back in the INCLUDE state, but adding just in case. - if propagation_score > low_score_threshold - || authority_high_quorum_round >= network_high_quorum_round - { - ancestor_info.state = AncestorState::Include; - ancestor_info.set_lock( - self.propagation_score_update_count + Self::STATE_LOCK_SCORE_UPDATES, - ); - info!( - "Authority {authority_id} moved to INCLUDE state with {propagation_score} > threshold of {low_score_threshold} or {authority_high_quorum_round} >= {network_high_quorum_round} and locked for {:?} score updates.", - Self::STATE_LOCK_SCORE_UPDATES - ); - self.context - .metrics - .node_metrics - .ancestor_state_change_by_authority - .with_label_values(&[block_hostname.as_str(), "include"]) - .inc(); - } - } - } - - // If any updates were made to state ensure they are persisted. - self.state_map[authority_id] = ancestor_info; - } - - /// Calculate the network's quorum round based on what information is - /// available via RoundProber. - /// When consensus_round_prober_probe_accepted_rounds is true, uses accepted - /// rounds. Otherwise falls back to received rounds. - fn calculate_network_high_quorum_round(&self) -> u32 { - if self - .context - .protocol_config - .consensus_round_prober_probe_accepted_rounds() - { - self.calculate_network_high_accepted_quorum_round() - } else { - self.calculate_network_high_received_quorum_round() - } - } - - fn calculate_network_high_accepted_quorum_round(&self) -> u32 { - let committee = &self.context.committee; - - let high_quorum_rounds_with_stake = self - .accepted_quorum_round_per_authority - .iter() - .zip(committee.authorities()) - .map(|((_low, high), (_, authority))| (*high, authority.stake)) - .collect::>(); - - self.calculate_network_high_quorum_round_internal(high_quorum_rounds_with_stake) - } - - fn calculate_network_high_received_quorum_round(&self) -> u32 { - let committee = &self.context.committee; - - let high_quorum_rounds_with_stake = self - .received_quorum_round_per_authority - .iter() - .zip(committee.authorities()) - .map(|((_low, high), (_, authority))| (*high, authority.stake)) - .collect::>(); - - self.calculate_network_high_quorum_round_internal(high_quorum_rounds_with_stake) - } - - /// Calculate the network's high quorum round. - /// The authority high quorum round is the lowest round higher or equal to - /// rounds from a quorum of authorities. The network high quorum round - /// is using the high quorum round of each authority as reported by the - /// [`RoundProber`] and then finding the high quorum round of those high - /// quorum rounds. - fn calculate_network_high_quorum_round_internal( - &self, - mut high_quorum_rounds_with_stake: Vec<(u32, u64)>, - ) -> u32 { - high_quorum_rounds_with_stake.sort(); - - let mut total_stake = 0; - let mut network_high_quorum_round = 0; - - for (round, stake) in high_quorum_rounds_with_stake.iter() { - total_stake += stake; - if total_stake >= self.context.committee.quorum_threshold() { - network_high_quorum_round = *round; - break; - } - } - - network_high_quorum_round - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::leader_scoring::ReputationScores; - - #[tokio::test] - async fn test_calculate_network_high_received_quorum_round() { - telemetry_subscribers::init_for_testing(); - - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_round_prober_probe_accepted_rounds(false); - let context = Arc::new(context); - - let scores = ReputationScores::new((1..=300).into(), vec![1, 2, 4, 3]); - let mut ancestor_state_manager = AncestorStateManager::new(context); - ancestor_state_manager.set_propagation_scores(scores); - - // Quorum rounds are not set yet, so we should calculate a network - // quorum round of 0 to start. - let network_high_quorum_round = - ancestor_state_manager.calculate_network_high_quorum_round(); - assert_eq!(network_high_quorum_round, 0); - - let received_quorum_rounds = vec![(100, 229), (225, 229), (229, 300), (229, 300)]; - let accepted_quorum_rounds = vec![(50, 229), (175, 229), (179, 229), (179, 300)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - - // When probe_accepted_rounds is false, should use received rounds - let network_high_quorum_round = - ancestor_state_manager.calculate_network_high_quorum_round(); - assert_eq!(network_high_quorum_round, 300); - } - - #[tokio::test] - async fn test_calculate_network_high_accepted_quorum_round() { - telemetry_subscribers::init_for_testing(); - - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_round_prober_probe_accepted_rounds(true); - let context = Arc::new(context); - - let scores = ReputationScores::new((1..=300).into(), vec![1, 2, 4, 3]); - let mut ancestor_state_manager = AncestorStateManager::new(context); - ancestor_state_manager.set_propagation_scores(scores); - - // Quorum rounds are not set yet, so we should calculate a network - // quorum round of 0 to start. - let network_high_quorum_round = - ancestor_state_manager.calculate_network_high_quorum_round(); - assert_eq!(network_high_quorum_round, 0); - - let received_quorum_rounds = vec![(100, 229), (225, 300), (229, 300), (229, 300)]; - let accepted_quorum_rounds = vec![(50, 229), (175, 229), (179, 229), (179, 300)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - - // When probe_accepted_rounds is true, should use accepted rounds - let network_high_quorum_round = - ancestor_state_manager.calculate_network_high_quorum_round(); - assert_eq!(network_high_quorum_round, 229); - } - - // Test all state transitions with probe_accepted_rounds = true - // Default all INCLUDE -> EXCLUDE - // EXCLUDE -> INCLUDE (Blocked due to lock) - // EXCLUDE -> INCLUDE (Pass due to lock expired) - // INCLUDE -> EXCLUDE (Blocked due to lock) - // INCLUDE -> EXCLUDE (Pass due to lock expired) - #[tokio::test] - async fn test_update_all_ancestor_state_using_accepted_rounds() { - telemetry_subscribers::init_for_testing(); - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_round_prober_probe_accepted_rounds(true); - let context = Arc::new(context); - - let scores = ReputationScores::new((1..=300).into(), vec![1, 2, 4, 3]); - let mut ancestor_state_manager = AncestorStateManager::new(context); - ancestor_state_manager.set_propagation_scores(scores); - - let received_quorum_rounds = vec![(300, 400), (300, 400), (300, 400), (300, 400)]; - let accepted_quorum_rounds = vec![(225, 229), (225, 229), (229, 300), (229, 300)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Score threshold for exclude is (4 * 10) / 100 = 0 - // No ancestors should be excluded in with this threshold - let state_map = ancestor_state_manager.get_ancestor_states(); - for state in state_map.iter() { - assert_eq!(*state, AncestorState::Include); - } - - let scores = ReputationScores::new((1..=300).into(), vec![10, 10, 100, 100]); - ancestor_state_manager.set_propagation_scores(scores); - ancestor_state_manager.update_all_ancestors_state(); - - // Score threshold for exclude is (100 * 10) / 100 = 10 - // 2 authorities should be excluded in with this threshold - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if (0..=1).contains(&authority) { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - ancestor_state_manager.update_all_ancestors_state(); - - // 2 authorities should still be excluded with these scores and no new - // quorum round updates have been set to expire the locks. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if (0..=1).contains(&authority) { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - // Updating the quorum rounds will expire the lock as we only need 1 - // quorum round update for tests. - let received_quorum_rounds = vec![(400, 500), (400, 500), (400, 500), (400, 500)]; - let accepted_quorum_rounds = vec![(229, 300), (225, 229), (229, 300), (229, 300)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Authority 0 should now be included again because high quorum round is - // at the network high quorum round of 300. Authority 1's quorum round is - // too low and will remain excluded. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if authority == 1 { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - let received_quorum_rounds = vec![(500, 600), (500, 600), (500, 600), (500, 600)]; - let accepted_quorum_rounds = vec![(229, 300), (229, 300), (229, 300), (229, 300)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Ancestor 1 can transition to the INCLUDE state. Ancestor 0 is still locked - // in the INCLUDE state until a score update is performed which is why - // even though the scores are still low it has not moved to the EXCLUDE - // state. - let state_map = ancestor_state_manager.get_ancestor_states(); - for state in state_map.iter() { - assert_eq!(*state, AncestorState::Include); - } - - // Updating the scores will expire the lock as we only need 1 update for tests. - let scores = ReputationScores::new((1..=300).into(), vec![100, 10, 100, 100]); - ancestor_state_manager.set_propagation_scores(scores); - ancestor_state_manager.update_all_ancestors_state(); - - // Ancestor 1 can transition to EXCLUDE state now that the lock expired - // and its scores are below the threshold. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if authority == 1 { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - } - - // Test all state transitions with probe_accepted_rounds = false - // Default all INCLUDE -> EXCLUDE - // EXCLUDE -> INCLUDE (Blocked due to lock) - // EXCLUDE -> INCLUDE (Pass due to lock expired) - // INCLUDE -> EXCLUDE (Blocked due to lock) - // INCLUDE -> EXCLUDE (Pass due to lock expired) - #[tokio::test] - async fn test_update_all_ancestor_state_using_received_rounds() { - telemetry_subscribers::init_for_testing(); - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_round_prober_probe_accepted_rounds(false); - let context = Arc::new(context); - - let scores = ReputationScores::new((1..=300).into(), vec![1, 2, 4, 3]); - let mut ancestor_state_manager = AncestorStateManager::new(context); - ancestor_state_manager.set_propagation_scores(scores); - - let received_quorum_rounds = vec![(225, 229), (225, 300), (229, 300), (229, 300)]; - let accepted_quorum_rounds = vec![(100, 150), (100, 150), (100, 150), (100, 150)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Score threshold for exclude is (4 * 10) / 100 = 0 - // No ancestors should be excluded in with this threshold - let state_map = ancestor_state_manager.get_ancestor_states(); - for state in state_map.iter() { - assert_eq!(*state, AncestorState::Include); - } - - let scores = ReputationScores::new((1..=300).into(), vec![10, 10, 100, 100]); - ancestor_state_manager.set_propagation_scores(scores); - ancestor_state_manager.update_all_ancestors_state(); - - // Score threshold for exclude is (100 * 10) / 100 = 10 - // 2 authorities should be excluded in with this threshold - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if (0..=1).contains(&authority) { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - ancestor_state_manager.update_all_ancestors_state(); - - // 2 authorities should still be excluded with these scores and no new - // quorum round updates have been set to expire the locks. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if (0..=1).contains(&authority) { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - // Updating the quorum rounds will expire the lock as we only need 1 - // quorum round update for tests. - let received_quorum_rounds = vec![(229, 300), (225, 229), (229, 300), (229, 300)]; - let accepted_quorum_rounds = vec![(100, 150), (100, 150), (100, 150), (100, 150)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Authority 0 should now be included again because high quorum round is - // at the network high quorum round of 300. Authority 1's quorum round is - // too low and will remain excluded. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if authority == 1 { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - - let received_quorum_rounds = vec![(229, 300), (229, 300), (229, 300), (229, 300)]; - let accepted_quorum_rounds = vec![(100, 150), (100, 150), (100, 150), (100, 150)]; - ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - ancestor_state_manager.update_all_ancestors_state(); - - // Ancestor 1 can transition to the INCLUDE state. Ancestor 0 is still locked - // in the INCLUDE state until a score update is performed which is why - // even though the scores are still low it has not moved to the EXCLUDE - // state. - let state_map = ancestor_state_manager.get_ancestor_states(); - for state in state_map.iter() { - assert_eq!(*state, AncestorState::Include); - } - - // Updating the scores will expire the lock as we only need 1 update for tests. - let scores = ReputationScores::new((1..=300).into(), vec![100, 10, 100, 100]); - ancestor_state_manager.set_propagation_scores(scores); - ancestor_state_manager.update_all_ancestors_state(); - - // Ancestor 1 can transition to EXCLUDE state now that the lock expired - // and its scores are below the threshold. - let state_map = ancestor_state_manager.get_ancestor_states(); - for (authority, state) in state_map.iter().enumerate() { - if authority == 1 { - assert_eq!(*state, AncestorState::Exclude(10)); - } else { - assert_eq!(*state, AncestorState::Include); - } - } - } -} diff --git a/consensus/core/src/authority_node.rs b/consensus/core/src/authority_node.rs deleted file mode 100644 index 6ce3f1d4287..00000000000 --- a/consensus/core/src/authority_node.rs +++ /dev/null @@ -1,911 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Instant}; - -use consensus_config::{AuthorityIndex, Committee, NetworkKeyPair, Parameters, ProtocolKeyPair}; -use iota_common::scoring_metrics::VersionedScoringMetrics; -use iota_protocol_config::{ConsensusNetwork, ProtocolConfig}; -use itertools::Itertools; -use parking_lot::RwLock; -use prometheus::Registry; -use tracing::{info, warn}; - -use crate::{ - CommitConsumer, CommitConsumerMonitor, - authority_service::AuthorityService, - block_manager::BlockManager, - block_verifier::SignedBlockVerifier, - broadcaster::Broadcaster, - commit_observer::CommitObserver, - commit_syncer::{CommitSyncer, CommitSyncerHandle}, - commit_vote_monitor::CommitVoteMonitor, - context::{Clock, Context}, - core::{Core, CoreSignals}, - core_thread::{ChannelCoreThreadDispatcher, CoreThreadHandle}, - dag_state::DagState, - leader_schedule::LeaderSchedule, - leader_timeout::{LeaderTimeoutTask, LeaderTimeoutTaskHandle}, - metrics::initialise_metrics, - network::{NetworkClient as _, NetworkManager, tonic_network::TonicManager}, - round_prober::{RoundProber, RoundProberHandle}, - scoring_metrics_store::MysticetiScoringMetricsStore, - storage::rocksdb_store::RocksDBStore, - subscriber::Subscriber, - synchronizer::{Synchronizer, SynchronizerHandle}, - transaction::{TransactionClient, TransactionConsumer, TransactionVerifier}, -}; - -/// ConsensusAuthority is used by IOTA to manage the lifetime of AuthorityNode. -/// It hides the details of the implementation from the caller, -/// MysticetiManager. -pub enum ConsensusAuthority { - #[expect(private_interfaces)] - WithTonic(AuthorityNode), -} - -impl ConsensusAuthority { - /// Starts the `ConsensusAuthority` for the specified network type. - pub async fn start( - network_type: ConsensusNetwork, - epoch_start_timestamp_ms: u64, - own_index: AuthorityIndex, - committee: Committee, - parameters: Parameters, - protocol_config: ProtocolConfig, - protocol_keypair: ProtocolKeyPair, - network_keypair: NetworkKeyPair, - clock: Arc, - transaction_verifier: Arc, - commit_consumer: CommitConsumer, - registry: Registry, - current_local_metrics_count: Arc, - // A counter that keeps track of how many times the authority node has been booted while - // the binary or the component that is calling the `ConsensusAuthority` has been - // running. It's mostly useful to make decisions on whether amnesia recovery should - // run or not. When `boot_counter` is 0, then `ConsensusAuthority` will initiate - // the process of amnesia recovery if that's enabled in the parameters. - boot_counter: u64, - ) -> Self { - match network_type { - ConsensusNetwork::Tonic => { - let authority = AuthorityNode::start( - epoch_start_timestamp_ms, - own_index, - committee, - parameters, - protocol_config, - protocol_keypair, - network_keypair, - clock, - transaction_verifier, - commit_consumer, - registry, - current_local_metrics_count, - boot_counter, - ) - .await; - Self::WithTonic(authority) - } - } - } - - pub async fn stop(self) { - match self { - Self::WithTonic(authority) => authority.stop().await, - } - } - - pub fn transaction_client(&self) -> Arc { - match self { - Self::WithTonic(authority) => authority.transaction_client(), - } - } - - pub async fn replay_complete(&self) { - match self { - Self::WithTonic(authority) => authority.replay_complete().await, - } - } - - #[cfg(test)] - fn context(&self) -> &Arc { - match self { - Self::WithTonic(authority) => &authority.context, - } - } - - #[cfg(test)] - fn sync_last_known_own_block_enabled(&self) -> bool { - match self { - Self::WithTonic(authority) => authority.sync_last_known_own_block, - } - } -} - -pub(crate) struct AuthorityNode -where - N: NetworkManager>, -{ - context: Arc, - start_time: Instant, - transaction_client: Arc, - synchronizer: Arc, - commit_consumer_monitor: Arc, - - commit_syncer_handle: CommitSyncerHandle, - round_prober_handle: Option, - leader_timeout_handle: LeaderTimeoutTaskHandle, - core_thread_handle: CoreThreadHandle, - // Only one of broadcaster and subscriber gets created, depending on - // if streaming is supported. - broadcaster: Option, - subscriber: Option>>, - network_manager: N, - #[cfg(test)] - sync_last_known_own_block: bool, -} - -impl AuthorityNode -where - N: NetworkManager>, -{ - /// This function initializes and starts the consensus authority node - /// It ensures that the authority node is fully initialized and - /// ready to participate in the consensus process. - pub(crate) async fn start( - epoch_start_timestamp_ms: u64, - own_index: AuthorityIndex, - committee: Committee, - parameters: Parameters, - protocol_config: ProtocolConfig, - // To avoid accidentally leaking the private key, the protocol key pair should only be - // kept in Core. - protocol_keypair: ProtocolKeyPair, - network_keypair: NetworkKeyPair, - clock: Arc, - transaction_verifier: Arc, - commit_consumer: CommitConsumer, - registry: Registry, - current_local_metrics_count: Arc, - boot_counter: u64, - ) -> Self { - assert!( - committee.is_valid_index(own_index), - "Invalid own index {own_index}" - ); - let own_hostname = &committee.authority(own_index).hostname; - info!( - "Starting consensus authority {own_index} {own_hostname}, {0:?}, epoch start timestamp {epoch_start_timestamp_ms}, boot counter {boot_counter}", - protocol_config.version - ); - info!( - "Consensus authorities: {}", - committee - .authorities() - .map(|(i, a)| format!("{}: {}", i, a.hostname)) - .join(", ") - ); - info!("Consensus parameters: {:?}", parameters); - info!("Consensus committee: {:?}", committee); - - let scoring_metrics_store = Arc::new(MysticetiScoringMetricsStore::new( - committee.size(), - current_local_metrics_count, - &protocol_config, - )); - - let context = Arc::new(Context::new( - epoch_start_timestamp_ms, - own_index, - committee, - parameters, - protocol_config, - initialise_metrics(registry), - scoring_metrics_store, - clock, - )); - let start_time = Instant::now(); - - let (tx_client, tx_receiver) = TransactionClient::new(context.clone()); - let tx_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - let (core_signals, signals_receivers) = CoreSignals::new(context.clone()); - - let mut network_manager = N::new(context.clone(), network_keypair); - let network_client = network_manager.client(); - - // REQUIRED: Broadcaster must be created before Core, to start listening on the - // broadcast channel in order to not miss blocks and cause test failures. - let broadcaster = if N::Client::SUPPORT_STREAMING { - None - } else { - Some(Broadcaster::new( - context.clone(), - network_client.clone(), - &signals_receivers, - )) - }; - - let store_path = context.parameters.db_path.as_path().to_str().unwrap(); - let store = Arc::new(RocksDBStore::new(store_path)); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let highest_known_commit_at_startup = dag_state.read().last_commit_index(); - - // Sync last known own block is enabled when: - // 1. This is the first boot of the authority node (e.g. disable if the - // validator was active in the previous epoch) and - // 2. The timeout for syncing last known own block is not set to zero. - let sync_last_known_own_block = boot_counter == 0 - && !context - .parameters - .sync_last_known_own_block_timeout - .is_zero(); - info!("Sync last known own block: {sync_last_known_own_block}"); - - let block_verifier = Arc::new(SignedBlockVerifier::new( - context.clone(), - transaction_verifier, - )); - - let block_manager = - BlockManager::new(context.clone(), dag_state.clone(), block_verifier.clone()); - - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let commit_consumer_monitor = commit_consumer.monitor(); - commit_consumer_monitor - .set_highest_observed_commit_at_startup(highest_known_commit_at_startup); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let core = Core::new( - context.clone(), - leader_schedule, - tx_consumer, - block_manager, - // For streaming RPC, Core will be notified when consumer is available. - // For non-streaming RPC, there is no way to know so default to true. - // When there is only one (this) authority, assume subscriber exists. - !N::Client::SUPPORT_STREAMING || context.committee.size() == 1, - commit_observer, - core_signals, - protocol_keypair, - dag_state.clone(), - sync_last_known_own_block, - ); - - let (core_dispatcher, core_thread_handle) = - ChannelCoreThreadDispatcher::start(context.clone(), &dag_state, core); - let core_dispatcher = Arc::new(core_dispatcher); - let leader_timeout_handle = - LeaderTimeoutTask::start(core_dispatcher.clone(), &signals_receivers, context.clone()); - - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - - let synchronizer = Synchronizer::start( - network_client.clone(), - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier.clone(), - dag_state.clone(), - sync_last_known_own_block, - ); - - let commit_syncer_handle = CommitSyncer::new( - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - commit_consumer_monitor.clone(), - network_client.clone(), - block_verifier.clone(), - dag_state.clone(), - ) - .start(); - - let round_prober_handle = if context.protocol_config.consensus_round_prober() { - Some( - RoundProber::new( - context.clone(), - core_dispatcher.clone(), - dag_state.clone(), - network_client.clone(), - ) - .start(), - ) - } else { - None - }; - - let network_service = Arc::new(AuthorityService::new( - context.clone(), - block_verifier, - commit_vote_monitor, - synchronizer.clone(), - core_dispatcher, - signals_receivers.block_broadcast_receiver(), - dag_state.clone(), - store, - )); - - let subscriber = if N::Client::SUPPORT_STREAMING { - let s = Subscriber::new( - context.clone(), - network_client, - network_service.clone(), - dag_state, - ); - for (peer, _) in context.committee.authorities() { - if peer != context.own_index { - s.subscribe(peer); - } - } - Some(s) - } else { - None - }; - - network_manager.install_service(network_service).await; - - info!( - "Consensus authority started, took {:?}", - start_time.elapsed() - ); - - Self { - context, - start_time, - transaction_client: Arc::new(tx_client), - synchronizer, - commit_consumer_monitor, - commit_syncer_handle, - round_prober_handle, - leader_timeout_handle, - core_thread_handle, - broadcaster, - subscriber, - network_manager, - #[cfg(test)] - sync_last_known_own_block, - } - } - - pub(crate) async fn stop(mut self) { - info!( - "Stopping authority. Total run time: {:?}", - self.start_time.elapsed() - ); - - // First shutdown components calling into Core. - if let Err(e) = self.synchronizer.stop().await { - if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } - warn!( - "Failed to stop synchronizer when shutting down consensus: {:?}", - e - ); - }; - self.commit_syncer_handle.stop().await; - if let Some(round_prober_handle) = self.round_prober_handle.take() { - round_prober_handle.stop().await; - } - self.leader_timeout_handle.stop().await; - // Shutdown Core to stop block productions and broadcast. - // When using streaming, all subscribers to broadcasted blocks stop after this. - self.core_thread_handle.stop().await; - if let Some(mut broadcaster) = self.broadcaster.take() { - broadcaster.stop(); - } - // Stop outgoing long lived streams before stopping network server. - if let Some(subscriber) = self.subscriber.take() { - subscriber.stop(); - } - self.network_manager.stop().await; - - self.context - .metrics - .node_metrics - .uptime - .observe(self.start_time.elapsed().as_secs_f64()); - } - - pub(crate) fn transaction_client(&self) -> Arc { - self.transaction_client.clone() - } - - pub(crate) async fn replay_complete(&self) { - self.commit_consumer_monitor.replay_complete().await; - } -} - -#[cfg(test)] -mod tests { - #![allow(non_snake_case)] - - use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - time::Duration, - }; - - use consensus_config::{Parameters, local_committee_and_keys}; - use iota_metrics::monitored_mpsc::{UnboundedReceiver, unbounded_channel}; - use iota_protocol_config::ProtocolConfig; - use prometheus::Registry; - use rstest::rstest; - use tempfile::TempDir; - use tokio::time::{sleep, timeout}; - use typed_store::DBMetrics; - - use super::*; - use crate::{ - CommittedSubDag, - block::{BlockAPI as _, GENESIS_ROUND}, - transaction::NoopTransactionVerifier, - }; - - #[rstest] - #[tokio::test] - async fn test_authority_start_and_stop( - #[values(ConsensusNetwork::Tonic)] network_type: ConsensusNetwork, - ) { - let (committee, keypairs) = local_committee_and_keys(0, vec![1]); - let registry = Registry::new(); - - let temp_dir = TempDir::new().unwrap(); - let parameters = Parameters { - db_path: temp_dir.keep(), - ..Default::default() - }; - let txn_verifier = NoopTransactionVerifier {}; - - let own_index = committee.to_authority_index(0).unwrap(); - let protocol_keypair = keypairs[own_index].1.clone(); - let network_keypair = keypairs[own_index].0.clone(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let protocol_config = ProtocolConfig::get_for_max_version_UNSAFE(); - let current_local_metrics_count = Arc::new(VersionedScoringMetrics::new( - committee.size(), - &protocol_config, - )); - - let authority = ConsensusAuthority::start( - network_type, - 0, - own_index, - committee, - parameters, - protocol_config, - protocol_keypair, - network_keypair, - Arc::new(Clock::default()), - Arc::new(txn_verifier), - commit_consumer, - registry, - current_local_metrics_count, - 0, - ) - .await; - - assert_eq!(authority.context().own_index, own_index); - assert_eq!(authority.context().committee.epoch(), 0); - assert_eq!(authority.context().committee.size(), 1); - - authority.stop().await; - } - - // TODO: build AuthorityFixture. - #[rstest] - #[tokio::test(flavor = "current_thread")] - async fn test_authority_committee( - #[values(ConsensusNetwork::Tonic)] network_type: ConsensusNetwork, - #[values(0, 5, 10)] gc_depth: u32, - ) { - let db_registry = Registry::new(); - DBMetrics::init(&db_registry); - - const NUM_OF_AUTHORITIES: usize = 4; - let (committee, keypairs) = local_committee_and_keys(0, [1; NUM_OF_AUTHORITIES].to_vec()); - let mut protocol_config = ProtocolConfig::get_for_max_version_UNSAFE(); - protocol_config.set_consensus_gc_depth_for_testing(gc_depth); - - if gc_depth == 0 { - protocol_config.set_consensus_linearize_subdag_v2_for_testing(false); - protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(false); - } - - let temp_dirs = (0..NUM_OF_AUTHORITIES) - .map(|_| TempDir::new().unwrap()) - .collect::>(); - - let mut output_receivers = Vec::with_capacity(committee.size()); - let mut authorities = Vec::with_capacity(committee.size()); - let mut boot_counters = [0; NUM_OF_AUTHORITIES]; - - for (index, _authority_info) in committee.authorities() { - let (authority, receiver) = make_authority( - index, - &temp_dirs[index.value()], - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index], - protocol_config.clone(), - ) - .await; - boot_counters[index] += 1; - output_receivers.push(receiver); - authorities.push(authority); - } - - const NUM_TRANSACTIONS: u8 = 15; - let mut submitted_transactions = BTreeSet::>::new(); - for i in 0..NUM_TRANSACTIONS { - let txn = vec![i; 16]; - submitted_transactions.insert(txn.clone()); - authorities[i as usize % authorities.len()] - .transaction_client() - .submit(vec![txn]) - .await - .unwrap(); - } - - for receiver in &mut output_receivers { - let mut expected_transactions = submitted_transactions.clone(); - loop { - let committed_subdag = - tokio::time::timeout(Duration::from_secs(1), receiver.recv()) - .await - .unwrap() - .unwrap(); - for b in committed_subdag.blocks { - for txn in b.transactions().iter().map(|t| t.data().to_vec()) { - assert!( - expected_transactions.remove(&txn), - "Transaction not submitted or already seen: {txn:?}" - ); - } - } - assert_eq!(committed_subdag.reputation_scores_desc, vec![]); - if expected_transactions.is_empty() { - break; - } - } - } - - // Stop authority 1. - let index = committee.to_authority_index(1).unwrap(); - authorities.remove(index.value()).stop().await; - sleep(Duration::from_secs(10)).await; - - // Restart authority 1 and let it run. - let (authority, receiver) = make_authority( - index, - &temp_dirs[index.value()], - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index], - protocol_config.clone(), - ) - .await; - boot_counters[index] += 1; - output_receivers[index] = receiver; - authorities.insert(index.value(), authority); - sleep(Duration::from_secs(10)).await; - - // Stop all authorities and exit. - for authority in authorities { - authority.stop().await; - } - } - - #[rstest] - #[tokio::test(flavor = "current_thread")] - async fn test_small_committee( - #[values(ConsensusNetwork::Tonic)] network_type: ConsensusNetwork, - #[values(1, 2, 3)] num_authorities: usize, - ) { - let db_registry = Registry::new(); - DBMetrics::init(&db_registry); - - let (committee, keypairs) = local_committee_and_keys(0, vec![1; num_authorities]); - let protocol_config: ProtocolConfig = ProtocolConfig::get_for_max_version_UNSAFE(); - - let temp_dirs = (0..num_authorities) - .map(|_| TempDir::new().unwrap()) - .collect::>(); - - let mut output_receivers = Vec::with_capacity(committee.size()); - let mut authorities: Vec = Vec::with_capacity(committee.size()); - let mut boot_counters = vec![0; num_authorities]; - - for (index, _authority_info) in committee.authorities() { - let (authority, receiver) = make_authority( - index, - &temp_dirs[index.value()], - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index], - protocol_config.clone(), - ) - .await; - boot_counters[index] += 1; - output_receivers.push(receiver); - authorities.push(authority); - } - - const NUM_TRANSACTIONS: u8 = 15; - let mut submitted_transactions = BTreeSet::>::new(); - for i in 0..NUM_TRANSACTIONS { - let txn = vec![i; 16]; - submitted_transactions.insert(txn.clone()); - authorities[i as usize % authorities.len()] - .transaction_client() - .submit(vec![txn]) - .await - .unwrap(); - } - - for receiver in &mut output_receivers { - let mut expected_transactions = submitted_transactions.clone(); - loop { - let committed_subdag = - tokio::time::timeout(Duration::from_secs(1), receiver.recv()) - .await - .unwrap() - .unwrap(); - for b in committed_subdag.blocks { - for txn in b.transactions().iter().map(|t| t.data().to_vec()) { - assert!( - expected_transactions.remove(&txn), - "Transaction not submitted or already seen: {txn:?}" - ); - } - } - assert_eq!(committed_subdag.reputation_scores_desc, vec![]); - if expected_transactions.is_empty() { - break; - } - } - } - - // Stop authority 0. - let index = committee.to_authority_index(0).unwrap(); - authorities.remove(index.value()).stop().await; - sleep(Duration::from_secs(10)).await; - - // Restart authority 0 and let it run. - let (authority, receiver) = make_authority( - index, - &temp_dirs[index.value()], - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index], - protocol_config.clone(), - ) - .await; - boot_counters[index] += 1; - output_receivers[index] = receiver; - authorities.insert(index.value(), authority); - sleep(Duration::from_secs(10)).await; - - // Stop all authorities and exit. - for authority in authorities { - authority.stop().await; - } - } - - #[rstest] - #[tokio::test(flavor = "current_thread")] - async fn test_amnesia_recovery_success( - #[values(ConsensusNetwork::Tonic)] network_type: ConsensusNetwork, - #[values(0, 5, 10)] gc_depth: u32, - ) { - telemetry_subscribers::init_for_testing(); - let db_registry = Registry::new(); - DBMetrics::init(&db_registry); - - const NUM_OF_AUTHORITIES: usize = 4; - let (committee, keypairs) = local_committee_and_keys(0, [1; NUM_OF_AUTHORITIES].to_vec()); - let mut output_receivers = vec![]; - let mut authorities = BTreeMap::new(); - let mut temp_dirs = BTreeMap::new(); - let mut boot_counters = [0; NUM_OF_AUTHORITIES]; - - let mut protocol_config = ProtocolConfig::get_for_max_version_UNSAFE(); - protocol_config.set_consensus_gc_depth_for_testing(gc_depth); - - if gc_depth == 0 { - protocol_config.set_consensus_linearize_subdag_v2_for_testing(false); - protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(false); - } - - for (index, _authority_info) in committee.authorities() { - let dir = TempDir::new().unwrap(); - let (authority, receiver) = make_authority( - index, - &dir, - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index], - protocol_config.clone(), - ) - .await; - assert!( - authority.sync_last_known_own_block_enabled(), - "Expected syncing of last known own block to be enabled as all authorities are of empty db and boot for first time." - ); - boot_counters[index] += 1; - output_receivers.push(receiver); - authorities.insert(index, authority); - temp_dirs.insert(index, dir); - } - - // Now we take the receiver of authority 1 and we wait until we see at least one - // block committed from this authority We wait until we see at least one - // committed block authored from this authority. That way we'll be 100% sure - // that at least one block has been proposed and successfully received - // by a quorum of nodes. - let index_1 = committee.to_authority_index(1).unwrap(); - 'outer: while let Some(result) = - timeout(Duration::from_secs(10), output_receivers[index_1].recv()) - .await - .expect("Timed out while waiting for at least one committed block from authority 1") - { - for block in result.blocks { - if block.round() > GENESIS_ROUND && block.author() == index_1 { - break 'outer; - } - } - } - - // Stop authority 1 & 2. - // * Authority 1 will be used to wipe out their DB and practically "force" the - // amnesia recovery. - // * Authority 2 is stopped in order to simulate less than f+1 availability - // which will - // make authority 1 retry during amnesia recovery until it has finally managed - // to successfully get back f+1 responses. once authority 2 is up and - // running again. - authorities.remove(&index_1).unwrap().stop().await; - let index_2 = committee.to_authority_index(2).unwrap(); - authorities.remove(&index_2).unwrap().stop().await; - sleep(Duration::from_secs(5)).await; - - // Authority 1: create a new directory to simulate amnesia. The node will start - // having participated previously to consensus but now will attempt to - // synchronize the last own block and recover from there. It won't be able - // to do that successfully as authority 2 is still down. - let dir = TempDir::new().unwrap(); - // We do reset the boot counter for this one to simulate a "binary" restart - boot_counters[index_1] = 0; - let (authority, mut receiver) = make_authority( - index_1, - &dir, - committee.clone(), - keypairs.clone(), - network_type, - boot_counters[index_1], - protocol_config.clone(), - ) - .await; - assert!( - authority.sync_last_known_own_block_enabled(), - "Authority should have the sync of last own block enabled" - ); - boot_counters[index_1] += 1; - authorities.insert(index_1, authority); - temp_dirs.insert(index_1, dir); - sleep(Duration::from_secs(5)).await; - - // Now spin up authority 2 using its earlier directly - so no amnesia recovery - // should be forced here. Authority 1 should be able to recover from - // amnesia successfully. - let (authority, _receiver) = make_authority( - index_2, - &temp_dirs[&index_2], - committee.clone(), - keypairs, - network_type, - boot_counters[index_2], - protocol_config.clone(), - ) - .await; - assert!( - !authority.sync_last_known_own_block_enabled(), - "Authority should not have attempted to sync the last own block" - ); - boot_counters[index_2] += 1; - authorities.insert(index_2, authority); - sleep(Duration::from_secs(5)).await; - - // We wait until we see at least one committed block authored from this - // authority - 'outer: while let Some(result) = receiver.recv().await { - for block in result.blocks { - if block.round() > GENESIS_ROUND && block.author() == index_1 { - break 'outer; - } - } - } - - // Stop all authorities and exit. - for (_, authority) in authorities { - authority.stop().await; - } - } - - // TODO: create a fixture - async fn make_authority( - index: AuthorityIndex, - db_dir: &TempDir, - committee: Committee, - keypairs: Vec<(NetworkKeyPair, ProtocolKeyPair)>, - network_type: ConsensusNetwork, - boot_counter: u64, - protocol_config: ProtocolConfig, - ) -> (ConsensusAuthority, UnboundedReceiver) { - let registry = Registry::new(); - - // Cache less blocks to exercise commit sync. - let parameters = Parameters { - db_path: db_dir.path().to_path_buf(), - dag_state_cached_rounds: 5, - commit_sync_parallel_fetches: 2, - commit_sync_batch_size: 3, - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - }; - let txn_verifier = NoopTransactionVerifier {}; - - let protocol_keypair = keypairs[index].1.clone(); - let network_keypair = keypairs[index].0.clone(); - - let (sender, receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let current_local_metrics_count = Arc::new(VersionedScoringMetrics::new( - committee.size(), - &protocol_config, - )); - - let authority = ConsensusAuthority::start( - network_type, - 0, - index, - committee, - parameters, - protocol_config, - protocol_keypair, - network_keypair, - Arc::new(Clock::default()), - Arc::new(txn_verifier), - commit_consumer, - registry, - current_local_metrics_count, - boot_counter, - ) - .await; - - (authority, receiver) - } -} diff --git a/consensus/core/src/authority_service.rs b/consensus/core/src/authority_service.rs deleted file mode 100644 index 74e812ae216..00000000000 --- a/consensus/core/src/authority_service.rs +++ /dev/null @@ -1,1337 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::BTreeMap, pin::Pin, sync::Arc, time::Duration}; - -use async_trait::async_trait; -use bytes::Bytes; -use consensus_config::AuthorityIndex; -use futures::{Stream, StreamExt, ready, stream, task}; -use iota_macros::fail_point_async; -use parking_lot::RwLock; -use tokio::{sync::broadcast, time::sleep}; -use tokio_util::sync::ReusableBoxFuture; -use tracing::{debug, info, warn}; - -use crate::{ - CommitIndex, Round, - block::{BlockAPI as _, BlockRef, ExtendedBlock, GENESIS_ROUND, SignedBlock, VerifiedBlock}, - block_verifier::BlockVerifier, - commit::{CommitAPI as _, CommitRange, TrustedCommit}, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::CoreThreadDispatcher, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - network::{BlockStream, ExtendedSerializedBlock, NetworkService}, - stake_aggregator::{QuorumThreshold, StakeAggregator}, - storage::Store, - synchronizer::{MAX_ADDITIONAL_BLOCKS, SynchronizerHandle}, -}; - -pub(crate) const COMMIT_LAG_MULTIPLIER: u32 = 5; - -/// Authority's network service implementation, agnostic to the actual -/// networking stack used. -pub(crate) struct AuthorityService { - context: Arc, - commit_vote_monitor: Arc, - block_verifier: Arc, - synchronizer: Arc, - core_dispatcher: Arc, - rx_block_broadcaster: broadcast::Receiver, - subscription_counter: Arc, - dag_state: Arc>, - store: Arc, -} - -impl AuthorityService { - pub(crate) fn new( - context: Arc, - block_verifier: Arc, - commit_vote_monitor: Arc, - synchronizer: Arc, - core_dispatcher: Arc, - rx_block_broadcaster: broadcast::Receiver, - dag_state: Arc>, - store: Arc, - ) -> Self { - let subscription_counter = Arc::new(SubscriptionCounter::new( - context.clone(), - core_dispatcher.clone(), - )); - Self { - context, - block_verifier, - commit_vote_monitor, - synchronizer, - core_dispatcher, - rx_block_broadcaster, - subscription_counter, - dag_state, - store, - } - } -} - -#[async_trait] -impl NetworkService for AuthorityService { - async fn handle_send_block( - &self, - peer: AuthorityIndex, - serialized_block: ExtendedSerializedBlock, - ) -> ConsensusResult<()> { - fail_point_async!("consensus-rpc-response"); - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["AuthorityService::handle_stream"]) - .start_timer(); - let peer_hostname = &self.context.committee.authority(peer).hostname; - - // TODO: dedup block verifications, here and with fetched blocks. - let signed_block: SignedBlock = - bcs::from_bytes(&serialized_block.block).map_err(ConsensusError::MalformedBlock)?; - - // Reject blocks not produced by the peer. - if peer != signed_block.author() { - self.context - .metrics - .node_metrics - .invalid_blocks - .with_label_values(&[ - peer_hostname.as_str(), - "handle_send_block", - "UnexpectedAuthority", - ]) - .inc(); - let e = ConsensusError::UnexpectedAuthority(signed_block.author(), peer); - info!("Block with wrong authority from {}: {}", peer, e); - return Err(e); - } - let peer_hostname = &self.context.committee.authority(peer).hostname; - - // Reject blocks failing validations. - if let Err(e) = self.block_verifier.verify(&signed_block) { - self.context - .metrics - .node_metrics - .invalid_blocks - .with_label_values(&[peer_hostname.as_str(), "handle_send_block", e.name()]) - .inc(); - info!("Invalid block from {}: {}", peer, e); - return Err(e); - } - let verified_block = VerifiedBlock::new_verified(signed_block, serialized_block.block); - let block_ref = verified_block.reference(); - debug!("Received block {} via send block.", block_ref); - - let now = self.context.clock.timestamp_utc_ms(); - let forward_time_drift = - Duration::from_millis(verified_block.timestamp_ms().saturating_sub(now)); - let latency_to_process_stream = - Duration::from_millis(now.saturating_sub(verified_block.timestamp_ms())); - self.context - .metrics - .node_metrics - .latency_to_process_stream - .with_label_values(&[peer_hostname.as_str()]) - .observe(latency_to_process_stream.as_secs_f64()); - - if !self - .context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - // Reject block with timestamp too far in the future. - if forward_time_drift > self.context.parameters.max_forward_time_drift { - self.context - .metrics - .node_metrics - .rejected_future_blocks - .with_label_values(&[peer_hostname]) - .inc(); - debug!( - "Block {:?} timestamp ({} > {}) is too far in the future, rejected.", - block_ref, - verified_block.timestamp_ms(), - now, - ); - return Err(ConsensusError::BlockRejected { - block_ref, - reason: format!( - "Block timestamp is too far in the future: {} > {}", - verified_block.timestamp_ms(), - now - ), - }); - } - - // Wait until the block's timestamp is current. - if forward_time_drift > Duration::ZERO { - self.context - .metrics - .node_metrics - .block_timestamp_drift_ms - .with_label_values(&[peer_hostname.as_str(), "handle_send_block"]) - .inc_by(forward_time_drift.as_millis() as u64); - debug!( - "Block {:?} timestamp ({} > {}) is in the future, waiting for {}ms", - block_ref, - verified_block.timestamp_ms(), - now, - forward_time_drift.as_millis(), - ); - sleep(forward_time_drift).await; - } - } else { - self.context - .metrics - .node_metrics - .block_timestamp_drift_ms - .with_label_values(&[peer_hostname.as_str(), "handle_send_block"]) - .inc_by(forward_time_drift.as_millis() as u64); - } - - // Observe the block for the commit votes. When local commit is lagging too - // much, commit sync loop will trigger fetching. - self.commit_vote_monitor.observe_block(&verified_block); - - // Reject blocks when local commit index is lagging too far from quorum commit - // index. - // - // IMPORTANT: this must be done after observing votes from the block, otherwise - // observed quorum commit will no longer progress. - // - // Since the main issue with too many suspended blocks is memory usage not CPU, - // it is ok to reject after block verifications instead of before. - let last_commit_index = self.dag_state.read().last_commit_index(); - let quorum_commit_index = self.commit_vote_monitor.quorum_commit_index(); - // The threshold to ignore block should be larger than commit_sync_batch_size, - // to avoid excessive block rejections and synchronizations. - if last_commit_index - + self.context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER - < quorum_commit_index - { - self.context - .metrics - .node_metrics - .rejected_blocks - .with_label_values(&["commit_lagging"]) - .inc(); - debug!( - "Block {:?} is rejected because last commit index is lagging quorum commit index too much ({} < {})", - block_ref, last_commit_index, quorum_commit_index, - ); - return Err(ConsensusError::BlockRejected { - block_ref, - reason: format!( - "Last commit index is lagging quorum commit index too much ({last_commit_index} < {quorum_commit_index})", - ), - }); - } - - self.context - .metrics - .node_metrics - .verified_blocks - .with_label_values(&[peer_hostname]) - .inc(); - - let missing_ancestors = self - .core_dispatcher - .add_blocks(vec![verified_block]) - .await - .map_err(|_| ConsensusError::Shutdown)?; - if !missing_ancestors.is_empty() { - // schedule the fetching of them from this peer - if let Err(err) = self - .synchronizer - .fetch_blocks(missing_ancestors, peer) - .await - { - warn!("Errored while trying to fetch missing ancestors via synchronizer: {err}"); - } - } - - // After processing the block, process the excluded ancestors - - let mut excluded_ancestors = serialized_block - .excluded_ancestors - .into_iter() - .map(|serialized| bcs::from_bytes::(&serialized)) - .collect::, bcs::Error>>() - .map_err(ConsensusError::MalformedBlock)?; - - let excluded_ancestors_limit = self.context.committee.size() * 2; - if excluded_ancestors.len() > excluded_ancestors_limit { - debug!( - "Dropping {} excluded ancestor(s) from {} {} due to size limit", - excluded_ancestors.len() - excluded_ancestors_limit, - peer, - peer_hostname, - ); - excluded_ancestors.truncate(excluded_ancestors_limit); - } - - self.context - .metrics - .node_metrics - .network_received_excluded_ancestors_from_authority - .with_label_values(&[peer_hostname]) - .inc_by(excluded_ancestors.len() as u64); - - for excluded_ancestor in &excluded_ancestors { - let excluded_ancestor_hostname = &self - .context - .committee - .authority(excluded_ancestor.author) - .hostname; - self.context - .metrics - .node_metrics - .network_excluded_ancestors_count_by_authority - .with_label_values(&[excluded_ancestor_hostname]) - .inc(); - } - - let missing_excluded_ancestors = self - .core_dispatcher - .check_block_refs(excluded_ancestors) - .await - .map_err(|_| ConsensusError::Shutdown)?; - - if !missing_excluded_ancestors.is_empty() { - self.context - .metrics - .node_metrics - .network_excluded_ancestors_sent_to_fetch - .with_label_values(&[peer_hostname]) - .inc_by(missing_excluded_ancestors.len() as u64); - - let synchronizer = self.synchronizer.clone(); - tokio::spawn(async move { - // schedule the fetching of them from this peer in the background - if let Err(err) = synchronizer - .fetch_blocks(missing_excluded_ancestors, peer) - .await - { - warn!( - "Errored while trying to fetch missing excluded ancestors via synchronizer: {err}" - ); - } - }); - } - - Ok(()) - } - - async fn handle_subscribe_blocks( - &self, - peer: AuthorityIndex, - last_received: Round, - ) -> ConsensusResult { - fail_point_async!("consensus-rpc-response"); - - let dag_state = self.dag_state.read(); - // Find recent own blocks that have not been received by the peer. - // If last_received is a valid and more blocks have been proposed since then, - // this call is guaranteed to return at least some recent blocks, which - // will help with liveness. - let missed_blocks = stream::iter( - dag_state - .get_cached_blocks(self.context.own_index, last_received + 1) - .into_iter() - .map(|block| ExtendedSerializedBlock { - block: block.serialized().clone(), - excluded_ancestors: vec![], - }), - ); - - let broadcasted_blocks = BroadcastedBlockStream::new( - peer, - self.rx_block_broadcaster.resubscribe(), - self.subscription_counter.clone(), - ); - - // Return a stream of blocks that first yields missed blocks as requested, then - // new blocks. - Ok(Box::pin(missed_blocks.chain( - broadcasted_blocks.map(ExtendedSerializedBlock::from), - ))) - } - - // Handles two types of requests: - // 1. Missing block for block sync: - // - uses highest_accepted_rounds. - // - at most max_blocks_per_sync blocks should be returned. - // 2. Committed block for commit sync: - // - does not use highest_accepted_rounds. - // - at most max_blocks_per_fetch blocks should be returned. - async fn handle_fetch_blocks( - &self, - peer: AuthorityIndex, - mut block_refs: Vec, - highest_accepted_rounds: Vec, - ) -> ConsensusResult> { - // This method is used for both commit sync and periodic/live synchronizer. - // For commit sync, we do not use highest_accepted_rounds and the fetch size is - // larger. - let commit_sync_handle = highest_accepted_rounds.is_empty(); - - fail_point_async!("consensus-rpc-response"); - - // Some quick validation of the requested block refs - for block in &block_refs { - if !self.context.committee.is_valid_index(block.author) { - return Err(ConsensusError::InvalidAuthorityIndex { - index: block.author, - max: self.context.committee.size(), - }); - } - if block.round == GENESIS_ROUND { - return Err(ConsensusError::UnexpectedGenesisBlockRequested); - } - } - - if !self.context.protocol_config.consensus_batched_block_sync() { - if block_refs.len() > self.context.parameters.max_blocks_per_fetch { - return Err(ConsensusError::TooManyFetchBlocksRequested(peer)); - } - - if !commit_sync_handle && highest_accepted_rounds.len() != self.context.committee.size() - { - return Err(ConsensusError::InvalidSizeOfHighestAcceptedRounds( - highest_accepted_rounds.len(), - self.context.committee.size(), - )); - } - - // For now ask dag state directly - let blocks = self.dag_state.read().get_blocks(&block_refs); - - // Now check if an ancestor's round is higher than the one that the peer has. If - // yes, then serve that ancestor blocks up to `MAX_ADDITIONAL_BLOCKS`. - let mut ancestor_blocks = vec![]; - if !commit_sync_handle { - let all_ancestors = blocks - .iter() - .flatten() - .flat_map(|block| block.ancestors().to_vec()) - .filter(|block_ref| highest_accepted_rounds[block_ref.author] < block_ref.round) - .take(MAX_ADDITIONAL_BLOCKS) - .collect::>(); - - if !all_ancestors.is_empty() { - ancestor_blocks = self.dag_state.read().get_blocks(&all_ancestors); - } - } - - // Return the serialised blocks & the ancestor blocks - let result = blocks - .into_iter() - .chain(ancestor_blocks) - .flatten() - .map(|block| block.serialized().clone()) - .collect::>(); - - return Ok(result); - } - - // For commit sync, the fetch size is larger. For periodic/live synchronizer, - // the fetch size is smaller.else { Instead of rejecting the request, we - // truncate the size to allow an easy update of this parameter in the future. - if commit_sync_handle { - block_refs.truncate(self.context.parameters.max_blocks_per_fetch); - } else { - block_refs.truncate(self.context.parameters.max_blocks_per_sync); - } - - // Get requested blocks from store. - let blocks = if commit_sync_handle { - // For commit sync, optimize by fetching from store for blocks below GC round - let gc_round = self.dag_state.read().gc_round(); - - // Separate indices for below/above GC while preserving original order - let mut below_gc_indices = Vec::new(); - let mut above_gc_indices = Vec::new(); - let mut below_gc_refs = Vec::new(); - let mut above_gc_refs = Vec::new(); - for (i, block_ref) in block_refs.iter().enumerate() { - if block_ref.round < gc_round { - below_gc_indices.push(i); - below_gc_refs.push(*block_ref); - } else { - above_gc_indices.push(i); - above_gc_refs.push(*block_ref); - } - } - - let mut blocks: Vec> = vec![None; block_refs.len()]; - - // Fetch blocks below GC from store - if !below_gc_refs.is_empty() { - for (idx, block) in below_gc_indices - .iter() - .zip(self.store.read_blocks(&below_gc_refs)?) - { - blocks[*idx] = block; - } - } - - // Fetch blocks at-or-above GC from dag_state - if !above_gc_refs.is_empty() { - for (idx, block) in above_gc_indices - .iter() - .zip(self.dag_state.read().get_blocks(&above_gc_refs)) - { - blocks[*idx] = block; - } - } - - blocks.into_iter().flatten().collect() - } else { - // For periodic or live synchronizer, we respond with requested blocks from the - // store and with additional blocks from the cache - block_refs.sort(); - block_refs.dedup(); - let dag_state = self.dag_state.read(); - let mut blocks = dag_state - .get_blocks(&block_refs) - .into_iter() - .flatten() - .collect::>(); - - // Get additional blocks for authorities with missing block, if they are - // available in cache. Compute the lowest missing round per - // requested authority. - let mut lowest_missing_rounds = BTreeMap::::new(); - for block_ref in blocks.iter().map(|b| b.reference()) { - let entry = lowest_missing_rounds - .entry(block_ref.author) - .or_insert(block_ref.round); - *entry = (*entry).min(block_ref.round); - } - - // Retrieve additional blocks per authority, from peer's highest accepted round - // + 1 to lowest missing round (exclusive) per requested authority. Start with - // own blocks. - let own_index = self.context.own_index; - - // Collect and sort so own_index comes first - let mut ordered_missing_rounds: Vec<_> = lowest_missing_rounds.into_iter().collect(); - ordered_missing_rounds.sort_by_key(|(auth, _)| if *auth == own_index { 0 } else { 1 }); - - for (authority, lowest_missing_round) in ordered_missing_rounds { - let highest_accepted_round = highest_accepted_rounds[authority]; - if highest_accepted_round >= lowest_missing_round { - continue; - } - - let missing_blocks = dag_state.get_cached_blocks_in_range( - authority, - highest_accepted_round + 1, - lowest_missing_round, - self.context - .parameters - .max_blocks_per_sync - .saturating_sub(blocks.len()), - ); - blocks.extend(missing_blocks); - if blocks.len() >= self.context.parameters.max_blocks_per_sync { - blocks.truncate(self.context.parameters.max_blocks_per_sync); - break; - } - } - - blocks - }; - - // Return the serialized blocks - let bytes = blocks - .into_iter() - .map(|block| block.serialized().clone()) - .collect::>(); - Ok(bytes) - } - - async fn handle_fetch_commits( - &self, - _peer: AuthorityIndex, - commit_range: CommitRange, - ) -> ConsensusResult<(Vec, Vec)> { - fail_point_async!("consensus-rpc-response"); - - // Compute an inclusive end index and bound the maximum number of commits - // scanned. - let inclusive_end = commit_range.end().min( - commit_range.start() + self.context.parameters.commit_sync_batch_size as CommitIndex - - 1, - ); - let mut commits = self - .store - .scan_commits((commit_range.start()..=inclusive_end).into())?; - let mut certifier_block_refs = vec![]; - 'commit: while let Some(c) = commits.last() { - let index = c.index(); - let votes = self.store.read_commit_votes(index)?; - let mut stake_aggregator = StakeAggregator::::new(); - for v in &votes { - stake_aggregator.add(v.author, &self.context.committee); - } - if stake_aggregator.reached_threshold(&self.context.committee) { - certifier_block_refs = votes; - break 'commit; - } else { - debug!( - "Commit {} votes did not reach quorum to certify, {} < {}, skipping", - index, - stake_aggregator.stake(), - stake_aggregator.threshold(&self.context.committee) - ); - self.context - .metrics - .node_metrics - .commit_sync_fetch_commits_handler_uncertified_skipped - .inc(); - commits.pop(); - } - } - let certifier_blocks = self - .store - .read_blocks(&certifier_block_refs)? - .into_iter() - .flatten() - .collect(); - Ok((commits, certifier_blocks)) - } - - async fn handle_fetch_latest_blocks( - &self, - peer: AuthorityIndex, - authorities: Vec, - ) -> ConsensusResult> { - fail_point_async!("consensus-rpc-response"); - - if authorities.len() > self.context.committee.size() { - return Err(ConsensusError::TooManyAuthoritiesProvided(peer)); - } - - // Ensure that those are valid authorities - for authority in &authorities { - if !self.context.committee.is_valid_index(*authority) { - return Err(ConsensusError::InvalidAuthorityIndex { - index: *authority, - max: self.context.committee.size(), - }); - } - } - - // Read from the dag state to find the latest blocks. - // TODO: at the moment we don't look into the block manager for suspended - // blocks. Ideally we want in the future if we think we would like to - // tackle the majority of cases. - let mut blocks = vec![]; - let dag_state = self.dag_state.read(); - for authority in authorities { - let block = dag_state.get_last_block_for_authority(authority); - - debug!("Latest block for {authority}: {block:?} as requested from {peer}"); - - // no reason to serve back the genesis block - it's equal as if it has not - // received any block - if block.round() != GENESIS_ROUND { - blocks.push(block); - } - } - - // Return the serialised blocks - let result = blocks - .into_iter() - .map(|block| block.serialized().clone()) - .collect::>(); - - Ok(result) - } - - async fn handle_get_latest_rounds( - &self, - _peer: AuthorityIndex, - ) -> ConsensusResult<(Vec, Vec)> { - fail_point_async!("consensus-rpc-response"); - - let mut highest_received_rounds = self.core_dispatcher.highest_received_rounds(); - - let blocks = self - .dag_state - .read() - .get_last_cached_block_per_authority(Round::MAX); - let highest_accepted_rounds = blocks - .into_iter() - .map(|(block, _)| block.round()) - .collect::>(); - - // Own blocks do not go through the core dispatcher, so they need to be set - // separately. - highest_received_rounds[self.context.own_index] = - highest_accepted_rounds[self.context.own_index]; - - Ok((highest_received_rounds, highest_accepted_rounds)) - } -} - -struct Counter { - count: usize, - subscriptions_by_authority: Vec, -} - -/// Atomically counts the number of active subscriptions to the block broadcast -/// stream, and dispatch commands to core based on the changes. -struct SubscriptionCounter { - context: Arc, - counter: parking_lot::Mutex, - dispatcher: Arc, -} - -impl SubscriptionCounter { - fn new(context: Arc, dispatcher: Arc) -> Self { - // Set the subscribed peers by default to 0 - for (_, authority) in context.committee.authorities() { - context - .metrics - .node_metrics - .subscribed_by - .with_label_values(&[authority.hostname.as_str()]) - .set(0); - } - - Self { - counter: parking_lot::Mutex::new(Counter { - count: 0, - subscriptions_by_authority: vec![0; context.committee.size()], - }), - dispatcher, - context, - } - } - - fn increment(&self, peer: AuthorityIndex) -> Result<(), ConsensusError> { - let mut counter = self.counter.lock(); - counter.count += 1; - let original_subscription_by_peer = counter.subscriptions_by_authority[peer]; - counter.subscriptions_by_authority[peer] += 1; - let mut total_stake = 0; - for (authority_index, _) in self.context.committee.authorities() { - if counter.subscriptions_by_authority[authority_index] >= 1 - || self.context.own_index == authority_index - { - total_stake += self.context.committee.stake(authority_index); - } - } - // Stake of subscriptions before a new peer was subscribed - let previous_stake = if original_subscription_by_peer == 0 { - total_stake - self.context.committee.stake(peer) - } else { - total_stake - }; - - let peer_hostname = &self.context.committee.authority(peer).hostname; - self.context - .metrics - .node_metrics - .subscribed_by - .with_label_values(&[peer_hostname]) - .set(1); - // If the subscription count reaches quorum, notify the dispatcher and get ready - // to propose blocks. - if !self.context.committee.reached_quorum(previous_stake) - && self.context.committee.reached_quorum(total_stake) - { - self.dispatcher - .set_quorum_subscribers_exists(true) - .map_err(|_| ConsensusError::Shutdown)?; - } - // Drop the counter after sending the command to the dispatcher - drop(counter); - Ok(()) - } - - fn decrement(&self, peer: AuthorityIndex) -> Result<(), ConsensusError> { - let mut counter = self.counter.lock(); - counter.count -= 1; - let original_subscription_by_peer = counter.subscriptions_by_authority[peer]; - counter.subscriptions_by_authority[peer] -= 1; - let mut total_stake = 0; - for (authority_index, _) in self.context.committee.authorities() { - if counter.subscriptions_by_authority[authority_index] >= 1 - || self.context.own_index == authority_index - { - total_stake += self.context.committee.stake(authority_index); - } - } - // Stake of subscriptions before a peer was dropped - let previous_stake = if original_subscription_by_peer == 1 { - total_stake + self.context.committee.stake(peer) - } else { - total_stake - }; - - if counter.subscriptions_by_authority[peer] == 0 { - let peer_hostname = &self.context.committee.authority(peer).hostname; - self.context - .metrics - .node_metrics - .subscribed_by - .with_label_values(&[peer_hostname]) - .set(0); - } - - // If the subscription count drops below quorum, notify the dispatcher to stop - // proposing blocks. - if self.context.committee.reached_quorum(previous_stake) - && !self.context.committee.reached_quorum(total_stake) - { - self.dispatcher - .set_quorum_subscribers_exists(false) - .map_err(|_| ConsensusError::Shutdown)?; - } - // Drop the counter after sending the command to the dispatcher - drop(counter); - Ok(()) - } -} - -/// Each broadcasted block stream wraps a broadcast receiver for blocks. -/// It yields blocks that are broadcasted after the stream is created. -type BroadcastedBlockStream = BroadcastStream; - -/// Adapted from `tokio_stream::wrappers::BroadcastStream`. The main difference -/// is that this tolerates lags with only logging, without yielding errors. -struct BroadcastStream { - peer: AuthorityIndex, - // Stores the receiver across poll_next() calls. - inner: ReusableBoxFuture< - 'static, - ( - Result, - broadcast::Receiver, - ), - >, - // Counts total subscriptions / active BroadcastStreams. - subscription_counter: Arc, -} - -impl BroadcastStream { - pub fn new( - peer: AuthorityIndex, - rx: broadcast::Receiver, - subscription_counter: Arc, - ) -> Self { - if let Err(err) = subscription_counter.increment(peer) { - match err { - ConsensusError::Shutdown => {} - _ => panic!("Unexpected error: {err}"), - } - } - Self { - peer, - inner: ReusableBoxFuture::new(make_recv_future(rx)), - subscription_counter, - } - } -} - -impl Stream for BroadcastStream { - type Item = T; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut task::Context<'_>, - ) -> task::Poll> { - let peer = self.peer; - let maybe_item = loop { - let (result, rx) = ready!(self.inner.poll(cx)); - self.inner.set(make_recv_future(rx)); - - match result { - Ok(item) => break Some(item), - Err(broadcast::error::RecvError::Closed) => { - info!("Block BroadcastedBlockStream {} closed", peer); - break None; - } - Err(broadcast::error::RecvError::Lagged(n)) => { - warn!( - "Block BroadcastedBlockStream {} lagged by {} messages", - peer, n - ); - continue; - } - } - }; - task::Poll::Ready(maybe_item) - } -} - -impl Drop for BroadcastStream { - fn drop(&mut self) { - if let Err(err) = self.subscription_counter.decrement(self.peer) { - match err { - ConsensusError::Shutdown => {} - _ => panic!("Unexpected error: {err}"), - } - } - } -} - -async fn make_recv_future( - mut rx: broadcast::Receiver, -) -> ( - Result, - broadcast::Receiver, -) { - let result = rx.recv().await; - (result, rx) -} - -// TODO: add a unit test for BroadcastStream. - -#[cfg(test)] -pub(crate) mod tests { - use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - time::Duration, - }; - - use async_trait::async_trait; - use bytes::Bytes; - use consensus_config::AuthorityIndex; - use parking_lot::{Mutex, RwLock}; - use rstest::rstest; - use tokio::{sync::broadcast, time::sleep}; - - use crate::{ - Round, - authority_service::AuthorityService, - block::{BlockAPI, BlockRef, GENESIS_ROUND, SignedBlock, TestBlock, VerifiedBlock}, - commit::{CertifiedCommits, CommitDigest, CommitRange, TrustedCommit}, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::{CoreError, CoreThreadDispatcher}, - dag_state::DagState, - error::ConsensusResult, - network::{BlockStream, ExtendedSerializedBlock, NetworkClient, NetworkService}, - round_prober::QuorumRound, - storage::{Store, WriteBatch, mem_store::MemStore}, - synchronizer::Synchronizer, - test_dag_builder::DagBuilder, - }; - - pub(crate) struct FakeCoreThreadDispatcher { - blocks: Mutex>, - } - - impl FakeCoreThreadDispatcher { - pub(crate) fn new() -> Self { - Self { - blocks: Mutex::new(vec![]), - } - } - - fn get_blocks(&self) -> Vec { - self.blocks.lock().clone() - } - } - - #[async_trait] - impl CoreThreadDispatcher for FakeCoreThreadDispatcher { - async fn add_blocks( - &self, - blocks: Vec, - ) -> Result, CoreError> { - let block_refs = blocks.iter().map(|b| b.reference()).collect(); - self.blocks.lock().extend(blocks); - Ok(block_refs) - } - - async fn check_block_refs( - &self, - _block_refs: Vec, - ) -> Result, CoreError> { - Ok(BTreeSet::new()) - } - - async fn add_certified_commits( - &self, - _commits: CertifiedCommits, - ) -> Result, CoreError> { - todo!() - } - - async fn new_block(&self, _round: Round, _force: bool) -> Result<(), CoreError> { - Ok(()) - } - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - Ok(Default::default()) - } - - fn set_quorum_subscribers_exists(&self, _exists: bool) -> Result<(), CoreError> { - todo!() - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - _delay: Round, - _received_quorum_rounds: Vec, - _accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - todo!() - } - - fn set_last_known_proposed_round(&self, _round: Round) -> Result<(), CoreError> { - todo!() - } - - fn highest_received_rounds(&self) -> Vec { - todo!() - } - } - - #[derive(Default)] - pub(crate) struct FakeNetworkClient {} - - #[async_trait] - impl NetworkClient for FakeNetworkClient { - const SUPPORT_STREAMING: bool = false; - - async fn send_block( - &self, - _peer: AuthorityIndex, - _block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - unimplemented!("Unimplemented") - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - unimplemented!("Unimplemented") - } - - async fn fetch_blocks( - &self, - _peer: AuthorityIndex, - _block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn get_latest_rounds( - &self, - _peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - } - - #[rstest] - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_handle_send_block(#[values(false, true)] median_based_timestamp: bool) { - let (mut context, _keys) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - median_based_timestamp, - ); - let context = Arc::new(context); - let block_verifier = Arc::new(crate::block_verifier::NoopBlockVerifier {}); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(FakeCoreThreadDispatcher::new()); - let (_tx_block_broadcast, rx_block_broadcast) = broadcast::channel(100); - let network_client = Arc::new(FakeNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let synchronizer = Synchronizer::start( - network_client, - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier.clone(), - dag_state.clone(), - false, - ); - let authority_service = Arc::new(AuthorityService::new( - context.clone(), - block_verifier, - commit_vote_monitor, - synchronizer, - core_dispatcher.clone(), - rx_block_broadcast, - dag_state, - store, - )); - - // Test delaying blocks with time drift. - let now = context.clock.timestamp_utc_ms(); - let max_drift = context.parameters.max_forward_time_drift; - let input_block = VerifiedBlock::new_for_test( - TestBlock::new(9, 0) - .set_timestamp_ms(now + max_drift.as_millis() as u64) - .build(), - ); - - let service = authority_service.clone(); - let serialized = ExtendedSerializedBlock { - block: input_block.serialized().clone(), - excluded_ancestors: vec![], - }; - - tokio::spawn(async move { - service - .handle_send_block(context.committee.to_authority_index(0).unwrap(), serialized) - .await - .unwrap(); - }); - - sleep(max_drift / 2).await; - - if !median_based_timestamp { - assert!(core_dispatcher.get_blocks().is_empty()); - sleep(max_drift).await; - } - - let blocks = core_dispatcher.get_blocks(); - assert_eq!(blocks.len(), 1); - assert_eq!(blocks[0], input_block); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_handle_fetch_latest_blocks() { - // GIVEN - let (context, _keys) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(crate::block_verifier::NoopBlockVerifier {}); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(FakeCoreThreadDispatcher::new()); - let (_tx_block_broadcast, rx_block_broadcast) = broadcast::channel(100); - let network_client = Arc::new(FakeNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let synchronizer = Synchronizer::start( - network_client, - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier.clone(), - dag_state.clone(), - true, - ); - let authority_service = Arc::new(AuthorityService::new( - context.clone(), - block_verifier, - commit_vote_monitor, - synchronizer, - core_dispatcher.clone(), - rx_block_broadcast, - dag_state.clone(), - store, - )); - - // Create some blocks for a few authorities. Create some equivocations as well - // and store in dag state. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder - .layers(1..=10) - .authorities(vec![AuthorityIndex::new_for_test(2)]) - .equivocate(1) - .build() - .persist_layers(dag_state); - - // WHEN - let authorities_to_request = vec![ - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - ]; - let results = authority_service - .handle_fetch_latest_blocks(AuthorityIndex::new_for_test(1), authorities_to_request) - .await; - - // THEN - let serialised_blocks = results.unwrap(); - for serialised_block in serialised_blocks { - let signed_block: SignedBlock = - bcs::from_bytes(&serialised_block).expect("Error while deserialising block"); - let verified_block = VerifiedBlock::new_verified(signed_block, serialised_block); - - assert_eq!(verified_block.round(), 10); - } - } - - /// Tests that handle_fetch_blocks preserves the original request order - /// of block refs when they span the GC boundary — i.e. some are fetched - /// from the persistent store (below GC) and others from in-memory - /// dag_state (at or above GC). The interleaved input order must be - /// maintained in the response. - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_handle_fetch_blocks_commit_sync_order_across_gc_boundary() { - // GIVEN - let rounds = 20; - let gc_depth = 5; - let (mut context, _keys) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_batched_block_sync_for_testing(true); - context.protocol_config.set_gc_depth_for_testing(gc_depth); - let context = Arc::new(context); - let block_verifier = Arc::new(crate::block_verifier::NoopBlockVerifier {}); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(FakeCoreThreadDispatcher::new()); - let (_tx_block_broadcast, rx_block_broadcast) = broadcast::channel(100); - let network_client = Arc::new(FakeNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let synchronizer = Synchronizer::start( - network_client, - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier.clone(), - dag_state.clone(), - true, - ); - let authority_service = Arc::new(AuthorityService::new( - context.clone(), - block_verifier, - commit_vote_monitor, - synchronizer, - core_dispatcher.clone(), - rx_block_broadcast, - dag_state.clone(), - store.clone(), - )); - - // Build DAG and persist all blocks to dag_state - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder - .layers(1..=rounds) - .build() - .persist_layers(dag_state.clone()); - - // Also write all blocks to the store so below-GC refs can be found - let all_blocks = dag_builder.blocks(1..=rounds); - store - .write(WriteBatch::new(all_blocks, vec![], vec![], vec![])) - .expect("Failed to write blocks to store"); - - // Set last_commit so gc_round() = leader_round - gc_depth = 15 - 5 = 10 - let leader_round = 15; - let leader_ref = dag_builder - .blocks(leader_round..=leader_round) - .first() - .unwrap() - .reference(); - let commit = - TrustedCommit::new_for_test(1, CommitDigest::MIN, 0, leader_ref, vec![leader_ref]); - dag_state.write().set_last_commit(commit); - - let gc_round = dag_state.read().gc_round(); - assert!( - gc_round > GENESIS_ROUND && gc_round < rounds, - "GC round {gc_round} should be between genesis and max round" - ); - - // Collect blocks per round for easy access - let mut blocks_by_round: Vec> = vec![vec![]; (rounds + 1) as usize]; - for round in 1..=rounds { - blocks_by_round[round as usize] = dag_builder.blocks(round..=round); - } - - // Create interleaved block_refs that alternate between below-GC and above-GC - let below_gc_rounds: Vec = (1..gc_round).collect(); - let above_gc_rounds: Vec = (gc_round..=rounds).collect(); - let validators = context.committee.size(); - let mut interleaved_refs = Vec::new(); - let max_pairs = std::cmp::min(below_gc_rounds.len(), above_gc_rounds.len()); - for i in 0..max_pairs { - let below_round = below_gc_rounds[i]; - let auth_idx = i % validators; - if auth_idx < blocks_by_round[below_round as usize].len() { - interleaved_refs.push(blocks_by_round[below_round as usize][auth_idx].reference()); - } - let above_round = above_gc_rounds[i]; - let auth_idx2 = (i + 1) % validators; - if auth_idx2 < blocks_by_round[above_round as usize].len() { - interleaved_refs.push(blocks_by_round[above_round as usize][auth_idx2].reference()); - } - } - - // Verify we have refs from both sides of the GC boundary - assert!( - interleaved_refs.iter().any(|r| r.round < gc_round), - "Should have refs below GC round" - ); - assert!( - interleaved_refs.iter().any(|r| r.round >= gc_round), - "Should have refs above GC round" - ); - - // WHEN: call handle_fetch_blocks with empty highest_accepted_rounds (commit - // sync path) - let peer = context.committee.to_authority_index(1).unwrap(); - let returned_blocks = authority_service - .handle_fetch_blocks(peer, interleaved_refs.clone(), vec![]) - .await - .expect("Should return valid serialized blocks"); - - // THEN: each returned block should match the corresponding input ref - assert_eq!( - returned_blocks.len(), - interleaved_refs.len(), - "Should receive all requested blocks" - ); - for (i, serialized_block) in returned_blocks.into_iter().enumerate() { - let signed_block: SignedBlock = - bcs::from_bytes(&serialized_block).expect("Error while deserialising block"); - let verified_block = VerifiedBlock::new_verified(signed_block, serialized_block); - assert_eq!( - verified_block.reference(), - interleaved_refs[i], - "Block at index {i} should match requested ref. \ - Expected {:?}, got {:?}", - interleaved_refs[i], - verified_block.reference() - ); - } - } -} diff --git a/consensus/core/src/base_committer.rs b/consensus/core/src/base_committer.rs deleted file mode 100644 index 565a1c13ec5..00000000000 --- a/consensus/core/src/base_committer.rs +++ /dev/null @@ -1,487 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashMap, fmt::Display, sync::Arc}; - -use consensus_config::{AuthorityIndex, Stake}; -use parking_lot::RwLock; -use tracing::warn; - -use crate::{ - block::{BlockAPI, BlockRef, Round, Slot, VerifiedBlock}, - commit::{DEFAULT_WAVE_LENGTH, LeaderStatus, MINIMUM_WAVE_LENGTH, WaveNumber}, - context::Context, - dag_state::DagState, - leader_schedule::LeaderSchedule, - stake_aggregator::{QuorumThreshold, StakeAggregator}, -}; - -#[cfg(test)] -#[path = "tests/base_committer_tests.rs"] -mod base_committer_tests; - -#[cfg(test)] -#[path = "tests/base_committer_declarative_tests.rs"] -mod base_committer_declarative_tests; - -pub(crate) struct BaseCommitterOptions { - /// TODO: Re-evaluate if we want this to be configurable after running - /// experiments. The length of a wave (minimum 3) - pub wave_length: u32, - /// The offset used in the leader-election protocol. This is used by the - /// multi-committer to ensure that each [`BaseCommitter`] instance elects - /// a different leader. - pub leader_offset: u32, - /// The offset of the first wave. This is used by the pipelined committer to - /// ensure that each[`BaseCommitter`] instances operates on a different - /// view of the dag. - pub round_offset: u32, -} - -impl Default for BaseCommitterOptions { - fn default() -> Self { - Self { - wave_length: DEFAULT_WAVE_LENGTH, - leader_offset: 0, - round_offset: 0, - } - } -} - -/// The [`BaseCommitter`] contains the bare bone commit logic. Once -/// instantiated, the method `try_direct_decide` and `try_indirect_decide` can -/// be called at any time and any number of times (it is idempotent) to -/// determine whether a leader can be committed or skipped. -pub(crate) struct BaseCommitter { - /// The per-epoch configuration of this authority. - context: Arc, - /// The consensus leader schedule to be used to resolve the leader for a - /// given round. - leader_schedule: Arc, - /// In memory block store representing the dag state - dag_state: Arc>, - /// The options used by this committer - options: BaseCommitterOptions, -} - -impl BaseCommitter { - pub fn new( - context: Arc, - leader_schedule: Arc, - dag_state: Arc>, - options: BaseCommitterOptions, - ) -> Self { - assert!(options.wave_length >= MINIMUM_WAVE_LENGTH); - Self { - context, - leader_schedule, - dag_state, - options, - } - } - - /// Apply the direct decision rule to the specified leader to see whether we - /// can direct-commit or direct-skip it. - #[tracing::instrument(skip_all, fields(leader = %leader))] - pub fn try_direct_decide(&self, leader: Slot) -> LeaderStatus { - // Check whether the leader has enough blame. That is, whether there are 2f+1 - // non-votes for that leader (which ensure there will never be a - // certificate for that leader). - let voting_round = leader.round + 1; - if self.enough_leader_blame(voting_round, leader.authority) { - return LeaderStatus::Skip(leader); - } - - // Check whether the leader(s) has enough support. That is, whether there are - // 2f+1 certificates over the leader. Note that there could be more than - // one leader block (created by Byzantine leaders). - let wave = self.wave_number(leader.round); - let decision_round = self.decision_round(wave); - let leader_blocks = self.dag_state.read().get_uncommitted_blocks_at_slot(leader); - let mut leaders_with_enough_support: Vec<_> = leader_blocks - .into_iter() - .filter(|l| self.enough_leader_support(decision_round, l)) - .map(LeaderStatus::Commit) - .collect(); - - // There can be at most one leader with enough support for each round, otherwise - // it means the BFT assumption is broken. - if leaders_with_enough_support.len() > 1 { - panic!("[{self}] More than one certified block for {leader}") - } - - leaders_with_enough_support - .pop() - .unwrap_or(LeaderStatus::Undecided(leader)) - } - - /// Apply the indirect decision rule to the specified leader to see whether - /// we can indirect-commit or indirect-skip it. - #[tracing::instrument(skip_all, fields(leader = %leader_slot))] - pub fn try_indirect_decide<'a>( - &self, - leader_slot: Slot, - leaders: impl Iterator, - ) -> LeaderStatus { - // The anchor is the first committed leader with round higher than the decision - // round of the target leader. We must stop the iteration upon - // encountering an undecided leader. - let anchors = leaders.filter(|x| leader_slot.round + self.options.wave_length <= x.round()); - - for anchor in anchors { - tracing::trace!( - "[{self}] Trying to indirect-decide {leader_slot} using anchor {anchor}", - ); - match anchor { - LeaderStatus::Commit(anchor) => { - return self.decide_leader_from_anchor(anchor, leader_slot); - } - LeaderStatus::Skip(..) => (), - LeaderStatus::Undecided(..) => break, - } - } - - LeaderStatus::Undecided(leader_slot) - } - - pub fn elect_leader(&self, round: Round) -> Option { - let wave = self.wave_number(round); - tracing::trace!( - "elect_leader: round={}, wave={}, leader_round={}, leader_offset={}", - round, - wave, - self.leader_round(wave), - self.options.leader_offset - ); - if self.leader_round(wave) != round { - return None; - } - - Some(Slot::new( - round, - self.leader_schedule - .elect_leader(round, self.options.leader_offset), - )) - } - - /// Return the leader round of the specified wave. The leader round is - /// always the first round of the wave. This takes into account round - /// offset for when pipelining is enabled. - pub(crate) fn leader_round(&self, wave: WaveNumber) -> Round { - (wave * self.options.wave_length) + self.options.round_offset - } - - /// Return the decision round of the specified wave. The decision round is - /// always the last round of the wave. This takes into account round offset - /// for when pipelining is enabled. - pub(crate) fn decision_round(&self, wave: WaveNumber) -> Round { - let wave_length = self.options.wave_length; - (wave * wave_length) + wave_length - 1 + self.options.round_offset - } - - /// Return the wave in which the specified round belongs. This takes into - /// account the round offset for when pipelining is enabled. - pub(crate) fn wave_number(&self, round: Round) -> WaveNumber { - round.saturating_sub(self.options.round_offset) / self.options.wave_length - } - - /// Find which block is supported at a slot (author, round) by the given - /// block. Blocks can indirectly reference multiple other blocks at a - /// slot, but only one block at a slot will be supported by the given - /// block. If block A supports B at a slot, it is guaranteed that any - /// processed block by the same author that directly or indirectly - /// includes A will also support B at that slot. - fn find_supported_block(&self, leader_slot: Slot, from: &VerifiedBlock) -> Option { - if from.round() < leader_slot.round { - return None; - } - for ancestor in from.ancestors() { - if Slot::from(*ancestor) == leader_slot { - return Some(*ancestor); - } - // Weak links may point to blocks with lower round numbers than strong links. - if ancestor.round <= leader_slot.round { - continue; - } - let ancestor = self - .dag_state - .read() - .get_block(ancestor) - .unwrap_or_else(|| panic!("Block not found in storage: {ancestor:?}")); - if let Some(support) = self.find_supported_block(leader_slot, &ancestor) { - return Some(support); - } - } - None - } - - /// Check whether the specified block (`potential_vote`) is a vote for - /// the specified leader (`leader_block`). - fn is_vote(&self, potential_vote: &VerifiedBlock, leader_block: &VerifiedBlock) -> bool { - let reference = leader_block.reference(); - let leader_slot = Slot::from(reference); - self.find_supported_block(leader_slot, potential_vote) == Some(reference) - } - - /// Check whether the specified block (`potential_certificate`) is a - /// certificate for the specified leader (`leader_block`). An - /// `all_votes` map can be provided as a cache to quickly skip checking - /// against the block store on whether a reference is a vote. This is - /// done for efficiency. Bear in mind that the `all_votes` should refer - /// to votes considered to the same `leader_block` and it can't be - /// reused for different leaders. - fn is_certificate( - &self, - potential_certificate: &VerifiedBlock, - leader_block: &VerifiedBlock, - all_votes: &mut HashMap, - ) -> bool { - let (gc_enabled, gc_round) = { - let dag_state = self.dag_state.read(); - (dag_state.gc_enabled(), dag_state.gc_round()) - }; - - let mut votes_stake_aggregator = StakeAggregator::::new(); - for reference in potential_certificate.ancestors() { - let is_vote = if let Some(is_vote) = all_votes.get(reference) { - *is_vote - } else { - let potential_vote = self.dag_state.read().get_block(reference); - - let is_vote = if gc_enabled { - if let Some(potential_vote) = potential_vote { - self.is_vote(&potential_vote, leader_block) - } else { - assert!( - reference.round <= gc_round, - "Block not found in storage: {reference:?} , and is not below gc_round: {gc_round}" - ); - false - } - } else { - let potential_vote = potential_vote - .unwrap_or_else(|| panic!("Block not found in storage: {reference:?}")); - self.is_vote(&potential_vote, leader_block) - }; - - all_votes.insert(*reference, is_vote); - is_vote - }; - - if is_vote { - tracing::trace!("[{self}] {reference} is a vote for {leader_block}"); - if votes_stake_aggregator.add(reference.author, &self.context.committee) { - tracing::trace!( - "[{self}] {potential_certificate} is a certificate for leader {leader_block}" - ); - return true; - } - } else { - tracing::trace!("[{self}] {reference} is not a vote for {leader_block}",); - } - } - tracing::trace!( - "[{self}] {potential_certificate} is not a certificate for leader {leader_block}" - ); - false - } - - /// Decide the status of a target leader from the specified anchor. We - /// commit the target leader if it has a certified link to the anchor. - /// Otherwise, we skip the target leader. - fn decide_leader_from_anchor(&self, anchor: &VerifiedBlock, leader_slot: Slot) -> LeaderStatus { - // Get the block(s) proposed by the leader. There could be more than one leader - // block in the slot from a Byzantine authority. - let leader_blocks = self - .dag_state - .read() - .get_uncommitted_blocks_at_slot(leader_slot); - - // TODO: Re-evaluate this check once we have a better way to handle/track - // byzantine authorities. - if leader_blocks.len() > 1 { - tracing::warn!( - "Multiple blocks found for leader slot {leader_slot}: {:?}", - leader_blocks - ); - } - - // Get all blocks that could be potential certificates for the target leader. - // These blocks are in the decision round of the target leader and are - // linked to the anchor. - let wave = self.wave_number(leader_slot.round); - let decision_round = self.decision_round(wave); - let potential_certificates = self - .dag_state - .read() - .ancestors_at_round(anchor, decision_round); - - // Use those potential certificates to determine which (if any) of the target - // leader blocks can be committed. - let mut certified_leader_blocks: Vec<_> = leader_blocks - .into_iter() - .filter(|leader_block| { - let mut all_votes = HashMap::new(); - potential_certificates.iter().any(|potential_certificate| { - self.is_certificate(potential_certificate, leader_block, &mut all_votes) - }) - }) - .collect(); - - // There can be at most one certified leader, otherwise it means the BFT - // assumption is broken. - if certified_leader_blocks.len() > 1 { - panic!("More than one certified block at wave {wave} from leader {leader_slot}") - } - - // We commit the target leader if it has a certificate that is an ancestor of - // the anchor. Otherwise skip it. - match certified_leader_blocks.pop() { - Some(certified_leader_block) => LeaderStatus::Commit(certified_leader_block), - None => LeaderStatus::Skip(leader_slot), - } - } - - /// Check whether the specified leader has 2f+1 non-votes (blames) to be - /// directly skipped. - fn enough_leader_blame(&self, voting_round: Round, leader: AuthorityIndex) -> bool { - let voting_blocks = self - .dag_state - .read() - .get_uncommitted_blocks_at_round(voting_round); - - let mut blame_stake_aggregator = StakeAggregator::::new(); - for voting_block in &voting_blocks { - let voter = voting_block.reference().author; - if voting_block - .ancestors() - .iter() - .all(|ancestor| ancestor.author != leader) - { - tracing::trace!( - "[{self}] {voting_block} is a blame for leader {}", - Slot::new(voting_round - 1, leader) - ); - if blame_stake_aggregator.add(voter, &self.context.committee) { - return true; - } - } else { - tracing::trace!( - "[{self}] {voting_block} is not a blame for leader {}", - Slot::new(voting_round - 1, leader) - ); - } - } - false - } - - /// Check whether the specified leader has 2f+1 certificates to be directly - /// committed. - fn enough_leader_support(&self, decision_round: Round, leader_block: &VerifiedBlock) -> bool { - let decision_blocks = self - .dag_state - .read() - .get_uncommitted_blocks_at_round(decision_round); - - // Quickly reject if there isn't enough stake to support the leader from - // the potential certificates. - let total_stake: Stake = decision_blocks - .iter() - .map(|b| self.context.committee.stake(b.author())) - .sum(); - if !self.context.committee.reached_quorum(total_stake) { - tracing::debug!( - "Not enough support for {leader_block}. Stake not enough: {total_stake} < {}", - self.context.committee.quorum_threshold() - ); - return false; - } - - let mut certificate_stake_aggregator = StakeAggregator::::new(); - let mut all_votes = HashMap::new(); - for decision_block in &decision_blocks { - let authority = decision_block.reference().author; - if self.is_certificate(decision_block, leader_block, &mut all_votes) - && certificate_stake_aggregator.add(authority, &self.context.committee) - { - return true; - } - } - false - } -} - -impl Display for BaseCommitter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Committer-L{}-R{}", - self.options.leader_offset, self.options.round_offset - ) - } -} - -/// A builder for the base committer. By default, the builder creates a base -/// committer that has no leader or round offset. Which indicates single leader -/// & pipelining disabled. -#[cfg(test)] -mod base_committer_builder { - use super::*; - use crate::leader_schedule::LeaderSwapTable; - - pub(crate) struct BaseCommitterBuilder { - context: Arc, - dag_state: Arc>, - wave_length: u32, - leader_offset: u32, - round_offset: u32, - } - - impl BaseCommitterBuilder { - pub(crate) fn new(context: Arc, dag_state: Arc>) -> Self { - Self { - context, - dag_state, - wave_length: DEFAULT_WAVE_LENGTH, - leader_offset: 0, - round_offset: 0, - } - } - - #[expect(unused)] - pub(crate) fn with_wave_length(mut self, wave_length: u32) -> Self { - self.wave_length = wave_length; - self - } - - #[expect(unused)] - pub(crate) fn with_leader_offset(mut self, leader_offset: u32) -> Self { - self.leader_offset = leader_offset; - self - } - - #[expect(unused)] - pub(crate) fn with_round_offset(mut self, round_offset: u32) -> Self { - self.round_offset = round_offset; - self - } - - pub(crate) fn build(self) -> BaseCommitter { - let options = BaseCommitterOptions { - wave_length: self.wave_length, - leader_offset: self.leader_offset, - round_offset: self.round_offset, - }; - BaseCommitter::new( - self.context.clone(), - Arc::new(LeaderSchedule::new( - self.context, - LeaderSwapTable::default(), - )), - self.dag_state, - options, - ) - } - } -} diff --git a/consensus/core/src/block.rs b/consensus/core/src/block.rs deleted file mode 100644 index 18160403c41..00000000000 --- a/consensus/core/src/block.rs +++ /dev/null @@ -1,718 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - fmt, - hash::{Hash, Hasher}, - ops::Deref, - sync::Arc, -}; - -use bytes::Bytes; -use consensus_config::{ - AuthorityIndex, DIGEST_LENGTH, DefaultHashFunction, Epoch, ProtocolKeyPair, - ProtocolKeySignature, ProtocolPublicKey, -}; -use enum_dispatch::enum_dispatch; -use fastcrypto::hash::{Digest, HashFunction}; -use iota_sdk_types::crypto::{Intent, IntentMessage, IntentScope}; -use serde::{Deserialize, Serialize}; -use tracing::instrument; - -use crate::{ - commit::CommitVote, - context::Context, - ensure, - error::{ConsensusError, ConsensusResult}, -}; - -/// Round number of a block. -pub type Round = u32; - -pub(crate) const GENESIS_ROUND: Round = 0; - -/// Block proposal as epoch UNIX timestamp in milliseconds. -pub type BlockTimestampMs = u64; - -/// IOTA transaction in serialised bytes -#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Default, Debug)] -pub struct Transaction { - data: Bytes, -} - -impl Transaction { - pub fn new(data: Vec) -> Self { - Self { data: data.into() } - } - - pub fn data(&self) -> &[u8] { - &self.data - } - - pub fn into_data(self) -> Bytes { - self.data - } -} - -/// A block includes references to previous round blocks and transactions that -/// the authority considers valid. -/// Well behaved authorities produce at most one block per round, but malicious -/// authorities can equivocate. -#[derive(Clone, Deserialize, Serialize)] -#[enum_dispatch(BlockAPI)] -pub enum Block { - V1(BlockV1), -} - -#[enum_dispatch] -pub trait BlockAPI { - fn epoch(&self) -> Epoch; - fn round(&self) -> Round; - fn author(&self) -> AuthorityIndex; - fn slot(&self) -> Slot; - fn timestamp_ms(&self) -> BlockTimestampMs; - fn ancestors(&self) -> &[BlockRef]; - fn transactions(&self) -> &[Transaction]; - fn commit_votes(&self) -> &[CommitVote]; - fn misbehavior_reports(&self) -> &[MisbehaviorReport]; -} - -#[derive(Clone, Default, Deserialize, Serialize)] -pub struct BlockV1 { - epoch: Epoch, - round: Round, - author: AuthorityIndex, - // TODO: during verification ensure that timestamp_ms >= ancestors.timestamp - timestamp_ms: BlockTimestampMs, - ancestors: Vec, - transactions: Vec, - commit_votes: Vec, - // This form of misbehavior report is now deprecated - misbehavior_reports: Vec, -} - -impl BlockV1 { - pub(crate) fn new( - epoch: Epoch, - round: Round, - author: AuthorityIndex, - timestamp_ms: BlockTimestampMs, - ancestors: Vec, - transactions: Vec, - commit_votes: Vec, - misbehavior_reports: Vec, - ) -> BlockV1 { - Self { - epoch, - round, - author, - timestamp_ms, - ancestors, - transactions, - commit_votes, - misbehavior_reports, - } - } - - fn genesis_block(context: &Context, author: AuthorityIndex) -> Self { - let timestamp_ms = if context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - context.epoch_start_timestamp_ms - } else { - 0 - }; - Self { - epoch: context.committee.epoch(), - round: GENESIS_ROUND, - author, - timestamp_ms, - ancestors: vec![], - transactions: vec![], - commit_votes: vec![], - misbehavior_reports: vec![], - } - } -} - -impl BlockAPI for BlockV1 { - fn epoch(&self) -> Epoch { - self.epoch - } - - fn round(&self) -> Round { - self.round - } - - fn author(&self) -> AuthorityIndex { - self.author - } - - fn slot(&self) -> Slot { - Slot::new(self.round, self.author) - } - - fn timestamp_ms(&self) -> BlockTimestampMs { - self.timestamp_ms - } - - fn ancestors(&self) -> &[BlockRef] { - &self.ancestors - } - - fn transactions(&self) -> &[Transaction] { - &self.transactions - } - - fn commit_votes(&self) -> &[CommitVote] { - &self.commit_votes - } - - fn misbehavior_reports(&self) -> &[MisbehaviorReport] { - &self.misbehavior_reports - } -} - -/// `BlockRef` uniquely identifies a `VerifiedBlock` via `digest`. It also -/// contains the slot info (round and author) so it can be used in logic such as -/// aggregating stakes for a round. -#[derive(Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] -pub struct BlockRef { - pub round: Round, - pub author: AuthorityIndex, - pub digest: BlockDigest, -} - -impl BlockRef { - pub const MIN: Self = Self { - round: 0, - author: AuthorityIndex::MIN, - digest: BlockDigest::MIN, - }; - - pub const MAX: Self = Self { - round: u32::MAX, - author: AuthorityIndex::MAX, - digest: BlockDigest::MAX, - }; - - pub fn new(round: Round, author: AuthorityIndex, digest: BlockDigest) -> Self { - Self { - round, - author, - digest, - } - } -} - -impl fmt::Display for BlockRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "B{}({},{})", self.round, self.author, self.digest) - } -} - -impl fmt::Debug for BlockRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - fmt::Display::fmt(self, f) - } -} - -impl Hash for BlockRef { - fn hash(&self, state: &mut H) { - state.write(&self.digest.0[..8]); - } -} - -/// Digest of a `VerifiedBlock` or verified `SignedBlock`, which covers the -/// `Block` and its signature. -/// -/// Note: the signature algorithm is assumed to be non-malleable, so it is -/// impossible for another party to create an altered but valid signature, -/// producing an equivocating `BlockDigest`. -#[derive(Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] -pub struct BlockDigest([u8; consensus_config::DIGEST_LENGTH]); - -impl BlockDigest { - /// Lexicographic min & max digest. - pub const MIN: Self = Self([u8::MIN; consensus_config::DIGEST_LENGTH]); - pub const MAX: Self = Self([u8::MAX; consensus_config::DIGEST_LENGTH]); -} - -impl Hash for BlockDigest { - fn hash(&self, state: &mut H) { - state.write(&self.0[..8]); - } -} - -impl From for Digest<{ DIGEST_LENGTH }> { - fn from(hd: BlockDigest) -> Self { - Digest::new(hd.0) - } -} - -impl fmt::Display for BlockDigest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!( - f, - "{}", - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, self.0) - .get(0..4) - .ok_or(fmt::Error)? - ) - } -} - -impl fmt::Debug for BlockDigest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!( - f, - "{}", - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, self.0) - ) - } -} - -impl AsRef<[u8]> for BlockDigest { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -/// Slot is the position of blocks in the DAG. It can contain 0, 1 or multiple -/// blocks from the same authority at the same round. -#[derive(Clone, Copy, PartialEq, PartialOrd, Default, Hash)] -pub struct Slot { - pub round: Round, - pub authority: AuthorityIndex, -} - -impl Slot { - pub fn new(round: Round, authority: AuthorityIndex) -> Self { - Self { round, authority } - } - - #[cfg(test)] - pub fn new_for_test(round: Round, authority: u32) -> Self { - Self { - round, - authority: AuthorityIndex::new_for_test(authority), - } - } -} - -impl From for Slot { - fn from(value: BlockRef) -> Self { - Slot::new(value.round, value.author) - } -} - -impl fmt::Display for Slot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "S{}{}", self.round, self.authority) - } -} - -impl fmt::Debug for Slot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self, f) - } -} - -/// A Block with its signature, before they are verified. -/// -/// Note: `BlockDigest` is computed over this struct, so any added field -/// (without `#[serde(skip)]`) will affect the values of `BlockDigest` and -/// `BlockRef`. -#[derive(Deserialize, Serialize)] -pub(crate) struct SignedBlock { - inner: Block, - signature: Bytes, -} - -impl SignedBlock { - /// Should only be used when constructing the genesis blocks - pub(crate) fn new_genesis(block: Block) -> Self { - Self { - inner: block, - signature: Bytes::default(), - } - } - - pub(crate) fn new(block: Block, protocol_keypair: &ProtocolKeyPair) -> ConsensusResult { - let signature = compute_block_signature(&block, protocol_keypair)?; - Ok(Self { - inner: block, - signature: Bytes::copy_from_slice(signature.to_bytes()), - }) - } - - pub(crate) fn signature(&self) -> &Bytes { - &self.signature - } - - /// This method only verifies this block's signature. Verification of the - /// full block should be done via BlockVerifier. - #[instrument(level = "trace", skip_all)] - pub(crate) fn verify_signature(&self, context: &Context) -> ConsensusResult<()> { - let block = &self.inner; - let committee = &context.committee; - ensure!( - committee.is_valid_index(block.author()), - ConsensusError::InvalidAuthorityIndex { - index: block.author(), - max: committee.size() - 1 - } - ); - let authority = committee.authority(block.author()); - verify_block_signature(block, self.signature(), &authority.protocol_key) - } - - /// Serialises the block using the bcs serializer - pub(crate) fn serialize(&self) -> Result { - let bytes = bcs::to_bytes(self)?; - Ok(bytes.into()) - } - - /// Clears signature for testing. - #[cfg(test)] - pub(crate) fn clear_signature(&mut self) { - self.signature = Bytes::default(); - } -} - -/// Digest of a block, covering all `Block` fields without its signature. -/// This is used during Block signing and signature verification. -/// This should never be used outside of this file, to avoid confusion with -/// `BlockDigest`. -#[derive(Serialize, Deserialize)] -struct InnerBlockDigest([u8; consensus_config::DIGEST_LENGTH]); - -/// Computes the digest of a Block, only for signing and verifications. -fn compute_inner_block_digest(block: &Block) -> ConsensusResult { - let mut hasher = DefaultHashFunction::new(); - hasher.update(bcs::to_bytes(block).map_err(ConsensusError::SerializationFailure)?); - Ok(InnerBlockDigest(hasher.finalize().into())) -} - -/// Wrap a InnerBlockDigest in the intent message. -fn to_consensus_block_intent(digest: InnerBlockDigest) -> IntentMessage { - IntentMessage::new(Intent::consensus_app(IntentScope::ConsensusBlock), digest) -} - -/// Process for signing & verying a block signature: -/// 1. Compute the digest of `Block`. -/// 2. Wrap the digest in `IntentMessage`. -/// 3. Sign the serialized `IntentMessage`, or verify signature against it. -#[instrument(level = "trace", skip_all)] -fn compute_block_signature( - block: &Block, - protocol_keypair: &ProtocolKeyPair, -) -> ConsensusResult { - let digest = compute_inner_block_digest(block)?; - let message = bcs::to_bytes(&to_consensus_block_intent(digest)) - .map_err(ConsensusError::SerializationFailure)?; - Ok(protocol_keypair.sign(&message)) -} -#[instrument(level = "trace", skip_all)] -fn verify_block_signature( - block: &Block, - signature: &[u8], - protocol_pubkey: &ProtocolPublicKey, -) -> ConsensusResult<()> { - let digest = compute_inner_block_digest(block)?; - let message = bcs::to_bytes(&to_consensus_block_intent(digest)) - .map_err(ConsensusError::SerializationFailure)?; - let sig = - ProtocolKeySignature::from_bytes(signature).map_err(ConsensusError::MalformedSignature)?; - protocol_pubkey - .verify(&message, &sig) - .map_err(ConsensusError::SignatureVerificationFailure) -} - -/// Allow quick access on the underlying Block without having to always refer to -/// the inner block ref. -impl Deref for SignedBlock { - type Target = Block; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -/// VerifiedBlock allows full access to its content. -/// Note: clone() is relatively cheap with most underlying data refcounted. -#[derive(Clone)] -pub struct VerifiedBlock { - block: Arc, - - // Cached Block digest and serialized SignedBlock, to avoid re-computing these values. - digest: BlockDigest, - serialized: Bytes, -} - -impl VerifiedBlock { - /// Creates VerifiedBlock from a verified SignedBlock and its serialized - /// bytes. - pub(crate) fn new_verified(signed_block: SignedBlock, serialized: Bytes) -> Self { - let digest = Self::compute_digest(&serialized); - VerifiedBlock { - block: Arc::new(signed_block), - digest, - serialized, - } - } - - pub(crate) fn new_verified_with_digest( - signed_block: SignedBlock, - serialized: Bytes, - digest: BlockDigest, - ) -> Self { - VerifiedBlock { - block: Arc::new(signed_block), - digest, - serialized, - } - } - - /// This method is public for testing in other crates. - pub fn new_for_test(block: Block) -> Self { - let signed_block = SignedBlock { - inner: block, - signature: Default::default(), - }; - let serialized: Bytes = bcs::to_bytes(&signed_block) - .expect("Serialization should not fail") - .into(); - let digest = Self::compute_digest(&serialized); - VerifiedBlock { - block: Arc::new(signed_block), - digest, - serialized, - } - } - - /// Returns reference to the block. - pub fn reference(&self) -> BlockRef { - BlockRef { - round: self.round(), - author: self.author(), - digest: self.digest(), - } - } - - pub(crate) fn digest(&self) -> BlockDigest { - self.digest - } - - /// Returns the serialized block with signature. - pub(crate) fn serialized(&self) -> &Bytes { - &self.serialized - } - - /// Computes digest from the serialized block with signature. - pub(crate) fn compute_digest(serialized: &[u8]) -> BlockDigest { - let mut hasher = DefaultHashFunction::new(); - hasher.update(serialized); - BlockDigest(hasher.finalize().into()) - } -} - -/// Allow quick access on the underlying Block without having to always refer to -/// the inner block ref. -impl Deref for VerifiedBlock { - type Target = Block; - - fn deref(&self) -> &Self::Target { - &self.block.inner - } -} - -impl PartialEq for VerifiedBlock { - fn eq(&self, other: &Self) -> bool { - self.digest() == other.digest() - } -} - -impl fmt::Display for VerifiedBlock { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "{}", self.reference()) - } -} - -impl fmt::Debug for VerifiedBlock { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!( - f, - "{:?}({}ms;{:?};{}t;{}c)", - self.reference(), - self.timestamp_ms(), - self.ancestors(), - self.transactions().len(), - self.commit_votes().len(), - ) - } -} - -/// Block with extended additional information, such as -/// local blocks that are excluded from the block's ancestors. -/// The extended information do not need to be certified or forwarded to other -/// authorities. -#[derive(Clone, Debug)] -pub(crate) struct ExtendedBlock { - pub block: VerifiedBlock, - pub excluded_ancestors: Vec, -} - -/// Generates the genesis blocks for the current Committee. -/// The blocks are returned in authority index order. -pub(crate) fn genesis_blocks(context: &Context) -> Vec { - context - .committee - .authorities() - .map(|(authority_index, _)| { - let signed_block = SignedBlock::new_genesis(Block::V1(BlockV1::genesis_block( - context, - authority_index, - ))); - let serialized = signed_block - .serialize() - .expect("Genesis block serialization failed."); - // Unnecessary to verify genesis blocks. - VerifiedBlock::new_verified(signed_block, serialized) - }) - .collect::>() -} - -/// This struct is public for testing in other crates. -#[derive(Clone)] -pub struct TestBlock { - block: BlockV1, -} - -impl TestBlock { - pub fn new(round: Round, author: u32) -> Self { - Self { - block: BlockV1 { - round, - author: AuthorityIndex::new_for_test(author), - ..Default::default() - }, - } - } - - pub fn set_epoch(mut self, epoch: Epoch) -> Self { - self.block.epoch = epoch; - self - } - - pub fn set_round(mut self, round: Round) -> Self { - self.block.round = round; - self - } - - pub fn set_author(mut self, author: AuthorityIndex) -> Self { - self.block.author = author; - self - } - - pub fn set_timestamp_ms(mut self, timestamp_ms: BlockTimestampMs) -> Self { - self.block.timestamp_ms = timestamp_ms; - self - } - - pub fn set_ancestors(mut self, ancestors: Vec) -> Self { - self.block.ancestors = ancestors; - self - } - - pub fn set_transactions(mut self, transactions: Vec) -> Self { - self.block.transactions = transactions; - self - } - - pub fn set_commit_votes(mut self, commit_votes: Vec) -> Self { - self.block.commit_votes = commit_votes; - self - } - - pub fn build(self) -> Block { - Block::V1(self.block) - } -} - -// This form of misbehavior report is now deprecated -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct MisbehaviorReport { - pub target: AuthorityIndex, - pub proof: MisbehaviorProof, -} - -/// Proof of misbehavior are usually signed block(s) from the misbehaving -/// authority. -#[derive(Clone, Serialize, Deserialize, Debug)] -pub enum MisbehaviorProof { - InvalidBlock(BlockRef), -} - -// TODO: add basic verification for BlockRef and BlockDigest. -// TODO: add tests for SignedBlock and VerifiedBlock conversion. - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use fastcrypto::error::FastCryptoError; - - use crate::{ - BlockAPI, - block::{SignedBlock, TestBlock, genesis_blocks}, - context::Context, - error::ConsensusError, - }; - - #[tokio::test] - async fn test_sign_and_verify() { - let (context, key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - - // Create a block that authority 2 has created - let block = TestBlock::new(10, 2).build(); - - // Create a signed block with authority's 2 private key - let author_two_key = &key_pairs[2].1; - let signed_block = SignedBlock::new(block, author_two_key).expect("Shouldn't fail signing"); - - // Now verify the block's signature - let result = signed_block.verify_signature(&context); - assert!(result.is_ok()); - - // Try to sign authority's 2 block with authority's 1 key - let block = TestBlock::new(10, 2).build(); - let author_one_key = &key_pairs[1].1; - let signed_block = SignedBlock::new(block, author_one_key).expect("Shouldn't fail signing"); - - // Now verify the block, it should fail - let result = signed_block.verify_signature(&context); - match result.err().unwrap() { - ConsensusError::SignatureVerificationFailure(err) => { - assert_eq!(err, FastCryptoError::InvalidSignature); - } - err => panic!("Unexpected error: {err:?}"), - } - } - - #[tokio::test] - async fn test_genesis_blocks() { - let (context, _) = Context::new_for_test(4); - const TIMESTAMP_MS: u64 = 1000; - let context = Arc::new(context.with_epoch_start_timestamp_ms(TIMESTAMP_MS)); - let blocks = genesis_blocks(&context); - for (i, block) in blocks.into_iter().enumerate() { - assert_eq!(block.author().value(), i); - assert_eq!(block.round(), 0); - assert_eq!(block.timestamp_ms(), TIMESTAMP_MS); - } - } -} diff --git a/consensus/core/src/block_manager.rs b/consensus/core/src/block_manager.rs deleted file mode 100644 index 23732e7248c..00000000000 --- a/consensus/core/src/block_manager.rs +++ /dev/null @@ -1,1661 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, BTreeSet, btree_map::Entry}, - sync::Arc, - time::Instant, -}; - -use consensus_config::AuthorityIndex; -use iota_metrics::monitored_scope; -use itertools::Itertools as _; -use parking_lot::RwLock; -use tracing::{debug, instrument, trace, warn}; - -use crate::{ - Round, - block::{BlockAPI, BlockRef, GENESIS_ROUND, VerifiedBlock}, - block_verifier::BlockVerifier, - context::Context, - dag_state::DagState, -}; - -#[derive(Clone)] -pub(crate) struct SuspendedBlock { - block: VerifiedBlock, - missing_ancestors: BTreeSet, - timestamp: Instant, -} - -impl SuspendedBlock { - pub(crate) fn new(block: VerifiedBlock, missing_ancestors: BTreeSet) -> Self { - Self { - block, - missing_ancestors, - timestamp: Instant::now(), - } - } -} - -/// Block manager suspends incoming blocks until they are connected to the -/// existing graph, returning newly connected blocks. -/// TODO: As it is possible to have Byzantine validators who produce Blocks -/// without valid causal history we need to make sure that BlockManager takes -/// care of that and avoid OOM (Out Of Memory) situations. -pub(crate) struct BlockManager { - context: Arc, - dag_state: Arc>, - block_verifier: Arc, - - /// Keeps all the suspended blocks. A suspended block is a block that is - /// missing part of its causal history and thus can't be immediately - /// processed. A block will remain in this map until all its causal history - /// has been successfully processed. - suspended_blocks: BTreeMap, - /// A map that keeps all the blocks that we are missing (keys) and the - /// corresponding blocks that reference the missing blocks as ancestors - /// and need them to get unsuspended. It is possible for a missing - /// dependency (key) to be a suspended block, so the block has been - /// already fetched but it self is still missing some of its ancestors to be - /// processed. - missing_ancestors: BTreeMap>, - /// A map of currently missing blocks to the set of authorities expected - /// to have them available locally. This set is approximated based on the - /// block's author and the authors of its direct children. - /// A block is considered missing if it appears in `missing_ancestors` - /// and has not yet been accepted or fetched. Blocks already stored or - /// present in `suspended_blocks` are excluded. - missing_blocks: BTreeMap>, - /// A vector that holds a tuple of (lowest_round, highest_round) of received - /// blocks per authority. This is used for metrics reporting purposes - /// and resets during restarts. - received_block_rounds: Vec>, -} - -impl BlockManager { - pub(crate) fn new( - context: Arc, - dag_state: Arc>, - block_verifier: Arc, - ) -> Self { - let committee_size = context.committee.size(); - Self { - context, - dag_state, - block_verifier, - suspended_blocks: BTreeMap::new(), - missing_ancestors: BTreeMap::new(), - missing_blocks: BTreeMap::new(), - received_block_rounds: vec![None; committee_size], - } - } - - /// Tries to accept the provided blocks assuming that all their causal - /// history exists. The method returns all the blocks that have been - /// successfully processed in round ascending order, that includes also - /// previously suspended blocks that have now been able to get accepted. - /// Method also returns a set with the missing ancestor blocks. - #[tracing::instrument(skip_all)] - pub(crate) fn try_accept_blocks( - &mut self, - blocks: Vec, - ) -> (Vec, BTreeSet) { - let _s = monitored_scope("BlockManager::try_accept_blocks"); - self.try_accept_blocks_internal(blocks, false) - } - - // Tries to accept blocks that have been committed. Returns all the blocks that - // have been accepted, both from the ones provided and any children blocks. - #[tracing::instrument(skip_all)] - pub(crate) fn try_accept_committed_blocks( - &mut self, - blocks: Vec, - ) -> Vec { - // If GC is disabled then should not run any of this logic. - if !self.dag_state.read().gc_enabled() { - return Vec::new(); - } - - // Just accept the blocks - let _s = monitored_scope("BlockManager::try_accept_committed_blocks"); - let (accepted_blocks, missing_blocks) = self.try_accept_blocks_internal(blocks, true); - assert!( - missing_blocks.is_empty(), - "No missing blocks should be returned for committed blocks" - ); - - accepted_blocks - } - - /// Attempts to accept the provided blocks. When `committed = true` then the - /// blocks are considered to be committed via certified commits and - /// are handled differently. - fn try_accept_blocks_internal( - &mut self, - mut blocks: Vec, - committed: bool, - ) -> (Vec, BTreeSet) { - let _s = monitored_scope("BlockManager::try_accept_blocks_internal"); - - blocks.sort_by_key(|b| b.round()); - debug!( - "Trying to accept blocks: {}", - blocks.iter().map(|b| b.reference().to_string()).join(",") - ); - - let mut accepted_blocks = vec![]; - let mut missing_blocks = BTreeSet::new(); - - for block in blocks { - self.update_block_received_metrics(&block); - - // Try to accept the input block. - let block_ref = block.reference(); - - let mut to_verify_timestamps_and_accept = vec![]; - if committed { - match self.try_accept_one_committed_block(block) { - TryAcceptResult::Accepted(block) => { - // As this is a committed block, then it's already accepted and there is no - // need to verify its timestamps. Just add it to the - // accepted blocks list. - accepted_blocks.push(block); - } - TryAcceptResult::Processed => continue, - TryAcceptResult::Suspended(_) | TryAcceptResult::Skipped => { - panic!("Did not expect to suspend or skip a committed block: {block_ref:?}") - } - }; - } else { - match self.try_accept_one_block(block) { - TryAcceptResult::Accepted(block) => { - to_verify_timestamps_and_accept.push(block); - } - TryAcceptResult::Suspended(ancestors_to_fetch) => { - debug!( - "Missing ancestors to fetch for block {block_ref}: {}", - ancestors_to_fetch.iter().map(|b| b.to_string()).join(",") - ); - missing_blocks.extend(ancestors_to_fetch); - continue; - } - TryAcceptResult::Processed | TryAcceptResult::Skipped => continue, - }; - }; - - // If the block is accepted, try to unsuspend its children blocks if any. - let unsuspended_blocks = self.try_unsuspend_children_blocks(block_ref); - to_verify_timestamps_and_accept.extend(unsuspended_blocks); - - // Verify block timestamps - let blocks_to_accept = - self.verify_block_timestamps_and_accept(to_verify_timestamps_and_accept); - accepted_blocks.extend(blocks_to_accept); - } - - self.update_stats(missing_blocks.len() as u64); - - // Figure out the new missing blocks - (accepted_blocks, missing_blocks) - } - - fn try_accept_one_committed_block(&mut self, block: VerifiedBlock) -> TryAcceptResult { - if self.dag_state.read().contains_block(&block.reference()) { - return TryAcceptResult::Processed; - } - - // Remove the block from missing and suspended blocks - self.missing_blocks.remove(&block.reference()); - - // If the block has been already fetched and parked as suspended block, then - // remove it. Also find all the references of missing ancestors to - // remove those as well. If we don't do that then it's possible once the missing - // ancestor is fetched to cause a panic when trying to unsuspend this - // children as it won't be found in the suspended blocks map. - if let Some(suspended_block) = self.suspended_blocks.remove(&block.reference()) { - suspended_block - .missing_ancestors - .iter() - .for_each(|ancestor| { - if let Some(references) = self.missing_ancestors.get_mut(ancestor) { - references.remove(&block.reference()); - } - }); - } - - // Accept this block before any unsuspended children blocks - self.dag_state.write().accept_blocks(vec![block.clone()]); - - TryAcceptResult::Accepted(block) - } - - /// Tries to find the provided block_refs in DagState and BlockManager, - /// and returns missing block refs. - pub(crate) fn try_find_blocks(&mut self, block_refs: Vec) -> BTreeSet { - let _s = monitored_scope("BlockManager::try_find_blocks"); - let gc_round = self.dag_state.read().gc_round(); - - // No need to fetch blocks that are <= gc_round as they won't get processed - // anyways and they'll get skipped. So keep only the ones above. - let mut block_refs = block_refs - .into_iter() - .filter(|block_ref| block_ref.round > gc_round) - .collect::>(); - - if block_refs.is_empty() { - return BTreeSet::new(); - } - - block_refs.sort_by_key(|b| b.round); - - debug!( - "Trying to find blocks: {}", - block_refs.iter().map(|b| b.to_string()).join(",") - ); - - let mut missing_blocks = BTreeSet::new(); - - for (found, block_ref) in self - .dag_state - .read() - .contains_blocks(block_refs.clone()) - .into_iter() - .zip(block_refs.iter()) - { - if found || self.suspended_blocks.contains_key(block_ref) { - continue; - } - // Fetches the block if it is not in dag state or suspended. - missing_blocks.insert(*block_ref); - if self - .missing_blocks - .insert(*block_ref, BTreeSet::from([block_ref.author])) - .is_none() - { - // We want to report this as a missing ancestor even if there is no block that - // is actually references it right now. That will allow us - // to seamlessly GC the block later if needed. - self.missing_ancestors.entry(*block_ref).or_default(); - - let block_ref_hostname = - &self.context.committee.authority(block_ref.author).hostname; - self.context - .metrics - .node_metrics - .block_manager_missing_blocks_by_authority - .with_label_values(&[block_ref_hostname]) - .inc(); - } - } - - let metrics = &self.context.metrics.node_metrics; - metrics - .missing_blocks_total - .inc_by(missing_blocks.len() as u64); - metrics - .block_manager_missing_blocks - .set(self.missing_blocks.len() as i64); - - missing_blocks - } - - // Persists in store all the valid blocks that should be accepted. Method - // returns the accepted and persisted blocks. - fn verify_block_timestamps_and_accept( - &mut self, - unsuspended_blocks: impl IntoIterator, - ) -> Vec { - // If the median based timestamp is enabled, then we skip all the timestamp - // verification and we go straight and accept the blocks. - let blocks_to_accept = if self - .context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - unsuspended_blocks.into_iter().collect::>() - } else { - self.verify_block_timestamps(unsuspended_blocks) - }; - - // Insert the accepted blocks into DAG state so future blocks including them as - // ancestors do not get suspended. - self.dag_state - .write() - .accept_blocks(blocks_to_accept.clone()); - - blocks_to_accept - } - - // TODO: remove once timestamping is refactored to the new approach. - // Verifies each block's timestamp based on its ancestors.Method returns blocks - // with valid timestamps. - fn verify_block_timestamps( - &mut self, - unsuspended_blocks: impl IntoIterator, - ) -> Vec { - let (gc_enabled, gc_round) = { - let dag_state = self.dag_state.read(); - (dag_state.gc_enabled(), dag_state.gc_round()) - }; - // Try to verify the block and its children for timestamp, with ancestor blocks. - let mut blocks_to_accept: BTreeMap = BTreeMap::new(); - let mut blocks_to_reject: BTreeMap = BTreeMap::new(); - { - 'block: for b in unsuspended_blocks { - let ancestors = self.dag_state.read().get_blocks(b.ancestors()); - assert_eq!(b.ancestors().len(), ancestors.len()); - let mut ancestor_blocks = vec![]; - 'ancestor: for (ancestor_ref, found) in b.ancestors().iter().zip(ancestors) { - if let Some(found_block) = found { - // This invariant should be guaranteed by DagState. - assert_eq!(ancestor_ref, &found_block.reference()); - ancestor_blocks.push(Some(found_block)); - continue 'ancestor; - } - // blocks_to_accept have not been added to DagState yet, but they - // can appear in ancestors. - if blocks_to_accept.contains_key(ancestor_ref) { - ancestor_blocks.push(Some(blocks_to_accept[ancestor_ref].clone())); - continue 'ancestor; - } - // If an ancestor is already rejected, reject this block as well. - if blocks_to_reject.contains_key(ancestor_ref) { - blocks_to_reject.insert(b.reference(), b); - continue 'block; - } - - // When gc is enabled it's possible that we indeed won't find any ancestors that - // are passed gc_round. That's ok. We don't need to panic here. - // We do want to panic if gc_enabled we and have an ancestor that is > gc_round, - // or gc is disabled. - if gc_enabled - && ancestor_ref.round > GENESIS_ROUND - && ancestor_ref.round <= gc_round - { - debug!( - "Block {:?} has a missing ancestor: {:?} passed GC round {}", - b.reference(), - ancestor_ref, - gc_round - ); - ancestor_blocks.push(None); - } else { - panic!( - "Unsuspended block {b:?} has a missing ancestor! Ancestor not found in DagState: {ancestor_ref:?}" - ); - } - } - if let Err(e) = - self.block_verifier - .check_ancestors(&b, &ancestor_blocks, gc_enabled, gc_round) - { - warn!("Block {:?} failed to verify ancestors: {}", b, e); - blocks_to_reject.insert(b.reference(), b); - } else { - blocks_to_accept.insert(b.reference(), b); - } - } - } - - // TODO: report blocks_to_reject to peers. - for (block_ref, block) in blocks_to_reject { - let hostname = self - .context - .committee - .authority(block_ref.author) - .hostname - .clone(); - - self.context - .metrics - .node_metrics - .invalid_blocks - .with_label_values(&[hostname.as_str(), "accept_block", "InvalidAncestors"]) - .inc(); - warn!("Invalid block {:?} is rejected", block); - } - - let blocks_to_accept = blocks_to_accept.values().cloned().collect::>(); - - // Insert the accepted blocks into DAG state so future blocks including them as - // ancestors do not get suspended. - self.dag_state - .write() - .accept_blocks(blocks_to_accept.clone()); - - blocks_to_accept - } - - /// Tries to accept the provided block. To accept a block its ancestors must - /// have been already successfully accepted. If block is accepted then - /// Some result is returned. None is returned when either the block is - /// suspended or the block has been already accepted before. - fn try_accept_one_block(&mut self, block: VerifiedBlock) -> TryAcceptResult { - let block_ref = block.reference(); - let mut missing_ancestors = BTreeSet::new(); - let mut ancestors_to_fetch = BTreeSet::new(); - let dag_state = self.dag_state.read(); - let gc_round = dag_state.gc_round(); - let gc_enabled = dag_state.gc_enabled(); - - // If block has been already received and suspended, or already processed and - // stored, or is a genesis block, then skip it. - if self.suspended_blocks.contains_key(&block_ref) || dag_state.contains_block(&block_ref) { - return TryAcceptResult::Processed; - } - - // If the block is <= gc_round, then we simply skip its processing as there is - // no meaning do any action on it or even store it. - if gc_enabled && block.round() <= gc_round { - let hostname = self - .context - .committee - .authority(block.author()) - .hostname - .as_str(); - self.context - .metrics - .node_metrics - .block_manager_skipped_blocks - .with_label_values(&[hostname]) - .inc(); - return TryAcceptResult::Skipped; - } - - // Keep only the ancestors that are greater than the GC round to check for their - // existence. Keep in mind that if GC is disabled then gc_round will be - // 0 and all ancestors will be considered. - let ancestors = if gc_enabled { - block - .ancestors() - .iter() - .filter(|ancestor| ancestor.round == GENESIS_ROUND || ancestor.round > gc_round) - .cloned() - .collect::>() - } else { - block.ancestors().to_vec() - }; - - // make sure that we have all the required ancestors in store - for (found, ancestor) in dag_state - .contains_blocks(ancestors.clone()) - .into_iter() - .zip(ancestors.iter()) - { - if !found { - missing_ancestors.insert(*ancestor); - - // mark the block as having missing ancestors - self.missing_ancestors - .entry(*ancestor) - .or_default() - .insert(block_ref); - - let ancestor_hostname = &self.context.committee.authority(ancestor.author).hostname; - self.context - .metrics - .node_metrics - .block_manager_missing_ancestors_by_authority - .with_label_values(&[ancestor_hostname]) - .inc(); - - // Add the ancestor to the missing blocks set only if it doesn't already exist - // in the suspended blocks - meaning that we already have its - // payload. - if !self.suspended_blocks.contains_key(ancestor) { - // Fetches the block if it is not in dag state or suspended. - ancestors_to_fetch.insert(*ancestor); - // We also want to keep track of the authorities that have this block. - // This block could be already missing, so we just update the set of - // authorities who have it. - let entry = self.missing_blocks.entry(*ancestor); - match entry { - Entry::Vacant(v) => { - v.insert(BTreeSet::from([ancestor.author, block_ref.author])); - self.context - .metrics - .node_metrics - .block_manager_missing_blocks_by_authority - .with_label_values(&[ancestor_hostname]) - .inc(); - } - Entry::Occupied(mut o) => { - o.get_mut().insert(block_ref.author); - } - } - } - } - } - - // Remove the block ref from the `missing_blocks` - if exists - since we now - // have received the block. The block might still get suspended, but we - // won't report it as missing in order to not re-fetch. - self.missing_blocks.remove(&block.reference()); - - if !missing_ancestors.is_empty() { - let hostname = self - .context - .committee - .authority(block.author()) - .hostname - .as_str(); - self.context - .metrics - .node_metrics - .block_suspensions - .with_label_values(&[hostname]) - .inc(); - self.suspended_blocks - .insert(block_ref, SuspendedBlock::new(block, missing_ancestors)); - return TryAcceptResult::Suspended(ancestors_to_fetch); - } - - TryAcceptResult::Accepted(block) - } - - /// Given an accepted block `accepted_block` it attempts to accept all the - /// suspended children blocks assuming such exist. All the unsuspended / - /// accepted blocks are returned as a vector in causal order. - fn try_unsuspend_children_blocks(&mut self, accepted_block: BlockRef) -> Vec { - let mut unsuspended_blocks = vec![]; - let mut to_process_blocks = vec![accepted_block]; - - while let Some(block_ref) = to_process_blocks.pop() { - // And try to check if its direct children can be unsuspended - if let Some(block_refs_with_missing_deps) = self.missing_ancestors.remove(&block_ref) { - for r in block_refs_with_missing_deps { - // For each dependency try to unsuspend it. If that's successful then we add it - // to the queue so we can recursively try to unsuspend its - // children. - if let Some(block) = self.try_unsuspend_block(&r, &block_ref) { - to_process_blocks.push(block.block.reference()); - unsuspended_blocks.push(block); - } - } - } - } - - let now = Instant::now(); - - // Report the unsuspended blocks - for block in &unsuspended_blocks { - let hostname = self - .context - .committee - .authority(block.block.author()) - .hostname - .as_str(); - self.context - .metrics - .node_metrics - .block_unsuspensions - .with_label_values(&[hostname]) - .inc(); - self.context - .metrics - .node_metrics - .suspended_block_time - .with_label_values(&[hostname]) - .observe(now.saturating_duration_since(block.timestamp).as_secs_f64()); - } - - unsuspended_blocks - .into_iter() - .map(|block| block.block) - .collect() - } - - /// Attempts to unsuspend a block by checking its ancestors and removing the - /// `accepted_dependency` by its local set. If there is no missing - /// dependency then this block can be unsuspended immediately and is removed - /// from the `suspended_blocks` map. - fn try_unsuspend_block( - &mut self, - block_ref: &BlockRef, - accepted_dependency: &BlockRef, - ) -> Option { - let block = self - .suspended_blocks - .get_mut(block_ref) - .expect("Block should be in suspended map"); - - assert!( - block.missing_ancestors.remove(accepted_dependency), - "Block reference {} should be present in missing dependencies of {:?}", - block_ref, - block.block - ); - - if block.missing_ancestors.is_empty() { - // we have no missing dependency, so we unsuspend the block and return it - return self.suspended_blocks.remove(block_ref); - } - None - } - - /// Tries to unsuspend any blocks for the latest gc round. If gc round - /// hasn't changed then no blocks will be unsuspended due to - /// this action. - #[instrument(level = "trace", skip_all)] - pub(crate) fn try_unsuspend_blocks_for_latest_gc_round(&mut self) { - let _s = monitored_scope("BlockManager::try_unsuspend_blocks_for_latest_gc_round"); - let (gc_enabled, gc_round) = { - let dag_state = self.dag_state.read(); - (dag_state.gc_enabled(), dag_state.gc_round()) - }; - let mut blocks_unsuspended_below_gc_round = 0; - let mut blocks_gc_ed = 0; - - if !gc_enabled { - trace!("GC is disabled, no blocks will attempt to get unsuspended."); - return; - } - - while let Some((block_ref, _children_refs)) = self.missing_ancestors.first_key_value() { - // If the first block in the missing ancestors is higher than the gc_round, then - // we can't unsuspend it yet. So we just put it back - // and we terminate the iteration as any next entry will be of equal or higher - // round anyways. - if block_ref.round > gc_round { - return; - } - - blocks_gc_ed += 1; - - let hostname = self - .context - .committee - .authority(block_ref.author) - .hostname - .as_str(); - self.context - .metrics - .node_metrics - .block_manager_gced_blocks - .with_label_values(&[hostname]) - .inc(); - - assert!( - !self.suspended_blocks.contains_key(block_ref), - "Block should not be suspended, as we are causally GC'ing and no suspended block should exist for a missing ancestor." - ); - - // Also remove it from the missing list - we don't want to keep looking for it. - self.missing_blocks.remove(block_ref); - - // Find all the children blocks that have a dependency on this one and try to - // unsuspend them - let unsuspended_blocks = self.try_unsuspend_children_blocks(*block_ref); - - unsuspended_blocks.iter().for_each(|block| { - if block.round() <= gc_round { - blocks_unsuspended_below_gc_round += 1; - } - }); - - // Now validate their timestamps and accept them - let accepted_blocks = self.verify_block_timestamps_and_accept(unsuspended_blocks); - for block in accepted_blocks { - let hostname = self - .context - .committee - .authority(block.author()) - .hostname - .as_str(); - self.context - .metrics - .node_metrics - .block_manager_gc_unsuspended_blocks - .with_label_values(&[hostname]) - .inc(); - } - } - - debug!( - "Total {} blocks unsuspended and total blocks {} gc'ed <= gc_round {}", - blocks_unsuspended_below_gc_round, blocks_gc_ed, gc_round - ); - } - - /// Returns all the blocks that are currently missing and needed in order to - /// accept suspended blocks. For each block reference it returns the set of - /// authorities who have this block. - pub(crate) fn missing_blocks(&self) -> BTreeMap> { - self.missing_blocks.clone() - } - - /// Returns all the block refs that are currently missing. - #[cfg(test)] - pub(crate) fn missing_block_refs(&self) -> BTreeSet { - self.missing_blocks.keys().cloned().collect() - } - - fn update_stats(&mut self, missing_blocks: u64) { - let metrics = &self.context.metrics.node_metrics; - metrics.missing_blocks_total.inc_by(missing_blocks); - metrics - .block_manager_suspended_blocks - .set(self.suspended_blocks.len() as i64); - metrics - .block_manager_missing_ancestors - .set(self.missing_ancestors.len() as i64); - metrics - .block_manager_missing_blocks - .set(self.missing_blocks.len() as i64); - } - - fn update_block_received_metrics(&mut self, block: &VerifiedBlock) { - let (min_round, max_round) = - if let Some((curr_min, curr_max)) = self.received_block_rounds[block.author()] { - (curr_min.min(block.round()), curr_max.max(block.round())) - } else { - (block.round(), block.round()) - }; - self.received_block_rounds[block.author()] = Some((min_round, max_round)); - - let hostname = &self.context.committee.authority(block.author()).hostname; - self.context - .metrics - .node_metrics - .lowest_verified_authority_round - .with_label_values(&[hostname]) - .set(min_round.into()); - self.context - .metrics - .node_metrics - .highest_verified_authority_round - .with_label_values(&[hostname]) - .set(max_round.into()); - } - - /// Checks if block manager is empty. - #[cfg(test)] - pub(crate) fn is_empty(&self) -> bool { - self.suspended_blocks.is_empty() - && self.missing_ancestors.is_empty() - && self.missing_blocks.is_empty() - } - - /// Returns all the suspended blocks whose causal history we miss hence we - /// can't accept them yet. - #[cfg(test)] - fn suspended_blocks_refs(&self) -> BTreeSet { - self.suspended_blocks.keys().cloned().collect() - } -} - -// Result of trying to accept one block. -enum TryAcceptResult { - // The block is accepted. Wraps the block itself. - Accepted(VerifiedBlock), - // The block is suspended. Wraps ancestors to be fetched. - Suspended(BTreeSet), - // The block has been processed before and already exists in BlockManager (and is suspended) - // or in DagState (so has been already accepted). No further processing has been done at - // this point. - Processed, - // When a received block is <= gc_round, then we simply skip its processing as there is no - // meaning do any action on it or even store it. - Skipped, -} - -#[cfg(test)] -mod tests { - use std::{collections::BTreeSet, sync::Arc}; - - use consensus_config::AuthorityIndex; - use parking_lot::RwLock; - use rand::{SeedableRng, prelude::StdRng, seq::SliceRandom}; - use rstest::rstest; - - use crate::{ - CommitDigest, Round, - block::{BlockAPI, BlockDigest, BlockRef, SignedBlock, VerifiedBlock}, - block_manager::BlockManager, - block_verifier::{BlockVerifier, NoopBlockVerifier, SignedBlockVerifier}, - commit::TrustedCommit, - context::Context, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - storage::mem_store::MemStore, - test_dag_builder::DagBuilder, - test_dag_parser::parse_dag, - transaction::NoopTransactionVerifier, - }; - - #[tokio::test] - async fn suspend_blocks_with_missing_ancestors() { - // GIVEN - let (context, _key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder - .layers(1..=2) // 2 rounds - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(2), - ]) // Create equivocating blocks for 2 authorities - .equivocate(3) - .build(); - - // Take only the blocks of round 2 and try to accept them - let round_2_blocks = dag_builder - .blocks - .into_iter() - .filter_map(|(_, block)| (block.round() == 2).then_some(block)) - .collect::>(); - - // WHEN - let (accepted_blocks, missing) = block_manager.try_accept_blocks(round_2_blocks.clone()); - - // THEN - assert!(accepted_blocks.is_empty()); - - // AND the returned missing ancestors should be the same as the provided block - // ancestors - let missing_block_refs = round_2_blocks.first().unwrap().ancestors(); - let missing_block_refs = missing_block_refs.iter().cloned().collect::>(); - assert_eq!(missing, missing_block_refs); - - // AND the missing blocks are the parents of the round 2 blocks. Since this is a - // fully connected DAG taking the ancestors of the first element - // suffices. - assert_eq!(block_manager.missing_block_refs(), missing_block_refs); - - // AND suspended blocks should return the round_2_blocks - assert_eq!( - block_manager.suspended_blocks_refs(), - round_2_blocks - .into_iter() - .map(|block| block.reference()) - .collect::>() - ); - - // AND each missing block should be known to all authorities - let known_by_manager = block_manager - .missing_blocks() - .iter() - .next() - .expect("We should expect at least two elements there") - .1 - .clone(); - assert_eq!( - known_by_manager, - context - .committee - .authorities() - .map(|(a, _)| a) - .collect::>() - ); - } - - #[tokio::test] - async fn try_accept_block_returns_missing_blocks() { - let (context, _key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=4) // 4 rounds - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(2), - ]) // Create equivocating blocks for 2 authorities - .equivocate(3) // Use 3 equivocations blocks per authority - .build(); - - // Take the blocks from round 4 up to 2 (included). Only the first block of each - // round should return missing ancestors when try to accept - for (_, block) in dag_builder - .blocks - .into_iter() - .rev() - .take_while(|(_, block)| block.round() >= 2) - { - // WHEN - let (accepted_blocks, missing) = block_manager.try_accept_blocks(vec![block.clone()]); - - // THEN - assert!(accepted_blocks.is_empty()); - - let block_ancestors = block.ancestors().iter().cloned().collect::>(); - assert_eq!(missing, block_ancestors); - } - } - - #[tokio::test] - async fn accept_blocks_with_complete_causal_history() { - // GIVEN - let (context, _key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG of 2 rounds - let mut dag_builder = DagBuilder::new(context); - dag_builder.layers(1..=2).build(); - - let all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - // WHEN - let (accepted_blocks, missing) = block_manager.try_accept_blocks(all_blocks.clone()); - - // THEN - assert_eq!(accepted_blocks.len(), 8); - assert_eq!( - accepted_blocks, - all_blocks - .iter() - .filter(|block| block.round() > 0) - .cloned() - .collect::>() - ); - assert!(missing.is_empty()); - assert!(block_manager.is_empty()); - - // WHEN trying to accept same blocks again, then none will be returned as those - // have been already accepted - let (accepted_blocks, _) = block_manager.try_accept_blocks(all_blocks); - assert!(accepted_blocks.is_empty()); - } - - /// Tests that the block manager accepts blocks when some or all of their - /// causal history is below or equal to the GC round. - #[tokio::test] - async fn accept_blocks_with_causal_history_below_gc_round() { - // GIVEN - let (mut context, _key_pairs) = Context::new_for_test(4); - - // We set the gc depth to 4 - context - .protocol_config - .set_consensus_gc_depth_for_testing(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // We "fake" the commit for round 10, so we can test the GC round 6 - // (commit_round - gc_depth = 10 - 4 = 6) - let last_commit = TrustedCommit::new_for_test( - 10, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - vec![], - ); - dag_state.write().set_last_commit(last_commit); - assert_eq!( - dag_state.read().gc_round(), - 6, - "GC round should have moved to round 6" - ); - - let mut block_manager = BlockManager::new(context, dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG of 10 rounds with some weak links for the blocks of round 9 - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { * }, - Round 5 : { * }, - Round 6 : { * }, - Round 7 : { - A -> [*], - B -> [*], - C -> [*], - } - Round 8 : { - A -> [*], - B -> [*], - C -> [*], - }, - Round 9 : { - A -> [A8, B8, C8, D6], - B -> [A8, B8, C8, D6], - C -> [A8, B8, C8, D6], - D -> [A8, B8, C8, D6], - }, - Round 10 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - - // Now take all the blocks for round 7 & 8 , which are above the gc_round = 6. - // All those blocks should eventually be returned as accepted. Pay attention - // that without GC none of those blocks should get accepted. - let blocks_ranges = vec![7..=8 as Round, 9..=10 as Round]; - - for rounds_range in blocks_ranges { - let all_blocks = dag_builder - .blocks - .values() - .filter(|block| rounds_range.contains(&block.round())) - .cloned() - .collect::>(); - - // WHEN - let mut reversed_blocks = all_blocks.clone(); - reversed_blocks.sort_by_key(|b| std::cmp::Reverse(b.reference())); - let (mut accepted_blocks, missing) = block_manager.try_accept_blocks(reversed_blocks); - accepted_blocks.sort_by_key(|a| a.reference()); - - // THEN - assert_eq!(accepted_blocks, all_blocks.to_vec()); - assert!(missing.is_empty()); - assert!(block_manager.is_empty()); - - let (accepted_blocks, _) = block_manager.try_accept_blocks(all_blocks); - assert!(accepted_blocks.is_empty()); - } - } - - /// Blocks that are attempted to be accepted but are <= gc_round they will - /// be skipped for processing. Nothing should be stored or trigger any - /// unsuspension etc. - #[tokio::test] - async fn skip_accepting_blocks_below_gc_round() { - // GIVEN - let (mut context, _key_pairs) = Context::new_for_test(4); - // We set the gc depth to 4 - context - .protocol_config - .set_consensus_gc_depth_for_testing(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // We "fake" the commit for round 10, so we can test the GC round 6 - // (commit_round - gc_depth = 10 - 4 = 6) - let last_commit = TrustedCommit::new_for_test( - 10, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - vec![], - ); - dag_state.write().set_last_commit(last_commit); - assert_eq!( - dag_state.read().gc_round(), - 6, - "GC round should have moved to round 6" - ); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG of 6 rounds - let mut dag_builder = DagBuilder::new(context); - dag_builder.layers(1..=6).build(); - - let all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - // WHEN - let (accepted_blocks, missing) = block_manager.try_accept_blocks(all_blocks); - - // THEN - assert!(accepted_blocks.is_empty()); - assert!(missing.is_empty()); - assert!(block_manager.is_empty()); - } - - /// The test generate blocks for a well connected DAG and feed them to block - /// manager in random order. In the end all the blocks should be - /// uniquely suspended and no missing blocks should exist. The test will run - /// for both gc_enabled/disabled. When gc is enabled we set a high - /// gc_depth value so in practice gc_round will be 0, but we'll be able to - /// test in the common case that this work exactly the same way as when - /// gc is disabled. - #[rstest] - #[tokio::test] - async fn accept_blocks_unsuspend_children_blocks(#[values(false, true)] gc_enabled: bool) { - // GIVEN - let (mut context, _key_pairs) = Context::new_for_test(4); - - if gc_enabled { - context - .protocol_config - .set_consensus_gc_depth_for_testing(10); - } - let context = Arc::new(context); - - // create a DAG of rounds 1 ~ 3 - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=3).build(); - - let mut all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - // Now randomize the sequence of sending the blocks to block manager. In the end - // all the blocks should be uniquely suspended and no missing blocks - // should exist. - for seed in 0..100u8 { - all_blocks.shuffle(&mut StdRng::from_seed([seed; 32])); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // WHEN - let mut all_accepted_blocks = vec![]; - for block in &all_blocks { - let (accepted_blocks, _) = block_manager.try_accept_blocks(vec![block.clone()]); - - all_accepted_blocks.extend(accepted_blocks); - } - - // THEN - all_accepted_blocks.sort_by_key(|b| b.reference()); - all_blocks.sort_by_key(|b| b.reference()); - - assert_eq!( - all_accepted_blocks, all_blocks, - "Failed acceptance sequence for seed {seed}" - ); - assert!(block_manager.is_empty()); - } - } - - /// Tests that `missing_blocks()` correctly infers the authorities - /// referencing each missing block based on accepted blocks in the DAG. - #[tokio::test] - async fn authorities_that_know_missing_blocks() { - let (context, _key_pairs) = Context::new_for_test(4); - - let context = Arc::new(context); - - // create a DAG of rounds 1 ~ 3 - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=3).build(); - - let all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - let blocks_round_2 = all_blocks - .iter() - .filter(|block| block.round() == 2) - .cloned() - .collect::>(); - - let blocks_round_1 = all_blocks - .iter() - .filter(|block| block.round() == 1) - .map(|block| block.reference()) - .collect::>(); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = BlockManager::new(context, dag_state, Arc::new(NoopBlockVerifier)); - - let (_, missing_blocks) = block_manager.try_accept_blocks(vec![blocks_round_2[0].clone()]); - // Blocks from round 1 are all missing, since the DAG is fully connected - assert_eq!(missing_blocks, blocks_round_1); - - let missing_blocks_with_authorities = block_manager.missing_blocks(); - - let block_round_1_authority_0 = all_blocks - .iter() - .filter(|block| block.round() == 1 && block.author() == AuthorityIndex::new_for_test(0)) - .map(|block| block.reference()) - .next() - .unwrap(); - let block_round_1_authority_1 = all_blocks - .iter() - .filter(|block| block.round() == 1 && block.author() == AuthorityIndex::new_for_test(1)) - .map(|block| block.reference()) - .next() - .unwrap(); - assert_eq!( - missing_blocks_with_authorities[&block_round_1_authority_0], - BTreeSet::from([AuthorityIndex::new_for_test(0)]) - ); - assert_eq!( - missing_blocks_with_authorities[&block_round_1_authority_1], - BTreeSet::from([ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1) - ]) - ); - - // Add a new block from round 2 from authority 1, which updates the set of - // authorities that are aware of the missing blocks - block_manager.try_accept_blocks(vec![blocks_round_2[1].clone()]); - let missing_blocks_with_authorities = block_manager.missing_blocks(); - assert_eq!( - missing_blocks_with_authorities[&block_round_1_authority_0], - BTreeSet::from([ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1) - ]) - ); - } - - #[rstest] - #[tokio::test] - async fn unsuspend_blocks_for_latest_gc_round(#[values(5, 10, 14)] gc_depth: u32) { - telemetry_subscribers::init_for_testing(); - // GIVEN - let (mut context, _key_pairs) = Context::new_for_test(4); - - if gc_depth > 0 { - context - .protocol_config - .set_consensus_gc_depth_for_testing(gc_depth); - } - let context = Arc::new(context); - - // create a DAG of rounds 1 ~ gc_depth * 2 - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=gc_depth * 2).build(); - - // Pay attention that we start from round 2. Round 1 will always be missing so - // no matter what we do we can't unsuspend it unless gc_round has - // advanced to round >= 1. - let mut all_blocks = dag_builder - .blocks - .values() - .filter(|block| block.round() > 1) - .cloned() - .collect::>(); - - // Now randomize the sequence of sending the blocks to block manager. In the end - // all the blocks should be uniquely suspended and no missing blocks - // should exist. - for seed in 0..100u8 { - all_blocks.shuffle(&mut StdRng::from_seed([seed; 32])); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let mut block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - - // WHEN - for block in &all_blocks { - let (accepted_blocks, _) = block_manager.try_accept_blocks(vec![block.clone()]); - assert!(accepted_blocks.is_empty()); - } - assert!(!block_manager.is_empty()); - - // AND also call the try_to_find method with some non existing block refs. Those - // should be cleaned up as well once GC kicks in. - let non_existing_refs = (1..=3) - .map(|round| { - BlockRef::new(round, AuthorityIndex::new_for_test(0), BlockDigest::MIN) - }) - .collect::>(); - assert_eq!(block_manager.try_find_blocks(non_existing_refs).len(), 3); - - // AND - // Trigger a commit which will advance GC round - let last_commit = TrustedCommit::new_for_test( - gc_depth * 2, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - BlockRef::new( - gc_depth * 2, - AuthorityIndex::new_for_test(0), - BlockDigest::MIN, - ), - vec![], - ); - dag_state.write().set_last_commit(last_commit); - - // AND - block_manager.try_unsuspend_blocks_for_latest_gc_round(); - - // THEN - assert!(block_manager.is_empty()); - - // AND ensure that all have been accepted to the DAG - for block in &all_blocks { - assert!(dag_state.read().contains_block(&block.reference())); - } - } - } - - #[rstest] - #[tokio::test] - async fn try_accept_committed_blocks() { - // GIVEN - let (mut context, _key_pairs) = Context::new_for_test(4); - // We set the gc depth to 4 - context - .protocol_config - .set_consensus_gc_depth_for_testing(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // We "fake" the commit for round 6, so GC round moves to (commit_round - - // gc_depth = 6 - 4 = 2) - let last_commit = TrustedCommit::new_for_test( - 10, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - BlockRef::new(6, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - vec![], - ); - dag_state.write().set_last_commit(last_commit); - assert_eq!( - dag_state.read().gc_round(), - 2, - "GC round should have moved to round 2" - ); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG of 12 rounds - let mut dag_builder = DagBuilder::new(context); - dag_builder.layers(1..=12).build(); - - // Now try to accept via the normal acceptance block path the blocks of rounds 7 - // ~ 12. None of them should be accepted - let blocks = dag_builder.blocks(7..=12); - let (accepted_blocks, missing) = block_manager.try_accept_blocks(blocks); - assert!(accepted_blocks.is_empty()); - assert_eq!(missing.len(), 4); - - // Now try to accept via the committed blocks path the blocks of rounds 3 ~ 6. - // All of them should be accepted and also the blocks of rounds 7 ~ 12 - // should be unsuspended and accepted as well. - let blocks = dag_builder.blocks(3..=6); - - // WHEN - let mut accepted_blocks = block_manager.try_accept_committed_blocks(blocks); - - // THEN - accepted_blocks.sort_by_key(|b| b.reference()); - - let mut all_blocks = dag_builder.blocks(3..=12); - all_blocks.sort_by_key(|b| b.reference()); - - assert_eq!(accepted_blocks, all_blocks); - assert!(block_manager.is_empty()); - } - - struct TestBlockVerifier { - fail: BTreeSet, - } - - impl TestBlockVerifier { - fn new(fail: BTreeSet) -> Self { - Self { fail } - } - } - - impl BlockVerifier for TestBlockVerifier { - fn verify(&self, _block: &SignedBlock) -> ConsensusResult<()> { - Ok(()) - } - - fn check_ancestors( - &self, - block: &VerifiedBlock, - _ancestors: &[Option], - _gc_enabled: bool, - _gc_round: Round, - ) -> ConsensusResult<()> { - if self.fail.contains(&block.reference()) { - Err(ConsensusError::InvalidBlockTimestamp { - max_timestamp_ms: 0, - block_timestamp_ms: block.timestamp_ms(), - }) - } else { - Ok(()) - } - } - } - - #[tokio::test] - async fn reject_blocks_failing_verifications() { - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(false); - let context = Arc::new(context); - - // create a DAG of rounds 1 ~ 5. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=5).build(); - - let all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - // Create a test verifier that fails the blocks of round 3 - let test_verifier = TestBlockVerifier::new( - all_blocks - .iter() - .filter(|block| block.round() == 3) - .map(|block| block.reference()) - .collect(), - ); - - // Create BlockManager. - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let mut block_manager = BlockManager::new(context, dag_state, Arc::new(test_verifier)); - - // Try to accept blocks from round 2 ~ 5 into block manager. All of them should - // be suspended. - let (accepted_blocks, missing_refs) = block_manager.try_accept_blocks( - all_blocks - .iter() - .filter(|block| block.round() > 1) - .cloned() - .collect(), - ); - - // Missing refs should all come from round 1. - assert!(accepted_blocks.is_empty()); - assert_eq!(missing_refs.len(), 4); - missing_refs.iter().for_each(|missing_ref| { - assert_eq!(missing_ref.round, 1); - }); - - // Now add round 1 blocks into block manager. - let (accepted_blocks, missing_refs) = block_manager.try_accept_blocks( - all_blocks - .iter() - .filter(|block| block.round() == 1) - .cloned() - .collect(), - ); - - // Only round 1 and round 2 blocks should be accepted. - assert_eq!(accepted_blocks.len(), 8); - accepted_blocks.iter().for_each(|block| { - assert!(block.round() <= 2); - }); - assert!(missing_refs.is_empty()); - - // Other blocks should be rejected and there should be no remaining suspended - // block. - assert!(block_manager.suspended_blocks_refs().is_empty()); - } - - #[tokio::test] - async fn try_find_blocks() { - // GIVEN - let (context, _key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = - BlockManager::new(context.clone(), dag_state, Arc::new(NoopBlockVerifier)); - - // create a DAG - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=2) // 2 rounds - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(2), - ]) // Create equivocating blocks for 2 authorities - .equivocate(3) - .build(); - - // Take only the blocks of round 2 and try to accept them - let round_2_blocks = dag_builder - .blocks - .iter() - .filter_map(|(_, block)| (block.round() == 2).then_some(block.clone())) - .collect::>(); - - // All blocks should be missing - let missing_block_refs_from_find = - block_manager.try_find_blocks(round_2_blocks.iter().map(|b| b.reference()).collect()); - assert_eq!(missing_block_refs_from_find.len(), 10); - assert!( - missing_block_refs_from_find - .iter() - .all(|block_ref| block_ref.round == 2) - ); - - // Try accept blocks which will cause blocks to be suspended and added to - // missing in block manager. - let (accepted_blocks, missing) = block_manager.try_accept_blocks(round_2_blocks.clone()); - assert!(accepted_blocks.is_empty()); - - let missing_block_refs = round_2_blocks.first().unwrap().ancestors(); - let missing_block_refs_from_accept = - missing_block_refs.iter().cloned().collect::>(); - assert_eq!(missing, missing_block_refs_from_accept); - assert_eq!( - block_manager.missing_block_refs(), - missing_block_refs_from_accept - ); - - // No blocks should be accepted and block manager should have made note - // of the missing & suspended blocks. - // Now we can check get the result of try find block with all of the blocks - // from newly created but not accepted round 3. - dag_builder.layer(3).build(); - - let round_3_blocks = dag_builder - .blocks - .iter() - .filter_map(|(_, block)| (block.round() == 3).then_some(block.reference())) - .collect::>(); - - let missing_block_refs_from_find = block_manager.try_find_blocks( - round_2_blocks - .iter() - .map(|b| b.reference()) - .chain(round_3_blocks.into_iter()) - .collect(), - ); - - assert_eq!(missing_block_refs_from_find.len(), 4); - assert!( - missing_block_refs_from_find - .iter() - .all(|block_ref| block_ref.round == 3) - ); - assert_eq!( - block_manager.missing_block_refs(), - missing_block_refs_from_accept - .into_iter() - .chain(missing_block_refs_from_find.into_iter()) - .collect() - ); - } - - #[rstest] - #[tokio::test] - async fn test_verify_block_timestamps_and_accept( - #[values(false, true)] median_based_timestamp: bool, - ) { - telemetry_subscribers::init_for_testing(); - let (mut context, _key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - median_based_timestamp, - ); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let mut block_manager = BlockManager::new( - context.clone(), - dag_state, - Arc::new(SignedBlockVerifier::new( - context.clone(), - Arc::new(NoopTransactionVerifier {}), - )), - ); - - // create a DAG where authority 0 timestamp is always higher than the others. - let mut dag_builder = DagBuilder::new(context.clone()); - let authorities = context - .committee - .authorities() - .map(|(index, _)| index) - .collect::>(); - dag_builder - .layers(1..=1) - .authorities(authorities.clone()) - .with_timestamps(vec![1000, 500, 550, 580]) - .build(); - dag_builder - .layers(2..=2) - .authorities(authorities.clone()) - .with_timestamps(vec![2000, 600, 650, 680]) - .build(); - dag_builder - .layers(3..=3) - .authorities(authorities) - .with_timestamps(vec![3000, 700, 750, 780]) - .build(); - - // take all the blocks and try to accept them. - let all_blocks = dag_builder.blocks.values().cloned().collect::>(); - - // All blocks should get accepted - let (accepted_blocks, missing) = block_manager.try_accept_blocks(all_blocks.clone()); - - if median_based_timestamp { - // If the median based timestamp is enabled then all the blocks should be - // accepted - assert_eq!(all_blocks, accepted_blocks); - assert!(missing.is_empty()); - } else { - // only the blocks of first round will be accepted (and the block of round 2 for - // authority 0) as the rest will be rejected - assert_eq!(accepted_blocks.len(), 5); - for block in accepted_blocks { - if block.author() == AuthorityIndex::new_for_test(0) { - assert!(block.round() == 1 || block.round() == 2); - } else { - assert_eq!(block.round(), 1); - } - } - } - } -} diff --git a/consensus/core/src/block_verifier.rs b/consensus/core/src/block_verifier.rs deleted file mode 100644 index 394e13e4044..00000000000 --- a/consensus/core/src/block_verifier.rs +++ /dev/null @@ -1,699 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::BTreeSet, sync::Arc}; - -use tracing::instrument; - -use crate::{ - Round, - block::{ - BlockAPI, BlockRef, BlockTimestampMs, GENESIS_ROUND, SignedBlock, VerifiedBlock, - genesis_blocks, - }, - context::Context, - error::{ConsensusError, ConsensusResult}, - transaction::TransactionVerifier, -}; - -pub(crate) trait BlockVerifier: Send + Sync + 'static { - /// Verifies a block's metadata and transactions. - /// This is called before examining a block's causal history. - fn verify(&self, block: &SignedBlock) -> ConsensusResult<()>; - - /// Verifies a block w.r.t. ancestor blocks. - /// This is called after a block has complete causal history locally, - /// and is ready to be accepted into the DAG. - /// - /// Caller must make sure ancestors corresponse to block.ancestors() 1-to-1, - /// in the same order. - fn check_ancestors( - &self, - block: &VerifiedBlock, - ancestors: &[Option], - gc_enabled: bool, - gc_round: Round, - ) -> ConsensusResult<()>; -} - -/// `SignedBlockVerifier` checks the validity of a block. -/// -/// Blocks that fail verification at one honest authority will be rejected by -/// all other honest authorities as well. The means invalid blocks, and blocks -/// with an invalid ancestor, will never be accepted into the DAG. -pub(crate) struct SignedBlockVerifier { - context: Arc, - genesis: BTreeSet, - transaction_verifier: Arc, -} - -impl SignedBlockVerifier { - pub(crate) fn new( - context: Arc, - transaction_verifier: Arc, - ) -> Self { - let genesis = genesis_blocks(&context) - .into_iter() - .map(|b| b.reference()) - .collect(); - Self { - context, - genesis, - transaction_verifier, - } - } - - pub(crate) fn check_transactions(&self, batch: &[&[u8]]) -> ConsensusResult<()> { - let max_transaction_size_limit = - self.context.protocol_config.max_transaction_size_bytes() as usize; - for t in batch { - if t.len() > max_transaction_size_limit && max_transaction_size_limit > 0 { - return Err(ConsensusError::TransactionTooLarge { - size: t.len(), - limit: max_transaction_size_limit, - }); - } - } - - let max_num_transactions_limit = - self.context.protocol_config.max_num_transactions_in_block() as usize; - if batch.len() > max_num_transactions_limit && max_num_transactions_limit > 0 { - return Err(ConsensusError::TooManyTransactions { - count: batch.len(), - limit: max_num_transactions_limit, - }); - } - - let total_transactions_size_limit = self - .context - .protocol_config - .max_transactions_in_block_bytes() as usize; - if batch.iter().map(|t| t.len()).sum::() > total_transactions_size_limit - && total_transactions_size_limit > 0 - { - return Err(ConsensusError::TooManyTransactionBytes { - size: batch.len(), - limit: total_transactions_size_limit, - }); - } - Ok(()) - } -} - -// All block verification logic are implemented below. -impl BlockVerifier for SignedBlockVerifier { - #[instrument(level = "trace", skip_all)] - fn verify(&self, block: &SignedBlock) -> ConsensusResult<()> { - let committee = &self.context.committee; - // The block must belong to the current epoch and have valid authority index, - // before having its signature verified. - if block.epoch() != committee.epoch() { - return Err(ConsensusError::WrongEpoch { - expected: committee.epoch(), - actual: block.epoch(), - }); - } - if block.round() == 0 { - return Err(ConsensusError::UnexpectedGenesisBlock); - } - if !committee.is_valid_index(block.author()) { - return Err(ConsensusError::InvalidAuthorityIndex { - index: block.author(), - max: committee.size() - 1, - }); - } - - // Verify the block's signature. - block.verify_signature(&self.context)?; - - // Verify the block's ancestor refs are consistent with the block's round, - // and total parent stakes reach quorum. - if block.ancestors().len() > committee.size() { - return Err(ConsensusError::TooManyAncestors( - block.ancestors().len(), - committee.size(), - )); - } - if block.ancestors().is_empty() { - return Err(ConsensusError::InsufficientParentStakes { - parent_stakes: 0, - quorum: committee.quorum_threshold(), - }); - } - let mut seen_ancestors = vec![false; committee.size()]; - let mut parent_stakes = 0; - for (i, ancestor) in block.ancestors().iter().enumerate() { - if !committee.is_valid_index(ancestor.author) { - return Err(ConsensusError::InvalidAncestorAuthorityIndex { - block_authority: block.author(), - ancestor_index: ancestor.author, - max: committee.size() - 1, - }); - } - if (i == 0 && ancestor.author != block.author()) - || (i > 0 && ancestor.author == block.author()) - { - return Err(ConsensusError::InvalidAncestorPosition { - block_authority: block.author(), - ancestor_authority: ancestor.author, - position: i, - }); - } - if ancestor.round >= block.round() { - return Err(ConsensusError::InvalidAncestorRound { - ancestor: ancestor.round, - block: block.round(), - }); - } - if ancestor.round == GENESIS_ROUND && !self.genesis.contains(ancestor) { - return Err(ConsensusError::InvalidGenesisAncestor(*ancestor)); - } - if seen_ancestors[ancestor.author] { - return Err(ConsensusError::DuplicatedAncestorsAuthority( - ancestor.author, - )); - } - seen_ancestors[ancestor.author] = true; - // Block must have round >= 1 so checked_sub(1) should be safe. - if ancestor.round == block.round().checked_sub(1).unwrap() { - parent_stakes += committee.stake(ancestor.author); - } - } - if !committee.reached_quorum(parent_stakes) { - return Err(ConsensusError::InsufficientParentStakes { - parent_stakes, - quorum: committee.quorum_threshold(), - }); - } - - let batch: Vec<_> = block.transactions().iter().map(|t| t.data()).collect(); - - self.check_transactions(&batch)?; - - self.transaction_verifier - .verify_batch(&batch) - .map_err(|e| ConsensusError::InvalidTransaction(format!("{e:?}"))) - } - - fn check_ancestors( - &self, - block: &VerifiedBlock, - ancestors: &[Option], - gc_enabled: bool, - gc_round: Round, - ) -> ConsensusResult<()> { - if gc_enabled { - // TODO: will be removed with new timestamp calculation is in place as all these - // will be irrelevant. When gc is enabled we don't have guarantees - // that all ancestors will be available. We'll take into account only the passed - // gc_round ones for the timestamp check. - let mut max_timestamp_ms = BlockTimestampMs::MIN; - for ancestor in ancestors.iter().flatten() { - if ancestor.round() <= gc_round { - continue; - } - max_timestamp_ms = max_timestamp_ms.max(ancestor.timestamp_ms()); - if max_timestamp_ms > block.timestamp_ms() { - return Err(ConsensusError::InvalidBlockTimestamp { - max_timestamp_ms, - block_timestamp_ms: block.timestamp_ms(), - }); - } - } - } else { - assert_eq!(block.ancestors().len(), ancestors.len()); - // This checks the invariant that block timestamp >= max ancestor timestamp. - let mut max_timestamp_ms = BlockTimestampMs::MIN; - for (ancestor_ref, ancestor_block) in block.ancestors().iter().zip(ancestors.iter()) { - let ancestor_block = ancestor_block - .as_ref() - .expect("There should never be an empty slot"); - assert_eq!(ancestor_ref, &ancestor_block.reference()); - max_timestamp_ms = max_timestamp_ms.max(ancestor_block.timestamp_ms()); - } - if max_timestamp_ms > block.timestamp_ms() { - return Err(ConsensusError::InvalidBlockTimestamp { - max_timestamp_ms, - block_timestamp_ms: block.timestamp_ms(), - }); - } - } - Ok(()) - } -} - -#[cfg(test)] -pub(crate) struct NoopBlockVerifier; - -#[cfg(test)] -impl BlockVerifier for NoopBlockVerifier { - fn verify(&self, _block: &SignedBlock) -> ConsensusResult<()> { - Ok(()) - } - - fn check_ancestors( - &self, - _block: &VerifiedBlock, - _ancestors: &[Option], - _gc_enabled: bool, - _gc_round: Round, - ) -> ConsensusResult<()> { - Ok(()) - } -} - -#[cfg(test)] -mod test { - use consensus_config::AuthorityIndex; - use rstest::rstest; - - use super::*; - use crate::{ - block::{BlockDigest, BlockRef, TestBlock, Transaction}, - context::Context, - transaction::{TransactionVerifier, ValidationError}, - }; - - struct TxnSizeVerifier {} - - impl TransactionVerifier for TxnSizeVerifier { - // Fails verification if any transaction is < 4 bytes. - fn verify_batch(&self, transactions: &[&[u8]]) -> Result<(), ValidationError> { - for txn in transactions { - if txn.len() < 4 { - return Err(ValidationError::InvalidTransaction(format!( - "Length {} is too short!", - txn.len() - ))); - } - } - Ok(()) - } - } - - #[tokio::test] - async fn test_verify_block() { - let (context, keypairs) = Context::new_for_test(4); - let context = Arc::new(context); - let authority_2_protocol_keypair = &keypairs[2].1; - let verifier = SignedBlockVerifier::new(context, Arc::new(TxnSizeVerifier {})); - - let test_block = TestBlock::new(10, 2) - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(7, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .set_transactions(vec![Transaction::new(vec![4; 8])]); - - // Valid SignedBlock. - { - let block = test_block.clone().build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - verifier.verify(&signed_block).unwrap(); - } - - // Block with wrong epoch. - { - let block = test_block.clone().set_epoch(1).build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::WrongEpoch { - expected: _, - actual: _ - }) - )); - } - - // Block at genesis round. - { - let block = test_block.clone().set_round(0).build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::UnexpectedGenesisBlock) - )); - } - - // Block with invalid authority index. - { - let block = test_block - .clone() - .set_author(AuthorityIndex::new_for_test(4)) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InvalidAuthorityIndex { index: _, max: _ }) - )); - } - - // Block with mismatched authority index and signature. - { - let block = test_block - .clone() - .set_author(AuthorityIndex::new_for_test(1)) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::SignatureVerificationFailure(_)) - )); - } - - // Block with wrong key. - { - let block = test_block.clone().build(); - let signed_block = SignedBlock::new(block, &keypairs[3].1).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::SignatureVerificationFailure(_)) - )); - } - - // Block without signature. - { - let block = test_block.clone().build(); - let mut signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - signed_block.clear_signature(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::MalformedSignature(_)) - )); - } - - // Block with invalid ancestor round. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InvalidAncestorRound { - ancestor: _, - block: _ - }) - )); - } - - // Block with parents not reaching quorum. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InsufficientParentStakes { - parent_stakes: _, - quorum: _ - }) - )); - } - - // Block with too many ancestors. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - BlockRef::new(9, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::TooManyAncestors(_, _)) - )); - } - - // Block without own ancestor. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InvalidAncestorPosition { - block_authority: _, - ancestor_authority: _, - position: _ - }) - )); - } - - // Block with own ancestor at wrong position. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(9, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InvalidAncestorPosition { - block_authority: _, - ancestor_authority: _, - position: _ - }) - )); - } - - // Block with ancestors from the same authority. - { - let block = test_block - .clone() - .set_ancestors(vec![ - BlockRef::new(8, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(1), BlockDigest::MIN), - ]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::DuplicatedAncestorsAuthority(_)) - )); - } - - // Block with invalid transaction. - { - let block = test_block - .clone() - .set_transactions(vec![Transaction::new(vec![1; 2])]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::InvalidTransaction(_)) - )); - } - - // Block with transaction too large. - { - let block = test_block - .clone() - .set_transactions(vec![Transaction::new(vec![4; 257 * 1024])]) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::TransactionTooLarge { size: _, limit: _ }) - )); - } - - // Block with too many transactions. - { - let block = test_block - .clone() - .set_transactions((0..1000).map(|_| Transaction::new(vec![4; 8])).collect()) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::TooManyTransactions { count: _, limit: _ }) - )); - } - - // Block with too many transaction bytes. - { - let block = test_block - .set_transactions( - (0..100) - .map(|_| Transaction::new(vec![4; 8 * 1024])) - .collect(), - ) - .build(); - let signed_block = SignedBlock::new(block, authority_2_protocol_keypair).unwrap(); - assert!(matches!( - verifier.verify(&signed_block), - Err(ConsensusError::TooManyTransactionBytes { size: _, limit: _ }) - )); - } - } - - /// Tests the block's ancestors for timestamp monotonicity. Test will run - /// for both when gc is enabled and disabled, but with none of the - /// ancestors being below the gc_round. - #[rstest] - #[tokio::test] - async fn test_check_ancestors(#[values(false, true)] gc_enabled: bool) { - let num_authorities = 4; - let (context, _keypairs) = Context::new_for_test(num_authorities); - let context = Arc::new(context); - let verifier = SignedBlockVerifier::new(context, Arc::new(TxnSizeVerifier {})); - let gc_round = 0; - - let mut ancestor_blocks = vec![]; - for i in 0..num_authorities { - let test_block = TestBlock::new(10, i as u32) - .set_timestamp_ms(1000 + 100 * i as BlockTimestampMs) - .build(); - ancestor_blocks.push(Some(VerifiedBlock::new_for_test(test_block))); - } - let ancestor_refs = ancestor_blocks - .iter() - .flatten() - .map(|block| block.reference()) - .collect::>(); - - // Block respecting timestamp invariant. - { - let block = TestBlock::new(11, 0) - .set_ancestors(ancestor_refs.clone()) - .set_timestamp_ms(1500) - .build(); - let verified_block = VerifiedBlock::new_for_test(block); - assert!( - verifier - .check_ancestors(&verified_block, &ancestor_blocks, gc_enabled, gc_round) - .is_ok() - ); - } - - // Block not respecting timestamp invariant. - { - let block = TestBlock::new(11, 0) - .set_ancestors(ancestor_refs) - .set_timestamp_ms(1000) - .build(); - let verified_block = VerifiedBlock::new_for_test(block); - assert!(matches!( - verifier.check_ancestors(&verified_block, &ancestor_blocks, gc_enabled, gc_round), - Err(ConsensusError::InvalidBlockTimestamp { - max_timestamp_ms: _, - block_timestamp_ms: _ - }) - )); - } - } - - #[tokio::test] - async fn test_check_ancestors_passed_gc_round() { - let num_authorities = 4; - let (context, _keypairs) = Context::new_for_test(num_authorities); - let context = Arc::new(context); - let verifier = SignedBlockVerifier::new(context, Arc::new(TxnSizeVerifier {})); - let gc_enabled = true; - let gc_round = 3; - - let mut ancestor_blocks = vec![]; - - // Create one block just on the `gc_round` (so it should be considered garbage - // collected). This has higher timestamp that the block we are testing. - let test_block = TestBlock::new(gc_round, 0_u32) - .set_timestamp_ms(1500 as BlockTimestampMs) - .build(); - ancestor_blocks.push(Some(VerifiedBlock::new_for_test(test_block))); - - // Rest of the blocks - for i in 1..=3 { - let test_block = TestBlock::new(gc_round + 1, i as u32) - .set_timestamp_ms(1000 + 100 * i as BlockTimestampMs) - .build(); - ancestor_blocks.push(Some(VerifiedBlock::new_for_test(test_block))); - } - - let ancestor_refs = ancestor_blocks - .iter() - .flatten() - .map(|block| block.reference()) - .collect::>(); - - // Block respecting timestamp invariant. - { - let block = TestBlock::new(gc_round + 2, 0) - .set_ancestors(ancestor_refs.clone()) - .set_timestamp_ms(1600) - .build(); - let verified_block = VerifiedBlock::new_for_test(block); - assert!( - verifier - .check_ancestors(&verified_block, &ancestor_blocks, gc_enabled, gc_round) - .is_ok() - ); - } - - // Block not respecting timestamp invariant for the block that is garbage - // collected Validation should pass. - { - let block = TestBlock::new(11, 0) - .set_ancestors(ancestor_refs.clone()) - .set_timestamp_ms(1400) - .build(); - let verified_block = VerifiedBlock::new_for_test(block); - assert!( - verifier - .check_ancestors(&verified_block, &ancestor_blocks, gc_enabled, gc_round) - .is_ok() - ); - } - - // Block not respecting timestamp invariant for the blocks that are not garbage - // collected - { - let block = TestBlock::new(11, 0) - .set_ancestors(ancestor_refs) - .set_timestamp_ms(1100) - .build(); - let verified_block = VerifiedBlock::new_for_test(block); - assert!(matches!( - verifier.check_ancestors(&verified_block, &ancestor_blocks, gc_enabled, gc_round), - Err(ConsensusError::InvalidBlockTimestamp { - max_timestamp_ms: _, - block_timestamp_ms: _ - }) - )); - } - } -} diff --git a/consensus/core/src/broadcaster.rs b/consensus/core/src/broadcaster.rs deleted file mode 100644 index 6a44a939c59..00000000000 --- a/consensus/core/src/broadcaster.rs +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - cmp::{max, min}, - sync::Arc, - time::Duration, -}; - -use consensus_config::AuthorityIndex; -use futures::{StreamExt as _, stream::FuturesUnordered}; -use tokio::{ - sync::broadcast, - task::JoinSet, - time::{Instant, error::Elapsed, sleep_until, timeout}, -}; -use tracing::{trace, warn}; - -use crate::{ - block::{BlockAPI as _, ExtendedBlock, VerifiedBlock}, - context::Context, - core::CoreSignalsReceivers, - error::ConsensusResult, - network::NetworkClient, -}; - -/// Number of Blocks that can be inflight sending to a peer. -const BROADCAST_CONCURRENCY: usize = 10; - -/// Broadcaster sends newly created blocks to each peer over the network. -/// -/// For a peer that lags behind or is disconnected, blocks are buffered and -/// retried until a limit is reached, then old blocks will get dropped from the -/// buffer. -pub(crate) struct Broadcaster { - // Background tasks listening for new blocks and pushing them to peers. - senders: JoinSet<()>, -} - -impl Broadcaster { - const LAST_BLOCK_RETRY_INTERVAL: Duration = Duration::from_secs(2); - const MIN_SEND_BLOCK_NETWORK_TIMEOUT: Duration = Duration::from_secs(5); - - pub(crate) fn new( - context: Arc, - network_client: Arc, - signals_receiver: &CoreSignalsReceivers, - ) -> Self { - let mut senders = JoinSet::new(); - for (index, _authority) in context.committee.authorities() { - // Skip sending Block to self. - if index == context.own_index { - continue; - } - senders.spawn(Self::push_blocks( - context.clone(), - network_client.clone(), - signals_receiver.block_broadcast_receiver(), - index, - )); - } - - Self { senders } - } - - pub(crate) fn stop(&mut self) { - // Intentionally not waiting for senders to exit, to speed up shutdown. - self.senders.abort_all(); - } - - /// Runs a loop that continuously pushes new blocks received from the - /// rx_block_broadcast channel to the target peer. - /// - /// The loop does not exit until the validator is shutting down. - async fn push_blocks( - context: Arc, - network_client: Arc, - mut rx_block_broadcast: broadcast::Receiver, - peer: AuthorityIndex, - ) { - let peer_hostname = &context.committee.authority(peer).hostname; - - // Record the last block to be broadcasted, to retry in case no new block is - // produced for awhile. Even if the peer has acknowledged the last - // block, the block might have been dropped afterwards if the peer - // crashed. - let mut last_block: Option = None; - - // Retry last block with an interval. - let mut retry_timer = tokio::time::interval(Self::LAST_BLOCK_RETRY_INTERVAL); - retry_timer.reset_after(Self::LAST_BLOCK_RETRY_INTERVAL); - retry_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - // Use a simple exponential-decay RTT estimator to adjust the timeout for each - // block sent. The estimation logic will be removed once the underlying - // transport switches to use streaming and the streaming implementation - // can be relied upon for retries. - const RTT_ESTIMATE_DECAY: f64 = 0.95; - const TIMEOUT_THRESHOLD_MULTIPLIER: f64 = 2.0; - const TIMEOUT_RTT_INCREMENT_FACTOR: f64 = 1.6; - let mut rtt_estimate = Duration::from_millis(200); - - let mut requests = FuturesUnordered::new(); - - async fn send_block( - network_client: Arc, - peer: AuthorityIndex, - rtt_estimate: Duration, - block: VerifiedBlock, - ) -> (Result, Elapsed>, Instant, VerifiedBlock) { - let start = Instant::now(); - let req_timeout = rtt_estimate.mul_f64(TIMEOUT_THRESHOLD_MULTIPLIER); - // Use a minimum timeout of 5s so the receiver does not terminate the request - // too early. - let network_timeout = - std::cmp::max(req_timeout, Broadcaster::MIN_SEND_BLOCK_NETWORK_TIMEOUT); - let resp = timeout( - req_timeout, - network_client.send_block(peer, &block, network_timeout), - ) - .await; - if matches!(resp, Ok(Err(_))) { - // Add a delay before retrying. - sleep_until(start + req_timeout).await; - } - (resp, start, block) - } - - loop { - tokio::select! { - result = rx_block_broadcast.recv(), if requests.len() < BROADCAST_CONCURRENCY => { - let block = match result { - // Other info from ExtendedBlock are ignored, because Broadcaster is not used in production. - Ok(block) => block.block, - Err(broadcast::error::RecvError::Closed) => { - trace!("Sender to {peer} is shutting down!"); - return; - } - Err(broadcast::error::RecvError::Lagged(e)) => { - warn!("Sender to {peer} is lagging! {e}"); - // Re-run the loop to receive again. - continue; - } - }; - requests.push(send_block(network_client.clone(), peer, rtt_estimate, block.clone())); - if last_block.is_none() || last_block.as_ref().unwrap().round() < block.round() { - last_block = Some(block); - } - } - - Some((resp, start, block)) = requests.next() => { - match resp { - Ok(Ok(_)) => { - let now = Instant::now(); - rtt_estimate = rtt_estimate.mul_f64(RTT_ESTIMATE_DECAY) + (now - start).mul_f64(1.0 - RTT_ESTIMATE_DECAY); - // Avoid immediately retrying a successfully sent block. - // Resetting timer is unnecessary otherwise because there are - // additional inflight requests. - retry_timer.reset_after(Self::LAST_BLOCK_RETRY_INTERVAL); - }, - Err(Elapsed { .. }) => { - rtt_estimate = rtt_estimate.mul_f64(TIMEOUT_RTT_INCREMENT_FACTOR); - requests.push(send_block(network_client.clone(), peer, rtt_estimate, block)); - }, - Ok(Err(_)) => { - requests.push(send_block(network_client.clone(), peer, rtt_estimate, block)); - }, - }; - } - - _ = retry_timer.tick() => { - if requests.is_empty() { - if let Some(block) = last_block.clone() { - requests.push(send_block(network_client.clone(), peer, rtt_estimate, block)); - } - } - } - }; - - // Limit RTT estimate to be between 5ms and 5s. - rtt_estimate = min(rtt_estimate, Duration::from_secs(5)); - rtt_estimate = max(rtt_estimate, Duration::from_millis(5)); - context - .metrics - .node_metrics - .broadcaster_rtt_estimate_ms - .with_label_values(&[peer_hostname]) - .set(rtt_estimate.as_millis() as i64); - } - } -} - -#[cfg(test)] -mod test { - use std::{collections::BTreeMap, ops::DerefMut, time::Duration}; - - use async_trait::async_trait; - use bytes::Bytes; - use parking_lot::Mutex; - use tokio::time::sleep; - - use super::*; - use crate::{ - Round, - block::{BlockRef, ExtendedBlock, TestBlock}, - commit::CommitRange, - core::CoreSignals, - network::BlockStream, - }; - - struct FakeNetworkClient { - blocks_sent: Mutex>>, - } - - impl FakeNetworkClient { - fn new() -> Self { - Self { - blocks_sent: Mutex::new(BTreeMap::new()), - } - } - - fn blocks_sent(&self) -> BTreeMap> { - let mut blocks_sent = self.blocks_sent.lock(); - let result = std::mem::take(blocks_sent.deref_mut()); - blocks_sent.clear(); - result - } - } - - #[async_trait] - impl NetworkClient for FakeNetworkClient { - const SUPPORT_STREAMING: bool = false; - - async fn send_block( - &self, - peer: AuthorityIndex, - block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - let mut blocks_sent = self.blocks_sent.lock(); - let blocks = blocks_sent.entry(peer).or_default(); - blocks.push(block.serialized().clone()); - Ok(()) - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - unimplemented!("Unimplemented") - } - - async fn fetch_blocks( - &self, - _peer: AuthorityIndex, - _block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn get_latest_rounds( - &self, - _peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_broadcaster() { - let (context, _keys) = Context::new_for_test(4); - let context = Arc::new(context); - let network_client = Arc::new(FakeNetworkClient::new()); - let (core_signals, signals_receiver) = CoreSignals::new(context.clone()); - let _broadcaster = - Broadcaster::new(context.clone(), network_client.clone(), &signals_receiver); - - let block = VerifiedBlock::new_for_test(TestBlock::new(9, 1).build()); - assert!( - core_signals - .new_block(ExtendedBlock { - block: block.clone(), - excluded_ancestors: vec![], - }) - .is_ok(), - "No subscriber active to receive the block" - ); - - // block should be broadcasted immediately to all peers. - sleep(Duration::from_millis(1)).await; - let blocks_sent = network_client.blocks_sent(); - for (index, _) in context.committee.authorities() { - if index == context.own_index { - continue; - } - assert_eq!(blocks_sent.get(&index).unwrap(), &vec![block.serialized()]); - } - - // block should not be re-broadcasted ... - sleep(Broadcaster::LAST_BLOCK_RETRY_INTERVAL / 2).await; - let blocks_sent = network_client.blocks_sent(); - for (index, _) in context.committee.authorities() { - if index == context.own_index { - continue; - } - assert!(!blocks_sent.contains_key(&index)); - } - - // ... until LAST_BLOCK_RETRY_INTERVAL - sleep(Broadcaster::LAST_BLOCK_RETRY_INTERVAL / 2).await; - let blocks_sent = network_client.blocks_sent(); - for (index, _) in context.committee.authorities() { - if index == context.own_index { - continue; - } - assert_eq!(blocks_sent.get(&index).unwrap(), &vec![block.serialized()]); - } - } -} diff --git a/consensus/core/src/commit.rs b/consensus/core/src/commit.rs deleted file mode 100644 index 7fb689ef791..00000000000 --- a/consensus/core/src/commit.rs +++ /dev/null @@ -1,783 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - cmp::Ordering, - fmt::{self, Debug, Display, Formatter}, - hash::{Hash, Hasher}, - ops::{Deref, Range, RangeInclusive}, - sync::Arc, -}; - -use bytes::Bytes; -use consensus_config::{AuthorityIndex, DIGEST_LENGTH, DefaultHashFunction}; -use enum_dispatch::enum_dispatch; -use fastcrypto::hash::{Digest, HashFunction as _}; -use serde::{Deserialize, Serialize}; - -use crate::{ - block::{BlockAPI, BlockRef, BlockTimestampMs, Round, Slot, VerifiedBlock}, - leader_scoring::ReputationScores, - storage::Store, -}; - -/// Index of a commit among all consensus commits. -pub type CommitIndex = u32; - -pub(crate) const GENESIS_COMMIT_INDEX: CommitIndex = 0; - -/// Default wave length for all committers. A longer wave length increases the -/// chance of committing the leader under asynchrony at the cost of latency in -/// the common case. -// TODO: merge DEFAULT_WAVE_LENGTH and MINIMUM_WAVE_LENGTH into a single -// constant, because we are unlikely to change them via config in the -// foreseeable future. -pub(crate) const DEFAULT_WAVE_LENGTH: Round = MINIMUM_WAVE_LENGTH; - -/// We need at least one leader round, one voting round, and one decision round. -pub(crate) const MINIMUM_WAVE_LENGTH: Round = 3; - -/// The consensus protocol operates in 'waves'. Each wave is composed of a -/// leader round, at least one voting round, and one decision round. -pub(crate) type WaveNumber = u32; - -/// [`Commit`] summarizes [`CommittedSubDag`] for storage and network -/// communications. -/// -/// Validators should be able to reconstruct a sequence of CommittedSubDag from -/// the corresponding Commit and blocks referenced in the Commit. -/// A field must meet these requirements to be added to Commit: -/// - helps with recovery locally and for peers catching up. -/// - cannot be derived from a sequence of Commits and other persisted values. -/// -/// For example, transactions in blocks should not be included in Commit, -/// because they can be retrieved from blocks specified in Commit. Last -/// committed round per authority also should not be included, because it can be -/// derived from the latest value in storage and the additional sequence of -/// Commits. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -#[enum_dispatch(CommitAPI)] -pub(crate) enum Commit { - V1(CommitV1), -} - -impl Commit { - /// Create a new commit. - pub(crate) fn new( - index: CommitIndex, - previous_digest: CommitDigest, - timestamp_ms: BlockTimestampMs, - leader: BlockRef, - blocks: Vec, - ) -> Self { - Commit::V1(CommitV1 { - index, - previous_digest, - timestamp_ms, - leader, - blocks, - }) - } - - pub(crate) fn serialize(&self) -> Result { - let bytes = bcs::to_bytes(self)?; - Ok(bytes.into()) - } -} - -/// Accessors to Commit info. -#[enum_dispatch] -pub(crate) trait CommitAPI { - fn round(&self) -> Round; - fn index(&self) -> CommitIndex; - fn previous_digest(&self) -> CommitDigest; - fn timestamp_ms(&self) -> BlockTimestampMs; - fn leader(&self) -> BlockRef; - fn blocks(&self) -> &[BlockRef]; -} - -/// Specifies one consensus commit. -/// It is stored on disk, so it does not contain blocks which are stored -/// individually. -#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub(crate) struct CommitV1 { - /// Index of the commit. - /// First commit after genesis has an index of 1, then every next commit has - /// an index incremented by 1. - index: CommitIndex, - /// Digest of the previous commit. - /// Set to CommitDigest::MIN for the first commit after genesis. - previous_digest: CommitDigest, - /// Timestamp of the commit, max of the timestamp of the leader block and - /// previous Commit timestamp. - timestamp_ms: BlockTimestampMs, - /// A reference to the commit leader. - leader: BlockRef, - /// Refs to committed blocks, in the commit order. - blocks: Vec, -} - -impl CommitAPI for CommitV1 { - fn round(&self) -> Round { - self.leader.round - } - - fn index(&self) -> CommitIndex { - self.index - } - - fn previous_digest(&self) -> CommitDigest { - self.previous_digest - } - - fn timestamp_ms(&self) -> BlockTimestampMs { - self.timestamp_ms - } - - fn leader(&self) -> BlockRef { - self.leader - } - - fn blocks(&self) -> &[BlockRef] { - &self.blocks - } -} - -/// A commit is trusted when it is produced locally or certified by a quorum of -/// authorities. Blocks referenced by TrustedCommit are assumed to be valid. -/// Only trusted Commit can be sent to execution. -/// -/// Note: clone() is relatively cheap with the underlying data refcounted. -#[derive(Clone, Debug, PartialEq)] -pub(crate) struct TrustedCommit { - inner: Arc, - - // Cached digest and serialized value, to avoid re-computing these values. - digest: CommitDigest, - serialized: Bytes, -} - -impl TrustedCommit { - pub(crate) fn new_trusted(commit: Commit, serialized: Bytes) -> Self { - let digest = Self::compute_digest(&serialized); - Self { - inner: Arc::new(commit), - digest, - serialized, - } - } - - #[cfg(test)] - pub(crate) fn new_for_test( - index: CommitIndex, - previous_digest: CommitDigest, - timestamp_ms: BlockTimestampMs, - leader: BlockRef, - blocks: Vec, - ) -> Self { - let commit = Commit::new(index, previous_digest, timestamp_ms, leader, blocks); - let serialized = commit.serialize().unwrap(); - Self::new_trusted(commit, serialized) - } - - pub(crate) fn reference(&self) -> CommitRef { - CommitRef { - index: self.index(), - digest: self.digest(), - } - } - - pub(crate) fn digest(&self) -> CommitDigest { - self.digest - } - - pub(crate) fn serialized(&self) -> &Bytes { - &self.serialized - } - - pub(crate) fn compute_digest(serialized: &[u8]) -> CommitDigest { - let mut hasher = DefaultHashFunction::new(); - hasher.update(serialized); - CommitDigest(hasher.finalize().into()) - } -} - -/// Allow easy access on the underlying Commit. -impl Deref for TrustedCommit { - type Target = Commit; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -/// `CertifiedCommits` keeps the synchronized certified commits along with the -/// corresponding votes received from the peer that provided these commits. -/// The `votes` contain the blocks as those provided by the peer, and certify -/// the tip of the synced commits. -#[derive(Clone, Debug)] -pub(crate) struct CertifiedCommits { - commits: Vec, - votes: Vec, -} - -impl CertifiedCommits { - pub(crate) fn new(commits: Vec, votes: Vec) -> Self { - Self { commits, votes } - } - - pub(crate) fn commits(&self) -> &[CertifiedCommit] { - &self.commits - } - - pub(crate) fn votes(&self) -> &[VerifiedBlock] { - &self.votes - } -} - -/// A commit that has been synced and certified by a quorum of authorities. -#[derive(Clone, Debug)] -pub(crate) struct CertifiedCommit { - commit: Arc, - blocks: Vec, -} - -impl CertifiedCommit { - pub(crate) fn new_certified(commit: TrustedCommit, blocks: Vec) -> Self { - Self { - commit: Arc::new(commit), - blocks, - } - } - - pub fn blocks(&self) -> &[VerifiedBlock] { - &self.blocks - } -} - -impl Deref for CertifiedCommit { - type Target = TrustedCommit; - - fn deref(&self) -> &Self::Target { - &self.commit - } -} - -/// Digest of a consensus commit. -#[derive(Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct CommitDigest([u8; consensus_config::DIGEST_LENGTH]); - -impl CommitDigest { - /// Lexicographic min & max digest. - pub const MIN: Self = Self([u8::MIN; consensus_config::DIGEST_LENGTH]); - pub const MAX: Self = Self([u8::MAX; consensus_config::DIGEST_LENGTH]); - - pub fn into_inner(self) -> [u8; consensus_config::DIGEST_LENGTH] { - self.0 - } -} - -impl Hash for CommitDigest { - fn hash(&self, state: &mut H) { - state.write(&self.0[..8]); - } -} - -impl From for Digest<{ DIGEST_LENGTH }> { - fn from(hd: CommitDigest) -> Self { - Digest::new(hd.0) - } -} - -impl fmt::Display for CommitDigest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!( - f, - "{}", - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, self.0) - .get(0..4) - .ok_or(fmt::Error)? - ) - } -} - -impl fmt::Debug for CommitDigest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!( - f, - "{}", - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, self.0) - ) - } -} - -/// Uniquely identifies a commit with its index and digest. -#[derive(Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq, PartialOrd, Ord)] -pub struct CommitRef { - pub index: CommitIndex, - pub digest: CommitDigest, -} - -impl CommitRef { - pub fn new(index: CommitIndex, digest: CommitDigest) -> Self { - Self { index, digest } - } -} - -impl fmt::Display for CommitRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "C{}({})", self.index, self.digest) - } -} - -impl fmt::Debug for CommitRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - write!(f, "C{}({:?})", self.index, self.digest) - } -} - -// Represents a vote on a Commit. -pub type CommitVote = CommitRef; - -/// The output of consensus to execution is an ordered list of -/// [`CommittedSubDag`]. Each CommittedSubDag contains the information needed to -/// execution transactions in the consensus commit. -/// -/// The application processing CommittedSubDag can arbitrarily sort the blocks -/// within each sub-dag (but using a deterministic algorithm). -#[derive(Clone, PartialEq)] -pub struct CommittedSubDag { - /// A reference to the leader of the sub-dag - pub leader: BlockRef, - /// All the committed blocks that are part of this sub-dag - pub blocks: Vec, - /// The timestamp of the commit, obtained from the timestamp of the leader - /// block. - pub timestamp_ms: BlockTimestampMs, - /// The reference of the commit. - /// First commit after genesis has a index of 1, then every next commit has - /// a index incremented by 1. - pub commit_ref: CommitRef, - /// Optional scores that are provided as part of the consensus output to - /// IOTA that can then be used by IOTA for future submission to - /// consensus. - pub reputation_scores_desc: Vec<(AuthorityIndex, u64)>, -} - -impl CommittedSubDag { - /// Creates a new committed sub dag. - pub fn new( - leader: BlockRef, - blocks: Vec, - timestamp_ms: BlockTimestampMs, - commit_ref: CommitRef, - reputation_scores_desc: Vec<(AuthorityIndex, u64)>, - ) -> Self { - Self { - leader, - blocks, - timestamp_ms, - commit_ref, - reputation_scores_desc, - } - } -} - -// Sort the blocks of the sub-dag blocks by round number then authority index. -// Any deterministic & stable algorithm works. -pub(crate) fn sort_sub_dag_blocks(blocks: &mut [VerifiedBlock]) { - blocks.sort_by(|a, b| { - a.round() - .cmp(&b.round()) - .then_with(|| a.author().cmp(&b.author())) - }) -} - -impl Display for CommittedSubDag { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!( - f, - "CommittedSubDag(leader={}, ref={}, blocks=[", - self.leader, self.commit_ref - )?; - for (idx, block) in self.blocks.iter().enumerate() { - if idx > 0 { - write!(f, ", ")?; - } - write!(f, "{}", block.digest())?; - } - write!(f, "])") - } -} - -impl fmt::Debug for CommittedSubDag { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}@{} ([", self.leader, self.commit_ref)?; - for block in &self.blocks { - write!(f, "{}, ", block.reference())?; - } - write!( - f, - "];{}ms;rs{:?})", - self.timestamp_ms, self.reputation_scores_desc - ) - } -} - -// Recovers the full CommittedSubDag from block store, based on Commit. -pub fn load_committed_subdag_from_store( - store: &dyn Store, - commit: TrustedCommit, - reputation_scores_desc: Vec<(AuthorityIndex, u64)>, -) -> CommittedSubDag { - let mut leader_block_idx = None; - let commit_blocks = store - .read_blocks(commit.blocks()) - .expect("We should have the block referenced in the commit data"); - let blocks = commit_blocks - .into_iter() - .enumerate() - .map(|(idx, commit_block_opt)| { - let commit_block = - commit_block_opt.expect("We should have the block referenced in the commit data"); - if commit_block.reference() == commit.leader() { - leader_block_idx = Some(idx); - } - commit_block - }) - .collect::>(); - let leader_block_idx = leader_block_idx.expect("Leader block must be in the sub-dag"); - let leader_block_ref = blocks[leader_block_idx].reference(); - CommittedSubDag::new( - leader_block_ref, - blocks, - commit.timestamp_ms(), - commit.reference(), - reputation_scores_desc, - ) -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub(crate) enum Decision { - Direct, - Indirect, - Certified, // This is a commit certified leader so no commit decision was made locally. -} - -/// The status of a leader slot from the direct and indirect commit rules. -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum LeaderStatus { - Commit(VerifiedBlock), - Skip(Slot), - Undecided(Slot), -} - -impl LeaderStatus { - pub(crate) fn round(&self) -> Round { - match self { - Self::Commit(block) => block.round(), - Self::Skip(leader) => leader.round, - Self::Undecided(leader) => leader.round, - } - } - - pub(crate) fn is_decided(&self) -> bool { - match self { - Self::Commit(_) => true, - Self::Skip(_) => true, - Self::Undecided(_) => false, - } - } - - pub(crate) fn into_decided_leader(self) -> Option { - match self { - Self::Commit(block) => Some(DecidedLeader::Commit(block)), - Self::Skip(slot) => Some(DecidedLeader::Skip(slot)), - Self::Undecided(..) => None, - } - } -} - -impl Display for LeaderStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Commit(block) => write!(f, "Commit({})", block.reference()), - Self::Skip(slot) => write!(f, "Skip({slot})"), - Self::Undecided(slot) => write!(f, "Undecided({slot})"), - } - } -} - -/// Decision of each leader slot. -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum DecidedLeader { - Commit(VerifiedBlock), - Skip(Slot), -} - -impl DecidedLeader { - // Slot where the leader is decided. - pub(crate) fn slot(&self) -> Slot { - match self { - Self::Commit(block) => block.reference().into(), - Self::Skip(slot) => *slot, - } - } - - // Converts to committed block if the decision is to commit. Returns None - // otherwise. - pub(crate) fn into_committed_block(self) -> Option { - match self { - Self::Commit(block) => Some(block), - Self::Skip(_) => None, - } - } - - #[cfg(test)] - pub(crate) fn round(&self) -> Round { - match self { - Self::Commit(block) => block.round(), - Self::Skip(leader) => leader.round, - } - } - - #[cfg(test)] - pub(crate) fn authority(&self) -> AuthorityIndex { - match self { - Self::Commit(block) => block.author(), - Self::Skip(leader) => leader.authority, - } - } -} - -impl Display for DecidedLeader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Commit(block) => write!(f, "Commit({})", block.reference()), - Self::Skip(slot) => write!(f, "Skip({slot})"), - } - } -} - -/// Per-commit properties that can be regenerated from past values, and do not -/// need to be part of the Commit struct. -/// Only the latest version is needed for recovery, but more versions are stored -/// for debugging, and potentially restoring from an earlier state. -// TODO: version this struct. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct CommitInfo { - pub(crate) committed_rounds: Vec, - pub(crate) reputation_scores: ReputationScores, -} - -/// CommitRange stores a range of CommitIndex. The range contains the start -/// (inclusive) and end (inclusive) commit indices and can be ordered for use as -/// the key of a table. -/// -/// NOTE: using Range for internal representation for backward -/// compatibility. The external semantics of CommitRange is closer to -/// RangeInclusive. -#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct CommitRange(Range); - -impl CommitRange { - pub(crate) fn new(range: RangeInclusive) -> Self { - // When end is CommitIndex::MAX, the range can be considered as unbounded - // so it is ok to saturate at the end. - Self(*range.start()..(*range.end()).saturating_add(1)) - } - - // Inclusive - pub(crate) fn start(&self) -> CommitIndex { - self.0.start - } - - // Inclusive - pub(crate) fn end(&self) -> CommitIndex { - self.0.end.saturating_sub(1) - } - - pub(crate) fn extend_to(&mut self, other: CommitIndex) { - let new_end = other.saturating_add(1); - assert!(self.0.end <= new_end); - self.0 = self.0.start..new_end; - } - - pub(crate) fn size(&self) -> usize { - self.0 - .end - .checked_sub(self.0.start) - .expect("Range should never have end < start") as usize - } - - /// Check whether the two ranges have the same size. - pub(crate) fn is_equal_size(&self, other: &Self) -> bool { - self.size() == other.size() - } - - /// Check if the provided range is sequentially after this range. - pub(crate) fn is_next_range(&self, other: &Self) -> bool { - self.0.end == other.0.start - } -} - -impl Ord for CommitRange { - fn cmp(&self, other: &Self) -> Ordering { - self.start() - .cmp(&other.start()) - .then_with(|| self.end().cmp(&other.end())) - } -} - -impl PartialOrd for CommitRange { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl From> for CommitRange { - fn from(range: RangeInclusive) -> Self { - Self::new(range) - } -} - -/// Display CommitRange as an inclusive range. -impl Debug for CommitRange { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "CommitRange({}..={})", self.start(), self.end()) - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; - use crate::{ - block::TestBlock, - context::Context, - storage::{WriteBatch, mem_store::MemStore}, - }; - - #[tokio::test] - async fn test_new_subdag_from_commit() { - let store = Arc::new(MemStore::new()); - let context = Arc::new(Context::new_for_test(4).0); - let wave_length = DEFAULT_WAVE_LENGTH; - - // Populate fully connected test blocks for round 0 ~ 3, authorities 0 ~ 3. - let first_wave_rounds: u32 = wave_length; - let num_authorities: u32 = 4; - - let mut blocks = Vec::new(); - let (genesis_references, genesis): (Vec<_>, Vec<_>) = context - .committee - .authorities() - .map(|index| { - let author_idx = index.0.value() as u32; - let block = TestBlock::new(0, author_idx).build(); - VerifiedBlock::new_for_test(block) - }) - .map(|block| (block.reference(), block)) - .unzip(); - // TODO: avoid writing genesis blocks? - store.write(WriteBatch::default().blocks(genesis)).unwrap(); - blocks.append(&mut genesis_references.clone()); - - let mut ancestors = genesis_references; - let mut leader = None; - for round in 1..=first_wave_rounds { - let mut new_ancestors = vec![]; - for author in 0..num_authorities { - let base_ts = round as BlockTimestampMs * 1000; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_timestamp_ms(base_ts + (author + round) as u64) - .set_ancestors(ancestors.clone()) - .build(), - ); - store - .write(WriteBatch::default().blocks(vec![block.clone()])) - .unwrap(); - new_ancestors.push(block.reference()); - blocks.push(block.reference()); - - // only write one block for the final round, which is the leader - // of the committed subdag. - if round == first_wave_rounds { - leader = Some(block.clone()); - break; - } - } - ancestors = new_ancestors; - } - - let leader_block = leader.unwrap(); - let leader_ref = leader_block.reference(); - let commit_index = 1; - let commit = TrustedCommit::new_for_test( - commit_index, - CommitDigest::MIN, - leader_block.timestamp_ms(), - leader_ref, - blocks.clone(), - ); - let subdag = load_committed_subdag_from_store(store.as_ref(), commit.clone(), vec![]); - assert_eq!(subdag.leader, leader_ref); - assert_eq!(subdag.timestamp_ms, leader_block.timestamp_ms()); - assert_eq!( - subdag.blocks.len(), - (num_authorities * wave_length) as usize + 1 - ); - assert_eq!(subdag.commit_ref, commit.reference()); - assert_eq!(subdag.reputation_scores_desc, vec![]); - } - - #[tokio::test] - async fn test_commit_range() { - telemetry_subscribers::init_for_testing(); - let mut range1 = CommitRange::new(1..=5); - let range2 = CommitRange::new(2..=6); - let range3 = CommitRange::new(5..=10); - let range4 = CommitRange::new(6..=10); - let range5 = CommitRange::new(6..=9); - let range6 = CommitRange::new(1..=1); - - assert_eq!(range1.start(), 1); - assert_eq!(range1.end(), 5); - - // Test range size - assert_eq!(range1.size(), 5); - assert_eq!(range3.size(), 6); - assert_eq!(range6.size(), 1); - - // Test next range check - assert!(!range1.is_next_range(&range2)); - assert!(!range1.is_next_range(&range3)); - assert!(range1.is_next_range(&range4)); - assert!(range1.is_next_range(&range5)); - - // Test equal size range check - assert!(range1.is_equal_size(&range2)); - assert!(!range1.is_equal_size(&range3)); - assert!(range1.is_equal_size(&range4)); - assert!(!range1.is_equal_size(&range5)); - - // Test range ordering - assert!(range1 < range2); - assert!(range2 < range3); - assert!(range3 < range4); - assert!(range5 < range4); - - // Test extending range - range1.extend_to(10); - assert_eq!(range1.start(), 1); - assert_eq!(range1.end(), 10); - assert_eq!(range1.size(), 10); - range1.extend_to(20); - assert_eq!(range1.start(), 1); - assert_eq!(range1.end(), 20); - assert_eq!(range1.size(), 20); - } -} diff --git a/consensus/core/src/commit_consumer.rs b/consensus/core/src/commit_consumer.rs deleted file mode 100644 index 8fe5d1bc746..00000000000 --- a/consensus/core/src/commit_consumer.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::{Arc, RwLock}; - -use iota_metrics::monitored_mpsc::UnboundedSender; -use tokio::sync::watch; -use tracing::{debug, info}; - -use crate::{CommitIndex, CommittedSubDag}; - -pub struct CommitConsumer { - // A channel to send the committed sub dags through - pub(crate) sender: UnboundedSender, - // Index of the last commit that the consumer has processed. This is useful for - // crash/recovery so mysticeti can replay the commits from the next index. - // First commit in the replayed sequence will have index last_processed_commit_index + 1. - // Set 0 to replay from the start (as generated commit sequence starts at index = 1). - pub(crate) last_processed_commit_index: CommitIndex, - // Allows the commit consumer to report its progress. - monitor: Arc, -} - -impl CommitConsumer { - pub fn new( - sender: UnboundedSender, - last_processed_commit_index: CommitIndex, - ) -> Self { - let monitor = Arc::new(CommitConsumerMonitor::new(last_processed_commit_index)); - Self { - sender, - last_processed_commit_index, - monitor, - } - } - - pub fn monitor(&self) -> Arc { - self.monitor.clone() - } -} - -pub struct CommitConsumerMonitor { - // highest commit that has been handled by IOTA - highest_handled_commit: watch::Sender, - - // the highest commit found in local storage at startup - highest_observed_commit_at_startup: RwLock, -} - -impl CommitConsumerMonitor { - pub(crate) fn new(last_handled_commit: CommitIndex) -> Self { - Self { - highest_handled_commit: watch::Sender::new(last_handled_commit), - highest_observed_commit_at_startup: RwLock::new(0), - } - } - - pub fn highest_handled_commit(&self) -> CommitIndex { - *self.highest_handled_commit.borrow() - } - - pub fn set_highest_handled_commit(&self, highest_handled_commit: CommitIndex) { - debug!("Highest handled commit set to {}", highest_handled_commit); - self.highest_handled_commit - .send_replace(highest_handled_commit); - } - - pub fn highest_observed_commit_at_startup(&self) -> CommitIndex { - *self.highest_observed_commit_at_startup.read().unwrap() - } - - pub fn set_highest_observed_commit_at_startup( - &self, - highest_observed_commit_at_startup: CommitIndex, - ) { - let highest_handled_commit = self.highest_handled_commit(); - assert!( - highest_observed_commit_at_startup >= highest_handled_commit, - "we cannot have handled a commit that we do not know about: {highest_observed_commit_at_startup} < {highest_handled_commit}", - ); - - let mut commit = self.highest_observed_commit_at_startup.write().unwrap(); - - assert!( - *commit == 0, - "highest_known_commit_at_startup can only be set once" - ); - *commit = highest_observed_commit_at_startup; - - info!( - "Highest observed commit at startup set to {}", - highest_observed_commit_at_startup - ); - } - - pub(crate) async fn replay_complete(&self) { - let highest_observed_commit_at_startup = self.highest_observed_commit_at_startup(); - let mut rx = self.highest_handled_commit.subscribe(); - loop { - let highest_handled = *rx.borrow_and_update(); - if highest_handled >= highest_observed_commit_at_startup { - return; - } - rx.changed().await.unwrap(); - } - } -} - -#[cfg(test)] -mod test { - use crate::CommitConsumerMonitor; - - #[test] - fn test_commit_consumer_monitor() { - let monitor = CommitConsumerMonitor::new(10); - assert_eq!(monitor.highest_handled_commit(), 10); - - monitor.set_highest_handled_commit(100); - assert_eq!(monitor.highest_handled_commit(), 100); - } -} diff --git a/consensus/core/src/commit_observer.rs b/consensus/core/src/commit_observer.rs deleted file mode 100644 index 8919ebe4e0f..00000000000 --- a/consensus/core/src/commit_observer.rs +++ /dev/null @@ -1,589 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use iota_metrics::monitored_mpsc::UnboundedSender; -use parking_lot::RwLock; -use tokio::time::Instant; -use tracing::{debug, info, instrument}; - -use crate::{ - CommitConsumer, CommittedSubDag, - block::{BlockAPI, VerifiedBlock}, - commit::{CommitAPI, CommitIndex, load_committed_subdag_from_store}, - context::Context, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - leader_schedule::LeaderSchedule, - linearizer::Linearizer, - storage::Store, -}; - -/// Role of CommitObserver -/// - Called by core when try_commit() returns newly committed leaders. -/// - The newly committed leaders are sent to commit observer and then commit -/// observer gets subdags for each leader via the commit interpreter -/// (linearizer) -/// - The committed subdags are sent as consensus output via an unbounded tokio -/// channel. -/// -/// No back pressure mechanism is needed as backpressure is handled as input -/// into consensus. -/// -/// - Commit metadata including index is persisted in store, before the -/// CommittedSubDag is sent to the consumer. -/// - When CommitObserver is initialized a last processed commit index can be -/// used to ensure any missing commits are re-sent. -pub(crate) struct CommitObserver { - context: Arc, - /// Component to deterministically collect subdags for committed leaders. - commit_interpreter: Linearizer, - /// An unbounded channel to send committed sub-dags to the consumer of - /// consensus output. - sender: UnboundedSender, - /// Persistent storage for blocks, commits and other consensus data. - store: Arc, - leader_schedule: Arc, -} - -impl CommitObserver { - pub(crate) fn new( - context: Arc, - commit_consumer: CommitConsumer, - dag_state: Arc>, - store: Arc, - leader_schedule: Arc, - ) -> Self { - let mut observer = Self { - commit_interpreter: Linearizer::new( - context.clone(), - dag_state, - leader_schedule.clone(), - ), - context, - sender: commit_consumer.sender, - store, - leader_schedule, - }; - - observer.recover_and_send_commits(commit_consumer.last_processed_commit_index); - observer - } - - #[instrument(level = "trace", skip_all)] - pub(crate) fn handle_commit( - &mut self, - committed_leaders: Vec, - ) -> ConsensusResult> { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["CommitObserver::handle_commit"]) - .start_timer(); - - let committed_sub_dags = self.commit_interpreter.handle_commit(committed_leaders); - let mut sent_sub_dags = Vec::with_capacity(committed_sub_dags.len()); - for committed_sub_dag in committed_sub_dags.into_iter() { - // Failures in sender.send() are assumed to be permanent - if let Err(err) = self.sender.send(committed_sub_dag.clone()) { - tracing::error!( - "Failed to send committed sub-dag, probably due to shutdown: {err:?}" - ); - return Err(ConsensusError::Shutdown); - } - tracing::debug!( - "Sending to execution commit {} leader {}", - committed_sub_dag.commit_ref, - committed_sub_dag.leader - ); - sent_sub_dags.push(committed_sub_dag); - } - - self.report_metrics(&sent_sub_dags); - tracing::trace!("Committed & sent {sent_sub_dags:#?}"); - Ok(sent_sub_dags) - } - - fn recover_and_send_commits(&mut self, last_processed_commit_index: CommitIndex) { - let now = Instant::now(); - // TODO: remove this check, to allow consensus to regenerate commits? - let last_commit = self - .store - .read_last_commit() - .expect("Reading the last commit should not fail"); - - if let Some(last_commit) = &last_commit { - let last_commit_index = last_commit.index(); - - assert!(last_commit_index >= last_processed_commit_index); - if last_commit_index == last_processed_commit_index { - debug!( - "Nothing to recover for commit observer as commit index {last_commit_index} = {last_processed_commit_index} last processed index" - ); - return; - } - }; - - // We should not send the last processed commit again, so - // last_processed_commit_index+1 - let unsent_commits = self - .store - .scan_commits(((last_processed_commit_index + 1)..=CommitIndex::MAX).into()) - .expect("Scanning commits should not fail"); - - info!( - "Recovering commit observer after index {last_processed_commit_index} with last commit {} and {} unsent commits", - last_commit.map(|c| c.index()).unwrap_or_default(), - unsent_commits.len() - ); - - // Resend all the committed subdags to the consensus output channel - // for all the commits above the last processed index. - let mut last_sent_commit_index = last_processed_commit_index; - let num_unsent_commits = unsent_commits.len(); - for (index, commit) in unsent_commits.into_iter().enumerate() { - // Commit index must be continuous. - assert_eq!(commit.index(), last_sent_commit_index + 1); - - // On recovery leader schedule will be updated with the current scores - // and the scores will be passed along with the last commit sent to - // iota so that the current scores are available for submission. - let reputation_scores = if index == num_unsent_commits - 1 { - self.leader_schedule - .leader_swap_table - .read() - .reputation_scores_desc - .clone() - } else { - vec![] - }; - - info!("Sending commit {} during recovery", commit.index()); - let committed_sub_dag = - load_committed_subdag_from_store(self.store.as_ref(), commit, reputation_scores); - self.sender.send(committed_sub_dag).unwrap_or_else(|e| { - panic!("Failed to send commit during recovery, probably due to shutdown: {e:?}") - }); - - last_sent_commit_index += 1; - } - - info!( - "Commit observer recovery completed, took {:?}", - now.elapsed() - ); - } - - fn report_metrics(&self, committed: &[CommittedSubDag]) { - let metrics = &self.context.metrics.node_metrics; - let utc_now = self.context.clock.timestamp_utc_ms(); - - for commit in committed { - debug!( - "Consensus commit {} with leader {} has {} blocks", - commit.commit_ref, - commit.leader, - commit.blocks.len() - ); - - metrics - .last_committed_leader_round - .set(commit.leader.round as i64); - metrics - .last_commit_index - .set(commit.commit_ref.index as i64); - metrics - .blocks_per_commit_count - .observe(commit.blocks.len() as f64); - - for block in &commit.blocks { - let latency_ms = utc_now.saturating_sub(block.timestamp_ms()); - metrics - .block_commit_latency - .observe(Duration::from_millis(latency_ms).as_secs_f64()); - } - } - - self.context - .metrics - .node_metrics - .sub_dags_per_commit_count - .observe(committed.len() as f64); - } -} - -#[cfg(test)] -mod tests { - use iota_metrics::monitored_mpsc::{UnboundedReceiver, unbounded_channel}; - use parking_lot::RwLock; - use rstest::rstest; - - use super::*; - use crate::{ - block::BlockRef, context::Context, dag_state::DagState, - linearizer::median_timestamp_by_stake, storage::mem_store::MemStore, - test_dag_builder::DagBuilder, - }; - - #[rstest] - #[tokio::test] - async fn test_handle_commit(#[values(true, false)] consensus_median_timestamp: bool) { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - - let (mut context, _keys) = Context::new_for_test(num_authorities); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - - let context = Arc::new(context); - - let mem_store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - mem_store.clone(), - ))); - let last_processed_commit_index = 0; - let (sender, mut receiver) = unbounded_channel("consensus_output"); - - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let mut observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender, last_processed_commit_index), - dag_state.clone(), - mem_store.clone(), - leader_schedule, - ); - - // Populate fully connected test blocks for round 0 ~ 10, authorities 0 ~ 3. - let num_rounds = 10; - let mut builder = DagBuilder::new(context.clone()); - builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - let leaders = builder - .leader_blocks(1..=num_rounds) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - let commits = observer.handle_commit(leaders.clone()).unwrap(); - - // Check commits are returned by CommitObserver::handle_commit is accurate - let mut expected_stored_refs: Vec = vec![]; - for (idx, subdag) in commits.iter().enumerate() { - tracing::info!("{subdag:?}"); - assert_eq!(subdag.leader, leaders[idx].reference()); - - let expected_ts = if consensus_median_timestamp { - let block_refs = leaders[idx] - .ancestors() - .iter() - .filter(|block_ref| block_ref.round == leaders[idx].round() - 1) - .cloned() - .collect::>(); - let blocks = dag_state - .read() - .get_blocks(&block_refs) - .into_iter() - .map(|block_opt| block_opt.expect("We should have all blocks in dag state.")); - median_timestamp_by_stake(&context, blocks).unwrap() - } else { - leaders[idx].timestamp_ms() - }; - - let expected_ts = if idx == 0 { - expected_ts - } else { - expected_ts.max(commits[idx - 1].timestamp_ms) - }; - assert_eq!(expected_ts, subdag.timestamp_ms); - if idx == 0 { - // First subdag includes the leader block plus all ancestor blocks - // of the leader minus the genesis round blocks - assert_eq!(subdag.blocks.len(), 1); - } else { - // Every subdag after will be missing the leader block from the previous - // committed subdag - assert_eq!(subdag.blocks.len(), num_authorities); - } - for block in subdag.blocks.iter() { - expected_stored_refs.push(block.reference()); - assert!(block.round() <= leaders[idx].round()); - } - assert_eq!(subdag.commit_ref.index, idx as CommitIndex + 1); - } - - // Check commits sent over consensus output channel is accurate - let mut processed_subdag_index = 0; - while let Ok(subdag) = receiver.try_recv() { - assert_eq!(subdag, commits[processed_subdag_index]); - assert_eq!(subdag.reputation_scores_desc, vec![]); - processed_subdag_index = subdag.commit_ref.index as usize; - if processed_subdag_index == leaders.len() { - break; - } - } - assert_eq!(processed_subdag_index, leaders.len()); - - verify_channel_empty(&mut receiver); - - // Check commits have been persisted to storage - let last_commit = mem_store.read_last_commit().unwrap().unwrap(); - assert_eq!( - last_commit.index(), - commits.last().unwrap().commit_ref.index - ); - let all_stored_commits = mem_store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), leaders.len()); - let blocks_existence = mem_store.contains_blocks(&expected_stored_refs).unwrap(); - assert!(blocks_existence.iter().all(|exists| *exists)); - } - - #[tokio::test] - async fn test_recover_and_send_commits() { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let context = Arc::new(Context::new_for_test(num_authorities).0); - let mem_store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - mem_store.clone(), - ))); - let last_processed_commit_index = 0; - let (sender, mut receiver) = unbounded_channel("consensus_output"); - - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let mut observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), last_processed_commit_index), - dag_state.clone(), - mem_store.clone(), - leader_schedule.clone(), - ); - - // Populate fully connected test blocks for round 0 ~ 10, authorities 0 ~ 3. - let num_rounds = 10; - let mut builder = DagBuilder::new(context.clone()); - builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - let leaders = builder - .leader_blocks(1..=num_rounds) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Commit first batch of leaders (2) and "receive" the subdags as the - // consumer of the consensus output channel. - let expected_last_processed_index: usize = 2; - let mut commits = observer - .handle_commit( - leaders - .clone() - .into_iter() - .take(expected_last_processed_index) - .collect::>(), - ) - .unwrap(); - - // Check commits sent over consensus output channel is accurate - let mut processed_subdag_index = 0; - while let Ok(subdag) = receiver.try_recv() { - tracing::info!("Processed {subdag}"); - assert_eq!(subdag, commits[processed_subdag_index]); - assert_eq!(subdag.reputation_scores_desc, vec![]); - processed_subdag_index = subdag.commit_ref.index as usize; - if processed_subdag_index == expected_last_processed_index { - break; - } - } - assert_eq!(processed_subdag_index, expected_last_processed_index); - - verify_channel_empty(&mut receiver); - - // Check last stored commit is correct - let last_commit = mem_store.read_last_commit().unwrap().unwrap(); - assert_eq!( - last_commit.index(), - expected_last_processed_index as CommitIndex - ); - - // Handle next batch of leaders (1), these will be sent by consensus but not - // "processed" by consensus output channel. Simulating something happened on - // the consumer side where the commits were not persisted. - commits.append( - &mut observer - .handle_commit( - leaders - .into_iter() - .skip(expected_last_processed_index) - .collect::>(), - ) - .unwrap(), - ); - - let expected_last_sent_index = num_rounds as usize; - while let Ok(subdag) = receiver.try_recv() { - tracing::info!("{subdag} was sent but not processed by consumer"); - assert_eq!(subdag, commits[processed_subdag_index]); - assert_eq!(subdag.reputation_scores_desc, vec![]); - processed_subdag_index = subdag.commit_ref.index as usize; - if processed_subdag_index == expected_last_sent_index { - break; - } - } - assert_eq!(processed_subdag_index, expected_last_sent_index); - - verify_channel_empty(&mut receiver); - - // Check last stored commit is correct. We should persist the last commit - // that was sent over the channel regardless of how the consumer handled - // the commit on their end. - let last_commit = mem_store.read_last_commit().unwrap().unwrap(); - assert_eq!(last_commit.index(), expected_last_sent_index as CommitIndex); - - // Re-create commit observer starting from index 2 which represents the - // last processed index from the consumer over consensus output channel - let _observer = CommitObserver::new( - context, - CommitConsumer::new(sender, expected_last_processed_index as CommitIndex), - dag_state, - mem_store, - leader_schedule, - ); - - // Check commits sent over consensus output channel is accurate starting - // from last processed index of 2 and finishing at last sent index of 3. - processed_subdag_index = expected_last_processed_index; - while let Ok(subdag) = receiver.try_recv() { - tracing::info!("Processed {subdag} on resubmission"); - assert_eq!(subdag, commits[processed_subdag_index]); - assert_eq!(subdag.reputation_scores_desc, vec![]); - processed_subdag_index = subdag.commit_ref.index as usize; - if processed_subdag_index == expected_last_sent_index { - break; - } - } - assert_eq!(processed_subdag_index, expected_last_sent_index); - - verify_channel_empty(&mut receiver); - } - - #[tokio::test] - async fn test_send_no_missing_commits() { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let context = Arc::new(Context::new_for_test(num_authorities).0); - let mem_store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - mem_store.clone(), - ))); - let last_processed_commit_index = 0; - let (sender, mut receiver) = unbounded_channel("consensus_output"); - - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let mut observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), last_processed_commit_index), - dag_state.clone(), - mem_store.clone(), - leader_schedule.clone(), - ); - - // Populate fully connected test blocks for round 0 ~ 10, authorities 0 ~ 3. - let num_rounds = 10; - let mut builder = DagBuilder::new(context.clone()); - builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - let leaders = builder - .leader_blocks(1..=num_rounds) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Commit all of the leaders and "receive" the subdags as the consumer of - // the consensus output channel. - let expected_last_processed_index: usize = 10; - let commits = observer.handle_commit(leaders).unwrap(); - - // Check commits sent over consensus output channel is accurate - let mut processed_subdag_index = 0; - while let Ok(subdag) = receiver.try_recv() { - tracing::info!("Processed {subdag}"); - assert_eq!(subdag, commits[processed_subdag_index]); - assert_eq!(subdag.reputation_scores_desc, vec![]); - processed_subdag_index = subdag.commit_ref.index as usize; - if processed_subdag_index == expected_last_processed_index { - break; - } - } - assert_eq!(processed_subdag_index, expected_last_processed_index); - - verify_channel_empty(&mut receiver); - - // Check last stored commit is correct - let last_commit = mem_store.read_last_commit().unwrap().unwrap(); - assert_eq!( - last_commit.index(), - expected_last_processed_index as CommitIndex - ); - - // Re-create commit observer starting from index 3 which represents the - // last processed index from the consumer over consensus output channel - let _observer = CommitObserver::new( - context, - CommitConsumer::new(sender, expected_last_processed_index as CommitIndex), - dag_state, - mem_store, - leader_schedule, - ); - - // No commits should be resubmitted as consensus store's last commit index - // is equal to last processed index by consumer - verify_channel_empty(&mut receiver); - } - - /// After receiving all expected subdags, ensure channel is empty - fn verify_channel_empty(receiver: &mut UnboundedReceiver) { - match receiver.try_recv() { - Ok(_) => { - panic!("Expected the consensus output channel to be empty, but found more subdags.") - } - Err(e) => match e { - tokio::sync::mpsc::error::TryRecvError::Empty => {} - tokio::sync::mpsc::error::TryRecvError::Disconnected => { - panic!("The consensus output channel was unexpectedly closed.") - } - }, - } - } -} diff --git a/consensus/core/src/commit_syncer.rs b/consensus/core/src/commit_syncer.rs deleted file mode 100644 index 9686e6a4c4d..00000000000 --- a/consensus/core/src/commit_syncer.rs +++ /dev/null @@ -1,1049 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! CommitSyncer implements efficient synchronization of committed data. -//! -//! During the operation of a committee of authorities for consensus, one or -//! more authorities can fall behind the quorum in their received and accepted -//! blocks. This can happen due to network disruptions, host crash, or other -//! reasons. Authorities fell behind need to catch up to the quorum to be able -//! to vote on the latest leaders. So efficient synchronization is necessary -//! to minimize the impact of temporary disruptions and maintain smooth -//! operations of the network. -//! CommitSyncer achieves efficient synchronization by relying on the following: -//! when blocks are included in commits with >= 2f+1 certifiers by stake, these -//! blocks must have passed verifications on some honest validators, so -//! re-verifying them is unnecessary. In fact, the quorum certified commits -//! themselves can be trusted to be sent to IOTA directly, but for simplicity -//! this is not done. Blocks from trusted commits still go through Core and -//! committer. -//! -//! Another way CommitSyncer improves the efficiency of synchronization is -//! parallel fetching: commits have a simple dependency graph (linear), so it is -//! easy to fetch ranges of commits in parallel. -//! -//! Commit synchronization is an expensive operation, involving transferring -//! large amount of data via the network. And it is not on the critical path of -//! block processing. So the heuristics for synchronization, including triggers -//! and retries, should be chosen to favor throughput and efficient resource -//! usage, over faster reactions. - -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - time::Duration, -}; - -use bytes::Bytes; -use consensus_config::AuthorityIndex; -use futures::{StreamExt as _, stream::FuturesOrdered}; -use iota_metrics::spawn_logged_monitored_task; -use itertools::Itertools as _; -use parking_lot::RwLock; -use rand::{prelude::SliceRandom as _, rngs::ThreadRng}; -use tokio::{ - runtime::Handle, - sync::oneshot, - task::{JoinHandle, JoinSet}, - time::{MissedTickBehavior, sleep}, -}; -use tracing::{debug, info, instrument, warn}; - -use crate::{ - CommitConsumerMonitor, CommitIndex, - block::{BlockAPI, BlockRef, SignedBlock, VerifiedBlock}, - block_verifier::BlockVerifier, - commit::{ - CertifiedCommit, CertifiedCommits, Commit, CommitAPI as _, CommitDigest, CommitRange, - CommitRef, TrustedCommit, - }, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::CoreThreadDispatcher, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - network::NetworkClient, - scoring_metrics_store::ErrorSource, - stake_aggregator::{QuorumThreshold, StakeAggregator}, -}; - -// Handle to stop the CommitSyncer loop. -pub(crate) struct CommitSyncerHandle { - schedule_task: JoinHandle<()>, - tx_shutdown: oneshot::Sender<()>, -} - -impl CommitSyncerHandle { - pub(crate) async fn stop(self) { - let _ = self.tx_shutdown.send(()); - // Do not abort schedule task, which waits for fetches to shut down. - if let Err(e) = self.schedule_task.await { - if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } - } - } -} - -pub(crate) struct CommitSyncer { - // States shared by scheduler and fetch tasks. - - // Shared components wrapper. - inner: Arc>, - - // States only used by the scheduler. - - // Inflight requests to fetch commits from different authorities. - inflight_fetches: JoinSet<(AuthorityIndex, u32, CertifiedCommits)>, - // Additional ranges of commits to fetch. - pending_fetches: BTreeSet, - // Fetched commits and blocks by commit range. - fetched_ranges: BTreeMap, - // Highest commit index among inflight and pending fetches. - // Used to determine the start of new ranges to be fetched. - highest_scheduled_index: Option, - // Highest index among fetched commits, after commits and blocks are verified. - // Used for metrics. - highest_fetched_commit_index: CommitIndex, - // The commit index that is the max of highest local commit index and commit index inflight to - // Core. Used to determine if fetched blocks can be sent to Core without gaps. - synced_commit_index: CommitIndex, -} - -impl CommitSyncer { - pub(crate) fn new( - context: Arc, - core_thread_dispatcher: Arc, - commit_vote_monitor: Arc, - commit_consumer_monitor: Arc, - network_client: Arc, - block_verifier: Arc, - dag_state: Arc>, - ) -> Self { - let inner = Arc::new(Inner { - context, - core_thread_dispatcher, - commit_vote_monitor, - commit_consumer_monitor, - network_client, - block_verifier, - dag_state, - }); - let synced_commit_index = inner.dag_state.read().last_commit_index(); - CommitSyncer { - inner, - inflight_fetches: JoinSet::new(), - pending_fetches: BTreeSet::new(), - fetched_ranges: BTreeMap::new(), - highest_scheduled_index: None, - highest_fetched_commit_index: 0, - synced_commit_index, - } - } - - pub(crate) fn start(self) -> CommitSyncerHandle { - let (tx_shutdown, rx_shutdown) = oneshot::channel(); - let schedule_task = spawn_logged_monitored_task!(self.schedule_loop(rx_shutdown,)); - CommitSyncerHandle { - schedule_task, - tx_shutdown, - } - } - - async fn schedule_loop(mut self, mut rx_shutdown: oneshot::Receiver<()>) { - let mut interval = tokio::time::interval(Duration::from_secs(2)); - interval.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - tokio::select! { - // Periodically, schedule new fetches if the node is falling behind. - _ = interval.tick() => { - self.try_schedule_once(); - } - // Handles results from fetch tasks. - Some(result) = self.inflight_fetches.join_next(), if !self.inflight_fetches.is_empty() => { - if let Err(e) = result { - if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } - warn!("Fetch cancelled. CommitSyncer shutting down: {}", e); - // If any fetch is cancelled or panicked, try to shutdown and exit the loop. - self.inflight_fetches.shutdown().await; - return; - } - let (authority, target_end, commits) = result.unwrap(); - self.handle_fetch_result(authority, target_end, commits).await; - } - _ = &mut rx_shutdown => { - // Shutdown requested. - info!("CommitSyncer shutting down ..."); - self.inflight_fetches.shutdown().await; - return; - } - } - - self.try_start_fetches(); - } - } - - fn try_schedule_once(&mut self) { - let quorum_commit_index = self.inner.commit_vote_monitor.quorum_commit_index(); - let local_commit_index = self.inner.dag_state.read().last_commit_index(); - let metrics = &self.inner.context.metrics.node_metrics; - metrics - .commit_sync_quorum_index - .set(quorum_commit_index as i64); - metrics - .commit_sync_local_index - .set(local_commit_index as i64); - let highest_handled_index = self.inner.commit_consumer_monitor.highest_handled_commit(); - let highest_scheduled_index = self.highest_scheduled_index.unwrap_or(0); - // Update synced_commit_index periodically to make sure it is no smaller than - // local commit index. - self.synced_commit_index = self.synced_commit_index.max(local_commit_index); - let unhandled_commits_threshold = self.unhandled_commits_threshold(); - info!( - "Checking to schedule fetches: synced_commit_index={}, highest_handled_index={}, highest_scheduled_index={}, quorum_commit_index={}, unhandled_commits_threshold={}", - self.synced_commit_index, - highest_handled_index, - highest_scheduled_index, - quorum_commit_index, - unhandled_commits_threshold, - ); - - // TODO: cleanup inflight fetches that are no longer needed. - let fetch_after_index = self - .synced_commit_index - .max(self.highest_scheduled_index.unwrap_or(0)); - // When the node is falling behind, schedule pending fetches which will be - // executed on later. - for prev_end in (fetch_after_index..=quorum_commit_index) - .step_by(self.inner.context.parameters.commit_sync_batch_size as usize) - { - // Create range with inclusive start and end. - let range_start = prev_end + 1; - let range_end = prev_end + self.inner.context.parameters.commit_sync_batch_size; - // Commit range is not fetched when [range_start, range_end] contains less - // number of commits than the target batch size. This is to avoid - // the cost of processing more and smaller batches. Block broadcast, - // subscription and synchronization will help the node catchup. - if quorum_commit_index < range_end { - break; - } - // Pause scheduling new fetches when handling of commits is lagging. - if highest_handled_index + unhandled_commits_threshold < range_end { - warn!( - "Skip scheduling new commit fetches: consensus handler is lagging. highest_handled_index={}, highest_scheduled_index={}", - highest_handled_index, highest_scheduled_index - ); - break; - } - self.pending_fetches - .insert((range_start..=range_end).into()); - // quorum_commit_index should be non-decreasing, so highest_scheduled_index - // should not decrease either. - self.highest_scheduled_index = Some(range_end); - } - } - - async fn handle_fetch_result( - &mut self, - authority_index: AuthorityIndex, - target_end: CommitIndex, - certified_commits: CertifiedCommits, - ) { - assert!(!certified_commits.commits().is_empty()); - - let (total_blocks_fetched, total_blocks_size_bytes) = certified_commits - .commits() - .iter() - .fold((0, 0), |(blocks, bytes), c| { - ( - blocks + c.blocks().len(), - bytes - + c.blocks() - .iter() - .map(|b| b.serialized().len()) - .sum::() as u64, - ) - }); - let hostname = &self - .inner - .context - .committee - .authority(authority_index) - .hostname; - let metrics = &self.inner.context.metrics.node_metrics; - metrics - .commit_sync_fetched_commits - .with_label_values(&[&hostname.as_str()]) - .inc_by(certified_commits.commits().len() as u64); - metrics - .commit_sync_fetched_blocks - .with_label_values(&[&hostname.as_str()]) - .inc_by(total_blocks_fetched as u64); - metrics - .commit_sync_total_fetched_blocks_size - .with_label_values(&[&hostname.as_str()]) - .inc_by(total_blocks_size_bytes); - - let (commit_start, commit_end) = ( - certified_commits.commits().first().unwrap().index(), - certified_commits.commits().last().unwrap().index(), - ); - self.highest_fetched_commit_index = self.highest_fetched_commit_index.max(commit_end); - metrics - .commit_sync_highest_fetched_index - .set(self.highest_fetched_commit_index as i64); - - // Allow returning partial results, and try fetching the rest separately. - if commit_end < target_end { - self.pending_fetches - .insert((commit_end + 1..=target_end).into()); - } - // Make sure synced_commit_index is up to date. - self.synced_commit_index = self - .synced_commit_index - .max(self.inner.dag_state.read().last_commit_index()); - // Only add new blocks if at least some of them are not already synced. - if self.synced_commit_index < commit_end { - self.fetched_ranges - .insert((commit_start..=commit_end).into(), certified_commits); - } - // Try to process as many fetched blocks as possible. - while let Some((fetched_commit_range, _commits)) = self.fetched_ranges.first_key_value() { - // Only pop fetched_ranges if there is no gap with blocks already synced. - // Note: start, end and synced_commit_index are all inclusive. - let (fetched_commit_range, commits) = - if fetched_commit_range.start() <= self.synced_commit_index + 1 { - self.fetched_ranges.pop_first().unwrap() - } else { - // Found gap between earliest fetched block and latest synced block, - // so not sending additional blocks to Core. - metrics.commit_sync_gap_on_processing.inc(); - break; - }; - // Avoid sending to Core a whole batch of already synced blocks. - if fetched_commit_range.end() <= self.synced_commit_index { - continue; - } - - debug!( - "Fetched certified blocks for commit range {:?}: {}", - fetched_commit_range, - commits - .commits() - .iter() - .flat_map(|c| c.blocks()) - .map(|b| b.reference().to_string()) - .join(","), - ); - - // If core thread cannot handle the incoming blocks, it is ok to block here. - // Also it is possible to have missing ancestors because an equivocating - // validator may produce blocks that are not included in commits but - // are ancestors to other blocks. Synchronizer is needed to fill in - // the missing ancestors in this case. - match self - .inner - .core_thread_dispatcher - .add_certified_commits(commits) - .await - { - Ok(missing) => { - if !missing.is_empty() { - warn!( - "Fetched blocks have missing ancestors: {:?} for commit range {:?}", - missing, fetched_commit_range - ); - } - for block_ref in missing { - let hostname = &self - .inner - .context - .committee - .authority(block_ref.author) - .hostname; - metrics - .commit_sync_fetch_missing_blocks - .with_label_values(&[hostname]) - .inc(); - } - } - Err(e) => { - info!("Failed to add blocks, shutting down: {}", e); - return; - } - }; - - // Once commits and blocks are sent to Core, ratchet up synced_commit_index - self.synced_commit_index = self.synced_commit_index.max(fetched_commit_range.end()); - } - - metrics - .commit_sync_inflight_fetches - .set(self.inflight_fetches.len() as i64); - metrics - .commit_sync_pending_fetches - .set(self.pending_fetches.len() as i64); - metrics - .commit_sync_highest_synced_index - .set(self.synced_commit_index as i64); - } - - fn try_start_fetches(&mut self) { - // Cap parallel fetches based on configured limit and committee size, to avoid - // overloading the network. Also when there are too many fetched blocks - // that cannot be sent to Core before an earlier fetch has not finished, - // reduce parallelism so the earlier fetch can retry on a better host and - // succeed. - let target_parallel_fetches = self - .inner - .context - .parameters - .commit_sync_parallel_fetches - .min(self.inner.context.committee.size() * 2 / 3) - .min( - self.inner - .context - .parameters - .commit_sync_batches_ahead - .saturating_sub(self.fetched_ranges.len()), - ) - .max(1); - // Start new fetches if there are pending batches and available slots. - loop { - if self.inflight_fetches.len() >= target_parallel_fetches { - break; - } - let Some(commit_range) = self.pending_fetches.pop_first() else { - break; - }; - self.inflight_fetches - .spawn(Self::fetch_loop(self.inner.clone(), commit_range)); - } - - let metrics = &self.inner.context.metrics.node_metrics; - metrics - .commit_sync_inflight_fetches - .set(self.inflight_fetches.len() as i64); - metrics - .commit_sync_pending_fetches - .set(self.pending_fetches.len() as i64); - metrics - .commit_sync_highest_synced_index - .set(self.synced_commit_index as i64); - } - - // Retries fetching commits and blocks from available authorities, until a - // request succeeds where at least a prefix of the commit range is fetched. - // Returns the fetched commits and blocks referenced by the commits. - async fn fetch_loop( - inner: Arc>, - commit_range: CommitRange, - ) -> (AuthorityIndex, CommitIndex, CertifiedCommits) { - // Individual request base timeout. - const TIMEOUT: Duration = Duration::from_secs(10); - // Max per-request timeout will be base timeout times a multiplier. - // At the extreme, this means there will be 120s timeout to fetch - // max_blocks_per_fetch blocks. - const MAX_TIMEOUT_MULTIPLIER: u32 = 12; - // timeout * max number of targets should be reasonably small, so the - // system can adjust to slow network or large data sizes quickly. - const MAX_NUM_TARGETS: usize = 24; - let mut timeout_multiplier = 0; - - let _timer = inner - .context - .metrics - .node_metrics - .commit_sync_fetch_loop_latency - .start_timer(); - info!("Starting to fetch commits in {commit_range:?} ...",); - loop { - // Attempt to fetch commits and blocks through min(committee size, - // MAX_NUM_TARGETS) peers. - let mut target_authorities = inner - .context - .committee - .authorities() - .filter_map(|(i, _)| { - if i != inner.context.own_index { - Some(i) - } else { - None - } - }) - .collect_vec(); - target_authorities.shuffle(&mut ThreadRng::default()); - target_authorities.truncate(MAX_NUM_TARGETS); - // Increase timeout multiplier for each loop until MAX_TIMEOUT_MULTIPLIER. - timeout_multiplier = (timeout_multiplier + 1).min(MAX_TIMEOUT_MULTIPLIER); - let request_timeout = TIMEOUT * timeout_multiplier; - // Give enough overall timeout for fetching commits and blocks. - // - Timeout for fetching commits and commit certifying blocks. - // - Timeout for fetching blocks referenced by the commits. - // - Time spent on pipelining requests to fetch blocks. - // - Another headroom to allow fetch_once() to timeout gracefully if possible. - let fetch_timeout = request_timeout * 4; - // Try fetching from selected target authority. - for authority in target_authorities { - match tokio::time::timeout( - fetch_timeout, - Self::fetch_once( - inner.clone(), - authority, - commit_range.clone(), - request_timeout, - ), - ) - .await - { - Ok(Ok(commits)) => { - info!("Finished fetching commits in {commit_range:?}",); - return (authority, commit_range.end(), commits); - } - Ok(Err(e)) => { - let hostname = inner - .context - .committee - .authority(authority) - .hostname - .clone(); - inner - .context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority, - hostname.as_str(), - e.clone(), - ErrorSource::CommitSyncer, - &inner.context.metrics.node_metrics, - ); - warn!("Failed to fetch {commit_range:?} from {hostname}: {}", e); - let error: &'static str = e.into(); - inner - .context - .metrics - .node_metrics - .commit_sync_fetch_once_errors - .with_label_values(&[hostname.as_str(), error]) - .inc(); - } - Err(_) => { - let hostname = inner - .context - .committee - .authority(authority) - .hostname - .clone(); - warn!("Timed out fetching {commit_range:?} from {authority}",); - inner - .context - .metrics - .node_metrics - .commit_sync_fetch_once_errors - .with_label_values(&[hostname.as_str(), "FetchTimeout"]) - .inc(); - } - } - } - // Avoid busy looping, by waiting for a while before retrying. - sleep(TIMEOUT).await; - } - } - - // Fetches commits and blocks from a single authority. At a high level, first - // the commits are fetched and verified. After that, blocks referenced in - // the certified commits are fetched and sent to Core for processing. - async fn fetch_once( - inner: Arc>, - target_authority: AuthorityIndex, - commit_range: CommitRange, - timeout: Duration, - ) -> ConsensusResult { - // Maximum delay between consecutive pipelined requests, to avoid - // overwhelming the peer while still maintaining reasonable throughput. - const MAX_PIPELINE_DELAY: Duration = Duration::from_secs(1); - - let hostname = inner - .context - .committee - .authority(target_authority) - .hostname - .clone(); - let _timer = inner - .context - .metrics - .node_metrics - .commit_sync_fetch_once_latency - .with_label_values(&[hostname.as_str()]) - .start_timer(); - - // 1. Fetch commits in the commit range from the target authority. - let (serialized_commits, serialized_blocks) = inner - .network_client - .fetch_commits(target_authority, commit_range.clone(), timeout) - .await?; - - // 2. Verify the response contains blocks that can certify the last returned - // commit, - // and the returned commits are chained by digest, so earlier commits are - // certified as well. - let (commits, vote_blocks) = Handle::current() - .spawn_blocking({ - let inner = inner.clone(); - move || { - inner.verify_commits( - target_authority, - commit_range, - serialized_commits, - serialized_blocks, - ) - } - }) - .await - .expect("Spawn blocking should not fail")?; - - // 3. Fetch blocks referenced by the commits, from the same authority. - let block_refs: Vec<_> = commits.iter().flat_map(|c| c.blocks()).cloned().collect(); - let num_chunks = block_refs - .len() - .div_ceil(inner.context.parameters.max_blocks_per_fetch) - as u32; - let mut requests: FuturesOrdered<_> = block_refs - .chunks(inner.context.parameters.max_blocks_per_fetch) - .enumerate() - .map(|(i, request_block_refs)| { - let inner = inner.clone(); - async move { - // 4. Send out pipelined fetch requests to avoid overloading the target - // authority. - let individual_delay = (timeout / num_chunks).min(MAX_PIPELINE_DELAY); - sleep(individual_delay * i as u32).await; - // TODO: add some retries. - let serialized_blocks = inner - .network_client - .fetch_blocks( - target_authority, - request_block_refs.to_vec(), - vec![], - timeout, - ) - .await?; - // 5. Verify the same number of blocks are returned as requested. - if request_block_refs.len() != serialized_blocks.len() { - return Err(ConsensusError::UnexpectedNumberOfBlocksFetched { - authority: target_authority, - requested: request_block_refs.len(), - received: serialized_blocks.len(), - }); - } - // 6. Verify returned blocks have valid formats. - let signed_blocks = serialized_blocks - .iter() - .map(|serialized| { - let block: SignedBlock = bcs::from_bytes(serialized) - .map_err(ConsensusError::MalformedBlock)?; - Ok(block) - }) - .collect::>>()?; - // 7. Verify the returned blocks match the requested block refs. - // If they do match, the returned blocks can be considered verified as well. - let mut blocks = Vec::new(); - for ((requested_block_ref, signed_block), serialized) in request_block_refs - .iter() - .zip(signed_blocks) - .zip(serialized_blocks) - { - let signed_block_digest = VerifiedBlock::compute_digest(&serialized); - let received_block_ref = BlockRef::new( - signed_block.round(), - signed_block.author(), - signed_block_digest, - ); - if *requested_block_ref != received_block_ref { - return Err(ConsensusError::UnexpectedBlockForCommit { - peer: target_authority, - requested: *requested_block_ref, - received: received_block_ref, - }); - } - blocks.push(VerifiedBlock::new_verified(signed_block, serialized)); - } - Ok(blocks) - } - }) - .collect(); - - let mut fetched_blocks = BTreeMap::new(); - while let Some(result) = requests.next().await { - for block in result? { - fetched_blocks.insert(block.reference(), block); - } - } - - // 8. Make sure fetched block (and votes) timestamps are lower than current - // time. - for block in fetched_blocks.values().chain(vote_blocks.iter()) { - let now_ms = inner.context.clock.timestamp_utc_ms(); - let forward_drift = block.timestamp_ms().saturating_sub(now_ms); - if forward_drift == 0 { - continue; - }; - let peer_hostname = &inner.context.committee.authority(target_authority).hostname; - inner - .context - .metrics - .node_metrics - .block_timestamp_drift_ms - .with_label_values(&[peer_hostname.as_str(), "commit_syncer"]) - .inc_by(forward_drift); - - // We want to run the following checks only if the median based commit timestamp - // is not enabled. - if !inner - .context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - let forward_drift = Duration::from_millis(forward_drift); - if forward_drift >= inner.context.parameters.max_forward_time_drift { - warn!( - "Local clock is behind a quorum of peers: local ts {}, committed block ts {}", - now_ms, - block.timestamp_ms() - ); - } - sleep(forward_drift).await; - } - } - - // 9. Now create the Certified commits by assigning the blocks to each commit - // and retaining the commit votes history. - let mut certified_commits = Vec::new(); - for commit in &commits { - let blocks = commit - .blocks() - .iter() - .map(|block_ref| { - fetched_blocks - .remove(block_ref) - .expect("Block should exist") - }) - .collect::>(); - certified_commits.push(CertifiedCommit::new_certified(commit.clone(), blocks)); - } - - Ok(CertifiedCommits::new(certified_commits, vote_blocks)) - } - - fn unhandled_commits_threshold(&self) -> CommitIndex { - self.inner.context.parameters.commit_sync_batch_size - * (self.inner.context.parameters.commit_sync_batches_ahead as u32) - } - - #[cfg(test)] - fn pending_fetches(&self) -> BTreeSet { - self.pending_fetches.clone() - } - - #[cfg(test)] - fn fetched_ranges(&self) -> BTreeMap { - self.fetched_ranges.clone() - } - - #[cfg(test)] - fn highest_scheduled_index(&self) -> Option { - self.highest_scheduled_index - } - - #[cfg(test)] - fn highest_fetched_commit_index(&self) -> CommitIndex { - self.highest_fetched_commit_index - } - - #[cfg(test)] - fn synced_commit_index(&self) -> CommitIndex { - self.synced_commit_index - } -} - -struct Inner { - context: Arc, - core_thread_dispatcher: Arc, - commit_vote_monitor: Arc, - commit_consumer_monitor: Arc, - network_client: Arc, - block_verifier: Arc, - dag_state: Arc>, -} - -impl Inner { - /// Verifies the commits and also certifies them using the provided vote - /// blocks for the last commit. The method returns the trusted commits - /// and the votes as verified blocks. - #[instrument(level = "trace", skip_all)] - fn verify_commits( - &self, - peer: AuthorityIndex, - commit_range: CommitRange, - serialized_commits: Vec, - serialized_vote_blocks: Vec, - ) -> ConsensusResult<(Vec, Vec)> { - // Parse and verify commits. - let mut commits = Vec::new(); - for serialized in &serialized_commits { - let commit: Commit = - bcs::from_bytes(serialized).map_err(ConsensusError::MalformedCommit)?; - let digest = TrustedCommit::compute_digest(serialized); - if commits.is_empty() { - // start is inclusive, so first commit must be at the start index. - if commit.index() != commit_range.start() { - return Err(ConsensusError::UnexpectedStartCommit { - peer, - start: commit_range.start(), - commit: Box::new(commit), - }); - } - } else { - // Verify next commit increments index and references the previous digest. - let (last_commit_digest, last_commit): &(CommitDigest, Commit) = - commits.last().unwrap(); - if commit.index() != last_commit.index() + 1 - || &commit.previous_digest() != last_commit_digest - { - return Err(ConsensusError::UnexpectedCommitSequence { - peer, - prev_commit: Box::new(last_commit.clone()), - curr_commit: Box::new(commit), - }); - } - } - // Do not process more commits past the end index. - if commit.index() > commit_range.end() { - break; - } - commits.push((digest, commit)); - } - let Some((end_commit_digest, end_commit)) = commits.last() else { - return Err(ConsensusError::NoCommitReceived { peer }); - }; - - // Parse and verify blocks. Then accumulate votes on the end commit. - let end_commit_ref = CommitRef::new(end_commit.index(), *end_commit_digest); - let mut stake_aggregator = StakeAggregator::::new(); - let mut vote_blocks = Vec::new(); - for serialized in serialized_vote_blocks { - let block: SignedBlock = - bcs::from_bytes(&serialized).map_err(ConsensusError::MalformedBlock)?; - // The block signature needs to be verified. - self.block_verifier.verify(&block)?; - for vote in block.commit_votes() { - if *vote == end_commit_ref { - stake_aggregator.add(block.author(), &self.context.committee); - } - } - vote_blocks.push(VerifiedBlock::new_verified(block, serialized)); - } - - // Check if the end commit has enough votes. - if !stake_aggregator.reached_threshold(&self.context.committee) { - return Err(ConsensusError::NotEnoughCommitVotes { - stake: stake_aggregator.stake(), - peer, - commit: Box::new(end_commit.clone()), - }); - } - - let trusted_commits = commits - .into_iter() - .zip(serialized_commits) - .map(|((_d, c), s)| TrustedCommit::new_trusted(c, s)) - .collect(); - Ok((trusted_commits, vote_blocks)) - } -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - use bytes::Bytes; - use consensus_config::{AuthorityIndex, Parameters}; - use parking_lot::RwLock; - - use crate::{ - CommitConsumerMonitor, CommitDigest, CommitRef, Round, - block::{BlockRef, TestBlock, VerifiedBlock}, - block_verifier::NoopBlockVerifier, - commit::CommitRange, - commit_syncer::CommitSyncer, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::tests::MockCoreThreadDispatcher, - dag_state::DagState, - error::ConsensusResult, - network::{BlockStream, NetworkClient}, - storage::mem_store::MemStore, - }; - - #[derive(Default)] - struct FakeNetworkClient {} - - #[async_trait::async_trait] - impl NetworkClient for FakeNetworkClient { - const SUPPORT_STREAMING: bool = true; - - async fn send_block( - &self, - _peer: AuthorityIndex, - _serialized_block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - unimplemented!("Unimplemented") - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - unimplemented!("Unimplemented") - } - - async fn fetch_blocks( - &self, - _peer: AuthorityIndex, - _block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn get_latest_rounds( - &self, - _peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn commit_syncer_start_and_pause_scheduling() { - // SETUP - let (context, _) = Context::new_for_test(4); - // Use smaller batches and fetch limits for testing. - let context = Context { - own_index: AuthorityIndex::new_for_test(3), - parameters: Parameters { - commit_sync_batch_size: 5, - commit_sync_batches_ahead: 5, - commit_sync_parallel_fetches: 5, - max_blocks_per_fetch: 5, - ..context.parameters - }, - ..context - }; - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_thread_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let network_client = Arc::new(FakeNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let commit_consumer_monitor = Arc::new(CommitConsumerMonitor::new(0)); - let mut commit_syncer = CommitSyncer::new( - context, - core_thread_dispatcher, - commit_vote_monitor.clone(), - commit_consumer_monitor.clone(), - network_client, - block_verifier, - dag_state, - ); - - // Check initial state. - assert!(commit_syncer.pending_fetches().is_empty()); - assert!(commit_syncer.fetched_ranges().is_empty()); - assert!(commit_syncer.highest_scheduled_index().is_none()); - assert_eq!(commit_syncer.highest_fetched_commit_index(), 0); - assert_eq!(commit_syncer.synced_commit_index(), 0); - - // Observe round 15 blocks voting for commit 10 from authorities 0 to 2 in - // CommitVoteMonitor - for i in 0..3 { - let test_block = TestBlock::new(15, i) - .set_commit_votes(vec![CommitRef::new(10, CommitDigest::MIN)]) - .build(); - let block = VerifiedBlock::new_for_test(test_block); - commit_vote_monitor.observe_block(&block); - } - - // Fetches should be scheduled after seeing progress of other validators. - commit_syncer.try_schedule_once(); - - // Verify state. - assert_eq!(commit_syncer.pending_fetches().len(), 2); - assert!(commit_syncer.fetched_ranges().is_empty()); - assert_eq!(commit_syncer.highest_scheduled_index(), Some(10)); - assert_eq!(commit_syncer.highest_fetched_commit_index(), 0); - assert_eq!(commit_syncer.synced_commit_index(), 0); - - // Observe round 40 blocks voting for commit 35 from authorities 0 to 2 in - // CommitVoteMonitor - for i in 0..3 { - let test_block = TestBlock::new(40, i) - .set_commit_votes(vec![CommitRef::new(35, CommitDigest::MIN)]) - .build(); - let block = VerifiedBlock::new_for_test(test_block); - commit_vote_monitor.observe_block(&block); - } - - // Fetches should be scheduled until the unhandled commits threshold. - commit_syncer.try_schedule_once(); - - // Verify commit syncer is paused after scheduling 15 commits to index 25. - assert_eq!(commit_syncer.unhandled_commits_threshold(), 25); - assert_eq!(commit_syncer.highest_scheduled_index(), Some(25)); - let pending_fetches = commit_syncer.pending_fetches(); - assert_eq!(pending_fetches.len(), 5); - - // Indicate commit index 25 is consumed, and try to schedule again. - commit_consumer_monitor.set_highest_handled_commit(25); - commit_syncer.try_schedule_once(); - - // Verify commit syncer schedules fetches up to index 35. - assert_eq!(commit_syncer.highest_scheduled_index(), Some(35)); - let pending_fetches = commit_syncer.pending_fetches(); - assert_eq!(pending_fetches.len(), 7); - - // Verify contiguous ranges are scheduled. - for (range, start) in pending_fetches.iter().zip((1..35).step_by(5)) { - assert_eq!(range.start(), start); - assert_eq!(range.end(), start + 4); - } - } -} diff --git a/consensus/core/src/commit_vote_monitor.rs b/consensus/core/src/commit_vote_monitor.rs deleted file mode 100644 index eb041d70406..00000000000 --- a/consensus/core/src/commit_vote_monitor.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use parking_lot::Mutex; - -use crate::{ - CommitIndex, - block::{BlockAPI as _, VerifiedBlock}, - commit::GENESIS_COMMIT_INDEX, - context::Context, -}; - -/// Monitors the progress of consensus commits across the network. -pub(crate) struct CommitVoteMonitor { - context: Arc, - // Highest commit index voted by each authority. - highest_voted_commits: Mutex>, -} - -impl CommitVoteMonitor { - pub(crate) fn new(context: Arc) -> Self { - let highest_voted_commits = Mutex::new(vec![0; context.committee.size()]); - Self { - context, - highest_voted_commits, - } - } - - /// Keeps track of the highest commit voted by each authority. - pub(crate) fn observe_block(&self, block: &VerifiedBlock) { - let mut highest_voted_commits = self.highest_voted_commits.lock(); - for vote in block.commit_votes() { - if vote.index > highest_voted_commits[block.author()] { - highest_voted_commits[block.author()] = vote.index; - } - } - } - - // Finds the highest commit index certified by a quorum. - // When an authority votes for commit index S, it is also voting for all commit - // indices 1 <= i < S. So the quorum commit index is the smallest index S - // such that the sum of stakes of authorities voting for commit indices >= S - // passes the quorum threshold. - pub(crate) fn quorum_commit_index(&self) -> CommitIndex { - let highest_voted_commits = self.highest_voted_commits.lock(); - let mut highest_voted_commits = highest_voted_commits - .iter() - .zip(self.context.committee.authorities()) - .map(|(commit_index, (_, a))| (*commit_index, a.stake)) - .collect::>(); - // Sort by commit index then stake, in descending order. - highest_voted_commits.sort_by(|a, b| a.cmp(b).reverse()); - let mut total_stake = 0; - for (commit_index, stake) in highest_voted_commits { - total_stake += stake; - if total_stake >= self.context.committee.quorum_threshold() { - return commit_index; - } - } - GENESIS_COMMIT_INDEX - } -} - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use super::CommitVoteMonitor; - use crate::{ - block::{TestBlock, VerifiedBlock}, - commit::{CommitDigest, CommitRef}, - context::Context, - }; - - #[tokio::test] - async fn test_commit_vote_monitor() { - let context = Arc::new(Context::new_for_test(4).0); - let monitor = CommitVoteMonitor::new(context); - - // Observe commit votes for indices 5, 6, 7, 8 from blocks. - let blocks = (0..4) - .map(|i| { - VerifiedBlock::new_for_test( - TestBlock::new(10, i) - .set_commit_votes(vec![CommitRef::new(5 + i, CommitDigest::MIN)]) - .build(), - ) - }) - .collect::>(); - for b in blocks { - monitor.observe_block(&b); - } - - // CommitIndex 6 is the highest index supported by a quorum. - assert_eq!(monitor.quorum_commit_index(), 6); - - // Observe new blocks with new votes from authority 0 and 1. - let blocks = (0..2) - .map(|i| { - VerifiedBlock::new_for_test( - TestBlock::new(11, i) - .set_commit_votes(vec![ - CommitRef::new(6 + i, CommitDigest::MIN), - CommitRef::new(7 + i, CommitDigest::MIN), - ]) - .build(), - ) - }) - .collect::>(); - for b in blocks { - monitor.observe_block(&b); - } - - // Highest commit index per authority should be 7, 8, 7, 8 now. - assert_eq!(monitor.quorum_commit_index(), 7); - } -} diff --git a/consensus/core/src/context.rs b/consensus/core/src/context.rs deleted file mode 100644 index c7c87157380..00000000000 --- a/consensus/core/src/context.rs +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::SystemTime}; - -use consensus_config::{AuthorityIndex, Committee, Parameters}; -#[cfg(test)] -use consensus_config::{NetworkKeyPair, ProtocolKeyPair}; -use iota_protocol_config::ProtocolConfig; -#[cfg(test)] -use tempfile::TempDir; -use tokio::time::Instant; - -#[cfg(test)] -use crate::metrics::test_metrics; -use crate::{ - block::BlockTimestampMs, metrics::Metrics, scoring_metrics_store::MysticetiScoringMetricsStore, -}; -/// Context contains per-epoch configuration and metrics shared by all -/// components of this authority. -#[derive(Clone)] -pub(crate) struct Context { - /// Timestamp of the start of the current epoch. - pub epoch_start_timestamp_ms: u64, - /// Index of this authority in the committee. - pub own_index: AuthorityIndex, - /// Committee of the current epoch. - pub committee: Committee, - /// Parameters of this authority. - pub parameters: Parameters, - /// Protocol configuration of current epoch. - pub protocol_config: ProtocolConfig, - /// Metrics of this authority. - pub metrics: Arc, - /// Store for scoring metrics collected by this authority. - pub(crate) scoring_metrics_store: Arc, - /// Access to local clock - pub clock: Arc, -} - -impl Context { - pub(crate) fn new( - epoch_start_timestamp_ms: u64, - own_index: AuthorityIndex, - committee: Committee, - parameters: Parameters, - protocol_config: ProtocolConfig, - metrics: Arc, - scoring_metrics_store: Arc, - - clock: Arc, - ) -> Self { - Self { - epoch_start_timestamp_ms, - own_index, - committee, - parameters, - protocol_config, - metrics, - scoring_metrics_store, - clock, - } - } - - /// Create a test context with a committee of given size and even stake - #[cfg(test)] - pub(crate) fn new_for_test( - committee_size: usize, - ) -> (Self, Vec<(NetworkKeyPair, ProtocolKeyPair)>) { - use iota_common::scoring_metrics::{ScoringMetricsV1, VersionedScoringMetrics}; - - let (committee, keypairs) = - consensus_config::local_committee_and_keys(0, vec![1; committee_size]); - let metrics = test_metrics(); - let temp_dir = TempDir::new().unwrap(); - let clock = Arc::new(Clock::default()); - let current_local_metrics_count = Arc::new(VersionedScoringMetrics::V1( - ScoringMetricsV1::new(committee_size), - )); - let scoring_metrics_store = Arc::new(MysticetiScoringMetricsStore::new( - committee_size, - current_local_metrics_count, - &ProtocolConfig::get_for_max_version_UNSAFE(), - )); - let context = Context::new( - 0, - AuthorityIndex::new_for_test(0), - committee, - Parameters { - db_path: temp_dir.keep(), - ..Default::default() - }, - ProtocolConfig::get_for_max_version_UNSAFE(), - metrics, - scoring_metrics_store, - clock, - ); - (context, keypairs) - } - - #[cfg(test)] - pub(crate) fn with_epoch_start_timestamp_ms(mut self, epoch_start_timestamp_ms: u64) -> Self { - self.epoch_start_timestamp_ms = epoch_start_timestamp_ms; - self - } - - #[cfg(test)] - pub(crate) fn with_authority_index(mut self, authority: AuthorityIndex) -> Self { - self.own_index = authority; - self - } - - #[cfg(test)] - pub(crate) fn with_committee(mut self, committee: Committee) -> Self { - self.committee = committee; - self - } - - #[cfg(test)] - pub(crate) fn with_parameters(mut self, parameters: Parameters) -> Self { - self.parameters = parameters; - self - } -} - -/// A clock that allows to derive the current UNIX system timestamp while -/// guaranteeing that timestamp will be monotonically incremented, tolerating -/// ntp and system clock changes and corrections. Explicitly avoid to make -/// `[Clock]` cloneable to ensure that a single instance is shared behind an -/// `[Arc]` wherever is needed in order to make sure that consecutive calls to -/// receive the system timestamp will remain monotonically increasing. -pub struct Clock { - initial_instant: Instant, - initial_system_time: SystemTime, - // `clock_drift` should be used only for testing - #[cfg(any(test, msim))] - clock_drift: BlockTimestampMs, -} - -impl Default for Clock { - fn default() -> Self { - Self { - initial_instant: Instant::now(), - initial_system_time: SystemTime::now(), - #[cfg(any(test, msim))] - clock_drift: 0, - } - } -} - -impl Clock { - #[cfg(any(test, msim))] - pub fn new_for_test(clock_drift: BlockTimestampMs) -> Self { - Self { - initial_instant: Instant::now(), - initial_system_time: SystemTime::now(), - clock_drift, - } - } - - // Returns the current time expressed as UNIX timestamp in milliseconds. - // Calculated with Tokio Instant to ensure monotonicity, - // and to allow testing with tokio clock. - pub(crate) fn timestamp_utc_ms(&self) -> BlockTimestampMs { - let now: Instant = Instant::now(); - let monotonic_system_time = self - .initial_system_time - .checked_add( - now.checked_duration_since(self.initial_instant) - .unwrap_or_else(|| { - panic!( - "current instant ({:?}) < initial instant ({:?})", - now, self.initial_instant - ) - }), - ) - .expect("Computing system time should not overflow"); - let timestamp = monotonic_system_time - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_else(|_| { - panic!( - "system time ({:?}) < UNIX_EPOCH ({:?})", - monotonic_system_time, - SystemTime::UNIX_EPOCH, - ) - }) - .as_millis() as BlockTimestampMs; - - // Apply clock drift only in test/msim environments to simulate clock skew - // between nodes. In production builds, clock_drift field doesn't exist - // and this returns timestamp directly. - #[cfg(any(test, msim))] - { - timestamp + self.clock_drift - } - #[cfg(not(any(test, msim)))] - { - timestamp - } - } -} diff --git a/consensus/core/src/core.rs b/consensus/core/src/core.rs deleted file mode 100644 index 8a992b63abb..00000000000 --- a/consensus/core/src/core.rs +++ /dev/null @@ -1,4269 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, BTreeSet}, - iter, mem, - sync::Arc, - time::Duration, - vec, -}; - -use consensus_config::{AuthorityIndex, ProtocolKeyPair}; -#[cfg(test)] -use consensus_config::{Stake, local_committee_and_keys}; -use iota_macros::fail_point; -#[cfg(test)] -use iota_metrics::monitored_mpsc::{UnboundedReceiver, unbounded_channel}; -use iota_metrics::monitored_scope; -use itertools::Itertools as _; -use parking_lot::RwLock; -use tokio::{ - sync::{broadcast, watch}, - time::Instant, -}; -use tracing::{debug, info, instrument, trace, warn}; - -#[cfg(test)] -use crate::{ - CommitConsumer, TransactionClient, block_verifier::NoopBlockVerifier, - storage::mem_store::MemStore, -}; -use crate::{ - ancestor::{AncestorState, AncestorStateManager}, - block::{ - Block, BlockAPI, BlockRef, BlockTimestampMs, BlockV1, ExtendedBlock, GENESIS_ROUND, Round, - SignedBlock, Slot, VerifiedBlock, - }, - block_manager::BlockManager, - commit::{ - CertifiedCommit, CertifiedCommits, CommitAPI, CommittedSubDag, DecidedLeader, Decision, - }, - commit_observer::CommitObserver, - context::Context, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - leader_schedule::LeaderSchedule, - round_prober::QuorumRound, - stake_aggregator::{QuorumThreshold, StakeAggregator}, - transaction::TransactionConsumer, - universal_committer::{ - UniversalCommitter, universal_committer_builder::UniversalCommitterBuilder, - }, -}; - -// Maximum number of commit votes to include in a block. -// TODO: Move to protocol config, and verify in BlockVerifier. -const MAX_COMMIT_VOTES_PER_BLOCK: usize = 100; - -pub(crate) struct Core { - context: Arc, - /// The consumer to use in order to pull transactions to be included for the - /// next proposals - transaction_consumer: TransactionConsumer, - /// The block manager which is responsible for keeping track of the DAG - /// dependencies when processing new blocks and accept them or suspend - /// if we are missing their causal history - block_manager: BlockManager, - /// Whether there is a quorum of 2f+1 subscribers waiting for new blocks - /// proposed by this authority. Core stops proposing new blocks when - /// there is not enough subscribers, because new proposed blocks will - /// not be sufficiently propagated to the network. - quorum_subscribers_exists: bool, - /// Estimated delay by round for propagating blocks to a quorum. - /// Because of the nature of TCP and block streaming, propagation delay is - /// expected to be 0 in most cases, even when the actual latency of - /// broadcasting blocks is high. When this value is higher than the - /// `propagation_delay_stop_proposal_threshold`, most likely this - /// validator cannot broadcast blocks to the network at all. Core stops - /// proposing new blocks in this case. - propagation_delay: Round, - - /// Used to make commit decisions for leader blocks in the dag. - committer: UniversalCommitter, - /// The last new round for which core has sent out a signal. - last_signaled_round: Round, - /// The blocks of the last included ancestors per authority. This vector is - /// basically used as a watermark in order to include in the next block - /// proposal only ancestors of higher rounds. By default, is initialised - /// with `None` values. - last_included_ancestors: Vec>, - /// The last decided leader returned from the universal committer. Important - /// to note that this does not signify that the leader has been - /// persisted yet as it still has to go through CommitObserver and - /// persist the commit in store. On recovery/restart - /// the last_decided_leader will be set to the last_commit leader in dag - /// state. - last_decided_leader: Slot, - /// The consensus leader schedule to be used to resolve the leader for a - /// given round. - leader_schedule: Arc, - /// The commit observer is responsible for observing the commits and - /// collecting - /// + sending subdags over the consensus output channel. - commit_observer: CommitObserver, - /// Sender of outgoing signals from Core. - signals: CoreSignals, - /// The keypair to be used for block signing - block_signer: ProtocolKeyPair, - /// Keeping track of state of the DAG, including blocks, commits and last - /// committed rounds. - dag_state: Arc>, - /// The last known round for which the node has proposed. Any proposal - /// should be for a round > of this. This is currently being used to - /// avoid equivocations during a node recovering from amnesia. When value is - /// None it means that the last block sync mechanism is enabled, but it - /// hasn't been initialised yet. - last_known_proposed_round: Option, - // The ancestor state manager will keep track of the quality of the authorities - // based on the distribution of their blocks to the network. It will use this - // information to decide whether to include that authority block in the next - // proposal or not. - ancestor_state_manager: AncestorStateManager, -} - -impl Core { - pub(crate) fn new( - context: Arc, - leader_schedule: Arc, - transaction_consumer: TransactionConsumer, - block_manager: BlockManager, - subscriber_exists: bool, - commit_observer: CommitObserver, - signals: CoreSignals, - block_signer: ProtocolKeyPair, - dag_state: Arc>, - sync_last_known_own_block: bool, - ) -> Self { - let last_decided_leader = dag_state.read().last_commit_leader(); - let committer = UniversalCommitterBuilder::new( - context.clone(), - leader_schedule.clone(), - dag_state.clone(), - ) - // we want to keep with_number_of_leaders param to be able to enable it in a future protocol - // upgrade - .with_number_of_leaders(1) - .with_pipeline(true) - .build(); - - // Recover the last proposed block - let last_proposed_block = dag_state.read().get_last_proposed_block(); - - let last_signaled_round = last_proposed_block.round(); - - // Recover the last included ancestor rounds based on the last proposed block. - // That will allow to perform the next block proposal by using ancestor - // blocks of higher rounds and avoid re-including blocks that have been - // already included in the last (or earlier) block proposal. - // This is only strongly guaranteed for a quorum of ancestors. It is still - // possible to re-include a block from an authority which hadn't been - // added as part of the last proposal hence its latest included ancestor - // is not accurately captured here. This is considered a small deficiency, - // and it mostly matters just for this next proposal without any actual - // penalties in performance or block proposal. - let mut last_included_ancestors = vec![None; context.committee.size()]; - for ancestor in last_proposed_block.ancestors() { - last_included_ancestors[ancestor.author] = Some(*ancestor); - } - - let min_propose_round = if sync_last_known_own_block { - None - } else { - // if the sync is disabled then we practically don't want to impose any - // restriction. - Some(0) - }; - - let propagation_scores = leader_schedule - .leader_swap_table - .read() - .reputation_scores - .clone(); - let mut ancestor_state_manager = AncestorStateManager::new(context.clone()); - ancestor_state_manager.set_propagation_scores(propagation_scores); - - Self { - context, - last_signaled_round, - last_included_ancestors, - last_decided_leader, - leader_schedule, - transaction_consumer, - block_manager, - quorum_subscribers_exists: subscriber_exists, - propagation_delay: 0, - committer, - commit_observer, - signals, - block_signer, - dag_state, - last_known_proposed_round: min_propose_round, - ancestor_state_manager, - } - .recover() - } - - fn recover(mut self) -> Self { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Core::recover"]) - .start_timer(); - // Ensure local time is after max ancestor timestamp. - let ancestor_blocks = self - .dag_state - .read() - .get_last_cached_block_per_authority(Round::MAX); - let max_ancestor_timestamp = ancestor_blocks - .iter() - .fold(0, |ts, (b, _)| ts.max(b.timestamp_ms())); - let wait_ms = max_ancestor_timestamp.saturating_sub(self.context.clock.timestamp_utc_ms()); - - if self - .context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - info!( - "Median based timestamp is enabled. Will not wait for {} ms while recovering ancestors from storage", - wait_ms - ); - } else if wait_ms > 0 { - warn!( - "Waiting for {} ms while recovering ancestors from storage", - wait_ms - ); - std::thread::sleep(Duration::from_millis(wait_ms)); - } - - // Try to commit and propose, since they may not have run after the last storage - // write. - self.try_commit(vec![]).unwrap(); - let last_proposed_block = if let Some(last_proposed_block) = self.try_propose(true).unwrap() - { - last_proposed_block - } else { - let last_proposed_block = self.dag_state.read().get_last_proposed_block(); - if self.should_propose() { - assert!( - last_proposed_block.round() > GENESIS_ROUND, - "At minimum a block of round higher than genesis should have been produced during recovery" - ); - } - - // if no new block proposed then just re-broadcast the last proposed one to - // ensure liveness. - self.signals - .new_block(ExtendedBlock { - block: last_proposed_block.clone(), - excluded_ancestors: vec![], - }) - .unwrap(); - last_proposed_block - }; - - // Try to set up leader timeout if needed. - // This needs to be called after try_commit() and try_propose(), which may - // have advanced the threshold clock round. - self.try_signal_new_round(); - - info!( - "Core recovery completed with last proposed block {:?}", - last_proposed_block - ); - - self - } - - /// Processes the provided blocks and accepts them if possible when their - /// causal history exists. The method returns: - /// - The references of ancestors missing their block - #[tracing::instrument("consensus_add_blocks", skip_all)] - pub(crate) fn add_blocks( - &mut self, - blocks: Vec, - ) -> ConsensusResult> { - let _scope = monitored_scope("Core::add_blocks"); - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Core::add_blocks"]) - .start_timer(); - self.context - .metrics - .node_metrics - .core_add_blocks_batch_size - .observe(blocks.len() as f64); - - let (accepted_blocks, missing_block_refs) = self.block_manager.try_accept_blocks(blocks); - - if !accepted_blocks.is_empty() { - debug!( - "Accepted blocks: {}", - accepted_blocks - .iter() - .map(|b| b.reference().to_string()) - .join(",") - ); - - // Try to commit the new blocks if possible. - self.try_commit(vec![])?; - - // Try to propose now since there are new blocks accepted. - self.try_propose(false)?; - - // Now set up leader timeout if needed. - // This needs to be called after try_commit() and try_propose(), which may - // have advanced the threshold clock round. - self.try_signal_new_round(); - } - - if !missing_block_refs.is_empty() { - trace!( - "Missing block refs: {}", - missing_block_refs.iter().map(|b| b.to_string()).join(", ") - ); - } - Ok(missing_block_refs) - } - - /// Checks if provided block refs have been accepted. If not, missing block - /// refs are kept for synchronizations. Returns the references of - /// missing blocks among the input blocks. - pub(crate) fn check_block_refs( - &mut self, - block_refs: Vec, - ) -> ConsensusResult> { - let _scope = monitored_scope("Core::check_block_refs"); - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Core::check_block_refs"]) - .start_timer(); - self.context - .metrics - .node_metrics - .core_check_block_refs_batch_size - .observe(block_refs.len() as f64); - - // Try to find them via the block manager - let missing_block_refs = self.block_manager.try_find_blocks(block_refs); - - if !missing_block_refs.is_empty() { - trace!( - "Missing block refs: {}", - missing_block_refs.iter().map(|b| b.to_string()).join(", ") - ); - } - Ok(missing_block_refs) - } - - // Adds the certified commits that have been synced via the commit syncer. We - // are using the commit info in order to skip running the decision - // rule and immediately commit the corresponding leaders and sub dags. Pay - // attention that no block acceptance is happening here, but rather - // internally in the `try_commit` method which ensures that everytime only the - // blocks corresponding to the certified commits that are about to - // be committed are accepted. - #[tracing::instrument(skip_all)] - pub(crate) fn add_certified_commits( - &mut self, - certified_commits: CertifiedCommits, - ) -> ConsensusResult> { - let _scope = monitored_scope("Core::add_certified_commits"); - - // We want to enable the commit process logic when GC is enabled. - if self.dag_state.read().gc_enabled() { - let votes = certified_commits.votes().to_vec(); - let commits = self - .validate_certified_commits(certified_commits.commits().to_vec()) - .expect("Certified commits validation failed"); - - // Accept the certified commit votes. This is optimistically done to increase - // the chances of having votes available when this node will need to - // sync commits to other nodes. - self.block_manager.try_accept_blocks(votes); - - // Try to commit the new blocks. Take into account the trusted commit that has - // been provided. - self.try_commit(commits)?; - - // Try to propose now since there are new blocks accepted. - self.try_propose(false)?; - - // Now set up leader timeout if needed. - // This needs to be called after try_commit() and try_propose(), which may - // have advanced the threshold clock round. - self.try_signal_new_round(); - - return Ok(BTreeSet::new()); - } - - // If GC is not enabled then process blocks as usual. - let blocks = certified_commits - .commits() - .iter() - .flat_map(|commit| commit.blocks()) - .cloned() - .collect::>(); - - self.add_blocks(blocks) - } - - /// If needed, signals a new clock round and sets up leader timeout. - fn try_signal_new_round(&mut self) { - // Signal only when the threshold clock round is more advanced than the last - // signaled round. - // - // NOTE: a signal is still sent even when a block has been proposed at the new - // round. We can consider changing this in the future. - let new_clock_round = self.dag_state.read().threshold_clock_round(); - if new_clock_round <= self.last_signaled_round { - return; - } - // Then send a signal to set up leader timeout. - tracing::trace!(round = ?new_clock_round, "new_consensus_round_sent"); - self.signals.new_round(new_clock_round); - self.last_signaled_round = new_clock_round; - - // Report the threshold clock round - self.context - .metrics - .node_metrics - .threshold_clock_round - .set(new_clock_round as i64); - } - - /// Creating a new block for the dictated round. This is used when a leader - /// timeout occurs, either when the min timeout expires or max. When - /// `force = true` , then any checks like previous round - /// leader existence will get skipped. - pub(crate) fn new_block( - &mut self, - round: Round, - force: bool, - ) -> ConsensusResult> { - let _scope = monitored_scope("Core::new_block"); - if self.last_proposed_round() < round { - self.context - .metrics - .node_metrics - .leader_timeout_total - .with_label_values(&[&format!("{force}")]) - .inc(); - let result = self.try_propose(force); - // The threshold clock round may have advanced, so a signal needs to be sent. - self.try_signal_new_round(); - return result; - } - Ok(None) - } - - /// Keeps only the certified commits that have a commit index > last commit - /// index. It also ensures that the first commit in the list is the next one - /// in line, otherwise it panics. - fn validate_certified_commits( - &mut self, - commits: Vec, - ) -> ConsensusResult> { - // Filter out the commits that have been already locally committed and keep only - // anything that is above the last committed index. - let last_commit_index = self.dag_state.read().last_commit_index(); - let commits = commits - .iter() - .filter(|commit| { - if commit.index() > last_commit_index { - true - } else { - tracing::debug!( - "Skip commit for index {} as it is already committed with last commit index {}", - commit.index(), - last_commit_index - ); - false - } - }) - .cloned() - .collect::>(); - - // Make sure that the first commit we find is the next one in line and there is - // no gap. - if let Some(commit) = commits.first() { - if commit.index() != last_commit_index + 1 { - return Err(ConsensusError::UnexpectedCertifiedCommitIndex { - expected_commit_index: last_commit_index + 1, - commit_index: commit.index(), - }); - } - } - - Ok(commits) - } - - // Attempts to create a new block, persist and propose it to all peers. - // When force is true, ignore if leader from the last round exists among - // ancestors and if the minimum round delay has passed. - #[instrument(level = "trace", skip_all)] - fn try_propose(&mut self, force: bool) -> ConsensusResult> { - if !self.should_propose() { - return Ok(None); - } - if let Some(extended_block) = self.try_new_block(force) { - self.signals.new_block(extended_block.clone())?; - - fail_point!("consensus-after-propose"); - - // The new block may help commit. - self.try_commit(vec![])?; - return Ok(Some(extended_block.block)); - } - Ok(None) - } - - /// Attempts to propose a new block for the next round. If a block has - /// already proposed for latest or earlier round, then no block is - /// created and None is returned. - #[instrument(level = "trace", skip_all)] - fn try_new_block(&mut self, force: bool) -> Option { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Core::try_new_block"]) - .start_timer(); - - // Ensure the new block has a higher round than the last proposed block. - let clock_round = { - let dag_state = self.dag_state.read(); - let clock_round = dag_state.threshold_clock_round(); - if clock_round <= dag_state.get_last_proposed_block().round() { - return None; - } - clock_round - }; - - // There must be a quorum of blocks from the previous round. - let quorum_round = clock_round.saturating_sub(1); - - // Create a new block either because we want to "forcefully" propose a block due - // to a leader timeout, or because we are actually ready to produce the - // block (leader exists and min delay has passed). - if !force { - if !self.leaders_exist(quorum_round) { - return None; - } - - if Duration::from_millis( - self.context - .clock - .timestamp_utc_ms() - .saturating_sub(self.last_proposed_timestamp_ms()), - ) < self.context.parameters.min_round_delay - { - return None; - } - } - - // Determine the ancestors to be included in proposal. - // Smart ancestor selection requires distributed scoring to be enabled. - let (ancestors, excluded_ancestors) = if self - .context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - && self - .context - .protocol_config - .consensus_smart_ancestor_selection() - { - let (ancestors, excluded_and_equivocating_ancestors) = - self.smart_ancestors_to_propose(clock_round, !force); - - // If we did not find enough good ancestors to propose, continue to wait before - // proposing. - if ancestors.is_empty() { - assert!( - !force, - "Ancestors should have been returned if force is true!" - ); - return None; - } - - let excluded_ancestors_limit = self.context.committee.size() * 2; - if excluded_and_equivocating_ancestors.len() > excluded_ancestors_limit { - debug!( - "Dropping {} excluded ancestor(s) during proposal due to size limit", - excluded_and_equivocating_ancestors.len() - excluded_ancestors_limit, - ); - } - let excluded_ancestors = excluded_and_equivocating_ancestors - .into_iter() - .take(excluded_ancestors_limit) - .collect(); - - (ancestors, excluded_ancestors) - } else { - (self.ancestors_to_propose(clock_round), vec![]) - }; - - // Update the last included ancestor block refs - for ancestor in &ancestors { - self.last_included_ancestors[ancestor.author()] = Some(ancestor.reference()); - } - - let leader_authority = &self - .context - .committee - .authority(self.first_leader(quorum_round)) - .hostname; - self.context - .metrics - .node_metrics - .block_proposal_leader_wait_ms - .with_label_values(&[leader_authority]) - .inc_by( - Instant::now() - .saturating_duration_since(self.dag_state.read().threshold_clock_quorum_ts()) - .as_millis() as u64, - ); - self.context - .metrics - .node_metrics - .block_proposal_leader_wait_count - .with_label_values(&[leader_authority]) - .inc(); - - self.context - .metrics - .node_metrics - .proposed_block_ancestors - .observe(ancestors.len() as f64); - for ancestor in &ancestors { - let authority = &self.context.committee.authority(ancestor.author()).hostname; - self.context - .metrics - .node_metrics - .proposed_block_ancestors_depth - .with_label_values(&[authority]) - .observe(clock_round.saturating_sub(ancestor.round()).into()); - } - - let now = self.context.clock.timestamp_utc_ms(); - ancestors.iter().for_each(|block| { - if self.context.protocol_config.consensus_median_timestamp_with_checkpoint_enforcement() { - if block.timestamp_ms() > now { - trace!("Ancestor block {:?} has timestamp {}, greater than current timestamp {now}. Proposing for round {}.", block, block.timestamp_ms(), clock_round); - let authority = &self.context.committee.authority(block.author()).hostname; - self.context - .metrics - .node_metrics - .proposed_block_ancestors_timestamp_drift_ms - .with_label_values(&[authority]) - .inc_by(block.timestamp_ms().saturating_sub(now)); - } - } else { - // Ensure ancestor timestamps are not more advanced than the current time. - // Also catch the issue if system's clock go backwards. - assert!( - block.timestamp_ms() <= now, - "Violation: ancestor block {:?} has timestamp {}, greater than current timestamp {now}. Proposing for round {}.", - block, block.timestamp_ms(), clock_round - ); - } - }); - - // Consume the next transactions to be included. Do not drop the guards yet as - // this would acknowledge the inclusion of transactions. Just let this - // be done in the end of the method. - let (transactions, ack_transactions, _limit_reached) = self.transaction_consumer.next(); - self.context - .metrics - .node_metrics - .proposed_block_transactions - .observe(transactions.len() as f64); - - // Consume the commit votes to be included. - let commit_votes = self - .dag_state - .write() - .take_commit_votes(MAX_COMMIT_VOTES_PER_BLOCK); - - // Create the block and insert to storage. - let block = Block::V1(BlockV1::new( - self.context.committee.epoch(), - clock_round, - self.context.own_index, - now, - ancestors.iter().map(|b| b.reference()).collect(), - transactions, - commit_votes, - vec![], - )); - let signed_block = - SignedBlock::new(block, &self.block_signer).expect("Block signing failed."); - let serialized = signed_block - .serialize() - .expect("Block serialization failed."); - self.context - .metrics - .node_metrics - .proposed_block_size - .observe(serialized.len() as f64); - // Own blocks are assumed to be valid. - let verified_block = VerifiedBlock::new_verified(signed_block, serialized); - - // Record the interval from last proposal, before accepting the proposed block. - let last_proposed_block = self.last_proposed_block(); - if last_proposed_block.round() > 0 { - self.context - .metrics - .node_metrics - .block_proposal_interval - .observe( - Duration::from_millis( - verified_block - .timestamp_ms() - .saturating_sub(last_proposed_block.timestamp_ms()), - ) - .as_secs_f64(), - ); - } - - // Accept the block into BlockManager and DagState. - let (accepted_blocks, missing) = self - .block_manager - .try_accept_blocks(vec![verified_block.clone()]); - assert_eq!(accepted_blocks.len(), 1); - assert!(missing.is_empty()); - - // Ensure the new block and its ancestors are persisted, before broadcasting it. - self.dag_state.write().flush(); - - // Now acknowledge the transactions for their inclusion to block - ack_transactions(verified_block.reference()); - - info!("Created block {verified_block:?} for round {clock_round}"); - - self.context - .metrics - .node_metrics - .proposed_blocks - .with_label_values(&[&force.to_string()]) - .inc(); - - Some(ExtendedBlock { - block: verified_block, - excluded_ancestors, - }) - } - - /// Runs commit rule to attempt to commit additional blocks from the DAG. If - /// any `certified_commits` are provided, then it will attempt to commit - /// those first before trying to commit any further leaders. - #[instrument(level = "trace", skip_all)] - fn try_commit( - &mut self, - mut certified_commits: Vec, - ) -> ConsensusResult> { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Core::try_commit"]) - .start_timer(); - - let mut certified_commits_map = BTreeMap::new(); - for c in &certified_commits { - certified_commits_map.insert(c.index(), c.reference()); - } - - if !certified_commits.is_empty() { - info!( - "Will try to commit synced commits first : {:?}", - certified_commits - .iter() - .map(|c| (c.index(), c.leader())) - .collect::>() - ); - } - - let mut committed_sub_dags = Vec::new(); - // TODO: Add optimization to abort early without quorum for a round. - loop { - // LeaderSchedule has a limit to how many sequenced leaders can be committed - // before a change is triggered. Calling into leader schedule will get you - // how many commits till next leader change. We will loop back and recalculate - // any discarded leaders with the new schedule. - let mut commits_until_update = self - .leader_schedule - .commits_until_leader_schedule_update(self.dag_state.clone()); - - if commits_until_update == 0 { - let last_commit_index = self.dag_state.read().last_commit_index(); - - tracing::info!( - "Leader schedule change triggered at commit index {last_commit_index}" - ); - if self - .context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - self.leader_schedule - .update_leader_schedule_v2(&self.dag_state); - - let propagation_scores = self - .leader_schedule - .leader_swap_table - .read() - .reputation_scores - .clone(); - self.ancestor_state_manager - .set_propagation_scores(propagation_scores); - } else { - self.leader_schedule - .update_leader_schedule_v1(&self.dag_state); - } - commits_until_update = self - .leader_schedule - .commits_until_leader_schedule_update(self.dag_state.clone()); - - fail_point!("consensus-after-leader-schedule-change"); - } - assert!(commits_until_update > 0); - - // Always try to process the synced commits first. If there are certified - // commits to process then the decided leaders and the commits will be returned. - let (mut decided_leaders, decided_certified_commits): ( - Vec, - Vec, - ) = self - .try_decide_certified(&mut certified_commits, commits_until_update) - .into_iter() - .unzip(); - - // Only accept blocks for the certified commits that we are certain to sequence. - // This ensures that only blocks corresponding to committed certified commits - // are flushed to disk. Blocks from non-committed certified commits - // will not be flushed, preventing issues during crash-recovery. - // This avoids scenarios where accepting and flushing blocks of non-committed - // certified commits could lead to premature commit rule execution. - // Due to GC, this could cause a panic if the commit rule tries to access - // missing causal history from blocks of certified commits. - let blocks = decided_certified_commits - .iter() - .flat_map(|c| c.blocks()) - .cloned() - .collect::>(); - self.block_manager.try_accept_committed_blocks(blocks); - - // If the certified `decided_leaders` is empty then try to run the decision - // rule. - if decided_leaders.is_empty() { - // TODO: limit commits by commits_until_update, which may be needed when leader - // schedule length is reduced. - decided_leaders = self.committer.try_decide(self.last_decided_leader); - - // Truncate the decided leaders to fit the commit schedule limit. - if decided_leaders.len() >= commits_until_update { - let _ = decided_leaders.split_off(commits_until_update); - } - } - - // If the decided leaders list is empty then just break the loop. - let Some(last_decided) = decided_leaders.last().cloned() else { - break; - }; - - self.last_decided_leader = last_decided.slot(); - - let sequenced_leaders = decided_leaders - .into_iter() - .filter_map(|leader| leader.into_committed_block()) - .collect::>(); - - tracing::debug!( - "Decided {} leaders and {commits_until_update} commits can be made before next leader schedule change", - sequenced_leaders.len() - ); - - self.context - .metrics - .node_metrics - .last_decided_leader_round - .set(self.last_decided_leader.round as i64); - - // It's possible to reach this point as the decided leaders might all of them be - // "Skip" decisions. In this case there is no leader to commit and - // we should break the loop. - if sequenced_leaders.is_empty() { - break; - } - - tracing::info!( - "Committing {} leaders: {}", - sequenced_leaders.len(), - sequenced_leaders - .iter() - .map(|b| b.reference().to_string()) - .join(",") - ); - - // TODO: refcount subdags - let subdags = self.commit_observer.handle_commit(sequenced_leaders)?; - if self - .context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - self.dag_state.write().add_scoring_subdags(subdags.clone()); - } else { - // TODO: Remove when DistributedVoteScoring is enabled. - self.dag_state - .write() - .add_unscored_committed_subdags(subdags.clone()); - } - - // Try to unsuspend blocks if gc_round has advanced. - self.block_manager - .try_unsuspend_blocks_for_latest_gc_round(); - committed_sub_dags.extend(subdags); - - fail_point!("consensus-after-handle-commit"); - } - - // Sanity check: for commits that have been linearized using the certified - // commits, ensure that the same sub dag has been committed. - for sub_dag in &committed_sub_dags { - if let Some(commit_ref) = certified_commits_map.remove(&sub_dag.commit_ref.index) { - assert_eq!( - commit_ref, sub_dag.commit_ref, - "Certified commit has different reference than the committed sub dag" - ); - } - } - - // Notify about our own committed blocks - let committed_block_refs = committed_sub_dags - .iter() - .flat_map(|sub_dag| sub_dag.blocks.iter()) - .filter_map(|block| { - (block.author() == self.context.own_index).then_some(block.reference()) - }) - .collect::>(); - self.transaction_consumer - .notify_own_blocks_status(committed_block_refs, self.dag_state.read().gc_round()); - - Ok(committed_sub_dags) - } - - pub(crate) fn get_missing_blocks(&self) -> BTreeMap> { - let _scope = monitored_scope("Core::get_missing_blocks"); - self.block_manager.missing_blocks() - } - - /// Sets if there is 2f+1 subscriptions to the block stream. - pub(crate) fn set_quorum_subscribers_exists(&mut self, exists: bool) { - info!("A quorum of block subscribers exists: {exists}"); - self.quorum_subscribers_exists = exists; - } - - /// Sets the delay by round for propagating blocks to a quorum and the - /// received & accepted quorum rounds per authority for ancestor state - /// manager. - pub(crate) fn set_propagation_delay_and_quorum_rounds( - &mut self, - delay: Round, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, - ) { - info!( - "Received quorum round per authority in ancestor state manager set to: {}", - self.context - .committee - .authorities() - .zip(received_quorum_rounds.iter()) - .map(|((i, _), rounds)| format!("{i}: {rounds:?}")) - .join(", ") - ); - info!( - "Accepted quorum round per authority in ancestor state manager set to: {}", - self.context - .committee - .authorities() - .zip(accepted_quorum_rounds.iter()) - .map(|((i, _), rounds)| format!("{i}: {rounds:?}")) - .join(", ") - ); - self.ancestor_state_manager - .set_quorum_rounds_per_authority(received_quorum_rounds, accepted_quorum_rounds); - info!("Propagation round delay set to: {delay}"); - self.propagation_delay = delay; - } - - /// Sets the min propose round for the proposer allowing to propose blocks - /// only for round numbers `> last_known_proposed_round`. At the moment - /// is allowed to call the method only once leading to a panic - /// if attempt to do multiple times. - pub(crate) fn set_last_known_proposed_round(&mut self, round: Round) { - if self.last_known_proposed_round.is_some() { - panic!( - "Should not attempt to set the last known proposed round if that has been already set" - ); - } - self.last_known_proposed_round = Some(round); - info!("Last known proposed round set to {round}"); - } - - /// Whether the core should propose new blocks. - pub(crate) fn should_propose(&self) -> bool { - let clock_round = self.dag_state.read().threshold_clock_round(); - let core_skipped_proposals = &self.context.metrics.node_metrics.core_skipped_proposals; - - if !self.quorum_subscribers_exists { - debug!("Skip proposing for round {clock_round}, don't have a quorum of subscribers."); - core_skipped_proposals - .with_label_values(&["no_quorum_subscriber"]) - .inc(); - return false; - } - - if self.propagation_delay - > self - .context - .parameters - .propagation_delay_stop_proposal_threshold - { - debug!( - "Skip proposing for round {clock_round}, high propagation delay {} > {}.", - self.propagation_delay, - self.context - .parameters - .propagation_delay_stop_proposal_threshold - ); - core_skipped_proposals - .with_label_values(&["high_propagation_delay"]) - .inc(); - return false; - } - - let Some(last_known_proposed_round) = self.last_known_proposed_round else { - debug!( - "Skip proposing for round {clock_round}, last known proposed round has not been synced yet." - ); - core_skipped_proposals - .with_label_values(&["no_last_known_proposed_round"]) - .inc(); - return false; - }; - if clock_round <= last_known_proposed_round { - debug!( - "Skip proposing for round {clock_round} as last known proposed round is {last_known_proposed_round}" - ); - core_skipped_proposals - .with_label_values(&["higher_last_known_proposed_round"]) - .inc(); - return false; - } - - true - } - - // Try to decide which of the certified commits will have to be committed next - // respecting the `limit`. If provided `limit` is zero, it will panic. - // The function returns the list of decided leaders and updates in place the - // remaining certified commits. If empty vector is returned, it means that - // there are no certified commits to be committed as `certified_commits` is - // either empty or all of the certified commits are already committed. - #[tracing::instrument(skip_all)] - fn try_decide_certified( - &mut self, - certified_commits: &mut Vec, - limit: usize, - ) -> Vec<(DecidedLeader, CertifiedCommit)> { - // If GC is disabled then should not run any of this logic. - if !self.dag_state.read().gc_enabled() { - return Vec::new(); - } - - assert!(limit > 0, "limit should be greater than 0"); - - let to_commit = if certified_commits.len() >= limit { - // We keep only the number of leaders as dictated by the `limit` - certified_commits.drain(..limit).collect::>() - } else { - // Otherwise just take all of them and leave the `synced_commits` empty. - mem::take(certified_commits) - }; - - tracing::debug!( - "Decided {} certified leaders: {}", - to_commit.len(), - to_commit.iter().map(|c| c.leader().to_string()).join(",") - ); - - let sequenced_leaders = to_commit - .into_iter() - .map(|commit| { - let leader = commit.blocks().last().expect("Certified commit should have at least one block"); - assert_eq!(leader.reference(), commit.leader(), "Last block of the committed sub dag should have the same digest as the leader of the commit"); - let leader = DecidedLeader::Commit(leader.clone()); - UniversalCommitter::update_metrics(&self.context, &leader, Decision::Certified); - (leader, commit) - }) - .collect::>(); - - sequenced_leaders - } - - /// Retrieves the next ancestors to propose to form a block at `clock_round` - /// round. - fn ancestors_to_propose(&mut self, clock_round: Round) -> Vec { - // Now take the ancestors before the clock_round (excluded) for each authority. - let (ancestors, gc_enabled, gc_round) = { - let dag_state = self.dag_state.read(); - ( - dag_state.get_last_cached_block_per_authority(clock_round), - dag_state.gc_enabled(), - dag_state.gc_round(), - ) - }; - - assert_eq!( - ancestors.len(), - self.context.committee.size(), - "Fatal error, number of returned ancestors don't match committee size." - ); - - // Propose only ancestors of higher rounds than what has already been proposed. - // And always include own last proposed block first among ancestors. - let (last_proposed_block, _) = ancestors[self.context.own_index].clone(); - assert_eq!(last_proposed_block.author(), self.context.own_index); - let ancestors = iter::once(last_proposed_block) - .chain( - ancestors - .into_iter() - .filter(|(block, _)| block.author() != self.context.own_index) - .filter(|(block, _)| { - if gc_enabled && gc_round > GENESIS_ROUND { - return block.round() > gc_round; - } - true - }) - .flat_map(|(block, _)| { - if let Some(last_block_ref) = self.last_included_ancestors[block.author()] { - return (last_block_ref.round < block.round()).then_some(block); - } - Some(block) - }), - ) - .collect::>(); - - // TODO: this is for temporary sanity check - we might want to remove later on - let mut quorum = StakeAggregator::::new(); - for ancestor in ancestors - .iter() - .filter(|block| block.round() == clock_round - 1) - { - quorum.add(ancestor.author(), &self.context.committee); - } - assert!( - quorum.reached_threshold(&self.context.committee), - "Fatal error, quorum not reached for parent round when proposing for round {clock_round}. Possible mismatch between DagState and Core." - ); - - ancestors - } - - /// Retrieves the next ancestors to propose to form a block at `clock_round` - /// round. If smart selection is enabled then this will try to select - /// the best ancestors based on the propagation scores of the - /// authorities. - fn smart_ancestors_to_propose( - &mut self, - clock_round: Round, - smart_select: bool, - ) -> (Vec, BTreeSet) { - let node_metrics = &self.context.metrics.node_metrics; - let _s = node_metrics - .scope_processing_time - .with_label_values(&["Core::smart_ancestors_to_propose"]) - .start_timer(); - - // Now take the ancestors before the clock_round (excluded) for each authority. - let all_ancestors = self - .dag_state - .read() - .get_last_cached_block_per_authority(clock_round); - - assert_eq!( - all_ancestors.len(), - self.context.committee.size(), - "Fatal error, number of returned ancestors don't match committee size." - ); - - // Ensure ancestor state is up to date before selecting for proposal. - self.ancestor_state_manager.update_all_ancestors_state(); - let ancestor_state_map = self.ancestor_state_manager.get_ancestor_states(); - - let quorum_round = clock_round.saturating_sub(1); - - let mut score_and_pending_excluded_ancestors = Vec::new(); - let mut excluded_and_equivocating_ancestors = BTreeSet::new(); - - // Propose only ancestors of higher rounds than what has already been proposed. - // And always include own last proposed block first among ancestors. - // Start by only including the high scoring ancestors. Low scoring ancestors - // will be included in a second pass below. - let included_ancestors = iter::once(self.last_proposed_block()) - .chain( - all_ancestors - .into_iter() - .flat_map(|(ancestor, equivocating_ancestors)| { - if ancestor.author() == self.context.own_index { - return None; - } - if let Some(last_block_ref) = - self.last_included_ancestors[ancestor.author()] - { - if last_block_ref.round >= ancestor.round() { - return None; - } - } - - // We will never include equivocating ancestors so add them immediately - excluded_and_equivocating_ancestors.extend(equivocating_ancestors); - - let ancestor_state = ancestor_state_map[ancestor.author()]; - match ancestor_state { - AncestorState::Include => { - trace!("Found ancestor {ancestor} with INCLUDE state for round {clock_round}"); - } - AncestorState::Exclude(score) => { - trace!("Added ancestor {ancestor} with EXCLUDE state with score {score} to temporary excluded ancestors for round {clock_round}"); - score_and_pending_excluded_ancestors.push((score, ancestor)); - return None; - } - } - - Some(ancestor) - }), - ) - .collect::>(); - - let mut parent_round_quorum = StakeAggregator::::new(); - - // Check total stake of high scoring parent round ancestors - for ancestor in included_ancestors - .iter() - .filter(|a| a.round() == quorum_round) - { - parent_round_quorum.add(ancestor.author(), &self.context.committee); - } - - if smart_select && !parent_round_quorum.reached_threshold(&self.context.committee) { - node_metrics.smart_selection_wait.inc(); - debug!( - "Only found {} stake of good ancestors to include for round {clock_round}, will wait for more.", - parent_round_quorum.stake() - ); - return (vec![], BTreeSet::new()); - } - - // Sort scores descending so we can include the best of the pending excluded - // ancestors first until we reach the threshold. - score_and_pending_excluded_ancestors.sort_by(|a, b| b.0.cmp(&a.0)); - - let mut ancestors_to_propose = included_ancestors; - let mut excluded_ancestors = Vec::new(); - for (score, ancestor) in score_and_pending_excluded_ancestors.into_iter() { - let block_hostname = &self.context.committee.authority(ancestor.author()).hostname; - if !parent_round_quorum.reached_threshold(&self.context.committee) - && ancestor.round() == quorum_round - { - debug!( - "Including temporarily excluded parent round ancestor {ancestor} with score {score} to propose for round {clock_round}" - ); - parent_round_quorum.add(ancestor.author(), &self.context.committee); - ancestors_to_propose.push(ancestor); - node_metrics - .included_excluded_proposal_ancestors_count_by_authority - .with_label_values(&[block_hostname.as_str(), "timeout"]) - .inc(); - } else { - excluded_ancestors.push((score, ancestor)); - } - } - - // Iterate through excluded ancestors and include the ancestor or the ancestor's - // ancestor that has been accepted by a quorum of the network. If the - // original ancestor itself is not included then it will be part of - // excluded ancestors that are not included in the block but will still - // be broadcasted to peers. - for (score, ancestor) in excluded_ancestors.iter() { - let excluded_author = ancestor.author(); - let block_hostname = &self.context.committee.authority(excluded_author).hostname; - // A quorum of validators reported to have accepted blocks from the - // excluded_author up to the low quorum round. - let mut accepted_low_quorum_round = self - .ancestor_state_manager - .accepted_quorum_round_per_authority[excluded_author] - .0; - // If the accepted quorum round of this ancestor is greater than or equal - // to the clock round then we want to make sure to set it to clock_round - 1 - // as that is the max round the new block can include as an ancestor. - accepted_low_quorum_round = accepted_low_quorum_round.min(quorum_round); - - let last_included_round = self.last_included_ancestors[excluded_author] - .map(|block_ref| block_ref.round) - .unwrap_or(GENESIS_ROUND); - if ancestor.round() <= last_included_round { - // This should have already been filtered out when filtering all_ancestors. - // Still, ensure previously included ancestors are filtered out. - continue; - } - - if last_included_round >= accepted_low_quorum_round { - excluded_and_equivocating_ancestors.insert(ancestor.reference()); - trace!( - "Excluded low score ancestor {} with score {score} to propose for round {clock_round}: last included round {last_included_round} >= accepted low quorum round {accepted_low_quorum_round}", - ancestor.reference() - ); - node_metrics - .excluded_proposal_ancestors_count_by_authority - .with_label_values(&[block_hostname]) - .inc(); - continue; - } - - let ancestor = if ancestor.round() <= accepted_low_quorum_round { - // Include the ancestor block as it has been seen & accepted by a strong quorum. - ancestor.clone() - } else { - // Exclude this ancestor since it hasn't been accepted by a strong quorum - excluded_and_equivocating_ancestors.insert(ancestor.reference()); - trace!( - "Excluded low score ancestor {} with score {score} to propose for round {clock_round}: ancestor round {} > accepted low quorum round {accepted_low_quorum_round} ", - ancestor.reference(), - ancestor.round() - ); - node_metrics - .excluded_proposal_ancestors_count_by_authority - .with_label_values(&[block_hostname]) - .inc(); - - // Look for an earlier block in the ancestor chain that we can include as there - // is a gap between the last included round and the accepted low quorum round. - // - // Note: Only cached blocks need to be propagated. Committed and GC'ed blocks - // do not need to be propagated. - match self.dag_state.read().get_last_cached_block_in_range( - excluded_author, - last_included_round + 1, - accepted_low_quorum_round + 1, - ) { - Some(earlier_ancestor) => { - // Found an earlier block that has been propagated well - include it instead - earlier_ancestor - } - None => { - // No suitable earlier block found - continue; - } - } - }; - self.last_included_ancestors[excluded_author] = Some(ancestor.reference()); - ancestors_to_propose.push(ancestor.clone()); - trace!( - "Included low scoring ancestor {} with score {score} seen at accepted low quorum round {accepted_low_quorum_round} to propose for round {clock_round}", - ancestor.reference() - ); - node_metrics - .included_excluded_proposal_ancestors_count_by_authority - .with_label_values(&[block_hostname.as_str(), "quorum"]) - .inc(); - } - - assert!( - parent_round_quorum.reached_threshold(&self.context.committee), - "Fatal error, quorum not reached for parent round when proposing for round {clock_round}. Possible mismatch between DagState and Core." - ); - - info!( - "Included {} ancestors & excluded {} low performing or equivocating ancestors for proposal in round {clock_round}", - ancestors_to_propose.len(), - excluded_and_equivocating_ancestors.len() - ); - - (ancestors_to_propose, excluded_and_equivocating_ancestors) - } - - /// Checks whether all the leaders of the round exist. - /// TODO: we can leverage some additional signal here in order to more - /// cleverly manipulate later the leader timeout Ex if we already have - /// one leader - the first in order - we might don't want to wait as much. - fn leaders_exist(&self, round: Round) -> bool { - let dag_state = self.dag_state.read(); - for leader in self.leaders(round) { - // Search for all the leaders. If at least one is not found, then return false. - // A linear search should be fine here as the set of elements is not expected to - // be small enough and more sophisticated data structures might not - // give us much here. - if !dag_state.contains_cached_block_at_slot(leader) { - return false; - } - } - - true - } - - /// Returns the leaders of the provided round. - fn leaders(&self, round: Round) -> Vec { - self.committer - .get_leaders(round) - .into_iter() - .map(|authority_index| Slot::new(round, authority_index)) - .collect() - } - - /// Returns the 1st leader of the round. - fn first_leader(&self, round: Round) -> AuthorityIndex { - self.leaders(round).first().unwrap().authority - } - - fn last_proposed_timestamp_ms(&self) -> BlockTimestampMs { - self.last_proposed_block().timestamp_ms() - } - - fn last_proposed_round(&self) -> Round { - self.last_proposed_block().round() - } - - fn last_proposed_block(&self) -> VerifiedBlock { - self.dag_state.read().get_last_proposed_block() - } -} - -/// Senders of signals from Core, for outputs and events (ex new block -/// produced). -pub(crate) struct CoreSignals { - tx_block_broadcast: broadcast::Sender, - new_round_sender: watch::Sender, - context: Arc, -} - -impl CoreSignals { - pub fn new(context: Arc) -> (Self, CoreSignalsReceivers) { - // Blocks buffered in broadcast channel should be roughly equal to thosed cached - // in dag state, since the underlying blocks are ref counted so a lower - // buffer here will not reduce memory usage significantly. - let (tx_block_broadcast, rx_block_broadcast) = broadcast::channel::( - context.parameters.dag_state_cached_rounds as usize, - ); - let (new_round_sender, new_round_receiver) = watch::channel(0); - - let me = Self { - tx_block_broadcast, - new_round_sender, - context, - }; - - let receivers = CoreSignalsReceivers { - rx_block_broadcast, - new_round_receiver, - }; - - (me, receivers) - } - - /// Sends a signal to all the waiters that a new block has been produced. - /// The method will return true if block has reached even one - /// subscriber, false otherwise. - pub(crate) fn new_block(&self, extended_block: ExtendedBlock) -> ConsensusResult<()> { - // When there is only one authority in committee, it is unnecessary to broadcast - // the block which will fail anyway without subscribers to the signal. - if self.context.committee.size() > 1 { - if extended_block.block.round() == GENESIS_ROUND { - debug!("Ignoring broadcasting genesis block to peers"); - return Ok(()); - } - - if let Err(err) = self.tx_block_broadcast.send(extended_block) { - warn!("Couldn't broadcast the block to any receiver: {err}"); - return Err(ConsensusError::Shutdown); - } - } else { - debug!( - "Did not broadcast block {extended_block:?} to receivers as committee size is <= 1" - ); - } - Ok(()) - } - - /// Sends a signal that threshold clock has advanced to new round. The - /// `round_number` is the round at which the threshold clock has - /// advanced to. - pub(crate) fn new_round(&mut self, round_number: Round) { - let _ = self.new_round_sender.send_replace(round_number); - } -} - -/// Receivers of signals from Core. -/// Intentionally un-cloneable. Components should only subscribe to channels -/// they need. -pub(crate) struct CoreSignalsReceivers { - rx_block_broadcast: broadcast::Receiver, - new_round_receiver: watch::Receiver, -} - -impl CoreSignalsReceivers { - pub(crate) fn block_broadcast_receiver(&self) -> broadcast::Receiver { - self.rx_block_broadcast.resubscribe() - } - - pub(crate) fn new_round_receiver(&self) -> watch::Receiver { - self.new_round_receiver.clone() - } -} - -/// Creates cores for the specified number of authorities for their -/// corresponding stakes. The method returns the cores and their respective -/// signal receivers are returned in `AuthorityIndex` order asc. -#[cfg(test)] -pub(crate) fn create_cores(context: Context, authorities: Vec) -> Vec { - let mut cores = Vec::new(); - - for index in 0..authorities.len() { - let own_index = AuthorityIndex::new_for_test(index as u32); - let core = CoreTextFixture::new(context.clone(), authorities.clone(), own_index, false); - cores.push(core); - } - cores -} - -#[cfg(test)] -pub(crate) struct CoreTextFixture { - pub core: Core, - pub signal_receivers: CoreSignalsReceivers, - pub block_receiver: broadcast::Receiver, - #[expect(unused)] - pub commit_receiver: UnboundedReceiver, - pub store: Arc, -} - -#[cfg(test)] -impl CoreTextFixture { - fn new( - context: Context, - authorities: Vec, - own_index: AuthorityIndex, - sync_last_known_own_block: bool, - ) -> Self { - let (committee, mut signers) = local_committee_and_keys(0, authorities); - let mut context = context; - context = context - .with_committee(committee) - .with_authority_index(own_index); - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new( - LeaderSchedule::from_store(context.clone(), dag_state.clone()) - .with_num_commits_per_schedule(10), - ); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let block_receiver = signal_receivers.block_broadcast_receiver(); - - let (commit_sender, commit_receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(commit_sender, 0), - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let block_signer = signers.remove(own_index.value()).1; - - let core = Core::new( - context, - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - block_signer, - dag_state, - sync_last_known_own_block, - ); - - Self { - core, - signal_receivers, - block_receiver, - commit_receiver, - store, - } - } -} - -#[cfg(test)] -mod test { - use std::{collections::BTreeSet, time::Duration}; - - use consensus_config::{AuthorityIndex, Parameters}; - use futures::{StreamExt, stream::FuturesUnordered}; - use iota_metrics::monitored_mpsc::unbounded_channel; - use iota_protocol_config::ProtocolConfig; - use rstest::rstest; - use tokio::time::sleep; - - use super::*; - use crate::{ - CommitConsumer, CommitIndex, - block::{TestBlock, genesis_blocks}, - block_verifier::NoopBlockVerifier, - commit::CommitAPI, - leader_scoring::ReputationScores, - storage::{Store, WriteBatch, mem_store::MemStore}, - test_dag_builder::DagBuilder, - test_dag_parser::parse_dag, - transaction::{BlockStatus, TransactionClient}, - }; - - /// Recover Core and continue proposing from the last round which forms a - /// quorum. - #[tokio::test] - async fn test_core_recover_from_store_for_full_round() { - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let mut block_status_subscriptions = FuturesUnordered::new(); - - // Create test blocks for all the authorities for 4 rounds and populate them in - // store - let mut last_round_blocks = genesis_blocks(&context); - let mut all_blocks: Vec = last_round_blocks.clone(); - for round in 1..=4 { - let mut this_round_blocks = Vec::new(); - for (index, _authority) in context.committee.authorities() { - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, index.value() as u32) - .set_ancestors(last_round_blocks.iter().map(|b| b.reference()).collect()) - .build(), - ); - - // If it's round 1, that one will be committed later on, and it's our "own" - // block, then subscribe to listen for the block status. - if round == 1 && index == context.own_index { - let subscription = - transaction_consumer.subscribe_for_block_status_testing(block.reference()); - block_status_subscriptions.push(subscription); - } - - this_round_blocks.push(block); - } - all_blocks.extend(this_round_blocks.clone()); - last_round_blocks = this_round_blocks; - } - // write them in store - store - .write(WriteBatch::default().blocks(all_blocks)) - .expect("Storage error"); - - // create dag state after all blocks have been written to store - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), 0), - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - // Check no commits have been persisted to dag_state or store. - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - - // Now spin up core - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let mut block_receiver = signal_receivers.block_broadcast_receiver(); - let _core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - // New round should be 5 - let mut new_round = signal_receivers.new_round_receiver(); - assert_eq!(*new_round.borrow_and_update(), 5); - - // Block for round 5 should have been proposed. - let proposed_block = block_receiver - .recv() - .await - .expect("A block should have been created"); - assert_eq!(proposed_block.block.round(), 5); - let ancestors = proposed_block.block.ancestors(); - - // Only ancestors of round 4 should be included. - assert_eq!(ancestors.len(), 4); - for ancestor in ancestors { - assert_eq!(ancestor.round, 4); - } - - let last_commit = store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - - // There were no commits prior to the core starting up but there was completed - // rounds up to and including round 4. So we should commit leaders in round 1 & - // 2 as soon as the new block for round 5 is proposed. - assert_eq!(last_commit.index(), 2); - assert_eq!(dag_state.read().last_commit_index(), 2); - let all_stored_commits = store.scan_commits((0..=CommitIndex::MAX).into()).unwrap(); - assert_eq!(all_stored_commits.len(), 2); - - // And ensure that our "own" block 1 sent to TransactionConsumer as notification - // alongside with gc_round - while let Some(result) = block_status_subscriptions.next().await { - let status = result.unwrap(); - assert!(matches!(status, BlockStatus::Sequenced(_))); - } - } - - /// Recover Core and continue proposing when having a partial last round - /// which doesn't form a quorum and we haven't proposed for that round - /// yet. - #[tokio::test] - async fn test_core_recover_from_store_for_partial_round() { - telemetry_subscribers::init_for_testing(); - - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // Create test blocks for all authorities except our's (index = 0). - let mut last_round_blocks = genesis_blocks(&context); - let mut all_blocks = last_round_blocks.clone(); - for round in 1..=4 { - let mut this_round_blocks = Vec::new(); - - // For round 4 only produce f+1 blocks. Skip our validator 0 and that of - // position 1 from creating blocks. - let authorities_to_skip = if round == 4 { - context.committee.validity_threshold() as usize - } else { - // otherwise always skip creating a block for our authority - 1 - }; - - for (index, _authority) in context.committee.authorities().skip(authorities_to_skip) { - let block = TestBlock::new(round, index.value() as u32) - .set_ancestors(last_round_blocks.iter().map(|b| b.reference()).collect()) - .build(); - this_round_blocks.push(VerifiedBlock::new_for_test(block)); - } - all_blocks.extend(this_round_blocks.clone()); - last_round_blocks = this_round_blocks; - } - - // write them in store - store - .write(WriteBatch::default().blocks(all_blocks)) - .expect("Storage error"); - - // create dag state after all blocks have been written to store - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), 0), - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - // Check no commits have been persisted to dag_state & store - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - - // Now spin up core - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let mut block_receiver = signal_receivers.block_broadcast_receiver(); - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - // Clock round should have advanced to 5 during recovery because - // a quorum has formed in round 4. - let mut new_round = signal_receivers.new_round_receiver(); - assert_eq!(*new_round.borrow_and_update(), 5); - - // During recovery, round 4 block should have been proposed. - let proposed_block = block_receiver - .recv() - .await - .expect("A block should have been created"); - assert_eq!(proposed_block.block.round(), 4); - let ancestors = proposed_block.block.ancestors(); - - assert_eq!(ancestors.len(), 4); - for ancestor in ancestors { - if ancestor.author == context.own_index { - assert_eq!(ancestor.round, 0); - } else { - assert_eq!(ancestor.round, 3); - } - } - - // Run commit rule. - core.try_commit(vec![]).ok(); - let last_commit = store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - - // There were no commits prior to the core starting up but there was completed - // rounds up to round 4. So we should commit leaders in round 1 & 2 as soon - // as the new block for round 4 is proposed. - assert_eq!(last_commit.index(), 2); - assert_eq!(dag_state.read().last_commit_index(), 2); - let all_stored_commits = store.scan_commits((0..=CommitIndex::MAX).into()).unwrap(); - assert_eq!(all_stored_commits.len(), 2); - } - - #[tokio::test] - async fn test_core_propose_after_genesis() { - telemetry_subscribers::init_for_testing(); - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(2_000); - config.set_consensus_max_transactions_in_block_bytes_for_testing(2_000); - config - }); - - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let (transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let mut block_receiver = signal_receivers.block_broadcast_receiver(); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), 0), - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - // Send some transactions - let mut total = 0; - let mut index = 0; - loop { - let transaction = - bcs::to_bytes(&format!("Transaction {index}")).expect("Shouldn't fail"); - total += transaction.len(); - index += 1; - let _w = transaction_client - .submit_no_wait(vec![transaction]) - .await - .unwrap(); - - // Create total size of transactions up to 1KB - if total >= 1_000 { - break; - } - } - - // a new block should have been created during recovery. - let extended_block = block_receiver - .recv() - .await - .expect("A new block should have been created"); - - // A new block created - assert the details - assert_eq!(extended_block.block.round(), 1); - assert_eq!(extended_block.block.author().value(), 0); - assert_eq!(extended_block.block.ancestors().len(), 4); - - let mut total = 0; - for (i, transaction) in extended_block.block.transactions().iter().enumerate() { - total += transaction.data().len() as u64; - let transaction: String = bcs::from_bytes(transaction.data()).unwrap(); - assert_eq!(format!("Transaction {i}"), transaction); - } - assert!( - total - <= context - .protocol_config - .consensus_max_transactions_in_block_bytes() - ); - - // genesis blocks should be referenced - let all_genesis = genesis_blocks(&context); - - for ancestor in extended_block.block.ancestors() { - all_genesis - .iter() - .find(|block| block.reference() == *ancestor) - .expect("Block should be found amongst genesis blocks"); - } - - // Try to propose again - with or without ignore leaders check, it will not - // return any block - assert!(core.try_propose(false).unwrap().is_none()); - assert!(core.try_propose(true).unwrap().is_none()); - - // Check no commits have been persisted to dag_state & store - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - } - - #[tokio::test] - async fn test_core_propose_once_receiving_a_quorum() { - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), 0), - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - let mut expected_ancestors = BTreeSet::new(); - - // Adding one block now will trigger the creation of new block for round 1 - let block_1 = VerifiedBlock::new_for_test(TestBlock::new(1, 1).build()); - expected_ancestors.insert(block_1.reference()); - // Wait for min round delay to allow blocks to be proposed. - sleep(context.parameters.min_round_delay).await; - // add blocks to trigger proposal. - _ = core.add_blocks(vec![block_1]); - - assert_eq!(core.last_proposed_round(), 1); - expected_ancestors.insert(core.last_proposed_block().reference()); - // attempt to create a block - none will be produced. - assert!(core.try_propose(false).unwrap().is_none()); - - // Adding another block now forms a quorum for round 1, so block at round 2 will - // proposed - let block_3 = VerifiedBlock::new_for_test(TestBlock::new(1, 2).build()); - expected_ancestors.insert(block_3.reference()); - // Wait for min round delay to allow blocks to be proposed. - sleep(context.parameters.min_round_delay).await; - // add blocks to trigger proposal. - _ = core.add_blocks(vec![block_3]); - - assert_eq!(core.last_proposed_round(), 2); - - let proposed_block = core.last_proposed_block(); - assert_eq!(proposed_block.round(), 2); - assert_eq!(proposed_block.author(), context.own_index); - assert_eq!(proposed_block.ancestors().len(), 3); - let ancestors = proposed_block.ancestors(); - let ancestors = ancestors.iter().cloned().collect::>(); - assert_eq!(ancestors, expected_ancestors); - - // Check no commits have been persisted to dag_state & store - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - } - - #[rstest] - #[tokio::test] - async fn test_commit_and_notify_for_block_status(#[values(0, 2)] gc_depth: u32) { - telemetry_subscribers::init_for_testing(); - let (mut context, mut key_pairs) = Context::new_for_test(4); - - if gc_depth > 0 { - context - .protocol_config - .set_consensus_gc_depth_for_testing(gc_depth); - } - - let context = Arc::new(context); - - let store = Arc::new(MemStore::new()); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let mut block_status_subscriptions = FuturesUnordered::new(); - - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { - A -> [*], - B -> [-A2], - C -> [-A2], - D -> [-A2], - }, - Round 4 : { - B -> [-A3], - C -> [-A3], - D -> [-A3], - }, - Round 5 : { - A -> [A3, B4, C4, D4] - B -> [*], - C -> [*], - D -> [*], - }, - Round 6 : { * }, - Round 7 : { * }, - Round 8 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - dag_builder.print(); - - // Subscribe to all created "own" blocks. We know that for our node (A) we'll be - // able to commit up to round 5. - for block in dag_builder.blocks(1..=5) { - if block.author() == context.own_index { - let subscription = - transaction_consumer.subscribe_for_block_status_testing(block.reference()); - block_status_subscriptions.push(subscription); - } - } - - // write them in store - store - .write(WriteBatch::default().blocks(dag_builder.blocks(1..=8))) - .expect("Storage error"); - - // create dag state after all blocks have been written to store - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender.clone(), 0); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - // Check no commits have been persisted to dag_state or store. - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - - // Now spin up core - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - let _core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - let last_commit = store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - - assert_eq!(last_commit.index(), 5); - - while let Some(result) = block_status_subscriptions.next().await { - let status = result.unwrap(); - - // If gc is enabled, then we expect some blocks to be garbage collected. - if gc_depth > 0 { - match status { - BlockStatus::Sequenced(block_ref) => { - assert!(block_ref.round == 1 || block_ref.round == 5); - } - BlockStatus::GarbageCollected(block_ref) => { - assert!(block_ref.round == 2 || block_ref.round == 3); - } - } - } else { - // otherwise all of them should be committed - assert!(matches!(status, BlockStatus::Sequenced(_))); - } - } - } - - // Tests that the threshold clock advances when blocks get unsuspended due to - // GC'ed blocks and newly created blocks are always higher than the last - // advanced gc round. - #[tokio::test] - async fn test_multiple_commits_advance_threshold_clock() { - telemetry_subscribers::init_for_testing(); - let (mut context, mut key_pairs) = Context::new_for_test(4); - const GC_DEPTH: u32 = 2; - - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - - let context = Arc::new(context); - - let store = Arc::new(MemStore::new()); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // On round 1 we do produce the block for authority D but we do not link it - // until round 6. This is making round 6 unable to get processed - // until leader of round 3 is committed where round 1 gets garbage collected. - // Then we add more rounds so we can trigger a commit for leader of round 9 - // which will move the gc round to 7. - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { - B -> [-D1], - C -> [-D1], - D -> [-D1], - }, - Round 3 : { - B -> [*], - C -> [*] - D -> [*], - }, - Round 4 : { - A -> [*], - B -> [*], - C -> [*] - D -> [*], - }, - Round 5 : { - A -> [*], - B -> [*], - C -> [*], - D -> [*], - }, - Round 6 : { - B -> [A5, B5, C5, D1], - C -> [A5, B5, C5, D1], - D -> [A5, B5, C5, D1], - }, - Round 7 : { - B -> [*], - C -> [*], - D -> [*], - }, - Round 8 : { - B -> [*], - C -> [*], - D -> [*], - }, - Round 9 : { - B -> [*], - C -> [*], - D -> [*], - }, - Round 10 : { - B -> [*], - C -> [*], - D -> [*], - }, - Round 11 : { - B -> [*], - C -> [*], - D -> [*], - }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - dag_builder.print(); - - // create dag state after all blocks have been written to store - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - // Check no commits have been persisted to dag_state or store. - let last_commit = store.read_last_commit().unwrap(); - assert!(last_commit.is_none()); - assert_eq!(dag_state.read().last_commit_index(), 0); - - // Now spin up core - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state, - true, - ); - // We set the last known round to 4 so we avoid creating new blocks until then - - // otherwise it will crash as the already created DAG contains blocks for this - // authority. - core.set_last_known_proposed_round(4); - - // We add all the blocks except D1. The only ones we can immediately accept are - // the ones up to round 5 as they don't have a dependency on D1. Rest of blocks - // do have causal dependency to D1 so they can't be processed until the - // leader of round 3 can get committed and gc round moves to 1. That will make - // all the blocks that depend to D1 get accepted. However, our threshold - // clock is now at round 6 as the last quorum that we managed to process was the - // round 5. As commits happen blocks of later rounds get accepted and - // more leaders get committed. Eventually the leader of round 9 gets committed - // and gc is moved to 9 - 2 = 7. If our node attempts to produce a block - // for the threshold clock 6, that will make the acceptance checks fail as now - // gc has moved far past this round. - core.add_blocks( - dag_builder - .blocks(1..=11) - .into_iter() - .filter(|b| !(b.round() == 1 && b.author() == AuthorityIndex::new_for_test(3))) - .collect(), - ) - .expect("Should not fail"); - - assert_eq!(core.last_proposed_round(), 12); - } - - #[tokio::test] - async fn test_core_set_min_propose_round() { - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - })); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender, 0), - dag_state.clone(), - store, - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state, - true, - ); - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // Trying to explicitly propose a block will not produce anything - assert!(core.try_propose(true).unwrap().is_none()); - - // Create blocks for the whole network - even "our" node in order to replicate - // an "amnesia" recovery. - let mut builder = DagBuilder::new(context.clone()); - builder.layers(1..=10).build(); - - let blocks = builder.blocks.values().cloned().collect::>(); - - // Process all the blocks - assert!(core.add_blocks(blocks).unwrap().is_empty()); - - // Try to propose - no block should be produced. - assert!(core.try_propose(true).unwrap().is_none()); - - // Now set the last known proposed round which is the highest round for which - // the network informed us that we do have proposed a block about. - core.set_last_known_proposed_round(10); - - let block = core.try_propose(true).expect("No error").unwrap(); - assert_eq!(block.round(), 11); - assert_eq!(block.ancestors().len(), 4); - - let our_ancestor_included = block.ancestors()[0]; - assert_eq!(our_ancestor_included.author, context.own_index); - assert_eq!(our_ancestor_included.round, 10); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_core_try_new_block_leader_timeout() { - telemetry_subscribers::init_for_testing(); - - // Since we run the test with started_paused = true, any time-dependent - // operations using Tokio's time facilities, such as tokio::time::sleep - // or tokio::time::Instant, will not advance. So practically each Core's - // clock will have initialised potentially with different values but it never - // advances. To ensure that blocks won't get rejected by cores we'll - // need to manually wait for the time diff before processing them. By - // calling the `tokio::time::sleep` we implicitly also advance the tokio - // clock. - async fn wait_blocks(blocks: &[VerifiedBlock], context: &Context) { - // Simulate the time wait before processing a block to ensure that - // block.timestamp <= now - let now = context.clock.timestamp_utc_ms(); - let max_timestamp = blocks - .iter() - .max_by_key(|block| block.timestamp_ms() as BlockTimestampMs) - .map(|block| block.timestamp_ms()) - .unwrap_or(0); - - let wait_time = Duration::from_millis(max_timestamp.saturating_sub(now)); - sleep(wait_time).await; - } - - let (context, _) = Context::new_for_test(4); - // Create the cores for all authorities - let mut all_cores = create_cores(context, vec![1, 1, 1, 1]); - - // Create blocks for rounds 1..=3 from all Cores except last Core of authority - // 3, so we miss the block from it. As it will be the leader of round 3 - // then no-one will be able to progress to round 4 unless we explicitly trigger - // the block creation. - // create the cores and their signals for all the authorities - let (_last_core, cores) = all_cores.split_last_mut().unwrap(); - - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::::new(); - for round in 1..=3 { - let mut this_round_blocks = Vec::new(); - - for core_fixture in cores.iter_mut() { - wait_blocks(&last_round_blocks, &core_fixture.core.context).await; - - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // Only when round > 1 and using non-genesis parents. - if let Some(r) = last_round_blocks.first().map(|b| b.round()) { - assert_eq!(round - 1, r); - if core_fixture.core.last_proposed_round() == r { - // Force propose new block regardless of min round delay. - core_fixture - .core - .try_propose(true) - .unwrap() - .unwrap_or_else(|| { - panic!("Block should have been proposed for round {round}") - }); - } - } - - assert_eq!(core_fixture.core.last_proposed_round(), round); - - this_round_blocks.push(core_fixture.core.last_proposed_block()); - } - - last_round_blocks = this_round_blocks; - } - - // Try to create the blocks for round 4 by calling the try_propose() method. No - // block should be created as the leader - authority 3 - hasn't proposed - // any block. - for core_fixture in cores.iter_mut() { - wait_blocks(&last_round_blocks, &core_fixture.core.context).await; - - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - assert!(core_fixture.core.try_propose(false).unwrap().is_none()); - } - - // Now try to create the blocks for round 4 via the leader timeout method which - // should ignore any leader checks or min round delay. - for core_fixture in cores.iter_mut() { - assert!(core_fixture.core.new_block(4, true).unwrap().is_some()); - assert_eq!(core_fixture.core.last_proposed_round(), 4); - - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 1 leader rounds with rounds completed up to and including - // round 4 - assert_eq!(last_commit.index(), 1); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), 1); - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_core_try_new_block_with_leader_timeout_and_low_scoring_authority() { - telemetry_subscribers::init_for_testing(); - - // Since we run the test with started_paused = true, any time-dependent - // operations using Tokio's time facilities, such as tokio::time::sleep - // or tokio::time::Instant, will not advance. So practically each Core's - // clock will have initialised potentially with different values but it never - // advances. To ensure that blocks won't get rejected by cores we'll - // need to manually wait for the time diff before processing them. By - // calling the `tokio::time::sleep` we implicitly also advance the tokio - // clock. - async fn wait_blocks(blocks: &[VerifiedBlock], context: &Context) { - // Simulate the time wait before processing a block to ensure that - // block.timestamp <= now - let now = context.clock.timestamp_utc_ms(); - let max_timestamp = blocks - .iter() - .max_by_key(|block| block.timestamp_ms() as BlockTimestampMs) - .map(|block| block.timestamp_ms()) - .unwrap_or(0); - - let wait_time = Duration::from_millis(max_timestamp.saturating_sub(now)); - sleep(wait_time).await; - } - - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_smart_ancestor_selection_for_testing(true); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(true); - - // Create the cores for all authorities - let mut all_cores = create_cores(context, vec![1, 1, 1, 1]); - let (_last_core, cores) = all_cores.split_last_mut().unwrap(); - - // Create blocks for rounds 1..=30 from all Cores except last Core of authority - // 3. - let mut last_round_blocks = Vec::::new(); - for round in 1..=30 { - let mut this_round_blocks = Vec::new(); - - for core_fixture in cores.iter_mut() { - wait_blocks(&last_round_blocks, &core_fixture.core.context).await; - - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // Only when round > 1 and using non-genesis parents. - if let Some(r) = last_round_blocks.first().map(|b| b.round()) { - assert_eq!(round - 1, r); - if core_fixture.core.last_proposed_round() == r { - // Force propose new block regardless of min round delay. - core_fixture - .core - .try_propose(true) - .unwrap() - .unwrap_or_else(|| { - panic!("Block should have been proposed for round {round}") - }); - } - } - - assert_eq!(core_fixture.core.last_proposed_round(), round); - - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - } - - last_round_blocks = this_round_blocks; - } - - // Now produce blocks for all Cores - for round in 31..=40 { - let mut this_round_blocks = Vec::new(); - - for core_fixture in all_cores.iter_mut() { - wait_blocks(&last_round_blocks, &core_fixture.core.context).await; - - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // Only when round > 1 and using non-genesis parents. - if let Some(r) = last_round_blocks.first().map(|b| b.round()) { - assert_eq!(round - 1, r); - if core_fixture.core.last_proposed_round() == r { - // Force propose new block regardless of min round delay. - core_fixture - .core - .try_propose(true) - .unwrap() - .unwrap_or_else(|| { - panic!("Block should have been proposed for round {round}") - }); - } - } - - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - - for block in this_round_blocks.iter() { - if block.author() != AuthorityIndex::new_for_test(3) { - // Assert blocks created include only 3 ancestors per block as one - // should be excluded - assert_eq!(block.ancestors().len(), 3); - } else { - // Authority 3 is the low scoring authority so it will still include - // its own blocks. - assert_eq!(block.ancestors().len(), 4); - } - } - } - - last_round_blocks = this_round_blocks; - } - } - - #[tokio::test] - async fn test_smart_ancestor_selection() { - telemetry_subscribers::init_for_testing(); - let (mut context, mut key_pairs) = Context::new_for_test(7); - context - .protocol_config - .set_consensus_smart_ancestor_selection_for_testing(true); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(true); - let context = Arc::new(context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - })); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new( - LeaderSchedule::from_store(context.clone(), dag_state.clone()) - .with_num_commits_per_schedule(10), - ); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let mut block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - true, - ); - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // Trying to explicitly propose a block will not produce anything - assert!(core.try_propose(true).unwrap().is_none()); - - // Create blocks for the whole network but not for authority 1 - let mut builder = DagBuilder::new(context.clone()); - builder - .layers(1..=12) - .authorities(vec![AuthorityIndex::new_for_test(1)]) - .skip_block() - .build(); - let blocks = builder.blocks(1..=12); - // Process all the blocks - assert!(core.add_blocks(blocks).unwrap().is_empty()); - core.set_last_known_proposed_round(12); - - let block = core.try_propose(true).expect("No error").unwrap(); - assert_eq!(block.round(), 13); - assert_eq!(block.ancestors().len(), 7); - - // Build blocks for rest of the network other than own index - builder - .layers(13..=14) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - let blocks = builder.blocks(13..=14); - assert!(core.add_blocks(blocks).unwrap().is_empty()); - - // We now have triggered a leader schedule change so we should have - // one EXCLUDE authority (1) when we go to select ancestors for the next - // proposal - let block = core.try_propose(true).expect("No error").unwrap(); - assert_eq!(block.round(), 15); - assert_eq!(block.ancestors().len(), 6); - - // Build blocks for a quorum of the network including the EXCLUDE authority (1) - // which will trigger smart select and we will not propose a block - builder - .layer(15) - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(5), - AuthorityIndex::new_for_test(6), - ]) - .skip_block() - .build(); - let blocks = builder.blocks(15..=15); - let authority_1_excluded_block_reference = blocks - .iter() - .find(|block| block.author() == AuthorityIndex::new_for_test(1)) - .unwrap() - .reference(); - // Wait for min round delay to allow blocks to be proposed. - sleep(context.parameters.min_round_delay).await; - // Smart select should be triggered and no block should be proposed. - assert!(core.add_blocks(blocks).unwrap().is_empty()); - assert_eq!(core.last_proposed_block().round(), 15); - - builder - .layer(15) - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - AuthorityIndex::new_for_test(3), - AuthorityIndex::new_for_test(4), - ]) - .skip_block() - .build(); - let blocks = builder.blocks(15..=15); - let included_block_references = iter::once(&core.last_proposed_block()) - .chain(blocks.iter()) - .filter(|block| block.author() != AuthorityIndex::new_for_test(1)) - .map(|block| block.reference()) - .collect::>(); - - // Have enough ancestor blocks to propose now. - assert!(core.add_blocks(blocks).unwrap().is_empty()); - assert_eq!(core.last_proposed_block().round(), 16); - - // Check that a new block has been proposed & signaled. - let extended_block = loop { - let extended_block = - tokio::time::timeout(Duration::from_secs(1), block_receiver.recv()) - .await - .unwrap() - .unwrap(); - if extended_block.block.round() == 16 { - break extended_block; - } - }; - assert_eq!(extended_block.block.round(), 16); - assert_eq!(extended_block.block.author(), core.context.own_index); - assert_eq!(extended_block.block.ancestors().len(), 6); - assert_eq!(extended_block.block.ancestors(), included_block_references); - assert_eq!(extended_block.excluded_ancestors.len(), 1); - assert_eq!( - extended_block.excluded_ancestors[0], - authority_1_excluded_block_reference - ); - - // Build blocks for a quorum of the network including the EXCLUDE ancestor - // which will trigger smart select and we will not propose a block. - // This time we will force propose by hitting the leader timeout after which - // should cause us to include this EXCLUDE ancestor. - builder - .layer(16) - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(5), - AuthorityIndex::new_for_test(6), - ]) - .skip_block() - .build(); - let blocks = builder.blocks(16..=16); - // Wait for leader timeout to force blocks to be proposed. - sleep(context.parameters.min_round_delay).await; - // Smart select should be triggered and no block should be proposed. - assert!(core.add_blocks(blocks).unwrap().is_empty()); - assert_eq!(core.last_proposed_block().round(), 16); - - // Simulate a leader timeout and a force proposal where we will include - // one EXCLUDE ancestor when we go to select ancestors for the next proposal - let block = core.try_propose(true).expect("No error").unwrap(); - assert_eq!(block.round(), 17); - assert_eq!(block.ancestors().len(), 5); - - // Check that a new block has been proposed & signaled. - let extended_block = tokio::time::timeout(Duration::from_secs(1), block_receiver.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), 17); - assert_eq!(extended_block.block.author(), core.context.own_index); - assert_eq!(extended_block.block.ancestors().len(), 5); - assert_eq!(extended_block.excluded_ancestors.len(), 0); - - // Set quorum rounds for authority which will unlock the Excluded - // authority (1) and then we should be able to create a new layer of blocks - // which will then all be included as ancestors for the next proposal - core.set_propagation_delay_and_quorum_rounds( - 0, - vec![ - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - ], - vec![ - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - (16, 16), - ], - ); - - builder - .layer(17) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - let blocks = builder.blocks(17..=17); - let included_block_references = iter::once(&core.last_proposed_block()) - .chain(blocks.iter()) - .map(|block| block.reference()) - .collect::>(); - - // Have enough ancestor blocks to propose now. - sleep(context.parameters.min_round_delay).await; - assert!(core.add_blocks(blocks).unwrap().is_empty()); - assert_eq!(core.last_proposed_block().round(), 18); - - // Check that a new block has been proposed & signaled. - let extended_block = tokio::time::timeout(Duration::from_secs(1), block_receiver.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), 18); - assert_eq!(extended_block.block.author(), core.context.own_index); - assert_eq!(extended_block.block.ancestors().len(), 7); - assert_eq!(extended_block.block.ancestors(), included_block_references); - assert_eq!(extended_block.excluded_ancestors.len(), 0); - } - - #[tokio::test] - async fn test_excluded_ancestor_limit() { - telemetry_subscribers::init_for_testing(); - let (mut context, mut key_pairs) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_smart_ancestor_selection_for_testing(true); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(true); - let context = Arc::new(context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - })); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new( - LeaderSchedule::from_store(context.clone(), dag_state.clone()) - .with_num_commits_per_schedule(10), - ); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let mut block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store.clone(), - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - true, - ); - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // Create blocks for the whole network - let mut builder = DagBuilder::new(context.clone()); - builder.layers(1..=3).build(); - - // This will equivocate 9 blocks for authority 1 which will be excluded on - // the proposal but because of the limits set will be dropped and not included - // as part of the ExtendedBlock structure sent to the rest of the network - builder - .layer(4) - .authorities(vec![AuthorityIndex::new_for_test(1)]) - .equivocate(9) - .build(); - let blocks = builder.blocks(1..=4); - - // Process all the blocks - assert!(core.add_blocks(blocks).unwrap().is_empty()); - core.set_last_known_proposed_round(3); - - let block = core.try_propose(true).expect("No error").unwrap(); - assert_eq!(block.round(), 5); - assert_eq!(block.ancestors().len(), 4); - - // Check that a new block has been proposed & signaled. - let extended_block = tokio::time::timeout(Duration::from_secs(1), block_receiver.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), 5); - assert_eq!(extended_block.block.author(), core.context.own_index); - assert_eq!(extended_block.block.ancestors().len(), 4); - assert_eq!(extended_block.excluded_ancestors.len(), 8); - } - - #[tokio::test] - async fn test_core_set_subscriber_exists() { - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender, 0), - dag_state.clone(), - store, - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - // Set to no subscriber exists initially. - false, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state, - false, - ); - - // There is no proposal during recovery because there is no subscriber. - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // There is no proposal even with forced proposing. - assert!(core.try_propose(true).unwrap().is_none()); - - // Let Core know subscriber exists. - core.set_quorum_subscribers_exists(true); - - // Proposing now would succeed. - assert!(core.try_propose(true).unwrap().is_some()); - } - - #[tokio::test] - async fn test_core_set_propagation_delay_per_authority() { - // TODO: create helper to avoid the duplicated code here. - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender, 0), - dag_state.clone(), - store, - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - // Set to no subscriber exists initially. - false, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state, - false, - ); - - // There is no proposal during recovery because there is no subscriber. - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // Use a large propagation delay to disable proposing. - core.set_propagation_delay_and_quorum_rounds(1000, vec![], vec![]); - - // Make propagation delay the only reason for not proposing. - core.set_quorum_subscribers_exists(true); - - // There is no proposal even with forced proposing. - assert!(core.try_propose(true).unwrap().is_none()); - - // Let Core know there is no propagation delay. - core.set_propagation_delay_and_quorum_rounds(0, vec![], vec![]); - - // Proposing now would succeed. - assert!(core.try_propose(true).unwrap().is_some()); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_leader_schedule_change() { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - - let (context, _) = Context::new_for_test(4); - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1]); - - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::new(); - for round in 1..=30 { - let mut this_round_blocks = Vec::new(); - - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - - for core_fixture in &mut cores { - // add the blocks from last round - // this will trigger a block creation for the round and a signal should be - // emitted - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // A "new round" signal should be received given that all the blocks of previous - // round have been processed - let new_round = receive( - Duration::from_secs(1), - core_fixture.signal_receivers.new_round_receiver(), - ) - .await; - assert_eq!(new_round, round); - - // Check that a new block has been proposed. - let extended_block = tokio::time::timeout( - Duration::from_secs(1), - core_fixture.block_receiver.recv(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), round); - assert_eq!( - extended_block.block.author(), - core_fixture.core.context.own_index - ); - - // append the new block to this round blocks - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - - let block = core_fixture.core.last_proposed_block(); - - // ensure that produced block is referring to the blocks of last_round - assert_eq!( - block.ancestors().len(), - core_fixture.core.context.committee.size() - ); - for ancestor in block.ancestors() { - if block.round() > 1 { - // don't bother with round 1 block which just contains the genesis blocks. - assert!( - last_round_blocks - .iter() - .any(|block| block.reference() == *ancestor), - "Reference from previous round should be added" - ); - } - } - } - - last_round_blocks = this_round_blocks; - } - - for core_fixture in cores { - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 28 leader rounds with rounds completed up to and including - // round 29. Round 30 blocks will only include their own blocks, so the - // 28th leader will not be committed. - assert_eq!(last_commit.index(), 27); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), 27); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .bad_nodes - .len(), - 1 - ); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .good_nodes - .len(), - 1 - ); - let expected_reputation_scores = - ReputationScores::new((11..=20).into(), vec![29, 29, 29, 29]); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .reputation_scores, - expected_reputation_scores - ); - } - } - - // TODO: Remove this when DistributedVoteScoring is enabled. - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_leader_schedule_change_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1]); - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::new(); - for round in 1..=30 { - let mut this_round_blocks = Vec::new(); - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - for core_fixture in &mut cores { - // add the blocks from last round - // this will trigger a block creation for the round and a signal should be - // emitted - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - // A "new round" signal should be received given that all the blocks of previous - // round have been processed - let new_round = receive( - Duration::from_secs(1), - core_fixture.signal_receivers.new_round_receiver(), - ) - .await; - assert_eq!(new_round, round); - // Check that a new block has been proposed. - let extended_block = tokio::time::timeout( - Duration::from_secs(1), - core_fixture.block_receiver.recv(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), round); - assert_eq!( - extended_block.block.author(), - core_fixture.core.context.own_index - ); - - // append the new block to this round blocks - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - let block = core_fixture.core.last_proposed_block(); - // ensure that produced block is referring to the blocks of last_round - assert_eq!( - block.ancestors().len(), - core_fixture.core.context.committee.size() - ); - for ancestor in block.ancestors() { - if block.round() > 1 { - // don't bother with round 1 block which just contains the genesis blocks. - assert!( - last_round_blocks - .iter() - .any(|block| block.reference() == *ancestor), - "Reference from previous round should be added" - ); - } - } - } - last_round_blocks = this_round_blocks; - } - for core_fixture in cores { - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 28 leader rounds with rounds completed up to and including - // round 29. Round 30 blocks will only include their own blocks, so the - // 28th leader will not be committed. - assert_eq!(last_commit.index(), 27); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), 27); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .bad_nodes - .len(), - 1 - ); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .good_nodes - .len(), - 1 - ); - let expected_reputation_scores = - ReputationScores::new((11..=20).into(), vec![9, 8, 8, 8]); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .reputation_scores, - expected_reputation_scores - ); - } - } - - #[tokio::test] - async fn test_validate_certified_commits() { - telemetry_subscribers::init_for_testing(); - - let (context, _key_pairs) = Context::new_for_test(4); - let context = context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - }); - - let authority_index = AuthorityIndex::new_for_test(0); - let core = CoreTextFixture::new(context, vec![1, 1, 1, 1], authority_index, true); - let mut core = core.core; - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // create a DAG of 12 rounds - let mut dag_builder = DagBuilder::new(core.context.clone()); - dag_builder.layers(1..=12).build(); - - // Store all blocks up to round 6 which should be enough to decide up to leader - // 4 - dag_builder.print(); - let blocks = dag_builder.blocks(1..=6); - - for block in blocks { - core.dag_state.write().accept_block(block); - } - - // Get all the committed sub dags up to round 10 - let sub_dags_and_commits = dag_builder.get_sub_dag_and_certified_commits(1..=10); - - // Now try to commit up to the latest leader (round = 4). Do not provide any - // certified commits. - let committed_sub_dags = core.try_commit(vec![]).unwrap(); - - // We should have committed up to round 4 - assert_eq!(committed_sub_dags.len(), 4); - - // Now validate the certified commits. We'll try 3 different scenarios: - println!("Case 1. Provide certified commits that are all before the last committed round."); - - // Highest certified commit should be for leader of round 4. - let certified_commits = sub_dags_and_commits - .iter() - .take(4) - .map(|(_, c)| c) - .cloned() - .collect::>(); - assert!( - certified_commits.last().unwrap().index() - <= committed_sub_dags.last().unwrap().commit_ref.index, - "Highest certified commit should older than the highest committed index." - ); - - let certified_commits = core.validate_certified_commits(certified_commits).unwrap(); - - // No commits should be processed - assert!(certified_commits.is_empty()); - - println!("Case 2. Provide certified commits that are all after the last committed round."); - - // Highest certified commit should be for leader of round 4. - let certified_commits = sub_dags_and_commits - .iter() - .take(5) - .map(|(_, c)| c.clone()) - .collect::>(); - - let certified_commits = core.validate_certified_commits(certified_commits).unwrap(); - - // The certified commit of index 5 should be processed. - assert_eq!(certified_commits.len(), 1); - assert_eq!(certified_commits.first().unwrap().reference().index, 5); - - println!( - "Case 3. Provide certified commits where the first certified commit index is not the last_committed_index + 1." - ); - - // Highest certified commit should be for leader of round 4. - let certified_commits = sub_dags_and_commits - .iter() - .skip(5) - .take(1) - .map(|(_, c)| c.clone()) - .collect::>(); - - let err = core - .validate_certified_commits(certified_commits) - .unwrap_err(); - match err { - ConsensusError::UnexpectedCertifiedCommitIndex { - expected_commit_index: 5, - commit_index: 6, - } => (), - _ => panic!("Unexpected error: {err:?}"), - } - } - - #[tokio::test] - async fn test_add_certified_commits() { - telemetry_subscribers::init_for_testing(); - - let (context, _key_pairs) = Context::new_for_test(4); - let context = context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - }); - - let authority_index = AuthorityIndex::new_for_test(0); - let core = CoreTextFixture::new(context, vec![1, 1, 1, 1], authority_index, true); - let store = core.store.clone(); - let mut core = core.core; - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - // create a DAG of 12 rounds - let mut dag_builder = DagBuilder::new(core.context.clone()); - dag_builder.layers(1..=12).build(); - - // Store all blocks up to round 6 which should be enough to decide up to leader - // 4 - dag_builder.print(); - let blocks = dag_builder.blocks(1..=6); - - for block in blocks { - core.dag_state.write().accept_block(block); - } - - // Get all the committed sub dags up to round 10 - let sub_dags_and_commits = dag_builder.get_sub_dag_and_certified_commits(1..=10); - - // Now try to commit up to the latest leader (round = 4). Do not provide any - // certified commits. - let committed_sub_dags = core.try_commit(vec![]).unwrap(); - - // We should have committed up to round 4 - assert_eq!(committed_sub_dags.len(), 4); - - let last_commit = store - .read_last_commit() - .unwrap() - .expect("Last commit should be set"); - assert_eq!(last_commit.reference().index, 4); - - println!("Case 1. Provide no certified commits. No commit should happen."); - - let last_commit = store - .read_last_commit() - .unwrap() - .expect("Last commit should be set"); - assert_eq!(last_commit.reference().index, 4); - - println!( - "Case 2. Provide certified commits that before and after the last committed round and also there are additional blocks so can run the direct decide rule as well." - ); - - // The commits of leader rounds 5-8 should be committed via the certified - // commits. - let certified_commits = sub_dags_and_commits - .iter() - .skip(3) - .take(5) - .map(|(_, c)| c.clone()) - .collect::>(); - - // Now only add the blocks of rounds 8..=12. The blocks up to round 7 should be - // accepted via the certified commits processing. - let blocks = dag_builder.blocks(8..=12); - for block in blocks { - core.dag_state.write().accept_block(block); - } - - // The corresponding blocks of the certified commits should be accepted and - // stored before linearizing and committing the DAG. - core.add_certified_commits(CertifiedCommits::new(certified_commits, vec![])) - .expect("Should not fail"); - - let commits = store.scan_commits((6..=10).into()).unwrap(); - - // We expect all the sub dags up to leader round 10 to be committed. - assert_eq!(commits.len(), 5); - - for i in 6..=10 { - let commit = &commits[i - 6]; - assert_eq!(commit.reference().index, i as u32); - } - } - - #[tokio::test] - async fn try_commit_with_certified_commits_gced_blocks() { - const GC_DEPTH: u32 = 3; - telemetry_subscribers::init_for_testing(); - - let (mut context, mut key_pairs) = Context::new_for_test(5); - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - // context.protocol_config. - // set_new_leader_election_schedule_for_testing(val); - let context = Arc::new(context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - })); - - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let leader_schedule = Arc::new( - LeaderSchedule::from_store(context.clone(), dag_state.clone()) - .with_num_commits_per_schedule(10), - ); - - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - // Need at least one subscriber to the block broadcast channel. - let _block_receiver = signal_receivers.block_broadcast_receiver(); - - let (sender, _receiver) = unbounded_channel("consensus_output"); - let commit_consumer = CommitConsumer::new(sender, 0); - let commit_observer = CommitObserver::new( - context.clone(), - commit_consumer, - dag_state.clone(), - store, - leader_schedule.clone(), - ); - - let mut core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state, - true, - ); - - // No new block should have been produced - assert_eq!( - core.last_proposed_round(), - GENESIS_ROUND, - "No block should have been created other than genesis" - ); - - let dag_str = "DAG { - Round 0 : { 5 }, - Round 1 : { * }, - Round 2 : { - A -> [-E1], - B -> [-E1], - C -> [-E1], - D -> [-E1], - }, - Round 3 : { - A -> [*], - B -> [*], - C -> [*], - D -> [*], - }, - Round 4 : { - A -> [*], - B -> [*], - C -> [*], - D -> [*], - }, - Round 5 : { - A -> [*], - B -> [*], - C -> [*], - D -> [*], - E -> [A4, B4, C4, D4, E1] - }, - Round 6 : { * }, - Round 7 : { * }, - }"; - - let (_, mut dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - dag_builder.print(); - - // Now get all the committed sub dags from the DagBuilder - let (_sub_dags, certified_commits): (Vec<_>, Vec<_>) = dag_builder - .get_sub_dag_and_certified_commits(1..=5) - .into_iter() - .unzip(); - - // Now try to commit up to the latest leader (round = 5) with the provided - // certified commits. Not that we have not accepted any blocks. That - // should happen during the commit process. - let committed_sub_dags = core.try_commit(certified_commits).unwrap(); - - // We should have committed up to round 4 - assert_eq!(committed_sub_dags.len(), 4); - for (index, committed_sub_dag) in committed_sub_dags.iter().enumerate() { - assert_eq!(committed_sub_dag.commit_ref.index as usize, index + 1); - - // ensure that block from E1 node has not been committed - for block in committed_sub_dag.blocks.iter() { - if block.round() == 1 && block.author() == AuthorityIndex::new_for_test(5) { - panic!("Did not expect to commit block E1"); - } - } - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_commit_on_leader_schedule_change_boundary_without_multileader() { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - - let (context, _) = Context::new_for_test(6); - - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1, 1, 1]); - - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::new(); - for round in 1..=33 { - let mut this_round_blocks = Vec::new(); - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - for core_fixture in &mut cores { - // add the blocks from last round - // this will trigger a block creation for the round and a signal should be - // emitted - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - // A "new round" signal should be received given that all the blocks of previous - // round have been processed - let new_round = receive( - Duration::from_secs(1), - core_fixture.signal_receivers.new_round_receiver(), - ) - .await; - assert_eq!(new_round, round); - // Check that a new block has been proposed. - let extended_block = tokio::time::timeout( - Duration::from_secs(1), - core_fixture.block_receiver.recv(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), round); - assert_eq!( - extended_block.block.author(), - core_fixture.core.context.own_index - ); - - // append the new block to this round blocks - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - let block = core_fixture.core.last_proposed_block(); - // ensure that produced block is referring to the blocks of last_round - assert_eq!( - block.ancestors().len(), - core_fixture.core.context.committee.size() - ); - for ancestor in block.ancestors() { - if block.round() > 1 { - // don't bother with round 1 block which just contains the genesis blocks. - assert!( - last_round_blocks - .iter() - .any(|block| block.reference() == *ancestor), - "Reference from previous round should be added" - ); - } - } - } - last_round_blocks = this_round_blocks; - } - for core_fixture in cores { - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 31 leader rounds with rounds completed up to and including - // round 33. Round 33 blocks will only include their own blocks, so there - // should only be 30 commits. - // However on a leader schedule change boundary its is possible for a - // new leader to get selected for the same round if the leader elected - // gets swapped allowing for multiple leaders to be committed at a round. - // Meaning with multi leader per round explicitly set to 1 we will have 30, - // otherwise 31. - // NOTE: We used 31 leader rounds to specifically trigger the scenario - // where the leader schedule boundary occurred AND we had a swap to a new - // leader for the same round - let expected_commit_count = 30; - // Leave the code for re-use. - // let expected_commit_count = match num_leaders_per_round { - // Some(1) => 30, - // _ => 31, - //}; - assert_eq!(last_commit.index(), expected_commit_count); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), expected_commit_count as usize); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .bad_nodes - .len(), - 1 - ); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .good_nodes - .len(), - 1 - ); - let expected_reputation_scores = - ReputationScores::new((21..=30).into(), vec![43, 43, 43, 43, 43, 43]); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .reputation_scores, - expected_reputation_scores - ); - } - } - - // TODO: Remove two tests below this when DistributedVoteScoring is enabled. - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn test_commit_on_leader_schedule_change_boundary_without_multileader_with_vote_scoring() - { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - - let (mut context, _) = Context::new_for_test(6); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1, 1, 1]); - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::new(); - for round in 1..=63 { - let mut this_round_blocks = Vec::new(); - - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - - for core_fixture in &mut cores { - // add the blocks from last round - // this will trigger a block creation for the round and a signal should be - // emitted - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // A "new round" signal should be received given that all the blocks of previous - // round have been processed - let new_round = receive( - Duration::from_secs(1), - core_fixture.signal_receivers.new_round_receiver(), - ) - .await; - assert_eq!(new_round, round); - - // Check that a new block has been proposed. - let extended_block = tokio::time::timeout( - Duration::from_secs(1), - core_fixture.block_receiver.recv(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), round); - assert_eq!( - extended_block.block.author(), - core_fixture.core.context.own_index - ); - - // append the new block to this round blocks - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - - let block = core_fixture.core.last_proposed_block(); - - // ensure that produced block is referring to the blocks of last_round - assert_eq!( - block.ancestors().len(), - core_fixture.core.context.committee.size() - ); - for ancestor in block.ancestors() { - if block.round() > 1 { - // don't bother with round 1 block which just contains the genesis blocks. - assert!( - last_round_blocks - .iter() - .any(|block| block.reference() == *ancestor), - "Reference from previous round should be added" - ); - } - } - } - - last_round_blocks = this_round_blocks; - } - - for core_fixture in cores { - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 61 leader rounds with rounds completed up to and including - // round 63. Round 63 blocks will only include their own blocks, so there - // should only be 60 commits. - // However on a leader schedule change boundary its is possible for a - // new leader to get selected for the same round if the leader elected - // gets swapped allowing for multiple leaders to be committed at a round. - // Meaning with multi leader per round explicitly set to 1 we will have 30, - // otherwise 61. - // NOTE: We used 61 leader rounds to specifically trigger the scenario - // where the leader schedule boundary occurred AND we had a swap to a new - // leader for the same round - let expected_commit_count = 60; - // Leave the code for re-use. - // let expected_commit_count = match num_leaders_per_round { - // Some(1) => 60, - // _ => 61, - //}; - assert_eq!(last_commit.index(), expected_commit_count); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), expected_commit_count as usize); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .bad_nodes - .len(), - 1 - ); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .good_nodes - .len(), - 1 - ); - let expected_reputation_scores = - ReputationScores::new((51..=60).into(), vec![8, 8, 9, 8, 8, 8]); - assert_eq!( - core_fixture - .core - .leader_schedule - .leader_swap_table - .read() - .reputation_scores, - expected_reputation_scores - ); - } - } - - #[tokio::test] - async fn test_core_signals() { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - - let (context, _) = Context::new_for_test(4); - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1]); - - // Now iterate over a few rounds and ensure the corresponding signals are - // created while network advances - let mut last_round_blocks = Vec::new(); - for round in 1..=10 { - let mut this_round_blocks = Vec::new(); - - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - - for core_fixture in &mut cores { - // add the blocks from last round - // this will trigger a block creation for the round and a signal should be - // emitted - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - - // A "new round" signal should be received given that all the blocks of previous - // round have been processed - let new_round = receive( - Duration::from_secs(1), - core_fixture.signal_receivers.new_round_receiver(), - ) - .await; - assert_eq!(new_round, round); - - // Check that a new block has been proposed. - let extended_block = tokio::time::timeout( - Duration::from_secs(1), - core_fixture.block_receiver.recv(), - ) - .await - .unwrap() - .unwrap(); - assert_eq!(extended_block.block.round(), round); - assert_eq!( - extended_block.block.author(), - core_fixture.core.context.own_index - ); - - // append the new block to this round blocks - this_round_blocks.push(core_fixture.core.last_proposed_block().clone()); - - let block = core_fixture.core.last_proposed_block(); - - // ensure that produced block is referring to the blocks of last_round - assert_eq!( - block.ancestors().len(), - core_fixture.core.context.committee.size() - ); - for ancestor in block.ancestors() { - if block.round() > 1 { - // don't bother with round 1 block which just contains the genesis blocks. - assert!( - last_round_blocks - .iter() - .any(|block| block.reference() == *ancestor), - "Reference from previous round should be added" - ); - } - } - } - - last_round_blocks = this_round_blocks; - } - - for core_fixture in cores { - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 8 leader rounds with rounds completed up to and including - // round 9. Round 10 blocks will only include their own blocks, so the - // 8th leader will not be committed. - assert_eq!(last_commit.index(), 7); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), 7); - } - } - - #[tokio::test] - async fn test_core_compress_proposal_references() { - telemetry_subscribers::init_for_testing(); - let default_params = Parameters::default(); - - let (context, _) = Context::new_for_test(4); - // create the cores and their signals for all the authorities - let mut cores = create_cores(context, vec![1, 1, 1, 1]); - - let mut last_round_blocks = Vec::new(); - let mut all_blocks = Vec::new(); - - let excluded_authority = AuthorityIndex::new_for_test(3); - - for round in 1..=10 { - let mut this_round_blocks = Vec::new(); - - for core_fixture in &mut cores { - // do not produce any block for authority 3 - if core_fixture.core.context.own_index == excluded_authority { - continue; - } - - // try to propose to ensure that we are covering the case where we miss the - // leader authority 3 - core_fixture - .core - .add_blocks(last_round_blocks.clone()) - .unwrap(); - core_fixture.core.new_block(round, true).unwrap(); - - let block = core_fixture.core.last_proposed_block(); - assert_eq!(block.round(), round); - - // append the new block to this round blocks - this_round_blocks.push(block.clone()); - } - - last_round_blocks = this_round_blocks.clone(); - all_blocks.extend(this_round_blocks); - } - - // Now send all the produced blocks to core of authority 3. It should produce a - // new block. If no compression would be applied the we should expect - // all the previous blocks to be referenced from round 0..=10. However, since - // compression is applied only the last round's (10) blocks should be - // referenced + the authority's block of round 0. - let core_fixture = &mut cores[excluded_authority]; - // Wait for min round delay to allow blocks to be proposed. - sleep(default_params.min_round_delay).await; - // add blocks to trigger proposal. - core_fixture.core.add_blocks(all_blocks).unwrap(); - - // Assert that a block has been created for round 11 and it references to blocks - // of round 10 for the other peers, and to round 1 for its own block - // (created after recovery). - let block = core_fixture.core.last_proposed_block(); - assert_eq!(block.round(), 11); - assert_eq!(block.ancestors().len(), 4); - for block_ref in block.ancestors() { - if block_ref.author == excluded_authority { - assert_eq!(block_ref.round, 1); - } else { - assert_eq!(block_ref.round, 10); - } - } - - // Check commits have been persisted to store - let last_commit = core_fixture - .store - .read_last_commit() - .unwrap() - .expect("last commit should be set"); - // There are 8 leader rounds with rounds completed up to and including - // round 10. However because there were no blocks produced for authority 3 - // 2 leader rounds will be skipped. - assert_eq!(last_commit.index(), 6); - let all_stored_commits = core_fixture - .store - .scan_commits((0..=CommitIndex::MAX).into()) - .unwrap(); - assert_eq!(all_stored_commits.len(), 6); - } - - #[tokio::test] - async fn try_decide_certified() { - // GIVEN - telemetry_subscribers::init_for_testing(); - - let (context, _) = Context::new_for_test(4); - - let authority_index = AuthorityIndex::new_for_test(0); - let core = CoreTextFixture::new(context.clone(), vec![1, 1, 1, 1], authority_index, true); - let mut core = core.core; - - let mut dag_builder = DagBuilder::new(Arc::new(context)); - dag_builder.layers(1..=12).build(); - - let limit = 2; - - let blocks = dag_builder.blocks(1..=12); - - for block in blocks { - core.dag_state.write().accept_block(block); - } - - // WHEN - let sub_dags_and_commits = dag_builder.get_sub_dag_and_certified_commits(1..=4); - let mut certified_commits = sub_dags_and_commits - .into_iter() - .map(|(_, commit)| commit) - .collect::>(); - - let leaders = core.try_decide_certified(&mut certified_commits, limit); - - // THEN - assert_eq!(leaders.len(), 2); - assert_eq!(certified_commits.len(), 2); - } - - pub(crate) async fn receive(timeout: Duration, mut receiver: watch::Receiver) -> T { - tokio::time::timeout(timeout, receiver.changed()) - .await - .expect("Timeout while waiting to read from receiver") - .expect("Signal receive channel shouldn't be closed"); - *receiver.borrow_and_update() - } -} diff --git a/consensus/core/src/core_thread.rs b/consensus/core/src/core_thread.rs deleted file mode 100644 index 5d15a959bf1..00000000000 --- a/consensus/core/src/core_thread.rs +++ /dev/null @@ -1,581 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, - sync::{ - Arc, - atomic::{AtomicU32, Ordering}, - }, -}; - -use async_trait::async_trait; -use consensus_config::AuthorityIndex; -use iota_metrics::{ - monitored_mpsc::{Receiver, Sender, WeakSender, channel}, - monitored_scope, spawn_logged_monitored_task, -}; -use parking_lot::RwLock; -use thiserror::Error; -use tokio::sync::{oneshot, watch}; -use tracing::warn; - -use crate::{ - BlockAPI as _, - block::{BlockRef, Round, VerifiedBlock}, - commit::CertifiedCommits, - context::Context, - core::Core, - core_thread::CoreError::Shutdown, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - round_prober::QuorumRound, -}; - -const CORE_THREAD_COMMANDS_CHANNEL_SIZE: usize = 2000; - -enum CoreThreadCommand { - /// Add blocks to be processed and accepted - AddBlocks(Vec, oneshot::Sender>), - /// Checks if block refs exist locally and sync missing ones. - CheckBlockRefs(Vec, oneshot::Sender>), - /// Add committed sub dag blocks for processing and acceptance. - AddCertifiedCommits(CertifiedCommits, oneshot::Sender>), - /// Called when the min round has passed or the leader timeout occurred and - /// a block should be produced. When the command is called with `force = - /// true`, then the block will be created for `round` skipping - /// any checks (ex leader existence of previous round). More information can - /// be found on the `Core` component. - NewBlock(Round, oneshot::Sender<()>, bool), - /// Request missing blocks that need to be synced together with authorities - /// that have these blocks. - GetMissingBlocks(oneshot::Sender>>), -} - -#[derive(Error, Debug)] -pub enum CoreError { - #[error("Core thread shutdown: {0}")] - Shutdown(String), -} - -/// The interface to dispatch commands to CoreThread and Core. -/// Also this allows the easier mocking during unit tests. -#[async_trait] -pub trait CoreThreadDispatcher: Sync + Send + 'static { - async fn add_blocks(&self, blocks: Vec) - -> Result, CoreError>; - - async fn check_block_refs( - &self, - block_refs: Vec, - ) -> Result, CoreError>; - - async fn add_certified_commits( - &self, - commits: CertifiedCommits, - ) -> Result, CoreError>; - - async fn new_block(&self, round: Round, force: bool) -> Result<(), CoreError>; - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError>; - - /// Informs the core whether consumer of produced blocks exists. - /// This is only used by core to decide if it should propose new blocks. - /// It is not a guarantee that produced blocks will be accepted by peers. - fn set_quorum_subscribers_exists(&self, exists: bool) -> Result<(), CoreError>; - - /// Sets the estimated delay to propagate a block to a quorum of peers, in - /// number of rounds, and the received & accepted quorum rounds for all - /// authorities. - fn set_propagation_delay_and_quorum_rounds( - &self, - delay: Round, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError>; - - fn set_last_known_proposed_round(&self, round: Round) -> Result<(), CoreError>; - - /// Returns the highest round received for each authority by Core. - fn highest_received_rounds(&self) -> Vec; -} - -pub(crate) struct CoreThreadHandle { - sender: Sender, - join_handle: tokio::task::JoinHandle<()>, -} - -impl CoreThreadHandle { - pub async fn stop(self) { - // drop the sender, that will force all the other weak senders to not able to - // upgrade. - drop(self.sender); - self.join_handle.await.ok(); - } -} - -struct CoreThread { - core: Core, - receiver: Receiver, - rx_quorum_subscribers_exists: watch::Receiver, - rx_propagation_delay_and_quorum_rounds: watch::Receiver, - rx_last_known_proposed_round: watch::Receiver, - context: Arc, -} - -impl CoreThread { - pub async fn run(mut self) -> ConsensusResult<()> { - tracing::debug!("Started core thread"); - - loop { - tokio::select! { - command = self.receiver.recv() => { - let Some(command) = command else { - break; - }; - self.context.metrics.node_metrics.core_lock_dequeued.inc(); - match command { - CoreThreadCommand::AddBlocks(blocks, sender) => { - let _scope = monitored_scope("CoreThread::loop::add_blocks"); - let missing_block_refs = self.core.add_blocks(blocks)?; - sender.send(missing_block_refs).ok(); - } - CoreThreadCommand::CheckBlockRefs(blocks, sender) => { - let _scope = monitored_scope("CoreThread::loop::find_excluded_blocks"); - let missing_block_refs = self.core.check_block_refs(blocks)?; - sender.send(missing_block_refs).ok(); - } - CoreThreadCommand::AddCertifiedCommits(commits, sender) => { - let _scope = monitored_scope("CoreThread::loop::add_certified_commits"); - let missing_block_refs = self.core.add_certified_commits(commits)?; - sender.send(missing_block_refs).ok(); - } - CoreThreadCommand::NewBlock(round, sender, force) => { - let _scope = monitored_scope("CoreThread::loop::new_block"); - self.core.new_block(round, force)?; - sender.send(()).ok(); - } - CoreThreadCommand::GetMissingBlocks(sender) => { - let _scope = monitored_scope("CoreThread::loop::get_missing_blocks"); - sender.send(self.core.get_missing_blocks()).ok(); - } - } - } - _ = self.rx_last_known_proposed_round.changed() => { - let _scope = monitored_scope("CoreThread::loop::set_last_known_proposed_round"); - let round = *self.rx_last_known_proposed_round.borrow(); - self.core.set_last_known_proposed_round(round); - self.core.new_block(round + 1, true)?; - } - _ = self.rx_quorum_subscribers_exists.changed() => { - let _scope = monitored_scope("CoreThread::loop::set_subscriber_exists"); - let should_propose_before = self.core.should_propose(); - let exists = *self.rx_quorum_subscribers_exists.borrow(); - self.core.set_quorum_subscribers_exists(exists); - if !should_propose_before && self.core.should_propose() { - // If core cannot propose before but can propose now, try to produce a new block to ensure liveness, - // because block proposal could have been skipped. - self.core.new_block(Round::MAX, true)?; - } - } - _ = self.rx_propagation_delay_and_quorum_rounds.changed() => { - let _scope = monitored_scope("CoreThread::loop::set_propagation_delay_and_quorum_rounds"); - let should_propose_before = self.core.should_propose(); - let state = self.rx_propagation_delay_and_quorum_rounds.borrow().clone(); - self.core.set_propagation_delay_and_quorum_rounds( - state.delay, - state.received_quorum_rounds, - state.accepted_quorum_rounds - ); - if !should_propose_before && self.core.should_propose() { - // If core cannot propose before but can propose now, try to produce a new block to ensure liveness, - // because block proposal could have been skipped. - self.core.new_block(Round::MAX, true)?; - } - } - } - } - - Ok(()) - } -} - -#[derive(Clone)] -pub(crate) struct ChannelCoreThreadDispatcher { - context: Arc, - sender: WeakSender, - tx_quorum_subscribers_exists: Arc>, - tx_propagation_delay_and_quorum_rounds: Arc>, - tx_last_known_proposed_round: Arc>, - highest_received_rounds: Arc>, -} - -impl ChannelCoreThreadDispatcher { - /// Starts the core thread for the consensus authority and returns a - /// dispatcher and handle for managing the core thread. - pub(crate) fn start( - context: Arc, - dag_state: &RwLock, - core: Core, - ) -> (Self, CoreThreadHandle) { - // Initialize highest received rounds. - let highest_received_rounds = { - let dag_state = dag_state.read(); - let highest_received_rounds = context - .committee - .authorities() - .map(|(index, _)| { - AtomicU32::new(dag_state.get_last_block_for_authority(index).round()) - }) - .collect(); - - highest_received_rounds - }; - let (sender, receiver) = - channel("consensus_core_commands", CORE_THREAD_COMMANDS_CHANNEL_SIZE); - let (tx_quorum_subscribers_exists, mut rx_quorum_subscriber_exists) = watch::channel(false); - let (tx_propagation_delay_and_quorum_rounds, mut rx_propagation_delay_and_quorum_rounds) = - watch::channel(PropagationDelayAndQuorumRounds { - delay: 0, - received_quorum_rounds: vec![(0, 0); context.committee.size()], - accepted_quorum_rounds: vec![(0, 0); context.committee.size()], - }); - let (tx_last_known_proposed_round, mut rx_last_known_proposed_round) = watch::channel(0); - rx_quorum_subscriber_exists.mark_unchanged(); - rx_propagation_delay_and_quorum_rounds.mark_unchanged(); - rx_last_known_proposed_round.mark_unchanged(); - let core_thread = CoreThread { - core, - receiver, - rx_quorum_subscribers_exists: rx_quorum_subscriber_exists, - rx_propagation_delay_and_quorum_rounds, - rx_last_known_proposed_round, - context: context.clone(), - }; - - let join_handle = spawn_logged_monitored_task!( - async move { - if let Err(err) = core_thread.run().await { - if !matches!(err, ConsensusError::Shutdown) { - panic!("Fatal error occurred: {err}"); - } - } - }, - "ConsensusCoreThread" - ); - - // Explicitly using downgraded sender in order to allow sharing the - // CoreThreadDispatcher but able to shutdown the CoreThread by dropping - // the original sender. - let dispatcher = ChannelCoreThreadDispatcher { - context, - sender: sender.downgrade(), - tx_quorum_subscribers_exists: Arc::new(tx_quorum_subscribers_exists), - tx_propagation_delay_and_quorum_rounds: Arc::new( - tx_propagation_delay_and_quorum_rounds, - ), - tx_last_known_proposed_round: Arc::new(tx_last_known_proposed_round), - highest_received_rounds: Arc::new(highest_received_rounds), - }; - let handle = CoreThreadHandle { - join_handle, - sender, - }; - (dispatcher, handle) - } - - async fn send(&self, command: CoreThreadCommand) { - self.context.metrics.node_metrics.core_lock_enqueued.inc(); - if let Some(sender) = self.sender.upgrade() { - if let Err(err) = sender.send(command).await { - warn!( - "Couldn't send command to core thread, probably is shutting down: {}", - err - ); - } - } - } -} - -#[async_trait] -impl CoreThreadDispatcher for ChannelCoreThreadDispatcher { - async fn add_blocks( - &self, - blocks: Vec, - ) -> Result, CoreError> { - for block in &blocks { - self.highest_received_rounds[block.author()].fetch_max(block.round(), Ordering::AcqRel); - } - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::AddBlocks(blocks.clone(), sender)) - .await; - let missing_block_refs = receiver.await.map_err(|e| Shutdown(e.to_string()))?; - - Ok(missing_block_refs) - } - - async fn check_block_refs( - &self, - block_refs: Vec, - ) -> Result, CoreError> { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::CheckBlockRefs( - block_refs.clone(), - sender, - )) - .await; - let missing_block_refs = receiver.await.map_err(|e| Shutdown(e.to_string()))?; - - Ok(missing_block_refs) - } - - async fn add_certified_commits( - &self, - commits: CertifiedCommits, - ) -> Result, CoreError> { - for commit in commits.commits() { - for block in commit.blocks() { - self.highest_received_rounds[block.author()] - .fetch_max(block.round(), Ordering::AcqRel); - } - } - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::AddCertifiedCommits(commits, sender)) - .await; - let missing_block_refs = receiver.await.map_err(|e| Shutdown(e.to_string()))?; - Ok(missing_block_refs) - } - - async fn new_block(&self, round: Round, force: bool) -> Result<(), CoreError> { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::NewBlock(round, sender, force)) - .await; - receiver.await.map_err(|e| Shutdown(e.to_string())) - } - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - let (sender, receiver) = oneshot::channel(); - self.send(CoreThreadCommand::GetMissingBlocks(sender)).await; - receiver.await.map_err(|e| Shutdown(e.to_string())) - } - - fn set_quorum_subscribers_exists(&self, exists: bool) -> Result<(), CoreError> { - self.tx_quorum_subscribers_exists - .send(exists) - .map_err(|e| Shutdown(e.to_string())) - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - delay: Round, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - self.tx_propagation_delay_and_quorum_rounds - .send(PropagationDelayAndQuorumRounds { - delay, - received_quorum_rounds, - accepted_quorum_rounds, - }) - .map_err(|e| Shutdown(e.to_string())) - } - - fn set_last_known_proposed_round(&self, round: Round) -> Result<(), CoreError> { - self.tx_last_known_proposed_round - .send(round) - .map_err(|e| Shutdown(e.to_string())) - } - - fn highest_received_rounds(&self) -> Vec { - self.highest_received_rounds - .iter() - .map(|round| round.load(Ordering::Relaxed)) - .collect() - } -} - -#[derive(Clone)] -struct PropagationDelayAndQuorumRounds { - delay: Round, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, -} - -#[cfg(test)] -pub(crate) mod tests { - use iota_metrics::monitored_mpsc::unbounded_channel; - use parking_lot::RwLock; - - use super::*; - use crate::{ - CommitConsumer, - block_manager::BlockManager, - block_verifier::NoopBlockVerifier, - commit_observer::CommitObserver, - context::Context, - core::CoreSignals, - dag_state::DagState, - leader_schedule::LeaderSchedule, - storage::mem_store::MemStore, - transaction::{TransactionClient, TransactionConsumer}, - }; - - // TODO: complete the Mock for thread dispatcher to be used from several tests - #[derive(Default)] - pub(crate) struct MockCoreThreadDispatcher { - add_blocks: parking_lot::Mutex>, - missing_blocks: parking_lot::Mutex>>, - last_known_proposed_round: parking_lot::Mutex>, - } - - impl MockCoreThreadDispatcher { - pub(crate) async fn get_add_blocks(&self) -> Vec { - let mut add_blocks = self.add_blocks.lock(); - add_blocks.drain(0..).collect() - } - - pub(crate) async fn stub_missing_blocks(&self, block_refs: BTreeSet) { - let mut missing_blocks = self.missing_blocks.lock(); - for block_ref in &block_refs { - missing_blocks.insert(*block_ref, BTreeSet::from([block_ref.author])); - } - } - - pub(crate) async fn get_last_own_proposed_round(&self) -> Vec { - let last_known_proposed_round = self.last_known_proposed_round.lock(); - last_known_proposed_round.clone() - } - } - - #[async_trait] - impl CoreThreadDispatcher for MockCoreThreadDispatcher { - async fn add_blocks( - &self, - blocks: Vec, - ) -> Result, CoreError> { - let mut add_blocks = self.add_blocks.lock(); - add_blocks.extend(blocks); - Ok(BTreeSet::new()) - } - - async fn check_block_refs( - &self, - _block_refs: Vec, - ) -> Result, CoreError> { - Ok(BTreeSet::new()) - } - - async fn add_certified_commits( - &self, - _commits: CertifiedCommits, - ) -> Result, CoreError> { - todo!() - } - - async fn new_block(&self, _round: Round, _force: bool) -> Result<(), CoreError> { - Ok(()) - } - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - let mut missing_blocks = self.missing_blocks.lock(); - let result = missing_blocks.clone(); - missing_blocks.clear(); - Ok(result) - } - - fn set_quorum_subscribers_exists(&self, _exists: bool) -> Result<(), CoreError> { - todo!() - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - _delay: Round, - _received_quorum_rounds: Vec, - _accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - todo!() - } - - fn set_last_known_proposed_round(&self, round: Round) -> Result<(), CoreError> { - let mut last_known_proposed_round = self.last_known_proposed_round.lock(); - last_known_proposed_round.push(round); - Ok(()) - } - - fn highest_received_rounds(&self) -> Vec { - todo!() - } - } - - #[tokio::test] - async fn test_core_thread() { - telemetry_subscribers::init_for_testing(); - let (context, mut key_pairs) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - let (_transaction_client, tx_receiver) = TransactionClient::new(context.clone()); - let transaction_consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let (signals, signal_receivers) = CoreSignals::new(context.clone()); - let _block_receiver = signal_receivers.block_broadcast_receiver(); - let (sender, _receiver) = unbounded_channel("consensus_output"); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - let commit_observer = CommitObserver::new( - context.clone(), - CommitConsumer::new(sender.clone(), 0), - dag_state.clone(), - store, - leader_schedule.clone(), - ); - let leader_schedule = Arc::new(LeaderSchedule::from_store( - context.clone(), - dag_state.clone(), - )); - let core = Core::new( - context.clone(), - leader_schedule, - transaction_consumer, - block_manager, - true, - commit_observer, - signals, - key_pairs.remove(context.own_index.value()).1, - dag_state.clone(), - false, - ); - - let (core_dispatcher, handle) = - ChannelCoreThreadDispatcher::start(context, &dag_state, core); - - // Now create some clones of the dispatcher - let dispatcher_1 = core_dispatcher.clone(); - let dispatcher_2 = core_dispatcher.clone(); - - // Try to send some commands - assert!(dispatcher_1.add_blocks(vec![]).await.is_ok()); - assert!(dispatcher_2.add_blocks(vec![]).await.is_ok()); - - // Now shutdown the dispatcher - handle.stop().await; - - // Try to send some commands - assert!(dispatcher_1.add_blocks(vec![]).await.is_err()); - assert!(dispatcher_2.add_blocks(vec![]).await.is_err()); - } -} diff --git a/consensus/core/src/dag_state.rs b/consensus/core/src/dag_state.rs deleted file mode 100644 index 3510d0212c4..00000000000 --- a/consensus/core/src/dag_state.rs +++ /dev/null @@ -1,2770 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - cmp::max, - collections::{BTreeMap, BTreeSet, VecDeque}, - ops::Bound::{Excluded, Included, Unbounded}, - panic, - sync::Arc, - vec, -}; - -use consensus_config::AuthorityIndex; -use itertools::Itertools as _; -use tokio::time::Instant; -use tracing::{debug, info, trace, warn}; - -use crate::{ - CommittedSubDag, - block::{ - BlockAPI, BlockDigest, BlockRef, BlockTimestampMs, GENESIS_ROUND, Round, Slot, - VerifiedBlock, genesis_blocks, - }, - commit::{ - CommitAPI as _, CommitDigest, CommitIndex, CommitInfo, CommitRef, CommitVote, - GENESIS_COMMIT_INDEX, TrustedCommit, load_committed_subdag_from_store, - }, - context::Context, - leader_scoring::{ReputationScores, ScoringSubdag}, - storage::{Store, WriteBatch}, - threshold_clock::ThresholdClock, -}; - -/// DagState provides the API to write and read accepted blocks from the DAG. -/// Only uncommitted and last committed blocks are cached in memory. -/// The rest of blocks are stored on disk. -/// Refs to cached blocks and additional refs are cached as well, to speed up -/// existence checks. -/// -/// Note: DagState should be wrapped with Arc>, to allow -/// concurrent access from multiple components. -pub(crate) struct DagState { - context: Arc, - - // The genesis blocks - genesis: BTreeMap, - - // Contains recent blocks within CACHED_ROUNDS from the last committed round per authority. - // Note: all uncommitted blocks are kept in memory. - // - // When GC is enabled, this map has a different semantic. It holds all the recent data for each - // authority making sure that it always have available CACHED_ROUNDS worth of data. The - // entries are evicted based on the latest GC round, however the eviction process will respect - // the CACHED_ROUNDS. For each authority, blocks are only evicted when their round is less - // than or equal to both `gc_round`, and `highest authority round - cached rounds`. - // This ensures that the GC requirements are respected (we never clean up any block above - // `gc_round`), and there are enough blocks cached. - recent_blocks: BTreeMap, - - // Indexes recent block refs by their authorities. - // Vec position corresponds to the authority index. - recent_refs_by_authority: Vec>, - - // Keeps track of the threshold clock for proposing blocks. - threshold_clock: ThresholdClock, - - // Keeps track of the highest round that has been evicted for each authority. Any blocks that - // are of round <= evict_round should be considered evicted, and if any exist we should not - // consider the causauly complete in the order they appear. The `evicted_rounds` size - // should be the same as the committee size. - evicted_rounds: Vec, - - // Highest round of blocks accepted. - highest_accepted_round: Round, - - // Last consensus commit of the dag. - last_commit: Option, - - // Last wall time when commit round advanced. Does not persist across restarts. - last_commit_round_advancement_time: Option, - - // Last committed rounds per authority. - last_committed_rounds: Vec, - - /// The committed subdags that have been scored but scores have not been - /// used for leader schedule yet. - scoring_subdag: ScoringSubdag, - // TODO: Remove when DistributedVoteScoring is enabled. - /// The list of committed subdags that have been sequenced by the universal - /// committer but have yet to be used to calculate reputation scores for the - /// next leader schedule. Until then we consider it as "unscored" subdags. - unscored_committed_subdags: Vec, - - // Commit votes pending to be included in new blocks. - // TODO: limit to 1st commit per round with multi-leader. - pending_commit_votes: VecDeque, - - // Data to be flushed to storage. - blocks_to_write: Vec, - commits_to_write: Vec, - - // Buffer the reputation scores & last_committed_rounds to be flushed with the - // next dag state flush. This is okay because we can recover reputation scores - // & last_committed_rounds from the commits as needed. - commit_info_to_write: Vec<(CommitRef, CommitInfo)>, - - // Persistent storage for blocks, commits and other consensus data. - store: Arc, - - // The number of cached rounds - cached_rounds: Round, -} - -impl DagState { - /// Initializes DagState from storage. - pub(crate) fn new(context: Arc, store: Arc) -> Self { - let cached_rounds = context.parameters.dag_state_cached_rounds as Round; - let num_authorities = context.committee.size(); - - let genesis = genesis_blocks(&context) - .into_iter() - .map(|block| (block.reference(), block)) - .collect(); - - let threshold_clock = ThresholdClock::new(1, context.clone()); - - let last_commit = store - .read_last_commit() - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")); - - let commit_info = store - .read_last_commit_info() - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")); - let (mut last_committed_rounds, commit_recovery_start_index) = - if let Some((commit_ref, commit_info)) = commit_info { - tracing::info!("Recovering committed state from {commit_ref} {commit_info:?}"); - (commit_info.committed_rounds, commit_ref.index + 1) - } else { - tracing::info!("Found no stored CommitInfo to recover from"); - (vec![0; num_authorities], GENESIS_COMMIT_INDEX + 1) - }; - - let mut unscored_committed_subdags = Vec::new(); - let mut scoring_subdag = ScoringSubdag::new(context.clone()); - - if let Some(last_commit) = last_commit.as_ref() { - store - .scan_commits((commit_recovery_start_index..=last_commit.index()).into()) - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")) - .iter() - .for_each(|commit| { - for block_ref in commit.blocks() { - last_committed_rounds[block_ref.author] = - max(last_committed_rounds[block_ref.author], block_ref.round); - } - - let committed_subdag = - load_committed_subdag_from_store(store.as_ref(), commit.clone(), vec![]); // We don't need to recover reputation scores for unscored_committed_subdags - unscored_committed_subdags.push(committed_subdag); - }); - } - - tracing::info!( - "DagState was initialized with the following state: \ - {last_commit:?}; {last_committed_rounds:?}; {} unscored committed subdags;", - unscored_committed_subdags.len() - ); - - if context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - scoring_subdag.add_subdags(std::mem::take(&mut unscored_committed_subdags)); - } - - let mut state = Self { - context, - genesis, - recent_blocks: BTreeMap::new(), - recent_refs_by_authority: vec![BTreeSet::new(); num_authorities], - threshold_clock, - highest_accepted_round: 0, - last_commit: last_commit.clone(), - last_commit_round_advancement_time: None, - last_committed_rounds: last_committed_rounds.clone(), - pending_commit_votes: VecDeque::new(), - blocks_to_write: vec![], - commits_to_write: vec![], - commit_info_to_write: vec![], - scoring_subdag, - unscored_committed_subdags, - store: store.clone(), - cached_rounds, - evicted_rounds: vec![0; num_authorities], - }; - - for (i, round) in last_committed_rounds.into_iter().enumerate() { - let authority_index = state.context.committee.to_authority_index(i).unwrap(); - let (blocks, eviction_round) = if state.gc_enabled() { - // Find the latest block for the authority to calculate the eviction round. Then - // we want to scan and load the blocks from the eviction round and onwards only. - // As reminder, the eviction round is taking into account the gc_round. - let last_block = state - .store - .scan_last_blocks_by_author(authority_index, 1, None) - .expect("Database error"); - let last_block_round = last_block - .last() - .map(|b| b.round()) - .unwrap_or(GENESIS_ROUND); - - let eviction_round = Self::gc_eviction_round( - last_block_round, - state.gc_round(), - state.cached_rounds, - ); - let blocks = state - .store - .scan_blocks_by_author(authority_index, eviction_round + 1) - .expect("Database error"); - (blocks, eviction_round) - } else { - let eviction_round = Self::eviction_round(round, cached_rounds); - let blocks = state - .store - .scan_blocks_by_author(authority_index, eviction_round + 1) - .expect("Database error"); - (blocks, eviction_round) - }; - - state.evicted_rounds[authority_index] = eviction_round; - - // Update the block metadata for the authority. - for block in &blocks { - state.update_block_metadata(block); - } - - info!( - "Recovered blocks {}: {:?}", - authority_index, - blocks - .iter() - .map(|b| b.reference()) - .collect::>() - ); - } - - // Initialize scoring metrics according to the metrics in store and the blocks - // that were loaded to cache. - let recovered_scoring_metrics = state.store.scan_scoring_metrics().expect("Database error"); - state - .context - .scoring_metrics_store - .initialize_scoring_metrics( - recovered_scoring_metrics, - &state.recent_refs_by_authority, - state.threshold_clock_round(), - &state.evicted_rounds, - state.context.clone(), - ); - - if state.gc_enabled() { - if let Some(last_commit) = last_commit { - let mut index = last_commit.index(); - let gc_round = state.gc_round(); - info!( - "Recovering block commit statuses from commit index {} and backwards until leader of round <= gc_round {:?}", - index, gc_round - ); - - loop { - let commits = store - .scan_commits((index..=index).into()) - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")); - let Some(commit) = commits.first() else { - info!( - "Recovering finished up to index {index}, no more commits to recover" - ); - break; - }; - - // Check the commit leader round to see if it is within the gc_round. If it is - // not then we can stop the recovery process. - if gc_round > 0 && commit.leader().round <= gc_round { - info!( - "Recovering finished, reached commit leader round {} <= gc_round {}", - commit.leader().round, - gc_round - ); - break; - } - - commit.blocks().iter().filter(|b| b.round > gc_round).for_each(|block_ref|{ - debug!( - "Setting block {:?} as committed based on commit {:?}", - block_ref, - commit.index() - ); - assert!(state.set_committed(block_ref), "Attempted to set again a block {block_ref:?} as committed when recovering commit {commit:?}"); - }); - - // All commits are indexed starting from 1, so one reach zero exit. - index = index.saturating_sub(1); - if index == 0 { - break; - } - } - } - } - - state - } - - /// Accepts a block into DagState and keeps it in memory. - pub(crate) fn accept_block(&mut self, block: VerifiedBlock) { - assert_ne!( - block.round(), - 0, - "Genesis block should not be accepted into DAG." - ); - - let block_ref = block.reference(); - if self.contains_block(&block_ref) { - return; - } - - let now = self.context.clock.timestamp_utc_ms(); - if block.timestamp_ms() > now { - if self - .context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - trace!( - "Block {:?} with timestamp {} is greater than local timestamp {}.", - block, - block.timestamp_ms(), - now, - ); - } else { - panic!( - "Block {:?} cannot be accepted! Block timestamp {} is greater than local timestamp {}.", - block, - block.timestamp_ms(), - now, - ); - } - } - let hostname = &self.context.committee.authority(block_ref.author).hostname; - self.context - .metrics - .node_metrics - .accepted_block_time_drift_ms - .with_label_values(&[hostname]) - .inc_by(block.timestamp_ms().saturating_sub(now)); - - // TODO: Move this check to core - // Ensure we don't write multiple blocks per slot for our own index - if block_ref.author == self.context.own_index { - let existing_blocks = self.get_uncommitted_blocks_at_slot(block_ref.into()); - assert!( - existing_blocks.is_empty(), - "Block Rejected! Attempted to add block {block:#?} to own slot where \ - block(s) {existing_blocks:#?} already exists." - ); - } - self.update_block_metadata(&block); - self.blocks_to_write.push(block); - let source = if self.context.own_index == block_ref.author { - "own" - } else { - "others" - }; - self.context - .metrics - .node_metrics - .accepted_blocks - .with_label_values(&[source]) - .inc(); - } - - /// Updates internal metadata for a block. - fn update_block_metadata(&mut self, block: &VerifiedBlock) { - let block_ref = block.reference(); - self.recent_blocks - .insert(block_ref, BlockInfo::new(block.clone())); - self.recent_refs_by_authority[block_ref.author].insert(block_ref); - self.threshold_clock.add_block(block_ref); - self.highest_accepted_round = max(self.highest_accepted_round, block.round()); - self.context - .metrics - .node_metrics - .highest_accepted_round - .set(self.highest_accepted_round as i64); - - let highest_accepted_round_for_author = self.recent_refs_by_authority[block_ref.author] - .last() - .map(|block_ref| block_ref.round) - .expect("There should be by now at least one block ref"); - let hostname = &self.context.committee.authority(block_ref.author).hostname; - self.context - .metrics - .node_metrics - .highest_accepted_authority_round - .with_label_values(&[hostname]) - .set(highest_accepted_round_for_author as i64); - } - - /// Accepts a blocks into DagState and keeps it in memory. - pub(crate) fn accept_blocks(&mut self, blocks: Vec) { - debug!( - "Accepting blocks: {}", - blocks.iter().map(|b| b.reference().to_string()).join(",") - ); - for block in blocks { - self.accept_block(block); - } - } - - /// Gets a block by checking cached recent blocks then storage. - /// Returns None when the block is not found. - pub(crate) fn get_block(&self, reference: &BlockRef) -> Option { - self.get_blocks(&[*reference]) - .pop() - .expect("Exactly one element should be returned") - } - - /// Gets blocks by checking genesis, cached recent blocks in memory, then - /// storage. An element is None when the corresponding block is not - /// found. - pub(crate) fn get_blocks(&self, block_refs: &[BlockRef]) -> Vec> { - let mut blocks = vec![None; block_refs.len()]; - let mut missing = Vec::new(); - - for (index, block_ref) in block_refs.iter().enumerate() { - if block_ref.round == GENESIS_ROUND { - // Allow the caller to handle the invalid genesis ancestor error. - if let Some(block) = self.genesis.get(block_ref) { - blocks[index] = Some(block.clone()); - } - continue; - } - if let Some(block_info) = self.recent_blocks.get(block_ref) { - blocks[index] = Some(block_info.block.clone()); - continue; - } - missing.push((index, block_ref)); - } - - if missing.is_empty() { - return blocks; - } - - let missing_refs = missing - .iter() - .map(|(_, block_ref)| **block_ref) - .collect::>(); - let store_results = self - .store - .read_blocks(&missing_refs) - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")); - self.context - .metrics - .node_metrics - .dag_state_store_read_count - .with_label_values(&["get_blocks"]) - .inc(); - - for ((index, _), result) in missing.into_iter().zip(store_results) { - blocks[index] = result; - } - - blocks - } - - // Sets the block as committed in the cache. If the block is set as committed - // for first time, then true is returned, otherwise false is returned instead. - // Method will panic if the block is not found in the cache. - pub(crate) fn set_committed(&mut self, block_ref: &BlockRef) -> bool { - if let Some(block_info) = self.recent_blocks.get_mut(block_ref) { - if !block_info.committed { - block_info.committed = true; - return true; - } - false - } else { - panic!("Block {block_ref:?} not found in cache to set as committed."); - } - } - - pub(crate) fn is_committed(&self, block_ref: &BlockRef) -> bool { - self.recent_blocks - .get(block_ref) - .unwrap_or_else(|| panic!("Attempted to query for commit status for a block not in cached data {block_ref}")) - .committed - } - - /// Gets all uncommitted blocks in a slot. - /// Uncommitted blocks must exist in memory, so only in-memory blocks are - /// checked. - pub(crate) fn get_uncommitted_blocks_at_slot(&self, slot: Slot) -> Vec { - // TODO: either panic below when the slot is at or below the last committed - // round, or support reading from storage while limiting storage reads - // to edge cases. - - let mut blocks = vec![]; - for (_block_ref, block_info) in self.recent_blocks.range(( - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MIN)), - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MAX)), - )) { - blocks.push(block_info.block.clone()) - } - blocks - } - - /// Gets all uncommitted blocks in a round. - /// Uncommitted blocks must exist in memory, so only in-memory blocks are - /// checked. - pub(crate) fn get_uncommitted_blocks_at_round(&self, round: Round) -> Vec { - if round <= self.last_commit_round() { - panic!("Round {round} have committed blocks!"); - } - - let mut blocks = vec![]; - for (_block_ref, block_info) in self.recent_blocks.range(( - Included(BlockRef::new(round, AuthorityIndex::ZERO, BlockDigest::MIN)), - Excluded(BlockRef::new( - round + 1, - AuthorityIndex::ZERO, - BlockDigest::MIN, - )), - )) { - blocks.push(block_info.block.clone()) - } - blocks - } - - /// Gets all ancestors in the history of a block at a certain round. - pub(crate) fn ancestors_at_round( - &self, - later_block: &VerifiedBlock, - earlier_round: Round, - ) -> Vec { - // Iterate through ancestors of later_block in round descending order. - let mut linked: BTreeSet = later_block.ancestors().iter().cloned().collect(); - while !linked.is_empty() { - let round = linked.last().unwrap().round; - // Stop after finishing traversal for ancestors above earlier_round. - if round <= earlier_round { - break; - } - let block_ref = linked.pop_last().unwrap(); - let Some(block) = self.get_block(&block_ref) else { - panic!("Block {block_ref:?} should exist in DAG!"); - }; - linked.extend(block.ancestors().iter().cloned()); - } - linked - .range(( - Included(BlockRef::new( - earlier_round, - AuthorityIndex::ZERO, - BlockDigest::MIN, - )), - Unbounded, - )) - .map(|r| { - self.get_block(r) - .unwrap_or_else(|| panic!("Block {r:?} should exist in DAG!")) - }) - .collect() - } - - /// Gets the last proposed block from this authority. - /// If no block is proposed yet, returns the genesis block. - pub(crate) fn get_last_proposed_block(&self) -> VerifiedBlock { - self.get_last_block_for_authority(self.context.own_index) - } - - /// Retrieves the last accepted block from the specified `authority`. If no - /// block is found in cache then the genesis block is returned as no other - /// block has been received from that authority. - pub(crate) fn get_last_block_for_authority(&self, authority: AuthorityIndex) -> VerifiedBlock { - if let Some(last) = self.recent_refs_by_authority[authority].last() { - return self - .recent_blocks - .get(last) - .expect("Block should be found in recent blocks") - .block - .clone(); - } - - // if none exists, then fallback to genesis - let (_, genesis_block) = self - .genesis - .iter() - .find(|(block_ref, _)| block_ref.author == authority) - .expect("Genesis should be found for authority {authority_index}"); - genesis_block.clone() - } - - /// Returns cached recent blocks from the specified authority. - /// Blocks returned are limited to round >= `start`, and cached. - /// NOTE: caller should not assume returned blocks are always chained. - /// "Disconnected" blocks can be returned when there are byzantine blocks, - /// or a previously evicted block is accepted again. - pub(crate) fn get_cached_blocks( - &self, - authority: AuthorityIndex, - start: Round, - ) -> Vec { - self.get_cached_blocks_in_range(authority, start, Round::MAX, usize::MAX) - } - - // Retrieves the cached block within the range [start_round, end_round) from a - // given authority, limited in total number of blocks. - pub(crate) fn get_cached_blocks_in_range( - &self, - authority: AuthorityIndex, - start_round: Round, - end_round: Round, - limit: usize, - ) -> Vec { - if start_round >= end_round || limit == 0 { - return vec![]; - } - - let mut blocks = vec![]; - for block_ref in self.recent_refs_by_authority[authority].range(( - Included(BlockRef::new(start_round, authority, BlockDigest::MIN)), - Excluded(BlockRef::new( - end_round, - AuthorityIndex::MIN, - BlockDigest::MIN, - )), - )) { - let block_info = self - .recent_blocks - .get(block_ref) - .expect("Block should exist in recent blocks"); - blocks.push(block_info.block.clone()); - if blocks.len() >= limit { - break; - } - } - blocks - } - - // Retrieves the cached block within the range [start_round, end_round) from a - // given authority. NOTE: end_round must be greater than GENESIS_ROUND. - pub(crate) fn get_last_cached_block_in_range( - &self, - authority: AuthorityIndex, - start_round: Round, - end_round: Round, - ) -> Option { - if start_round >= end_round { - return None; - } - - let block_ref = self.recent_refs_by_authority[authority] - .range(( - Included(BlockRef::new(start_round, authority, BlockDigest::MIN)), - Excluded(BlockRef::new( - end_round, - AuthorityIndex::MIN, - BlockDigest::MIN, - )), - )) - .last()?; - - self.recent_blocks - .get(block_ref) - .map(|block_info| block_info.block.clone()) - } - - /// Returns the last block proposed per authority with `evicted round < - /// round < end_round`. The method is guaranteed to return results only - /// when the `end_round` is not earlier of the available cached data for - /// each authority (evicted round + 1), otherwise the method will panic. - /// It's the caller's responsibility to ensure that is not requesting for - /// earlier rounds. In case of equivocation for an authority's last - /// slot, one block will be returned (the last in order) and the other - /// equivocating blocks will be returned. - pub(crate) fn get_last_cached_block_per_authority( - &self, - end_round: Round, - ) -> Vec<(VerifiedBlock, Vec)> { - // Initialize with the genesis blocks as fallback - let mut blocks = self.genesis.values().cloned().collect::>(); - let mut equivocating_blocks = vec![vec![]; self.context.committee.size()]; - - if end_round == GENESIS_ROUND { - panic!( - "Attempted to retrieve blocks earlier than the genesis round which is not possible" - ); - } - - if end_round == GENESIS_ROUND + 1 { - return blocks.into_iter().map(|b| (b, vec![])).collect(); - } - - for (authority_index, block_refs) in self.recent_refs_by_authority.iter().enumerate() { - let authority_index = self - .context - .committee - .to_authority_index(authority_index) - .unwrap(); - - let last_evicted_round = self.evicted_rounds[authority_index]; - if end_round.saturating_sub(1) <= last_evicted_round { - panic!( - "Attempted to request for blocks of rounds < {end_round}, when the last evicted round is {last_evicted_round} for authority {authority_index}", - ); - } - - let block_ref_iter = block_refs - .range(( - Included(BlockRef::new( - last_evicted_round + 1, - authority_index, - BlockDigest::MIN, - )), - Excluded(BlockRef::new(end_round, authority_index, BlockDigest::MIN)), - )) - .rev(); - - let mut last_round = 0; - for block_ref in block_ref_iter { - if last_round == 0 { - last_round = block_ref.round; - let block_info = self - .recent_blocks - .get(block_ref) - .expect("Block should exist in recent blocks"); - blocks[authority_index] = block_info.block.clone(); - continue; - } - if block_ref.round < last_round { - break; - } - equivocating_blocks[authority_index].push(*block_ref); - } - } - - blocks.into_iter().zip(equivocating_blocks).collect() - } - - /// Checks whether a block exists in the slot. The method checks only - /// against the cached data. If the user asks for a slot that is not - /// within the cached data then a panic is thrown. - pub(crate) fn contains_cached_block_at_slot(&self, slot: Slot) -> bool { - // Always return true for genesis slots. - if slot.round == GENESIS_ROUND { - return true; - } - - let eviction_round = self.evicted_rounds[slot.authority]; - if slot.round <= eviction_round { - panic!( - "{}", - format!( - "Attempted to check for slot {slot} that is <= the last{}evicted round {eviction_round}", - if self.gc_enabled() { " gc " } else { " " } - ) - ); - } - - let mut result = self.recent_refs_by_authority[slot.authority].range(( - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MIN)), - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MAX)), - )); - result.next().is_some() - } - - /// Checks whether the required blocks are in cache, if exist, or otherwise - /// will check in store. The method is not caching back the results, so - /// its expensive if keep asking for cache missing blocks. - pub(crate) fn contains_blocks(&self, block_refs: Vec) -> Vec { - let mut exist = vec![false; block_refs.len()]; - let mut missing = Vec::new(); - - for (index, block_ref) in block_refs.into_iter().enumerate() { - let recent_refs = &self.recent_refs_by_authority[block_ref.author]; - if recent_refs.contains(&block_ref) || self.genesis.contains_key(&block_ref) { - exist[index] = true; - } else if recent_refs.is_empty() || recent_refs.last().unwrap().round < block_ref.round - { - // Optimization: recent_refs contain the most recent blocks known to this - // authority. If a block ref is not found there and has a higher - // round, it definitely is missing from this authority and there - // is no need to check disk. - exist[index] = false; - } else { - missing.push((index, block_ref)); - } - } - - if missing.is_empty() { - return exist; - } - - let missing_refs = missing - .iter() - .map(|(_, block_ref)| *block_ref) - .collect::>(); - let store_results = self - .store - .contains_blocks(&missing_refs) - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")); - self.context - .metrics - .node_metrics - .dag_state_store_read_count - .with_label_values(&["contains_blocks"]) - .inc(); - - for ((index, _), result) in missing.into_iter().zip(store_results) { - exist[index] = result; - } - - exist - } - - pub(crate) fn contains_block(&self, block_ref: &BlockRef) -> bool { - let blocks = self.contains_blocks(vec![*block_ref]); - blocks.first().cloned().unwrap() - } - - pub(crate) fn threshold_clock_round(&self) -> Round { - self.threshold_clock.get_round() - } - - pub(crate) fn threshold_clock_quorum_ts(&self) -> Instant { - self.threshold_clock.get_quorum_ts() - } - - pub(crate) fn highest_accepted_round(&self) -> Round { - self.highest_accepted_round - } - - // Buffers a new commit in memory and updates last committed rounds. - // REQUIRED: must not skip over any commit index. - pub(crate) fn add_commit(&mut self, commit: TrustedCommit) { - let time_diff = if let Some(last_commit) = &self.last_commit { - if commit.index() <= last_commit.index() { - warn!( - "New commit index {} <= last commit index {}!", - commit.index(), - last_commit.index() - ); // This could happen in case of fast commit syncer downloading transactions from last solid commit (not pending). - return; - } - assert_eq!(commit.index(), last_commit.index() + 1); - - if commit.timestamp_ms() < last_commit.timestamp_ms() { - panic!( - "Commit timestamps do not monotonically increment, prev commit {last_commit:?}, new commit {commit:?}" - ); - } - commit - .timestamp_ms() - .saturating_sub(last_commit.timestamp_ms()) - } else { - assert_eq!(commit.index(), 1); - 0 - }; - - self.context - .metrics - .node_metrics - .last_commit_time_diff - .observe(time_diff as f64); - - let commit_round_advanced = if let Some(previous_commit) = &self.last_commit { - previous_commit.round() < commit.round() - } else { - true - }; - - self.last_commit = Some(commit.clone()); - - if commit_round_advanced { - let now = std::time::Instant::now(); - if let Some(previous_time) = self.last_commit_round_advancement_time { - self.context - .metrics - .node_metrics - .commit_round_advancement_interval - .observe(now.duration_since(previous_time).as_secs_f64()) - } - self.last_commit_round_advancement_time = Some(now); - } - - for block_ref in commit.blocks().iter() { - self.last_committed_rounds[block_ref.author] = max( - self.last_committed_rounds[block_ref.author], - block_ref.round, - ); - } - - for (i, round) in self.last_committed_rounds.iter().enumerate() { - let index = self.context.committee.to_authority_index(i).unwrap(); - let hostname = &self.context.committee.authority(index).hostname; - self.context - .metrics - .node_metrics - .last_committed_authority_round - .with_label_values(&[hostname]) - .set((*round).into()); - } - - self.pending_commit_votes.push_back(commit.reference()); - self.commits_to_write.push(commit); - } - - pub(crate) fn add_commit_info(&mut self, reputation_scores: ReputationScores) { - // We empty the unscored committed subdags to calculate reputation scores. - assert!(self.unscored_committed_subdags.is_empty()); - - // We create an empty scoring subdag once reputation scores are calculated. - // Note: It is okay for this to not be gated by protocol config as the - // scoring_subdag should be empty in either case at this point. - assert!(self.scoring_subdag.is_empty()); - - let commit_info = CommitInfo { - committed_rounds: self.last_committed_rounds.clone(), - reputation_scores, - }; - let last_commit = self - .last_commit - .as_ref() - .expect("Last commit should already be set."); - self.commit_info_to_write - .push((last_commit.reference(), commit_info)); - } - - pub(crate) fn take_commit_votes(&mut self, limit: usize) -> Vec { - let mut votes = Vec::new(); - while !self.pending_commit_votes.is_empty() && votes.len() < limit { - votes.push(self.pending_commit_votes.pop_front().unwrap()); - } - votes - } - - /// Index of the last commit. - pub(crate) fn last_commit_index(&self) -> CommitIndex { - match &self.last_commit { - Some(commit) => commit.index(), - None => 0, - } - } - - /// Digest of the last commit. - pub(crate) fn last_commit_digest(&self) -> CommitDigest { - match &self.last_commit { - Some(commit) => commit.digest(), - None => CommitDigest::MIN, - } - } - - /// Timestamp of the last commit. - pub(crate) fn last_commit_timestamp_ms(&self) -> BlockTimestampMs { - match &self.last_commit { - Some(commit) => commit.timestamp_ms(), - None => 0, - } - } - - /// Leader slot of the last commit. - pub(crate) fn last_commit_leader(&self) -> Slot { - match &self.last_commit { - Some(commit) => commit.leader().into(), - None => self - .genesis - .iter() - .next() - .map(|(genesis_ref, _)| *genesis_ref) - .expect("Genesis blocks should always be available.") - .into(), - } - } - - /// Highest round where a block is committed, which is last commit's leader - /// round. - pub(crate) fn last_commit_round(&self) -> Round { - match &self.last_commit { - Some(commit) => commit.leader().round, - None => 0, - } - } - - /// Last committed round per authority. - pub(crate) fn last_committed_rounds(&self) -> Vec { - self.last_committed_rounds.clone() - } - - /// The GC round is the highest round that blocks of equal or lower round - /// are considered obsolete and no longer possible to be committed. - /// There is no meaning accepting any blocks with round <= gc_round. The - /// Garbage Collection (GC) round is calculated based on the latest - /// committed leader round. When GC is disabled that will return the genesis - /// round. - pub(crate) fn gc_round(&self) -> Round { - self.calculate_gc_round(self.last_commit_round()) - } - - pub(crate) fn calculate_gc_round(&self, commit_round: Round) -> Round { - let gc_depth = self.context.protocol_config.gc_depth(); - if gc_depth > 0 { - // GC is enabled, only then calculate the diff - commit_round.saturating_sub(gc_depth) - } else { - // Otherwise just return genesis round. That also acts as a safety mechanism so - // we never attempt to truncate anything even accidentally. - GENESIS_ROUND - } - } - - pub(crate) fn gc_enabled(&self) -> bool { - self.context.protocol_config.gc_depth() > 0 - } - - /// After each flush, DagState becomes persisted in storage and it expected - /// to recover all internal states from storage after restarts. - pub(crate) fn flush(&mut self) { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["DagState::flush"]) - .start_timer(); - // Flush buffered data to storage. - let blocks = std::mem::take(&mut self.blocks_to_write); - let commits = std::mem::take(&mut self.commits_to_write); - let commit_info_to_write = std::mem::take(&mut self.commit_info_to_write); - - if blocks.is_empty() && commits.is_empty() { - return; - } - debug!( - "Flushing {} blocks ({}), {} commits ({}) and {} commit info ({}) to storage.", - blocks.len(), - blocks.iter().map(|b| b.reference().to_string()).join(","), - commits.len(), - commits.iter().map(|c| c.reference().to_string()).join(","), - commit_info_to_write.len(), - commit_info_to_write - .iter() - .map(|(commit_ref, _)| commit_ref.to_string()) - .join(","), - ); - - // Update the scoring metrics accordingly to the blocks being flushed. - let mut metrics_to_write = vec![]; - let threshold_clock_round = self.threshold_clock_round(); - for (authority_index, authority) in self.context.committee.authorities() { - let last_eviction_round = self.evicted_rounds[authority_index]; - let current_eviction_round = self.calculate_authority_eviction_round(authority_index); - let metrics_to_write_from_authority = self - .context - .scoring_metrics_store - .update_scoring_metrics_on_eviction( - authority_index, - authority.hostname.as_str(), - &self.recent_refs_by_authority[authority_index], - current_eviction_round, - last_eviction_round, - threshold_clock_round, - &self.context.metrics.node_metrics, - ); - if let Some(metrics_to_write_from_authority) = metrics_to_write_from_authority { - metrics_to_write.push((authority_index, metrics_to_write_from_authority)); - } - } - - self.store - .write(WriteBatch::new( - blocks, - commits, - commit_info_to_write, - metrics_to_write, - )) - .unwrap_or_else(|e| panic!("Failed to write to storage: {e:?}")); - self.context - .metrics - .node_metrics - .dag_state_store_write_count - .inc(); - - // Clean up old cached data. After flushing, all cached blocks are guaranteed to - // be persisted. This clean up also triggers some of the scoring metrics - // updates. - for (authority_index, _) in self.context.committee.authorities() { - let eviction_round = self.calculate_authority_eviction_round(authority_index); - while let Some(block_ref) = self.recent_refs_by_authority[authority_index].first() { - let block_round = block_ref.round; - if block_round <= eviction_round { - self.recent_blocks.remove(block_ref); - self.recent_refs_by_authority[authority_index].pop_first(); - } else { - break; - } - } - self.evicted_rounds[authority_index] = eviction_round; - } - - let metrics = &self.context.metrics.node_metrics; - metrics - .dag_state_recent_blocks - .set(self.recent_blocks.len() as i64); - metrics.dag_state_recent_refs.set( - self.recent_refs_by_authority - .iter() - .map(BTreeSet::len) - .sum::() as i64, - ); - } - - pub(crate) fn recover_last_commit_info(&self) -> Option<(CommitRef, CommitInfo)> { - self.store - .read_last_commit_info() - .unwrap_or_else(|e| panic!("Failed to read from storage: {e:?}")) - } - - // TODO: Remove four methods below this when DistributedVoteScoring is enabled. - pub(crate) fn unscored_committed_subdags_count(&self) -> u64 { - self.unscored_committed_subdags.len() as u64 - } - - #[cfg(test)] - pub(crate) fn unscored_committed_subdags(&self) -> Vec { - self.unscored_committed_subdags.clone() - } - - pub(crate) fn add_unscored_committed_subdags( - &mut self, - committed_subdags: Vec, - ) { - self.unscored_committed_subdags.extend(committed_subdags); - } - - pub(crate) fn take_unscored_committed_subdags(&mut self) -> Vec { - std::mem::take(&mut self.unscored_committed_subdags) - } - - pub(crate) fn add_scoring_subdags(&mut self, scoring_subdags: Vec) { - self.scoring_subdag.add_subdags(scoring_subdags); - } - - pub(crate) fn clear_scoring_subdag(&mut self) { - self.scoring_subdag.clear(); - } - - pub(crate) fn scoring_subdags_count(&self) -> usize { - self.scoring_subdag.scored_subdags_count() - } - - pub(crate) fn is_scoring_subdag_empty(&self) -> bool { - self.scoring_subdag.is_empty() - } - - pub(crate) fn calculate_scoring_subdag_scores(&self) -> ReputationScores { - self.scoring_subdag.calculate_distributed_vote_scores() - } - - pub(crate) fn scoring_subdag_commit_range(&self) -> CommitIndex { - self.scoring_subdag - .commit_range - .as_ref() - .expect("commit range should exist for scoring subdag") - .end() - } - - /// The last round that should get evicted after a cache clean up operation. - /// After this round we are guaranteed to have all the produced blocks - /// from that authority. For any round that is <= `last_evicted_round` - /// we don't have such guarantees as out of order blocks might exist. - fn calculate_authority_eviction_round(&self, authority_index: AuthorityIndex) -> Round { - if self.gc_enabled() { - let last_round = self.recent_refs_by_authority[authority_index] - .last() - .map(|block_ref| block_ref.round) - .unwrap_or(GENESIS_ROUND); - - Self::gc_eviction_round(last_round, self.gc_round(), self.cached_rounds) - } else { - let commit_round = self.last_committed_rounds[authority_index]; - Self::eviction_round(commit_round, self.cached_rounds) - } - } - - /// Calculates the last eviction round based on the provided `commit_round`. - /// Any blocks with round <= the evict round have been cleaned up. - fn eviction_round(commit_round: Round, cached_rounds: Round) -> Round { - commit_round.saturating_sub(cached_rounds) - } - - /// Calculates the eviction round for the given authority. The goal is to - /// keep at least `cached_rounds` of the latest blocks in the cache (if - /// enough data is available), while evicting blocks with rounds <= - /// `gc_round` when possible. - fn gc_eviction_round(last_round: Round, gc_round: Round, cached_rounds: u32) -> Round { - gc_round.min(last_round.saturating_sub(cached_rounds)) - } - - /// Detects and returns the blocks of the round that forms the last quorum. - /// The method will return the quorum even if that's genesis. - #[cfg(test)] - pub(crate) fn last_quorum(&self) -> Vec { - // the quorum should exist either on the highest accepted round or the one - // before. If we fail to detect a quorum then it means that our DAG has - // advanced with missing causal history. - for round in - (self.highest_accepted_round.saturating_sub(1)..=self.highest_accepted_round).rev() - { - if round == GENESIS_ROUND { - return self.genesis_blocks(); - } - use crate::stake_aggregator::{QuorumThreshold, StakeAggregator}; - let mut quorum = StakeAggregator::::new(); - - // Since the minimum wave length is 3 we expect to find a quorum in the - // uncommitted rounds. - let blocks = self.get_uncommitted_blocks_at_round(round); - for block in &blocks { - if quorum.add(block.author(), &self.context.committee) { - return blocks; - } - } - } - - panic!("Fatal error, no quorum has been detected in our DAG on the last two rounds."); - } - - #[cfg(test)] - pub(crate) fn genesis_blocks(&self) -> Vec { - self.genesis.values().cloned().collect() - } - - #[cfg(test)] - pub(crate) fn set_last_commit(&mut self, commit: TrustedCommit) { - self.last_commit = Some(commit); - } -} - -struct BlockInfo { - block: VerifiedBlock, - // Whether the block has been committed - committed: bool, -} - -impl BlockInfo { - fn new(block: VerifiedBlock) -> Self { - Self { - block, - committed: false, - } - } -} - -#[cfg(test)] -mod test { - use std::vec; - - use parking_lot::RwLock; - use rstest::rstest; - - use super::*; - use crate::{ - block::{BlockDigest, BlockRef, BlockTimestampMs, TestBlock, VerifiedBlock}, - storage::{WriteBatch, mem_store::MemStore}, - test_dag_builder::DagBuilder, - test_dag_parser::parse_dag, - }; - - #[tokio::test] - async fn test_get_blocks() { - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - let own_index = AuthorityIndex::new_for_test(0); - - // Populate test blocks for round 1 ~ 10, authorities 0 ~ 2. - let num_rounds: u32 = 10; - let non_existent_round: u32 = 100; - let num_authorities: u32 = 3; - let num_blocks_per_slot: usize = 3; - let mut blocks = BTreeMap::new(); - for round in 1..=num_rounds { - for author in 0..num_authorities { - // Create 3 blocks per slot, with different timestamps and digests. - let base_ts = round as BlockTimestampMs * 1000; - for timestamp in base_ts..base_ts + num_blocks_per_slot as u64 { - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_timestamp_ms(timestamp) - .build(), - ); - dag_state.accept_block(block.clone()); - blocks.insert(block.reference(), block); - - // Only write one block per slot for own index - if AuthorityIndex::new_for_test(author) == own_index { - break; - } - } - } - } - - // Check uncommitted blocks that exist. - for (r, block) in &blocks { - assert_eq!(&dag_state.get_block(r).unwrap(), block); - } - - // Check uncommitted blocks that do not exist. - let last_ref = blocks.keys().last().unwrap(); - assert!( - dag_state - .get_block(&BlockRef::new( - last_ref.round, - last_ref.author, - BlockDigest::MIN - )) - .is_none() - ); - - // Check slots with uncommitted blocks. - for round in 1..=num_rounds { - for author in 0..num_authorities { - let slot = Slot::new( - round, - context - .committee - .to_authority_index(author as usize) - .unwrap(), - ); - let blocks = dag_state.get_uncommitted_blocks_at_slot(slot); - - // We only write one block per slot for own index - if AuthorityIndex::new_for_test(author) == own_index { - assert_eq!(blocks.len(), 1); - } else { - assert_eq!(blocks.len(), num_blocks_per_slot); - } - - for b in blocks { - assert_eq!(b.round(), round); - assert_eq!( - b.author(), - context - .committee - .to_authority_index(author as usize) - .unwrap() - ); - } - } - } - - // Check slots without uncommitted blocks. - let slot = Slot::new(non_existent_round, AuthorityIndex::ZERO); - assert!(dag_state.get_uncommitted_blocks_at_slot(slot).is_empty()); - - // Check rounds with uncommitted blocks. - for round in 1..=num_rounds { - let blocks = dag_state.get_uncommitted_blocks_at_round(round); - // Expect 3 blocks per authority except for own authority which should - // have 1 block. - assert_eq!( - blocks.len(), - (num_authorities - 1) as usize * num_blocks_per_slot + 1 - ); - for b in blocks { - assert_eq!(b.round(), round); - } - } - - // Check rounds without uncommitted blocks. - assert!( - dag_state - .get_uncommitted_blocks_at_round(non_existent_round) - .is_empty() - ); - } - - #[tokio::test] - async fn test_ancestors_at_uncommitted_round() { - // Initialize DagState. - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context, store); - - // Populate DagState. - - // Round 10 refs will not have their blocks in DagState. - let round_10_refs: Vec<_> = (0..4) - .map(|a| { - VerifiedBlock::new_for_test(TestBlock::new(10, a).set_timestamp_ms(1000).build()) - .reference() - }) - .collect(); - - // Round 11 blocks. - let round_11 = [ - // This will connect to round 12. - VerifiedBlock::new_for_test( - TestBlock::new(11, 0) - .set_timestamp_ms(1100) - .set_ancestors(round_10_refs.clone()) - .build(), - ), - // Slot(11, 1) has 3 blocks. - // This will connect to round 12. - VerifiedBlock::new_for_test( - TestBlock::new(11, 1) - .set_timestamp_ms(1110) - .set_ancestors(round_10_refs.clone()) - .build(), - ), - // This will connect to round 13. - VerifiedBlock::new_for_test( - TestBlock::new(11, 1) - .set_timestamp_ms(1111) - .set_ancestors(round_10_refs.clone()) - .build(), - ), - // This will not connect to any block. - VerifiedBlock::new_for_test( - TestBlock::new(11, 1) - .set_timestamp_ms(1112) - .set_ancestors(round_10_refs.clone()) - .build(), - ), - // This will not connect to any block. - VerifiedBlock::new_for_test( - TestBlock::new(11, 2) - .set_timestamp_ms(1120) - .set_ancestors(round_10_refs.clone()) - .build(), - ), - // This will connect to round 12. - VerifiedBlock::new_for_test( - TestBlock::new(11, 3) - .set_timestamp_ms(1130) - .set_ancestors(round_10_refs) - .build(), - ), - ]; - - // Round 12 blocks. - let ancestors_for_round_12 = vec![ - round_11[0].reference(), - round_11[1].reference(), - round_11[5].reference(), - ]; - let round_12 = [ - VerifiedBlock::new_for_test( - TestBlock::new(12, 0) - .set_timestamp_ms(1200) - .set_ancestors(ancestors_for_round_12.clone()) - .build(), - ), - VerifiedBlock::new_for_test( - TestBlock::new(12, 2) - .set_timestamp_ms(1220) - .set_ancestors(ancestors_for_round_12.clone()) - .build(), - ), - VerifiedBlock::new_for_test( - TestBlock::new(12, 3) - .set_timestamp_ms(1230) - .set_ancestors(ancestors_for_round_12) - .build(), - ), - ]; - - // Round 13 blocks. - let ancestors_for_round_13 = vec![ - round_12[0].reference(), - round_12[1].reference(), - round_12[2].reference(), - round_11[2].reference(), - ]; - let round_13 = [ - VerifiedBlock::new_for_test( - TestBlock::new(12, 1) - .set_timestamp_ms(1300) - .set_ancestors(ancestors_for_round_13.clone()) - .build(), - ), - VerifiedBlock::new_for_test( - TestBlock::new(12, 2) - .set_timestamp_ms(1320) - .set_ancestors(ancestors_for_round_13.clone()) - .build(), - ), - VerifiedBlock::new_for_test( - TestBlock::new(12, 3) - .set_timestamp_ms(1330) - .set_ancestors(ancestors_for_round_13) - .build(), - ), - ]; - - // Round 14 anchor block. - let ancestors_for_round_14 = round_13.iter().map(|b| b.reference()).collect(); - let anchor = VerifiedBlock::new_for_test( - TestBlock::new(14, 1) - .set_timestamp_ms(1410) - .set_ancestors(ancestors_for_round_14) - .build(), - ); - - // Add all blocks (at and above round 11) to DagState. - for b in round_11 - .iter() - .chain(round_12.iter()) - .chain(round_13.iter()) - .chain([anchor.clone()].iter()) - { - dag_state.accept_block(b.clone()); - } - - // Check ancestors connected to anchor. - let ancestors = dag_state.ancestors_at_round(&anchor, 11); - let mut ancestors_refs: Vec = ancestors.iter().map(|b| b.reference()).collect(); - ancestors_refs.sort(); - let mut expected_refs = vec![ - round_11[0].reference(), - round_11[1].reference(), - round_11[2].reference(), - round_11[5].reference(), - ]; - expected_refs.sort(); // we need to sort as blocks with same author and round of round 11 (position 1 - // & 2) might not be in right lexicographical order. - assert_eq!( - ancestors_refs, expected_refs, - "Expected round 11 ancestors: {expected_refs:?}. Got: {ancestors_refs:?}" - ); - } - - #[tokio::test] - async fn test_contains_blocks_in_cache_or_store() { - /// Only keep elements up to 2 rounds before the last committed round - const CACHED_ROUNDS: Round = 2; - - let (mut context, _) = Context::new_for_test(4); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context, store.clone()); - - // Create test blocks for round 1 ~ 10 - let num_rounds: u32 = 10; - let num_authorities: u32 = 4; - let mut blocks = Vec::new(); - - for round in 1..=num_rounds { - for author in 0..num_authorities { - let block = VerifiedBlock::new_for_test(TestBlock::new(round, author).build()); - blocks.push(block); - } - } - - // Now write in store the blocks from first 4 rounds and the rest to the dag - // state - blocks.clone().into_iter().for_each(|block| { - if block.round() <= 4 { - store - .write(WriteBatch::default().blocks(vec![block])) - .unwrap(); - } else { - dag_state.accept_blocks(vec![block]); - } - }); - - // Now when trying to query whether we have all the blocks, we should - // successfully retrieve a positive answer where the blocks of first 4 - // round should be found in DagState and the rest in store. - let mut block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state.contains_blocks(block_refs.clone()); - - // Ensure everything is found - let mut expected = vec![true; (num_rounds * num_authorities) as usize]; - assert_eq!(result, expected); - - // Now try to ask also for one block ref that is neither in cache nor in store - block_refs.insert( - 3, - BlockRef::new(11, AuthorityIndex::new_for_test(3), BlockDigest::default()), - ); - let result = dag_state.contains_blocks(block_refs.clone()); - - // Then all should be found apart from the last one - expected.insert(3, false); - assert_eq!(result, expected.clone()); - } - - #[tokio::test] - async fn test_contains_cached_block_at_slot() { - /// Only keep elements up to 2 rounds before the last committed round - const CACHED_ROUNDS: Round = 2; - - let num_authorities: u32 = 4; - let (mut context, _) = Context::new_for_test(num_authorities as usize); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Create test blocks for round 1 ~ 10 - let num_rounds: u32 = 10; - let mut blocks = Vec::new(); - - for round in 1..=num_rounds { - for author in 0..num_authorities { - let block = VerifiedBlock::new_for_test(TestBlock::new(round, author).build()); - blocks.push(block.clone()); - dag_state.accept_block(block); - } - } - - // Query for genesis round 0, genesis blocks should be returned - for (author, _) in context.committee.authorities() { - assert!( - dag_state.contains_cached_block_at_slot(Slot::new(GENESIS_ROUND, author)), - "Genesis should always be found" - ); - } - - // Now when trying to query whether we have all the blocks, we should - // successfully retrieve a positive answer where the blocks of first 4 - // round should be found in DagState and the rest in store. - let mut block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - - for block_ref in block_refs.clone() { - let slot = block_ref.into(); - let found = dag_state.contains_cached_block_at_slot(slot); - assert!(found, "A block should be found at slot {slot}"); - } - - // Now try to ask also for one block ref that is not in cache - // Then all should be found apart from the last one - block_refs.insert( - 3, - BlockRef::new(11, AuthorityIndex::new_for_test(3), BlockDigest::default()), - ); - let mut expected = vec![true; (num_rounds * num_authorities) as usize]; - expected.insert(3, false); - - // Attempt to check the same for via the contains slot method - for block_ref in block_refs { - let slot = block_ref.into(); - let found = dag_state.contains_cached_block_at_slot(slot); - - assert_eq!(expected.remove(0), found); - } - } - - #[tokio::test] - #[should_panic( - expected = "Attempted to check for slot S8[0] that is <= the last gc evicted round 8" - )] - async fn test_contains_cached_block_at_slot_panics_when_ask_out_of_range() { - /// Only keep elements up to 2 rounds before the last committed round - const CACHED_ROUNDS: Round = 2; - const GC_DEPTH: u32 = 1; - let (mut context, _) = Context::new_for_test(4); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context, store); - - // Create test blocks for round 1 ~ 10 for authority 0 - let mut blocks = Vec::new(); - for round in 1..=10 { - let block = VerifiedBlock::new_for_test(TestBlock::new(round, 0).build()); - blocks.push(block.clone()); - dag_state.accept_block(block); - } - - // Now add a commit to trigger an eviction - dag_state.add_commit(TrustedCommit::new_for_test( - 1 as CommitIndex, - CommitDigest::MIN, - 0, - blocks.last().unwrap().reference(), - blocks - .into_iter() - .map(|block| block.reference()) - .collect::>(), - )); - - dag_state.flush(); - - // When trying to request for authority 0 at block slot 8 it should panic, as - // anything that is <= commit_round - cached_rounds = 10 - 2 = 8 should - // be evicted - let _ = - dag_state.contains_cached_block_at_slot(Slot::new(8, AuthorityIndex::new_for_test(0))); - } - - #[tokio::test] - #[should_panic( - expected = "Attempted to check for slot S3[1] that is <= the last gc evicted round 3" - )] - async fn test_contains_cached_block_at_slot_panics_when_ask_out_of_range_gc_enabled() { - /// Keep 2 rounds from the highest committed round. This is considered - /// universal and minimum necessary blocks to hold - /// for the correct node operation. - const GC_DEPTH: u32 = 2; - /// Keep at least 3 rounds in cache for each authority. - const CACHED_ROUNDS: Round = 3; - - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Create for rounds 1..=6. Skip creating blocks for authority 0 for rounds 4 - - // 6. - let mut dag_builder = DagBuilder::new(context); - dag_builder.layers(1..=3).build(); - dag_builder - .layers(4..=6) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - - // Accept all blocks - dag_builder - .all_blocks() - .into_iter() - .for_each(|block| dag_state.accept_block(block)); - - // Now add a commit for leader round 5 to trigger an eviction - dag_state.add_commit(TrustedCommit::new_for_test( - 1 as CommitIndex, - CommitDigest::MIN, - 0, - dag_builder.leader_block(5).unwrap().reference(), - vec![], - )); - - dag_state.flush(); - - // Ensure that gc round has been updated - assert_eq!(dag_state.gc_round(), 3, "GC round should be 3"); - - // Now what we expect to happen is for: - // * Nodes 1 - 3 should have in cache blocks from gc_round (3) and onwards. - // * Node 0 should have in cache blocks from it's latest round, 3, up to round - // 1, which is the number of cached_rounds. - for authority_index in 1..=3 { - for round in 4..=6 { - assert!(dag_state.contains_cached_block_at_slot(Slot::new( - round, - AuthorityIndex::new_for_test(authority_index) - ))); - } - } - - for round in 1..=3 { - assert!( - dag_state.contains_cached_block_at_slot(Slot::new( - round, - AuthorityIndex::new_for_test(0) - )) - ); - } - - // When trying to request for authority 1 at block slot 3 it should panic, as - // anything that is <= 3 should be evicted - let _ = - dag_state.contains_cached_block_at_slot(Slot::new(3, AuthorityIndex::new_for_test(1))); - } - - #[tokio::test] - async fn test_get_blocks_in_cache_or_store() { - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context, store.clone()); - - // Create test blocks for round 1 ~ 10 - let num_rounds: u32 = 10; - let num_authorities: u32 = 4; - let mut blocks = Vec::new(); - - for round in 1..=num_rounds { - for author in 0..num_authorities { - let block = VerifiedBlock::new_for_test(TestBlock::new(round, author).build()); - blocks.push(block); - } - } - - // Now write in store the blocks from first 4 rounds and the rest to the dag - // state - blocks.clone().into_iter().for_each(|block| { - if block.round() <= 4 { - store - .write(WriteBatch::default().blocks(vec![block])) - .unwrap(); - } else { - dag_state.accept_blocks(vec![block]); - } - }); - - // Now when trying to query whether we have all the blocks, we should - // successfully retrieve a positive answer where the blocks of first 4 - // round should be found in DagState and the rest in store. - let mut block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state.get_blocks(&block_refs); - - let mut expected = blocks - .into_iter() - .map(Some) - .collect::>>(); - - // Ensure everything is found - assert_eq!(result, expected.clone()); - - // Now try to ask also for one block ref that is neither in cache nor in store - block_refs.insert( - 3, - BlockRef::new(11, AuthorityIndex::new_for_test(3), BlockDigest::default()), - ); - let result = dag_state.get_blocks(&block_refs); - - // Then all should be found apart from the last one - expected.insert(3, None); - assert_eq!(result, expected); - } - - // TODO: Remove when DistributedVoteScoring is enabled. - #[rstest] - #[tokio::test] - async fn test_flush_and_recovery_with_unscored_subdag(#[values(0, 5)] gc_depth: u32) { - telemetry_subscribers::init_for_testing(); - let num_authorities: u32 = 4; - let (mut context, _) = Context::new_for_test(num_authorities as usize); - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - - if gc_depth > 0 { - context - .protocol_config - .set_consensus_gc_depth_for_testing(gc_depth); - } - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - // Create test blocks and commits for round 1 ~ 10 - let num_rounds: u32 = 10; - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=num_rounds).build(); - let mut commits = vec![]; - - for (_subdag, commit) in dag_builder.get_sub_dag_and_commits(1..=num_rounds) { - commits.push(commit); - } - - // Add the blocks from first 5 rounds and first 5 commits to the dag state - let temp_commits = commits.split_off(5); - dag_state.accept_blocks(dag_builder.blocks(1..=5)); - for commit in commits.clone() { - dag_state.add_commit(commit); - } - - // Flush the dag state - dag_state.flush(); - - // Add the rest of the blocks and commits to the dag state - dag_state.accept_blocks(dag_builder.blocks(6..=num_rounds)); - for commit in temp_commits { - dag_state.add_commit(commit); - } - - // All blocks should be found in DagState. - let all_blocks = dag_builder.blocks(6..=num_rounds); - let block_refs = all_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, all_blocks); - - // Last commit index should be 10. - assert_eq!(dag_state.last_commit_index(), 10); - assert_eq!( - dag_state.last_committed_rounds(), - dag_builder.last_committed_rounds.clone() - ); - - // Destroy the dag state. - drop(dag_state); - - // Recover the state from the store - let dag_state = DagState::new(context, store); - - // Blocks of first 5 rounds should be found in DagState. - let blocks = dag_builder.blocks(1..=5); - let block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, blocks); - - // Blocks above round 5 should not be in DagState, because they are not flushed. - let missing_blocks = dag_builder.blocks(6..=num_rounds); - let block_refs = missing_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let retrieved_blocks = dag_state - .get_blocks(&block_refs) - .into_iter() - .flatten() - .collect::>(); - assert!(retrieved_blocks.is_empty()); - - // Last commit index should be 5. - assert_eq!(dag_state.last_commit_index(), 5); - - // This is the last_commit_rounds of the first 5 commits that were flushed - let expected_last_committed_rounds = vec![4, 5, 4, 4]; - assert_eq!( - dag_state.last_committed_rounds(), - expected_last_committed_rounds - ); - - // Unscored subdags will be recovered based on the flushed commits and no commit - // info - assert_eq!(dag_state.unscored_committed_subdags_count(), 5); - } - - #[tokio::test] - async fn test_flush_and_recovery() { - telemetry_subscribers::init_for_testing(); - let num_authorities: u32 = 4; - let (context, _) = Context::new_for_test(num_authorities as usize); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - // Create test blocks and commits for round 1 ~ 10 - let num_rounds: u32 = 10; - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=num_rounds).build(); - let mut commits = vec![]; - for (_subdag, commit) in dag_builder.get_sub_dag_and_commits(1..=num_rounds) { - commits.push(commit); - } - - // Add the blocks from first 5 rounds and first 5 commits to the dag state - let temp_commits = commits.split_off(5); - dag_state.accept_blocks(dag_builder.blocks(1..=5)); - for commit in commits.clone() { - dag_state.add_commit(commit); - } - - // Flush the dag state - dag_state.flush(); - - // Add the rest of the blocks and commits to the dag state - dag_state.accept_blocks(dag_builder.blocks(6..=num_rounds)); - for commit in temp_commits { - dag_state.add_commit(commit); - } - - // All blocks should be found in DagState. - let all_blocks = dag_builder.blocks(6..=num_rounds); - let block_refs = all_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, all_blocks); - - // Last commit index should be 10. - assert_eq!(dag_state.last_commit_index(), 10); - assert_eq!( - dag_state.last_committed_rounds(), - dag_builder.last_committed_rounds.clone() - ); - - // Destroy the dag state. - drop(dag_state); - - // Recover the state from the store - let dag_state = DagState::new(context, store); - - // Blocks of first 5 rounds should be found in DagState. - let blocks = dag_builder.blocks(1..=5); - let block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, blocks); - - // Blocks above round 5 should not be in DagState, because they are not flushed. - let missing_blocks = dag_builder.blocks(6..=num_rounds); - let block_refs = missing_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let retrieved_blocks = dag_state - .get_blocks(&block_refs) - .into_iter() - .flatten() - .collect::>(); - assert!(retrieved_blocks.is_empty()); - - // Last commit index should be 5. - assert_eq!(dag_state.last_commit_index(), 5); - - // This is the last_commit_rounds of the first 5 commits that were flushed - let expected_last_committed_rounds = vec![4, 5, 4, 4]; - assert_eq!( - dag_state.last_committed_rounds(), - expected_last_committed_rounds - ); - // Unscored subdags will be recovered based on the flushed commits and no commit - // info - assert_eq!(dag_state.scoring_subdags_count(), 5); - } - - #[tokio::test] - async fn test_flush_and_recovery_gc_enabled() { - telemetry_subscribers::init_for_testing(); - - const GC_DEPTH: u32 = 3; - const CACHED_ROUNDS: u32 = 4; - - let num_authorities: u32 = 4; - let (mut context, _) = Context::new_for_test(num_authorities as usize); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - context - .protocol_config - .set_consensus_linearize_subdag_v2_for_testing(true); - - let context = Arc::new(context); - - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - let num_rounds: u32 = 10; - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=5).build(); - dag_builder - .layers(6..=8) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - dag_builder.layers(9..=num_rounds).build(); - - let mut commits = dag_builder - .get_sub_dag_and_commits(1..=num_rounds) - .into_iter() - .map(|(_subdag, commit)| commit) - .collect::>(); - - // Add the blocks from first 8 rounds and first 7 commits to the dag state - // It's 7 commits because we missing the commit of round 8 where authority 0 is - // the leader, but produced no block - let temp_commits = commits.split_off(7); - dag_state.accept_blocks(dag_builder.blocks(1..=8)); - for commit in commits.clone() { - dag_state.add_commit(commit); - } - - // Holds all the committed blocks from the commits that ended up being persisted - // (flushed). Any commits that not flushed will not be considered. - let mut all_committed_blocks = BTreeSet::::new(); - for commit in commits.iter() { - all_committed_blocks.extend(commit.blocks()); - } - // Flush the dag state - dag_state.flush(); - - // Add the rest of the blocks and commits to the dag state - dag_state.accept_blocks(dag_builder.blocks(9..=num_rounds)); - for commit in temp_commits { - dag_state.add_commit(commit); - } - - // All blocks should be found in DagState. - let all_blocks = dag_builder.blocks(1..=num_rounds); - let block_refs = all_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, all_blocks); - - // Last commit index should be 9 - assert_eq!(dag_state.last_commit_index(), 9); - assert_eq!( - dag_state.last_committed_rounds(), - dag_builder.last_committed_rounds.clone() - ); - - // Destroy the dag state. - drop(dag_state); - - // Recover the state from the store - let dag_state = DagState::new(context.clone(), store); - - // Blocks of first 5 rounds should be found in DagState. - let blocks = dag_builder.blocks(1..=5); - let block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let result = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|b| b.unwrap()) - .collect::>(); - assert_eq!(result, blocks); - - // Blocks above round 9 should not be in DagState, because they are not flushed. - let missing_blocks = dag_builder.blocks(9..=num_rounds); - let block_refs = missing_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - let retrieved_blocks = dag_state - .get_blocks(&block_refs) - .into_iter() - .flatten() - .collect::>(); - assert!(retrieved_blocks.is_empty()); - - // Last commit index should be 7. - assert_eq!(dag_state.last_commit_index(), 7); - - // This is the last_commit_rounds of the first 7 commits that were flushed - let expected_last_committed_rounds = vec![5, 6, 6, 7]; - assert_eq!( - dag_state.last_committed_rounds(), - expected_last_committed_rounds - ); - // Unscored subdags will be recoverd based on the flushed commits and no commit - // info - assert_eq!(dag_state.scoring_subdags_count(), 7); - // Ensure that cached blocks exist only for specific rounds per authority - for (authority_index, _) in context.committee.authorities() { - let blocks = dag_state.get_cached_blocks(authority_index, 1); - - // Ensure that eviction rounds have been properly recovered - // DagState should hold cached blocks for authority 0 for rounds [2..=5] as no - // higher blocks exist and due to CACHED_ROUNDS = 4 we want at max - // to hold blocks for 4 rounds in cache. - if authority_index == AuthorityIndex::new_for_test(0) { - assert_eq!(blocks.len(), 4); - assert_eq!(dag_state.evicted_rounds[authority_index.value()], 1); - assert!( - blocks - .into_iter() - .all(|block| block.round() >= 2 && block.round() <= 5) - ); - } else { - assert_eq!(blocks.len(), 4); - assert_eq!(dag_state.evicted_rounds[authority_index.value()], 4); - assert!( - blocks - .into_iter() - .all(|block| block.round() >= 5 && block.round() <= 8) - ); - } - } - // Ensure that committed blocks from > gc_round have been correctly marked as - // committed according to committed sub dags - let gc_round = dag_state.gc_round(); - assert_eq!(gc_round, 4); - dag_state - .recent_blocks - .iter() - .for_each(|(block_ref, block_info)| { - if block_ref.round > gc_round && all_committed_blocks.contains(block_ref) { - assert!( - block_info.committed, - "Block {block_ref:?} should be committed" - ); - }; - }); - } - - #[tokio::test] - async fn test_block_info_as_committed() { - let num_authorities: u32 = 4; - let (context, _) = Context::new_for_test(num_authorities as usize); - let context = Arc::new(context); - - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context, store); - - // Accept a block - let block = VerifiedBlock::new_for_test( - TestBlock::new(1, 0) - .set_timestamp_ms(1000) - .set_ancestors(vec![]) - .build(), - ); - - dag_state.accept_block(block.clone()); - - // Query is committed - assert!(!dag_state.is_committed(&block.reference())); - - // Set block as committed for first time should return true - assert!( - dag_state.set_committed(&block.reference()), - "Block should be successfully set as committed for first time" - ); - - // Now it should appear as committed - assert!(dag_state.is_committed(&block.reference())); - - // Trying to set the block as committed again, it should return false. - assert!( - !dag_state.set_committed(&block.reference()), - "Block should not be successfully set as committed" - ); - } - - #[tokio::test] - async fn test_get_cached_blocks() { - let (mut context, _) = Context::new_for_test(4); - context.parameters.dag_state_cached_rounds = 5; - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Create no blocks for authority 0 - // Create one block (round 10) for authority 1 - // Create two blocks (rounds 10,11) for authority 2 - // Create three blocks (rounds 10,11,12) for authority 3 - let mut all_blocks = Vec::new(); - for author in 1..=3 { - for round in 10..(10 + author) { - let block = VerifiedBlock::new_for_test(TestBlock::new(round, author).build()); - all_blocks.push(block.clone()); - dag_state.accept_block(block); - } - } - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(0).unwrap(), 0); - assert!(cached_blocks.is_empty()); - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(1).unwrap(), 10); - assert_eq!(cached_blocks.len(), 1); - assert_eq!(cached_blocks[0].round(), 10); - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(2).unwrap(), 10); - assert_eq!(cached_blocks.len(), 2); - assert_eq!(cached_blocks[0].round(), 10); - assert_eq!(cached_blocks[1].round(), 11); - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(2).unwrap(), 11); - assert_eq!(cached_blocks.len(), 1); - assert_eq!(cached_blocks[0].round(), 11); - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(3).unwrap(), 10); - assert_eq!(cached_blocks.len(), 3); - assert_eq!(cached_blocks[0].round(), 10); - assert_eq!(cached_blocks[1].round(), 11); - assert_eq!(cached_blocks[2].round(), 12); - - let cached_blocks = - dag_state.get_cached_blocks(context.committee.to_authority_index(3).unwrap(), 12); - assert_eq!(cached_blocks.len(), 1); - assert_eq!(cached_blocks[0].round(), 12); - - // Test get_cached_blocks_in_range() - - // Start == end - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(3).unwrap(), - 10, - 10, - 1, - ); - assert!(cached_blocks.is_empty()); - - // Start > end - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(3).unwrap(), - 11, - 10, - 1, - ); - assert!(cached_blocks.is_empty()); - - // Empty result. - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(0).unwrap(), - 9, - 10, - 1, - ); - assert!(cached_blocks.is_empty()); - - // Single block, one round before the end. - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(1).unwrap(), - 9, - 11, - 1, - ); - assert_eq!(cached_blocks.len(), 1); - assert_eq!(cached_blocks[0].round(), 10); - - // Respect end round. - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(2).unwrap(), - 9, - 12, - 5, - ); - assert_eq!(cached_blocks.len(), 2); - assert_eq!(cached_blocks[0].round(), 10); - assert_eq!(cached_blocks[1].round(), 11); - - // Respect start round. - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(3).unwrap(), - 11, - 20, - 5, - ); - assert_eq!(cached_blocks.len(), 2); - assert_eq!(cached_blocks[0].round(), 11); - assert_eq!(cached_blocks[1].round(), 12); - - // Respect limit - let cached_blocks = dag_state.get_cached_blocks_in_range( - context.committee.to_authority_index(3).unwrap(), - 10, - 20, - 1, - ); - assert_eq!(cached_blocks.len(), 1); - assert_eq!(cached_blocks[0].round(), 10); - } - - #[rstest] - #[tokio::test] - async fn test_get_last_cached_block(#[values(0, 1)] gc_depth: u32) { - // GIVEN - const CACHED_ROUNDS: Round = 2; - let (mut context, _) = Context::new_for_test(4); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - - if gc_depth > 0 { - context - .protocol_config - .set_consensus_gc_depth_for_testing(gc_depth); - } - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Create no blocks for authority 0 - // Create one block (round 1) for authority 1 - // Create two blocks (rounds 1,2) for authority 2 - // Create three blocks (rounds 1,2,3) for authority 3 - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { - B -> [*], - C -> [*], - D -> [*], - }, - Round 2 : { - C -> [*], - D -> [*], - }, - Round 3 : { - D -> [*], - }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - - // Add equivocating block for round 2 authority 3 - let block = VerifiedBlock::new_for_test(TestBlock::new(2, 2).build()); - - // Accept all blocks - for block in dag_builder - .all_blocks() - .into_iter() - .chain(std::iter::once(block)) - { - dag_state.accept_block(block); - } - - dag_state.add_commit(TrustedCommit::new_for_test( - 1 as CommitIndex, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - dag_builder.leader_block(3).unwrap().reference(), - vec![], - )); - - // WHEN search for the latest blocks - let end_round = 4; - let expected_rounds = vec![0, 1, 2, 3]; - let expected_excluded_and_equivocating_blocks = vec![0, 0, 1, 0]; - // THEN - let last_blocks = dag_state.get_last_cached_block_per_authority(end_round); - assert_eq!( - last_blocks.iter().map(|b| b.0.round()).collect::>(), - expected_rounds - ); - assert_eq!( - last_blocks.iter().map(|b| b.1.len()).collect::>(), - expected_excluded_and_equivocating_blocks - ); - - // THEN - for (i, expected_round) in expected_rounds.iter().enumerate() { - let round = dag_state - .get_last_cached_block_in_range( - context.committee.to_authority_index(i).unwrap(), - 0, - end_round, - ) - .map(|b| b.round()) - .unwrap_or_default(); - assert_eq!(round, *expected_round, "Authority {i}"); - } - - // WHEN starting from round 2 - let start_round = 2; - let expected_rounds = [0, 0, 2, 3]; - - // THEN - for (i, expected_round) in expected_rounds.iter().enumerate() { - let round = dag_state - .get_last_cached_block_in_range( - context.committee.to_authority_index(i).unwrap(), - start_round, - end_round, - ) - .map(|b| b.round()) - .unwrap_or_default(); - assert_eq!(round, *expected_round, "Authority {i}"); - } - - // WHEN we flush the DagState - after adding a - // commit with all the blocks, we expect this to trigger a clean up in - // the internal cache. That will keep the all the blocks with rounds >= - // authority_commit_round - CACHED_ROUND. - // - // When GC is enabled then we'll keep all the blocks that are > gc_round (2) and - // for those who don't have blocks > gc_round, we'll keep - // all their highest round blocks for CACHED_ROUNDS. - dag_state.flush(); - - // AND we request before round 3 - let end_round = 3; - let expected_rounds = vec![0, 1, 2, 2]; - - // THEN - let last_blocks = dag_state.get_last_cached_block_per_authority(end_round); - assert_eq!( - last_blocks.iter().map(|b| b.0.round()).collect::>(), - expected_rounds - ); - - // THEN - for (i, expected_round) in expected_rounds.iter().enumerate() { - let round = dag_state - .get_last_cached_block_in_range( - context.committee.to_authority_index(i).unwrap(), - 0, - end_round, - ) - .map(|b| b.round()) - .unwrap_or_default(); - assert_eq!(round, *expected_round, "Authority {i}"); - } - } - - #[tokio::test] - #[should_panic( - expected = "Attempted to request for blocks of rounds < 2, when the last evicted round is 1 for authority [2]" - )] - async fn test_get_cached_last_block_per_authority_requesting_out_of_round_range() { - // GIVEN - const CACHED_ROUNDS: Round = 1; - const GC_DEPTH: u32 = 1; - let (mut context, _) = Context::new_for_test(4); - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Create no blocks for authority 0 - // Create one block (round 1) for authority 1 - // Create two blocks (rounds 1,2) for authority 2 - // Create three blocks (rounds 1,2,3) for authority 3 - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=1) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - dag_builder - .layers(2..=2) - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1), - ]) - .skip_block() - .build(); - dag_builder - .layers(3..=3) - .authorities(vec![ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - ]) - .skip_block() - .build(); - - // Accept all blocks - for block in dag_builder.all_blocks() { - dag_state.accept_block(block); - } - - dag_state.add_commit(TrustedCommit::new_for_test( - 1 as CommitIndex, - CommitDigest::MIN, - 0, - dag_builder.leader_block(3).unwrap().reference(), - vec![], - )); - - // Flush the store so we update the evict rounds - dag_state.flush(); - - // THEN the method should panic, as some authorities have already evicted rounds - // <= round 2 - dag_state.get_last_cached_block_per_authority(2); - } - - #[tokio::test] - async fn test_last_quorum() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // WHEN no blocks exist then genesis should be returned - { - let genesis = genesis_blocks(&context); - - assert_eq!(dag_state.read().last_quorum(), genesis); - } - - // WHEN a fully connected DAG up to round 4 is created, then round 4 blocks - // should be returned as quorum - { - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=4) - .build() - .persist_layers(dag_state.clone()); - let round_4_blocks: Vec<_> = dag_builder - .blocks(4..=4) - .into_iter() - .map(|block| block.reference()) - .collect(); - - let last_quorum = dag_state.read().last_quorum(); - - assert_eq!( - last_quorum - .into_iter() - .map(|block| block.reference()) - .collect::>(), - round_4_blocks - ); - } - - // WHEN adding one more block at round 5, still round 4 should be returned as - // quorum - { - let block = VerifiedBlock::new_for_test(TestBlock::new(5, 0).build()); - dag_state.write().accept_block(block); - - let round_4_blocks = dag_state.read().get_uncommitted_blocks_at_round(4); - - let last_quorum = dag_state.read().last_quorum(); - - assert_eq!(last_quorum, round_4_blocks); - } - } - - #[tokio::test] - async fn test_last_block_for_authority() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // WHEN no blocks exist then genesis should be returned - { - let genesis = genesis_blocks(&context); - let my_genesis = genesis - .into_iter() - .find(|block| block.author() == context.own_index) - .unwrap(); - - assert_eq!(dag_state.read().get_last_proposed_block(), my_genesis); - } - - // WHEN adding some blocks for authorities, only the last ones should be - // returned - { - // add blocks up to round 4 - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder - .layers(1..=4) - .build() - .persist_layers(dag_state.clone()); - - // add block 5 for authority 0 - let block = VerifiedBlock::new_for_test(TestBlock::new(5, 0).build()); - dag_state.write().accept_block(block); - - let block = dag_state - .read() - .get_last_block_for_authority(AuthorityIndex::new_for_test(0)); - assert_eq!(block.round(), 5); - - for (authority_index, _) in context.committee.authorities() { - let block = dag_state - .read() - .get_last_block_for_authority(authority_index); - - if authority_index.value() == 0 { - assert_eq!(block.round(), 5); - } else { - assert_eq!(block.round(), 4); - } - } - } - } - - #[tokio::test] - #[should_panic] - async fn test_accept_block_panics_when_timestamp_is_ahead() { - // GIVEN - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(false); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Set a timestamp for the block that is ahead of the current time - let block_timestamp = context.clock.timestamp_utc_ms() + 5_000; - - let block = VerifiedBlock::new_for_test( - TestBlock::new(10, 0) - .set_timestamp_ms(block_timestamp) - .build(), - ); - - // Try to accept the block - it will panic as accepted block timestamp is ahead - // of the current time - dag_state.accept_block(block); - } - - #[tokio::test] - async fn test_accept_block_not_panics_when_timestamp_is_ahead_and_median_timestamp() { - // GIVEN - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(true); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store); - - // Set a timestamp for the block that is ahead of the current time - let block_timestamp = context.clock.timestamp_utc_ms() + 5_000; - - let block = VerifiedBlock::new_for_test( - TestBlock::new(10, 0) - .set_timestamp_ms(block_timestamp) - .build(), - ); - - // Try to accept the block - it should not panic - dag_state.accept_block(block); - } -} diff --git a/consensus/core/src/error.rs b/consensus/core/src/error.rs deleted file mode 100644 index 03320a14a62..00000000000 --- a/consensus/core/src/error.rs +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use consensus_config::{AuthorityIndex, Epoch, Stake}; -use fastcrypto::error::FastCryptoError; -use strum_macros::IntoStaticStr; -use thiserror::Error; -use typed_store::TypedStoreError; - -use crate::{ - block::{BlockRef, Round}, - commit::{Commit, CommitIndex}, -}; - -/// Errors that can occur when processing blocks, reading from storage, or -/// encountering shutdown. -#[derive(Clone, Debug, Error, IntoStaticStr)] -pub(crate) enum ConsensusError { - #[error("Error deserializing block: {0}")] - MalformedBlock(bcs::Error), - - #[error("Error deserializing commit: {0}")] - MalformedCommit(bcs::Error), - - #[error("Error serializing: {0}")] - SerializationFailure(bcs::Error), - - #[error("Block contains a transaction that is too large: {size} > {limit}")] - TransactionTooLarge { size: usize, limit: usize }, - - #[error("Block contains too many transactions: {count} > {limit}")] - TooManyTransactions { count: usize, limit: usize }, - - #[error("Block contains too many transaction bytes: {size} > {limit}")] - TooManyTransactionBytes { size: usize, limit: usize }, - - #[error("Unexpected block authority {0} from peer {1}")] - UnexpectedAuthority(AuthorityIndex, AuthorityIndex), - - #[error("Block has wrong epoch: expected {expected}, actual {actual}")] - WrongEpoch { expected: Epoch, actual: Epoch }, - - #[error("Genesis blocks should only be generated from Committee!")] - UnexpectedGenesisBlock, - - #[error("Genesis blocks should not be queried!")] - UnexpectedGenesisBlockRequested, - - #[error( - "Expected {requested} but received {received} blocks returned from authority {authority}" - )] - UnexpectedNumberOfBlocksFetched { - authority: AuthorityIndex, - requested: usize, - received: usize, - }, - - #[error("Unexpected block returned while fetching missing blocks")] - UnexpectedFetchedBlock { - index: AuthorityIndex, - block_ref: BlockRef, - }, - - #[error( - "Unexpected block {block_ref} returned while fetching last own block from peer {index}" - )] - UnexpectedLastOwnBlock { - index: AuthorityIndex, - block_ref: BlockRef, - }, - - #[error( - "Too many blocks have been returned from authority {0} when requesting to fetch missing blocks" - )] - TooManyFetchedBlocksReturned(AuthorityIndex), - - #[error("Too many blocks have been requested from authority {0}")] - TooManyFetchBlocksRequested(AuthorityIndex), - - #[error("Too many authorities have been provided from authority {0}")] - TooManyAuthoritiesProvided(AuthorityIndex), - - #[error( - "Provided size of highest accepted rounds parameter, {0}, is different than committee size, {1}" - )] - InvalidSizeOfHighestAcceptedRounds(usize, usize), - - #[error("Invalid authority index: {index} > {max}")] - InvalidAuthorityIndex { index: AuthorityIndex, max: usize }, - - #[error( - "Invalid ancestor authority index: block {block_authority}, ancestor {ancestor_index} > {max}" - )] - InvalidAncestorAuthorityIndex { - block_authority: AuthorityIndex, - ancestor_index: AuthorityIndex, - max: usize, - }, - - #[error("Failed to deserialize signature: {0}")] - MalformedSignature(FastCryptoError), - - #[error("Failed to verify the block's signature: {0}")] - SignatureVerificationFailure(FastCryptoError), - - #[error("Synchronizer for fetching blocks directly from ([{0}],{1}) is saturated")] - SynchronizerSaturated(AuthorityIndex, String), - - #[error("Block {block_ref:?} rejected: {reason}")] - BlockRejected { block_ref: BlockRef, reason: String }, - - #[error( - "Ancestor is in wrong position: block {block_authority}, ancestor {ancestor_authority}, position {position}" - )] - InvalidAncestorPosition { - block_authority: AuthorityIndex, - ancestor_authority: AuthorityIndex, - position: usize, - }, - - #[error("Ancestor's round ({ancestor}) should be lower than the block's round ({block})")] - InvalidAncestorRound { ancestor: Round, block: Round }, - - #[error("Ancestor {0} not found among genesis blocks!")] - InvalidGenesisAncestor(BlockRef), - - #[error("Too many ancestors in the block: {0} > {1}")] - TooManyAncestors(usize, usize), - - #[error("Ancestors from the same authority {0}")] - DuplicatedAncestorsAuthority(AuthorityIndex), - - #[error("Insufficient stake from parents: {parent_stakes} < {quorum}")] - InsufficientParentStakes { parent_stakes: Stake, quorum: Stake }, - - #[error("Invalid transaction: {0}")] - InvalidTransaction(String), - - #[error("Ancestors max timestamp {max_timestamp_ms} > block timestamp {block_timestamp_ms}")] - InvalidBlockTimestamp { - max_timestamp_ms: u64, - block_timestamp_ms: u64, - }, - - #[error("Received no commit from peer {peer}")] - NoCommitReceived { peer: AuthorityIndex }, - - #[error( - "Received unexpected start commit from peer {peer}: requested {start}, received {commit:?}" - )] - UnexpectedStartCommit { - peer: AuthorityIndex, - start: CommitIndex, - commit: Box, - }, - - #[error( - "Received unexpected commit sequence from peer {peer}: {prev_commit:?}, {curr_commit:?}" - )] - UnexpectedCommitSequence { - peer: AuthorityIndex, - prev_commit: Box, - curr_commit: Box, - }, - - #[error("Not enough votes ({stake}) on end commit from peer {peer}: {commit:?}")] - NotEnoughCommitVotes { - stake: Stake, - peer: AuthorityIndex, - commit: Box, - }, - - #[error("Received unexpected block from peer {peer}: {requested:?} vs {received:?}")] - UnexpectedBlockForCommit { - peer: AuthorityIndex, - requested: BlockRef, - received: BlockRef, - }, - - #[error( - "Unexpected certified commit index and last committed index. Expected next commit index to be {expected_commit_index}, but found {commit_index}" - )] - UnexpectedCertifiedCommitIndex { - expected_commit_index: CommitIndex, - commit_index: CommitIndex, - }, - - #[error("RocksDB failure: {0}")] - RocksDBFailure(#[from] TypedStoreError), - - #[error("Network config error: {0:?}")] - NetworkConfig(String), - - #[error("Failed to connect as client: {0:?}")] - NetworkClientConnection(String), - - #[error("Failed to send request: {0:?}")] - NetworkRequest(String), - - #[error("Request timeout: {0:?}")] - NetworkRequestTimeout(String), - - #[error("Consensus has shut down!")] - Shutdown, -} - -impl ConsensusError { - /// Returns the error name - only the enun name without any parameters - as - /// a static string. - pub fn name(&self) -> &'static str { - self.into() - } -} - -pub type ConsensusResult = Result; - -#[macro_export] -macro_rules! bail { - ($e:expr) => { - return Err($e); - }; -} - -#[macro_export(local_inner_macros)] -macro_rules! ensure { - ($cond:expr, $e:expr) => { - if !($cond) { - bail!($e); - } - }; -} - -#[cfg(test)] -mod test { - use super::*; - /// This test ensures that consensus errors when converted to a static - /// string are the same as the enum name without any parameterers - /// included to the result string. - #[test] - fn test_error_name() { - { - let error = ConsensusError::InvalidAncestorRound { - ancestor: 10, - block: 11, - }; - let error: &'static str = error.into(); - assert_eq!(error, "InvalidAncestorRound"); - } - { - let error = ConsensusError::InvalidAuthorityIndex { - index: AuthorityIndex::new_for_test(3), - max: 10, - }; - assert_eq!(error.name(), "InvalidAuthorityIndex"); - } - { - let error = ConsensusError::InsufficientParentStakes { - parent_stakes: 5, - quorum: 20, - }; - assert_eq!(error.name(), "InsufficientParentStakes"); - } - } -} diff --git a/consensus/core/src/leader_schedule.rs b/consensus/core/src/leader_schedule.rs deleted file mode 100644 index ec04a373aa5..00000000000 --- a/consensus/core/src/leader_schedule.rs +++ /dev/null @@ -1,1368 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::BTreeMap, - fmt::{Debug, Formatter}, - sync::Arc, -}; - -use consensus_config::{AuthorityIndex, Stake}; -use parking_lot::RwLock; -use rand::{SeedableRng, prelude::SliceRandom, rngs::StdRng}; - -use crate::{ - CommitIndex, Round, - commit::CommitRange, - context::Context, - dag_state::DagState, - leader_scoring::{ReputationScoreCalculator, ReputationScores}, -}; - -/// The `LeaderSchedule` is responsible for producing the leader schedule across -/// an epoch. The leader schedule is subject to change periodically based on -/// calculated `ReputationScores` of the authorities. -#[derive(Clone)] -pub(crate) struct LeaderSchedule { - pub leader_swap_table: Arc>, - context: Arc, - num_commits_per_schedule: u64, -} - -impl LeaderSchedule { - /// The window where the schedule change takes place in consensus. It - /// represents number of committed sub dags. - /// TODO: move this to protocol config - #[cfg(not(msim))] - const CONSENSUS_COMMITS_PER_SCHEDULE: u64 = 300; - #[cfg(msim)] - const CONSENSUS_COMMITS_PER_SCHEDULE: u64 = 10; - - pub(crate) fn new(context: Arc, leader_swap_table: LeaderSwapTable) -> Self { - Self { - context, - num_commits_per_schedule: Self::CONSENSUS_COMMITS_PER_SCHEDULE, - leader_swap_table: Arc::new(RwLock::new(leader_swap_table)), - } - } - - #[cfg(test)] - pub(crate) fn with_num_commits_per_schedule(mut self, num_commits_per_schedule: u64) -> Self { - self.num_commits_per_schedule = num_commits_per_schedule; - self - } - - /// Restores the `LeaderSchedule` from storage. It will attempt to retrieve - /// the last stored `ReputationScores` and use them to build a - /// `LeaderSwapTable`. - pub(crate) fn from_store(context: Arc, dag_state: Arc>) -> Self { - let leader_swap_table = dag_state.read().recover_last_commit_info().map_or( - LeaderSwapTable::default(), - |(last_commit_ref, last_commit_info)| { - LeaderSwapTable::new( - context.clone(), - last_commit_ref.index, - last_commit_info.reputation_scores, - ) - }, - ); - - if context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - tracing::info!( - "LeaderSchedule recovered using {leader_swap_table:?}. There are {} committed subdags scored in DagState.", - dag_state.read().scoring_subdags_count(), - ); - } else { - // TODO: Remove when DistributedVoteScoring is enabled. - tracing::info!( - "LeaderSchedule recovered using {leader_swap_table:?}. There are {} pending unscored subdags in DagState.", - dag_state.read().unscored_committed_subdags_count(), - ); - } - - // create the schedule - Self::new(context, leader_swap_table) - } - - pub(crate) fn commits_until_leader_schedule_update( - &self, - dag_state: Arc>, - ) -> usize { - let subdag_count = if self - .context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - dag_state.read().scoring_subdags_count() as u64 - } else { - // TODO: Remove when DistributedVoteScoring is enabled. - dag_state.read().unscored_committed_subdags_count() - }; - assert!( - subdag_count <= self.num_commits_per_schedule, - "Committed subdags count exceeds the number of commits per schedule" - ); - self.num_commits_per_schedule - .checked_sub(subdag_count) - .unwrap() as usize - } - - /// Checks whether the dag state sub dags list is empty. If yes - /// then that means that either (1) the system has just started and - /// there is no unscored sub dag available (2) the schedule has updated, - /// new scores have been calculated. Both cases we consider as valid cases - /// where the schedule has been updated. - pub(crate) fn leader_schedule_updated(&self, dag_state: &RwLock) -> bool { - if self - .context - .protocol_config - .consensus_distributed_vote_scoring_strategy() - { - dag_state.read().is_scoring_subdag_empty() - } else { - // TODO: Remove when DistributedVoteScoring is enabled. - dag_state.read().unscored_committed_subdags_count() == 0 - } - } - - pub(crate) fn update_leader_schedule_v2(&self, dag_state: &RwLock) { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["LeaderSchedule::update_leader_schedule"]) - .start_timer(); - let (reputation_scores, last_commit_index) = { - let dag_state = dag_state.read(); - let reputation_scores = dag_state.calculate_scoring_subdag_scores(); - let last_commit_index = dag_state.scoring_subdag_commit_range(); - (reputation_scores, last_commit_index) - }; - { - let mut dag_state = dag_state.write(); - // Clear scoring subdag as we have updated the leader schedule - dag_state.clear_scoring_subdag(); - // Buffer score and last commit rounds in dag state to be persisted later - dag_state.add_commit_info(reputation_scores.clone()); - } - self.update_leader_swap_table(LeaderSwapTable::new( - self.context.clone(), - last_commit_index, - reputation_scores.clone(), - )); - reputation_scores.update_metrics(self.context.clone()); - self.context - .metrics - .node_metrics - .num_of_bad_nodes - .set(self.leader_swap_table.read().bad_nodes.len() as i64); - } - - // TODO: Remove when DistributedVoteScoring is enabled. - pub(crate) fn update_leader_schedule_v1(&self, dag_state: &RwLock) { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["LeaderSchedule::update_leader_schedule"]) - .start_timer(); - - let mut dag_state = dag_state.write(); - let unscored_subdags = dag_state.take_unscored_committed_subdags(); - - let score_calculation_timer = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["ReputationScoreCalculator::calculate"]) - .start_timer(); - let reputation_scores = - ReputationScoreCalculator::new(self.context.clone(), &unscored_subdags).calculate(); - drop(score_calculation_timer); - - reputation_scores.update_metrics(self.context.clone()); - - let last_commit_index = unscored_subdags.last().unwrap().commit_ref.index; - self.update_leader_swap_table(LeaderSwapTable::new( - self.context.clone(), - last_commit_index, - reputation_scores.clone(), - )); - - self.context - .metrics - .node_metrics - .num_of_bad_nodes - .set(self.leader_swap_table.read().bad_nodes.len() as i64); - - // Buffer score and last commit rounds in dag state to be persisted later - dag_state.add_commit_info(reputation_scores); - } - - pub(crate) fn elect_leader(&self, round: u32, leader_offset: u32) -> AuthorityIndex { - cfg_if::cfg_if! { - // TODO: we need to differentiate the leader strategy in tests, so for - // some type of testing (ex sim tests) we can use the staked approach. - if #[cfg(test)] { - let leader = AuthorityIndex::new_for_test((round + leader_offset) % self.context.committee.size() as u32); - let table = self.leader_swap_table.read(); - table.swap(leader, round, leader_offset).unwrap_or(leader) - } else { - let leader = self.elect_leader_stake_based(round, leader_offset); - let table = self.leader_swap_table.read(); - table.swap(leader, round, leader_offset).unwrap_or(leader) - } - } - } - - pub(crate) fn elect_leader_stake_based(&self, round: u32, offset: u32) -> AuthorityIndex { - assert!((offset as usize) < self.context.committee.size()); - - // To ensure that we elect different leaders for the same round (using - // different offset) we are using the round number as seed to shuffle in - // a weighted way the results, but skip based on the offset. - // TODO: use a cache in case this proves to be computationally expensive - let mut seed_bytes = [0u8; 32]; - seed_bytes[32 - 4..].copy_from_slice(&(round).to_le_bytes()); - let mut rng = StdRng::from_seed(seed_bytes); - - let choices = self - .context - .committee - .authorities() - .map(|(index, authority)| (index, authority.stake as f32)) - .collect::>(); - - let leader_index = *choices - .choose_multiple_weighted(&mut rng, self.context.committee.size(), |item| item.1) - .expect("Weighted choice error: stake values incorrect!") - .skip(offset as usize) - .map(|(index, _)| index) - .next() - .unwrap(); - - leader_index - } - - /// Atomically updates the `LeaderSwapTable` with the new provided one. Any - /// leader queried from now on will get calculated according to this swap - /// table until a new one is provided again. - fn update_leader_swap_table(&self, table: LeaderSwapTable) { - let read = self.leader_swap_table.read(); - let old_commit_range = &read.reputation_scores.commit_range; - let new_commit_range = &table.reputation_scores.commit_range; - - // Unless LeaderSchedule is brand new and using the default commit range - // of CommitRange(0..0) all future LeaderSwapTables should be calculated - // from a CommitRange of equal length and immediately following the - // preceding commit range of the old swap table. - if *old_commit_range != CommitRange::default() { - assert!( - old_commit_range.is_next_range(new_commit_range) - && old_commit_range.is_equal_size(new_commit_range), - "The new LeaderSwapTable has an invalid CommitRange. Old LeaderSwapTable {old_commit_range:?} vs new LeaderSwapTable {new_commit_range:?}", - ); - } - drop(read); - - tracing::trace!("Updating {table:?}"); - - let mut write = self.leader_swap_table.write(); - *write = table; - } -} - -#[derive(Default, Clone)] -pub(crate) struct LeaderSwapTable { - /// The list of `f` (by configurable stake) authorities with best scores as - /// those defined by the provided `ReputationScores`. Those authorities will - /// be used in the position of the `bad_nodes` on the final leader schedule. - /// Storing the hostname & stake along side the authority index for - /// debugging. - pub(crate) good_nodes: Vec<(AuthorityIndex, String, Stake)>, - - /// The set of `f` (by configurable stake) authorities with the worst scores - /// as those defined by the provided `ReputationScores`. Every time where - /// such authority is elected as leader on the schedule, it will swapped - /// by one of the authorities of the `good_nodes`. - /// Storing the hostname & stake along side the authority index for - /// debugging. - pub(crate) bad_nodes: BTreeMap, - - /// Scores by authority in descending order, needed by other parts of the - /// system for a consistent view on how each validator performs in - /// consensus. - pub(crate) reputation_scores_desc: Vec<(AuthorityIndex, u64)>, - - // The scores for which the leader swap table was built from. This struct is - // used for debugging purposes. Once `good_nodes` & `bad_nodes` are identified - // the `reputation_scores` are no longer needed functionally for the swap table. - pub(crate) reputation_scores: ReputationScores, -} - -impl LeaderSwapTable { - // Constructs a new table based on the provided reputation scores. The - // `swap_stake_threshold` designates the total (by stake) nodes that will be - // considered as "bad" based on their scores and will be replaced by good nodes. - // The `swap_stake_threshold` should be in the range of [0 - 33]. - pub(crate) fn new( - context: Arc, - commit_index: CommitIndex, - reputation_scores: ReputationScores, - ) -> Self { - let swap_stake_threshold = context - .protocol_config - .consensus_bad_nodes_stake_threshold(); - Self::new_inner( - context, - swap_stake_threshold, - commit_index, - reputation_scores, - ) - } - - fn new_inner( - context: Arc, - // Ignore linter warning in simtests. - // TODO: maybe override protocol configs in tests for swap_stake_threshold, and call new(). - #[cfg_attr(msim, expect(unused_variables))] swap_stake_threshold: u64, - commit_index: CommitIndex, - reputation_scores: ReputationScores, - ) -> Self { - #[cfg(msim)] - let swap_stake_threshold = 33; - - assert!( - (0..=33).contains(&swap_stake_threshold), - "The swap_stake_threshold ({swap_stake_threshold}) should be in range [0 - 33], out of bounds parameter detected" - ); - - // When reputation scores are disabled or at genesis, use the default value. - if reputation_scores.scores_per_authority.is_empty() { - return Self::default(); - } - - // Randomize order of authorities when they have the same score, - // to avoid bias in the selection of the good and bad nodes. - let mut seed_bytes = [0u8; 32]; - seed_bytes[28..32].copy_from_slice(&commit_index.to_le_bytes()); - let mut rng = StdRng::from_seed(seed_bytes); - let mut authorities_by_score = reputation_scores.authorities_by_score(context.clone()); - assert_eq!(authorities_by_score.len(), context.committee.size()); - authorities_by_score.shuffle(&mut rng); - // Stable sort the authorities by score descending. Order of authorities with - // the same score is preserved. - authorities_by_score.sort_by(|a1, a2| a2.1.cmp(&a1.1)); - - // Calculating the good nodes - let good_nodes = Self::retrieve_first_nodes( - context.clone(), - authorities_by_score.iter(), - swap_stake_threshold, - ) - .into_iter() - .collect::>(); - - // Calculating the bad nodes - // Reverse the sorted authorities to score ascending so we get the first - // low scorers up to the provided stake threshold. - let bad_nodes = Self::retrieve_first_nodes( - context, - authorities_by_score.iter().rev(), - swap_stake_threshold, - ) - .into_iter() - .map(|(idx, hostname, stake)| (idx, (hostname, stake))) - .collect::>(); - - good_nodes.iter().for_each(|(idx, hostname, stake)| { - tracing::debug!( - "Good node {hostname} with stake {stake} has score {} for {:?}", - reputation_scores.scores_per_authority[idx.to_owned()], - reputation_scores.commit_range, - ); - }); - - bad_nodes.iter().for_each(|(idx, (hostname, stake))| { - tracing::debug!( - "Bad node {hostname} with stake {stake} has score {} for {:?}", - reputation_scores.scores_per_authority[idx.to_owned()], - reputation_scores.commit_range, - ); - }); - - tracing::info!("Scores used for new LeaderSwapTable: {reputation_scores:?}"); - - Self { - good_nodes, - bad_nodes, - reputation_scores_desc: authorities_by_score, - reputation_scores, - } - } - - /// Checks whether the provided leader is a bad performer and needs to be - /// swapped in the schedule with a good performer. If not, then the method - /// returns None. Otherwise the leader to swap with is returned instead. The - /// `leader_round` & `leader_offset` represents the DAG slot on which the - /// provided `AuthorityIndex` is a leader on and is used as a seed to random - /// function in order to calculate the good node that will swap in that - /// round with the bad node. We are intentionally not doing weighted - /// randomness as we want to give to all the good nodes equal - /// opportunity to get swapped with bad nodes and nothave one node with - /// enough stake end up swapping bad nodes more frequently than the - /// others on the final schedule. - pub(crate) fn swap( - &self, - leader: AuthorityIndex, - leader_round: Round, - leader_offset: u32, - ) -> Option { - if self.bad_nodes.contains_key(&leader) { - // TODO: Re-work swap for the multileader case - assert!( - leader_offset == 0, - "Swap for multi-leader case not implemented yet." - ); - let mut seed_bytes = [0u8; 32]; - seed_bytes[24..28].copy_from_slice(&leader_round.to_le_bytes()); - seed_bytes[28..32].copy_from_slice(&leader_offset.to_le_bytes()); - let mut rng = StdRng::from_seed(seed_bytes); - - let (idx, _hostname, _stake) = self - .good_nodes - .choose(&mut rng) - .expect("There should be at least one good node available"); - - tracing::trace!( - "Swapping bad leader {} -> {} for round {}", - leader, - idx, - leader_round - ); - - return Some(*idx); - } - None - } - - /// Retrieves the first nodes provided by the iterator `authorities` until - /// the `stake_threshold` has been reached. The `stake_threshold` should - /// be between [0, 100] and expresses the percentage of stake that is - /// considered the cutoff. It's the caller's responsibility to ensure - /// that the elements of the `authorities` input is already sorted. - fn retrieve_first_nodes<'a>( - context: Arc, - authorities: impl Iterator, - stake_threshold: u64, - ) -> Vec<(AuthorityIndex, String, Stake)> { - let mut filtered_authorities = Vec::new(); - - let mut stake = 0; - for &(authority_idx, _score) in authorities { - stake += context.committee.stake(authority_idx); - - // If the total accumulated stake has surpassed the stake threshold - // then we omit this last authority and we exit the loop. Important to - // note that this means if the threshold is too low we may not have - // any nodes returned. - if stake > (stake_threshold * context.committee.total_stake()) / 100 as Stake { - break; - } - - let authority = context.committee.authority(authority_idx); - filtered_authorities.push((authority_idx, authority.hostname.clone(), authority.stake)); - } - - filtered_authorities - } -} - -impl Debug for LeaderSwapTable { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&format!( - "LeaderSwapTable for {:?}, good_nodes: {:?} with stake: {}, bad_nodes: {:?} with stake: {}", - self.reputation_scores.commit_range, - self.good_nodes - .iter() - .map(|(idx, _hostname, _stake)| idx.to_owned()) - .collect::>(), - self.good_nodes - .iter() - .map(|(_idx, _hostname, stake)| stake) - .sum::(), - self.bad_nodes.keys().map(|idx| idx.to_owned()), - self.bad_nodes - .values() - .map(|(_hostname, stake)| stake) - .sum::(), - )) - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use crate::{ - block::{BlockDigest, BlockRef, BlockTimestampMs, TestBlock, VerifiedBlock}, - commit::{CommitDigest, CommitInfo, CommitRef, CommittedSubDag, TrustedCommit}, - storage::{Store, WriteBatch, mem_store::MemStore}, - test_dag_builder::DagBuilder, - }; - - #[tokio::test] - async fn test_elect_leader() { - let context = Arc::new(Context::new_for_test(4).0); - let leader_schedule = LeaderSchedule::new(context, LeaderSwapTable::default()); - - assert_eq!( - leader_schedule.elect_leader(0, 0), - AuthorityIndex::new_for_test(0) - ); - assert_eq!( - leader_schedule.elect_leader(1, 0), - AuthorityIndex::new_for_test(1) - ); - assert_eq!( - leader_schedule.elect_leader(5, 0), - AuthorityIndex::new_for_test(1) - ); - // ensure we elect different leaders for the same round for the multi-leader - // case - assert_ne!( - leader_schedule.elect_leader_stake_based(1, 1), - leader_schedule.elect_leader_stake_based(1, 2) - ); - } - - #[tokio::test] - async fn test_elect_leader_stake_based() { - let context = Arc::new(Context::new_for_test(4).0); - let leader_schedule = LeaderSchedule::new(context, LeaderSwapTable::default()); - - assert_eq!( - leader_schedule.elect_leader_stake_based(0, 0), - AuthorityIndex::new_for_test(1) - ); - assert_eq!( - leader_schedule.elect_leader_stake_based(1, 0), - AuthorityIndex::new_for_test(1) - ); - assert_eq!( - leader_schedule.elect_leader_stake_based(5, 0), - AuthorityIndex::new_for_test(3) - ); - // ensure we elect different leaders for the same round for the multi-leader - // case - assert_ne!( - leader_schedule.elect_leader_stake_based(1, 1), - leader_schedule.elect_leader_stake_based(1, 2) - ); - } - - #[tokio::test] - async fn test_leader_schedule_from_store() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - - // Populate fully connected test blocks for round 0 ~ 11, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=11).build(); - let mut subdags = vec![]; - let mut expected_commits = vec![]; - - let mut blocks_to_write = vec![]; - - for (sub_dag, commit) in dag_builder.get_sub_dag_and_commits(1..=11) { - for block in sub_dag.blocks.iter() { - blocks_to_write.push(block.clone()); - } - - expected_commits.push(commit); - subdags.push(sub_dag); - } - - // The CommitInfo for the first 10 commits are written to store. This is the - // info that LeaderSchedule will be recovered from - let commit_range = (1..=10).into(); - let reputation_scores = ReputationScores::new(commit_range, vec![4, 1, 1, 3]); - let committed_rounds = vec![9, 9, 10, 9]; - let commit_ref = expected_commits[9].reference(); - let commit_info = CommitInfo { - reputation_scores, - committed_rounds, - }; - - // CommitIndex '11' will be written to store. This should result in the cached - // last_committed_rounds & unscored subdags in DagState to be updated with the - // latest commit information on recovery. - store - .write( - WriteBatch::default() - .commit_info(vec![(commit_ref, commit_info)]) - .blocks(blocks_to_write) - .commits(expected_commits), - ) - .unwrap(); - - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - dag_builder.last_committed_rounds.clone(), - dag_state.read().last_committed_rounds() - ); - assert_eq!(1, dag_state.read().scoring_subdags_count()); - let recovered_scores = dag_state.read().calculate_scoring_subdag_scores(); - let expected_scores = ReputationScores::new((11..=11).into(), vec![0, 0, 0, 0]); - assert_eq!(recovered_scores, expected_scores); - - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 1); - assert_eq!( - leader_swap_table.good_nodes[0].0, - AuthorityIndex::new_for_test(0) - ); - assert_eq!(leader_swap_table.bad_nodes.len(), 1); - assert!( - leader_swap_table - .bad_nodes - .contains_key(&AuthorityIndex::new_for_test(2)), - "{:?}", - leader_swap_table.bad_nodes - ); - } - - #[tokio::test] - async fn test_leader_schedule_from_store_no_commits() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let expected_last_committed_rounds = vec![0, 0, 0, 0]; - - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - expected_last_committed_rounds, - dag_state.read().last_committed_rounds() - ); - assert_eq!(0, dag_state.read().scoring_subdags_count()); - - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 0); - assert_eq!(leader_swap_table.bad_nodes.len(), 0); - } - - #[tokio::test] - async fn test_leader_schedule_from_store_no_commit_info() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - - // Populate fully connected test blocks for round 0 ~ 2, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=2).build(); - - let mut expected_scored_subdags = vec![]; - let mut expected_commits = vec![]; - - let mut blocks_to_write = vec![]; - - for (sub_dag, commit) in dag_builder.get_sub_dag_and_commits(1..=2) { - for block in sub_dag.blocks.iter() { - blocks_to_write.push(block.clone()); - } - expected_commits.push(commit); - expected_scored_subdags.push(sub_dag); - } - - // The CommitInfo for the first 2 commits are written to store. 10 commits - // would have been required for a leader schedule update so at this point - // no commit info should have been persisted and no leader schedule should - // be recovered. However dag state should have properly recovered the - // unscored subdags & last committed rounds. - store - .write( - WriteBatch::default() - .blocks(blocks_to_write) - .commits(expected_commits), - ) - .unwrap(); - - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - dag_builder.last_committed_rounds.clone(), - dag_state.read().last_committed_rounds() - ); - assert_eq!( - expected_scored_subdags.len(), - dag_state.read().scoring_subdags_count() - ); - let recovered_scores = dag_state.read().calculate_scoring_subdag_scores(); - let expected_scores = ReputationScores::new((1..=2).into(), vec![0, 0, 0, 0]); - assert_eq!(recovered_scores, expected_scores); - - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 0); - assert_eq!(leader_swap_table.bad_nodes.len(), 0); - } - - #[tokio::test] - async fn test_leader_schedule_commits_until_leader_schedule_update() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let leader_schedule = LeaderSchedule::new(context.clone(), LeaderSwapTable::default()); - - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let unscored_subdags = vec![CommittedSubDag::new( - BlockRef::new(1, AuthorityIndex::ZERO, BlockDigest::MIN), - vec![], - context.clock.timestamp_utc_ms(), - CommitRef::new(1, CommitDigest::MIN), - vec![], - )]; - dag_state.write().add_scoring_subdags(unscored_subdags); - - let commits_until_leader_schedule_update = - leader_schedule.commits_until_leader_schedule_update(dag_state); - assert_eq!(commits_until_leader_schedule_update, 299); - } - - #[tokio::test] - async fn test_leader_schedule_update_leader_schedule() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - - // Populate fully connected test blocks for round 0 ~ 4, authorities 0 ~ 3. - let max_round: u32 = 4; - let num_authorities: u32 = 4; - - let mut blocks = Vec::new(); - let (genesis_references, genesis): (Vec<_>, Vec<_>) = context - .committee - .authorities() - .map(|index| { - let author_idx = index.0.value() as u32; - let block = TestBlock::new(0, author_idx).build(); - VerifiedBlock::new_for_test(block) - }) - .map(|block| (block.reference(), block)) - .unzip(); - blocks.extend(genesis); - - let mut ancestors = genesis_references; - let mut leader = None; - for round in 1..=max_round { - let mut new_ancestors = vec![]; - for author in 0..num_authorities { - let base_ts = round as BlockTimestampMs * 1000; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_timestamp_ms(base_ts + (author + round) as u64) - .set_ancestors(ancestors.clone()) - .build(), - ); - new_ancestors.push(block.reference()); - - // Simulate referenced block which was part of another committed - // subdag. - if round == 3 && author == 0 { - tracing::info!("Skipping {block} in committed subdags blocks"); - continue; - } - - blocks.push(block.clone()); - - // only write one block for the final round, which is the leader - // of the committed subdag. - if round == max_round { - leader = Some(block.clone()); - break; - } - } - ancestors = new_ancestors; - } - - let leader_block = leader.unwrap(); - let leader_ref = leader_block.reference(); - let commit_index = 1; - - let last_commit = TrustedCommit::new_for_test( - commit_index, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - leader_ref, - blocks - .iter() - .map(|block| block.reference()) - .collect::>(), - ); - - let unscored_subdags = vec![CommittedSubDag::new( - leader_ref, - blocks, - context.clock.timestamp_utc_ms(), - last_commit.reference(), - vec![], - )]; - - let mut dag_state_write = dag_state.write(); - dag_state_write.set_last_commit(last_commit); - dag_state_write.add_scoring_subdags(unscored_subdags); - drop(dag_state_write); - - assert_eq!( - leader_schedule.elect_leader(4, 0), - AuthorityIndex::new_for_test(0) - ); - - leader_schedule.update_leader_schedule_v2(&dag_state); - - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 1); - assert_eq!( - leader_swap_table.good_nodes[0].0, - AuthorityIndex::new_for_test(2) - ); - assert_eq!(leader_swap_table.bad_nodes.len(), 1); - assert!( - leader_swap_table - .bad_nodes - .contains_key(&AuthorityIndex::new_for_test(0)) - ); - assert_eq!( - leader_schedule.elect_leader(4, 0), - AuthorityIndex::new_for_test(2) - ); - } - - #[tokio::test] - async fn test_leader_schedule_from_store_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - // Populate fully connected test blocks for round 0 ~ 11, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=11).build(); - let mut subdags = vec![]; - let mut expected_commits = vec![]; - let mut blocks_to_write = vec![]; - - for (sub_dag, commit) in dag_builder.get_sub_dag_and_commits(1..=11) { - for block in sub_dag.blocks.iter() { - blocks_to_write.push(block.clone()); - } - expected_commits.push(commit); - subdags.push(sub_dag); - } - // The CommitInfo for the first 10 commits are written to store. This is the - // info that LeaderSchedule will be recovered from - let commit_range = (1..=10).into(); - let reputation_scores = ReputationScores::new(commit_range, vec![4, 1, 1, 3]); - let committed_rounds = vec![9, 9, 10, 9]; - let commit_ref = expected_commits[9].reference(); - let commit_info = CommitInfo { - reputation_scores, - committed_rounds, - }; - // CommitIndex '11' will be written to store. This should result in the cached - // last_committed_rounds & unscored subdags in DagState to be updated with the - // latest commit information on recovery. - store - .write( - WriteBatch::default() - .commit_info(vec![(commit_ref, commit_info)]) - .blocks(blocks_to_write) - .commits(expected_commits), - ) - .unwrap(); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - dag_builder.last_committed_rounds.clone(), - dag_state.read().last_committed_rounds() - ); - let actual_unscored_subdags = dag_state.read().unscored_committed_subdags(); - assert_eq!(1, dag_state.read().unscored_committed_subdags_count()); - let actual_subdag = actual_unscored_subdags[0].clone(); - assert_eq!(*subdags.last().unwrap(), actual_subdag); - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 1); - assert_eq!( - leader_swap_table.good_nodes[0].0, - AuthorityIndex::new_for_test(0) - ); - assert_eq!(leader_swap_table.bad_nodes.len(), 1); - assert!( - leader_swap_table - .bad_nodes - .contains_key(&AuthorityIndex::new_for_test(2)), - "{:?}", - leader_swap_table.bad_nodes - ); - } - - #[tokio::test] - async fn test_leader_schedule_from_store_no_commits_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let expected_last_committed_rounds = vec![0, 0, 0, 0]; - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - expected_last_committed_rounds, - dag_state.read().last_committed_rounds() - ); - assert_eq!(0, dag_state.read().unscored_committed_subdags_count()); - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 0); - assert_eq!(leader_swap_table.bad_nodes.len(), 0); - } - - #[tokio::test] - async fn test_leader_schedule_from_store_no_commit_info_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - // Populate fully connected test blocks for round 0 ~ 2, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=2).build(); - - let mut expected_unscored_subdags = vec![]; - let mut expected_commits = vec![]; - let mut blocks_to_write = vec![]; - - for (sub_dag, commit) in dag_builder.get_sub_dag_and_commits(1..=2) { - for block in sub_dag.blocks.iter() { - blocks_to_write.push(block.clone()); - } - expected_commits.push(commit); - expected_unscored_subdags.push(sub_dag); - } - // The CommitInfo for the first 2 commits are written to store. 10 commits - // would have been required for a leader schedule update so at this point - // no commit info should have been persisted and no leader schedule should - // be recovered. However dag state should have properly recovered the - // unscored subdags & last committed rounds. - store - .write( - WriteBatch::default() - .blocks(blocks_to_write) - .commits(expected_commits), - ) - .unwrap(); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - // Check that DagState recovery from stored CommitInfo worked correctly - assert_eq!( - dag_builder.last_committed_rounds.clone(), - dag_state.read().last_committed_rounds() - ); - let actual_unscored_subdags = dag_state.read().unscored_committed_subdags(); - assert_eq!( - expected_unscored_subdags.len() as u64, - dag_state.read().unscored_committed_subdags_count() - ); - for (idx, expected_subdag) in expected_unscored_subdags.into_iter().enumerate() { - let actual_subdag = actual_unscored_subdags[idx].clone(); - assert_eq!(expected_subdag, actual_subdag); - } - let leader_schedule = LeaderSchedule::from_store(context, dag_state); - // Check that LeaderSchedule recovery from stored CommitInfo worked correctly - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 0); - assert_eq!(leader_swap_table.bad_nodes.len(), 0); - } - - #[tokio::test] - async fn test_leader_schedule_commits_until_leader_schedule_update_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - let context = Arc::new(context); - let leader_schedule = LeaderSchedule::new(context.clone(), LeaderSwapTable::default()); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let unscored_subdags = vec![CommittedSubDag::new( - BlockRef::new(1, AuthorityIndex::ZERO, BlockDigest::MIN), - vec![], - context.clock.timestamp_utc_ms(), - CommitRef::new(1, CommitDigest::MIN), - vec![], - )]; - dag_state - .write() - .add_unscored_committed_subdags(unscored_subdags); - let commits_until_leader_schedule_update = - leader_schedule.commits_until_leader_schedule_update(dag_state); - assert_eq!(commits_until_leader_schedule_update, 299); - } - - // TODO: Remove when DistributedVoteScoring is enabled. - #[tokio::test] - async fn test_leader_schedule_update_leader_schedule_with_vote_scoring() { - telemetry_subscribers::init_for_testing(); - let mut context = Context::new_for_test(4).0; - context - .protocol_config - .set_consensus_distributed_vote_scoring_strategy_for_testing(false); - context - .protocol_config - .set_consensus_bad_nodes_stake_threshold_for_testing(33); - let context = Arc::new(context); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - // Populate fully connected test blocks for round 0 ~ 4, authorities 0 ~ 3. - let max_round: u32 = 4; - let num_authorities: u32 = 4; - let mut blocks = Vec::new(); - let (genesis_references, genesis): (Vec<_>, Vec<_>) = context - .committee - .authorities() - .map(|index| { - let author_idx = index.0.value() as u32; - let block = TestBlock::new(0, author_idx).build(); - VerifiedBlock::new_for_test(block) - }) - .map(|block| (block.reference(), block)) - .unzip(); - blocks.extend(genesis); - let mut ancestors = genesis_references; - let mut leader = None; - for round in 1..=max_round { - let mut new_ancestors = vec![]; - for author in 0..num_authorities { - let base_ts = round as BlockTimestampMs * 1000; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_timestamp_ms(base_ts + (author + round) as u64) - .set_ancestors(ancestors.clone()) - .build(), - ); - new_ancestors.push(block.reference()); - // Simulate referenced block which was part of another committed - // subdag. - if round == 3 && author == 0 { - tracing::info!("Skipping {block} in committed subdags blocks"); - continue; - } - blocks.push(block.clone()); - // only write one block for the final round, which is the leader - // of the committed subdag. - if round == max_round { - leader = Some(block.clone()); - break; - } - } - ancestors = new_ancestors; - } - let leader_block = leader.unwrap(); - let leader_ref = leader_block.reference(); - let commit_index = 1; - let last_commit = TrustedCommit::new_for_test( - commit_index, - CommitDigest::MIN, - context.clock.timestamp_utc_ms(), - leader_ref, - blocks - .iter() - .map(|block| block.reference()) - .collect::>(), - ); - let unscored_subdags = vec![CommittedSubDag::new( - leader_ref, - blocks, - context.clock.timestamp_utc_ms(), - last_commit.reference(), - vec![], - )]; - let mut dag_state_write = dag_state.write(); - dag_state_write.set_last_commit(last_commit); - dag_state_write.add_unscored_committed_subdags(unscored_subdags); - drop(dag_state_write); - assert_eq!( - leader_schedule.elect_leader(4, 0), - AuthorityIndex::new_for_test(0) - ); - leader_schedule.update_leader_schedule_v1(&dag_state); - let leader_swap_table = leader_schedule.leader_swap_table.read(); - assert_eq!(leader_swap_table.good_nodes.len(), 1); - assert_eq!( - leader_swap_table.good_nodes[0].0, - AuthorityIndex::new_for_test(2) - ); - assert_eq!(leader_swap_table.bad_nodes.len(), 1); - assert!( - leader_swap_table - .bad_nodes - .contains_key(&AuthorityIndex::new_for_test(0)) - ); - assert_eq!( - leader_schedule.elect_leader(4, 0), - AuthorityIndex::new_for_test(2) - ); - } - - #[tokio::test] - async fn test_leader_swap_table() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let swap_stake_threshold = 33; - let reputation_scores = ReputationScores::new( - (0..=10).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context, swap_stake_threshold, 0, reputation_scores); - - assert_eq!(leader_swap_table.good_nodes.len(), 1); - assert_eq!( - leader_swap_table.good_nodes[0].0, - AuthorityIndex::new_for_test(3) - ); - assert_eq!(leader_swap_table.bad_nodes.len(), 1); - assert!( - leader_swap_table - .bad_nodes - .contains_key(&AuthorityIndex::new_for_test(0)) - ); - } - - #[tokio::test] - async fn test_leader_swap_table_swap() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let swap_stake_threshold = 33; - let reputation_scores = ReputationScores::new( - (0..=10).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context, swap_stake_threshold, 0, reputation_scores); - - // Test swapping a bad leader - let leader = AuthorityIndex::new_for_test(0); - let leader_round = 1; - let leader_offset = 0; - let swapped_leader = leader_swap_table.swap(leader, leader_round, leader_offset); - assert_eq!(swapped_leader, Some(AuthorityIndex::new_for_test(3))); - - // Test not swapping a good leader - let leader = AuthorityIndex::new_for_test(1); - let leader_round = 1; - let leader_offset = 0; - let swapped_leader = leader_swap_table.swap(leader, leader_round, leader_offset); - assert_eq!(swapped_leader, None); - } - - #[tokio::test] - async fn test_leader_swap_table_retrieve_first_nodes() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let authorities = [ - (AuthorityIndex::new_for_test(0), 1), - (AuthorityIndex::new_for_test(1), 2), - (AuthorityIndex::new_for_test(2), 3), - (AuthorityIndex::new_for_test(3), 4), - ]; - - let stake_threshold = 50; - let filtered_authorities = LeaderSwapTable::retrieve_first_nodes( - context.clone(), - authorities.iter(), - stake_threshold, - ); - - // Test setup includes 4 validators with even stake. Therefore with a - // stake_threshold of 50% we should see 2 validators filtered. - assert_eq!(filtered_authorities.len(), 2); - let authority_0_idx = AuthorityIndex::new_for_test(0); - let authority_0 = context.committee.authority(authority_0_idx); - assert!(filtered_authorities.contains(&( - authority_0_idx, - authority_0.hostname.clone(), - authority_0.stake - ))); - let authority_1_idx = AuthorityIndex::new_for_test(1); - let authority_1 = context.committee.authority(authority_1_idx); - assert!(filtered_authorities.contains(&( - authority_1_idx, - authority_1.hostname.clone(), - authority_1.stake - ))); - } - - #[tokio::test] - #[should_panic( - expected = "The swap_stake_threshold (34) should be in range [0 - 33], out of bounds parameter detected" - )] - async fn test_leader_swap_table_swap_stake_threshold_out_of_bounds() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let swap_stake_threshold = 34; - let reputation_scores = ReputationScores::new( - (0..=10).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - LeaderSwapTable::new_inner(context, swap_stake_threshold, 0, reputation_scores); - } - - #[tokio::test] - async fn test_update_leader_swap_table() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let swap_stake_threshold = 33; - let reputation_scores = ReputationScores::new( - (1..=10).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context.clone(), swap_stake_threshold, 0, reputation_scores); - - let leader_schedule = LeaderSchedule::new(context.clone(), LeaderSwapTable::default()); - - // Update leader from brand new schedule to first real schedule - leader_schedule.update_leader_swap_table(leader_swap_table); - - let reputation_scores = ReputationScores::new( - (11..=20).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context, swap_stake_threshold, 0, reputation_scores); - - // Update leader from old swap table to new valid swap table - leader_schedule.update_leader_swap_table(leader_swap_table); - } - - #[tokio::test] - #[should_panic( - expected = "The new LeaderSwapTable has an invalid CommitRange. Old LeaderSwapTable CommitRange(11..=20) vs new LeaderSwapTable CommitRange(21..=25)" - )] - async fn test_update_bad_leader_swap_table() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let swap_stake_threshold = 33; - let reputation_scores = ReputationScores::new( - (1..=10).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context.clone(), swap_stake_threshold, 0, reputation_scores); - - let leader_schedule = LeaderSchedule::new(context.clone(), LeaderSwapTable::default()); - - // Update leader from brand new schedule to first real schedule - leader_schedule.update_leader_swap_table(leader_swap_table); - - let reputation_scores = ReputationScores::new( - (11..=20).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context.clone(), swap_stake_threshold, 0, reputation_scores); - - // Update leader from old swap table to new valid swap table - leader_schedule.update_leader_swap_table(leader_swap_table); - - let reputation_scores = ReputationScores::new( - (21..=25).into(), - (0..4).map(|i| i as u64).collect::>(), - ); - let leader_swap_table = - LeaderSwapTable::new_inner(context, swap_stake_threshold, 0, reputation_scores); - - // Update leader from old swap table to new invalid swap table - leader_schedule.update_leader_swap_table(leader_swap_table); - } -} diff --git a/consensus/core/src/leader_scoring.rs b/consensus/core/src/leader_scoring.rs deleted file mode 100644 index 883f58caed5..00000000000 --- a/consensus/core/src/leader_scoring.rs +++ /dev/null @@ -1,638 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, HashSet}, - fmt::Debug, - ops::Bound::{Excluded, Included}, - sync::Arc, -}; - -use consensus_config::AuthorityIndex; -use serde::{Deserialize, Serialize}; -use tracing::instrument; - -use crate::{ - Round, VerifiedBlock, - block::{BlockAPI, BlockDigest, BlockRef, Slot}, - commit::{CommitRange, CommittedSubDag}, - context::Context, - stake_aggregator::{QuorumThreshold, StakeAggregator}, -}; - -pub(crate) struct ReputationScoreCalculator { - // The range of commits that these scores are calculated from. - pub(crate) commit_range: CommitRange, - // The scores per authority. Vec index is the `AuthorityIndex`. - pub(crate) scores_per_authority: Vec, - - // As leaders are sequenced the subdags are collected and cached in `DagState`. - // Then when there are enough commits to trigger a `LeaderSchedule` change, - // the subdags are then combined into one `UnscoredSubdag` so that we can - // calculate the scores for the leaders in this subdag. - unscored_subdag: UnscoredSubdag, -} - -impl ReputationScoreCalculator { - pub(crate) fn new(context: Arc, unscored_subdags: &[CommittedSubDag]) -> Self { - let num_authorities = context.committee.size(); - let scores_per_authority = vec![0_u64; num_authorities]; - - assert!( - !unscored_subdags.is_empty(), - "Attempted to calculate scores with no unscored subdags" - ); - - let unscored_subdag = UnscoredSubdag::new(context, unscored_subdags); - let commit_range = unscored_subdag.commit_range.clone(); - - Self { - unscored_subdag, - commit_range, - scores_per_authority, - } - } - - pub(crate) fn calculate(&mut self) -> ReputationScores { - let leaders = self.unscored_subdag.committed_leaders.clone(); - for leader in leaders { - let leader_slot = Slot::from(leader); - tracing::trace!("Calculating score for leader {leader_slot}"); - self.add_scores(self.calculate_scores_for_leader(&self.unscored_subdag, leader_slot)); - } - - ReputationScores::new(self.commit_range.clone(), self.scores_per_authority.clone()) - } - - fn add_scores(&mut self, scores: Vec) { - assert_eq!(scores.len(), self.scores_per_authority.len()); - - for (authority_idx, score) in scores.iter().enumerate() { - self.scores_per_authority[authority_idx] += *score; - } - } - - // VoteScoringStrategy - // This scoring strategy will give one point to any votes for the leader. - fn calculate_scores_for_leader(&self, subdag: &UnscoredSubdag, leader_slot: Slot) -> Vec { - let num_authorities = subdag.context.committee.size(); - let mut scores_per_authority = vec![0_u64; num_authorities]; - let leader_blocks = subdag.get_blocks_at_slot(leader_slot); - if leader_blocks.is_empty() { - tracing::trace!( - "[{}] No block for leader slot {leader_slot} in this set of unscored committed subdags, skip scoring", - subdag.context.own_index - ); - return scores_per_authority; - } - // At this point we are guaranteed that there is only one leader per slot - // because we are operating on committed subdags. - assert!(leader_blocks.len() == 1); - let leader_block = leader_blocks.first().unwrap(); - let voting_round = leader_slot.round + 1; - let voting_blocks = subdag.get_blocks_at_round(voting_round); - for potential_vote in voting_blocks { - // TODO: use the decided leader as input instead of leader slot. If the leader - // was skipped, votes to skip should be included in the score as - // well. - if subdag.is_vote(&potential_vote, leader_block) { - let authority = potential_vote.author(); - tracing::trace!( - "Found a vote {} for leader {leader_block} from authority {authority}", - potential_vote.reference() - ); - tracing::trace!( - "[{}] scores +1 reputation for {authority}!", - subdag.context.own_index - ); - scores_per_authority[authority] += 1; - } - } - scores_per_authority - } -} - -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(crate) struct ReputationScores { - /// Score per authority. Vec index is the `AuthorityIndex`. - pub(crate) scores_per_authority: Vec, - // The range of commits these scores were calculated from. - pub(crate) commit_range: CommitRange, -} - -impl ReputationScores { - pub(crate) fn new(commit_range: CommitRange, scores_per_authority: Vec) -> Self { - Self { - scores_per_authority, - commit_range, - } - } - - pub(crate) fn highest_score(&self) -> u64 { - *self.scores_per_authority.iter().max().unwrap_or(&0) - } - - // Returns the authorities index with score tuples. - pub(crate) fn authorities_by_score(&self, context: Arc) -> Vec<(AuthorityIndex, u64)> { - self.scores_per_authority - .iter() - .enumerate() - .map(|(index, score)| { - ( - context - .committee - .to_authority_index(index) - .expect("Should be a valid AuthorityIndex"), - *score, - ) - }) - .collect() - } - - pub(crate) fn update_metrics(&self, context: Arc) { - for (index, score) in self.scores_per_authority.iter().enumerate() { - let authority_index = context - .committee - .to_authority_index(index) - .expect("Should be a valid AuthorityIndex"); - let authority = context.committee.authority(authority_index); - if !authority.hostname.is_empty() { - context - .metrics - .node_metrics - .reputation_scores - .with_label_values(&[&authority.hostname]) - .set(*score as i64); - } - } - } -} - -/// ScoringSubdag represents the scoring votes in a collection of subdags across -/// multiple commits. -/// These subdags are "scoring" for the purposes of leader schedule change. As -/// new subdags are added, the DAG is traversed and votes for leaders are -/// recorded and scored along with stake. On a leader schedule change, finalized -/// reputation scores will be calculated based on the votes & stake collected in -/// this struct. -pub(crate) struct ScoringSubdag { - pub(crate) context: Arc, - pub(crate) commit_range: Option, - // Only includes committed leaders for now. - // TODO: Include skipped leaders as well - pub(crate) leaders: HashSet, - // A map of votes to the stake of strongly linked blocks that include that vote - // Note: Including stake aggregator so that we can quickly check if it exceeds - // quourum threshold and only include those scores for certain scoring strategies. - pub(crate) votes: BTreeMap>, -} - -impl ScoringSubdag { - pub(crate) fn new(context: Arc) -> Self { - Self { - context, - commit_range: None, - leaders: HashSet::new(), - votes: BTreeMap::new(), - } - } - - #[instrument(level = "trace", skip_all)] - pub(crate) fn add_subdags(&mut self, committed_subdags: Vec) { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["ScoringSubdag::add_unscored_committed_subdags"]) - .start_timer(); - for subdag in committed_subdags { - if let Some(commit_range) = self.commit_range.as_mut() { - commit_range.extend_to(subdag.commit_ref.index); - } else { - // If the commit range is not set, then set it to the range of the first - // committed subdag index. - self.commit_range = Some(CommitRange::new( - subdag.commit_ref.index..=subdag.commit_ref.index, - )); - } - // Add the committed leader to the list of leaders we will be scoring. - tracing::trace!("Adding new committed leader {} for scoring", subdag.leader); - self.leaders.insert(subdag.leader); - // Check each block in subdag. Blocks are in order so we should traverse the - // oldest blocks first - for block in subdag.blocks { - for ancestor in block.ancestors() { - // Weak links may point to blocks with lower round numbers - // than strong links. - if ancestor.round != block.round().saturating_sub(1) { - continue; - } - // If a blocks strong linked ancestor is in leaders, then - // it's a vote for leader. - if self.leaders.contains(ancestor) { - // There should never be duplicate references to blocks - // with strong linked ancestors to leader. - tracing::trace!( - "Found a vote {} for leader {ancestor} from authority {}", - block.reference(), - block.author() - ); - assert!( - self.votes - .insert(block.reference(), StakeAggregator::new()) - .is_none(), - "Vote {block} already exists. Duplicate vote found for leader {ancestor}" - ); - } - if let Some(stake) = self.votes.get_mut(ancestor) { - // Vote is strongly linked to a future block, so we - // consider this a distributed vote. - tracing::trace!( - "Found a distributed vote {ancestor} from authority {}", - ancestor.author - ); - stake.add(block.author(), &self.context.committee); - } - } - } - } - } - - // Iterate through votes and calculate scores for each authority based on - // distributed vote scoring strategy. - pub(crate) fn calculate_distributed_vote_scores(&self) -> ReputationScores { - let scores_per_authority = self.distributed_votes_scores(); - - // TODO: Normalize scores - ReputationScores::new( - self.commit_range - .clone() - .expect("CommitRange should be set if calculate_scores is called."), - scores_per_authority, - ) - } - - /// This scoring strategy aims to give scores based on overall vote - /// distribution. Instead of only giving one point for each vote that is - /// included in 2f+1 blocks. We give a score equal to the amount of - /// stake of all blocks that included the vote. - fn distributed_votes_scores(&self) -> Vec { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["ScoringSubdag::score_distributed_votes"]) - .start_timer(); - - let num_authorities = self.context.committee.size(); - let mut scores_per_authority = vec![0_u64; num_authorities]; - - for (vote, stake_agg) in self.votes.iter() { - let authority = vote.author; - let stake = stake_agg.stake(); - tracing::trace!( - "[{}] scores +{stake} reputation for {authority}!", - self.context.own_index, - ); - scores_per_authority[authority.value()] += stake; - } - scores_per_authority - } - - pub(crate) fn scored_subdags_count(&self) -> usize { - if let Some(commit_range) = &self.commit_range { - commit_range.size() - } else { - 0 - } - } - - pub(crate) fn is_empty(&self) -> bool { - self.leaders.is_empty() && self.votes.is_empty() && self.commit_range.is_none() - } - - pub(crate) fn clear(&mut self) { - self.leaders.clear(); - self.votes.clear(); - self.commit_range = None; - } -} - -/// UnscoredSubdag represents a collection of subdags across multiple commits. -/// These subdags are considered unscored for the purposes of leader schedule -/// change. On a leader schedule change, reputation scores will be calculated -/// based on the dags collected in this struct. Similar graph traversal methods -/// that are provided in DagState are also added here to help calculate the -/// scores. -pub(crate) struct UnscoredSubdag { - pub(crate) context: Arc, - pub(crate) commit_range: CommitRange, - pub(crate) committed_leaders: Vec, - // When the blocks are collected form the list of provided subdags we ensure - // that the CommittedSubDag instances are contiguous in commit index order. - // Therefore we can guarantee the blocks of UnscoredSubdag are also sorted - // via the commit index. - pub(crate) blocks: BTreeMap, -} - -impl UnscoredSubdag { - pub(crate) fn new(context: Arc, subdags: &[CommittedSubDag]) -> Self { - let mut committed_leaders = vec![]; - let blocks = subdags - .iter() - .enumerate() - .flat_map(|(subdag_index, subdag)| { - committed_leaders.push(subdag.leader); - if subdag_index == 0 { - subdag.blocks.iter() - } else { - let previous_subdag = &subdags[subdag_index - 1]; - let expected_next_subdag_index = previous_subdag.commit_ref.index + 1; - assert_eq!( - subdag.commit_ref.index, expected_next_subdag_index, - "Non-contiguous commit index (expected: {}, found: {})", - expected_next_subdag_index, subdag.commit_ref.index - ); - subdag.blocks.iter() - } - }) - .map(|block| (block.reference(), block.clone())) - .collect::>(); - - // Guaranteed to have a contiguous list of commit indices - let commit_range = CommitRange::new( - subdags.first().unwrap().commit_ref.index..=subdags.last().unwrap().commit_ref.index, - ); - - assert!( - !blocks.is_empty(), - "Attempted to create UnscoredSubdag with no blocks" - ); - - Self { - context, - commit_range, - committed_leaders, - blocks, - } - } - - pub(crate) fn find_supported_leader_block( - &self, - leader_slot: Slot, - from: &VerifiedBlock, - ) -> Option { - if from.round() < leader_slot.round { - return None; - } - for ancestor in from.ancestors() { - if Slot::from(*ancestor) == leader_slot { - return Some(*ancestor); - } - // Weak links may point to blocks with lower round numbers than strong links. - if ancestor.round <= leader_slot.round { - continue; - } - if let Some(ancestor) = self.get_block(ancestor) { - if let Some(support) = self.find_supported_leader_block(leader_slot, &ancestor) { - return Some(support); - } - } else { - // TODO: Add unit test for this case once dagbuilder is ready. - tracing::trace!( - "Potential vote's ancestor block not found in unscored committed subdags: {:?}", - ancestor - ); - return None; - } - } - None - } - - pub(crate) fn is_vote( - &self, - potential_vote: &VerifiedBlock, - leader_block: &VerifiedBlock, - ) -> bool { - let reference = leader_block.reference(); - let leader_slot = Slot::from(reference); - self.find_supported_leader_block(leader_slot, potential_vote) == Some(reference) - } - - pub(crate) fn get_blocks_at_slot(&self, slot: Slot) -> Vec { - let mut blocks = vec![]; - for (_block_ref, block) in self.blocks.range(( - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MIN)), - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MAX)), - )) { - blocks.push(block.clone()) - } - blocks - } - - pub(crate) fn get_blocks_at_round(&self, round: Round) -> Vec { - let mut blocks = vec![]; - for (_block_ref, block) in self.blocks.range(( - Included(BlockRef::new(round, AuthorityIndex::ZERO, BlockDigest::MIN)), - Excluded(BlockRef::new( - round + 1, - AuthorityIndex::ZERO, - BlockDigest::MIN, - )), - )) { - blocks.push(block.clone()) - } - blocks - } - - pub(crate) fn get_block(&self, block_ref: &BlockRef) -> Option { - self.blocks.get(block_ref).cloned() - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use crate::{CommitDigest, CommitRef, test_dag_builder::DagBuilder}; - - #[tokio::test] - async fn test_reputation_scores_authorities_by_score() { - let context = Arc::new(Context::new_for_test(4).0); - let scores = ReputationScores::new((1..=300).into(), vec![4, 1, 1, 3]); - let authorities = scores.authorities_by_score(context); - assert_eq!( - authorities, - vec![ - (AuthorityIndex::new_for_test(0), 4), - (AuthorityIndex::new_for_test(1), 1), - (AuthorityIndex::new_for_test(2), 1), - (AuthorityIndex::new_for_test(3), 3), - ] - ); - } - - #[tokio::test] - async fn test_reputation_scores_update_metrics() { - let context = Arc::new(Context::new_for_test(4).0); - let scores = ReputationScores::new((1..=300).into(), vec![1, 2, 4, 3]); - scores.update_metrics(context.clone()); - let metrics = context.metrics.node_metrics.reputation_scores.clone(); - assert_eq!( - metrics - .get_metric_with_label_values(&["test_host_0"]) - .unwrap() - .get(), - 1 - ); - assert_eq!( - metrics - .get_metric_with_label_values(&["test_host_1"]) - .unwrap() - .get(), - 2 - ); - assert_eq!( - metrics - .get_metric_with_label_values(&["test_host_2"]) - .unwrap() - .get(), - 4 - ); - assert_eq!( - metrics - .get_metric_with_label_values(&["test_host_3"]) - .unwrap() - .get(), - 3 - ); - } - - #[tokio::test] - async fn test_scoring_subdag() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - // Populate fully connected test blocks for round 0 ~ 3, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=3).build(); - // Build round 4 but with just the leader block - dag_builder - .layer(4) - .authorities(vec![ - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - AuthorityIndex::new_for_test(3), - ]) - .skip_block() - .build(); - - let mut scoring_subdag = ScoringSubdag::new(context); - - for (sub_dag, _commit) in dag_builder.get_sub_dag_and_commits(1..=4) { - scoring_subdag.add_subdags(vec![sub_dag]); - } - - let scores = scoring_subdag.calculate_distributed_vote_scores(); - assert_eq!(scores.scores_per_authority, vec![5, 5, 5, 5]); - assert_eq!(scores.commit_range, (1..=4).into()); - } - - // TODO: Remove all tests below this when DistributedVoteScoring is enabled. - #[tokio::test] - async fn test_reputation_score_calculator() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - // Populate fully connected test blocks for round 0 ~ 3, authorities 0 ~ 3. - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=3).build(); - // Build round 4 but with just the leader block - dag_builder - .layer(4) - .authorities(vec![ - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - AuthorityIndex::new_for_test(3), - ]) - .skip_block() - .build(); - - let mut unscored_subdags = vec![]; - for (sub_dag, _commit) in dag_builder.get_sub_dag_and_commits(1..=4) { - unscored_subdags.push(sub_dag); - } - let mut calculator = ReputationScoreCalculator::new(context, &unscored_subdags); - let scores = calculator.calculate(); - assert_eq!(scores.scores_per_authority, vec![3, 2, 2, 2]); - assert_eq!(scores.commit_range, (1..=4).into()); - } - - #[tokio::test] - #[should_panic(expected = "Attempted to calculate scores with no unscored subdags")] - async fn test_reputation_score_calculator_no_subdags() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let unscored_subdags = vec![]; - let mut calculator = ReputationScoreCalculator::new(context, &unscored_subdags); - calculator.calculate(); - } - - #[tokio::test] - #[should_panic(expected = "Attempted to create UnscoredSubdag with no blocks")] - async fn test_reputation_score_calculator_no_subdag_blocks() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let blocks = vec![]; - let unscored_subdags = vec![CommittedSubDag::new( - BlockRef::new(1, AuthorityIndex::ZERO, BlockDigest::MIN), - blocks, - context.clock.timestamp_utc_ms(), - CommitRef::new(1, CommitDigest::MIN), - vec![], - )]; - let mut calculator = ReputationScoreCalculator::new(context, &unscored_subdags); - calculator.calculate(); - } - - #[tokio::test] - async fn test_scoring_with_missing_block_in_subdag() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - - let mut dag_builder = DagBuilder::new(context.clone()); - // Build layer 1 with missing leader block, simulating it was committed - // as part of another committed subdag. - dag_builder - .layer(1) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - // Build fully connected layers 2 ~ 3. - dag_builder.layers(2..=3).build(); - // Build round 4 but with just the leader block - dag_builder - .layer(4) - .authorities(vec![ - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - AuthorityIndex::new_for_test(3), - ]) - .skip_block() - .build(); - - let mut unscored_subdags = vec![]; - for (sub_dag, _commit) in dag_builder.get_sub_dag_and_commits(1..=4) { - unscored_subdags.push(sub_dag); - } - - let mut calculator = ReputationScoreCalculator::new(context, &unscored_subdags); - let scores = calculator.calculate(); - assert_eq!(scores.scores_per_authority, vec![3, 2, 2, 2]); - assert_eq!(scores.commit_range, (1..=4).into()); - } -} diff --git a/consensus/core/src/leader_timeout.rs b/consensus/core/src/leader_timeout.rs deleted file mode 100644 index 776ad201dd7..00000000000 --- a/consensus/core/src/leader_timeout.rs +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use tokio::{ - sync::{ - oneshot::{Receiver, Sender}, - watch, - }, - task::JoinHandle, - time::{Instant, sleep_until}, -}; -use tracing::{debug, warn}; - -use crate::{ - block::Round, context::Context, core::CoreSignalsReceivers, core_thread::CoreThreadDispatcher, -}; - -pub(crate) struct LeaderTimeoutTaskHandle { - handle: JoinHandle<()>, - stop: Sender<()>, -} - -impl LeaderTimeoutTaskHandle { - pub async fn stop(self) { - self.stop.send(()).ok(); - self.handle.await.ok(); - } -} - -pub(crate) struct LeaderTimeoutTask { - dispatcher: Arc, - new_round_receiver: watch::Receiver, - leader_timeout: Duration, - min_round_delay: Duration, - stop: Receiver<()>, -} - -impl LeaderTimeoutTask { - /// Starts the leader timeout task, which monitors and manages the leader - /// election timeout mechanism. - pub fn start( - dispatcher: Arc, - signals_receivers: &CoreSignalsReceivers, - context: Arc, - ) -> LeaderTimeoutTaskHandle { - let (stop_sender, stop) = tokio::sync::oneshot::channel(); - let mut me = Self { - dispatcher, - stop, - new_round_receiver: signals_receivers.new_round_receiver(), - leader_timeout: context.parameters.leader_timeout, - min_round_delay: context.parameters.min_round_delay, - }; - let handle = tokio::spawn(async move { me.run().await }); - - LeaderTimeoutTaskHandle { - handle, - stop: stop_sender, - } - } - - /// Runs the leader timeout task, managing the leader election timeout - /// mechanism in an asynchronous loop. - /// This mechanism ensures that if the current leader fails to produce a new - /// block within the specified timeout, the task forces the creation of a - /// new block, maintaining the continuity and robustness of the leader - /// election process. - async fn run(&mut self) { - let new_round = &mut self.new_round_receiver; - let mut leader_round: Round = *new_round.borrow_and_update(); - let mut min_leader_round_timed_out = false; - let mut max_leader_round_timed_out = false; - let timer_start = Instant::now(); - let min_leader_timeout = sleep_until(timer_start + self.min_round_delay); - let max_leader_timeout = sleep_until(timer_start + self.leader_timeout); - - tokio::pin!(min_leader_timeout); - tokio::pin!(max_leader_timeout); - - loop { - tokio::select! { - // when the min leader timer expires then we attempt to trigger the creation of a new block. - // If we already timed out before then the branch gets disabled so we don't attempt - // all the time to produce already produced blocks for that round. - () = &mut min_leader_timeout, if !min_leader_round_timed_out => { - if let Err(err) = self.dispatcher.new_block(leader_round, false).await { - warn!("Error received while calling dispatcher, probably dispatcher is shutting down, will now exit: {err:?}"); - return; - } - min_leader_round_timed_out = true; - }, - // When the max leader timer expires then we attempt to trigger the creation of a new block. This - // call is made with `force = true` to bypass any checks that allow to propose immediately if block - // not already produced. - // Keep in mind that first the min timeout should get triggered and then the max timeout, only - // if the round has not advanced in the meantime. Otherwise, the max timeout will not get - // triggered at all. - () = &mut max_leader_timeout, if !max_leader_round_timed_out => { - if let Err(err) = self.dispatcher.new_block(leader_round, true).await { - warn!("Error received while calling dispatcher, probably dispatcher is shutting down, will now exit: {err:?}"); - return; - } - max_leader_round_timed_out = true; - } - - // a new round has been produced. Reset the leader timeout. - Ok(_) = new_round.changed() => { - leader_round = *new_round.borrow_and_update(); - debug!("New round has been received {leader_round}, resetting timer"); - let _span = tracing::trace_span!("new_consensus_round_received", round = ?leader_round).entered(); - - min_leader_round_timed_out = false; - max_leader_round_timed_out = false; - - let now = Instant::now(); - min_leader_timeout - .as_mut() - .reset(now + self.min_round_delay); - max_leader_timeout - .as_mut() - .reset(now + self.leader_timeout); - }, - _ = &mut self.stop => { - debug!("Stop signal has been received, now shutting down"); - return; - } - } - } - } -} - -#[cfg(test)] -mod tests { - use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - time::Duration, - }; - - use async_trait::async_trait; - use consensus_config::{AuthorityIndex, Parameters}; - use parking_lot::Mutex; - use tokio::time::{Instant, sleep}; - - use crate::{ - block::{BlockRef, Round, VerifiedBlock}, - commit::CertifiedCommits, - context::Context, - core::CoreSignals, - core_thread::{CoreError, CoreThreadDispatcher}, - leader_timeout::LeaderTimeoutTask, - round_prober::QuorumRound, - }; - - #[derive(Clone, Default)] - struct MockCoreThreadDispatcher { - new_block_calls: Arc>>, - } - - impl MockCoreThreadDispatcher { - async fn get_new_block_calls(&self) -> Vec<(Round, bool, Instant)> { - let mut binding = self.new_block_calls.lock(); - let all_calls = binding.drain(0..); - all_calls.into_iter().collect() - } - } - - #[async_trait] - impl CoreThreadDispatcher for MockCoreThreadDispatcher { - async fn add_blocks( - &self, - _blocks: Vec, - ) -> Result, CoreError> { - todo!() - } - - async fn add_certified_commits( - &self, - _commits: CertifiedCommits, - ) -> Result, CoreError> { - todo!() - } - - async fn check_block_refs( - &self, - _block_refs: Vec, - ) -> Result, CoreError> { - todo!() - } - - async fn new_block(&self, round: Round, force: bool) -> Result<(), CoreError> { - self.new_block_calls - .lock() - .push((round, force, Instant::now())); - Ok(()) - } - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - todo!() - } - - fn set_quorum_subscribers_exists(&self, _exists: bool) -> Result<(), CoreError> { - todo!() - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - _delay: Round, - _received_quorum_rounds: Vec, - _accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - todo!() - } - - fn set_last_known_proposed_round(&self, _round: Round) -> Result<(), CoreError> { - todo!() - } - - fn highest_received_rounds(&self) -> Vec { - todo!() - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn basic_leader_timeout() { - let (context, _signers) = Context::new_for_test(4); - let dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let leader_timeout = Duration::from_millis(500); - let min_round_delay = Duration::from_millis(50); - let parameters = Parameters { - leader_timeout, - min_round_delay, - ..Default::default() - }; - let context = Arc::new(context.with_parameters(parameters)); - let start = Instant::now(); - - let (mut signals, signal_receivers) = CoreSignals::new(context.clone()); - - // spawn the task - let _handle = LeaderTimeoutTask::start(dispatcher.clone(), &signal_receivers, context); - - // send a signal that a new round has been produced. - signals.new_round(10); - - // wait enough until the min round delay has passed and a new_block call is - // triggered - sleep(2 * min_round_delay).await; - let all_calls = dispatcher.get_new_block_calls().await; - assert_eq!(all_calls.len(), 1); - - let (round, force, timestamp) = all_calls[0]; - assert_eq!(round, 10); - assert!(!force); - assert!( - min_round_delay <= timestamp - start, - "Leader timeout min setting {:?} should be less than actual time difference {:?}", - min_round_delay, - timestamp - start - ); - - // wait enough until a new_block has been received - sleep(2 * leader_timeout).await; - let all_calls = dispatcher.get_new_block_calls().await; - assert_eq!(all_calls.len(), 1); - - let (round, force, timestamp) = all_calls[0]; - assert_eq!(round, 10); - assert!(force); - assert!( - leader_timeout <= timestamp - start, - "Leader timeout setting {:?} should be less than actual time difference {:?}", - leader_timeout, - timestamp - start - ); - - // now wait another 2 * leader_timeout, no other call should be received - sleep(2 * leader_timeout).await; - let all_calls = dispatcher.get_new_block_calls().await; - - assert_eq!(all_calls.len(), 0); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn multiple_leader_timeouts() { - let (context, _signers) = Context::new_for_test(4); - let dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let leader_timeout = Duration::from_millis(500); - let min_round_delay = Duration::from_millis(50); - let parameters = Parameters { - leader_timeout, - min_round_delay, - ..Default::default() - }; - let context = Arc::new(context.with_parameters(parameters)); - let now = Instant::now(); - - let (mut signals, signal_receivers) = CoreSignals::new(context.clone()); - - // spawn the task - let _handle = LeaderTimeoutTask::start(dispatcher.clone(), &signal_receivers, context); - - // now send some signals with some small delay between them, but not enough so - // every round manages to timeout and call the force new block method. - signals.new_round(13); - sleep(min_round_delay / 2).await; - signals.new_round(14); - sleep(min_round_delay / 2).await; - signals.new_round(15); - sleep(2 * leader_timeout).await; - - // only the last one should be received - let all_calls = dispatcher.get_new_block_calls().await; - let (round, force, timestamp) = all_calls[0]; - assert_eq!(round, 15); - assert!(!force); - assert!(min_round_delay < timestamp - now); - - let (round, force, timestamp) = all_calls[1]; - assert_eq!(round, 15); - assert!(force); - assert!(leader_timeout < timestamp - now); - } -} diff --git a/consensus/core/src/lib.rs b/consensus/core/src/lib.rs deleted file mode 100644 index 25f2c2b720b..00000000000 --- a/consensus/core/src/lib.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -mod ancestor; -mod authority_node; -mod authority_service; -mod base_committer; -mod block; -mod block_manager; -mod block_verifier; -mod broadcaster; -mod commit; -mod commit_consumer; -mod commit_observer; -mod commit_syncer; -mod commit_vote_monitor; -mod context; -mod core; -mod core_thread; -mod dag_state; -mod error; -mod leader_schedule; -mod leader_scoring; -mod leader_timeout; -mod linearizer; -mod metrics; -#[cfg(not(msim))] -mod network; -#[cfg(msim)] -pub mod network; - -mod stake_aggregator; -mod storage; -mod subscriber; -mod synchronizer; -mod threshold_clock; -#[cfg(not(msim))] -mod transaction; -#[cfg(msim)] -pub mod transaction; - -mod universal_committer; - -#[cfg(test)] -#[path = "tests/randomized_tests.rs"] -mod randomized_tests; - -mod round_prober; -#[cfg(test)] -mod test_dag; -#[cfg(test)] -mod test_dag_builder; -#[cfg(test)] -mod test_dag_parser; - -pub mod scoring_metrics_store; - -/// Exported consensus API. -pub use authority_node::ConsensusAuthority; -pub use block::{BlockAPI, BlockRef, Round}; -/// Exported API for testing. -pub use block::{BlockTimestampMs, TestBlock, Transaction, VerifiedBlock}; -pub use commit::{CommitDigest, CommitIndex, CommitRef, CommittedSubDag}; -pub use commit_consumer::{CommitConsumer, CommitConsumerMonitor}; -pub use context::Clock; -pub use network::tonic_network::to_socket_addr; -#[cfg(msim)] -pub use transaction::NoopTransactionVerifier; -pub use transaction::{ - BlockStatus, ClientError, TransactionClient, TransactionVerifier, ValidationError, -}; diff --git a/consensus/core/src/linearizer.rs b/consensus/core/src/linearizer.rs deleted file mode 100644 index b64f80e0b8c..00000000000 --- a/consensus/core/src/linearizer.rs +++ /dev/null @@ -1,1300 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashSet, sync::Arc}; - -use consensus_config::{AuthorityIndex, Stake}; -use itertools::Itertools; -use parking_lot::RwLock; -use tracing::instrument; - -use crate::{ - Round, - block::{BlockAPI, BlockRef, BlockTimestampMs, VerifiedBlock}, - commit::{Commit, CommittedSubDag, TrustedCommit, sort_sub_dag_blocks}, - context::Context, - dag_state::DagState, - leader_schedule::LeaderSchedule, -}; - -/// The `StorageAPI` trait provides an interface for the block store and has -/// been mostly introduced for allowing to inject the test store in -/// `DagBuilder`. -pub(crate) trait BlockStoreAPI { - fn get_blocks(&self, refs: &[BlockRef]) -> Vec>; - - fn gc_round(&self) -> Round; - - fn gc_enabled(&self) -> bool; - - fn set_committed(&mut self, block_ref: &BlockRef) -> bool; - - fn is_committed(&self, block_ref: &BlockRef) -> bool; -} - -impl BlockStoreAPI - for parking_lot::lock_api::RwLockWriteGuard<'_, parking_lot::RawRwLock, DagState> -{ - fn get_blocks(&self, refs: &[BlockRef]) -> Vec> { - DagState::get_blocks(self, refs) - } - - fn gc_round(&self) -> Round { - DagState::gc_round(self) - } - - fn gc_enabled(&self) -> bool { - DagState::gc_enabled(self) - } - - fn set_committed(&mut self, block_ref: &BlockRef) -> bool { - DagState::set_committed(self, block_ref) - } - - fn is_committed(&self, block_ref: &BlockRef) -> bool { - DagState::is_committed(self, block_ref) - } -} - -/// Expand a committed sequence of leader into a sequence of sub-dags. -#[derive(Clone)] -pub(crate) struct Linearizer { - /// In memory block store representing the dag state - context: Arc, - dag_state: Arc>, - leader_schedule: Arc, -} - -impl Linearizer { - pub(crate) fn new( - context: Arc, - dag_state: Arc>, - leader_schedule: Arc, - ) -> Self { - Self { - dag_state, - leader_schedule, - context, - } - } - - /// Collect the sub-dag and the corresponding commit from a specific leader - /// excluding any duplicates or blocks that have already been committed - /// (within previous sub-dags). - fn collect_sub_dag_and_commit( - &mut self, - leader_block: VerifiedBlock, - reputation_scores_desc: Vec<(AuthorityIndex, u64)>, - ) -> (CommittedSubDag, TrustedCommit) { - let _s = self - .context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Linearizer::collect_sub_dag_and_commit"]) - .start_timer(); - // Grab latest commit state from dag state - let mut dag_state = self.dag_state.write(); - let last_commit_index = dag_state.last_commit_index(); - let last_commit_digest = dag_state.last_commit_digest(); - let last_commit_timestamp_ms = dag_state.last_commit_timestamp_ms(); - let last_committed_rounds = dag_state.last_committed_rounds(); - - // Now linearize the sub-dag starting from the leader block - let to_commit = Self::linearize_sub_dag( - &self.context, - leader_block.clone(), - last_committed_rounds, - &mut dag_state, - ); - - let timestamp_ms = Self::calculate_commit_timestamp( - &self.context, - &mut dag_state, - &leader_block, - last_commit_timestamp_ms, - ); - - drop(dag_state); - - // Create the Commit. - let commit = Commit::new( - last_commit_index + 1, - last_commit_digest, - timestamp_ms, - leader_block.reference(), - to_commit - .iter() - .map(|block| block.reference()) - .collect::>(), - ); - let serialized = commit - .serialize() - .unwrap_or_else(|e| panic!("Failed to serialize commit: {e}")); - let commit = TrustedCommit::new_trusted(commit, serialized); - - // Create the corresponding committed sub dag - let sub_dag = CommittedSubDag::new( - leader_block.reference(), - to_commit, - timestamp_ms, - commit.reference(), - reputation_scores_desc, - ); - - (sub_dag, commit) - } - - /// Calculates the commit's timestamp. If the median-based timestamp - /// calculation is enabled, then the timestamp will be calculated as the - /// median of leader's parents (leader.round - 1) timestamps by stake. - /// Otherwise, the leader's timestamp will be used. To ensure that commit - /// timestamp monotonicity is respected, it is compared against the - /// `last_commit_timestamp_ms` and the maximum of the two is returned. - pub(crate) fn calculate_commit_timestamp( - context: &Context, - dag_state: &mut impl BlockStoreAPI, - leader_block: &VerifiedBlock, - last_commit_timestamp_ms: BlockTimestampMs, - ) -> BlockTimestampMs { - let timestamp_ms = if context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - // Select leaders' parent blocks. - let block_refs = leader_block - .ancestors() - .iter() - .filter(|block_ref| block_ref.round == leader_block.round() - 1) - .cloned() - .collect::>(); - // Get the blocks from dag state which should not fail. - let blocks = dag_state - .get_blocks(&block_refs) - .into_iter() - .map(|block_opt| block_opt.expect("We should have all blocks in dag state.")); - median_timestamp_by_stake(context, blocks).unwrap_or_else(|e| { - panic!( - "Cannot compute median timestamp for leader block {leader_block:?} ancestors: {e}" - ) - }) - } else { - leader_block.timestamp_ms() - }; - - // Always make sure that commit timestamps are monotonic, so override if - // necessary. - timestamp_ms.max(last_commit_timestamp_ms) - } - - pub(crate) fn linearize_sub_dag( - context: &Context, - leader_block: VerifiedBlock, - last_committed_rounds: Vec, - dag_state: &mut impl BlockStoreAPI, - ) -> Vec { - let gc_enabled = dag_state.gc_enabled(); - // The GC round here is calculated based on the last committed round of the - // leader block. The algorithm will attempt to commit blocks up to this - // GC round. Once this commit has been processed and written to DagState, then - // gc round will update and on the processing of the next commit we'll - // have it already updated, so no need to do any gc_round recalculations here. - // We just use whatever is currently in DagState. - let gc_round: Round = dag_state.gc_round(); - let leader_block_ref = leader_block.reference(); - let mut buffer = vec![leader_block]; - - let mut to_commit = Vec::new(); - - // The new logic will perform the recursion without stopping at the highest - // round that has been committed per authority. Instead it will - // allow to commit blocks that are lower than the highest committed round for an - // authority but higher than gc_round. - if context.protocol_config.consensus_linearize_subdag_v2() { - assert!( - dag_state.set_committed(&leader_block_ref), - "Leader block with reference {leader_block_ref:?} attempted to be committed twice" - ); - - while let Some(x) = buffer.pop() { - to_commit.push(x.clone()); - - let ancestors: Vec = dag_state - .get_blocks( - &x.ancestors() - .iter() - .copied() - .filter(|ancestor| { - ancestor.round > gc_round && !dag_state.is_committed(ancestor) - }) - .collect::>(), - ) - .into_iter() - .map(|ancestor_opt| { - ancestor_opt.expect("We should have all uncommitted blocks in dag state.") - }) - .collect(); - - for ancestor in ancestors { - buffer.push(ancestor.clone()); - assert!( - dag_state.set_committed(&ancestor.reference()), - "Block with reference {:?} attempted to be committed twice", - ancestor.reference() - ); - } - } - } else { - let mut committed = HashSet::new(); - assert!(committed.insert(leader_block_ref)); - - while let Some(x) = buffer.pop() { - to_commit.push(x.clone()); - - let ancestors: Vec = dag_state - .get_blocks( - &x.ancestors() - .iter() - .copied() - .filter(|ancestor| { - // We skip the block if we already committed it or we reached a - // round that we already committed. - // TODO: for Fast Path we need to amend the recursion rule here and - // allow us to commit blocks all the way up to the `gc_round`. - // Some additional work will be needed to make sure that we keep the - // uncommitted blocks up to the `gc_round` across commits. - !committed.contains(ancestor) - && last_committed_rounds[ancestor.author] < ancestor.round - }) - .filter(|ancestor| { - // Keep the block if GC is not enabled or it is enabled and the - // block is above the gc_round. We do this - // to stop the recursion early and avoid going to deep when it's - // unnecessary. - !gc_enabled || ancestor.round > gc_round - }) - .collect::>(), - ) - .into_iter() - .map(|ancestor_opt| { - ancestor_opt.expect("We should have all uncommitted blocks in dag state.") - }) - .collect(); - - for ancestor in ancestors { - buffer.push(ancestor.clone()); - assert!(committed.insert(ancestor.reference())); - } - } - } - - // The above code should have not yielded any blocks that are <= gc_round, but - // just to make sure that we'll never commit anything that should be - // garbage collected we attempt to prune here as well. - if gc_enabled { - assert!( - to_commit.iter().all(|block| block.round() > gc_round), - "No blocks <= {gc_round} should be committed. Leader round {leader_block_ref}, blocks {to_commit:?}." - ); - } - - // Sort the blocks of the sub-dag blocks - sort_sub_dag_blocks(&mut to_commit); - - to_commit - } - - // This function should be called whenever a new commit is observed. This will - // iterate over the sequence of committed leaders and produce a list of - // committed sub-dags. - #[instrument(level = "trace", skip_all)] - pub(crate) fn handle_commit( - &mut self, - committed_leaders: Vec, - ) -> Vec { - if committed_leaders.is_empty() { - return vec![]; - } - - // We check whether the leader schedule has been updated. If yes, then we'll - // send the scores as part of the first sub dag. - let schedule_updated = self - .leader_schedule - .leader_schedule_updated(&self.dag_state); - - let mut committed_sub_dags = vec![]; - for (i, leader_block) in committed_leaders.into_iter().enumerate() { - let reputation_scores_desc = if schedule_updated && i == 0 { - self.leader_schedule - .leader_swap_table - .read() - .reputation_scores_desc - .clone() - } else { - vec![] - }; - - // Collect the sub-dag generated using each of these leaders and the - // corresponding commit. - let (sub_dag, commit) = - self.collect_sub_dag_and_commit(leader_block, reputation_scores_desc); - - self.update_blocks_pruned_metric(&sub_dag); - - // Buffer commit in dag state for persistence later. - // This also updates the last committed rounds. - self.dag_state.write().add_commit(commit.clone()); - - committed_sub_dags.push(sub_dag); - } - - // Committed blocks must be persisted to storage before sending them to IOTA and - // executing their transactions. - // Commit metadata can be persisted more lazily because they are recoverable. - // Uncommitted blocks can wait to persist too. - // But for simplicity, all unpersisted blocks and commits are flushed to - // storage. - self.dag_state.write().flush(); - - committed_sub_dags - } - - // Try to measure the number of blocks that get pruned due to GC. This is not - // very accurate, but it can give us a good enough idea. We consider a block - // as pruned when it is an ancestor of a block that has been committed as part - // of the provided `sub_dag`, but it has not been committed as part of - // previous commits. Right now we measure this via checking that highest - // committed round for the authority as we don't an efficient look up - // functionality to check if a block has been committed or not. - fn update_blocks_pruned_metric(&self, sub_dag: &CommittedSubDag) { - let (last_committed_rounds, gc_round) = { - let dag_state = self.dag_state.read(); - (dag_state.last_committed_rounds(), dag_state.gc_round()) - }; - - for block_ref in sub_dag - .blocks - .iter() - .flat_map(|block| block.ancestors()) - .filter( - |ancestor_ref| { - ancestor_ref.round <= gc_round - && last_committed_rounds[ancestor_ref.author] != ancestor_ref.round - }, /* If the last committed round is the same as the pruned block's round, then - * we know for sure that it has been committed and it doesn't count here - * as pruned block. */ - ) - .unique() - { - let hostname = &self.context.committee.authority(block_ref.author).hostname; - - // If the last committed round from this authority is lower than the pruned - // ancestor in question, then we know for sure that it has not been committed. - let label_values = if last_committed_rounds[block_ref.author] < block_ref.round { - &[hostname, "uncommitted"] - } else { - // If last committed round is higher for this authority, then we don't really - // know it's status, but we know that there is a higher committed block from - // this authority. - &[hostname, "higher_committed"] - }; - - self.context - .metrics - .node_metrics - .blocks_pruned_on_commit - .with_label_values(label_values) - .inc(); - } - } -} - -/// Computes the median timestamp of the blocks weighted by the stake of their -/// authorities. This function assumes each block comes from a different -/// authority of the same round. Error is returned if no blocks are provided or -/// total stake is less than quorum threshold. -pub(crate) fn median_timestamp_by_stake( - context: &Context, - blocks: impl Iterator, -) -> Result { - let mut total_stake = 0; - let mut timestamps = vec![]; - for block in blocks { - let stake = context.committee.authority(block.author()).stake; - timestamps.push((block.timestamp_ms(), stake)); - total_stake += stake; - } - - if timestamps.is_empty() { - return Err("No blocks provided".to_string()); - } - if total_stake < context.committee.quorum_threshold() { - return Err(format!( - "Total stake {} < quorum threshold {}", - total_stake, - context.committee.quorum_threshold() - )); - } - - Ok(median_timestamps_by_stake_inner(timestamps, total_stake)) -} - -fn median_timestamps_by_stake_inner( - mut timestamps: Vec<(BlockTimestampMs, Stake)>, - total_stake: Stake, -) -> BlockTimestampMs { - timestamps.sort_by_key(|(ts, _)| *ts); - - let mut cumulative_stake = 0; - for (ts, stake) in ×tamps { - cumulative_stake += stake; - if cumulative_stake > total_stake / 2 { - return *ts; - } - } - - timestamps.last().unwrap().0 -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - use crate::{ - CommitIndex, TestBlock, - commit::{CommitAPI as _, CommitDigest, DEFAULT_WAVE_LENGTH}, - context::Context, - leader_schedule::{LeaderSchedule, LeaderSwapTable}, - storage::mem_store::MemStore, - test_dag_builder::DagBuilder, - test_dag_parser::parse_dag, - }; - - #[rstest] - #[tokio::test] - async fn test_handle_commit(#[values(true, false)] consensus_median_timestamp: bool) { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let (mut context, _keys) = Context::new_for_test(num_authorities); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - - let context = Arc::new(context); - - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let mut linearizer = Linearizer::new(context.clone(), dag_state.clone(), leader_schedule); - - // Populate fully connected test blocks for round 0 ~ 10, authorities 0 ~ 3. - let num_rounds: u32 = 10; - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - let leaders = dag_builder - .leader_blocks(1..=num_rounds) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - let commits = linearizer.handle_commit(leaders.clone()); - for (idx, subdag) in commits.into_iter().enumerate() { - tracing::info!("{subdag:?}"); - assert_eq!(subdag.leader, leaders[idx].reference()); - - let expected_ts = if consensus_median_timestamp { - let block_refs = leaders[idx] - .ancestors() - .iter() - .filter(|block_ref| block_ref.round == leaders[idx].round() - 1) - .cloned() - .collect::>(); - let blocks = dag_state - .read() - .get_blocks(&block_refs) - .into_iter() - .map(|block_opt| block_opt.expect("We should have all blocks in dag state.")); - - median_timestamp_by_stake(&context, blocks).unwrap() - } else { - leaders[idx].timestamp_ms() - }; - assert_eq!(subdag.timestamp_ms, expected_ts); - - if idx == 0 { - // First subdag includes the leader block only - assert_eq!(subdag.blocks.len(), 1); - } else { - // Every subdag after will be missing the leader block from the previous - // committed subdag - assert_eq!(subdag.blocks.len(), num_authorities); - } - for block in subdag.blocks.iter() { - assert!(block.round() <= leaders[idx].round()); - } - assert_eq!(subdag.commit_ref.index, idx as CommitIndex + 1); - } - } - - #[tokio::test] - async fn test_handle_commit_with_schedule_update() { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let context = Arc::new(Context::new_for_test(num_authorities).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - const NUM_OF_COMMITS_PER_SCHEDULE: u64 = 10; - let leader_schedule = Arc::new( - LeaderSchedule::new(context.clone(), LeaderSwapTable::default()) - .with_num_commits_per_schedule(NUM_OF_COMMITS_PER_SCHEDULE), - ); - let mut linearizer = - Linearizer::new(context.clone(), dag_state.clone(), leader_schedule.clone()); - - // Populate fully connected test blocks for round 0 ~ 20, authorities 0 ~ 3. - let num_rounds: u32 = 20; - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - // Take the first 10 leaders - let leaders = dag_builder - .leader_blocks(1..=10) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Create some commits - let commits = linearizer.handle_commit(leaders); - - // Write them in DagState - dag_state.write().add_scoring_subdags(commits); - // Now update the leader schedule - leader_schedule.update_leader_schedule_v2(&dag_state); - assert!( - leader_schedule.leader_schedule_updated(&dag_state), - "Leader schedule should have been updated" - ); - - // Try to commit now the rest of the 10 leaders - let leaders = dag_builder - .leader_blocks(11..=20) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Now on the commits only the first one should contain the updated scores, the - // other should be empty - let commits = linearizer.handle_commit(leaders); - assert_eq!(commits.len(), 10); - let scores = vec![ - (AuthorityIndex::new_for_test(1), 29), - (AuthorityIndex::new_for_test(0), 29), - (AuthorityIndex::new_for_test(3), 29), - (AuthorityIndex::new_for_test(2), 29), - ]; - assert_eq!(commits[0].reputation_scores_desc, scores); - for commit in commits.into_iter().skip(1) { - assert_eq!(commit.reputation_scores_desc, vec![]); - } - } - - // TODO: Remove when DistributedVoteScoring is enabled. - #[tokio::test] - async fn test_handle_commit_with_schedule_update_with_unscored_subdags() { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let context = Arc::new(Context::new_for_test(num_authorities).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - const NUM_OF_COMMITS_PER_SCHEDULE: u64 = 10; - let leader_schedule = Arc::new( - LeaderSchedule::new(context.clone(), LeaderSwapTable::default()) - .with_num_commits_per_schedule(NUM_OF_COMMITS_PER_SCHEDULE), - ); - let mut linearizer = - Linearizer::new(context.clone(), dag_state.clone(), leader_schedule.clone()); - - // Populate fully connected test blocks for round 0 ~ 20, authorities 0 ~ 3. - let num_rounds: u32 = 20; - let mut dag_builder = DagBuilder::new(context); - dag_builder - .layers(1..=num_rounds) - .build() - .persist_layers(dag_state.clone()); - - // Take the first 10 leaders - let leaders = dag_builder - .leader_blocks(1..=10) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Create some commits - let commits = linearizer.handle_commit(leaders); - - // Write them in DagState - dag_state.write().add_unscored_committed_subdags(commits); - - // Now update the leader schedule - leader_schedule.update_leader_schedule_v1(&dag_state); - - assert!( - leader_schedule.leader_schedule_updated(&dag_state), - "Leader schedule should have been updated" - ); - - // Try to commit now the rest of the 10 leaders - let leaders = dag_builder - .leader_blocks(11..=20) - .into_iter() - .map(Option::unwrap) - .collect::>(); - - // Now on the commits only the first one should contain the updated scores, the - // other should be empty - let commits = linearizer.handle_commit(leaders); - assert_eq!(commits.len(), 10); - let scores = vec![ - (AuthorityIndex::new_for_test(2), 9), - (AuthorityIndex::new_for_test(1), 8), - (AuthorityIndex::new_for_test(0), 8), - (AuthorityIndex::new_for_test(3), 8), - ]; - assert_eq!(commits[0].reputation_scores_desc, scores); - - for commit in commits.into_iter().skip(1) { - assert_eq!(commit.reputation_scores_desc, vec![]); - } - } - - #[rstest] - #[tokio::test] - async fn test_handle_already_committed( - #[values(true, false)] consensus_median_timestamp: bool, - ) { - telemetry_subscribers::init_for_testing(); - let num_authorities = 4; - let (mut context, _) = Context::new_for_test(num_authorities); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - - let context = Arc::new(context); - - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let mut linearizer = - Linearizer::new(context.clone(), dag_state.clone(), leader_schedule.clone()); - let wave_length = DEFAULT_WAVE_LENGTH; - - let leader_round_wave_1 = 3; - let leader_round_wave_2 = leader_round_wave_1 + wave_length; - - // Build a Dag from round 1..=6 - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=leader_round_wave_2).build(); - - // Now retrieve all the blocks up to round leader_round_wave_1 - 1 - // And then only the leader of round leader_round_wave_1 - // Also store those to DagState - let mut blocks = dag_builder.blocks(0..=leader_round_wave_1 - 1); - blocks.push( - dag_builder - .leader_block(leader_round_wave_1) - .expect("Leader block should have been found"), - ); - dag_state.write().accept_blocks(blocks.clone()); - - let first_leader = dag_builder - .leader_block(leader_round_wave_1) - .expect("Wave 1 leader round block should exist"); - let mut last_commit_index = 1; - let first_commit_data = TrustedCommit::new_for_test( - last_commit_index, - CommitDigest::MIN, - 0, - first_leader.reference(), - blocks.iter().map(|block| block.reference()).collect(), - ); - dag_state.write().add_commit(first_commit_data); - - // Mark the blocks as committed in DagState. This will allow to correctly detect - // the committed blocks when the new linearizer logic is enabled. - for block in blocks.iter() { - dag_state.write().set_committed(&block.reference()); - } - - // Now take all the blocks from round `leader_round_wave_1` up to round - // `leader_round_wave_2-1` - let mut blocks = dag_builder.blocks(leader_round_wave_1..=leader_round_wave_2 - 1); - // Filter out leader block of round `leader_round_wave_1` - blocks.retain(|block| { - !(block.round() == leader_round_wave_1 - && block.author() == leader_schedule.elect_leader(leader_round_wave_1, 0)) - }); - // Add the leader block of round `leader_round_wave_2` - blocks.push( - dag_builder - .leader_block(leader_round_wave_2) - .expect("Leader block should have been found"), - ); - // Write them in dag state - dag_state.write().accept_blocks(blocks.clone()); - - let mut blocks: Vec<_> = blocks.into_iter().map(|block| block.reference()).collect(); - - // Now get the latest leader which is the leader round of wave 2 - let leader = dag_builder - .leader_block(leader_round_wave_2) - .expect("Leader block should exist"); - - last_commit_index += 1; - let expected_second_commit = TrustedCommit::new_for_test( - last_commit_index, - CommitDigest::MIN, - 0, - leader.reference(), - blocks.clone(), - ); - - let commit = linearizer.handle_commit(vec![leader.clone()]); - assert_eq!(commit.len(), 1); - - let subdag = &commit[0]; - tracing::info!("{subdag:?}"); - assert_eq!(subdag.leader, leader.reference()); - assert_eq!(subdag.commit_ref.index, expected_second_commit.index()); - - let expected_ts = if consensus_median_timestamp { - median_timestamp_by_stake( - &context, - subdag.blocks.iter().filter_map(|block| { - if block.round() == subdag.leader.round - 1 { - Some(block.clone()) - } else { - None - } - }), - ) - .unwrap() - } else { - leader.timestamp_ms() - }; - assert_eq!(subdag.timestamp_ms, expected_ts); - - // Using the same sorting as used in CommittedSubDag::sort - blocks.sort_by(|a, b| a.round.cmp(&b.round).then_with(|| a.author.cmp(&b.author))); - assert_eq!( - subdag - .blocks - .clone() - .into_iter() - .map(|b| b.reference()) - .collect::>(), - blocks - ); - for block in subdag.blocks.iter() { - assert!(block.round() <= expected_second_commit.leader().round); - } - } - - /// This test will run the linearizer with GC disabled (gc_depth = 0) and gc - /// enabled (gc_depth = 3) and make sure that for the exact same DAG the - /// linearizer will commit different blocks according to the rules. - #[rstest] - #[case(0, false)] - #[case(3, false)] - #[case(3, true)] - #[tokio::test] - async fn test_handle_commit_with_gc_simple( - #[case] gc_depth: u32, - #[case] consensus_median_timestamp: bool, - ) { - telemetry_subscribers::init_for_testing(); - - let num_authorities = 4; - let (mut context, _keys) = Context::new_for_test(num_authorities); - context.protocol_config.set_gc_depth_for_testing(gc_depth); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - if gc_depth == 0 { - context - .protocol_config - .set_consensus_linearize_subdag_v2_for_testing(false); - } - - let context = Arc::new(context); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let mut linearizer = Linearizer::new(context.clone(), dag_state.clone(), leader_schedule); - - // Authorities of index 0->2 will always creates blocks that see each other, but - // until round 5 they won't see the blocks of authority 3. For authority - // 3 we create blocks that connect to all the other authorities. - // On round 5 we finally make the other authorities see the blocks of authority - // 3. Practically we "simulate" here a long chain created by authority 3 - // that is visible in round 5, but due to GC blocks of only round >=2 will - // be committed, when GC is enabled. When GC is disabled all blocks will be - // committed for rounds >= 1. - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { - A -> [-D1], - B -> [-D1], - C -> [-D1], - D -> [*], - }, - Round 3 : { - A -> [-D2], - B -> [-D2], - C -> [-D2], - }, - Round 4 : { - A -> [-D3], - B -> [-D3], - C -> [-D3], - D -> [A3, B3, C3, D2], - }, - Round 5 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - dag_builder.print(); - dag_builder.persist_all_blocks(dag_state.clone()); - - let leaders = dag_builder - .leader_blocks(1..=6) - .into_iter() - .flatten() - .collect::>(); - - let commits = linearizer.handle_commit(leaders.clone()); - for (idx, subdag) in commits.into_iter().enumerate() { - tracing::info!("{subdag:?}"); - assert_eq!(subdag.leader, leaders[idx].reference()); - - let expected_ts = if consensus_median_timestamp { - let block_refs = leaders[idx] - .ancestors() - .iter() - .filter(|block_ref| block_ref.round == leaders[idx].round() - 1) - .cloned() - .collect::>(); - let blocks = dag_state - .read() - .get_blocks(&block_refs) - .into_iter() - .map(|block_opt| block_opt.expect("We should have all blocks in dag state.")); - - median_timestamp_by_stake(&context, blocks).unwrap() - } else { - leaders[idx].timestamp_ms() - }; - assert_eq!(subdag.timestamp_ms, expected_ts); - - if idx == 0 { - // First subdag includes the leader block only - assert_eq!(subdag.blocks.len(), 1); - } else if idx == 1 { - assert_eq!(subdag.blocks.len(), 3); - } else if idx == 2 { - // We commit: - // * 1 block on round 4, the leader block - // * 3 blocks on round 3, as no commit happened on round 3 since the leader was - // missing - // * 2 blocks on round 2, again as no commit happened on round 3, we commit the - // "sub dag" of leader of round 3, which will be another 2 blocks - assert_eq!(subdag.blocks.len(), 6); - } else { - // GC is enabled, so we expect to see only blocks of round >= 2 - if gc_depth > 0 { - // Now it's going to be the first time that a leader will see the blocks of - // authority 3 and will attempt to commit the long chain. - // However, due to GC it will only commit blocks of round > 1. That's because it - // will commit blocks up to previous leader's round (round = - // 4) minus the gc_depth = 3, so that will be gc_round = 4 - 3 = 1. So we expect - // to see on the sub dag committed blocks of round >= 2. - assert_eq!(subdag.blocks.len(), 5); - - assert!( - subdag.blocks.iter().all(|block| block.round() >= 2), - "Found blocks that are of round < 2." - ); - - // Also ensure that gc_round has advanced with the latest committed leader - assert_eq!(dag_state.read().gc_round(), subdag.leader.round - gc_depth); - } else { - // GC is disabled, so we expect to see all blocks of round >= 1 - assert_eq!(subdag.blocks.len(), 6); - assert!( - subdag.blocks.iter().all(|block| block.round() >= 1), - "Found blocks that are of round < 1." - ); - - // GC round should never have moved - assert_eq!(dag_state.read().gc_round(), 0); - } - } - for block in subdag.blocks.iter() { - assert!(block.round() <= leaders[idx].round()); - } - assert_eq!(subdag.commit_ref.index, idx as CommitIndex + 1); - } - } - - #[rstest] - #[case(3, false)] - #[case(3, true)] - #[tokio::test] - async fn test_handle_commit_below_highest_committed_round( - #[case] gc_depth: u32, - #[case] consensus_median_timestamp: bool, - ) { - telemetry_subscribers::init_for_testing(); - - let num_authorities = 4; - let (mut context, _keys) = Context::new_for_test(num_authorities); - context - .protocol_config - .set_consensus_gc_depth_for_testing(gc_depth); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - context - .protocol_config - .set_consensus_linearize_subdag_v2_for_testing(true); - - let context = Arc::new(context); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let mut linearizer = Linearizer::new(context.clone(), dag_state.clone(), leader_schedule); - - // Authority D will create an "orphaned" block on round 1 as it won't reference - // to it on the block of round 2. Similar, no other authority will reference to - // it on round 2. Then on round 3 the authorities A, B & C will link to - // block D1. Once the DAG gets committed we should see the block D1 getting - // committed as well. Normally ,as block D2 would have been committed - // first block D1 should be omitted. With the new logic this is no longer true. - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { - A -> [-D1], - B -> [-D1], - C -> [-D1], - D -> [-D1], - }, - Round 3 : { - A -> [A2, B2, C2, D1], - B -> [A2, B2, C2, D1], - C -> [A2, B2, C2, D1], - D -> [A2, B2, C2, D2] - }, - Round 4 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - dag_builder.print(); - dag_builder.persist_all_blocks(dag_state.clone()); - - let leaders = dag_builder - .leader_blocks(1..=4) - .into_iter() - .flatten() - .collect::>(); - - let commits = linearizer.handle_commit(leaders.clone()); - for (idx, subdag) in commits.into_iter().enumerate() { - tracing::info!("{subdag:?}"); - assert_eq!(subdag.leader, leaders[idx].reference()); - - let expected_ts = if consensus_median_timestamp { - let block_refs = leaders[idx] - .ancestors() - .iter() - .filter(|block_ref| block_ref.round == leaders[idx].round() - 1) - .cloned() - .collect::>(); - let blocks = dag_state - .read() - .get_blocks(&block_refs) - .into_iter() - .map(|block_opt| block_opt.expect("We should have all blocks in dag state.")); - - median_timestamp_by_stake(&context, blocks).unwrap() - } else { - leaders[idx].timestamp_ms() - }; - assert_eq!(subdag.timestamp_ms, expected_ts); - - if idx == 0 { - // First subdag includes the leader block only B1 - assert_eq!(subdag.blocks.len(), 1); - } else if idx == 1 { - // We commit: - // * 1 block on round 2, the leader block C2 - // * 2 blocks on round 1, A1, C1 - assert_eq!(subdag.blocks.len(), 3); - } else if idx == 2 { - // We commit: - // * 1 block on round 3, the leader block D3 - // * 3 blocks on round 2, A2, B2, D2 - assert_eq!(subdag.blocks.len(), 4); - - assert!( - subdag.blocks.iter().any(|block| block.round() == 2 - && block.author() == AuthorityIndex::new_for_test(3)), - "Block D2 should have been committed." - ); - } else if idx == 3 { - // We commit: - // * 1 block on round 4, the leader block A4 - // * 3 blocks on round 3, A3, B3, C3 - // * 1 block of round 1, D1 - assert_eq!(subdag.blocks.len(), 5); - assert!( - subdag.blocks.iter().any(|block| block.round() == 1 - && block.author() == AuthorityIndex::new_for_test(3)), - "Block D1 should have been committed." - ); - } else { - panic!("Unexpected subdag with index {idx:?}"); - } - - for block in subdag.blocks.iter() { - assert!(block.round() <= leaders[idx].round()); - } - assert_eq!(subdag.commit_ref.index, idx as CommitIndex + 1); - } - } - - #[rstest] - #[case(false, 5_000, 5_000, 6_000)] - #[case(true, 3_000, 3_000, 6_000)] - #[tokio::test] - async fn test_calculate_commit_timestamp( - #[case] consensus_median_timestamp: bool, - #[case] timestamp_1: u64, - #[case] timestamp_2: u64, - #[case] timestamp_3: u64, - ) { - // GIVEN - telemetry_subscribers::init_for_testing(); - - let num_authorities = 4; - let (mut context, _keys) = Context::new_for_test(num_authorities); - - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - - let context = Arc::new(context); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let mut dag_state = dag_state.write(); - - let ancestors = vec![ - VerifiedBlock::new_for_test(TestBlock::new(4, 0).set_timestamp_ms(1_000).build()), - VerifiedBlock::new_for_test(TestBlock::new(4, 1).set_timestamp_ms(2_000).build()), - VerifiedBlock::new_for_test(TestBlock::new(4, 2).set_timestamp_ms(3_000).build()), - VerifiedBlock::new_for_test(TestBlock::new(4, 3).set_timestamp_ms(4_000).build()), - ]; - - let leader_block = VerifiedBlock::new_for_test( - TestBlock::new(5, 0) - .set_timestamp_ms(5_000) - .set_ancestors( - ancestors - .iter() - .map(|block| block.reference()) - .collect::>(), - ) - .build(), - ); - - for block in &ancestors { - dag_state.accept_block(block.clone()); - } - - let last_commit_timestamp_ms = 0; - - // WHEN - let timestamp = Linearizer::calculate_commit_timestamp( - &context, - &mut dag_state, - &leader_block, - last_commit_timestamp_ms, - ); - assert_eq!(timestamp, timestamp_1); - - // AND skip the block of authority 0 and round 4. - let leader_block = VerifiedBlock::new_for_test( - TestBlock::new(5, 0) - .set_timestamp_ms(5_000) - .set_ancestors( - ancestors - .iter() - .skip(1) - .map(|block| block.reference()) - .collect::>(), - ) - .build(), - ); - - let timestamp = Linearizer::calculate_commit_timestamp( - &context, - &mut dag_state, - &leader_block, - last_commit_timestamp_ms, - ); - assert_eq!(timestamp, timestamp_2); - - // AND set the `last_commit_timestamp_ms` to 6_000 - let last_commit_timestamp_ms = 6_000; - let timestamp = Linearizer::calculate_commit_timestamp( - &context, - &mut dag_state, - &leader_block, - last_commit_timestamp_ms, - ); - assert_eq!(timestamp, timestamp_3); - - // AND there is only one ancestor block to commit - let (mut context, _) = Context::new_for_test(1); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - consensus_median_timestamp, - ); - let leader_block = VerifiedBlock::new_for_test( - TestBlock::new(5, 0) - .set_timestamp_ms(5_000) - .set_ancestors( - ancestors - .iter() - .take(1) - .map(|block| block.reference()) - .collect::>(), - ) - .build(), - ); - let last_commit_timestamp_ms = 0; - let timestamp = Linearizer::calculate_commit_timestamp( - &context, - &mut dag_state, - &leader_block, - last_commit_timestamp_ms, - ); - if consensus_median_timestamp { - assert_eq!(timestamp, 1_000); - } else { - assert_eq!(timestamp, leader_block.timestamp_ms()); - } - } - - #[test] - fn test_median_timestamps_by_stake() { - // One total stake. - let timestamps = vec![(1_000, 1)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 1), 1_000); - - // Odd number of total stakes. - let timestamps = vec![(1_000, 1), (2_000, 1), (3_000, 1)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 3), 2_000); - - // Even number of total stakes. - let timestamps = vec![(1_000, 1), (2_000, 1), (3_000, 1), (4_000, 1)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 4), 3_000); - - // Even number of total stakes, different order. - let timestamps = vec![(4_000, 1), (3_000, 1), (1_000, 1), (2_000, 1)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 4), 3_000); - - // Unequal stakes. - let timestamps = vec![(2_000, 2), (4_000, 2), (1_000, 3), (3_000, 3)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 10), 3_000); - - // Unequal stakes. - let timestamps = vec![ - (500, 2), - (4_000, 2), - (2_500, 3), - (1_000, 5), - (3_000, 3), - (2_000, 4), - ]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 19), 2_000); - - // One authority dominates. - let timestamps = vec![(1_000, 1), (2_000, 1), (3_000, 1), (4_000, 1), (5_000, 10)]; - assert_eq!(median_timestamps_by_stake_inner(timestamps, 14), 5_000); - } - - #[tokio::test] - async fn test_median_timestamps_by_stake_errors() { - let num_authorities = 4; - let (mut context, _keys) = Context::new_for_test(num_authorities); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(true); - - let context = Arc::new(context); - - // No blocks provided - let err = median_timestamp_by_stake(&context, vec![].into_iter()).unwrap_err(); - assert_eq!(err, "No blocks provided"); - - // Blocks provided but total stake is less than quorum threshold - let block = VerifiedBlock::new_for_test(TestBlock::new(5, 0).build()); - let err = median_timestamp_by_stake(&context, vec![block].into_iter()).unwrap_err(); - assert_eq!(err, "Total stake 1 < quorum threshold 3"); - } -} diff --git a/consensus/core/src/metrics.rs b/consensus/core/src/metrics.rs deleted file mode 100644 index 855a8c9eff2..00000000000 --- a/consensus/core/src/metrics.rs +++ /dev/null @@ -1,909 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use prometheus::{ - Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, Registry, - exponential_buckets, register_histogram_vec_with_registry, register_histogram_with_registry, - register_int_counter_vec_with_registry, register_int_counter_with_registry, - register_int_gauge_vec_with_registry, register_int_gauge_with_registry, -}; - -use crate::network::metrics::NetworkMetrics; - -// starts from 1μs, 50μs, 100μs... -const FINE_GRAINED_LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.000_001, 0.000_050, 0.000_100, 0.000_500, 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, - 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, - 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10., 20., 30., 60., 120., -]; - -const NUM_BUCKETS: &[f64] = &[ - 1.0, - 2.0, - 4.0, - 8.0, - 10.0, - 20.0, - 40.0, - 80.0, - 100.0, - 150.0, - 200.0, - 400.0, - 800.0, - 1000.0, - 2000.0, - 3000.0, - 5000.0, - 10000.0, - 20000.0, - 30000.0, - 50000.0, - 100_000.0, - 200_000.0, - 300_000.0, - 500_000.0, - 1_000_000.0, -]; - -const LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.001, 0.005, 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, - 1.0, 1.2, 1.4, 1.6, 1.8, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, - 9.0, 9.5, 10., 12.5, 15., 17.5, 20., 25., 30., 60., 90., 120., 180., 300., -]; - -const SIZE_BUCKETS: &[f64] = &[ - 100., - 400., - 800., - 1_000., - 2_000., - 5_000., - 10_000., - 20_000., - 50_000., - 100_000., - 200_000.0, - 300_000.0, - 400_000.0, - 500_000.0, - 1_000_000.0, - 2_000_000.0, - 3_000_000.0, - 5_000_000.0, - 10_000_000.0, -]; // size in bytes - -pub(crate) struct Metrics { - pub(crate) node_metrics: NodeMetrics, - pub(crate) network_metrics: NetworkMetrics, -} - -pub(crate) struct NodeMetrics { - pub(crate) block_commit_latency: Histogram, - pub(crate) proposed_blocks: IntCounterVec, - pub(crate) proposed_block_size: Histogram, - pub(crate) proposed_block_transactions: Histogram, - pub(crate) proposed_block_ancestors: Histogram, - pub(crate) proposed_block_ancestors_depth: HistogramVec, - pub(crate) proposed_block_ancestors_timestamp_drift_ms: IntCounterVec, - pub(crate) highest_verified_authority_round: IntGaugeVec, - pub(crate) lowest_verified_authority_round: IntGaugeVec, - pub(crate) block_proposal_interval: Histogram, - pub(crate) block_proposal_leader_wait_ms: IntCounterVec, - pub(crate) block_proposal_leader_wait_count: IntCounterVec, - pub(crate) block_timestamp_drift_ms: IntCounterVec, - pub(crate) blocks_per_commit_count: Histogram, - pub(crate) latency_to_process_stream: HistogramVec, - pub(crate) blocks_pruned_on_commit: IntCounterVec, - pub(crate) broadcaster_rtt_estimate_ms: IntGaugeVec, - pub(crate) core_add_blocks_batch_size: Histogram, - pub(crate) core_check_block_refs_batch_size: Histogram, - pub(crate) core_lock_dequeued: IntCounter, - pub(crate) core_lock_enqueued: IntCounter, - pub(crate) core_skipped_proposals: IntCounterVec, - pub(crate) highest_accepted_authority_round: IntGaugeVec, - pub(crate) highest_accepted_round: IntGauge, - pub(crate) accepted_block_time_drift_ms: IntCounterVec, - pub(crate) accepted_blocks: IntCounterVec, - pub(crate) dag_state_recent_blocks: IntGauge, - pub(crate) dag_state_recent_refs: IntGauge, - pub(crate) dag_state_store_read_count: IntCounterVec, - pub(crate) dag_state_store_write_count: IntCounter, - pub(crate) fetch_blocks_scheduler_inflight: IntGauge, - pub(crate) fetch_blocks_scheduler_skipped: IntCounterVec, - pub(crate) synchronizer_fetched_blocks_by_peer: IntCounterVec, - pub(crate) synchronizer_skipped_blocks_by_peer: IntCounterVec, - pub(crate) synchronizer_requested_blocks_by_peer: IntCounterVec, - pub(crate) synchronizer_missing_blocks_by_authority: IntCounterVec, - pub(crate) synchronizer_current_missing_blocks_by_authority: IntGaugeVec, - pub(crate) synchronizer_fetched_blocks_by_authority: IntCounterVec, - pub(crate) synchronizer_requested_blocks_by_authority: IntCounterVec, - pub(crate) synchronizer_fetch_failures_by_peer: IntCounterVec, - pub(crate) synchronizer_process_fetched_failures_by_peer: IntCounterVec, - pub(crate) network_received_excluded_ancestors_from_authority: IntCounterVec, - pub(crate) network_excluded_ancestors_sent_to_fetch: IntCounterVec, - pub(crate) network_excluded_ancestors_count_by_authority: IntCounterVec, - pub(crate) invalid_blocks: IntCounterVec, - pub(crate) faulty_blocks_provable_by_authority: IntCounterVec, - pub(crate) faulty_blocks_unprovable_by_authority: IntCounterVec, - pub(crate) uncached_equivocations_by_authority: IntCounterVec, - pub(crate) uncached_missing_proposals_by_authority: IntCounterVec, - pub(crate) equivocations_in_cache_by_authority: IntGaugeVec, - pub(crate) missing_proposals_in_cache_by_authority: IntGaugeVec, - #[allow(dead_code)] - pub(crate) score_by_authority: IntGaugeVec, - #[allow(dead_code)] - pub(crate) invalid_misbehavior_reports_by_authority: IntCounterVec, - pub(crate) rejected_blocks: IntCounterVec, - pub(crate) rejected_future_blocks: IntCounterVec, - pub(crate) subscribed_blocks: IntCounterVec, - pub(crate) verified_blocks: IntCounterVec, - pub(crate) committed_leaders_total: IntCounterVec, - pub(crate) last_committed_authority_round: IntGaugeVec, - pub(crate) last_committed_leader_round: IntGauge, - pub(crate) last_commit_index: IntGauge, - pub(crate) last_commit_time_diff: Histogram, - pub(crate) last_known_own_block_round: IntGauge, - pub(crate) sync_last_known_own_block_retries: IntCounter, - pub(crate) commit_round_advancement_interval: Histogram, - pub(crate) last_decided_leader_round: IntGauge, - pub(crate) leader_timeout_total: IntCounterVec, - pub(crate) smart_selection_wait: IntCounter, - pub(crate) ancestor_state_change_by_authority: IntCounterVec, - pub(crate) excluded_proposal_ancestors_count_by_authority: IntCounterVec, - pub(crate) included_excluded_proposal_ancestors_count_by_authority: IntCounterVec, - pub(crate) missing_blocks_total: IntCounter, - pub(crate) missing_blocks_after_fetch_total: IntCounter, - pub(crate) num_of_bad_nodes: IntGauge, - pub(crate) quorum_receive_latency: Histogram, - pub(crate) reputation_scores: IntGaugeVec, - pub(crate) scope_processing_time: HistogramVec, - pub(crate) sub_dags_per_commit_count: Histogram, - pub(crate) block_suspensions: IntCounterVec, - pub(crate) block_unsuspensions: IntCounterVec, - pub(crate) suspended_block_time: HistogramVec, - pub(crate) block_manager_suspended_blocks: IntGauge, - pub(crate) block_manager_missing_ancestors: IntGauge, - pub(crate) block_manager_missing_blocks: IntGauge, - pub(crate) block_manager_missing_blocks_by_authority: IntCounterVec, - pub(crate) block_manager_missing_ancestors_by_authority: IntCounterVec, - pub(crate) block_manager_gced_blocks: IntCounterVec, - pub(crate) block_manager_gc_unsuspended_blocks: IntCounterVec, - pub(crate) block_manager_skipped_blocks: IntCounterVec, - pub(crate) threshold_clock_round: IntGauge, - pub(crate) subscriber_connection_attempts: IntCounterVec, - pub(crate) subscribed_to: IntGaugeVec, - pub(crate) subscribed_by: IntGaugeVec, - pub(crate) commit_sync_inflight_fetches: IntGauge, - pub(crate) commit_sync_pending_fetches: IntGauge, - pub(crate) commit_sync_fetch_commits_handler_uncertified_skipped: IntCounter, - pub(crate) commit_sync_fetched_commits: IntCounterVec, - pub(crate) commit_sync_fetched_blocks: IntCounterVec, - pub(crate) commit_sync_total_fetched_blocks_size: IntCounterVec, - pub(crate) commit_sync_quorum_index: IntGauge, - pub(crate) commit_sync_highest_synced_index: IntGauge, - pub(crate) commit_sync_highest_fetched_index: IntGauge, - pub(crate) commit_sync_local_index: IntGauge, - pub(crate) commit_sync_gap_on_processing: IntCounter, - pub(crate) commit_sync_fetch_loop_latency: Histogram, - pub(crate) commit_sync_fetch_once_latency: HistogramVec, - pub(crate) commit_sync_fetch_once_errors: IntCounterVec, - pub(crate) commit_sync_fetch_missing_blocks: IntCounterVec, - pub(crate) round_prober_received_quorum_round_gaps: IntGaugeVec, - pub(crate) round_prober_accepted_quorum_round_gaps: IntGaugeVec, - pub(crate) round_prober_low_received_quorum_round: IntGaugeVec, - pub(crate) round_prober_low_accepted_quorum_round: IntGaugeVec, - pub(crate) round_prober_current_received_round_gaps: IntGaugeVec, - pub(crate) round_prober_current_accepted_round_gaps: IntGaugeVec, - pub(crate) round_prober_propagation_delays: Histogram, - pub(crate) round_prober_last_propagation_delay: IntGauge, - pub(crate) round_prober_request_errors: IntCounterVec, - pub(crate) uptime: Histogram, -} - -impl NodeMetrics { - pub(crate) fn new(registry: &Registry) -> Self { - Self { - block_commit_latency: register_histogram_with_registry!( - "block_commit_latency", - "The time taken between block creation and block commit.", - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - proposed_blocks: register_int_counter_vec_with_registry!( - "proposed_blocks", - "Total number of proposed blocks. If force is true then this block has been created forcefully via a leader timeout event.", - &["force"], - registry, - ).unwrap(), - proposed_block_size: register_histogram_with_registry!( - "proposed_block_size", - "The size (in bytes) of proposed blocks", - SIZE_BUCKETS.to_vec(), - registry - ).unwrap(), - proposed_block_transactions: register_histogram_with_registry!( - "proposed_block_transactions", - "# of transactions contained in proposed blocks", - NUM_BUCKETS.to_vec(), - registry - ).unwrap(), - proposed_block_ancestors: register_histogram_with_registry!( - "proposed_block_ancestors", - "Number of ancestors in proposed blocks", - exponential_buckets(1.0, 1.4, 20).unwrap(), - registry, - ).unwrap(), - proposed_block_ancestors_timestamp_drift_ms: register_int_counter_vec_with_registry!( - "proposed_block_ancestors_timestamp_drift_ms", - "The drift in ms of ancestors' timestamps included in newly proposed blocks", - &["authority"], - registry, - ).unwrap(), - latency_to_process_stream: register_histogram_vec_with_registry!( - "latency_to_process_stream", - "The latency between block creation and processing stream from peer", - &["peer"], - exponential_buckets(0.002, 1.5, 18).unwrap(), - registry, - ).unwrap(), - proposed_block_ancestors_depth: register_histogram_vec_with_registry!( - "proposed_block_ancestors_depth", - "The depth in rounds of ancestors included in newly proposed blocks", - &["authority"], - exponential_buckets(1.0, 2.0, 14).unwrap(), - registry, - ).unwrap(), - highest_verified_authority_round: register_int_gauge_vec_with_registry!( - "highest_verified_authority_round", - "The highest round of verified block for the corresponding authority", - &["authority"], - registry, - ).unwrap(), - lowest_verified_authority_round: register_int_gauge_vec_with_registry!( - "lowest_verified_authority_round", - "The lowest round of verified block for the corresponding authority", - &["authority"], - registry, - ).unwrap(), - block_proposal_interval: register_histogram_with_registry!( - "block_proposal_interval", - "Intervals (in secs) between block proposals.", - FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - block_proposal_leader_wait_ms: register_int_counter_vec_with_registry!( - "block_proposal_leader_wait_ms", - "Total time in ms spent waiting for a leader when proposing blocks.", - &["authority"], - registry, - ).unwrap(), - block_proposal_leader_wait_count: register_int_counter_vec_with_registry!( - "block_proposal_leader_wait_count", - "Total times waiting for a leader when proposing blocks.", - &["authority"], - registry, - ).unwrap(), - block_timestamp_drift_ms: register_int_counter_vec_with_registry!( - "block_timestamp_drift_ms", - "The clock drift time between a received block and the current node's time.", - &["authority", "source"], - registry, - ).unwrap(), - blocks_per_commit_count: register_histogram_with_registry!( - "blocks_per_commit_count", - "The number of blocks per commit.", - NUM_BUCKETS.to_vec(), - registry, - ).unwrap(), - blocks_pruned_on_commit: register_int_counter_vec_with_registry!( - "blocks_pruned_on_commit", - "Number of blocks that got pruned due to garbage collection during a commit. This is not an accurate metric and measures the pruned blocks on the edge of the commit.", - &["authority", "commit_status"], - registry, - ).unwrap(), - broadcaster_rtt_estimate_ms: register_int_gauge_vec_with_registry!( - "broadcaster_rtt_estimate_ms", - "Estimated RTT latency per peer authority, for block sending in Broadcaster", - &["peer"], - registry, - ).unwrap(), - core_add_blocks_batch_size: register_histogram_with_registry!( - "core_add_blocks_batch_size", - "The number of blocks received from Core for processing on a single batch", - NUM_BUCKETS.to_vec(), - registry, - ).unwrap(), - core_check_block_refs_batch_size: register_histogram_with_registry!( - "core_check_block_refs_batch_size", - "The number of excluded blocks received from Core for search on a single batch", - NUM_BUCKETS.to_vec(), - registry, - ).unwrap(), - core_lock_dequeued: register_int_counter_with_registry!( - "core_lock_dequeued", - "Number of dequeued core requests", - registry, - ).unwrap(), - core_lock_enqueued: register_int_counter_with_registry!( - "core_lock_enqueued", - "Number of enqueued core requests", - registry, - ).unwrap(), - core_skipped_proposals: register_int_counter_vec_with_registry!( - "core_skipped_proposals", - "Number of proposals skipped in the Core, per reason", - &["reason"], - registry, - ).unwrap(), - highest_accepted_authority_round: register_int_gauge_vec_with_registry!( - "highest_accepted_authority_round", - "The highest round where a block has been accepted per authority. Resets on restart.", - &["authority"], - registry, - ).unwrap(), - highest_accepted_round: register_int_gauge_with_registry!( - "highest_accepted_round", - "The highest round where a block has been accepted. Resets on restart.", - registry, - ).unwrap(), - accepted_block_time_drift_ms: register_int_counter_vec_with_registry!( - "accepted_block_time_drift_ms", - "The time drift in ms of an accepted block compared to local time", - &["authority"], - registry, - ).unwrap(), - accepted_blocks: register_int_counter_vec_with_registry!( - "accepted_blocks", - "Number of accepted blocks by source (own, others)", - &["source"], - registry, - ).unwrap(), - dag_state_recent_blocks: register_int_gauge_with_registry!( - "dag_state_recent_blocks", - "Number of recent blocks cached in the DagState", - registry, - ).unwrap(), - dag_state_recent_refs: register_int_gauge_with_registry!( - "dag_state_recent_refs", - "Number of recent refs cached in the DagState", - registry, - ).unwrap(), - dag_state_store_read_count: register_int_counter_vec_with_registry!( - "dag_state_store_read_count", - "Number of times DagState needs to read from store per operation type", - &["type"], - registry, - ).unwrap(), - dag_state_store_write_count: register_int_counter_with_registry!( - "dag_state_store_write_count", - "Number of times DagState needs to write to store", - registry, - ).unwrap(), - fetch_blocks_scheduler_inflight: register_int_gauge_with_registry!( - "fetch_blocks_scheduler_inflight", - "Designates whether the synchronizer scheduler task to fetch blocks is currently running", - registry, - ).unwrap(), - fetch_blocks_scheduler_skipped: register_int_counter_vec_with_registry!( - "fetch_blocks_scheduler_skipped", - "Number of times the scheduler skipped fetching blocks", - &["reason"], - registry - ).unwrap(), - synchronizer_fetched_blocks_by_peer: register_int_counter_vec_with_registry!( - "synchronizer_fetched_blocks_by_peer", - "Number of fetched blocks per peer authority via the synchronizer and also by block authority", - &["peer", "type"], - registry, - ).unwrap(), - synchronizer_skipped_blocks_by_peer: register_int_counter_vec_with_registry!( - "synchronizer_skipped_blocks_by_peer", - "Number of skipped blocks per peer authority via the synchronizer due to already being verified", - &["peer", "type"], - registry, - ).unwrap(), - synchronizer_requested_blocks_by_peer: register_int_counter_vec_with_registry!( - "synchronizer_requested_blocks_by_peer", - "Number of requested blocks per peer authority via the synchronizer and also by block authority", - &["peer", "type"], - registry, - ).unwrap(), - synchronizer_missing_blocks_by_authority: register_int_counter_vec_with_registry!( - "synchronizer_missing_blocks_by_authority", - "Number of missing blocks per block author, as observed by the synchronizer during periodic sync.", - &["authority"], - registry, - ).unwrap(), - synchronizer_fetch_failures_by_peer: register_int_counter_vec_with_registry!( - "synchronizer_fetch_failures", - "Number of fetch failures against each peer", - &["peer", "type"], - registry, - ).unwrap(), - synchronizer_process_fetched_failures_by_peer: register_int_counter_vec_with_registry!( - "synchronizer_process_fetched_failures", - "Number of failures for processing fetched blocks against each peer", - &["peer", "type"], - registry, - ).unwrap(), - synchronizer_current_missing_blocks_by_authority: register_int_gauge_vec_with_registry!( - "synchronizer_current_missing_blocks_by_authority", - "Current number of missing blocks per block author, as observed by the synchronizer during periodic sync.", - &["authority"], - registry, - ).unwrap(), - synchronizer_fetched_blocks_by_authority: register_int_counter_vec_with_registry!( - "synchronizer_fetched_blocks_by_authority", - "Number of fetched blocks per block author via the synchronizer", - &["authority", "type"], - registry, - ).unwrap(), - synchronizer_requested_blocks_by_authority: register_int_counter_vec_with_registry!( - "synchronizer_requested_blocks_by_authority", - "Number of requested blocks per block author via the synchronizer", - &["authority", "type"], - registry, - ).unwrap(), - network_received_excluded_ancestors_from_authority: register_int_counter_vec_with_registry!( - "network_received_excluded_ancestors_from_authority", - "Number of excluded ancestors received from each authority.", - &["authority"], - registry, - ).unwrap(), - network_excluded_ancestors_count_by_authority: register_int_counter_vec_with_registry!( - "network_excluded_ancestors_count_by_authority", - "Total number of excluded ancestors per authority.", - &["authority"], - registry, - ).unwrap(), - network_excluded_ancestors_sent_to_fetch: register_int_counter_vec_with_registry!( - "network_excluded_ancestors_sent_to_fetch", - "Number of excluded ancestors sent to fetch.", - &["authority"], - registry, - ).unwrap(), - last_known_own_block_round: register_int_gauge_with_registry!( - "last_known_own_block_round", - "The highest round of our own block as this has been synced from peers during an amnesia recovery", - registry, - ).unwrap(), - sync_last_known_own_block_retries: register_int_counter_with_registry!( - "sync_last_known_own_block_retries", - "Number of times this node tried to fetch the last own block from peers", - registry, - ).unwrap(), - // TODO: add a short status label. - invalid_blocks: register_int_counter_vec_with_registry!( - "invalid_blocks", - "Number of invalid blocks per peer authority", - &["authority", "source", "error"], - registry, - ).unwrap(), - faulty_blocks_provable_by_authority: register_int_counter_vec_with_registry!( - "faulty_blocks_provable_by_authority", - "Number of provably faulty blocks per peer authority", - &["authority", "source", "error"], - registry, - ).unwrap(), - faulty_blocks_unprovable_by_authority: register_int_counter_vec_with_registry!( - "faulty_blocks_unprovable_by_authority", - "Number of unprovably faulty blocks per peer authority", - &["authority", "source", "error"], - registry, - ).unwrap(), - uncached_equivocations_by_authority: register_int_counter_vec_with_registry!( - "uncached_equivocations_by_authority", - "Registers the number of equivocations per authority that were already evicted from cache.", - &["authority"], - registry, - ).unwrap(), - uncached_missing_proposals_by_authority: register_int_counter_vec_with_registry!( - "uncached_missing_proposals_by_authority", - "Registers the number of blocks that should be already evicted from cache but authority failed to send.", - &["authority"], - registry, - ).unwrap(), - equivocations_in_cache_by_authority: register_int_gauge_vec_with_registry!( - "equivocations_in_cache_by_authority", - "Registers the number of equivocations per authority stored on cache.", - &["authority"], - registry, - ).unwrap(), - missing_proposals_in_cache_by_authority: register_int_gauge_vec_with_registry!( - "missing_proposals_in_cache_by_authority", - "Registers the number of blocks on the cache that an authority failed to send.", - &["authority"], - registry, - ).unwrap(), - score_by_authority: register_int_gauge_vec_with_registry!( - "score_by_authority", - "Registers the authority score.", - &["authority"], - registry, - ).unwrap(), - invalid_misbehavior_reports_by_authority: register_int_counter_vec_with_registry!( - "invalid_misbehavior_reports_by_authority", - "Number of invalid misbehavior reports received from each authority", - &["authority"], - registry, - ).unwrap(), - rejected_blocks: register_int_counter_vec_with_registry!( - "rejected_blocks", - "Number of blocks rejected before verifications", - &["reason"], - registry, - ).unwrap(), - rejected_future_blocks: register_int_counter_vec_with_registry!( - "rejected_future_blocks", - "Number of blocks rejected because their timestamp is too far in the future", - &["authority"], - registry, - ).unwrap(), - subscribed_blocks: register_int_counter_vec_with_registry!( - "subscribed_blocks", - "Number of blocks received from each peer before verification", - &["authority"], - registry, - ).unwrap(), - verified_blocks: register_int_counter_vec_with_registry!( - "verified_blocks", - "Number of blocks received from each peer that are verified", - &["authority"], - registry, - ).unwrap(), - committed_leaders_total: register_int_counter_vec_with_registry!( - "committed_leaders_total", - "Total number of (direct or indirect) committed leaders per authority", - &["authority", "commit_type"], - registry, - ).unwrap(), - last_committed_authority_round: register_int_gauge_vec_with_registry!( - "last_committed_authority_round", - "The last round committed by authority.", - &["authority"], - registry, - ).unwrap(), - last_committed_leader_round: register_int_gauge_with_registry!( - "last_committed_leader_round", - "The last round where a leader was committed to store and sent to commit consumer.", - registry, - ).unwrap(), - last_commit_index: register_int_gauge_with_registry!( - "last_commit_index", - "Index of the last commit.", - registry, - ).unwrap(), - last_commit_time_diff: register_histogram_with_registry!( - "last_commit_time_diff", - "The time diff between the last commit and previous one.", - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - commit_round_advancement_interval: register_histogram_with_registry!( - "commit_round_advancement_interval", - "Intervals (in secs) between commit round advancements.", - FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - last_decided_leader_round: register_int_gauge_with_registry!( - "last_decided_leader_round", - "The last round where a commit decision was made.", - registry, - ).unwrap(), - leader_timeout_total: register_int_counter_vec_with_registry!( - "leader_timeout_total", - "Total number of leader timeouts, either when the min round time has passed, or max leader timeout", - &["timeout_type"], - registry, - ).unwrap(), - smart_selection_wait: register_int_counter_with_registry!( - "smart_selection_wait", - "Number of times we waited for smart ancestor selection.", - registry, - ).unwrap(), - ancestor_state_change_by_authority: register_int_counter_vec_with_registry!( - "ancestor_state_change_by_authority", - "The total number of times an ancestor state changed to EXCLUDE or INCLUDE.", - &["authority", "state"], - registry, - ).unwrap(), - excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!( - "excluded_proposal_ancestors_count_by_authority", - "Total number of excluded ancestors per authority during proposal.", - &["authority"], - registry, - ).unwrap(), - included_excluded_proposal_ancestors_count_by_authority: register_int_counter_vec_with_registry!( - "included_excluded_proposal_ancestors_count_by_authority", - "Total number of ancestors per authority with 'excluded' status that got included in proposal. Either weak or strong type.", - &["authority", "type"], - registry, - ).unwrap(), - missing_blocks_total: register_int_counter_with_registry!( - "missing_blocks_total", - "Total cumulative number of missing blocks", - registry, - ).unwrap(), - missing_blocks_after_fetch_total: register_int_counter_with_registry!( - "missing_blocks_after_fetch_total", - "Total number of missing blocks after fetching blocks from peer", - registry, - ).unwrap(), - num_of_bad_nodes: register_int_gauge_with_registry!( - "num_of_bad_nodes", - "The number of bad nodes in the new leader schedule", - registry - ).unwrap(), - quorum_receive_latency: register_histogram_with_registry!( - "quorum_receive_latency", - "The time it took to receive a new round quorum of blocks", - registry - ).unwrap(), - reputation_scores: register_int_gauge_vec_with_registry!( - "reputation_scores", - "Reputation scores for each authority", - &["authority"], - registry, - ).unwrap(), - scope_processing_time: register_histogram_vec_with_registry!( - "scope_processing_time", - "The processing time of a specific code scope", - &["scope"], - FINE_GRAINED_LATENCY_SEC_BUCKETS.to_vec(), - registry - ).unwrap(), - sub_dags_per_commit_count: register_histogram_with_registry!( - "sub_dags_per_commit_count", - "The number of subdags per commit.", - registry, - ).unwrap(), - block_suspensions: register_int_counter_vec_with_registry!( - "block_suspensions", - "The number block suspensions. The counter is reported uniquely, so if a block is sent for reprocessing while already suspended then is not double counted", - &["authority"], - registry, - ).unwrap(), - block_unsuspensions: register_int_counter_vec_with_registry!( - "block_unsuspensions", - "The number of block unsuspensions.", - &["authority"], - registry, - ).unwrap(), - suspended_block_time: register_histogram_vec_with_registry!( - "suspended_block_time", - "The time for which a block remains suspended", - &["authority"], - registry, - ).unwrap(), - block_manager_suspended_blocks: register_int_gauge_with_registry!( - "block_manager_suspended_blocks", - "The number of blocks currently suspended in the block manager", - registry, - ).unwrap(), - block_manager_missing_ancestors: register_int_gauge_with_registry!( - "block_manager_missing_ancestors", - "The number of missing ancestors tracked in the block manager", - registry, - ).unwrap(), - block_manager_missing_blocks: register_int_gauge_with_registry!( - "block_manager_missing_blocks", - "The number of blocks missing content tracked in the block manager", - registry, - ).unwrap(), - block_manager_missing_blocks_by_authority: register_int_counter_vec_with_registry!( - "block_manager_missing_blocks_by_authority", - "The number of new missing blocks by block authority", - &["authority"], - registry, - ).unwrap(), - block_manager_missing_ancestors_by_authority: register_int_counter_vec_with_registry!( - "block_manager_missing_ancestors_by_authority", - "The number of missing ancestors by ancestor authority across received blocks", - &["authority"], - registry, - ).unwrap(), - block_manager_gced_blocks: register_int_counter_vec_with_registry!( - "block_manager_gced_blocks", - "The number of blocks that garbage collected and did not get accepted, counted by block's source authority", - &["authority"], - registry, - ).unwrap(), - block_manager_gc_unsuspended_blocks: register_int_counter_vec_with_registry!( - "block_manager_gc_unsuspended_blocks", - "The number of blocks unsuspended because their missing ancestors are garbage collected by the block manager, counted by block's source authority", - &["authority"], - registry, - ).unwrap(), - block_manager_skipped_blocks: register_int_counter_vec_with_registry!( - "block_manager_skipped_blocks", - "The number of blocks skipped by the block manager due to block round being <= gc_round", - &["authority"], - registry, - ).unwrap(), - threshold_clock_round: register_int_gauge_with_registry!( - "threshold_clock_round", - "The current threshold clock round. We only advance to a new round when a quorum of parents have been synced.", - registry, - ).unwrap(), - subscriber_connection_attempts: register_int_counter_vec_with_registry!( - "subscriber_connection_attempts", - "The number of connection attempts per peer", - &["authority", "status"], - registry, - ).unwrap(), - subscribed_to: register_int_gauge_vec_with_registry!( - "subscribed_to", - "Peers that this authority subscribed to for block streams.", - &["authority"], - registry, - ).unwrap(), - subscribed_by: register_int_gauge_vec_with_registry!( - "subscribed_by", - "Peers subscribing for block streams from this authority.", - &["authority"], - registry, - ).unwrap(), - commit_sync_inflight_fetches: register_int_gauge_with_registry!( - "commit_sync_inflight_fetches", - "The number of inflight fetches in commit syncer", - registry, - ).unwrap(), - commit_sync_pending_fetches: register_int_gauge_with_registry!( - "commit_sync_pending_fetches", - "The number of pending fetches in commit syncer", - registry, - ).unwrap(), - commit_sync_fetched_commits: register_int_counter_vec_with_registry!( - "commit_sync_fetched_commits", - "The number of commits fetched via commit syncer, labeled by authority.", - &["authority"], - registry, - ).unwrap(), - commit_sync_fetched_blocks: register_int_counter_vec_with_registry!( - "commit_sync_fetched_blocks", - "The number of blocks fetched via commit syncer, labeled by authority", - &["authority"], - registry, - ).unwrap(), - commit_sync_total_fetched_blocks_size: register_int_counter_vec_with_registry!( - "commit_sync_total_fetched_blocks_size", - "The total size in bytes of blocks fetched via commit syncer", - &["authority"], - registry, - ).unwrap(), - commit_sync_quorum_index: register_int_gauge_with_registry!( - "commit_sync_quorum_index", - "The maximum commit index voted by a quorum of authorities", - registry, - ).unwrap(), - commit_sync_highest_synced_index: register_int_gauge_with_registry!( - "commit_sync_fetched_index", - "The max commit index among local and fetched commits", - registry, - ).unwrap(), - commit_sync_highest_fetched_index: register_int_gauge_with_registry!( - "commit_sync_highest_fetched_index", - "The max commit index that has been fetched via network", - registry, - ).unwrap(), - commit_sync_local_index: register_int_gauge_with_registry!( - "commit_sync_local_index", - "The local commit index", - registry, - ).unwrap(), - commit_sync_gap_on_processing: register_int_counter_with_registry!( - "commit_sync_gap_on_processing", - "Number of instances where a gap was found in fetched commit processing", - registry, - ).unwrap(), - commit_sync_fetch_loop_latency: register_histogram_with_registry!( - "commit_sync_fetch_loop_latency", - "The time taken to finish fetching commits and blocks from a given range", - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - commit_sync_fetch_once_latency: register_histogram_vec_with_registry!( - "commit_sync_fetch_once_latency", - "The time taken to fetch commits and blocks once, labeled by target authority.", - &["authority"], - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - commit_sync_fetch_once_errors: register_int_counter_vec_with_registry!( - "commit_sync_fetch_once_errors", - "Number of errors when attempting to fetch commits and blocks from single authority during commit sync.", - &["authority", "error"], - registry - ).unwrap(), - commit_sync_fetch_commits_handler_uncertified_skipped: register_int_counter_with_registry!( - "commit_sync_fetch_commits_handler_uncertified_skipped", - "Number of uncertified commits that got skipped when fetching commits due to lack of votes", - registry, - ).unwrap(), - commit_sync_fetch_missing_blocks: register_int_counter_vec_with_registry!( - "commit_sync_fetch_missing_blocks", - "Number of ancestor blocks that are missing when processing blocks via commit sync.", - &["authority"], - registry - ).unwrap(), - round_prober_received_quorum_round_gaps: register_int_gauge_vec_with_registry!( - "round_prober_received_quorum_round_gaps", - "Received round gaps among peers for blocks proposed from each authority", - &["authority"], - registry - ).unwrap(), - round_prober_accepted_quorum_round_gaps: register_int_gauge_vec_with_registry!( - "round_prober_accepted_quorum_round_gaps", - "Accepted round gaps among peers for blocks proposed & accepted from each authority", - &["authority"], - registry - ).unwrap(), - round_prober_low_received_quorum_round: register_int_gauge_vec_with_registry!( - "round_prober_low_received_quorum_round", - "Low quorum round among peers for blocks proposed from each authority", - &["authority"], - registry - ).unwrap(), - round_prober_low_accepted_quorum_round: register_int_gauge_vec_with_registry!( - "round_prober_low_accepted_quorum_round", - "Low quorum round among peers for blocks proposed & accepted from each authority", - &["authority"], - registry - ).unwrap(), - round_prober_current_received_round_gaps: register_int_gauge_vec_with_registry!( - "round_prober_current_received_round_gaps", - "Received round gaps from local last proposed round to the low received quorum round of each peer. Can be negative.", - &["authority"], - registry - ).unwrap(), - round_prober_current_accepted_round_gaps: register_int_gauge_vec_with_registry!( - "round_prober_current_accepted_round_gaps", - "Accepted round gaps from local last proposed & accepted round to the low accepted quorum round of each peer. Can be negative.", - &["authority"], - registry - ).unwrap(), - round_prober_propagation_delays: register_histogram_with_registry!( - "round_prober_propagation_delays", - "Round gaps between the last proposed block round and the lower bound of own quorum round", - NUM_BUCKETS.to_vec(), - registry - ).unwrap(), - round_prober_last_propagation_delay: register_int_gauge_with_registry!( - "round_prober_last_propagation_delay", - "Most recent propagation delay observed by RoundProber", - registry - ).unwrap(), - round_prober_request_errors: register_int_counter_vec_with_registry!( - "round_prober_request_errors", - "Number of errors when probing against peers per error type", - &["error_type"], - registry - ).unwrap(), - uptime: register_histogram_with_registry!( - "uptime", - "Total node uptime", - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ).unwrap(), - } - } -} - -pub(crate) fn initialise_metrics(registry: Registry) -> Arc { - let node_metrics = NodeMetrics::new(®istry); - let network_metrics = NetworkMetrics::new(®istry); - Arc::new(Metrics { - node_metrics, - network_metrics, - }) -} - -#[cfg(test)] -pub(crate) fn test_metrics() -> Arc { - initialise_metrics(Registry::new()) -} diff --git a/consensus/core/src/network/metrics.rs b/consensus/core/src/network/metrics.rs deleted file mode 100644 index 459afeae2d5..00000000000 --- a/consensus/core/src/network/metrics.rs +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use prometheus::{ - HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, - register_histogram_vec_with_registry, register_int_counter_vec_with_registry, - register_int_gauge_vec_with_registry, register_int_gauge_with_registry, -}; - -// Fields for network-agnostic metrics can be added here -pub(crate) struct NetworkMetrics { - pub(crate) network_type: IntGaugeVec, - pub(crate) inbound: Arc, - pub(crate) outbound: Arc, - #[cfg_attr(msim, allow(dead_code))] - pub(crate) tcp_connection_metrics: Arc, -} - -impl NetworkMetrics { - pub(crate) fn new(registry: &Registry) -> Self { - Self { - network_type: register_int_gauge_vec_with_registry!( - "network_type", - "Type of the network used: tonic", - &["type"], - registry - ) - .unwrap(), - inbound: Arc::new(NetworkRouteMetrics::new("inbound", registry)), - outbound: Arc::new(NetworkRouteMetrics::new("outbound", registry)), - tcp_connection_metrics: Arc::new(TcpConnectionMetrics::new(registry)), - } - } -} - -#[cfg_attr(msim, allow(dead_code))] -pub(crate) struct TcpConnectionMetrics { - /// Send buffer size of consensus TCP socket. - pub(crate) socket_send_buffer_size: IntGauge, - /// Receive buffer size of consensus TCP socket. - pub(crate) socket_recv_buffer_size: IntGauge, - /// Max send buffer size of TCP socket. - pub(crate) socket_send_buffer_max_size: IntGauge, - /// Max receive buffer size of TCP socket. - pub(crate) socket_recv_buffer_max_size: IntGauge, -} - -impl TcpConnectionMetrics { - pub fn new(registry: &Registry) -> Self { - Self { - socket_send_buffer_size: register_int_gauge_with_registry!( - "tcp_socket_send_buffer_size", - "Send buffer size of consensus TCP socket.", - registry - ) - .unwrap(), - socket_recv_buffer_size: register_int_gauge_with_registry!( - "tcp_socket_recv_buffer_size", - "Receive buffer size of consensus TCP socket.", - registry - ) - .unwrap(), - socket_send_buffer_max_size: register_int_gauge_with_registry!( - "tcp_socket_send_buffer_max_size", - "Max send buffer size of TCP socket.", - registry - ) - .unwrap(), - socket_recv_buffer_max_size: register_int_gauge_with_registry!( - "tcp_socket_recv_buffer_max_size", - "Max receive buffer size of TCP socket.", - registry - ) - .unwrap(), - } - } -} - -#[derive(Clone)] -pub(crate) struct NetworkRouteMetrics { - /// Counter of requests by route - pub requests: IntCounterVec, - /// Request latency by route - pub request_latency: HistogramVec, - /// Request size by route - pub request_size: HistogramVec, - /// Response size by route - pub response_size: HistogramVec, - /// Counter of requests exceeding the "excessive" size limit - pub excessive_size_requests: IntCounterVec, - /// Counter of responses exceeding the "excessive" size limit - pub excessive_size_responses: IntCounterVec, - /// Gauge of the number of inflight requests at any given time by route - pub inflight_requests: IntGaugeVec, - /// Failed requests by route - pub errors: IntCounterVec, -} - -const LATENCY_SEC_BUCKETS: &[f64] = &[ - 0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1., 2.5, 5., 10., 20., 30., 60., 90., -]; - -// Arbitrarily chosen buckets for message size, with gradually-lowering exponent -// to give us better resolution at high sizes. -const SIZE_BYTE_BUCKETS: &[f64] = &[ - 2048., 8192., // *4 - 16384., 32768., 65536., 131072., 262144., 524288., 1048576., // *2 - 1572864., 2359256., 3538944., // *1.5 - 4600627., 5980815., 7775060., 10107578., 13139851., 17081807., 22206349., 28868253., 37528729., - 48787348., 63423553., // *1.3 -]; - -impl NetworkRouteMetrics { - pub fn new(direction: &'static str, registry: &Registry) -> Self { - let requests = register_int_counter_vec_with_registry!( - format!("{direction}_requests"), - "The number of requests made on the network", - &["route"], - registry - ) - .unwrap(); - - let request_latency = register_histogram_vec_with_registry!( - format!("{direction}_request_latency"), - "Latency of a request by route", - &["route"], - LATENCY_SEC_BUCKETS.to_vec(), - registry, - ) - .unwrap(); - - let request_size = register_histogram_vec_with_registry!( - format!("{direction}_request_size"), - "Size of a request by route", - &["route"], - SIZE_BYTE_BUCKETS.to_vec(), - registry, - ) - .unwrap(); - - let response_size = register_histogram_vec_with_registry!( - format!("{direction}_response_size"), - "Size of a response by route", - &["route"], - SIZE_BYTE_BUCKETS.to_vec(), - registry, - ) - .unwrap(); - - let excessive_size_requests = register_int_counter_vec_with_registry!( - format!("{direction}_excessive_size_requests"), - "The number of excessively large request messages sent", - &["route"], - registry - ) - .unwrap(); - - let excessive_size_responses = register_int_counter_vec_with_registry!( - format!("{direction}_excessive_size_responses"), - "The number of excessively large response messages seen", - &["route"], - registry - ) - .unwrap(); - - let inflight_requests = register_int_gauge_vec_with_registry!( - format!("{direction}_inflight_requests"), - "The number of inflight network requests", - &["route"], - registry - ) - .unwrap(); - - let errors = register_int_counter_vec_with_registry!( - format!("{direction}_request_errors"), - "Number of errors by route", - &["route", "status"], - registry, - ) - .unwrap(); - - Self { - requests, - request_latency, - request_size, - response_size, - excessive_size_requests, - excessive_size_responses, - inflight_requests, - errors, - } - } -} diff --git a/consensus/core/src/network/metrics_layer.rs b/consensus/core/src/network/metrics_layer.rs deleted file mode 100644 index 47c25484ff3..00000000000 --- a/consensus/core/src/network/metrics_layer.rs +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! Tower layer adapters that allow specifying callbacks for request and -//! response handling can be implemented for different networking stacks. - -use std::sync::Arc; - -use prometheus::HistogramTimer; - -use super::metrics::NetworkRouteMetrics; - -pub(crate) trait SizedRequest { - fn size(&self) -> usize; - fn route(&self) -> String; -} - -pub(crate) trait SizedResponse { - fn size(&self) -> usize; - fn error_type(&self) -> Option; -} - -#[derive(Clone)] -pub(crate) struct MetricsCallbackMaker { - metrics: Arc, - /// Size in bytes above which a request or response message is considered - /// excessively large - excessive_message_size: usize, -} - -impl MetricsCallbackMaker { - pub(crate) fn new(metrics: Arc, excessive_message_size: usize) -> Self { - Self { - metrics, - excessive_message_size, - } - } - - // Update request metrics. And create a callback that should be called on - // response. - pub(crate) fn handle_request(&self, request: &dyn SizedRequest) -> MetricsResponseCallback { - let route = request.route(); - - self.metrics.requests.with_label_values(&[&route]).inc(); - self.metrics - .inflight_requests - .with_label_values(&[&route]) - .inc(); - let request_size = request.size(); - if request_size > 0 { - self.metrics - .request_size - .with_label_values(&[&route]) - .observe(request_size as f64); - } - if request_size > self.excessive_message_size { - self.metrics - .excessive_size_requests - .with_label_values(&[&route]) - .inc(); - } - - let timer = self - .metrics - .request_latency - .with_label_values(&[&route]) - .start_timer(); - - MetricsResponseCallback { - metrics: self.metrics.clone(), - timer, - route, - excessive_message_size: self.excessive_message_size, - response_body_size: None, - } - } -} - -pub(crate) struct MetricsResponseCallback { - metrics: Arc, - // The timer is held on to and "observed" once dropped - #[expect(unused)] - timer: HistogramTimer, - route: String, - excessive_message_size: usize, - /// If Some, response size has already been observed (exact size was known - /// from headers). If None, response size should be tracked via body - /// chunks. - response_body_size: Option, -} - -impl MetricsResponseCallback { - /// Track response metrics from HTTP response parts. - /// Handles both size tracking and error tracking. - pub(crate) fn on_response(&mut self, response: &dyn SizedResponse, headers: &http::HeaderMap) { - let mut response_size = response.size(); - - // Try to get exact body size from Content-Length header - // This is calculated outside of response.size() to properly handle - // streaming/chunked responses later to avoid calculating the same bytes twice - - // once from the header value and when parsing the response body chunks. - let body_size = headers - .get(http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - - if let Some(body_size) = body_size { - // Exact size is known, update response size - response_size += body_size; - - // Mark as already observed - self.response_body_size = Some(body_size); - } - if response_size > 0 { - self.metrics - .response_size - .with_label_values(&[&self.route]) - .observe(response_size as f64); - } - if response_size > self.excessive_message_size { - self.metrics - .excessive_size_responses - .with_label_values(&[&self.route]) - .inc(); - } - - if let Some(err) = response.error_type() { - self.metrics - .errors - .with_label_values(&[&self.route, &err]) - .inc(); - } - } - - pub(crate) fn on_error(&mut self, _error: &E) { - self.metrics - .errors - .with_label_values(&[self.route.as_str(), "unknown"]) - .inc(); - } - - pub(crate) fn on_chunk(&mut self, chunk_size: usize) { - // Only track chunks if the exact size wasn't known from headers - if self.response_body_size.is_none() && chunk_size > 0 { - self.metrics - .response_size - .with_label_values(&[&self.route]) - .observe(chunk_size as f64); - } - } -} - -impl Drop for MetricsResponseCallback { - fn drop(&mut self) { - self.metrics - .inflight_requests - .with_label_values(&[&self.route]) - .dec(); - } -} diff --git a/consensus/core/src/network/mod.rs b/consensus/core/src/network/mod.rs deleted file mode 100644 index 2b94cf62136..00000000000 --- a/consensus/core/src/network/mod.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! This module defines the network interface, and provides network -//! implementations for the consensus protocol. -//! -//! Having an abstract network interface allows -//! - simplifying the semantics of sending data and serving requests over the -//! network -//! - hiding implementation specific types and semantics from the consensus -//! protocol -//! - allowing easy swapping of network implementations, for better performance -//! or testing -//! -//! When modifying the client and server interfaces, the principle is to keep -//! the interfaces low level, close to underlying implementations in semantics. -//! For example, the client interface exposes sending messages to a specific -//! peer, instead of broadcasting to all peers. Subscribing to a stream of -//! blocks gets back the stream via response, instead of delivering the stream -//! directly to the server. This keeps the logic agnostics to the underlying -//! network outside of this module, so they can be reused easily across network -//! implementations. - -use std::{pin::Pin, sync::Arc, time::Duration}; - -use async_trait::async_trait; -use bytes::Bytes; -use consensus_config::{AuthorityIndex, NetworkKeyPair}; -use futures::Stream; - -use crate::{ - Round, - block::{BlockRef, ExtendedBlock, VerifiedBlock}, - commit::{CommitRange, TrustedCommit}, - context::Context, - error::ConsensusResult, -}; - -// Tonic generated RPC stubs. -mod tonic_gen { - include!(concat!(env!("OUT_DIR"), "/consensus.ConsensusService.rs")); -} - -pub(crate) mod metrics; -mod metrics_layer; -#[cfg(all(test, not(msim)))] -mod network_tests; -#[cfg(test)] -pub(crate) mod test_network; -#[cfg(not(msim))] -pub(crate) mod tonic_network; -#[cfg(msim)] -pub mod tonic_network; -mod tonic_tls; - -/// A stream of serialized filtered blocks returned over the network. -pub(crate) type BlockStream = Pin + Send>>; - -/// Network client for communicating with peers. -/// -/// NOTE: the timeout parameters help saving resources at client and potentially -/// server. But it is up to the server implementation if the timeout is honored. -/// - To bound server resources, server should implement own timeout for -/// incoming requests. -#[async_trait] -pub(crate) trait NetworkClient: Send + Sync + Sized + 'static { - // Whether the network client streams blocks to subscribed peers. - const SUPPORT_STREAMING: bool; - - /// Sends a serialized SignedBlock to a peer. - async fn send_block( - &self, - peer: AuthorityIndex, - block: &VerifiedBlock, - timeout: Duration, - ) -> ConsensusResult<()>; - - /// Subscribes to blocks from a peer after last_received round. - async fn subscribe_blocks( - &self, - peer: AuthorityIndex, - last_received: Round, - timeout: Duration, - ) -> ConsensusResult; - - // TODO: add a parameter for maximum total size of blocks returned. - /// Fetches serialized `SignedBlock`s from a peer. It also might return - /// additional ancestor blocks of the requested blocks according to the - /// provided `highest_accepted_rounds`. The `highest_accepted_rounds` - /// length should be equal to the committee size. If - /// `highest_accepted_rounds` is empty then it will be simply ignored. - async fn fetch_blocks( - &self, - peer: AuthorityIndex, - block_refs: Vec, - highest_accepted_rounds: Vec, - timeout: Duration, - ) -> ConsensusResult>; - - /// Fetches serialized commits in the commit range from a peer. - /// Returns a tuple of both the serialized commits, and serialized blocks - /// that contain votes certifying the last commit. - async fn fetch_commits( - &self, - peer: AuthorityIndex, - commit_range: CommitRange, - timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)>; - - /// Fetches the latest block from `peer` for the requested `authorities`. - /// The latest blocks are returned in the serialised format of - /// `SignedBlocks`. The method can return multiple blocks per peer as - /// its possible to have equivocations. - async fn fetch_latest_blocks( - &self, - peer: AuthorityIndex, - authorities: Vec, - timeout: Duration, - ) -> ConsensusResult>; - - /// Gets the latest received & accepted rounds of all authorities from the - /// peer. - async fn get_latest_rounds( - &self, - peer: AuthorityIndex, - timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)>; -} - -/// Network service for handling requests from peers. -#[async_trait] -pub(crate) trait NetworkService: Send + Sync + 'static { - /// Handles the block sent from the peer via either unicast RPC or - /// subscription stream. Peer value can be trusted to be a valid - /// authority index. But serialized_block must be verified before its - /// contents are trusted. - /// Excluded ancestors are also included as part of an effort to further - /// propagate blocks to peers despite the current exclusion. - async fn handle_send_block( - &self, - peer: AuthorityIndex, - block: ExtendedSerializedBlock, - ) -> ConsensusResult<()>; - - /// Handles the subscription request from the peer. - /// A stream of newly proposed blocks is returned to the peer. - /// The stream continues until the end of epoch, peer unsubscribes, or a - /// network error / crash occurs. - async fn handle_subscribe_blocks( - &self, - peer: AuthorityIndex, - last_received: Round, - ) -> ConsensusResult; - - /// Handles the request to fetch blocks by references from the peer. - async fn handle_fetch_blocks( - &self, - peer: AuthorityIndex, - block_refs: Vec, - highest_accepted_rounds: Vec, - ) -> ConsensusResult>; - - /// Handles the request to fetch commits by index range from the peer. - async fn handle_fetch_commits( - &self, - peer: AuthorityIndex, - commit_range: CommitRange, - ) -> ConsensusResult<(Vec, Vec)>; - - /// Handles the request to fetch the latest block for the provided - /// `authorities`. - async fn handle_fetch_latest_blocks( - &self, - peer: AuthorityIndex, - authorities: Vec, - ) -> ConsensusResult>; - - /// Handles the request to get the latest received & accepted rounds of all - /// authorities. - async fn handle_get_latest_rounds( - &self, - peer: AuthorityIndex, - ) -> ConsensusResult<(Vec, Vec)>; -} - -/// An `AuthorityNode` holds a `NetworkManager` until shutdown. -/// Dropping `NetworkManager` will shutdown the network service. -pub(crate) trait NetworkManager: Send + Sync -where - S: NetworkService, -{ - type Client: NetworkClient; - - /// Creates a new network manager. - fn new(context: Arc, network_keypair: NetworkKeyPair) -> Self; - - /// Returns the network client. - fn client(&self) -> Arc; - - /// Installs network service. - async fn install_service(&mut self, service: Arc); - - /// Stops the network service. - async fn stop(&mut self); -} - -/// Serialized block with extended information from the proposing authority. -#[derive(Clone, PartialEq, Eq, Debug)] -pub(crate) struct ExtendedSerializedBlock { - pub(crate) block: Bytes, - // Serialized BlockRefs that are excluded from the blocks ancestors. - pub(crate) excluded_ancestors: Vec>, -} - -impl From for ExtendedSerializedBlock { - fn from(extended_block: ExtendedBlock) -> Self { - Self { - block: extended_block.block.serialized().clone(), - excluded_ancestors: extended_block - .excluded_ancestors - .iter() - .filter_map(|r| match bcs::to_bytes(r) { - Ok(serialized) => Some(serialized), - Err(e) => { - tracing::debug!("Failed to serialize block ref {:?}: {e:?}", r); - None - } - }) - .collect(), - } - } -} diff --git a/consensus/core/src/network/network_tests.rs b/consensus/core/src/network/network_tests.rs deleted file mode 100644 index 37faa6dc3ef..00000000000 --- a/consensus/core/src/network/network_tests.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use bytes::Bytes; -use consensus_config::NetworkKeyPair; -use futures::StreamExt as _; -use parking_lot::Mutex; -use rstest::rstest; -use tokio::time::sleep; - -use super::{ - ExtendedSerializedBlock, NetworkClient, NetworkManager, test_network::TestService, - tonic_network::TonicManager, -}; -use crate::{ - Round, - block::{TestBlock, VerifiedBlock}, - context::Context, -}; - -trait ManagerBuilder { - fn build( - &self, - context: Arc, - network_keypair: NetworkKeyPair, - ) -> impl NetworkManager>; -} - -struct TonicManagerBuilder {} - -impl ManagerBuilder for TonicManagerBuilder { - fn build( - &self, - context: Arc, - network_keypair: NetworkKeyPair, - ) -> impl NetworkManager> { - TonicManager::new(context, network_keypair) - } -} - -fn block_for_round(round: Round) -> ExtendedSerializedBlock { - ExtendedSerializedBlock { - block: Bytes::from(vec![round as u8; 16]), - excluded_ancestors: vec![], - } -} - -fn service_with_own_blocks() -> Arc> { - let service = Arc::new(Mutex::new(TestService::new())); - { - let mut service = service.lock(); - let own_blocks = (0..=100u8) - .map(|i| block_for_round(i as Round)) - .collect::>(); - service.add_own_blocks(own_blocks); - } - service -} - -// TODO: figure out the issue with using simulated time with tonic in this test. -// When waiting for the server to become ready, it may need to use -// std::thread::sleep() instead of tokio::time::sleep(). -#[rstest] -#[tokio::test] -async fn send_and_receive_blocks_with_auth( - #[values(TonicManagerBuilder {})] manager_builder: impl ManagerBuilder, -) { - let (context, keys) = Context::new_for_test(4); - - let context_0 = Arc::new( - context - .clone() - .with_authority_index(context.committee.to_authority_index(0).unwrap()), - ); - let mut manager_0 = manager_builder.build(context_0.clone(), keys[0].0.clone()); - let client_0 = manager_0.client(); - let service_0 = service_with_own_blocks(); - manager_0.install_service(service_0.clone()).await; - - let context_1 = Arc::new( - context - .clone() - .with_authority_index(context.committee.to_authority_index(1).unwrap()), - ); - let mut manager_1 = manager_builder.build(context_1.clone(), keys[1].0.clone()); - let client_1 = manager_1.client(); - let service_1 = service_with_own_blocks(); - manager_1.install_service(service_1.clone()).await; - - // Wait for tonic to initialize. - sleep(Duration::from_secs(5)).await; - - // Test that servers can receive client RPCs. - let test_block_0 = VerifiedBlock::new_for_test(TestBlock::new(9, 0).build()); - client_0 - .send_block( - context.committee.to_authority_index(1).unwrap(), - &test_block_0, - Duration::from_secs(5), - ) - .await - .unwrap(); - let test_block_1 = VerifiedBlock::new_for_test(TestBlock::new(9, 1).build()); - client_1 - .send_block( - context.committee.to_authority_index(0).unwrap(), - &test_block_1, - Duration::from_secs(5), - ) - .await - .unwrap(); - - assert_eq!(service_0.lock().handle_send_block.len(), 1); - assert_eq!(service_0.lock().handle_send_block[0].0.value(), 1); - assert_eq!( - service_0.lock().handle_send_block[0].1, - ExtendedSerializedBlock { - block: test_block_1.serialized().clone(), - excluded_ancestors: vec![], - }, - ); - assert_eq!(service_1.lock().handle_send_block.len(), 1); - assert_eq!(service_1.lock().handle_send_block[0].0.value(), 0); - assert_eq!( - service_1.lock().handle_send_block[0].1, - ExtendedSerializedBlock { - block: test_block_0.serialized().clone(), - excluded_ancestors: vec![], - }, - ); - - // `Committee` is generated with the same random seed in - // Context::new_for_test(), so the first 4 authorities are the same. - let (context_4, keys_4) = Context::new_for_test(5); - let context_4 = Arc::new( - context_4 - .clone() - .with_authority_index(context_4.committee.to_authority_index(4).unwrap()), - ); - let mut manager_4 = manager_builder.build(context_4.clone(), keys_4[4].0.clone()); - let client_4 = manager_4.client(); - let service_4 = service_with_own_blocks(); - manager_4.install_service(service_4.clone()).await; - - // client_4 should not be able to reach service_0 or service_1, because of the - // AllowedPeers filter. - let test_block_2 = VerifiedBlock::new_for_test(TestBlock::new(9, 2).build()); - assert!( - client_4 - .send_block( - context.committee.to_authority_index(0).unwrap(), - &test_block_2, - Duration::from_secs(5), - ) - .await - .is_err() - ); - let test_block_3 = VerifiedBlock::new_for_test(TestBlock::new(9, 3).build()); - assert!( - client_4 - .send_block( - context.committee.to_authority_index(1).unwrap(), - &test_block_3, - Duration::from_secs(5), - ) - .await - .is_err() - ); -} - -#[rstest] -#[tokio::test] -async fn subscribe_and_receive_blocks( - // Only network supporting streaming can be tested. - #[values(TonicManagerBuilder {})] manager_builder: impl ManagerBuilder, -) { - let (context, keys) = Context::new_for_test(4); - - let context_0 = Arc::new( - context - .clone() - .with_authority_index(context.committee.to_authority_index(0).unwrap()), - ); - let mut manager_0 = manager_builder.build(context_0.clone(), keys[0].0.clone()); - let client_0 = manager_0.client(); - let service_0 = service_with_own_blocks(); - manager_0.install_service(service_0.clone()).await; - - let context_1 = Arc::new( - context - .clone() - .with_authority_index(context.committee.to_authority_index(1).unwrap()), - ); - let mut manager_1 = manager_builder.build(context_1.clone(), keys[1].0.clone()); - let client_1 = manager_1.client(); - let service_1 = service_with_own_blocks(); - manager_1.install_service(service_1.clone()).await; - - let client_0_round = 50; - let receive_stream_0 = client_0 - .subscribe_blocks( - context_0.committee.to_authority_index(1).unwrap(), - client_0_round, - Duration::from_secs(5), - ) - .await - .unwrap(); - - let count = receive_stream_0 - .enumerate() - .then(|(i, item)| async move { - assert_eq!(item, block_for_round(client_0_round + i as Round + 1)); - 1 - }) - .fold(0, |a, b| async move { a + b }) - .await; - // Round 51 to 100 blocks should have been received. - assert_eq!(count, 50); - - let client_1_round = 100; - let mut receive_stream_1 = client_1 - .subscribe_blocks( - context_1.committee.to_authority_index(0).unwrap(), - client_1_round, - Duration::from_secs(5), - ) - .await - .unwrap(); - assert!(receive_stream_1.next().await.is_none()); -} diff --git a/consensus/core/src/network/test_network.rs b/consensus/core/src/network/test_network.rs deleted file mode 100644 index d0cc00d65ab..00000000000 --- a/consensus/core/src/network/test_network.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use async_trait::async_trait; -use bytes::Bytes; -use consensus_config::AuthorityIndex; -use futures::stream; -use parking_lot::Mutex; - -use super::ExtendedSerializedBlock; -use crate::{ - Round, - block::{BlockRef, VerifiedBlock}, - commit::{CommitRange, TrustedCommit}, - error::ConsensusResult, - network::{BlockStream, NetworkService}, -}; - -pub(crate) struct TestService { - pub(crate) handle_send_block: Vec<(AuthorityIndex, ExtendedSerializedBlock)>, - pub(crate) handle_fetch_blocks: Vec<(AuthorityIndex, Vec)>, - pub(crate) handle_subscribe_blocks: Vec<(AuthorityIndex, Round)>, - pub(crate) handle_fetch_commits: Vec<(AuthorityIndex, CommitRange)>, - pub(crate) own_blocks: Vec, -} - -impl TestService { - pub(crate) fn new() -> Self { - Self { - handle_send_block: Vec::new(), - handle_fetch_blocks: Vec::new(), - handle_subscribe_blocks: Vec::new(), - handle_fetch_commits: Vec::new(), - own_blocks: Vec::new(), - } - } - - #[cfg_attr(msim, allow(dead_code))] - pub(crate) fn add_own_blocks(&mut self, blocks: Vec) { - self.own_blocks.extend(blocks); - } -} - -#[async_trait] -impl NetworkService for Mutex { - async fn handle_send_block( - &self, - peer: AuthorityIndex, - block: ExtendedSerializedBlock, - ) -> ConsensusResult<()> { - let mut state = self.lock(); - state.handle_send_block.push((peer, block)); - Ok(()) - } - - async fn handle_subscribe_blocks( - &self, - peer: AuthorityIndex, - last_received: Round, - ) -> ConsensusResult { - let mut state = self.lock(); - state.handle_subscribe_blocks.push((peer, last_received)); - let own_blocks = state - .own_blocks - .iter() - // Let index in own_blocks be the round, and skip blocks <= last_received round. - .skip(last_received as usize + 1) - .cloned() - .collect::>(); - Ok(Box::pin(stream::iter(own_blocks))) - } - - async fn handle_fetch_blocks( - &self, - peer: AuthorityIndex, - block_refs: Vec, - _highest_accepted_rounds: Vec, - ) -> ConsensusResult> { - self.lock().handle_fetch_blocks.push((peer, block_refs)); - Ok(vec![]) - } - - async fn handle_fetch_commits( - &self, - peer: AuthorityIndex, - commit_range: CommitRange, - ) -> ConsensusResult<(Vec, Vec)> { - self.lock().handle_fetch_commits.push((peer, commit_range)); - Ok((vec![], vec![])) - } - - async fn handle_fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn handle_get_latest_rounds( - &self, - _peer: AuthorityIndex, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } -} diff --git a/consensus/core/src/network/tonic_network.rs b/consensus/core/src/network/tonic_network.rs deleted file mode 100644 index fb2cb808d04..00000000000 --- a/consensus/core/src/network/tonic_network.rs +++ /dev/null @@ -1,1169 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::BTreeMap, - net::{SocketAddr, SocketAddrV4, SocketAddrV6}, - pin::Pin, - sync::Arc, - time::{Duration, Instant}, -}; - -use async_trait::async_trait; -use bytes::Bytes; -use consensus_config::{AuthorityIndex, NetworkKeyPair, NetworkPublicKey}; -use futures::{Stream, StreamExt as _, stream}; -use iota_http::ServerHandle; -use iota_network_stack::{ - Multiaddr, - callback::{CallbackLayer, MakeCallbackHandler, ResponseHandler}, - multiaddr::Protocol, -}; -use iota_tls::AllowPublicKeys; -use parking_lot::RwLock; -use tokio_stream::{Iter, iter}; -use tonic::{Request, Response, Streaming, codec::CompressionEncoding}; -use tower_http::trace::{DefaultMakeSpan, DefaultOnFailure, TraceLayer}; -use tracing::{debug, error, info, trace, warn}; - -use super::{ - BlockStream, ExtendedSerializedBlock, NetworkClient, NetworkManager, NetworkService, - metrics_layer::{MetricsCallbackMaker, MetricsResponseCallback, SizedRequest, SizedResponse}, - tonic_gen::{ - consensus_service_client::ConsensusServiceClient, - consensus_service_server::ConsensusService, - }, -}; -use crate::{ - CommitIndex, Round, - block::{BlockRef, VerifiedBlock}, - commit::CommitRange, - context::Context, - error::{ConsensusError, ConsensusResult}, - network::{ - tonic_gen::consensus_service_server::ConsensusServiceServer, - tonic_tls::certificate_server_name, - }, -}; - -// Maximum bytes size in a single fetch_blocks()response. -// TODO: put max RPC response size in protocol config. -const MAX_FETCH_RESPONSE_BYTES: usize = 4 * 1024 * 1024; - -// Maximum total bytes fetched in a single fetch_blocks() call, after combining -// the responses. -const MAX_TOTAL_FETCHED_BYTES: usize = 128 * 1024 * 1024; - -// Implements Tonic RPC client for Consensus. -pub(crate) struct TonicClient { - context: Arc, - network_keypair: NetworkKeyPair, - channel_pool: Arc, -} - -impl TonicClient { - pub(crate) fn new(context: Arc, network_keypair: NetworkKeyPair) -> Self { - Self { - context: context.clone(), - network_keypair, - channel_pool: Arc::new(ChannelPool::new(context)), - } - } - - async fn get_client( - &self, - peer: AuthorityIndex, - timeout: Duration, - ) -> ConsensusResult> { - let config = &self.context.parameters.tonic; - let channel = self - .channel_pool - .get_channel(self.network_keypair.clone(), peer, timeout) - .await?; - let mut client = ConsensusServiceClient::new(channel) - .max_encoding_message_size(config.message_size_limit) - .max_decoding_message_size(config.message_size_limit); - - if self.context.protocol_config.consensus_zstd_compression() { - client = client - .send_compressed(CompressionEncoding::Zstd) - .accept_compressed(CompressionEncoding::Zstd); - } - Ok(client) - } -} - -// TODO: make sure callsites do not send request to own index, and return error -// otherwise. -#[async_trait] -impl NetworkClient for TonicClient { - const SUPPORT_STREAMING: bool = true; - - async fn send_block( - &self, - peer: AuthorityIndex, - block: &VerifiedBlock, - timeout: Duration, - ) -> ConsensusResult<()> { - let mut client = self.get_client(peer, timeout).await?; - let mut request = Request::new(SendBlockRequest { - block: block.serialized().clone(), - }); - request.set_timeout(timeout); - client - .send_block(request) - .await - .map_err(|e| ConsensusError::NetworkRequest(format!("send_block failed: {e:?}")))?; - Ok(()) - } - - async fn subscribe_blocks( - &self, - peer: AuthorityIndex, - last_received: Round, - timeout: Duration, - ) -> ConsensusResult { - let mut client = self.get_client(peer, timeout).await?; - // TODO: add sampled block acknowledgments for latency measurements. - let request = Request::new(stream::once(async move { - SubscribeBlocksRequest { - last_received_round: last_received, - } - })); - let response = client.subscribe_blocks(request).await.map_err(|e| { - ConsensusError::NetworkRequest(format!("subscribe_blocks failed: {e:?}")) - })?; - let stream = response - .into_inner() - .take_while(|b| futures::future::ready(b.is_ok())) - .filter_map(move |b| async move { - match b { - Ok(response) => Some(ExtendedSerializedBlock { - block: response.block, - excluded_ancestors: response.excluded_ancestors, - }), - Err(e) => { - debug!("Network error received from {}: {e:?}", peer); - None - } - } - }); - let rate_limited_stream = - tokio_stream::StreamExt::throttle(stream, self.context.parameters.min_round_delay / 2) - .boxed(); - Ok(rate_limited_stream) - } - - async fn fetch_blocks( - &self, - peer: AuthorityIndex, - block_refs: Vec, - highest_accepted_rounds: Vec, - timeout: Duration, - ) -> ConsensusResult> { - let mut client = self.get_client(peer, timeout).await?; - let mut request = Request::new(FetchBlocksRequest { - block_refs: block_refs - .iter() - .filter_map(|r| match bcs::to_bytes(r) { - Ok(serialized) => Some(serialized), - Err(e) => { - debug!("Failed to serialize block ref {:?}: {e:?}", r); - None - } - }) - .collect(), - highest_accepted_rounds, - }); - request.set_timeout(timeout); - let mut stream = client - .fetch_blocks(request) - .await - .map_err(|e| { - if e.code() == tonic::Code::DeadlineExceeded { - ConsensusError::NetworkRequestTimeout(format!("fetch_blocks failed: {e:?}")) - } else { - ConsensusError::NetworkRequest(format!("fetch_blocks failed: {e:?}")) - } - })? - .into_inner(); - let mut blocks = vec![]; - let mut total_fetched_bytes = 0; - loop { - match stream.message().await { - Ok(Some(response)) => { - for b in &response.blocks { - total_fetched_bytes += b.len(); - } - blocks.extend(response.blocks); - if total_fetched_bytes > MAX_TOTAL_FETCHED_BYTES { - info!( - "fetch_blocks() fetched bytes exceeded limit: {} > {}, terminating stream.", - total_fetched_bytes, MAX_TOTAL_FETCHED_BYTES, - ); - break; - } - } - Ok(None) => { - break; - } - Err(e) => { - if blocks.is_empty() { - if e.code() == tonic::Code::DeadlineExceeded { - return Err(ConsensusError::NetworkRequestTimeout(format!( - "fetch_blocks failed mid-stream: {e:?}" - ))); - } - return Err(ConsensusError::NetworkRequest(format!( - "fetch_blocks failed mid-stream: {e:?}" - ))); - } else { - warn!("fetch_blocks failed mid-stream: {e:?}"); - break; - } - } - } - } - Ok(blocks) - } - - async fn fetch_commits( - &self, - peer: AuthorityIndex, - commit_range: CommitRange, - timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - let mut client = self.get_client(peer, timeout).await?; - let mut request = Request::new(FetchCommitsRequest { - start: commit_range.start(), - end: commit_range.end(), - }); - request.set_timeout(timeout); - let response = client - .fetch_commits(request) - .await - .map_err(|e| ConsensusError::NetworkRequest(format!("fetch_commits failed: {e:?}")))?; - let response = response.into_inner(); - Ok((response.commits, response.certifier_blocks)) - } - - async fn fetch_latest_blocks( - &self, - peer: AuthorityIndex, - authorities: Vec, - timeout: Duration, - ) -> ConsensusResult> { - let mut client = self.get_client(peer, timeout).await?; - let mut request = Request::new(FetchLatestBlocksRequest { - authorities: authorities - .iter() - .map(|authority| authority.value() as u32) - .collect(), - }); - request.set_timeout(timeout); - let mut stream = client - .fetch_latest_blocks(request) - .await - .map_err(|e| { - if e.code() == tonic::Code::DeadlineExceeded { - ConsensusError::NetworkRequestTimeout(format!("fetch_blocks failed: {e:?}")) - } else { - ConsensusError::NetworkRequest(format!("fetch_blocks failed: {e:?}")) - } - })? - .into_inner(); - let mut blocks = vec![]; - let mut total_fetched_bytes = 0; - loop { - match stream.message().await { - Ok(Some(response)) => { - for b in &response.blocks { - total_fetched_bytes += b.len(); - } - blocks.extend(response.blocks); - if total_fetched_bytes > MAX_TOTAL_FETCHED_BYTES { - info!( - "fetch_blocks() fetched bytes exceeded limit: {} > {}, terminating stream.", - total_fetched_bytes, MAX_TOTAL_FETCHED_BYTES, - ); - break; - } - } - Ok(None) => { - break; - } - Err(e) => { - if blocks.is_empty() { - if e.code() == tonic::Code::DeadlineExceeded { - return Err(ConsensusError::NetworkRequestTimeout(format!( - "fetch_blocks failed mid-stream: {e:?}" - ))); - } - return Err(ConsensusError::NetworkRequest(format!( - "fetch_blocks failed mid-stream: {e:?}" - ))); - } else { - warn!("fetch_latest_blocks failed mid-stream: {e:?}"); - break; - } - } - } - } - Ok(blocks) - } - - async fn get_latest_rounds( - &self, - peer: AuthorityIndex, - timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - let mut client = self.get_client(peer, timeout).await?; - let mut request = Request::new(GetLatestRoundsRequest {}); - request.set_timeout(timeout); - let response = client.get_latest_rounds(request).await.map_err(|e| { - ConsensusError::NetworkRequest(format!("get_latest_rounds failed: {e:?}")) - })?; - let response = response.into_inner(); - Ok((response.highest_received, response.highest_accepted)) - } -} - -// Tonic channel wrapped with layers. -type Channel = iota_network_stack::callback::Callback< - tower_http::trace::Trace< - tonic_rustls::Channel, - tower_http::classify::SharedClassifier, - >, - MetricsCallbackMaker, ->; - -/// Manages a pool of connections to peers to avoid constantly reconnecting, -/// which can be expensive. -struct ChannelPool { - context: Arc, - // Size is limited by known authorities in the committee. - channels: RwLock>, -} - -impl ChannelPool { - fn new(context: Arc) -> Self { - Self { - context, - channels: RwLock::new(BTreeMap::new()), - } - } - - async fn get_channel( - &self, - network_keypair: NetworkKeyPair, - peer: AuthorityIndex, - timeout: Duration, - ) -> ConsensusResult { - { - let channels = self.channels.read(); - if let Some(channel) = channels.get(&peer) { - return Ok(channel.clone()); - } - } - - let authority = self.context.committee.authority(peer); - let address = to_host_port_str(&authority.address).map_err(|e| { - ConsensusError::NetworkConfig(format!("Cannot convert address to host:port: {e:?}")) - })?; - let address = format!("https://{address}"); - let config = &self.context.parameters.tonic; - let buffer_size = config.connection_buffer_size; - let client_tls_config = iota_tls::create_rustls_client_config( - self.context - .committee - .authority(peer) - .network_key - .clone() - .into_inner(), - certificate_server_name(&self.context), - Some(network_keypair.private_key().into_inner()), - ); - let endpoint = tonic_rustls::Channel::from_shared(address.clone()) - .unwrap() - .connect_timeout(timeout) - .initial_connection_window_size(Some(buffer_size as u32)) - .initial_stream_window_size(Some(buffer_size as u32 / 2)) - .keep_alive_while_idle(true) - .keep_alive_timeout(config.keepalive_interval) - .http2_keep_alive_interval(config.keepalive_interval) - // tcp keepalive is probably unnecessary and is unsupported by msim. - .user_agent("mysticeti") - .unwrap() - .tls_config(client_tls_config) - .unwrap(); - - let deadline = tokio::time::Instant::now() + timeout; - let channel = loop { - trace!("Connecting to endpoint at {address}"); - match endpoint.connect().await { - Ok(channel) => break channel, - Err(e) => { - debug!("Failed to connect to endpoint at {address}: {e:?}"); - if tokio::time::Instant::now() >= deadline { - return Err(ConsensusError::NetworkClientConnection(format!( - "Timed out connecting to endpoint at {address}: {e:?}" - ))); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - }; - trace!("Connected to {address}"); - - let channel = tower::ServiceBuilder::new() - .layer(CallbackLayer::new(MetricsCallbackMaker::new( - self.context.metrics.network_metrics.outbound.clone(), - self.context.parameters.tonic.excessive_message_size, - ))) - .layer( - TraceLayer::new_for_grpc() - .make_span_with(DefaultMakeSpan::new().level(tracing::Level::TRACE)) - .on_failure(DefaultOnFailure::new().level(tracing::Level::DEBUG)), - ) - .service(channel); - - let mut channels = self.channels.write(); - // There should not be many concurrent attempts at connecting to the same peer. - let channel = channels.entry(peer).or_insert(channel); - Ok(channel.clone()) - } -} - -/// Proxies Tonic requests to NetworkService with actual handler implementation. -struct TonicServiceProxy { - context: Arc, - service: Arc, -} - -impl TonicServiceProxy { - fn new(context: Arc, service: Arc) -> Self { - Self { context, service } - } -} - -#[async_trait] -impl ConsensusService for TonicServiceProxy { - async fn send_block( - &self, - request: Request, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let block = request.into_inner().block; - let block = ExtendedSerializedBlock { - block, - excluded_ancestors: vec![], - }; - self.service - .handle_send_block(peer_index, block) - .await - .map_err(|e| tonic::Status::invalid_argument(format!("{e:?}")))?; - Ok(Response::new(SendBlockResponse {})) - } - - type SubscribeBlocksStream = - Pin> + Send>>; - - async fn subscribe_blocks( - &self, - request: Request>, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let mut request_stream = request.into_inner(); - let first_request = match request_stream.next().await { - Some(Ok(r)) => r, - Some(Err(e)) => { - debug!( - "subscribe_blocks() request from {} failed: {e:?}", - peer_index - ); - return Err(tonic::Status::invalid_argument("Request error")); - } - None => { - return Err(tonic::Status::invalid_argument("Missing request")); - } - }; - let stream = self - .service - .handle_subscribe_blocks(peer_index, first_request.last_received_round) - .await - .map_err(|e| tonic::Status::internal(format!("{e:?}")))? - .map(|block| { - Ok(SubscribeBlocksResponse { - block: block.block, - excluded_ancestors: block.excluded_ancestors, - }) - }); - let rate_limited_stream = - tokio_stream::StreamExt::throttle(stream, self.context.parameters.min_round_delay / 2) - .boxed(); - Ok(Response::new(rate_limited_stream)) - } - - type FetchBlocksStream = Iter>>; - - async fn fetch_blocks( - &self, - request: Request, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let inner = request.into_inner(); - let block_refs = inner - .block_refs - .into_iter() - .filter_map(|serialized| match bcs::from_bytes(&serialized) { - Ok(r) => Some(r), - Err(e) => { - debug!("Failed to deserialize block ref {:?}: {e:?}", serialized); - None - } - }) - .collect(); - let highest_accepted_rounds = inner.highest_accepted_rounds; - let blocks = self - .service - .handle_fetch_blocks(peer_index, block_refs, highest_accepted_rounds) - .await - .map_err(|e| tonic::Status::internal(format!("{e:?}")))?; - let responses: std::vec::IntoIter> = - chunk_blocks(blocks, MAX_FETCH_RESPONSE_BYTES) - .into_iter() - .map(|blocks| Ok(FetchBlocksResponse { blocks })) - .collect::>() - .into_iter(); - let stream = iter(responses); - Ok(Response::new(stream)) - } - - async fn fetch_commits( - &self, - request: Request, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let request = request.into_inner(); - let (commits, certifier_blocks) = self - .service - .handle_fetch_commits(peer_index, (request.start..=request.end).into()) - .await - .map_err(|e| tonic::Status::internal(format!("{e:?}")))?; - let commits = commits - .into_iter() - .map(|c| c.serialized().clone()) - .collect(); - let certifier_blocks = certifier_blocks - .into_iter() - .map(|b| b.serialized().clone()) - .collect(); - Ok(Response::new(FetchCommitsResponse { - commits, - certifier_blocks, - })) - } - - type FetchLatestBlocksStream = - Iter>>; - - async fn fetch_latest_blocks( - &self, - request: Request, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let inner = request.into_inner(); - - // Convert the authority indexes and validate them - let mut authorities = vec![]; - for authority in inner.authorities.into_iter() { - let Some(authority) = self - .context - .committee - .to_authority_index(authority as usize) - else { - return Err(tonic::Status::internal(format!( - "Invalid authority index provided {authority}" - ))); - }; - authorities.push(authority); - } - - let blocks = self - .service - .handle_fetch_latest_blocks(peer_index, authorities) - .await - .map_err(|e| tonic::Status::internal(format!("{e:?}")))?; - let responses: std::vec::IntoIter> = - chunk_blocks(blocks, MAX_FETCH_RESPONSE_BYTES) - .into_iter() - .map(|blocks| Ok(FetchLatestBlocksResponse { blocks })) - .collect::>() - .into_iter(); - let stream = iter(responses); - Ok(Response::new(stream)) - } - - async fn get_latest_rounds( - &self, - request: Request, - ) -> Result, tonic::Status> { - let Some(peer_index) = request - .extensions() - .get::() - .map(|p| p.authority_index) - else { - return Err(tonic::Status::internal("PeerInfo not found")); - }; - let (highest_received, highest_accepted) = self - .service - .handle_get_latest_rounds(peer_index) - .await - .map_err(|e| tonic::Status::internal(format!("{e:?}")))?; - Ok(Response::new(GetLatestRoundsResponse { - highest_received, - highest_accepted, - })) - } -} - -/// Manages the lifecycle of Tonic network client and service. Typical usage -/// during initialization: -/// 1. Create a new `TonicManager`. -/// 2. Take `TonicClient` from `TonicManager::client()`. -/// 3. Create consensus components. -/// 4. Create `TonicService` for consensus service handler. -/// 5. Install `TonicService` to `TonicManager` with -/// `TonicManager::install_service()`. -pub(crate) struct TonicManager { - context: Arc, - network_keypair: NetworkKeyPair, - client: Arc, - server: Option, -} - -impl TonicManager { - pub(crate) fn new(context: Arc, network_keypair: NetworkKeyPair) -> Self { - Self { - context: context.clone(), - network_keypair: network_keypair.clone(), - client: Arc::new(TonicClient::new(context, network_keypair)), - server: None, - } - } -} - -impl NetworkManager for TonicManager { - type Client = TonicClient; - - fn new(context: Arc, network_keypair: NetworkKeyPair) -> Self { - TonicManager::new(context, network_keypair) - } - - fn client(&self) -> Arc { - self.client.clone() - } - - async fn install_service(&mut self, service: Arc) { - self.context - .metrics - .network_metrics - .network_type - .with_label_values(&["tonic"]) - .set(1); - - info!("Starting tonic service"); - - let authority = self.context.committee.authority(self.context.own_index); - // By default, bind to the unspecified address to allow the actual address to be - // assigned. But bind to localhost if it is requested. - let own_address = if authority.address.is_localhost_ip() { - authority.address.clone() - } else { - authority.address.with_zero_ip() - }; - let own_address = to_socket_addr(&own_address).unwrap(); - let service = TonicServiceProxy::new(self.context.clone(), service); - let config = &self.context.parameters.tonic; - - let connections_info = Arc::new(ConnectionsInfo::new(self.context.clone())); - let layers = tower::ServiceBuilder::new() - // Add a layer to extract a peer's PeerInfo from their TLS certs - .map_request(move |mut request: http::Request<_>| { - if let Some(peer_certificates) = - request.extensions().get::() - { - if let Some(peer_info) = - peer_info_from_certs(&connections_info, peer_certificates) - { - request.extensions_mut().insert(peer_info); - } - } - request - }) - .layer(CallbackLayer::new(MetricsCallbackMaker::new( - self.context.metrics.network_metrics.inbound.clone(), - self.context.parameters.tonic.excessive_message_size, - ))) - .layer( - TraceLayer::new_for_grpc() - .make_span_with(DefaultMakeSpan::new().level(tracing::Level::TRACE)) - .on_failure(DefaultOnFailure::new().level(tracing::Level::DEBUG)), - ) - .layer_fn(|service| iota_network_stack::grpc_timeout::GrpcTimeout::new(service, None)); - - let mut consensus_service_server = ConsensusServiceServer::new(service) - .max_encoding_message_size(config.message_size_limit) - .max_decoding_message_size(config.message_size_limit); - - if self.context.protocol_config.consensus_zstd_compression() { - consensus_service_server = consensus_service_server - .send_compressed(CompressionEncoding::Zstd) - .accept_compressed(CompressionEncoding::Zstd); - } - - let consensus_service = tonic::service::Routes::new(consensus_service_server) - .into_axum_router() - .route_layer(layers); - - let tls_server_config = iota_tls::create_rustls_server_config_with_client_verifier( - self.network_keypair.clone().private_key().into_inner(), - certificate_server_name(&self.context), - AllowPublicKeys::new( - self.context - .committee - .authorities() - .map(|(_i, a)| a.network_key.clone().into_inner()) - .collect(), - ), - ); - - // Calculate some metrics around send/recv buffer sizes for the current - // machine/OS - #[cfg(not(msim))] - { - let tcp_connection_metrics = - &self.context.metrics.network_metrics.tcp_connection_metrics; - - // Try creating an ephemeral port to test the highest allowed send and recv - // buffer sizes. Buffer sizes are not set explicitly on the socket - // used for real traffic, to allow the OS to set appropriate values. - { - let ephemeral_addr = SocketAddr::new(own_address.ip(), 0); - let ephemeral_socket = create_socket(&ephemeral_addr); - tcp_connection_metrics - .socket_send_buffer_size - .set(ephemeral_socket.send_buffer_size().unwrap_or(0) as i64); - tcp_connection_metrics - .socket_recv_buffer_size - .set(ephemeral_socket.recv_buffer_size().unwrap_or(0) as i64); - - if let Err(e) = ephemeral_socket.set_send_buffer_size(32 << 20) { - info!("Failed to set send buffer size: {e:?}"); - } - if let Err(e) = ephemeral_socket.set_recv_buffer_size(32 << 20) { - info!("Failed to set recv buffer size: {e:?}"); - } - if ephemeral_socket.bind(ephemeral_addr).is_ok() { - tcp_connection_metrics - .socket_send_buffer_max_size - .set(ephemeral_socket.send_buffer_size().unwrap_or(0) as i64); - tcp_connection_metrics - .socket_recv_buffer_max_size - .set(ephemeral_socket.recv_buffer_size().unwrap_or(0) as i64); - }; - } - } - - let http_config = iota_http::Config::default() - .tcp_nodelay(true) - .initial_connection_window_size(64 << 20) - .initial_stream_window_size(32 << 20) - .http2_keepalive_interval(Some(config.keepalive_interval)) - .http2_keepalive_timeout(Some(config.keepalive_interval)) - .accept_http1(false); - - // Create server - // - // During simtest crash/restart tests there may be an older instance of - // consensus running that is bound to the TCP port of `own_address` that - // hasn't finished relinquishing control of the port yet. So instead of - // crashing when the address is inuse, we will retry for a short/ - // reasonable period of time before giving up. - let deadline = Instant::now() + Duration::from_secs(20); - let server = loop { - match iota_http::Builder::new() - .config(http_config.clone()) - .tls_config(tls_server_config.clone()) - .serve(own_address, consensus_service.clone()) - { - Ok(server) => break server, - Err(err) => { - warn!("Error starting consensus server: {err:?}"); - if Instant::now() > deadline { - panic!("Failed to start consensus server within required deadline"); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - } - }; - - info!("Server started at: {own_address}"); - self.server = Some(server); - } - - async fn stop(&mut self) { - if let Some(server) = self.server.take() { - server.shutdown().await; - } - - self.context - .metrics - .network_metrics - .network_type - .with_label_values(&["tonic"]) - .set(0); - } -} - -// Ensure that if there is an active network running that it is shutdown when -// the TonicManager is dropped. -impl Drop for TonicManager { - fn drop(&mut self) { - if let Some(server) = self.server.as_ref() { - server.trigger_shutdown(); - } - } -} - -// TODO: improve iota-http to allow for providing a MakeService so that this can -// be done once per connection -fn peer_info_from_certs( - connections_info: &ConnectionsInfo, - peer_certificates: &iota_http::PeerCertificates, -) -> Option { - let certs = peer_certificates.peer_certs(); - - if certs.len() != 1 { - trace!( - "Unexpected number of certificates from TLS stream: {}", - certs.len() - ); - return None; - } - trace!("Received {} certificates", certs.len()); - let public_key = iota_tls::public_key_from_certificate(&certs[0]) - .map_err(|e| { - trace!("Failed to extract public key from certificate: {e:?}"); - e - }) - .ok()?; - let client_public_key = NetworkPublicKey::new(public_key); - let Some(authority_index) = connections_info.authority_index(&client_public_key) else { - error!("Failed to find the authority with public key {client_public_key:?}"); - return None; - }; - Some(PeerInfo { authority_index }) -} - -/// Attempts to convert a multiaddr of the form `/[ip4,ip6,dns]/{}/udp/{port}` -/// into a host:port string. -fn to_host_port_str(addr: &Multiaddr) -> Result { - let mut iter = addr.iter(); - - match (iter.next(), iter.next()) { - (Some(Protocol::Ip4(ipaddr)), Some(Protocol::Udp(port))) => Ok(format!("{ipaddr}:{port}")), - (Some(Protocol::Ip6(ipaddr)), Some(Protocol::Udp(port))) => Ok(format!("{ipaddr}:{port}")), - (Some(Protocol::Dns(hostname)), Some(Protocol::Udp(port))) => { - Ok(format!("{hostname}:{port}")) - } - - _ => Err(format!("unsupported multiaddr: {addr}")), - } -} - -/// Attempts to convert a multiaddr of the form `/[ip4,ip6]/{}/[udp,tcp]/{port}` -/// into a SocketAddr value. -pub fn to_socket_addr(addr: &Multiaddr) -> Result { - let mut iter = addr.iter(); - - match (iter.next(), iter.next()) { - (Some(Protocol::Ip4(ipaddr)), Some(Protocol::Udp(port))) - | (Some(Protocol::Ip4(ipaddr)), Some(Protocol::Tcp(port))) => { - Ok(SocketAddr::V4(SocketAddrV4::new(ipaddr, port))) - } - - (Some(Protocol::Ip6(ipaddr)), Some(Protocol::Udp(port))) - | (Some(Protocol::Ip6(ipaddr)), Some(Protocol::Tcp(port))) => { - Ok(SocketAddr::V6(SocketAddrV6::new(ipaddr, port, 0, 0))) - } - - _ => Err(format!("unsupported multiaddr: {addr}")), - } -} - -#[cfg(not(msim))] -fn create_socket(address: &SocketAddr) -> tokio::net::TcpSocket { - let socket = if address.is_ipv4() { - tokio::net::TcpSocket::new_v4() - } else if address.is_ipv6() { - tokio::net::TcpSocket::new_v6() - } else { - panic!("Invalid own address: {address:?}"); - } - .unwrap_or_else(|e| panic!("Cannot create TCP socket: {e:?}")); - if let Err(e) = socket.set_nodelay(true) { - info!("Failed to set TCP_NODELAY: {e:?}"); - } - if let Err(e) = socket.set_reuseaddr(true) { - info!("Failed to set SO_REUSEADDR: {e:?}"); - } - socket -} - -/// Looks up authority index by authority public key. -/// -/// TODO: Add connection monitoring, and keep track of connected peers. -/// TODO: Maybe merge with connection_monitor.rs -struct ConnectionsInfo { - authority_key_to_index: BTreeMap, -} - -impl ConnectionsInfo { - fn new(context: Arc) -> Self { - let authority_key_to_index = context - .committee - .authorities() - .map(|(index, authority)| (authority.network_key.clone(), index)) - .collect(); - Self { - authority_key_to_index, - } - } - - fn authority_index(&self, key: &NetworkPublicKey) -> Option { - self.authority_key_to_index.get(key).copied() - } -} - -/// Information about the client peer, set per connection. -#[derive(Clone, Debug)] -struct PeerInfo { - authority_index: AuthorityIndex, -} - -// Adapt MetricsCallbackMaker and MetricsResponseCallback to http. - -/// Calculate approximate size of HTTP headers. -/// Note: This is an approximation of uncompressed size. Actual wire size will -/// be smaller due to HTTP/2 HPACK compression. -fn calculate_header_size(headers: &http::HeaderMap) -> usize { - headers - .iter() - .map(|(name, value)| { - // +4 bytes for ": " and "\r\n" separator in HTTP/1.1 format - name.as_str().len() + value.len() + 4 - }) - .sum() -} - -impl SizedRequest for http::request::Parts { - fn size(&self) -> usize { - let header_size = calculate_header_size(&self.headers); - let body_size = self - .headers - .get(http::header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()) - .unwrap_or(0); - header_size + body_size - } - - fn route(&self) -> String { - let path = self.uri.path(); - path.rsplit_once('/') - .map(|(_, route)| route) - .unwrap_or("unknown") - .to_string() - } -} - -impl SizedResponse for http::response::Parts { - fn size(&self) -> usize { - // Return header size only. Body size is tracked separately via - // ResponseHandler::on_body_chunk callback to support streaming responses. - calculate_header_size(&self.headers) - } - - fn error_type(&self) -> Option { - if self.status.is_success() { - None - } else { - Some(self.status.to_string()) - } - } -} - -impl MakeCallbackHandler for MetricsCallbackMaker { - type Handler = MetricsResponseCallback; - - fn make_handler(&self, request: &http::request::Parts) -> Self::Handler { - self.handle_request(request) - } -} - -impl ResponseHandler for MetricsResponseCallback { - fn on_response(&mut self, response: &http::response::Parts) { - MetricsResponseCallback::on_response(self, response, &response.headers) - } - - fn on_error(&mut self, err: &E) { - MetricsResponseCallback::on_error(self, err) - } - - fn on_body_chunk(&mut self, chunk: &B) - where - B: bytes::Buf, - { - let chunk_size = chunk.chunk().len(); - self.on_chunk(chunk_size); - } -} - -/// Network message types. -#[derive(Clone, prost::Message)] -pub(crate) struct SendBlockRequest { - // Serialized SignedBlock. - #[prost(bytes = "bytes", tag = "1")] - block: Bytes, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct SendBlockResponse {} - -#[derive(Clone, prost::Message)] -pub(crate) struct SubscribeBlocksRequest { - #[prost(uint32, tag = "1")] - last_received_round: Round, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct SubscribeBlocksResponse { - #[prost(bytes = "bytes", tag = "1")] - block: Bytes, - // Serialized BlockRefs that are excluded from the blocks ancestors. - #[prost(bytes = "vec", repeated, tag = "2")] - excluded_ancestors: Vec>, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchBlocksRequest { - #[prost(bytes = "vec", repeated, tag = "1")] - block_refs: Vec>, - // The highest accepted round per authority. The vector represents the round for each authority - // and its length should be the same as the committee size. - #[prost(uint32, repeated, tag = "2")] - highest_accepted_rounds: Vec, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchBlocksResponse { - // The response of the requested blocks as Serialized SignedBlock. - #[prost(bytes = "bytes", repeated, tag = "1")] - blocks: Vec, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchCommitsRequest { - #[prost(uint32, tag = "1")] - start: CommitIndex, - #[prost(uint32, tag = "2")] - end: CommitIndex, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchCommitsResponse { - // Serialized consecutive Commit. - #[prost(bytes = "bytes", repeated, tag = "1")] - commits: Vec, - // Serialized SignedBlock that certify the last commit from above. - #[prost(bytes = "bytes", repeated, tag = "2")] - certifier_blocks: Vec, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchLatestBlocksRequest { - #[prost(uint32, repeated, tag = "1")] - authorities: Vec, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct FetchLatestBlocksResponse { - // The response of the requested blocks as Serialized SignedBlock. - #[prost(bytes = "bytes", repeated, tag = "1")] - blocks: Vec, -} - -#[derive(Clone, prost::Message)] -pub(crate) struct GetLatestRoundsRequest {} - -#[derive(Clone, prost::Message)] -pub(crate) struct GetLatestRoundsResponse { - // Highest received round per authority. - #[prost(uint32, repeated, tag = "1")] - highest_received: Vec, - // Highest accepted round per authority. - #[prost(uint32, repeated, tag = "2")] - highest_accepted: Vec, -} - -fn chunk_blocks(blocks: Vec, chunk_limit: usize) -> Vec> { - let mut chunks = vec![]; - let mut chunk = vec![]; - let mut chunk_size = 0; - for block in blocks { - let block_size = block.len(); - if !chunk.is_empty() && chunk_size + block_size > chunk_limit { - chunks.push(chunk); - chunk = vec![]; - chunk_size = 0; - } - chunk.push(block); - chunk_size += block_size; - } - if !chunk.is_empty() { - chunks.push(chunk); - } - chunks -} diff --git a/consensus/core/src/network/tonic_tls.rs b/consensus/core/src/network/tonic_tls.rs deleted file mode 100644 index f6cabb83076..00000000000 --- a/consensus/core/src/network/tonic_tls.rs +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use crate::context::Context; - -pub(crate) fn certificate_server_name(context: &Context) -> String { - format!("consensus_epoch_{}", context.committee.epoch()) -} diff --git a/consensus/core/src/round_prober.rs b/consensus/core/src/round_prober.rs deleted file mode 100644 index a70a7b101fd..00000000000 --- a/consensus/core/src/round_prober.rs +++ /dev/null @@ -1,718 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -//! RoundProber periodically checks each peer for the latest rounds they -//! received and accepted from others. This provides insight into how -//! effectively each authority's blocks are propagated and accepted across the -//! network. -//! -//! Unlike inferring accepted rounds from the DAG of each block, RoundProber has -//! the benefit that it remains active even when peers are not proposing. This -//! makes it essential for determining when to disable optimizations that -//! improve DAG quality but may compromise liveness. -//! -//! RoundProber's data sources include the `highest_received_rounds` & -//! `highest_accepted_rounds` tracked by the CoreThreadDispatcher and DagState. -//! The received rounds are updated after blocks are verified but before -//! checking for dependencies. This should make the values more indicative of -//! how well authorities propagate blocks, and less influenced by the quality of -//! ancestors in the proposed blocks. The accepted rounds are updated after -//! checking for dependencies which should indicate the quality of the proposed -//! blocks including its ancestors. - -use std::{sync::Arc, time::Duration}; - -use consensus_config::{AuthorityIndex, Committee}; -use futures::stream::{FuturesUnordered, StreamExt as _}; -use iota_common::sync::notify_once::NotifyOnce; -use iota_metrics::monitored_scope; -use parking_lot::RwLock; -use tokio::{task::JoinHandle, time::MissedTickBehavior}; - -use crate::{ - BlockAPI as _, Round, context::Context, core_thread::CoreThreadDispatcher, dag_state::DagState, - network::NetworkClient, -}; - -/// A [`QuorumRound`] is a round range [low, high]. It is computed from -/// highest received or accepted rounds of an authority reported by all -/// authorities. -/// The bounds represent: -/// - the highest round lower or equal to rounds from a quorum (low) -/// - the lowest round higher or equal to rounds from a quorum (high) -/// -/// [`QuorumRound`] is useful because: -/// - [low, high] range is BFT, always between the lowest and highest rounds of -/// honest validators, with < validity threshold of malicious stake. -/// - It provides signals about how well blocks from an authority propagates in -/// the network. If low bound for an authority is lower than its last proposed -/// round, the last proposed block has not propagated to a quorum. If a new -/// block is proposed from the authority, it will not get accepted immediately -/// by a quorum. -pub(crate) type QuorumRound = (Round, Round); - -// Handle to control the RoundProber loop and read latest round gaps. -pub(crate) struct RoundProberHandle { - prober_task: JoinHandle<()>, - shutdown_notify: Arc, -} - -impl RoundProberHandle { - pub(crate) async fn stop(self) { - let _ = self.shutdown_notify.notify(); - // Do not abort prober task, which waits for requests to be cancelled. - if let Err(e) = self.prober_task.await { - if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } - } - } -} - -pub(crate) struct RoundProber { - context: Arc, - core_thread_dispatcher: Arc, - dag_state: Arc>, - network_client: Arc, - shutdown_notify: Arc, -} - -impl RoundProber { - pub(crate) fn new( - context: Arc, - core_thread_dispatcher: Arc, - dag_state: Arc>, - network_client: Arc, - ) -> Self { - Self { - context, - core_thread_dispatcher, - dag_state, - network_client, - shutdown_notify: Arc::new(NotifyOnce::new()), - } - } - - pub(crate) fn start(self) -> RoundProberHandle { - let shutdown_notify = self.shutdown_notify.clone(); - let loop_shutdown_notify = shutdown_notify.clone(); - let prober_task = tokio::spawn(async move { - // With 200 validators, this would result in 200 * 4 * 200 / 2 = 80KB of - // additional bandwidth usage per sec. We can consider using - // adaptive intervals, for example 10s by default but reduced to 2s - // when the propagation delay is higher. - let mut interval = tokio::time::interval(Duration::from_millis( - self.context.parameters.round_prober_interval_ms, - )); - interval.set_missed_tick_behavior(MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = interval.tick() => { - self.probe().await; - } - _ = loop_shutdown_notify.wait() => { - break; - } - } - } - }); - RoundProberHandle { - prober_task, - shutdown_notify, - } - } - - // Probes each peer for the latest rounds they received from others. - // Returns the quorum round for each authority, and the propagation delay - // of own blocks. - pub(crate) async fn probe(&self) -> (Vec, Vec, Round) { - let _scope = monitored_scope("RoundProber"); - - let node_metrics = &self.context.metrics.node_metrics; - let request_timeout = - Duration::from_millis(self.context.parameters.round_prober_request_timeout_ms); - let own_index = self.context.own_index; - let mut requests = FuturesUnordered::new(); - - for (peer, _) in self.context.committee.authorities() { - if peer == own_index { - continue; - } - let network_client = self.network_client.clone(); - requests.push(async move { - let result = tokio::time::timeout( - request_timeout, - network_client.get_latest_rounds(peer, request_timeout), - ) - .await; - (peer, result) - }); - } - - let mut highest_received_rounds = - vec![vec![0; self.context.committee.size()]; self.context.committee.size()]; - let mut highest_accepted_rounds = - vec![vec![0; self.context.committee.size()]; self.context.committee.size()]; - - let blocks = self - .dag_state - .read() - .get_last_cached_block_per_authority(Round::MAX); - let local_highest_accepted_rounds = blocks - .into_iter() - .map(|(block, _)| block.round()) - .collect::>(); - let last_proposed_round = local_highest_accepted_rounds[own_index]; - - // For our own index, the highest received & accepted round is our last - // accepted round or our last proposed round. - highest_received_rounds[own_index] = self.core_thread_dispatcher.highest_received_rounds(); - highest_accepted_rounds[own_index] = local_highest_accepted_rounds; - highest_received_rounds[own_index][own_index] = last_proposed_round; - highest_accepted_rounds[own_index][own_index] = last_proposed_round; - - loop { - tokio::select! { - result = requests.next() => { - let Some((peer, result)) = result else { break }; - match result { - Ok(Ok((received, accepted))) => { - if received.len() == self.context.committee.size() - { - highest_received_rounds[peer] = received; - } else { - node_metrics.round_prober_request_errors.with_label_values(&["invalid_received_rounds"]).inc(); - tracing::warn!("Received invalid number of received rounds from peer {}", peer); - } - - if self - .context - .protocol_config - .consensus_round_prober_probe_accepted_rounds() { - if accepted.len() == self.context.committee.size() { - highest_accepted_rounds[peer] = accepted; - } else { - node_metrics.round_prober_request_errors.with_label_values(&["invalid_accepted_rounds"]).inc(); - tracing::warn!("Received invalid number of accepted rounds from peer {}", peer); - } - } - - }, - // When a request fails, the highest received rounds from that authority will be 0 - // for the subsequent computations. - // For propagation delay, this behavior is desirable because the computed delay - // increases as this authority has more difficulty communicating with peers. Logic - // triggered by high delay should usually be triggered with frequent probing failures - // as well. - // For quorum rounds computed for peer, this means the values should be used for - // positive signals (peer A can propagate its blocks well) rather than negative signals - // (peer A cannot propagate its blocks well). It can be difficult to distinguish between - // own probing failures and actual propagation issues. - Ok(Err(err)) => { - node_metrics.round_prober_request_errors.with_label_values(&["failed_fetch"]).inc(); - tracing::debug!("Failed to get latest rounds from peer {}: {:?}", peer, err); - }, - Err(_) => { - node_metrics.round_prober_request_errors.with_label_values(&["timeout"]).inc(); - tracing::debug!("Timeout while getting latest rounds from peer {}", peer); - }, - } - } - _ = self.shutdown_notify.wait() => break, - } - } - - let received_quorum_rounds: Vec<_> = self - .context - .committee - .authorities() - .map(|(peer, _)| { - compute_quorum_round(&self.context.committee, peer, &highest_received_rounds) - }) - .collect(); - for ((low, high), (_, authority)) in received_quorum_rounds - .iter() - .zip(self.context.committee.authorities()) - { - node_metrics - .round_prober_received_quorum_round_gaps - .with_label_values(&[&authority.hostname]) - .set((high - low) as i64); - node_metrics - .round_prober_low_received_quorum_round - .with_label_values(&[&authority.hostname]) - .set(*low as i64); - // The gap can be negative if this validator is lagging behind the network. - node_metrics - .round_prober_current_received_round_gaps - .with_label_values(&[&authority.hostname]) - .set(last_proposed_round as i64 - *low as i64); - } - - let accepted_quorum_rounds: Vec<_> = self - .context - .committee - .authorities() - .map(|(peer, _)| { - compute_quorum_round(&self.context.committee, peer, &highest_accepted_rounds) - }) - .collect(); - for ((low, high), (_, authority)) in accepted_quorum_rounds - .iter() - .zip(self.context.committee.authorities()) - { - node_metrics - .round_prober_accepted_quorum_round_gaps - .with_label_values(&[&authority.hostname]) - .set((high - low) as i64); - node_metrics - .round_prober_low_accepted_quorum_round - .with_label_values(&[&authority.hostname]) - .set(*low as i64); - // The gap can be negative if this validator is lagging behind the network. - node_metrics - .round_prober_current_accepted_round_gaps - .with_label_values(&[&authority.hostname]) - .set(last_proposed_round as i64 - *low as i64); - } - // TODO: consider using own quorum round gap to control proposing in addition to - // propagation delay. For now they seem to be about the same. - - // It is possible more blocks arrive at a quorum of peers before the - // get_latest_rounds requests arrive. - // Using the lower bound to increase sensitivity about block propagation issues - // that can reduce round rate. - // Because of the nature of TCP and block streaming, propagation delay is - // expected to be 0 in most cases, even when the actual latency of - // broadcasting blocks is high. - let propagation_delay = - last_proposed_round.saturating_sub(received_quorum_rounds[own_index].0); - node_metrics - .round_prober_propagation_delays - .observe(propagation_delay as f64); - node_metrics - .round_prober_last_propagation_delay - .set(propagation_delay as i64); - if let Err(e) = self - .core_thread_dispatcher - .set_propagation_delay_and_quorum_rounds( - propagation_delay, - received_quorum_rounds.clone(), - accepted_quorum_rounds.clone(), - ) - { - tracing::warn!( - "Failed to set propagation delay and quorum rounds {received_quorum_rounds:?} on Core: {:?}", - e - ); - } - - ( - received_quorum_rounds, - accepted_quorum_rounds, - propagation_delay, - ) - } -} - -/// For the peer specified with target_index, compute and return its -/// [`QuorumRound`]. -fn compute_quorum_round( - committee: &Committee, - target_index: AuthorityIndex, - highest_received_rounds: &[Vec], -) -> QuorumRound { - let mut rounds_with_stake = highest_received_rounds - .iter() - .zip(committee.authorities()) - .map(|(rounds, (_, authority))| (rounds[target_index], authority.stake)) - .collect::>(); - rounds_with_stake.sort(); - - // Forward iteration and stopping at validity threshold would produce the same - // result currently, with fault tolerance of f/3f+1 votes. But it is not - // semantically correct, and will provide an incorrect value when fault - // tolerance and validity threshold are different. - let mut total_stake = 0; - let mut low = 0; - for (round, stake) in rounds_with_stake.iter().rev() { - total_stake += stake; - if total_stake >= committee.quorum_threshold() { - low = *round; - break; - } - } - - let mut total_stake = 0; - let mut high = 0; - for (round, stake) in rounds_with_stake.iter() { - total_stake += stake; - if total_stake >= committee.quorum_threshold() { - high = *round; - break; - } - } - - (low, high) -} - -#[cfg(test)] -mod test { - use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - time::Duration, - }; - - use async_trait::async_trait; - use bytes::Bytes; - use consensus_config::AuthorityIndex; - use parking_lot::{Mutex, RwLock}; - - use super::QuorumRound; - use crate::{ - Round, TestBlock, VerifiedBlock, - block::BlockRef, - commit::{CertifiedCommits, CommitRange}, - context::Context, - core_thread::{CoreError, CoreThreadDispatcher}, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - network::{BlockStream, NetworkClient}, - round_prober::{RoundProber, compute_quorum_round}, - storage::mem_store::MemStore, - }; - - struct FakeThreadDispatcher { - highest_received_rounds: Vec, - propagation_delay: Mutex, - received_quorum_rounds: Mutex>, - accepted_quorum_rounds: Mutex>, - } - - impl FakeThreadDispatcher { - fn new(highest_received_rounds: Vec) -> Self { - Self { - highest_received_rounds, - propagation_delay: Mutex::new(0), - received_quorum_rounds: Mutex::new(Vec::new()), - accepted_quorum_rounds: Mutex::new(Vec::new()), - } - } - - fn propagation_delay(&self) -> Round { - *self.propagation_delay.lock() - } - - fn received_quorum_rounds(&self) -> Vec { - self.received_quorum_rounds.lock().clone() - } - - fn accepted_quorum_rounds(&self) -> Vec { - self.accepted_quorum_rounds.lock().clone() - } - } - - #[async_trait] - impl CoreThreadDispatcher for FakeThreadDispatcher { - async fn add_blocks( - &self, - _blocks: Vec, - ) -> Result, CoreError> { - unimplemented!() - } - - async fn add_certified_commits( - &self, - _commits: CertifiedCommits, - ) -> Result, CoreError> { - unimplemented!() - } - - async fn check_block_refs( - &self, - _block_refs: Vec, - ) -> Result, CoreError> { - unimplemented!() - } - - async fn new_block(&self, _round: Round, _force: bool) -> Result<(), CoreError> { - unimplemented!() - } - - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - unimplemented!() - } - - fn set_quorum_subscribers_exists(&self, _exists: bool) -> Result<(), CoreError> { - unimplemented!() - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - delay: Round, - received_quorum_rounds: Vec, - accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - let mut received_quorum_round_per_authority = self.received_quorum_rounds.lock(); - *received_quorum_round_per_authority = received_quorum_rounds; - let mut accepted_quorum_round_per_authority = self.accepted_quorum_rounds.lock(); - *accepted_quorum_round_per_authority = accepted_quorum_rounds; - let mut propagation_delay = self.propagation_delay.lock(); - *propagation_delay = delay; - Ok(()) - } - - fn set_last_known_proposed_round(&self, _round: Round) -> Result<(), CoreError> { - unimplemented!() - } - - fn highest_received_rounds(&self) -> Vec { - self.highest_received_rounds.clone() - } - } - - struct FakeNetworkClient { - highest_received_rounds: Vec>, - highest_accepted_rounds: Vec>, - } - - impl FakeNetworkClient { - fn new( - highest_received_rounds: Vec>, - highest_accepted_rounds: Vec>, - ) -> Self { - Self { - highest_received_rounds, - highest_accepted_rounds, - } - } - } - - #[async_trait] - #[async_trait::async_trait] - impl NetworkClient for FakeNetworkClient { - const SUPPORT_STREAMING: bool = true; - - async fn send_block( - &self, - _peer: AuthorityIndex, - _serialized_block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - unimplemented!("Unimplemented") - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - unimplemented!("Unimplemented") - } - - async fn fetch_blocks( - &self, - _peer: AuthorityIndex, - _block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn get_latest_rounds( - &self, - peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - let received_rounds = self.highest_received_rounds[peer].clone(); - let accepted_rounds = self.highest_accepted_rounds[peer].clone(); - if received_rounds.is_empty() && accepted_rounds.is_empty() { - Err(ConsensusError::NetworkRequestTimeout("test".to_string())) - } else { - Ok((received_rounds, accepted_rounds)) - } - } - } - - #[tokio::test] - async fn test_round_prober() { - const NUM_AUTHORITIES: usize = 7; - let context = Arc::new(Context::new_for_test(NUM_AUTHORITIES).0); - let core_thread_dispatcher = Arc::new(FakeThreadDispatcher::new(vec![ - 110, 120, 130, 140, 150, 160, 170, - ])); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - // Have some peers return error or incorrect number of rounds. - let network_client = Arc::new(FakeNetworkClient::new( - vec![ - vec![], - vec![109, 121, 131, 0, 151, 161, 171], - vec![101, 0, 103, 104, 105, 166, 107], - vec![], - vec![100, 102, 133, 0, 155, 106, 177], - vec![105, 115, 103, 0, 125, 126, 127], - vec![10, 20, 30, 40, 50, 60], - ], // highest_received_rounds - vec![ - vec![], - vec![0, 121, 131, 0, 151, 161, 171], - vec![1, 0, 103, 104, 105, 166, 107], - vec![], - vec![0, 102, 133, 0, 155, 106, 177], - vec![1, 115, 103, 0, 125, 126, 127], - vec![1, 20, 30, 40, 50, 60], - ], // highest_accepted_rounds - )); - let prober = RoundProber::new( - context.clone(), - core_thread_dispatcher.clone(), - dag_state.clone(), - network_client.clone(), - ); - - // Create test blocks for each authority with incrementing rounds starting at - // 110 - let blocks = (0..NUM_AUTHORITIES) - .map(|authority| { - let round = 110 + (authority as u32 * 10); - VerifiedBlock::new_for_test(TestBlock::new(round, authority as u32).build()) - }) - .collect::>(); - - dag_state.write().accept_blocks(blocks); - - // Compute quorum rounds and propagation delay based on last proposed round = - // 110, and highest received rounds: - // 110, 120, 130, 140, 150, 160, 170, - // 109, 121, 131, 0, 151, 161, 171, - // 101, 0, 103, 104, 105, 166, 107, - // 0, 0, 0, 0, 0, 0, 0, - // 100, 102, 133, 0, 155, 106, 177, - // 105, 115, 103, 0, 125, 126, 127, - // 0, 0, 0, 0, 0, 0, 0, - - let (received_quorum_rounds, accepted_quorum_rounds, propagation_delay) = - prober.probe().await; - - assert_eq!( - received_quorum_rounds, - vec![ - (100, 105), - (0, 115), - (103, 130), - (0, 0), - (105, 150), - (106, 160), - (107, 170) - ] - ); - - assert_eq!( - core_thread_dispatcher.received_quorum_rounds(), - vec![ - (100, 105), - (0, 115), - (103, 130), - (0, 0), - (105, 150), - (106, 160), - (107, 170) - ] - ); - // 110 - 100 = 10 - assert_eq!(propagation_delay, 10); - assert_eq!(core_thread_dispatcher.propagation_delay(), 10); - - assert_eq!( - accepted_quorum_rounds, - vec![ - (0, 1), - (0, 115), - (103, 130), - (0, 0), - (105, 150), - (106, 160), - (107, 170) - ] - ); - - assert_eq!( - core_thread_dispatcher.accepted_quorum_rounds(), - vec![ - (0, 1), - (0, 115), - (103, 130), - (0, 0), - (105, 150), - (106, 160), - (107, 170) - ] - ); - } - - #[tokio::test] - async fn test_compute_quorum_round() { - let (context, _) = Context::new_for_test(4); - - // Observe latest rounds from peers. - let highest_received_rounds = vec![ - vec![10, 11, 12, 13], - vec![5, 2, 7, 4], - vec![0, 0, 0, 0], - vec![3, 4, 5, 6], - ]; - - let round = compute_quorum_round( - &context.committee, - AuthorityIndex::new_for_test(0), - &highest_received_rounds, - ); - assert_eq!(round, (3, 5)); - - let round = compute_quorum_round( - &context.committee, - AuthorityIndex::new_for_test(1), - &highest_received_rounds, - ); - assert_eq!(round, (2, 4)); - - let round = compute_quorum_round( - &context.committee, - AuthorityIndex::new_for_test(2), - &highest_received_rounds, - ); - assert_eq!(round, (5, 7)); - - let round = compute_quorum_round( - &context.committee, - AuthorityIndex::new_for_test(3), - &highest_received_rounds, - ); - assert_eq!(round, (4, 6)); - } -} diff --git a/consensus/core/src/scoring_metrics_store.rs b/consensus/core/src/scoring_metrics_store.rs deleted file mode 100644 index 14162570d25..00000000000 --- a/consensus/core/src/scoring_metrics_store.rs +++ /dev/null @@ -1,2045 +0,0 @@ -// Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::BTreeSet, - sync::{Arc, atomic::Ordering}, -}; - -use consensus_config::AuthorityIndex; -use iota_common::scoring_metrics::{ScoringMetricsV1, VersionedScoringMetrics}; -use iota_protocol_config::ProtocolConfig; -use itertools::izip; - -use crate::{ - BlockRef, context::Context, error::ConsensusError, metrics::NodeMetrics, - storage::StorageScoringMetrics, -}; -/// Struct that holds the scoring metrics for all authorities in the committee, -/// both cached and uncached. It also holds a shared reference to the current -/// local metrics count used by Scorer. -pub(crate) struct MysticetiScoringMetricsStore { - pub current_local_metrics_count: Arc, - pub cached_metrics: VersionedScoringMetrics, - pub uncached_metrics: VersionedScoringMetrics, -} - -impl MysticetiScoringMetricsStore { - pub(crate) fn new( - committee_size: usize, - current_local_metrics_count: Arc, - protocol_config: &ProtocolConfig, - ) -> Self { - match protocol_config.scorer_version_as_option() { - None | Some(1) => Self { - current_local_metrics_count, - cached_metrics: VersionedScoringMetrics::V1(ScoringMetricsV1::new(committee_size)), - - uncached_metrics: VersionedScoringMetrics::V1(ScoringMetricsV1::new( - committee_size, - )), - }, - _ => panic!("Unsupported scorer version"), - } - } - - // Initializes the scoring metrics store according to the - // recovered_scoring_metrics and blocks_in_cache_by_authority. - pub(crate) fn initialize_scoring_metrics( - &self, - mut recovered_scoring_metrics: Vec<(AuthorityIndex, StorageScoringMetrics)>, - blocks_in_cache_by_authority: &Vec>, - threshold_clock_round: u32, - eviction_rounds: &Vec, - context: Arc, - ) { - let hostnames = context - .committee - .authorities() - .map(|(_, x)| x.hostname.as_str()) - .collect::>(); - - // It is possible that the vector recovered_scoring_metrics does not have a - // component for every authority. A perfectly functioning validator, for - // example, will never have its metrics updated, so no metric will ever be - // stored. For this reason, we manually "fill" this vector. - if recovered_scoring_metrics.len() < context.committee.size() { - for i in 0..context.committee.size() { - if !recovered_scoring_metrics - .iter() - .any(|(index, _)| index.value() == i) - { - // We add a component with zeroed metrics for the authority with index i. - // This will ensure that every authority has its metrics initialized. - // They are initialized as zero because if an authority does not have any - // recovered metrics, it means that it never misbehaved in a way that was - // detected by the node. - recovered_scoring_metrics.insert( - i, - ( - AuthorityIndex::new_for_test(i as u32), - StorageScoringMetrics { - faulty_blocks_provable: 0, - faulty_blocks_unprovable: 0, - equivocations: 0, - missing_proposals: 0, - }, - ), - ); - } - } - } - for ((authority_index, metrics), hostname, blocks_in_cache, &eviction_round) in izip!( - recovered_scoring_metrics, - hostnames, - blocks_in_cache_by_authority, - eviction_rounds - ) { - // Initialize the uncached scoring metrics according to - // recovered_scoring_metrics - let StorageScoringMetrics { - faulty_blocks_provable, - faulty_blocks_unprovable, - equivocations, - missing_proposals, - } = metrics; - self.initialize_faulty_blocks_metrics( - faulty_blocks_provable, - faulty_blocks_unprovable, - hostname, - authority_index, - &context.metrics.node_metrics, - ); - self.update_missing_blocks_and_equivocations( - missing_proposals, - equivocations, - hostname, - authority_index, - StoreType::Uncached, - &context.metrics.node_metrics, - ); - - // Initialize the cached scoring metrics according to blocks_in_cache. - let block_rounds_in_cache = blocks_in_cache - .iter() - .map(|block_ref| block_ref.round) - .collect(); - let (cached_equivocations, missing_blocks_in_cached_rounds) = - calculate_scoring_metrics_for_range( - block_rounds_in_cache, - eviction_round + 1, - threshold_clock_round - 1, - ); - self.update_missing_blocks_and_equivocations( - missing_blocks_in_cached_rounds, - cached_equivocations, - hostname, - authority_index, - StoreType::Cached, - &context.metrics.node_metrics, - ); - } - } - - // Updates the scoring metrics according to the received block's - // authority and error encountered during its processing. - pub(crate) fn update_scoring_metrics_on_block_receival( - &self, - authority_index: AuthorityIndex, - hostname: &str, - error: ConsensusError, - source: ErrorSource, - node_metrics: &NodeMetrics, - ) { - // authority_index will be always a valid index. However, this method will - // panic if authority_index >= committee_size. We run this check only to avoid - // this panic. - if authority_index.value() >= self.cached_metrics.faulty_blocks_provable().len() { - return; - } - - let (metric_type, source_str) = match source { - ErrorSource::CommitSyncer => (classify_commit_syncer_error(&error), "fetch_once"), - ErrorSource::Subscriber => (classify_subscriber_error(&error), "handle_send_block"), - ErrorSource::Synchronizer => ( - classify_synchronizer_error(&error), - "process_fetched_blocks", - ), - }; - match metric_type { - MetricType::Provable => { - self.uncached_metrics - .increment_faulty_blocks_provable(authority_index.value(), 1); - node_metrics - .faulty_blocks_provable_by_authority - .with_label_values(&[hostname, source_str, error.name()]) - .inc(); - } - MetricType::Unprovable => { - self.uncached_metrics - .increment_faulty_blocks_unprovable(authority_index.value(), 1); - node_metrics - .faulty_blocks_unprovable_by_authority - .with_label_values(&[hostname, source_str, error.name()]) - .inc(); - } - MetricType::Untracked => { - // No scoring metrics need to be updated. - } - } - } - - // Auxiliary function to initialize scoring metrics relative to faulty blocks. - // The `authority` parameter should be a valid index, otherwise the function - // will panic. This check is not performed here, as it is assumed that the - // caller has already checked it. - pub(crate) fn initialize_faulty_blocks_metrics( - &self, - faulty_blocks_provable: u64, - faulty_blocks_unprovable: u64, - hostname: &str, - authority_index: AuthorityIndex, - node_metrics: &NodeMetrics, - ) { - node_metrics - .faulty_blocks_provable_by_authority - .with_label_values(&[hostname, "loaded from storage", "loaded from storage"]) - .inc_by(faulty_blocks_provable); - node_metrics - .faulty_blocks_unprovable_by_authority - .with_label_values(&[hostname, "loaded from storage", "loaded from storage"]) - .inc_by(faulty_blocks_unprovable); - self.uncached_metrics - .store_faulty_blocks_provable(authority_index.value(), faulty_blocks_provable); - self.uncached_metrics - .store_faulty_blocks_unprovable(authority_index.value(), faulty_blocks_unprovable); - } - - // Auxiliary function to update scoring metrics relative to missing blocks - // and equivocations. The `authority` parameter should be a valid index, - // otherwise the function will panic. This check is not performed here, as - // it is assumed that the caller has already checked it. - pub(crate) fn update_missing_blocks_and_equivocations( - &self, - missing_blocks: u64, - equivocations: u64, - hostname: &str, - authority: AuthorityIndex, - metric_type: StoreType, - node_metrics: &NodeMetrics, - ) { - match metric_type { - StoreType::Cached => { - self.cached_metrics - .store_equivocations(authority.value(), equivocations); - self.cached_metrics - .store_missing_proposals(authority.value(), missing_blocks); - node_metrics - .equivocations_in_cache_by_authority - .with_label_values(&[hostname]) - .set(equivocations as i64); - node_metrics - .missing_proposals_in_cache_by_authority - .with_label_values(&[hostname]) - .set(missing_blocks as i64); - } - - StoreType::Uncached => { - self.uncached_metrics - .increment_equivocations(authority.value(), equivocations); - self.uncached_metrics - .increment_missing_proposals(authority.value(), missing_blocks); - node_metrics - .uncached_equivocations_by_authority - .with_label_values(&[hostname]) - .inc_by(equivocations); - node_metrics - .uncached_missing_proposals_by_authority - .with_label_values(&[hostname]) - .inc_by(missing_blocks); - } - } - } - - // Updates the authority's scoring metrics according to the recent changes in - // the DAG state, i.e., recent evictions and additions to cache. It also - // updates the current local metrics count used by Scorer. It returns metrics - // changes that should be updated in disk storage. - pub(crate) fn update_scoring_metrics_on_eviction( - &self, - authority_index: AuthorityIndex, - hostname: &str, - recent_refs: &BTreeSet, - eviction_round: u32, - last_eviction_round: u32, - threshold_clock_round: u32, - node_metrics: &NodeMetrics, - ) -> Option { - // threshold_clock_round should be always at least 1. - // Analogously, authority_index should be a valid index. - if threshold_clock_round == 0 - || authority_index.value() >= self.uncached_metrics.faulty_blocks_provable().len() - { - return None; - } - - // Get the blocks rounds that were not evicted. - let cached_block_rounds = recent_refs - .iter() - .map(|block| block.round) - .filter(|&round| round > eviction_round && round < threshold_clock_round) - .collect::>(); - - // Update metrics according to the blocks from rounds still in cache. - let (cached_equivocations, missing_blocks_in_cached_rounds) = - calculate_scoring_metrics_for_range( - cached_block_rounds, - eviction_round + 1, - threshold_clock_round.saturating_sub(1), - ); - - self.update_missing_blocks_and_equivocations( - missing_blocks_in_cached_rounds, - cached_equivocations, - hostname, - authority_index, - StoreType::Cached, - node_metrics, - ); - - // If no eviction happened, we do not update the metrics on storage. - if eviction_round == last_eviction_round { - return None; - } - - // Get the evicted blocks rounds. - let evicted_block_rounds = recent_refs - .iter() - .map(|block| block.round) - .filter(|&round| round <= eviction_round) - .collect::>(); - - // Update metrics according to the blocks from evicted rounds. - let (evicted_equivocations, missing_blocks_in_evicted_rounds) = - calculate_scoring_metrics_for_range( - evicted_block_rounds, - last_eviction_round + 1, - eviction_round, - ); - - self.update_missing_blocks_and_equivocations( - missing_blocks_in_evicted_rounds, - evicted_equivocations, - hostname, - authority_index, - StoreType::Uncached, - node_metrics, - ); - - // Update current local metrics count. - self.update_current_local_metrics_count(authority_index); - - Some(StorageScoringMetrics { - faulty_blocks_provable: self.uncached_metrics.faulty_blocks_provable()[authority_index] - .load(Ordering::Relaxed), - faulty_blocks_unprovable: self.uncached_metrics.faulty_blocks_unprovable() - [authority_index] - .load(Ordering::Relaxed), - equivocations: self.uncached_metrics.equivocations()[authority_index] - .load(Ordering::Relaxed), - missing_proposals: self.uncached_metrics.missing_proposals()[authority_index] - .load(Ordering::Relaxed), - }) - } - - pub(crate) fn update_current_local_metrics_count(&self, authority_index: AuthorityIndex) { - let faulty_blocks_provable = - self.uncached_metrics.faulty_blocks_provable()[authority_index].load(Ordering::Relaxed); - let faulty_blocks_unprovable = self.uncached_metrics.faulty_blocks_unprovable() - [authority_index] - .load(Ordering::Relaxed); - let equivocations = - self.uncached_metrics.equivocations()[authority_index].load(Ordering::Relaxed); - let missing_proposals = - self.uncached_metrics.missing_proposals()[authority_index].load(Ordering::Relaxed); - self.current_local_metrics_count - .store_faulty_blocks_provable(authority_index.value(), faulty_blocks_provable); - self.current_local_metrics_count - .store_faulty_blocks_unprovable(authority_index.value(), faulty_blocks_unprovable); - self.current_local_metrics_count - .store_equivocations(authority_index.value(), equivocations); - self.current_local_metrics_count - .store_missing_proposals(authority_index.value(), missing_proposals); - } -} - -// Given the set of blocks issued by an authority in rounds in the inclusive -// range [start, end], this function calculates and returns the number of -// equivocations and missing blocks in that range . The function should receive -// the vector with the rounds of such blocks and the range start and end points. -fn calculate_scoring_metrics_for_range( - mut block_rounds: Vec, - start: u32, - end: u32, -) -> (u64, u64) { - // Filter out rounds that are not in the range [start, end]. - block_rounds.retain(|&round| round >= start && round <= end); - let number_of_blocks = block_rounds.len(); - block_rounds.dedup(); - let unique_block_rounds = block_rounds.len(); - // We use saturating_sub to avoid unexpected underflows, but the subtractions - // below should never result in negative values by construction: - // 1) unique_block_rounds <= number_of_blocks - // 2) end - start + 1 >= unique_block_rounds - let number_of_equivocations = number_of_blocks.saturating_sub(unique_block_rounds) as u64; - let number_of_missing_blocks = - (end + 1).saturating_sub(start + unique_block_rounds as u32) as u64; - - (number_of_equivocations, number_of_missing_blocks) -} - -pub(crate) enum StoreType { - Cached, - Uncached, -} - -#[derive(PartialEq)] -// Enum to classify errors into provable, unprovable, or untracked metrics. -// Provable metrics are those that can be proven to a third party by providing -// some cryptographic proof, such the signed block itself. Untracked metrics -// are those that are not of interest for scoring. -pub(crate) enum MetricType { - Provable, - Unprovable, - Untracked, -} - -// Classifies errors returned by the commit syncer as unprovable, and errors not -// returned by it as untracked. We do not classify any error as provable here -// because we cannot prove to a third party that a block or commit was fetched -// from a particular authority. -fn classify_commit_syncer_error(error: &ConsensusError) -> MetricType { - match error { - ConsensusError::MalformedCommit(_) => MetricType::Unprovable, - ConsensusError::UnexpectedStartCommit { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedCommitSequence { .. } => MetricType::Unprovable, - ConsensusError::NoCommitReceived { .. } => MetricType::Unprovable, - ConsensusError::MalformedBlock(_) => MetricType::Unprovable, - ConsensusError::NotEnoughCommitVotes { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedNumberOfBlocksFetched { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedBlockForCommit { .. } => MetricType::Unprovable, - // Overwrite block verifier classification to return unprovable. - error => match classify_block_verifier_error(error) { - MetricType::Provable => MetricType::Unprovable, - metric_type => metric_type, - }, - } -} - -// Classifies errors returned by the block verifier into provable or unprovable, -// and errors not returned by it as untracked. Errors classified as provable are -// those that can be proven to a third party by providing the signed faulty -// block itself -fn classify_block_verifier_error(error: &ConsensusError) -> MetricType { - match error { - ConsensusError::WrongEpoch { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedGenesisBlock => MetricType::Unprovable, - ConsensusError::InvalidAuthorityIndex { .. } => MetricType::Unprovable, - ConsensusError::SerializationFailure(_) => MetricType::Unprovable, - ConsensusError::MalformedSignature(_) => MetricType::Unprovable, - ConsensusError::SignatureVerificationFailure { .. } => MetricType::Unprovable, - // Signed block verification - ConsensusError::TooManyAncestors { .. } => MetricType::Provable, - ConsensusError::InsufficientParentStakes { .. } => MetricType::Provable, - ConsensusError::InvalidAncestorAuthorityIndex { .. } => MetricType::Provable, - ConsensusError::InvalidAncestorPosition { .. } => MetricType::Provable, - ConsensusError::InvalidAncestorRound { .. } => MetricType::Provable, - ConsensusError::InvalidGenesisAncestor { .. } => MetricType::Provable, - ConsensusError::DuplicatedAncestorsAuthority { .. } => MetricType::Provable, - ConsensusError::TransactionTooLarge { .. } => MetricType::Provable, - ConsensusError::TooManyTransactions { .. } => MetricType::Provable, - ConsensusError::TooManyTransactionBytes { .. } => MetricType::Provable, - ConsensusError::InvalidTransaction { .. } => MetricType::Provable, - _ => MetricType::Untracked, - } -} - -// Classifies errors returned by the subscriber into provable or unprovable, and -// errors not returned by it as untracked. Errors classified as provable are -// those that can be proven to a third party by providing the signed faulty -// block itself. Obs: BlockRejected errors are untracked because even though -// the rejected block signature can be verified, the reason for the rejection -// is not objective nor clearly the block author's fault. -fn classify_subscriber_error(error: &ConsensusError) -> MetricType { - match error { - ConsensusError::MalformedBlock { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedAuthority(..) => MetricType::Unprovable, - ConsensusError::BlockRejected { .. } => MetricType::Untracked, - error => classify_block_verifier_error(error), - } -} - -// Classifies errors returned by the synchronizer as unprovable, and errors not -// returned by it as untracked. We do not classify any error as provable here -// because we cannot prove to a third party that a block was fetched from a -// particular authority. -fn classify_synchronizer_error(error: &ConsensusError) -> MetricType { - match error { - ConsensusError::TooManyFetchedBlocksReturned { .. } => MetricType::Unprovable, - ConsensusError::MalformedBlock { .. } => MetricType::Unprovable, - ConsensusError::UnexpectedFetchedBlock { .. } => MetricType::Unprovable, - // Overwrite block verifier classification to return unprovable. - error => match classify_block_verifier_error(error) { - MetricType::Provable => MetricType::Unprovable, - metric_type => metric_type, - }, - } -} - -#[derive(Clone)] -pub(crate) enum ErrorSource { - // Errors from the fetch loop, returned from fetch_once. - CommitSyncer, - // Errors from the subscription loop, returned from handle_send_block. - Subscriber, - // Errors returned from process_fetched_blocks. - Synchronizer, -} - -#[cfg(test)] -mod tests { - use std::{collections::BTreeSet, sync::Arc, vec}; - - use consensus_config::{AuthorityIndex, NetworkKeyPair, ProtocolKeyPair}; - use parking_lot::RwLock; - use tokio::sync::broadcast; - - use crate::{ - TransactionVerifier, ValidationError, - authority_service::{ - AuthorityService, - tests::{FakeCoreThreadDispatcher, FakeNetworkClient}, - }, - block::{BlockDigest, BlockRef}, - block_verifier::SignedBlockVerifier, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - dag_state::DagState, - error::ConsensusError, - scoring_metrics_store::{ErrorSource, MysticetiScoringMetricsStore}, - storage::{StorageScoringMetrics, mem_store::MemStore}, - synchronizer::Synchronizer, - test_dag_builder::DagBuilder, - }; - - struct TxnSizeVerifier {} - - impl TransactionVerifier for TxnSizeVerifier { - fn verify_batch(&self, _transactions: &[&[u8]]) -> Result<(), ValidationError> { - unimplemented!("Unimplemented") - } - } - - // Creates a new authority service for scoring metrics testing purposes. - fn new_authority_service_for_metrics_tests( - committee_size: usize, - ) -> ( - Vec<(NetworkKeyPair, ProtocolKeyPair)>, - Arc, - Arc, - Arc>, - ) { - let (context, keys) = Context::new_for_test(committee_size); - let context = Arc::new(context); - let block_verifier = Arc::new(SignedBlockVerifier::new( - context.clone(), - Arc::new(TxnSizeVerifier {}), - )); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(FakeCoreThreadDispatcher::new()); - let (_tx_block_broadcast, rx_block_broadcast) = broadcast::channel(100); - let network_client = Arc::new(FakeNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store.clone()))); - let synchronizer = Synchronizer::start( - network_client, - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier.clone(), - dag_state.clone(), - false, - ); - let authority_service = Arc::new(AuthorityService::new( - context.clone(), - block_verifier, - commit_vote_monitor, - synchronizer, - core_dispatcher.clone(), - rx_block_broadcast, - dag_state, - store, - )); - (keys, context, core_dispatcher, authority_service) - } - - impl MysticetiScoringMetricsStore { - pub(crate) fn uncached_missing_proposals_by_authority(&self) -> Vec { - self.uncached_metrics.load_missing_proposals() - } - - pub(crate) fn equivocations_in_cache_by_authority(&self) -> Vec { - self.cached_metrics.load_equivocations() - } - - pub(crate) fn missing_proposals_in_cache_by_authority(&self) -> Vec { - self.cached_metrics.load_missing_proposals() - } - - pub(crate) fn uncached_equivocations_by_authority(&self) -> Vec { - self.uncached_metrics.load_equivocations() - } - - pub(crate) fn faulty_blocks_provable_by_authority(&self) -> Vec { - self.uncached_metrics.load_faulty_blocks_provable() - } - - pub(crate) fn faulty_blocks_unprovable_by_authority(&self) -> Vec { - self.uncached_metrics.load_faulty_blocks_unprovable() - } - } - - fn get_uncached_missing_proposals(context: &Arc) -> Vec { - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .uncached_missing_proposals_by_authority - .get_metric_with_label_values(&[hostname]) - .unwrap() - .get(), - ) - } - metrics - } - - fn get_missing_proposals_in_cache(context: &Arc) -> Vec { - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .missing_proposals_in_cache_by_authority - .get_metric_with_label_values(&[hostname]) - .unwrap() - .get() - .unsigned_abs(), - ) - } - metrics - } - - fn get_uncached_equivocations(context: &Arc) -> Vec { - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .uncached_equivocations_by_authority - .get_metric_with_label_values(&[hostname]) - .unwrap() - .get(), - ) - } - metrics - } - - fn get_equivocations_in_cache(context: &Arc) -> Vec { - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .equivocations_in_cache_by_authority - .get_metric_with_label_values(&[hostname]) - .unwrap() - .get() - .unsigned_abs(), - ) - } - metrics - } - - fn get_faulty_blocks_provable( - context: &Arc, - source: &ErrorSource, - error: &str, - ) -> Vec { - let source_str = match source { - ErrorSource::CommitSyncer => "fetch_once", - ErrorSource::Subscriber => "handle_send_block", - ErrorSource::Synchronizer => "process_fetched_blocks", - }; - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .faulty_blocks_provable_by_authority - .get_metric_with_label_values(&[hostname, source_str, error]) - .unwrap() - .get(), - ) - } - metrics - } - - fn get_faulty_blocks_unprovable( - context: &Arc, - source: &ErrorSource, - error: &str, - ) -> Vec { - let source_str = match source { - ErrorSource::CommitSyncer => "fetch_once", - ErrorSource::Subscriber => "handle_send_block", - ErrorSource::Synchronizer => "process_fetched_blocks", - }; - let mut metrics = Vec::new(); - for authority in context.committee.authorities() { - let hostname = authority.1.hostname.as_str(); - metrics.push( - context - .metrics - .node_metrics - .faulty_blocks_unprovable_by_authority - .get_metric_with_label_values(&[hostname, source_str, error]) - .unwrap() - .get(), - ) - } - metrics - } - - #[tokio::test] - async fn test_update_scoring_metrics_on_eviction_edge_cases() { - let context = Context::new_for_test(4); - let scoring_metrics_store = context.0.scoring_metrics_store; - let authority_index = AuthorityIndex::new_for_test(0); - let hostname = "test_host"; - let recent_refs_by_authority = BTreeSet::new(); - let node_metrics = &context.0.metrics.node_metrics; - // Test different unexpected combinations of eviction_round, last_evicted_round, - // and threshold_clock_round. Since recent_refs_by_authority is empty, the - // function should never panic or return more than zero equivocations. - // Each of the cases below have a small explanation of why they are unexpected - // and why they are supposed to return what they return. - - // Unexpected because: threshold_clock_round = last_evicted_round means that a - // round with blocks from less than 2f+1 stake was evicted. - // Return: None, because nothing is currently being evicted. - let last_evicted_round = 5; - let eviction_round = 5; - let threshold_clock_round = 5; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(stored_metrics.is_none()); - - // Unexpected because: threshold_clock_round = 0 means that genesis is missing. - // Return: None, because nothing is currently being evicted. - let last_evicted_round = 0; - let eviction_round = 0; - let threshold_clock_round = 0; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(stored_metrics.is_none()); - - // Unexpected because: threshold_clock_round < eviction_round means that a - // round with blocks from less than 2f+1 stake in being evicted. - // Return: 3 missing proposals, from rounds 1 to 3(eviction_round). - let last_evicted_round = 0; - let eviction_round = 3; - let threshold_clock_round = 2; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(matches!( - stored_metrics, - Some(StorageScoringMetrics { - faulty_blocks_provable: 0, - faulty_blocks_unprovable: 0, - equivocations: 0, - missing_proposals: 3 - }) - )); - - // Unexpected because: eviction_round < last_evicted_round means that blocks - // below or in last_evicted_round were accepted. - // Return: metrics won't be updated here, so it should return the same as in the - // last step. - let last_evicted_round = 1; - let eviction_round = 0; - let threshold_clock_round = 2; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(matches!( - stored_metrics, - Some(StorageScoringMetrics { - faulty_blocks_provable: 0, - faulty_blocks_unprovable: 0, - equivocations: 0, - missing_proposals: 3 - }) - )); - - // Unexpected because: threshold_clock_round < eviction_round < - // last_evicted_round and threshold_clock_round. Return: metrics won't - // be updated here, so it should return the same as in the last step. - let last_evicted_round = 2; - let eviction_round = 0; - let threshold_clock_round = 1; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(matches!( - stored_metrics, - Some(StorageScoringMetrics { - faulty_blocks_provable: 0, - faulty_blocks_unprovable: 0, - equivocations: 0, - missing_proposals: 3 - }) - )); - - // Unexpected because: threshold_clock_round < last_evicted_round means that a - // round with blocks from less than 2f+1 stake was evicted. - // Return: None, because nothing is currently being evicted. - let last_evicted_round = 1; - let eviction_round = 2; - let threshold_clock_round = 0; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(stored_metrics.is_none()); - - let last_evicted_round = 2; - let eviction_round = 1; - let threshold_clock_round = 0; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(stored_metrics.is_none()); - - // The function should not panic if the authority index is out of - // bounds. - // Unexpected because: threshold_clock_round = last_evicted_round means that a - // round with blocks from less than 2f+1 stake was evicted. - // Return: None, because nothing is currently being evicted. - let out_of_bounds_authority_index = AuthorityIndex::new_for_test(4); - let last_evicted_round = 1; - let eviction_round = 2; - let threshold_clock_round = 3; - let stored_metrics = scoring_metrics_store.update_scoring_metrics_on_eviction( - out_of_bounds_authority_index, - hostname, - &recent_refs_by_authority, - eviction_round, - last_evicted_round, - threshold_clock_round, - node_metrics, - ); - assert!(stored_metrics.is_none()); - } - - #[tokio::test] - async fn test_metrics_flush_and_recovery_gc_enabled() { - telemetry_subscribers::init_for_testing(); - - const GC_DEPTH: u32 = 3; - const CACHED_ROUNDS: u32 = 4; - - let committee_size = 4; - let (mut context, _) = Context::new_for_test(committee_size); - - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - context - .protocol_config - .set_consensus_linearize_subdag_v2_for_testing(true); - - let context = Arc::new(context); - let hostnames: Vec<&str> = context - .committee - .authorities() - .map(|a| a.1.hostname.as_str()) - .collect(); - let scoring_metrics = &context.scoring_metrics_store; - let node_metrics = &context.metrics.node_metrics; - let store = Arc::new(MemStore::new()); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - // Initialize the DAG builder with 20 layers. Blocks in the DAG will reference - // all blocks from the previous round. - // - Rounds 1 to 5 will have unique blocks from all authorities. - // - Rounds 6 to 8 will have unique blocks from all authorities, except 0, who - // will not propose any block. - // - Rounds 9 to 10 will have unique blocks from all authorities. - // - Rounds 11 to 20 will have unique blocks from all authorities, except: - // - Authority 1, who will produce 1 equivocating blocks at round 11 (i.e., - // 1+1 blocks) - // - Authority 2, who will produce 2 equivocating blocks at round 13 (i.e., - // 1+2 blocks) - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=5).build(); - dag_builder - .layers(6..=8) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - dag_builder.layers(9..=10).build(); - dag_builder - .layers(11..=11) - .authorities(vec![AuthorityIndex::new_for_test(1)]) - .equivocate(1) - .build(); - dag_builder.layers(12..=12).build(); - dag_builder - .layers(13..=13) - .authorities(vec![AuthorityIndex::new_for_test(2)]) - .equivocate(2) - .build(); - dag_builder.layers(14..=20).build(); - - let mut commits = dag_builder - .get_sub_dag_and_commits(1..=20) - .into_iter() - .map(|(_subdag, commit)| commit) - .collect::>(); - - // Add the blocks and commits from first 10 rounds to the dag state. Since - // authority 0 skipped a leader round, we use the 9 first items of the commits - // vector - let mut temp_commits = commits.split_off(9); - dag_state.accept_blocks(dag_builder.blocks(1..=10)); - for commit in commits.clone() { - dag_state.add_commit(commit); - } - - // Checks that metrics are still all zeroed, since even though we accepted - // blocks to the dag state, the metrics updates are done when the dag state is - // flushed. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Flush the dag state - dag_state.flush(); - - // Check that metrics were updated correctly after flushing. - // - // Equivocations: - // - We only accepted blocks from rounds <= 10, thus, no equivocations were - // accepted yet. Equivocations metrics, then, should be still all zeroed. - // - // Missing proposals: - // - The last_commit_round should be 10, so gc_round should be 6. The eviction - // round, then, should be 6 for all authorities. - // - The threshold_clock_round should be 11, since we already accepted all - // blocks from epoch 10. - // - Then, finally, we should have counted: - // - 1 uncached missing proposal for authority 0; - // - 2 missing proposal in cache for authority 0; - // - 0 missing proposals for authorities 1, 2, and 3. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0; committee_size], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0], - ] - ); - - // Clear and check all metrics - scoring_metrics.uncached_metrics.reset(); - scoring_metrics.cached_metrics.reset(); - node_metrics - .uncached_missing_proposals_by_authority - .with_label_values(&[hostnames[0]]) - .reset(); - node_metrics - .missing_proposals_in_cache_by_authority - .with_label_values(&[hostnames[0]]) - .set(0); - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Destroy and recover dag state from storage. - drop(dag_state); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - // Metrics should have been initialized as before the recovery. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0; committee_size], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0] - ] - ); - - // Add blocks and commits from rounds 11 and 12 to the dag state. - let second_temp_commits = temp_commits.split_off(2); - dag_state.accept_blocks(dag_builder.blocks(11..=12)); - for commit in temp_commits.clone() { - dag_state.add_commit(commit); - } - - // Flush the dag state - dag_state.flush(); - - // Check that metrics were updated correctly after flushing. - // - // Missing proposals: - // - The last_commit_round should be 12, so gc_round should be 8. The eviction - // round, then, should be 8 for all authorities. Then, we should have counted: - // - 3 missing proposal in cache for authority 0; - // - 0 missing proposals for authorities 1, 2, and 3. - // Equivocations: - // - We only removed from cache blocks from rounds <= 8, thus, no equivocations - // should be uncached. Then, we should have counted: - // - 0 uncached equivocations; - // - 1 equivocation in cache for authority 1; - // - 0 equivocations in cache for authorities 0, 2 and 3; - // - - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![3, 0, 0, 0], - vec![0, 1, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - vec![3, 0, 0, 0], - vec![0, 1, 0, 0], - vec![0; committee_size], - ] - ); - - // Accept all the rest of blocks and commits. - dag_state.accept_blocks(dag_builder.blocks(13..=20)); - for commit in second_temp_commits.clone() { - dag_state.add_commit(commit); - } - - // Clear and check all metrics - scoring_metrics.uncached_metrics.reset(); - scoring_metrics.cached_metrics.reset(); - node_metrics - .uncached_missing_proposals_by_authority - .with_label_values(&[hostnames[0]]) - .reset(); - node_metrics - .equivocations_in_cache_by_authority - .with_label_values(&[hostnames[1]]) - .set(0); - - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Destroy and recover dag state from storage. - drop(dag_state); - let mut dag_state = DagState::new(context.clone(), store); - - // Since the last accepted blocks were not flushed, the equivocations from - // rounds 13 to 20 should not be accounted for. The metrics should remain - // the same as before this acceptance. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![3, 0, 0, 0], - vec![0, 1, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - vec![3, 0, 0, 0], - vec![0, 1, 0, 0], - vec![0; committee_size], - ] - ); - - // Now we accept those lost blocks again and flush the dag state - dag_state.accept_blocks(dag_builder.blocks(13..=20)); - for commit in second_temp_commits { - dag_state.add_commit(commit); - } - dag_state.flush(); - - // Now all misbehaviors should be accounted for in the uncached metrics. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context), - ], - [ - vec![0, 1, 2, 0], - vec![3, 0, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - vec![0, 1, 2, 0], - vec![3, 0, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - ] - ); - } - - #[tokio::test] - async fn test_metrics_flush_and_recovery() { - telemetry_subscribers::init_for_testing(); - - const GC_DEPTH: u32 = 0; - const CACHED_ROUNDS: u32 = 5; - - let committee_size = 4; - let (mut context, _) = Context::new_for_test(committee_size); - - context.parameters.dag_state_cached_rounds = CACHED_ROUNDS; - context - .protocol_config - .set_consensus_gc_depth_for_testing(GC_DEPTH); - context - .protocol_config - .set_consensus_linearize_subdag_v2_for_testing(false); - context - .protocol_config - .set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing(false); - - let context = Arc::new(context); - let hostnames: Vec<&str> = context - .committee - .authorities() - .map(|a| a.1.hostname.as_str()) - .collect(); - let scoring_metrics = &context.scoring_metrics_store; - let node_metrics = &context.metrics.node_metrics; - - let store = Arc::new(MemStore::new()); - // `cached_rounds` is initialized here as 5. - let mut dag_state = DagState::new(context.clone(), store.clone()); - - // Initialize the DAG builder with 20 layers. Blocks in the DAG will reference - // all blocks from the previous round. - // - Rounds 1 to 5 will have unique blocks from all authorities. - // - Rounds 6 to 8 will have unique blocks from all authorities, except 0, who - // will not propose any block. - // - Rounds 9 to 10 will have unique blocks from all authorities. - // - Rounds 11 to 20 will have unique blocks from all authorities, except: - // - Authority 1, who will produce 1 equivocating blocks at round 11 (i.e., - // 1+1 blocks) - // - Authority 2, who will produce 2 equivocating blocks at round 13 (i.e., - // 1+2 blocks) - let mut dag_builder = DagBuilder::new(context.clone()); - dag_builder.layers(1..=5).build(); - dag_builder - .layers(6..=8) - .authorities(vec![AuthorityIndex::new_for_test(0)]) - .skip_block() - .build(); - dag_builder.layers(9..=10).build(); - dag_builder - .layers(11..=11) - .authorities(vec![AuthorityIndex::new_for_test(1)]) - .equivocate(1) - .build(); - dag_builder.layers(12..=12).build(); - dag_builder - .layers(13..=13) - .authorities(vec![AuthorityIndex::new_for_test(2)]) - .equivocate(2) - .build(); - dag_builder.layers(14..=20).build(); - - let mut commits = dag_builder - .get_sub_dag_and_commits(1..=20) - .into_iter() - .map(|(_subdag, commit)| commit) - .collect::>(); - - // Add the blocks and commits from first 10 rounds to the dag state. Since - // authority 0 skipped a leader round, we use the 9 first items of the commits - // vector - let mut temp_commits = commits.split_off(9); - dag_state.accept_blocks(dag_builder.blocks(1..=10)); - for commit in commits.clone() { - dag_state.add_commit(commit); - } - - // Checks that metrics are still all zeroed, since even though we accepted - // blocks to the dag state, the metrics updates are done when the dag state is - // flushed. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Flush the dag state - dag_state.flush(); - - // Check that metrics were updated correctly after flushing. - // - // Equivocations: - // - We only accepted blocks from rounds <= 10, thus, no equivocations were - // accepted yet. Equivocations metrics, then, should be still all zeroed. - // - // Missing proposals: - // - The last committed round is 10, so the eviction round should be 5 for - // authority 2 (leader of round 10) and 4 for all other authorities. - // - The threshold_clock_round should be 11, since we already accepted all - // blocks from epoch 10. - // - Then, finally, we should have counted: - // - 0 uncached missing proposals for authority 0; - // - 3 missing proposal in cache for authority 0; - // - 0 missing proposals for authorities 1, 2, and 3. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![3, 0, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![3, 0, 0, 0], - ] - ); - - // Clear and check all metrics - scoring_metrics.uncached_metrics.reset(); - scoring_metrics.cached_metrics.reset(); - node_metrics - .uncached_missing_proposals_by_authority - .with_label_values(&[hostnames[0]]) - .reset(); - node_metrics - .missing_proposals_in_cache_by_authority - .with_label_values(&[hostnames[0]]) - .set(0); - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Destroy and recover dag state from storage. - drop(dag_state); - let mut dag_state = DagState::new(context.clone(), store.clone()); - - assert_eq!(dag_state.last_commit_index(), 9); - assert_eq!(dag_state.last_committed_rounds(), [9, 9, 10, 9]); - - // Metrics should have been initialized as before the recovery. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority() - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![3, 0, 0, 0], - ] - ); - - // Add blocks and commits from rounds 11 and 12 to the dag state. - let second_temp_commits = temp_commits.split_off(2); - dag_state.accept_blocks(dag_builder.blocks(11..=12)); - for commit in temp_commits.clone() { - dag_state.add_commit(commit); - } - - // Flush the dag state - dag_state.flush(); - - // Check that metrics were updated correctly after flushing. - // - // Missing proposals: - // - The last commit round is 12, so the eviction round should be 7 for - // authority 0 (leader of round 12) and 6 for all other authorities. Then, we - // should have counted: - // - 2 uncached missing proposals for authority 0; - // - 1 missing proposal in cache for authority 0; - // - 0 missing proposals for authorities 1, 2, and 3. - // - // Equivocations: - // - We only removed from cache blocks from rounds <= 7, thus, no equivocations - // should be uncached. Then, we should have counted: - // - 0 uncached equivocations; - // - 1 equivocation in cache for authority 1; - // - 0 equivocations in cache for authorities 0, 2 and 3; - // - - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0, 1, 0, 0], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0, 1, 0, 0], - vec![1, 0, 0, 0], - ] - ); - - // Accept all the rest of blocks and commits. - dag_state.accept_blocks(dag_builder.blocks(13..=20)); - for commit in second_temp_commits.clone() { - dag_state.add_commit(commit); - } - - // Clear and check all metrics - scoring_metrics.uncached_metrics.reset(); - scoring_metrics.cached_metrics.reset(); - scoring_metrics.cached_metrics.reset(); - node_metrics - .uncached_missing_proposals_by_authority - .with_label_values(&[hostnames[0]]) - .reset(); - node_metrics - .missing_proposals_in_cache_by_authority - .with_label_values(&[hostnames[0]]) - .set(0); - node_metrics - .equivocations_in_cache_by_authority - .with_label_values(&[hostnames[1]]) - .set(0); - - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size], - vec![0; committee_size] - ] - ); - - // Destroy and recover dag state from storage. - drop(dag_state); - let mut dag_state = DagState::new(context.clone(), store); - - // Since the last accepted blocks were not flushed, the equivocations from - // rounds 13 to 20 should not be accounted for. The metrics should remain - // the same as before this acceptance. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0, 1, 0, 0], - vec![1, 0, 0, 0], - vec![0; committee_size], - vec![2, 0, 0, 0], - vec![0, 1, 0, 0], - vec![1, 0, 0, 0], - ] - ); - - // Now we accept those lost blocks again and flush the dag state - dag_state.accept_blocks(dag_builder.blocks(13..=20)); - for commit in second_temp_commits { - dag_state.add_commit(commit); - } - dag_state.flush(); - - // Now all misbehaviors should be accounted for in the uncached metrics. - assert_eq!( - [ - scoring_metrics.uncached_equivocations_by_authority(), - scoring_metrics.uncached_missing_proposals_by_authority(), - scoring_metrics.equivocations_in_cache_by_authority(), - scoring_metrics.missing_proposals_in_cache_by_authority(), - get_uncached_equivocations(&context), - get_uncached_missing_proposals(&context), - get_equivocations_in_cache(&context), - get_missing_proposals_in_cache(&context) - ], - [ - vec![0, 1, 2, 0], - vec![3, 0, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - vec![0, 1, 2, 0], - vec![3, 0, 0, 0], - vec![0; committee_size], - vec![0; committee_size], - ] - ); - } - - #[tokio::test] - async fn test_metrics_handle_send_block() { - // Initialize context and authority service given a committee_size - let committee_size = 4; - let (_, context, _, _) = new_authority_service_for_metrics_tests(committee_size); - let scoring_metrics = &context.scoring_metrics_store; - let source = ErrorSource::Subscriber; - // Create a set of errors to test - let ignored_error = ConsensusError::Shutdown; - let parsing_error = ConsensusError::MalformedBlock(bcs::Error::Eof); - let block_verification_error = ConsensusError::InvalidAuthorityIndex { - index: AuthorityIndex::new_for_test(5), - max: 4, - }; - let block_rejected_error = ConsensusError::BlockRejected { - block_ref: BlockRef::new(10, AuthorityIndex::new_for_test(10), BlockDigest::MIN), - reason: "string".to_string(), - }; - // Update metrics for each authority with an error that should be ignored. - // Metrics should not be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - ignored_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_rejected_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - ] - ); - - // Update metrics for each authority with a parsing error. - // Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - parsing_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_rejected_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0] - ] - ); - - // Update metrics for each authority with a unsigned block verification error. - // Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - block_verification_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_rejected_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![2, 2, 2, 2], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - ] - ); - - // Update metrics for each authority with a block rejected verification error. - // No metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - block_rejected_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_rejected_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![2, 2, 2, 2], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - ] - ); - } - - #[tokio::test] - async fn test_metrics_fetch_once() { - // Initialize context and authority service given a committee_size - let committee_size = 4; - let (_, context, _, _) = new_authority_service_for_metrics_tests(committee_size); - let scoring_metrics = &context.scoring_metrics_store; - let source = ErrorSource::CommitSyncer; - // Create a set of errors to test - let ignored_error = ConsensusError::Shutdown; - let parsing_error = ConsensusError::MalformedBlock(bcs::Error::Eof); - let block_verification_error = ConsensusError::TooManyAncestors(2, 2); - - // Update metrics for each authority with an error that should be ignored. - // Metrics should not be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - ignored_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0] - ] - ); - - // Update metrics for each authority with a parsing error. - // Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - parsing_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0] - ] - ); - - // Update metrics for each authority with a signed block verification error. - // Since for error comes from the commit syncer, blocks received are not - // necessarily from the peer. Thus, it is not provable that the peer actually - // sent this block. Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - block_verification_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![2, 2, 2, 2], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![1, 1, 1, 1], - ] - ); - } - - #[tokio::test] - async fn test_metrics_process_fetched_blocks() { - // Initialize context and authority service given a committee_size - let committee_size = 4; - let (_, context, _, _) = new_authority_service_for_metrics_tests(committee_size); - let scoring_metrics = &context.scoring_metrics_store; - let source = ErrorSource::Synchronizer; - // Create a set of errors to test - let ignored_error = ConsensusError::Shutdown; - let parsing_error = ConsensusError::MalformedBlock(bcs::Error::Eof); - let block_verification_error = ConsensusError::TooManyAncestors(2, 2); - - // Update metrics for each authority with an error that should be ignored. - // Metrics should not be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - ignored_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0] - ] - ); - - // Update metrics for each authority with a parsing error. - // Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - parsing_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![0, 0, 0, 0] - ] - ); - - // Update metrics for each authority with a signed block verification error. - // Since for error comes from the synchronizer, blocks received are not - // necessarily from the peer. Thus, it is not provable that the peer actually - // sent this block. Only unprovable metrics should be updated for this error. - for authority in context.committee.authorities() { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - authority.0, - authority.1.hostname.as_str(), - block_verification_error.clone(), - source.clone(), - &context.metrics.node_metrics, - ); - } - assert_eq!( - [ - scoring_metrics.faulty_blocks_provable_by_authority(), - scoring_metrics.faulty_blocks_unprovable_by_authority(), - get_faulty_blocks_provable(&context, &source, ignored_error.name()), - get_faulty_blocks_provable(&context, &source, parsing_error.name()), - get_faulty_blocks_provable(&context, &source, block_verification_error.name()), - get_faulty_blocks_unprovable(&context, &source, ignored_error.name()), - get_faulty_blocks_unprovable(&context, &source, parsing_error.name()), - get_faulty_blocks_unprovable(&context, &source, block_verification_error.name()) - ], - [ - vec![0, 0, 0, 0], - vec![2, 2, 2, 2], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![0, 0, 0, 0], - vec![1, 1, 1, 1], - vec![1, 1, 1, 1], - ] - ); - } -} diff --git a/consensus/core/src/stake_aggregator.rs b/consensus/core/src/stake_aggregator.rs deleted file mode 100644 index e2d678c7c47..00000000000 --- a/consensus/core/src/stake_aggregator.rs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::BTreeSet, marker::PhantomData}; - -use consensus_config::{AuthorityIndex, Committee, Stake}; - -pub(crate) trait CommitteeThreshold { - fn is_threshold(committee: &Committee, amount: Stake) -> bool; - fn threshold(committee: &Committee) -> Stake; -} - -pub(crate) struct QuorumThreshold; - -#[cfg(test)] -pub(crate) struct ValidityThreshold; - -impl CommitteeThreshold for QuorumThreshold { - fn is_threshold(committee: &Committee, amount: Stake) -> bool { - committee.reached_quorum(amount) - } - fn threshold(committee: &Committee) -> Stake { - committee.quorum_threshold() - } -} - -#[cfg(test)] -impl CommitteeThreshold for ValidityThreshold { - fn is_threshold(committee: &Committee, amount: Stake) -> bool { - committee.reached_validity(amount) - } - fn threshold(committee: &Committee) -> Stake { - committee.validity_threshold() - } -} - -pub(crate) struct StakeAggregator { - votes: BTreeSet, - stake: Stake, - _phantom: PhantomData, -} - -impl StakeAggregator { - pub(crate) fn new() -> Self { - Self { - votes: Default::default(), - stake: 0, - _phantom: Default::default(), - } - } - - /// Adds a vote for the specified authority index to the aggregator. It is - /// guaranteed to count the vote only once for an authority. The method - /// returns true when the required threshold has been reached. - pub(crate) fn add(&mut self, vote: AuthorityIndex, committee: &Committee) -> bool { - if self.votes.insert(vote) { - self.stake += committee.stake(vote); - } - T::is_threshold(committee, self.stake) - } - - pub(crate) fn stake(&self) -> Stake { - self.stake - } - - pub(crate) fn reached_threshold(&self, committee: &Committee) -> bool { - T::is_threshold(committee, self.stake) - } - - pub(crate) fn threshold(&self, committee: &Committee) -> Stake { - T::threshold(committee) - } - - pub(crate) fn clear(&mut self) { - self.votes.clear(); - self.stake = 0; - } -} - -#[cfg(test)] -mod tests { - use consensus_config::{AuthorityIndex, local_committee_and_keys}; - - use super::*; - - #[test] - fn test_aggregator_quorum_threshold() { - let committee = local_committee_and_keys(0, vec![1, 1, 1, 1]).0; - let mut aggregator = StakeAggregator::::new(); - - assert!(!aggregator.add(AuthorityIndex::new_for_test(0), &committee)); - assert!(!aggregator.add(AuthorityIndex::new_for_test(1), &committee)); - assert!(aggregator.add(AuthorityIndex::new_for_test(2), &committee)); - assert!(aggregator.add(AuthorityIndex::new_for_test(3), &committee)); - } - - #[test] - fn test_aggregator_validity_threshold() { - let committee = local_committee_and_keys(0, vec![1, 1, 1, 1]).0; - let mut aggregator = StakeAggregator::::new(); - - assert!(!aggregator.add(AuthorityIndex::new_for_test(0), &committee)); - assert!(aggregator.add(AuthorityIndex::new_for_test(1), &committee)); - } - - #[test] - fn test_aggregator_clear() { - let committee = local_committee_and_keys(0, vec![1, 1, 1, 1]).0; - let mut aggregator = StakeAggregator::::new(); - - assert!(!aggregator.add(AuthorityIndex::new_for_test(0), &committee)); - assert!(aggregator.add(AuthorityIndex::new_for_test(1), &committee)); - - // clear the aggregator - aggregator.clear(); - - // now add them again - assert!(!aggregator.add(AuthorityIndex::new_for_test(0), &committee)); - assert!(aggregator.add(AuthorityIndex::new_for_test(1), &committee)); - } -} diff --git a/consensus/core/src/storage/mem_store.rs b/consensus/core/src/storage/mem_store.rs deleted file mode 100644 index eec3fd1198f..00000000000 --- a/consensus/core/src/storage/mem_store.rs +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, BTreeSet, VecDeque}, - ops::Bound::Included, -}; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; - -use super::{Store, WriteBatch}; -use crate::{ - block::{BlockAPI as _, BlockDigest, BlockRef, Round, Slot, VerifiedBlock}, - commit::{ - CommitAPI as _, CommitDigest, CommitIndex, CommitInfo, CommitRange, CommitRef, - TrustedCommit, - }, - error::ConsensusResult, - storage::StorageScoringMetrics, -}; -/// In-memory storage for testing. -pub(crate) struct MemStore { - inner: RwLock, -} - -struct Inner { - blocks: BTreeMap<(Round, AuthorityIndex, BlockDigest), VerifiedBlock>, - digests_by_authorities: BTreeSet<(AuthorityIndex, Round, BlockDigest)>, - commits: BTreeMap<(CommitIndex, CommitDigest), TrustedCommit>, - commit_votes: BTreeSet<(CommitIndex, CommitDigest, BlockRef)>, - commit_info: BTreeMap<(CommitIndex, CommitDigest), CommitInfo>, - scoring_metrics: BTreeMap, -} - -impl MemStore { - pub(crate) fn new() -> Self { - MemStore { - inner: RwLock::new(Inner { - blocks: BTreeMap::new(), - digests_by_authorities: BTreeSet::new(), - commits: BTreeMap::new(), - commit_votes: BTreeSet::new(), - commit_info: BTreeMap::new(), - scoring_metrics: BTreeMap::new(), - }), - } - } -} - -impl Store for MemStore { - fn write(&self, write_batch: WriteBatch) -> ConsensusResult<()> { - let mut inner = self.inner.write(); - - for block in write_batch.blocks { - let block_ref = block.reference(); - inner.blocks.insert( - (block_ref.round, block_ref.author, block_ref.digest), - block.clone(), - ); - inner.digests_by_authorities.insert(( - block_ref.author, - block_ref.round, - block_ref.digest, - )); - for vote in block.commit_votes() { - inner - .commit_votes - .insert((vote.index, vote.digest, block_ref)); - } - } - - for commit in write_batch.commits { - inner - .commits - .insert((commit.index(), commit.digest()), commit); - } - - for (commit_ref, commit_info) in write_batch.commit_info { - inner - .commit_info - .insert((commit_ref.index, commit_ref.digest), commit_info); - } - - for (authority, metrics) in write_batch.scoring_metrics { - inner.scoring_metrics.insert(authority, metrics); - } - Ok(()) - } - - fn read_blocks(&self, refs: &[BlockRef]) -> ConsensusResult>> { - let inner = self.inner.read(); - let blocks = refs - .iter() - .map(|r| inner.blocks.get(&(r.round, r.author, r.digest)).cloned()) - .collect(); - Ok(blocks) - } - - fn contains_blocks(&self, refs: &[BlockRef]) -> ConsensusResult> { - let inner = self.inner.read(); - let exist = refs - .iter() - .map(|r| inner.blocks.contains_key(&(r.round, r.author, r.digest))) - .collect(); - Ok(exist) - } - - fn scan_blocks_by_author( - &self, - author: AuthorityIndex, - start_round: Round, - ) -> ConsensusResult> { - let inner = self.inner.read(); - let mut refs = vec![]; - for &(author, round, digest) in inner.digests_by_authorities.range(( - Included((author, start_round, BlockDigest::MIN)), - Included((author, Round::MAX, BlockDigest::MAX)), - )) { - refs.push(BlockRef::new(round, author, digest)); - } - let results = self.read_blocks(refs.as_slice())?; - let mut blocks = vec![]; - for (r, block) in refs.into_iter().zip(results) { - if let Some(block) = block { - blocks.push(block); - } else { - panic!("Block {r:?} not found!"); - } - } - Ok(blocks) - } - - fn scan_scoring_metrics( - &self, - ) -> ConsensusResult> { - let inner = self.inner.read(); - let metrics_by_author = inner - .scoring_metrics - .iter() - .map(|(&authority_index, metrics)| (authority_index, metrics.clone())) - .collect::>(); - Ok(metrics_by_author) - } - - fn contains_block_at_slot(&self, slot: Slot) -> ConsensusResult { - let inner = self.inner.read(); - let found = inner - .digests_by_authorities - .range(( - Included((slot.authority, slot.round, BlockDigest::MIN)), - Included((slot.authority, slot.round, BlockDigest::MAX)), - )) - .next() - .is_some(); - Ok(found) - } - - fn scan_last_blocks_by_author( - &self, - author: AuthorityIndex, - num_of_rounds: u64, - before_round: Option, - ) -> ConsensusResult> { - let before_round = before_round.unwrap_or(Round::MAX); - let mut refs = VecDeque::new(); - for &(author, round, digest) in self - .inner - .read() - .digests_by_authorities - .range(( - Included((author, Round::MIN, BlockDigest::MIN)), - Included((author, before_round, BlockDigest::MAX)), - )) - .rev() - .take(num_of_rounds as usize) - { - refs.push_front(BlockRef::new(round, author, digest)); - } - let results = self.read_blocks(refs.as_slices().0)?; - let mut blocks = vec![]; - for (r, block) in refs.into_iter().zip(results) { - blocks.push( - block.unwrap_or_else(|| panic!("Storage inconsistency: block {r:?} not found!")), - ); - } - Ok(blocks) - } - - fn read_last_commit(&self) -> ConsensusResult> { - let inner = self.inner.read(); - Ok(inner - .commits - .last_key_value() - .map(|(_, commit)| commit.clone())) - } - - fn scan_commits(&self, range: CommitRange) -> ConsensusResult> { - let inner = self.inner.read(); - let mut commits = vec![]; - for (_, commit) in inner.commits.range(( - Included((range.start(), CommitDigest::MIN)), - Included((range.end(), CommitDigest::MAX)), - )) { - commits.push(commit.clone()); - } - Ok(commits) - } - - fn read_commit_votes(&self, commit_index: CommitIndex) -> ConsensusResult> { - let inner = self.inner.read(); - let votes = inner - .commit_votes - .range(( - Included((commit_index, CommitDigest::MIN, BlockRef::MIN)), - Included((commit_index, CommitDigest::MAX, BlockRef::MAX)), - )) - .map(|(_, _, block_ref)| *block_ref) - .collect(); - Ok(votes) - } - - fn read_last_commit_info(&self) -> ConsensusResult> { - let inner = self.inner.read(); - Ok(inner - .commit_info - .last_key_value() - .map(|(k, v)| (CommitRef::new(k.0, k.1), v.clone()))) - } -} diff --git a/consensus/core/src/storage/mod.rs b/consensus/core/src/storage/mod.rs deleted file mode 100644 index 15f0c795307..00000000000 --- a/consensus/core/src/storage/mod.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(test)] -pub(crate) mod mem_store; -pub(crate) mod rocksdb_store; - -#[cfg(test)] -mod store_tests; - -use consensus_config::AuthorityIndex; - -use crate::{ - CommitIndex, - block::{BlockRef, Round, VerifiedBlock}, - commit::{CommitInfo, CommitRange, CommitRef, TrustedCommit}, - error::ConsensusResult, -}; - -/// A common interface for consensus storage. -pub(crate) trait Store: Send + Sync { - /// Writes blocks, consensus commits and other data to store atomically. - fn write(&self, write_batch: WriteBatch) -> ConsensusResult<()>; - - /// Reads blocks for the given refs. - fn read_blocks(&self, refs: &[BlockRef]) -> ConsensusResult>>; - - /// Checks if blocks exist in the store. - fn contains_blocks(&self, refs: &[BlockRef]) -> ConsensusResult>; - - /// Checks whether there is any block at the given slot - #[allow(dead_code)] - fn contains_block_at_slot(&self, slot: crate::block::Slot) -> ConsensusResult; - - /// Reads blocks for an authority, from start_round. - fn scan_blocks_by_author( - &self, - authority: AuthorityIndex, - start_round: Round, - ) -> ConsensusResult>; - - // The method reads and returns all metrics stored. Used for restoring the - // scoring metrics in case of DagState initialization from storage - fn scan_scoring_metrics(&self) - -> ConsensusResult>; - - // The method returns the last `num_of_rounds` rounds blocks by author in round - // ascending order. When a `before_round` is defined then the blocks of - // round `<=before_round` are returned. If not then the max value for round - // will be used as cut off. - #[allow(dead_code)] - fn scan_last_blocks_by_author( - &self, - author: AuthorityIndex, - num_of_rounds: u64, - before_round: Option, - ) -> ConsensusResult>; - - /// Reads the last commit. - fn read_last_commit(&self) -> ConsensusResult>; - - /// Reads all commits from start (inclusive) until end (inclusive). - fn scan_commits(&self, range: CommitRange) -> ConsensusResult>; - - /// Reads all blocks voting on a particular commit. - fn read_commit_votes(&self, commit_index: CommitIndex) -> ConsensusResult>; - - /// Reads the last commit info, written atomically with the last commit. - fn read_last_commit_info(&self) -> ConsensusResult>; -} - -/// Represents data to be written to the store together atomically. -#[derive(Debug, Default)] -pub(crate) struct WriteBatch { - pub(crate) blocks: Vec, - pub(crate) commits: Vec, - pub(crate) commit_info: Vec<(CommitRef, CommitInfo)>, - pub(crate) scoring_metrics: Vec<(AuthorityIndex, StorageScoringMetrics)>, -} - -impl WriteBatch { - pub(crate) fn new( - blocks: Vec, - commits: Vec, - commit_info: Vec<(CommitRef, CommitInfo)>, - scoring_metrics: Vec<(AuthorityIndex, StorageScoringMetrics)>, - ) -> Self { - WriteBatch { - blocks, - commits, - commit_info, - scoring_metrics, - } - } - - // Test setters. - - #[cfg(test)] - pub(crate) fn blocks(mut self, blocks: Vec) -> Self { - self.blocks = blocks; - self - } - - #[cfg(test)] - pub(crate) fn commits(mut self, commits: Vec) -> Self { - self.commits = commits; - self - } - - #[cfg(test)] - pub(crate) fn commit_info(mut self, commit_info: Vec<(CommitRef, CommitInfo)>) -> Self { - self.commit_info = commit_info; - self - } - - #[cfg(test)] - pub(crate) fn scoring_metrics( - mut self, - scoring_metrics: Vec<(AuthorityIndex, StorageScoringMetrics)>, - ) -> Self { - self.scoring_metrics = scoring_metrics; - self - } -} - -// This struct is used in storage. It holds the same data as -// `UncachedScoringMetrics`, but uses `u64` instead of `AtomicU64`. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] -pub(crate) struct StorageScoringMetrics { - pub(crate) faulty_blocks_provable: u64, - pub(crate) faulty_blocks_unprovable: u64, - pub(crate) equivocations: u64, - pub(crate) missing_proposals: u64, -} diff --git a/consensus/core/src/storage/rocksdb_store.rs b/consensus/core/src/storage/rocksdb_store.rs deleted file mode 100644 index 5c1b97c30b6..00000000000 --- a/consensus/core/src/storage/rocksdb_store.rs +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ops::Bound::Included, time::Duration}; - -use bytes::Bytes; -use consensus_config::AuthorityIndex; -use iota_macros::fail_point; -use typed_store::{ - Map as _, - metrics::SamplingInterval, - reopen, - rocks::{DBMap, MetricConf, ReadWriteOptions, default_db_options, open_cf_opts}, -}; - -use super::{CommitInfo, Store, WriteBatch}; -use crate::{ - block::{BlockAPI as _, BlockDigest, BlockRef, Round, SignedBlock, VerifiedBlock}, - commit::{CommitAPI as _, CommitDigest, CommitIndex, CommitRange, CommitRef, TrustedCommit}, - error::{ConsensusError, ConsensusResult}, - storage::StorageScoringMetrics, -}; - -/// Persistent storage with RocksDB. -pub(crate) struct RocksDBStore { - /// Stores SignedBlock by refs. - blocks: DBMap<(Round, AuthorityIndex, BlockDigest), Bytes>, - /// A secondary index that orders refs first by authors. - digests_by_authorities: DBMap<(AuthorityIndex, Round, BlockDigest), ()>, - /// Maps commit index to Commit. - commits: DBMap<(CommitIndex, CommitDigest), Bytes>, - /// Collects votes on commits. - /// TODO: batch multiple votes into a single row. - commit_votes: DBMap<(CommitIndex, CommitDigest, BlockRef), ()>, - /// Stores info related to Commit that helps recovery. - commit_info: DBMap<(CommitIndex, CommitDigest), CommitInfo>, - /// Stores scoring metrics for each authority. - scoring_metrics: DBMap, -} - -impl RocksDBStore { - const BLOCKS_CF: &'static str = "blocks"; - const DIGESTS_BY_AUTHORITIES_CF: &'static str = "digests"; - const COMMITS_CF: &'static str = "commits"; - const COMMIT_VOTES_CF: &'static str = "commit_votes"; - const COMMIT_INFO_CF: &'static str = "commit_info"; - const SCORING_METRICS_CF: &'static str = "scoring_metrics"; - - /// Creates a new instance of RocksDB storage. - pub(crate) fn new(path: &str) -> Self { - // Consensus data has high write throughput (all transactions) and is rarely - // read (only during recovery and when helping peers catch up). - let db_options = default_db_options().optimize_db_for_write_throughput(2); - let mut metrics_conf = MetricConf::new("consensus"); - metrics_conf.read_sample_interval = SamplingInterval::new(Duration::from_secs(60), 0); - let cf_options = default_db_options().optimize_for_write_throughput().options; - let column_family_options = vec![ - ( - Self::BLOCKS_CF, - default_db_options() - .optimize_for_write_throughput_no_deletion() - // Using larger block is ok since there is not much point reads on the cf. - .set_block_options(512, 128 << 10) - .options, - ), - (Self::DIGESTS_BY_AUTHORITIES_CF, cf_options.clone()), - (Self::COMMITS_CF, cf_options.clone()), - (Self::COMMIT_VOTES_CF, cf_options.clone()), - (Self::COMMIT_INFO_CF, cf_options.clone()), - (Self::SCORING_METRICS_CF, cf_options), - ]; - let rocksdb = open_cf_opts( - path, - Some(db_options.options), - metrics_conf, - &column_family_options, - ) - .expect("Cannot open database"); - - let (blocks, digests_by_authorities, commits, commit_votes, commit_info, scoring_metrics) = reopen!(&rocksdb, - Self::BLOCKS_CF;<(Round, AuthorityIndex, BlockDigest), bytes::Bytes>, - Self::DIGESTS_BY_AUTHORITIES_CF;<(AuthorityIndex, Round, BlockDigest), ()>, - Self::COMMITS_CF;<(CommitIndex, CommitDigest), Bytes>, - Self::COMMIT_VOTES_CF;<(CommitIndex, CommitDigest, BlockRef), ()>, - Self::COMMIT_INFO_CF;<(CommitIndex, CommitDigest), CommitInfo>, - Self::SCORING_METRICS_CF; - ); - - Self { - blocks, - digests_by_authorities, - commits, - commit_votes, - commit_info, - scoring_metrics, - } - } -} - -impl Store for RocksDBStore { - fn write(&self, write_batch: WriteBatch) -> ConsensusResult<()> { - fail_point!("consensus-store-before-write"); - - let mut batch = self.blocks.batch(); - for block in write_batch.blocks { - let block_ref = block.reference(); - batch - .insert_batch( - &self.blocks, - [( - (block_ref.round, block_ref.author, block_ref.digest), - block.serialized(), - )], - ) - .map_err(ConsensusError::RocksDBFailure)?; - batch - .insert_batch( - &self.digests_by_authorities, - [((block_ref.author, block_ref.round, block_ref.digest), ())], - ) - .map_err(ConsensusError::RocksDBFailure)?; - for vote in block.commit_votes() { - batch - .insert_batch( - &self.commit_votes, - [((vote.index, vote.digest, block_ref), ())], - ) - .map_err(ConsensusError::RocksDBFailure)?; - } - } - - for commit in write_batch.commits { - batch - .insert_batch( - &self.commits, - [((commit.index(), commit.digest()), commit.serialized())], - ) - .map_err(ConsensusError::RocksDBFailure)?; - } - - for (commit_ref, commit_info) in write_batch.commit_info { - batch - .insert_batch( - &self.commit_info, - [((commit_ref.index, commit_ref.digest), commit_info)], - ) - .map_err(ConsensusError::RocksDBFailure)?; - } - for (authority, metrics) in write_batch.scoring_metrics { - batch - .insert_batch(&self.scoring_metrics, [(authority, metrics)]) - .map_err(ConsensusError::RocksDBFailure)?; - } - batch.write()?; - fail_point!("consensus-store-after-write"); - Ok(()) - } - - fn read_blocks(&self, refs: &[BlockRef]) -> ConsensusResult>> { - let keys = refs - .iter() - .map(|r| (r.round, r.author, r.digest)) - .collect::>(); - let serialized = self.blocks.multi_get(keys)?; - let mut blocks = vec![]; - for (key, serialized) in refs.iter().zip(serialized) { - if let Some(serialized) = serialized { - let signed_block: SignedBlock = - bcs::from_bytes(&serialized).map_err(ConsensusError::MalformedBlock)?; - // Only accepted blocks should have been written to storage. - let block = VerifiedBlock::new_verified(signed_block, serialized); - // Makes sure block data is not corrupted, by comparing digests. - assert_eq!(*key, block.reference()); - blocks.push(Some(block)); - } else { - blocks.push(None); - } - } - Ok(blocks) - } - - fn contains_blocks(&self, refs: &[BlockRef]) -> ConsensusResult> { - let refs = refs - .iter() - .map(|r| (r.round, r.author, r.digest)) - .collect::>(); - let exist = self.blocks.multi_contains_keys(refs)?; - Ok(exist) - } - - fn contains_block_at_slot(&self, slot: crate::block::Slot) -> ConsensusResult { - let found = self - .digests_by_authorities - .safe_range_iter(( - Included((slot.authority, slot.round, BlockDigest::MIN)), - Included((slot.authority, slot.round, BlockDigest::MAX)), - )) - .next() - .is_some(); - Ok(found) - } - - fn scan_blocks_by_author( - &self, - author: AuthorityIndex, - start_round: Round, - ) -> ConsensusResult> { - let mut refs = vec![]; - for kv in self.digests_by_authorities.safe_range_iter(( - Included((author, start_round, BlockDigest::MIN)), - Included((author, Round::MAX, BlockDigest::MAX)), - )) { - let ((author, round, digest), _) = kv?; - refs.push(BlockRef::new(round, author, digest)); - } - let results = self.read_blocks(refs.as_slice())?; - let mut blocks = Vec::with_capacity(refs.len()); - for (r, block) in refs.into_iter().zip(results) { - blocks.push( - block.unwrap_or_else(|| panic!("Storage inconsistency: block {r:?} not found!")), - ); - } - Ok(blocks) - } - - fn scan_scoring_metrics( - &self, - ) -> ConsensusResult> { - let mut metrics_by_author = vec![]; - for kv in self.scoring_metrics.safe_iter() { - metrics_by_author.push(kv?); - } - Ok(metrics_by_author) - } - - // The method returns the last `num_of_rounds` rounds blocks by author in round - // ascending order. When a `before_round` is defined then the blocks of - // round `<=before_round` are returned. If not then the max value for round - // will be used as cut off. - fn scan_last_blocks_by_author( - &self, - author: AuthorityIndex, - num_of_rounds: u64, - before_round: Option, - ) -> ConsensusResult> { - let before_round = before_round.unwrap_or(Round::MAX); - let mut refs = std::collections::VecDeque::new(); - for kv in self - .digests_by_authorities - .reversed_safe_iter_with_bounds( - Some((author, Round::MIN, BlockDigest::MIN)), - Some((author, before_round, BlockDigest::MAX)), - )? - .take(num_of_rounds as usize) - { - let ((author, round, digest), _) = kv?; - refs.push_front(BlockRef::new(round, author, digest)); - } - let results = self.read_blocks(refs.as_slices().0)?; - let mut blocks = vec![]; - for (r, block) in refs.into_iter().zip(results) { - blocks.push( - block.unwrap_or_else(|| panic!("Storage inconsistency: block {r:?} not found!")), - ); - } - Ok(blocks) - } - - fn read_last_commit(&self) -> ConsensusResult> { - let Some(result) = self - .commits - .reversed_safe_iter_with_bounds(None, None)? - .next() - else { - return Ok(None); - }; - let ((_index, digest), serialized) = result?; - let commit = TrustedCommit::new_trusted( - bcs::from_bytes(&serialized).map_err(ConsensusError::MalformedCommit)?, - serialized, - ); - assert_eq!(commit.digest(), digest); - Ok(Some(commit)) - } - - fn scan_commits(&self, range: CommitRange) -> ConsensusResult> { - let mut commits = vec![]; - for result in self.commits.safe_range_iter(( - Included((range.start(), CommitDigest::MIN)), - Included((range.end(), CommitDigest::MAX)), - )) { - let ((_index, digest), serialized) = result?; - let commit = TrustedCommit::new_trusted( - bcs::from_bytes(&serialized).map_err(ConsensusError::MalformedCommit)?, - serialized, - ); - assert_eq!(commit.digest(), digest); - commits.push(commit); - } - Ok(commits) - } - - fn read_commit_votes(&self, commit_index: CommitIndex) -> ConsensusResult> { - let mut votes = Vec::new(); - for vote in self.commit_votes.safe_range_iter(( - Included((commit_index, CommitDigest::MIN, BlockRef::MIN)), - Included((commit_index, CommitDigest::MAX, BlockRef::MAX)), - )) { - let ((_, _, block_ref), _) = vote?; - votes.push(block_ref); - } - Ok(votes) - } - - fn read_last_commit_info(&self) -> ConsensusResult> { - let Some(result) = self - .commit_info - .reversed_safe_iter_with_bounds(None, None)? - .next() - else { - return Ok(None); - }; - let (key, commit_info) = result.map_err(ConsensusError::RocksDBFailure)?; - Ok(Some((CommitRef::new(key.0, key.1), commit_info))) - } -} diff --git a/consensus/core/src/storage/store_tests.rs b/consensus/core/src/storage/store_tests.rs deleted file mode 100644 index 762ec218ae7..00000000000 --- a/consensus/core/src/storage/store_tests.rs +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use consensus_config::AuthorityIndex; -use rstest::rstest; -use tempfile::TempDir; - -use super::{Store, WriteBatch, mem_store::MemStore, rocksdb_store::RocksDBStore}; -use crate::{ - block::{BlockAPI, BlockDigest, BlockRef, Slot, TestBlock, VerifiedBlock}, - commit::{CommitDigest, TrustedCommit}, -}; - -/// Test fixture for store tests. Wraps around various store implementations. -#[expect(clippy::large_enum_variant)] -enum TestStore { - RocksDB((RocksDBStore, TempDir)), - Mem(MemStore), -} - -impl TestStore { - fn store(&self) -> &dyn Store { - match self { - TestStore::RocksDB((store, _)) => store, - TestStore::Mem(store) => store, - } - } -} - -fn new_rocksdb_teststore() -> TestStore { - let temp_dir = TempDir::new().unwrap(); - TestStore::RocksDB(( - RocksDBStore::new(temp_dir.path().to_str().unwrap()), - temp_dir, - )) -} - -fn new_mem_teststore() -> TestStore { - TestStore::Mem(MemStore::new()) -} - -#[rstest] -#[tokio::test] -async fn read_and_contain_blocks( - #[values(new_rocksdb_teststore(), new_mem_teststore())] test_store: TestStore, -) { - let store = test_store.store(); - - let written_blocks: Vec = vec![ - VerifiedBlock::new_for_test(TestBlock::new(1, 1).build()), - VerifiedBlock::new_for_test(TestBlock::new(1, 0).build()), - VerifiedBlock::new_for_test(TestBlock::new(1, 2).build()), - VerifiedBlock::new_for_test(TestBlock::new(2, 3).build()), - ]; - store - .write(WriteBatch::default().blocks(written_blocks.clone())) - .unwrap(); - - { - let refs = vec![written_blocks[0].reference()]; - let read_blocks = store - .read_blocks(&refs) - .expect("Read blocks should not fail"); - assert_eq!(read_blocks.len(), 1); - assert_eq!(read_blocks[0].as_ref().unwrap(), &written_blocks[0]); - } - - { - let refs = vec![ - written_blocks[2].reference(), - written_blocks[1].reference(), - written_blocks[1].reference(), - ]; - let read_blocks = store - .read_blocks(&refs) - .expect("Read blocks should not fail"); - assert_eq!(read_blocks.len(), 3); - assert_eq!(read_blocks[0].as_ref().unwrap(), &written_blocks[2]); - assert_eq!(read_blocks[1].as_ref().unwrap(), &written_blocks[1]); - assert_eq!(read_blocks[2].as_ref().unwrap(), &written_blocks[1]); - } - - { - let refs = vec![ - written_blocks[3].reference(), - BlockRef::new(1, AuthorityIndex::new_for_test(3), BlockDigest::default()), - written_blocks[2].reference(), - ]; - let read_blocks = store - .read_blocks(&refs) - .expect("Read blocks should not fail"); - assert_eq!(read_blocks.len(), 3); - assert_eq!(read_blocks[0].as_ref().unwrap(), &written_blocks[3]); - assert!(read_blocks[1].is_none()); - assert_eq!(read_blocks[2].as_ref().unwrap(), &written_blocks[2]); - - let contain_blocks = store - .contains_blocks(&refs) - .expect("Contain blocks should not fail"); - assert_eq!(contain_blocks.len(), 3); - assert!(contain_blocks[0]); - assert!(!contain_blocks[1]); - assert!(contain_blocks[2]); - } - - { - for block in &written_blocks { - let found = store - .contains_block_at_slot(block.slot()) - .expect("Read blocks should not fail"); - assert!(found); - } - - let found = store - .contains_block_at_slot(Slot::new(10, AuthorityIndex::new_for_test(0))) - .expect("Read blocks should not fail"); - assert!(!found); - } -} - -#[rstest] -#[tokio::test] -async fn scan_blocks( - #[values(new_rocksdb_teststore(), new_mem_teststore())] test_store: TestStore, -) { - let store = test_store.store(); - - let written_blocks = vec![ - VerifiedBlock::new_for_test(TestBlock::new(9, 0).build()), - VerifiedBlock::new_for_test(TestBlock::new(10, 0).build()), - VerifiedBlock::new_for_test(TestBlock::new(10, 1).build()), - VerifiedBlock::new_for_test(TestBlock::new(11, 1).build()), - VerifiedBlock::new_for_test(TestBlock::new(11, 3).build()), - VerifiedBlock::new_for_test(TestBlock::new(12, 1).build()), - VerifiedBlock::new_for_test(TestBlock::new(13, 2).build()), - VerifiedBlock::new_for_test(TestBlock::new(13, 1).build()), - ]; - store - .write(WriteBatch::default().blocks(written_blocks.clone())) - .unwrap(); - - { - let scanned_blocks = store - .scan_blocks_by_author(AuthorityIndex::new_for_test(1), 20) - .expect("Scan blocks should not fail"); - assert!(scanned_blocks.is_empty(), "{scanned_blocks:?}"); - } - - { - let scanned_blocks = store - .scan_blocks_by_author(AuthorityIndex::new_for_test(1), 12) - .expect("Scan blocks should not fail"); - assert_eq!(scanned_blocks.len(), 2, "{scanned_blocks:?}"); - assert_eq!( - scanned_blocks, - vec![written_blocks[5].clone(), written_blocks[7].clone()] - ); - } - - let additional_blocks = vec![ - VerifiedBlock::new_for_test(TestBlock::new(14, 2).build()), - VerifiedBlock::new_for_test(TestBlock::new(15, 0).build()), - VerifiedBlock::new_for_test(TestBlock::new(15, 1).build()), - VerifiedBlock::new_for_test(TestBlock::new(16, 3).build()), - ]; - store - .write(WriteBatch::default().blocks(additional_blocks.clone())) - .unwrap(); - - { - let scanned_blocks = store - .scan_blocks_by_author(AuthorityIndex::new_for_test(1), 10) - .expect("Scan blocks should not fail"); - assert_eq!(scanned_blocks.len(), 5, "{scanned_blocks:?}"); - assert_eq!( - scanned_blocks, - vec![ - written_blocks[2].clone(), - written_blocks[3].clone(), - written_blocks[5].clone(), - written_blocks[7].clone(), - additional_blocks[2].clone(), - ] - ); - } - - { - let scanned_blocks = store - .scan_last_blocks_by_author(AuthorityIndex::new_for_test(1), 2, None) - .expect("Scan blocks should not fail"); - assert_eq!(scanned_blocks.len(), 2, "{scanned_blocks:?}"); - assert_eq!( - scanned_blocks, - vec![written_blocks[7].clone(), additional_blocks[2].clone()] - ); - - let scanned_blocks = store - .scan_last_blocks_by_author(AuthorityIndex::new_for_test(1), 0, None) - .expect("Scan blocks should not fail"); - assert_eq!(scanned_blocks.len(), 0); - } -} - -#[rstest] -#[tokio::test] -async fn read_and_scan_commits( - #[values(new_rocksdb_teststore(), new_mem_teststore())] test_store: TestStore, -) { - let store = test_store.store(); - - { - let last_commit = store - .read_last_commit() - .expect("Read last commit should not fail"); - assert!(last_commit.is_none(), "{last_commit:?}"); - } - - let written_commits = vec![ - TrustedCommit::new_for_test( - 1, - CommitDigest::MIN, - 1, - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::default()), - vec![], - ), - TrustedCommit::new_for_test( - 2, - CommitDigest::MIN, - 2, - BlockRef::new(2, AuthorityIndex::new_for_test(0), BlockDigest::default()), - vec![], - ), - TrustedCommit::new_for_test( - 3, - CommitDigest::MIN, - 3, - BlockRef::new(3, AuthorityIndex::new_for_test(0), BlockDigest::default()), - vec![], - ), - TrustedCommit::new_for_test( - 4, - CommitDigest::MIN, - 4, - BlockRef::new(4, AuthorityIndex::new_for_test(0), BlockDigest::default()), - vec![], - ), - ]; - store - .write(WriteBatch::default().commits(written_commits.clone())) - .unwrap(); - - { - let last_commit = store - .read_last_commit() - .expect("Read last commit should not fail"); - assert_eq!( - last_commit.as_ref(), - written_commits.last(), - "{last_commit:?}" - ); - } - - { - let scanned_commits = store - .scan_commits((20..=24).into()) - .expect("Scan commits should not fail"); - assert!(scanned_commits.is_empty(), "{scanned_commits:?}"); - } - - { - let scanned_commits = store - .scan_commits((3..=4).into()) - .expect("Scan commits should not fail"); - assert_eq!(scanned_commits.len(), 2, "{scanned_commits:?}"); - assert_eq!( - scanned_commits, - vec![written_commits[2].clone(), written_commits[3].clone()] - ); - } - - { - let scanned_commits = store - .scan_commits((0..=2).into()) - .expect("Scan commits should not fail"); - assert_eq!(scanned_commits.len(), 2, "{scanned_commits:?}"); - assert_eq!( - scanned_commits, - vec![written_commits[0].clone(), written_commits[1].clone()] - ); - } - - { - let scanned_commits = store - .scan_commits((0..=4).into()) - .expect("Scan commits should not fail"); - assert_eq!(scanned_commits.len(), 4, "{scanned_commits:?}"); - assert_eq!(scanned_commits, written_commits,); - } -} - -#[rstest] -#[tokio::test] -async fn scan_scoring_metrics( - #[values(new_rocksdb_teststore(), new_mem_teststore())] test_store: TestStore, -) { - use crate::storage::StorageScoringMetrics; - - let store = test_store.store(); - let metrics_updates = [ - StorageScoringMetrics { - faulty_blocks_provable: 1, - faulty_blocks_unprovable: 2, - equivocations: 3, - missing_proposals: 4, - }, - StorageScoringMetrics { - faulty_blocks_provable: 0, - faulty_blocks_unprovable: 0, - equivocations: 0, - missing_proposals: 0, - }, - ]; - let authories = [ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(1), - AuthorityIndex::new_for_test(2), - ]; - - let metrics_to_write = vec![ - (authories[0], metrics_updates[0].clone()), - (authories[1], metrics_updates[1].clone()), - ]; - - store - .write(WriteBatch::default().scoring_metrics(metrics_to_write.clone())) - .unwrap(); - - { - let scanned_metrics = store - .scan_scoring_metrics() - .expect("Scan scoring_metrics should not fail"); - assert_eq!(&scanned_metrics, &metrics_to_write); - } - - let metrics_to_write = vec![(authories[0], metrics_updates[1].clone())]; - - store - .write(WriteBatch::default().scoring_metrics(metrics_to_write)) - .unwrap(); - - { - let scanned_metrics = store - .scan_scoring_metrics() - .expect("Scan scoring_metrics should not fail"); - assert_eq!( - &scanned_metrics, - &vec![ - (authories[0], metrics_updates[1].clone()), - (authories[1], metrics_updates[1].clone()) - ] - ); - } -} diff --git a/consensus/core/src/subscriber.rs b/consensus/core/src/subscriber.rs deleted file mode 100644 index b29b39edb5f..00000000000 --- a/consensus/core/src/subscriber.rs +++ /dev/null @@ -1,410 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use consensus_config::AuthorityIndex; -use futures::StreamExt; -use iota_metrics::spawn_monitored_task; -use parking_lot::{Mutex, RwLock}; -use tokio::{ - task::JoinHandle, - time::{sleep, timeout}, -}; -use tracing::{Instrument, debug, error, info, trace_span}; - -use crate::{ - Round, - block::BlockAPI as _, - context::Context, - dag_state::DagState, - error::ConsensusError, - network::{NetworkClient, NetworkService}, - scoring_metrics_store::ErrorSource, -}; - -/// Subscriber manages the block stream subscriptions to other peers, taking -/// care of retrying when subscription streams break. Blocks returned from the -/// peer are sent to the authority service for processing. -/// Currently subscription management for individual peer is not exposed, but it -/// could become useful in future. -pub(crate) struct Subscriber { - context: Arc, - network_client: Arc, - authority_service: Arc, - dag_state: Arc>, - subscriptions: Arc>]>>>, -} - -impl Subscriber { - pub(crate) fn new( - context: Arc, - network_client: Arc, - authority_service: Arc, - dag_state: Arc>, - ) -> Self { - let subscriptions = (0..context.committee.size()) - .map(|_| None) - .collect::>(); - Self { - context, - network_client, - authority_service, - dag_state, - subscriptions: Arc::new(Mutex::new(subscriptions.into_boxed_slice())), - } - } - - pub(crate) fn subscribe(&self, peer: AuthorityIndex) { - if peer == self.context.own_index { - error!("Attempt to subscribe to own validator {peer} is ignored!"); - return; - } - let context = self.context.clone(); - let network_client = self.network_client.clone(); - let authority_service = self.authority_service.clone(); - let (mut last_received, gc_round, gc_enabled) = { - let dag_state = self.dag_state.read(); - ( - dag_state.get_last_block_for_authority(peer).round(), - dag_state.gc_round(), - dag_state.gc_enabled(), - ) - }; - - // If the latest block we have accepted by an authority is older than the - // current gc round, then do not attempt to fetch any blocks from that - // point as they will simply be skipped. Instead do attempt to fetch - // from the gc round. - if gc_enabled && last_received < gc_round { - info!( - "Last received block for peer {peer} is older than GC round, {last_received} < {gc_round}, fetching from GC round" - ); - last_received = gc_round; - } - - let mut subscriptions = self.subscriptions.lock(); - self.unsubscribe_locked(peer, &mut subscriptions[peer.value()]); - subscriptions[peer.value()] = Some(spawn_monitored_task!(Self::subscription_loop( - context, - network_client, - authority_service, - peer, - last_received, - ))); - } - - pub(crate) fn stop(&self) { - let mut subscriptions = self.subscriptions.lock(); - for (peer, _) in self.context.committee.authorities() { - self.unsubscribe_locked(peer, &mut subscriptions[peer.value()]); - } - } - - fn unsubscribe_locked(&self, peer: AuthorityIndex, subscription: &mut Option>) { - let peer_hostname = &self.context.committee.authority(peer).hostname; - if let Some(subscription) = subscription.take() { - subscription.abort(); - } - // There is a race between shutting down the subscription task and clearing the - // metric here. TODO: fix the race when unsubscribe_locked() gets called - // outside of stop(). - self.context - .metrics - .node_metrics - .subscribed_to - .with_label_values(&[peer_hostname]) - .set(0); - } - - async fn subscription_loop( - context: Arc, - network_client: Arc, - authority_service: Arc, - peer: AuthorityIndex, - last_received: Round, - ) { - const IMMEDIATE_RETRIES: i64 = 3; - // When not immediately retrying, limit retry delay between 100ms and 10s. - const INITIAL_RETRY_INTERVAL: Duration = Duration::from_millis(100); - const MAX_RETRY_INTERVAL: Duration = Duration::from_secs(10); - const RETRY_INTERVAL_MULTIPLIER: f32 = 1.2; - let peer_hostname = &context.committee.authority(peer).hostname; - let mut retries: i64 = 0; - let mut delay = INITIAL_RETRY_INTERVAL; - 'subscription: loop { - context - .metrics - .node_metrics - .subscribed_to - .with_label_values(&[peer_hostname]) - .set(0); - - if retries > IMMEDIATE_RETRIES { - debug!( - "Delaying retry {} of peer {} subscription, in {} seconds", - retries, - peer_hostname, - delay.as_secs_f32(), - ); - sleep(delay).await; - // Update delay for the next retry. - delay = delay - .mul_f32(RETRY_INTERVAL_MULTIPLIER) - .min(MAX_RETRY_INTERVAL); - } else if retries > 0 { - // Retry immediately, but still yield to avoid monopolizing the thread. - tokio::task::yield_now().await; - } else { - // First attempt, reset delay for next retries but no waiting. - delay = INITIAL_RETRY_INTERVAL; - } - retries += 1; - - // Wrap subscribe_blocks in a timeout and increment metric on timeout - let subscribe_future = - network_client.subscribe_blocks(peer, last_received, MAX_RETRY_INTERVAL); - let subscribe_result = timeout(MAX_RETRY_INTERVAL * 5, subscribe_future).await; - let mut blocks = match subscribe_result { - Ok(inner_result) => match inner_result { - Ok(blocks) => { - debug!( - "Subscribed to peer {} {} after {} attempts", - peer, peer_hostname, retries - ); - context - .metrics - .node_metrics - .subscriber_connection_attempts - .with_label_values(&[peer_hostname.as_str(), "success"]) - .inc(); - blocks - } - Err(e) => { - debug!( - "Failed to subscribe to blocks from peer {} {}: {}", - peer, peer_hostname, e - ); - context - .metrics - .node_metrics - .subscriber_connection_attempts - .with_label_values(&[peer_hostname.as_str(), "failure"]) - .inc(); - continue 'subscription; - } - }, - Err(_) => { - debug!( - "Timeout subscribing to blocks from peer {} {}", - peer, peer_hostname - ); - context - .metrics - .node_metrics - .subscriber_connection_attempts - .with_label_values(&[peer_hostname.as_str(), "timeout"]) - .inc(); - continue 'subscription; - } - }; - - // Now can consider the subscription successful - context - .metrics - .node_metrics - .subscribed_to - .with_label_values(&[peer_hostname]) - .set(1); - - 'stream: loop { - match blocks.next().await { - Some(block) => { - context - .metrics - .node_metrics - .subscribed_blocks - .with_label_values(&[peer_hostname]) - .inc(); - let result = authority_service - .handle_send_block(peer, block.clone()) - .instrument(trace_span!("handle_send_block")) - .await; - if let Err(e) = result { - context - .scoring_metrics_store - .update_scoring_metrics_on_block_receival( - peer, - peer_hostname, - e.clone(), - ErrorSource::Subscriber, - &context.metrics.node_metrics, - ); - match e { - ConsensusError::BlockRejected { block_ref, reason } => { - debug!( - "Failed to process block from peer {} {} for block {:?}: {}", - peer, peer_hostname, block_ref, reason - ); - } - _ => { - info!( - "Invalid block received from peer {} {}: {}", - peer, peer_hostname, e - ); - } - } - } - // Reset retries when a block is received. - retries = 0; - } - None => { - debug!( - "Subscription to blocks from peer {} {} ended", - peer, peer_hostname - ); - retries += 1; - break 'stream; - } - } - } - } - } -} - -#[cfg(test)] -mod test { - use async_trait::async_trait; - use bytes::Bytes; - use futures::stream; - - use super::*; - use crate::{ - VerifiedBlock, - block::BlockRef, - commit::CommitRange, - error::ConsensusResult, - network::{BlockStream, ExtendedSerializedBlock, test_network::TestService}, - storage::mem_store::MemStore, - }; - - struct SubscriberTestClient {} - - impl SubscriberTestClient { - fn new() -> Self { - Self {} - } - } - - #[async_trait] - impl NetworkClient for SubscriberTestClient { - const SUPPORT_STREAMING: bool = true; - - async fn send_block( - &self, - _peer: AuthorityIndex, - _block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - unimplemented!("Unimplemented") - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - let block_stream = stream::unfold((), |_| async { - sleep(Duration::from_millis(1)).await; - let block = ExtendedSerializedBlock { - block: Bytes::from(vec![1u8; 8]), - excluded_ancestors: vec![], - }; - Some((block, ())) - }) - .take(10); - Ok(Box::pin(block_stream)) - } - - async fn fetch_blocks( - &self, - _peer: AuthorityIndex, - _block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - _peer: AuthorityIndex, - _authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - unimplemented!("Unimplemented") - } - - async fn get_latest_rounds( - &self, - _peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn subscriber_retries() { - let (context, _keys) = Context::new_for_test(4); - let context = Arc::new(context); - let authority_service = Arc::new(Mutex::new(TestService::new())); - let network_client = Arc::new(SubscriberTestClient::new()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let subscriber = Subscriber::new( - context.clone(), - network_client, - authority_service.clone(), - dag_state, - ); - - let peer = context.committee.to_authority_index(2).unwrap(); - subscriber.subscribe(peer); - - // Wait for enough blocks received. - for _ in 0..10 { - tokio::time::sleep(Duration::from_secs(1)).await; - let service = authority_service.lock(); - if service.handle_send_block.len() >= 100 { - break; - } - } - - // Even if the stream ends after 10 blocks, the subscriber should retry and get - // enough blocks eventually. - let service = authority_service.lock(); - assert!(service.handle_send_block.len() >= 100); - for (p, block) in service.handle_send_block.iter() { - assert_eq!(*p, peer); - assert_eq!( - *block, - ExtendedSerializedBlock { - block: Bytes::from(vec![1u8; 8]), - excluded_ancestors: vec![] - } - ); - } - } -} diff --git a/consensus/core/src/synchronizer.rs b/consensus/core/src/synchronizer.rs deleted file mode 100644 index dff328eb113..00000000000 --- a/consensus/core/src/synchronizer.rs +++ /dev/null @@ -1,3139 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, - num::NonZeroUsize, - sync::Arc, - time::Duration, -}; - -use bytes::Bytes; -use consensus_config::AuthorityIndex; -use futures::{StreamExt as _, stream::FuturesUnordered}; -use iota_macros::fail_point_async; -use iota_metrics::{ - monitored_future, - monitored_mpsc::{Receiver, Sender, channel}, - monitored_scope, -}; -use itertools::Itertools as _; -use lru::LruCache; -use parking_lot::{Mutex, RwLock}; -#[cfg(not(test))] -use rand::prelude::{IteratorRandom, SeedableRng, SliceRandom, StdRng}; -use tap::TapFallible; -use tokio::{ - runtime::Handle, - sync::{mpsc::error::TrySendError, oneshot}, - task::{JoinError, JoinSet}, - time::{Instant, sleep, sleep_until, timeout}, -}; -use tracing::{debug, error, info, instrument, trace, warn}; - -use crate::{ - BlockAPI, CommitIndex, Round, - authority_service::COMMIT_LAG_MULTIPLIER, - block::{BlockDigest, BlockRef, GENESIS_ROUND, SignedBlock, VerifiedBlock}, - block_verifier::BlockVerifier, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::CoreThreadDispatcher, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - network::NetworkClient, - scoring_metrics_store::ErrorSource, -}; - -/// The number of concurrent fetch blocks requests per authority -const FETCH_BLOCKS_CONCURRENCY: usize = 5; - -/// The maximal additional blocks (parents) that can be fetched. -// TODO: This is a temporary value, and should be removed once the protocol -// version is updated to support batching -pub(crate) const MAX_ADDITIONAL_BLOCKS: usize = 10; - -/// The maximum number of verified block references to cache for deduplication. -const VERIFIED_BLOCKS_CACHE_CAP: usize = 200_000; - -/// The timeout for synchronizer to fetch blocks from a given peer authority. -const FETCH_REQUEST_TIMEOUT: Duration = Duration::from_millis(2_000); - -/// The timeout for periodic synchronizer to fetch blocks from the peers. -const FETCH_FROM_PEERS_TIMEOUT: Duration = Duration::from_millis(4_000); - -/// The maximum number of authorities from which we will try to periodically -/// fetch blocks at the same moment. The guard will protect that we will not ask -/// from more than this number of authorities at the same time. -const MAX_AUTHORITIES_TO_FETCH_PER_BLOCK: usize = 2; - -/// The maximum number of authorities from which the live synchronizer will try -/// to fetch blocks at the same moment. This is lower than the periodic sync -/// limit to prioritize periodic sync. -const MAX_AUTHORITIES_TO_LIVE_FETCH_PER_BLOCK: usize = 1; - -/// The maximum number of peers from which the periodic synchronizer will -/// request blocks -const MAX_PERIODIC_SYNC_PEERS: usize = 4; - -/// The maximum number of peers in the periodic synchronizer which are chosen -/// totally random to fetch blocks from. The other peers will be chosen based on -/// their knowledge of the DAG. -const MAX_PERIODIC_SYNC_RANDOM_PEERS: usize = 2; - -/// Represents the different methods used for synchronization -#[derive(Clone)] -enum SyncMethod { - Live, - Periodic, -} - -struct BlocksGuard { - map: Arc, - block_refs: BTreeSet, - peer: AuthorityIndex, - method: SyncMethod, -} - -impl Drop for BlocksGuard { - fn drop(&mut self) { - self.map.unlock_blocks(&self.block_refs, self.peer); - } -} - -// Keeps a mapping between the missing blocks that have been instructed to be -// fetched and the authorities that are currently fetching them. For a block ref -// there is a maximum number of authorities that can concurrently fetch it. The -// authority ids that are currently fetching a block are set on the -// corresponding `BTreeSet` and basically they act as "locks". -struct InflightBlocksMap { - inner: Mutex>>, -} - -impl InflightBlocksMap { - fn new() -> Arc { - Arc::new(Self { - inner: Mutex::new(HashMap::new()), - }) - } - - /// Locks the blocks to be fetched for the assigned `peer_index`. We want to - /// avoid re-fetching the missing blocks from too many authorities at - /// the same time, thus we limit the concurrency per block by attempting - /// to lock per block. If a block is already fetched by the maximum allowed - /// number of authorities, then the block ref will not be included in the - /// returned set. The method returns all the block refs that have been - /// successfully locked and allowed to be fetched. - /// - /// Different limits apply based on the sync method: - /// - Periodic sync: Can lock if total authorities < - /// MAX_AUTHORITIES_TO_FETCH_PER_BLOCK (3) - /// - Live sync: Can lock if total authorities < - /// MAX_AUTHORITIES_TO_LIVE_FETCH_PER_BLOCK (2) - fn lock_blocks( - self: &Arc, - missing_block_refs: BTreeSet, - peer: AuthorityIndex, - method: SyncMethod, - ) -> Option { - let mut blocks = BTreeSet::new(); - let mut inner = self.inner.lock(); - - for block_ref in missing_block_refs { - let authorities = inner.entry(block_ref).or_default(); - - // Check if this peer is already fetching this block - if authorities.contains(&peer) { - continue; - } - - // Count total authorities currently fetching this block - let total_count = authorities.len(); - - // Determine the limit based on the sync method - let max_limit = match method { - SyncMethod::Live => MAX_AUTHORITIES_TO_LIVE_FETCH_PER_BLOCK, - SyncMethod::Periodic => MAX_AUTHORITIES_TO_FETCH_PER_BLOCK, - }; - - // Check if we can acquire the lock - if total_count < max_limit { - assert!(authorities.insert(peer)); - blocks.insert(block_ref); - } - } - - if blocks.is_empty() { - None - } else { - Some(BlocksGuard { - map: self.clone(), - block_refs: blocks, - peer, - method, - }) - } - } - - /// Unlocks the provided block references for the given `peer`. The - /// unlocking is strict, meaning that if this method is called for a - /// specific block ref and peer more times than the corresponding lock - /// has been called, it will panic. - fn unlock_blocks(self: &Arc, block_refs: &BTreeSet, peer: AuthorityIndex) { - // Now mark all the blocks as fetched from the map - let mut blocks_to_fetch = self.inner.lock(); - for block_ref in block_refs { - let authorities = blocks_to_fetch - .get_mut(block_ref) - .expect("Should have found a non empty map"); - - assert!(authorities.remove(&peer), "Peer index should be present!"); - - // if the last one then just clean up - if authorities.is_empty() { - blocks_to_fetch.remove(block_ref); - } - } - } - - /// Drops the provided `blocks_guard` which will force to unlock the blocks, - /// and lock now again the referenced block refs. The swap is best - /// effort and there is no guarantee that the `peer` will be able to - /// acquire the new locks. - fn swap_locks( - self: &Arc, - blocks_guard: BlocksGuard, - peer: AuthorityIndex, - ) -> Option { - let block_refs = blocks_guard.block_refs.clone(); - let method = blocks_guard.method.clone(); - - // Explicitly drop the guard - drop(blocks_guard); - - // Now create a new guard with the same sync method - self.lock_blocks(block_refs, peer, method) - } - - #[cfg(test)] - fn num_of_locked_blocks(self: &Arc) -> usize { - let inner = self.inner.lock(); - inner.len() - } -} - -enum Command { - FetchBlocks { - missing_block_refs: BTreeSet, - peer_index: AuthorityIndex, - result: oneshot::Sender>, - }, - FetchOwnLastBlock, - KickOffScheduler, -} - -pub(crate) struct SynchronizerHandle { - commands_sender: Sender, - tasks: tokio::sync::Mutex>, -} - -impl SynchronizerHandle { - /// Explicitly asks from the synchronizer to fetch the blocks - provided the - /// block_refs set - from the peer authority. - pub(crate) async fn fetch_blocks( - &self, - missing_block_refs: BTreeSet, - peer_index: AuthorityIndex, - ) -> ConsensusResult<()> { - let (sender, receiver) = oneshot::channel(); - self.commands_sender - .send(Command::FetchBlocks { - missing_block_refs, - peer_index, - result: sender, - }) - .await - .map_err(|_err| ConsensusError::Shutdown)?; - receiver.await.map_err(|_err| ConsensusError::Shutdown)? - } - - pub(crate) async fn stop(&self) -> Result<(), JoinError> { - let mut tasks = self.tasks.lock().await; - tasks.abort_all(); - while let Some(result) = tasks.join_next().await { - result? - } - Ok(()) - } -} - -/// `Synchronizer` oversees live block synchronization, crucial for node -/// progress. Live synchronization refers to the process of retrieving missing -/// blocks, particularly those essential for advancing a node when data from -/// only a few rounds is absent. If a node significantly lags behind the -/// network, `commit_syncer` handles fetching missing blocks via a more -/// efficient approach. `Synchronizer` aims for swift catch-up employing two -/// mechanisms: -/// -/// 1. Explicitly requesting missing blocks from designated authorities via the -/// "block send" path. This includes attempting to fetch any missing -/// ancestors necessary for processing a received block. Such requests -/// prioritize the block author, maximizing the chance of prompt retrieval. A -/// locking mechanism allows concurrent requests for missing blocks from up -/// to two authorities simultaneously, enhancing the chances of timely -/// retrieval. Notably, if additional missing blocks arise during block -/// processing, requests to the same authority are deferred to the scheduler. -/// -/// 2. Periodically requesting missing blocks via a scheduler. This primarily -/// serves to retrieve missing blocks that were not ancestors of a received -/// block via the "block send" path. The scheduler operates on either a fixed -/// periodic basis or is triggered immediately after explicit fetches -/// described in (1), ensuring continued block retrieval if gaps persist. -/// -/// Additionally to the above, the synchronizer can synchronize and fetch the -/// last own proposed block from the network peers as best effort approach to -/// recover node from amnesia and avoid making the node equivocate. -pub(crate) struct Synchronizer { - context: Arc, - commands_receiver: Receiver, - fetch_block_senders: BTreeMap>, - core_dispatcher: Arc, - commit_vote_monitor: Arc, - dag_state: Arc>, - fetch_blocks_scheduler_task: JoinSet<()>, - fetch_own_last_block_task: JoinSet<()>, - network_client: Arc, - block_verifier: Arc, - inflight_blocks_map: Arc, - verified_blocks_cache: Arc>>, - commands_sender: Sender, -} - -impl Synchronizer { - /// Starts the synchronizer, which is responsible for fetching blocks from - /// other authorities and managing block synchronization tasks. - pub fn start( - network_client: Arc, - context: Arc, - core_dispatcher: Arc, - commit_vote_monitor: Arc, - block_verifier: Arc, - dag_state: Arc>, - sync_last_known_own_block: bool, - ) -> Arc { - let (commands_sender, commands_receiver) = - channel("consensus_synchronizer_commands", 1_000); - let inflight_blocks_map = InflightBlocksMap::new(); - let verified_blocks_cache = Arc::new(Mutex::new(LruCache::new( - NonZeroUsize::new(VERIFIED_BLOCKS_CACHE_CAP).unwrap(), - ))); - - // Spawn the tasks to fetch the blocks from the others - let mut fetch_block_senders = BTreeMap::new(); - let mut tasks = JoinSet::new(); - for (index, _) in context.committee.authorities() { - if index == context.own_index { - continue; - } - let (sender, receiver) = - channel("consensus_synchronizer_fetches", FETCH_BLOCKS_CONCURRENCY); - let fetch_blocks_from_authority_async = Self::fetch_blocks_from_authority( - index, - network_client.clone(), - block_verifier.clone(), - verified_blocks_cache.clone(), - commit_vote_monitor.clone(), - context.clone(), - core_dispatcher.clone(), - dag_state.clone(), - receiver, - commands_sender.clone(), - ); - tasks.spawn(monitored_future!(fetch_blocks_from_authority_async)); - fetch_block_senders.insert(index, sender); - } - - let commands_sender_clone = commands_sender.clone(); - - if sync_last_known_own_block { - commands_sender - .try_send(Command::FetchOwnLastBlock) - .expect("Failed to sync our last block"); - } - - // Spawn the task to listen to the requests & periodic runs - tasks.spawn(monitored_future!(async move { - let mut s = Self { - context, - commands_receiver, - fetch_block_senders, - core_dispatcher, - commit_vote_monitor, - fetch_blocks_scheduler_task: JoinSet::new(), - fetch_own_last_block_task: JoinSet::new(), - network_client, - block_verifier, - inflight_blocks_map, - verified_blocks_cache, - commands_sender: commands_sender_clone, - dag_state, - }; - s.run().await; - })); - - Arc::new(SynchronizerHandle { - commands_sender, - tasks: tokio::sync::Mutex::new(tasks), - }) - } - - // The main loop to listen for the submitted commands. - async fn run(&mut self) { - // We want the synchronizer to run periodically every 200ms to fetch any missing - // blocks. - const PERIODIC_FETCH_TIMEOUT: Duration = Duration::from_millis(200); - let scheduler_timeout = sleep_until(Instant::now() + PERIODIC_FETCH_TIMEOUT); - - tokio::pin!(scheduler_timeout); - - loop { - tokio::select! { - Some(command) = self.commands_receiver.recv() => { - match command { - Command::FetchBlocks{ missing_block_refs, peer_index, result } => { - if peer_index == self.context.own_index { - error!("We should never attempt to fetch blocks from our own node"); - continue; - } - - let peer_hostname = self.context.committee.authority(peer_index).hostname.clone(); - - // Keep only the max allowed blocks to request. It is ok to reduce here as the scheduler - // task will take care syncing whatever is leftover. - let missing_block_refs = missing_block_refs - .into_iter() - .take(self.context.parameters.max_blocks_per_sync) - .collect(); - - let blocks_guard = self.inflight_blocks_map.lock_blocks(missing_block_refs, peer_index, SyncMethod::Live); - let Some(blocks_guard) = blocks_guard else { - result.send(Ok(())).ok(); - continue; - }; - - // We don't block if the corresponding peer task is saturated - but we rather drop the request. That's ok as the periodic - // synchronization task will handle any still missing blocks in next run. - let r = self - .fetch_block_senders - .get(&peer_index) - .expect("Fatal error, sender should be present") - .try_send(blocks_guard) - .map_err(|err| { - match err { - TrySendError::Full(_) => ConsensusError::SynchronizerSaturated(peer_index,peer_hostname), - TrySendError::Closed(_) => ConsensusError::Shutdown - } - }); - - result.send(r).ok(); - } - Command::FetchOwnLastBlock => { - if self.fetch_own_last_block_task.is_empty() { - self.start_fetch_own_last_block_task(); - } - } - Command::KickOffScheduler => { - // just reset the scheduler timeout timer to run immediately if not already running. - // If the scheduler is already running then just reduce the remaining time to run. - let timeout = if self.fetch_blocks_scheduler_task.is_empty() { - Instant::now() - } else { - Instant::now() + PERIODIC_FETCH_TIMEOUT.checked_div(2).unwrap() - }; - - // only reset if it is earlier than the next deadline - if timeout < scheduler_timeout.deadline() { - scheduler_timeout.as_mut().reset(timeout); - } - } - } - }, - Some(result) = self.fetch_own_last_block_task.join_next(), if !self.fetch_own_last_block_task.is_empty() => { - match result { - Ok(()) => {}, - Err(e) => { - if e.is_cancelled() { - } else if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } else { - panic!("fetch our last block task failed: {e}"); - } - }, - }; - }, - Some(result) = self.fetch_blocks_scheduler_task.join_next(), if !self.fetch_blocks_scheduler_task.is_empty() => { - match result { - Ok(()) => {}, - Err(e) => { - if e.is_cancelled() { - } else if e.is_panic() { - std::panic::resume_unwind(e.into_panic()); - } else { - panic!("fetch blocks scheduler task failed: {e}"); - } - }, - }; - }, - () = &mut scheduler_timeout => { - // we want to start a new task only if the previous one has already finished. - if self.fetch_blocks_scheduler_task.is_empty() { - if let Err(err) = self.start_fetch_missing_blocks_task().await { - debug!("Core is shutting down, synchronizer is shutting down: {err:?}"); - return; - }; - } - - scheduler_timeout - .as_mut() - .reset(Instant::now() + PERIODIC_FETCH_TIMEOUT); - } - } - } - } - - async fn fetch_blocks_from_authority( - peer_index: AuthorityIndex, - network_client: Arc, - block_verifier: Arc, - verified_cache: Arc>>, - commit_vote_monitor: Arc, - context: Arc, - core_dispatcher: Arc, - dag_state: Arc>, - mut receiver: Receiver, - commands_sender: Sender, - ) { - const MAX_RETRIES: u32 = 3; - let peer_hostname = &context.committee.authority(peer_index).hostname; - - let mut requests = FuturesUnordered::new(); - - loop { - tokio::select! { - Some(blocks_guard) = receiver.recv(), if requests.len() < FETCH_BLOCKS_CONCURRENCY => { - // get the highest accepted rounds - let highest_rounds = Self::get_highest_accepted_rounds(dag_state.clone(), &context); - - // Record metrics for live synchronizer requests - let metrics = &context.metrics.node_metrics; - metrics - .synchronizer_requested_blocks_by_peer - .with_label_values(&[peer_hostname.as_str(), "live"]) - .inc_by(blocks_guard.block_refs.len() as u64); - // Count requested blocks per authority and increment metric by one per authority - let mut authors = HashSet::new(); - for block_ref in &blocks_guard.block_refs { - authors.insert(block_ref.author); - } - for author in authors { - let host = &context.committee.authority(author).hostname; - metrics - .synchronizer_requested_blocks_by_authority - .with_label_values(&[host.as_str(), "live"]) - .inc(); - } - - requests.push(Self::fetch_blocks_request( - network_client.clone(), - peer_index, - blocks_guard, - highest_rounds, - FETCH_REQUEST_TIMEOUT, - 1, - )) - }, - Some((response, blocks_guard, retries, _peer, highest_rounds)) = requests.next() => { - match response { - Ok(blocks) => { - if let Err(err) = Self::process_fetched_blocks(blocks, - peer_index, - blocks_guard, - core_dispatcher.clone(), - block_verifier.clone(), - verified_cache.clone(), - commit_vote_monitor.clone(), - context.clone(), - commands_sender.clone(), - "live" - ).await { - context.scoring_metrics_store.update_scoring_metrics_on_block_receival( - peer_index, - peer_hostname, - err.clone(), - ErrorSource::Synchronizer, - &context.metrics.node_metrics, - ); - warn!("Error while processing fetched blocks from peer {peer_index} {peer_hostname}: {err}"); - context.metrics.node_metrics.synchronizer_process_fetched_failures_by_peer.with_label_values(&[peer_hostname.as_str(), "live"]).inc(); - } - }, - Err(_) => { - context.metrics.node_metrics.synchronizer_fetch_failures_by_peer.with_label_values(&[peer_hostname.as_str(), "live"]).inc(); - if retries <= MAX_RETRIES { - requests.push(Self::fetch_blocks_request(network_client.clone(), peer_index, blocks_guard, highest_rounds, FETCH_REQUEST_TIMEOUT, retries)) - } else { - warn!("Max retries {retries} reached while trying to fetch blocks from peer {peer_index} {peer_hostname}."); - // we don't necessarily need to do, but dropping the guard here to unlock the blocks - drop(blocks_guard); - } - } - } - }, - else => { - info!("Fetching blocks from authority {peer_index} task will now abort."); - break; - } - } - } - } - - /// Processes the requested raw fetched blocks from peer `peer_index`. If no - /// error is returned then the verified blocks are immediately sent to - /// Core for processing. - async fn process_fetched_blocks( - mut serialized_blocks: Vec, - peer_index: AuthorityIndex, - requested_blocks_guard: BlocksGuard, - core_dispatcher: Arc, - block_verifier: Arc, - verified_cache: Arc>>, - commit_vote_monitor: Arc, - context: Arc, - commands_sender: Sender, - sync_method: &str, - ) -> ConsensusResult<()> { - if serialized_blocks.is_empty() { - return Ok(()); - } - let _s = context - .metrics - .node_metrics - .scope_processing_time - .with_label_values(&["Synchronizer::process_fetched_blocks"]) - .start_timer(); - - // Limit the number of the returned blocks processed. - if context.protocol_config.consensus_batched_block_sync() { - serialized_blocks.truncate(context.parameters.max_blocks_per_sync); - } else { - // Ensure that all the returned blocks do not go over the total max allowed - // returned blocks - if serialized_blocks.len() - > requested_blocks_guard.block_refs.len() + MAX_ADDITIONAL_BLOCKS - { - return Err(ConsensusError::TooManyFetchedBlocksReturned(peer_index)); - } - } - - // Verify all the fetched blocks - let blocks = Handle::current() - .spawn_blocking({ - let block_verifier = block_verifier.clone(); - let verified_cache = verified_cache.clone(); - let context = context.clone(); - let sync_method = sync_method.to_string(); - move || { - Self::verify_blocks( - serialized_blocks, - block_verifier, - verified_cache, - &context, - peer_index, - &sync_method, - ) - } - }) - .await - .expect("Spawn blocking should not fail")?; - - if !context.protocol_config.consensus_batched_block_sync() { - // Get all the ancestors of the requested blocks only - let ancestors = blocks - .iter() - .filter(|b| requested_blocks_guard.block_refs.contains(&b.reference())) - .flat_map(|b| b.ancestors().to_vec()) - .collect::>(); - - // Now confirm that the blocks are either between the ones requested, or they - // are parents of the requested blocks - for block in &blocks { - if !requested_blocks_guard - .block_refs - .contains(&block.reference()) - && !ancestors.contains(&block.reference()) - { - return Err(ConsensusError::UnexpectedFetchedBlock { - index: peer_index, - block_ref: block.reference(), - }); - } - } - } - - // Record commit votes from the verified blocks. - for block in &blocks { - commit_vote_monitor.observe_block(block); - } - - let metrics = &context.metrics.node_metrics; - let peer_hostname = &context.committee.authority(peer_index).hostname; - metrics - .synchronizer_fetched_blocks_by_peer - .with_label_values(&[peer_hostname.as_str(), sync_method]) - .inc_by(blocks.len() as u64); - for block in &blocks { - let block_hostname = &context.committee.authority(block.author()).hostname; - metrics - .synchronizer_fetched_blocks_by_authority - .with_label_values(&[block_hostname.as_str(), sync_method]) - .inc(); - } - - debug!( - "Synced {} missing blocks from peer {peer_index} {peer_hostname}: {}", - blocks.len(), - blocks.iter().map(|b| b.reference().to_string()).join(", "), - ); - - // Now send them to core for processing. Ignore the returned missing blocks as - // we don't want this mechanism to keep feedback looping on fetching - // more blocks. The periodic synchronization will take care of that. - let missing_blocks = core_dispatcher - .add_blocks(blocks) - .await - .map_err(|_| ConsensusError::Shutdown)?; - - // now release all the locked blocks as they have been fetched, verified & - // processed - drop(requested_blocks_guard); - - // kick off immediately the scheduled synchronizer - if !missing_blocks.is_empty() { - // do not block here, so we avoid any possible cycles. - if let Err(TrySendError::Full(_)) = commands_sender.try_send(Command::KickOffScheduler) - { - warn!("Commands channel is full") - } - } - - context - .metrics - .node_metrics - .missing_blocks_after_fetch_total - .inc_by(missing_blocks.len() as u64); - - Ok(()) - } - - fn get_highest_accepted_rounds( - dag_state: Arc>, - context: &Arc, - ) -> Vec { - let blocks = dag_state - .read() - .get_last_cached_block_per_authority(Round::MAX); - assert_eq!(blocks.len(), context.committee.size()); - - blocks - .into_iter() - .map(|(block, _)| block.round()) - .collect::>() - } - - #[instrument(level = "trace", skip_all)] - fn verify_blocks( - serialized_blocks: Vec, - block_verifier: Arc, - verified_cache: Arc>>, - context: &Context, - peer_index: AuthorityIndex, - sync_method: &str, - ) -> ConsensusResult> { - let mut verified_blocks = Vec::new(); - let mut skipped_count = 0u64; - - for serialized_block in serialized_blocks { - let block_digest = VerifiedBlock::compute_digest(&serialized_block); - - // Check if this block has already been verified - if verified_cache.lock().get(&block_digest).is_some() { - skipped_count += 1; - continue; // Skip already verified blocks - } - - let signed_block: SignedBlock = - bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?; - - if let Err(e) = block_verifier.verify(&signed_block) { - // TODO: we might want to use a different metric to track the invalid "served" - // blocks from the invalid "proposed" ones. - let hostname = context.committee.authority(peer_index).hostname.clone(); - - context - .metrics - .node_metrics - .invalid_blocks - .with_label_values(&[hostname.as_str(), "synchronizer", e.name()]) - .inc(); - warn!("Invalid block received from {}: {}", peer_index, e); - return Err(e); - } - - // Add block to verified cache after successful verification - verified_cache.lock().put(block_digest, ()); - - let verified_block = VerifiedBlock::new_verified_with_digest( - signed_block, - serialized_block, - block_digest, - ); - - // Dropping is ok because the block will be refetched. - // TODO: improve efficiency, maybe suspend and continue processing the block - // asynchronously. - let now = context.clock.timestamp_utc_ms(); - let drift = verified_block.timestamp_ms().saturating_sub(now) as u64; - if drift > 0 { - let peer_hostname = &context - .committee - .authority(verified_block.author()) - .hostname; - context - .metrics - .node_metrics - .block_timestamp_drift_ms - .with_label_values(&[peer_hostname.as_str(), "synchronizer"]) - .inc_by(drift); - - if context - .protocol_config - .consensus_median_timestamp_with_checkpoint_enforcement() - { - trace!( - "Synced block {} timestamp {} is in the future (now={}). Will not ignore as median based timestamp is enabled.", - verified_block.reference(), - verified_block.timestamp_ms(), - now - ); - } else { - warn!( - "Synced block {} timestamp {} is in the future (now={}). Ignoring.", - verified_block.reference(), - verified_block.timestamp_ms(), - now - ); - continue; - } - } - - verified_blocks.push(verified_block); - } - - // Record skipped blocks metric - if skipped_count > 0 { - let peer_hostname = &context.committee.authority(peer_index).hostname; - context - .metrics - .node_metrics - .synchronizer_skipped_blocks_by_peer - .with_label_values(&[peer_hostname.as_str(), sync_method]) - .inc_by(skipped_count); - } - - Ok(verified_blocks) - } - - async fn fetch_blocks_request( - network_client: Arc, - peer: AuthorityIndex, - blocks_guard: BlocksGuard, - highest_rounds: Vec, - request_timeout: Duration, - mut retries: u32, - ) -> ( - ConsensusResult>, - BlocksGuard, - u32, - AuthorityIndex, - Vec, - ) { - let start = Instant::now(); - let resp = timeout( - request_timeout, - network_client.fetch_blocks( - peer, - blocks_guard - .block_refs - .clone() - .into_iter() - .collect::>(), - highest_rounds.clone(), - request_timeout, - ), - ) - .await; - - fail_point_async!("consensus-delay"); - - let resp = match resp { - Ok(Err(err)) => { - // Add a delay before retrying - if that is needed. If request has timed out - // then eventually this will be a no-op. - sleep_until(start + request_timeout).await; - retries += 1; - Err(err) - } // network error - Err(err) => { - // timeout - sleep_until(start + request_timeout).await; - retries += 1; - Err(ConsensusError::NetworkRequestTimeout(err.to_string())) - } - Ok(result) => result, - }; - (resp, blocks_guard, retries, peer, highest_rounds) - } - - fn start_fetch_own_last_block_task(&mut self) { - const FETCH_OWN_BLOCK_RETRY_DELAY: Duration = Duration::from_millis(1_000); - const MAX_RETRY_DELAY_STEP: Duration = Duration::from_millis(4_000); - - let context = self.context.clone(); - let dag_state = self.dag_state.clone(); - let network_client = self.network_client.clone(); - let block_verifier = self.block_verifier.clone(); - let core_dispatcher = self.core_dispatcher.clone(); - - self.fetch_own_last_block_task - .spawn(monitored_future!(async move { - let _scope = monitored_scope("FetchOwnLastBlockTask"); - - let fetch_own_block = |authority_index: AuthorityIndex, fetch_own_block_delay: Duration| { - let network_client_cloned = network_client.clone(); - let own_index = context.own_index; - async move { - sleep(fetch_own_block_delay).await; - let r = network_client_cloned.fetch_latest_blocks(authority_index, vec![own_index], FETCH_REQUEST_TIMEOUT).await; - (r, authority_index) - } - }; - - let process_blocks = |blocks: Vec, authority_index: AuthorityIndex| -> ConsensusResult> { - let mut result = Vec::new(); - for serialized_block in blocks { - let signed_block: SignedBlock = bcs::from_bytes(&serialized_block).map_err(ConsensusError::MalformedBlock)?; - block_verifier.verify(&signed_block).tap_err(|err|{ - let hostname = context.committee.authority(authority_index).hostname.clone(); - context - .metrics - .node_metrics - .invalid_blocks - .with_label_values(&[hostname.as_str(), "synchronizer_own_block", err.clone().name()]) - .inc(); - warn!("Invalid block received from {}: {}", authority_index, err); - })?; - - let verified_block = VerifiedBlock::new_verified(signed_block, serialized_block); - if verified_block.author() != context.own_index { - return Err(ConsensusError::UnexpectedLastOwnBlock { index: authority_index, block_ref: verified_block.reference()}); - } - result.push(verified_block); - } - Ok(result) - }; - - // Get the highest of all the results. Retry until at least `f+1` results have been gathered. - let mut highest_round = GENESIS_ROUND; - // Keep track of the received responses to avoid fetching the own block header from same peer - let mut received_response = vec![false; context.committee.size()]; - // Assume that our node is not Byzantine - received_response[context.own_index] = true; - let mut total_stake = context.committee.stake(context.own_index); - let mut retries = 0; - let mut retry_delay_step = Duration::from_millis(500); - 'main:loop { - if context.committee.size() == 1 { - highest_round = dag_state.read().get_last_proposed_block().round(); - info!("Only one node in the network, will not try fetching own last block from peers."); - break 'main; - } - - // Ask all the other peers about our last block - let mut results = FuturesUnordered::new(); - - for (authority_index, _authority) in context.committee.authorities() { - // Skip our own index and the ones that have already responded - if !received_response[authority_index] { - results.push(fetch_own_block(authority_index, Duration::from_millis(0))); - } - } - - // Gather the results but wait to timeout as well - let timer = sleep_until(Instant::now() + context.parameters.sync_last_known_own_block_timeout); - tokio::pin!(timer); - - 'inner: loop { - tokio::select! { - result = results.next() => { - let Some((result, authority_index)) = result else { - break 'inner; - }; - match result { - Ok(result) => { - match process_blocks(result, authority_index) { - Ok(blocks) => { - received_response[authority_index] = true; - let max_round = blocks.into_iter().map(|b|b.round()).max().unwrap_or(0); - highest_round = highest_round.max(max_round); - - total_stake += context.committee.stake(authority_index); - }, - Err(err) => { - warn!("Invalid result returned from {authority_index} while fetching last own block: {err}"); - } - } - }, - Err(err) => { - warn!("Error {err} while fetching our own block from peer {authority_index}. Will retry."); - results.push(fetch_own_block(authority_index, FETCH_OWN_BLOCK_RETRY_DELAY)); - } - } - }, - () = &mut timer => { - info!("Timeout while trying to sync our own last block from peers"); - break 'inner; - } - } - } - - // Request at least a quorum of 2f+1 stake to have replied back. - if context.committee.reached_quorum(total_stake) { - info!("A quorum, {} out of {} total stake, returned acceptable results for our own last block with highest round {}, with {retries} retries.", total_stake, context.committee.total_stake(), highest_round); - break 'main; - } else { - info!("Only {} out of {} total stake returned acceptable results for our own last block with highest round {}, with {retries} retries.", total_stake, context.committee.total_stake(), highest_round); - } - - retries += 1; - context.metrics.node_metrics.sync_last_known_own_block_retries.inc(); - warn!("Not enough stake: {} out of {} total stake returned acceptable results for our own last block with highest round {}. Will now retry {retries}.", total_stake, context.committee.total_stake(), highest_round); - - sleep(retry_delay_step).await; - - retry_delay_step = Duration::from_secs_f64(retry_delay_step.as_secs_f64() * 1.5); - retry_delay_step = retry_delay_step.min(MAX_RETRY_DELAY_STEP); - } - - // Update the Core with the highest detected round - context.metrics.node_metrics.last_known_own_block_round.set(highest_round as i64); - - if let Err(err) = core_dispatcher.set_last_known_proposed_round(highest_round) { - warn!("Error received while calling dispatcher, probably dispatcher is shutting down, will now exit: {err:?}"); - } - })); - } - - async fn start_fetch_missing_blocks_task(&mut self) -> ConsensusResult<()> { - let mut missing_blocks = self - .core_dispatcher - .get_missing_blocks() - .await - .map_err(|_err| ConsensusError::Shutdown)?; - - // No reason to kick off the scheduler if there are no missing blocks to fetch - if missing_blocks.is_empty() { - return Ok(()); - } - - let context = self.context.clone(); - let network_client = self.network_client.clone(); - let block_verifier = self.block_verifier.clone(); - let verified_cache = self.verified_blocks_cache.clone(); - let commit_vote_monitor = self.commit_vote_monitor.clone(); - let core_dispatcher = self.core_dispatcher.clone(); - let blocks_to_fetch = self.inflight_blocks_map.clone(); - let commands_sender = self.commands_sender.clone(); - let dag_state = self.dag_state.clone(); - - let (commit_lagging, last_commit_index, quorum_commit_index) = self.is_commit_lagging(); - trace!( - "Commit lagging: {commit_lagging}, last commit index: {last_commit_index}, quorum commit index: {quorum_commit_index}" - ); - if commit_lagging { - // If gc is enabled and we are commit lagging, then we don't want to enable the - // scheduler. As the new logic of processing the certified commits - // takes place we are guaranteed that commits will happen for all the certified - // commits. - if dag_state.read().gc_enabled() { - return Ok(()); - } - - // As node is commit lagging try to sync only the missing blocks that are within - // the acceptable round thresholds to sync. The rest we don't attempt to - // sync yet. - let highest_accepted_round = dag_state.read().highest_accepted_round(); - missing_blocks = missing_blocks - .into_iter() - .take_while(|(block_ref, _)| { - block_ref.round <= highest_accepted_round + self.missing_block_round_threshold() - }) - .collect::>(); - - // If no missing blocks are within the acceptable thresholds to sync while we - // commit lag, then we disable the scheduler completely for this run. - if missing_blocks.is_empty() { - trace!( - "Scheduled synchronizer temporarily disabled as local commit is falling behind from quorum {last_commit_index} << {quorum_commit_index} and missing blocks are too far in the future." - ); - self.context - .metrics - .node_metrics - .fetch_blocks_scheduler_skipped - .with_label_values(&["commit_lagging"]) - .inc(); - return Ok(()); - } - } - - self.fetch_blocks_scheduler_task - .spawn(monitored_future!(async move { - let _scope = monitored_scope("FetchMissingBlocksScheduler"); - - context - .metrics - .node_metrics - .fetch_blocks_scheduler_inflight - .inc(); - let total_requested = missing_blocks.len(); - - fail_point_async!("consensus-delay"); - - // Fetch blocks from peers - let results = Self::fetch_blocks_from_authorities( - context.clone(), - blocks_to_fetch.clone(), - network_client, - missing_blocks, - dag_state, - ) - .await; - context - .metrics - .node_metrics - .fetch_blocks_scheduler_inflight - .dec(); - if results.is_empty() { - warn!("No results returned while requesting missing blocks"); - return; - } - - // Now process the returned results - let mut total_fetched = 0; - for (blocks_guard, fetched_blocks, peer) in results { - total_fetched += fetched_blocks.len(); - - if let Err(err) = Self::process_fetched_blocks( - fetched_blocks, - peer, - blocks_guard, - core_dispatcher.clone(), - block_verifier.clone(), - verified_cache.clone(), - commit_vote_monitor.clone(), - context.clone(), - commands_sender.clone(), - "periodic", - ) - .await - { - warn!( - "Error occurred while processing fetched blocks from peer {peer}: {err}" - ); - } - } - - debug!( - "Total blocks requested to fetch: {}, total fetched: {}", - total_requested, total_fetched - ); - })); - Ok(()) - } - - fn is_commit_lagging(&self) -> (bool, CommitIndex, CommitIndex) { - let last_commit_index = self.dag_state.read().last_commit_index(); - let quorum_commit_index = self.commit_vote_monitor.quorum_commit_index(); - let commit_threshold = last_commit_index - + self.context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER; - ( - commit_threshold < quorum_commit_index, - last_commit_index, - quorum_commit_index, - ) - } - - /// The number of rounds above the highest accepted round to allow fetching - /// missing blocks via the periodic synchronization. Any missing blocks - /// of higher rounds are considered too far in the future to fetch. This - /// property is used only when it's detected that the node has fallen - /// behind on its commit compared to the rest of the network, - /// otherwise scheduler will attempt to fetch any missing block. - fn missing_block_round_threshold(&self) -> Round { - self.context.parameters.commit_sync_batch_size - } - - /// Fetches the given `missing_blocks` from up to `MAX_PEERS` authorities in - /// parallel: - /// - /// Randomly select `MAX_PEERS - MAX_RANDOM_PEERS` peers from those who - /// are known to hold some missing block, requesting up to - /// `MAX_BLOCKS_PER_FETCH` block refs per peer. - /// - /// Randomly select `MAX_RANDOM_PEERS` additional peers from the - /// committee (excluding self and those already selected). - /// - /// The method returns a vector with the fetched blocks from each peer that - /// successfully responded and any corresponding additional ancestor blocks. - /// Each element of the vector is a tuple which contains the requested - /// missing block refs, the returned blocks and the peer authority index. - async fn fetch_blocks_from_authorities( - context: Arc, - inflight_blocks: Arc, - network_client: Arc, - missing_blocks: BTreeMap>, - dag_state: Arc>, - ) -> Vec<(BlocksGuard, Vec, AuthorityIndex)> { - // Step 1: Map authorities to missing blocks that they are aware of - let mut authority_to_blocks: HashMap> = HashMap::new(); - for (missing_block_ref, authorities) in &missing_blocks { - for author in authorities { - if author == &context.own_index { - // Skip our own index as we don't want to fetch blocks from ourselves - continue; - } - authority_to_blocks - .entry(*author) - .or_default() - .push(*missing_block_ref); - } - } - - // Step 2: Choose at most MAX_PEERS-MAX_RANDOM_PEERS peers from those who are - // aware of some missing blocks - - #[cfg(not(test))] - let mut rng = StdRng::from_entropy(); - - // Randomly pick up MAX_PEERS - MAX_RANDOM_PEERS authorities that are aware of - // missing blocks - #[cfg(not(test))] - let mut chosen_peers_with_blocks: Vec<(AuthorityIndex, Vec, &str)> = - authority_to_blocks - .iter() - .choose_multiple( - &mut rng, - MAX_PERIODIC_SYNC_PEERS - MAX_PERIODIC_SYNC_RANDOM_PEERS, - ) - .into_iter() - .map(|(&peer, blocks)| { - let limited_blocks = blocks - .iter() - .copied() - .take(context.parameters.max_blocks_per_sync) - .collect(); - (peer, limited_blocks, "periodic_known") - }) - .collect(); - #[cfg(test)] - // Deterministically pick the smallest (MAX_PEERS - MAX_RANDOM_PEERS) authority indices - let mut chosen_peers_with_blocks: Vec<(AuthorityIndex, Vec, &str)> = { - let mut items: Vec<(AuthorityIndex, Vec, &str)> = authority_to_blocks - .iter() - .map(|(&peer, blocks)| { - let limited_blocks = blocks - .iter() - .copied() - .take(context.parameters.max_blocks_per_sync) - .collect(); - (peer, limited_blocks, "periodic_known") - }) - .collect(); - // Sort by AuthorityIndex (natural order), then take the first MAX_PEERS - - // MAX_RANDOM_PEERS - items.sort_by_key(|(peer, _, _)| *peer); - items - .into_iter() - .take(MAX_PERIODIC_SYNC_PEERS - MAX_PERIODIC_SYNC_RANDOM_PEERS) - .collect() - }; - - // Step 3: Choose at most two random peers not known to be aware of the missing - // blocks - let already_chosen: HashSet = chosen_peers_with_blocks - .iter() - .map(|(peer, _, _)| *peer) - .collect(); - - let random_candidates: Vec<_> = context - .committee - .authorities() - .filter_map(|(peer_index, _)| { - (peer_index != context.own_index && !already_chosen.contains(&peer_index)) - .then_some(peer_index) - }) - .collect(); - #[cfg(test)] - let random_peers: Vec = random_candidates - .into_iter() - .take(MAX_PERIODIC_SYNC_RANDOM_PEERS) - .collect(); - #[cfg(not(test))] - let random_peers: Vec = random_candidates - .into_iter() - .choose_multiple(&mut rng, MAX_PERIODIC_SYNC_RANDOM_PEERS); - - #[cfg_attr(test, allow(unused_mut))] - let mut all_missing_blocks: Vec = missing_blocks.keys().cloned().collect(); - // Shuffle the missing blocks in case the first ones are blocked by irresponsive - // peers - #[cfg(not(test))] - all_missing_blocks.shuffle(&mut rng); - - let mut block_chunks = all_missing_blocks.chunks(context.parameters.max_blocks_per_sync); - - for peer in random_peers { - if let Some(chunk) = block_chunks.next() { - chosen_peers_with_blocks.push((peer, chunk.to_vec(), "periodic_random")); - } else { - break; - } - } - - let mut request_futures = FuturesUnordered::new(); - - let highest_rounds = Self::get_highest_accepted_rounds(dag_state, &context); - - // Record the missing blocks per authority for metrics - let mut missing_blocks_per_authority = vec![0; context.committee.size()]; - for block in &all_missing_blocks { - missing_blocks_per_authority[block.author] += 1; - } - for (missing, (_, authority)) in missing_blocks_per_authority - .into_iter() - .zip(context.committee.authorities()) - { - context - .metrics - .node_metrics - .synchronizer_missing_blocks_by_authority - .with_label_values(&[&authority.hostname]) - .inc_by(missing as u64); - context - .metrics - .node_metrics - .synchronizer_current_missing_blocks_by_authority - .with_label_values(&[&authority.hostname]) - .set(missing as i64); - } - - // Look at peers that were not chosen yet and try to fetch blocks from them if - // needed later - #[cfg_attr(test, expect(unused_mut))] - let mut remaining_peers: Vec<_> = context - .committee - .authorities() - .filter_map(|(peer_index, _)| { - if peer_index != context.own_index - && !chosen_peers_with_blocks - .iter() - .any(|(chosen_peer, _, _)| *chosen_peer == peer_index) - { - Some(peer_index) - } else { - None - } - }) - .collect(); - - #[cfg(not(test))] - remaining_peers.shuffle(&mut rng); - let mut remaining_peers = remaining_peers.into_iter(); - - // Send the initial requests - for (peer, blocks_to_request, label) in chosen_peers_with_blocks { - let peer_hostname = &context.committee.authority(peer).hostname; - let block_refs = blocks_to_request.iter().cloned().collect::>(); - - // Lock the blocks to be fetched. If no lock can be acquired for any of the - // blocks then don't bother. - if let Some(blocks_guard) = - inflight_blocks.lock_blocks(block_refs.clone(), peer, SyncMethod::Periodic) - { - info!( - "Periodic sync of {} missing blocks from peer {} {}: {}", - block_refs.len(), - peer, - peer_hostname, - block_refs - .iter() - .map(|b| b.to_string()) - .collect::>() - .join(", ") - ); - // Record metrics about requested blocks - let metrics = &context.metrics.node_metrics; - metrics - .synchronizer_requested_blocks_by_peer - .with_label_values(&[peer_hostname.as_str(), label]) - .inc_by(block_refs.len() as u64); - for block_ref in &block_refs { - let block_hostname = &context.committee.authority(block_ref.author).hostname; - metrics - .synchronizer_requested_blocks_by_authority - .with_label_values(&[block_hostname.as_str(), label]) - .inc(); - } - request_futures.push(Self::fetch_blocks_request( - network_client.clone(), - peer, - blocks_guard, - highest_rounds.clone(), - FETCH_REQUEST_TIMEOUT, - 1, - )); - } - } - - let mut results = Vec::new(); - let fetcher_timeout = sleep(FETCH_FROM_PEERS_TIMEOUT); - - tokio::pin!(fetcher_timeout); - - loop { - tokio::select! { - Some((response, blocks_guard, _retries, peer_index, highest_rounds)) = request_futures.next() => { - let peer_hostname = &context.committee.authority(peer_index).hostname; - match response { - Ok(fetched_blocks) => { - info!("Fetched {} blocks from peer {}", fetched_blocks.len(), peer_hostname); - results.push((blocks_guard, fetched_blocks, peer_index)); - - // no more pending requests are left, just break the loop - if request_futures.is_empty() { - break; - } - }, - Err(_) => { - context.metrics.node_metrics.synchronizer_fetch_failures_by_peer.with_label_values(&[peer_hostname.as_str(), "periodic"]).inc(); - // try again if there is any peer left - if let Some(next_peer) = remaining_peers.next() { - // do best effort to lock guards. If we can't lock then don't bother at this run. - if let Some(blocks_guard) = inflight_blocks.swap_locks(blocks_guard, next_peer) { - info!( - "Retrying syncing {} missing blocks from peer {}: {}", - blocks_guard.block_refs.len(), - peer_hostname, - blocks_guard.block_refs - .iter() - .map(|b| b.to_string()) - .collect::>() - .join(", ") - ); - let block_refs = blocks_guard.block_refs.clone(); - // Record metrics about requested blocks - let metrics = &context.metrics.node_metrics; - metrics - .synchronizer_requested_blocks_by_peer - .with_label_values(&[peer_hostname.as_str(), "periodic_retry"]) - .inc_by(block_refs.len() as u64); - for block_ref in &block_refs { - let block_hostname = - &context.committee.authority(block_ref.author).hostname; - metrics - .synchronizer_requested_blocks_by_authority - .with_label_values(&[block_hostname.as_str(), "periodic_retry"]) - .inc(); - } - request_futures.push(Self::fetch_blocks_request( - network_client.clone(), - next_peer, - blocks_guard, - highest_rounds, - FETCH_REQUEST_TIMEOUT, - 1, - )); - } else { - debug!("Couldn't acquire locks to fetch blocks from peer {next_peer}.") - } - } else { - debug!("No more peers left to fetch blocks"); - } - } - } - }, - _ = &mut fetcher_timeout => { - debug!("Timed out while fetching missing blocks"); - break; - } - } - } - - results - } -} - -#[cfg(test)] -mod tests { - use std::{ - collections::{BTreeMap, BTreeSet}, - num::NonZeroUsize, - sync::Arc, - time::Duration, - }; - - use async_trait::async_trait; - use bytes::Bytes; - use consensus_config::{AuthorityIndex, Parameters}; - use iota_metrics::monitored_mpsc; - use lru::LruCache; - use parking_lot::{Mutex as SyncMutex, RwLock}; - use tokio::{sync::Mutex, time::sleep}; - - use crate::{ - CommitDigest, CommitIndex, - authority_service::COMMIT_LAG_MULTIPLIER, - block::{BlockDigest, BlockRef, Round, SignedBlock, TestBlock, VerifiedBlock}, - block_verifier::{BlockVerifier, NoopBlockVerifier}, - commit::{CertifiedCommits, CommitRange, CommitVote, TrustedCommit}, - commit_vote_monitor::CommitVoteMonitor, - context::Context, - core_thread::{CoreError, CoreThreadDispatcher, tests::MockCoreThreadDispatcher}, - dag_state::DagState, - error::{ConsensusError, ConsensusResult}, - network::{BlockStream, NetworkClient}, - round_prober::QuorumRound, - storage::mem_store::MemStore, - synchronizer::{ - FETCH_BLOCKS_CONCURRENCY, FETCH_REQUEST_TIMEOUT, InflightBlocksMap, SyncMethod, - Synchronizer, VERIFIED_BLOCKS_CACHE_CAP, - }, - }; - - type FetchRequestKey = (Vec, AuthorityIndex); - type FetchRequestResponse = (Vec, Option); - type FetchLatestBlockKey = (AuthorityIndex, Vec); - type FetchLatestBlockResponse = (Vec, Option); - - // Mock verifier that always fails verification - struct FailingBlockVerifier; - - impl BlockVerifier for FailingBlockVerifier { - fn verify(&self, _block: &SignedBlock) -> ConsensusResult<()> { - Err(ConsensusError::WrongEpoch { - expected: 1, - actual: 0, - }) - } - - fn check_ancestors( - &self, - _block: &VerifiedBlock, - _ancestors: &[Option], - _gc_enabled: bool, - _gc_round: Round, - ) -> ConsensusResult<()> { - Ok(()) - } - } - - #[derive(Default)] - struct MockNetworkClient { - fetch_blocks_requests: Mutex>, - fetch_latest_blocks_requests: - Mutex>>, - } - - impl MockNetworkClient { - async fn stub_fetch_blocks( - &self, - blocks: Vec, - peer: AuthorityIndex, - latency: Option, - ) { - let mut lock = self.fetch_blocks_requests.lock().await; - let block_refs = blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - lock.insert((block_refs, peer), (blocks, latency)); - } - - async fn stub_fetch_latest_blocks( - &self, - blocks: Vec, - peer: AuthorityIndex, - authorities: Vec, - latency: Option, - ) { - let mut lock = self.fetch_latest_blocks_requests.lock().await; - lock.entry((peer, authorities)) - .or_default() - .push((blocks, latency)); - } - - async fn fetch_latest_blocks_pending_calls(&self) -> usize { - let lock = self.fetch_latest_blocks_requests.lock().await; - lock.len() - } - } - - #[async_trait] - impl NetworkClient for MockNetworkClient { - const SUPPORT_STREAMING: bool = false; - - async fn send_block( - &self, - _peer: AuthorityIndex, - _serialized_block: &VerifiedBlock, - _timeout: Duration, - ) -> ConsensusResult<()> { - unimplemented!("Unimplemented") - } - - async fn subscribe_blocks( - &self, - _peer: AuthorityIndex, - _last_received: Round, - _timeout: Duration, - ) -> ConsensusResult { - unimplemented!("Unimplemented") - } - - async fn fetch_blocks( - &self, - peer: AuthorityIndex, - block_refs: Vec, - _highest_accepted_rounds: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - let mut lock = self.fetch_blocks_requests.lock().await; - let response = lock - .remove(&(block_refs, peer)) - .expect("Unexpected fetch blocks request made"); - - let serialised = response - .0 - .into_iter() - .map(|block| block.serialized().clone()) - .collect::>(); - - drop(lock); - - if let Some(latency) = response.1 { - sleep(latency).await; - } - - Ok(serialised) - } - - async fn fetch_commits( - &self, - _peer: AuthorityIndex, - _commit_range: CommitRange, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - - async fn fetch_latest_blocks( - &self, - peer: AuthorityIndex, - authorities: Vec, - _timeout: Duration, - ) -> ConsensusResult> { - let mut lock = self.fetch_latest_blocks_requests.lock().await; - let mut responses = lock - .remove(&(peer, authorities.clone())) - .expect("Unexpected fetch blocks request made"); - - let response = responses.remove(0); - let serialised = response - .0 - .into_iter() - .map(|block| block.serialized().clone()) - .collect::>(); - - if !responses.is_empty() { - lock.insert((peer, authorities), responses); - } - - drop(lock); - - if let Some(latency) = response.1 { - sleep(latency).await; - } - - Ok(serialised) - } - - async fn get_latest_rounds( - &self, - _peer: AuthorityIndex, - _timeout: Duration, - ) -> ConsensusResult<(Vec, Vec)> { - unimplemented!("Unimplemented") - } - } - - #[test] - fn test_inflight_blocks_map() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(12, AuthorityIndex::new_for_test(3), BlockDigest::MIN), - BlockRef::new(15, AuthorityIndex::new_for_test(2), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // Lock & unlock blocks - { - let mut all_guards = Vec::new(); - - // Try to acquire the block locks for authorities 1 & 2 (Periodic limit is 2) - for i in 1..=2 { - let authority = AuthorityIndex::new_for_test(i); - - let guard = - map.lock_blocks(missing_block_refs.clone(), authority, SyncMethod::Periodic); - let guard = guard.expect("Guard should be created"); - assert_eq!(guard.block_refs.len(), 4); - - all_guards.push(guard); - - // trying to acquire any of them again will not succeed - let guard = - map.lock_blocks(missing_block_refs.clone(), authority, SyncMethod::Periodic); - assert!(guard.is_none()); - } - - // Trying to acquire for authority 3 it will fail - as we have maxed out the - // number of allowed peers (Periodic limit is 2) - let authority_3 = AuthorityIndex::new_for_test(3); - - let guard = map.lock_blocks( - missing_block_refs.clone(), - authority_3, - SyncMethod::Periodic, - ); - assert!(guard.is_none()); - - // Explicitly drop the guard of authority 1 and try for authority 3 again - it - // will now succeed - drop(all_guards.remove(0)); - - let guard = map.lock_blocks( - missing_block_refs.clone(), - authority_3, - SyncMethod::Periodic, - ); - let guard = guard.expect("Guard should be successfully acquired"); - - assert_eq!(guard.block_refs, missing_block_refs); - - // Dropping all guards should unlock on the block refs - drop(guard); - drop(all_guards); - - assert_eq!(map.num_of_locked_blocks(), 0); - } - - // Swap locks - { - // acquire a lock for authority 1 - let authority_1 = AuthorityIndex::new_for_test(1); - let guard = map - .lock_blocks( - missing_block_refs.clone(), - authority_1, - SyncMethod::Periodic, - ) - .unwrap(); - - // Now swap the locks for authority 2 - let authority_2 = AuthorityIndex::new_for_test(2); - let guard = map.swap_locks(guard, authority_2); - - assert_eq!(guard.unwrap().block_refs, missing_block_refs); - } - } - - #[test] - fn test_inflight_blocks_map_live_sync_limit() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // WHEN authority 1 locks with Live sync - let authority_1 = AuthorityIndex::new_for_test(1); - let guard_1 = map - .lock_blocks(missing_block_refs.clone(), authority_1, SyncMethod::Live) - .expect("Should successfully lock with Live sync"); - - assert_eq!(guard_1.block_refs.len(), 2); - - // THEN authority 2 cannot lock with Live sync (limit of 1 reached) - let authority_2 = AuthorityIndex::new_for_test(2); - let guard_2 = map.lock_blocks(missing_block_refs.clone(), authority_2, SyncMethod::Live); - - assert!( - guard_2.is_none(), - "Should fail to lock - Live limit of 1 reached" - ); - - // WHEN authority 1 releases the lock - drop(guard_1); - - // THEN authority 2 can now lock with Live sync - let guard_2 = map - .lock_blocks(missing_block_refs, authority_2, SyncMethod::Live) - .expect("Should successfully lock after authority 1 released"); - - assert_eq!(guard_2.block_refs.len(), 2); - } - - #[test] - fn test_inflight_blocks_map_periodic_allows_more_concurrency() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // WHEN authority 1 locks with Periodic sync - let authority_1 = AuthorityIndex::new_for_test(1); - let guard_1 = map - .lock_blocks( - missing_block_refs.clone(), - authority_1, - SyncMethod::Periodic, - ) - .expect("Should successfully lock with Periodic sync"); - - assert_eq!(guard_1.block_refs.len(), 2); - - // THEN authority 2 can also lock with Periodic sync (limit is 2) - let authority_2 = AuthorityIndex::new_for_test(2); - let guard_2 = map - .lock_blocks( - missing_block_refs.clone(), - authority_2, - SyncMethod::Periodic, - ) - .expect("Should successfully lock - Periodic allows 2 authorities"); - - assert_eq!(guard_2.block_refs.len(), 2); - - // BUT authority 3 cannot lock with Periodic sync (limit of 2 reached) - let authority_3 = AuthorityIndex::new_for_test(3); - let guard_3 = map.lock_blocks( - missing_block_refs.clone(), - authority_3, - SyncMethod::Periodic, - ); - - assert!( - guard_3.is_none(), - "Should fail to lock - Periodic limit of 2 reached" - ); - - // WHEN authority 1 releases the lock - drop(guard_1); - - // THEN authority 3 can now lock with Periodic sync - let guard_3 = map - .lock_blocks(missing_block_refs, authority_3, SyncMethod::Periodic) - .expect("Should successfully lock after authority 1 released"); - - assert_eq!(guard_3.block_refs.len(), 2); - } - - #[test] - fn test_inflight_blocks_map_periodic_blocks_live_when_at_live_limit() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // WHEN authority 1 locks with Periodic sync (total=1, at Live's limit) - let authority_1 = AuthorityIndex::new_for_test(1); - let guard_1 = map - .lock_blocks( - missing_block_refs.clone(), - authority_1, - SyncMethod::Periodic, - ) - .expect("Should successfully lock with Periodic sync"); - - assert_eq!(guard_1.block_refs.len(), 2); - - // THEN authority 2 cannot lock with Live sync (total already at Live limit of - // 1) - let authority_2 = AuthorityIndex::new_for_test(2); - let guard_2_live = - map.lock_blocks(missing_block_refs.clone(), authority_2, SyncMethod::Live); - - assert!( - guard_2_live.is_none(), - "Should fail to lock with Live - total already at Live limit of 1" - ); - - // BUT authority 2 CAN lock with Periodic sync (total would be 2, at Periodic - // limit) - let guard_2_periodic = map - .lock_blocks(missing_block_refs, authority_2, SyncMethod::Periodic) - .expect("Should successfully lock with Periodic - under Periodic limit of 2"); - - assert_eq!(guard_2_periodic.block_refs.len(), 2); - } - - #[test] - fn test_inflight_blocks_map_live_then_periodic_interaction() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // WHEN authority 1 locks with Live sync (total=1, at Live limit) - let authority_1 = AuthorityIndex::new_for_test(1); - let guard_1 = map - .lock_blocks(missing_block_refs.clone(), authority_1, SyncMethod::Live) - .expect("Should successfully lock with Live sync"); - - assert_eq!(guard_1.block_refs.len(), 2); - - // THEN authority 2 cannot lock with Live sync (would exceed Live limit of 1) - let authority_2 = AuthorityIndex::new_for_test(2); - let guard_2_live = - map.lock_blocks(missing_block_refs.clone(), authority_2, SyncMethod::Live); - - assert!( - guard_2_live.is_none(), - "Should fail to lock with Live - would exceed Live limit of 1" - ); - - // BUT authority 2 CAN lock with Periodic sync (total=2, at Periodic limit) - let guard_2 = map - .lock_blocks( - missing_block_refs.clone(), - authority_2, - SyncMethod::Periodic, - ) - .expect("Should successfully lock with Periodic - total 2 is at Periodic limit"); - - assert_eq!(guard_2.block_refs.len(), 2); - - // AND authority 3 cannot lock with Periodic sync (would exceed Periodic limit - // of 2) - let authority_3 = AuthorityIndex::new_for_test(3); - let guard_3 = map.lock_blocks(missing_block_refs, authority_3, SyncMethod::Periodic); - - assert!( - guard_3.is_none(), - "Should fail to lock with Periodic - would exceed Periodic limit of 2" - ); - } - - #[test] - fn test_inflight_blocks_map_partial_locks_mixed_methods() { - // GIVEN 4 blocks - let map = InflightBlocksMap::new(); - let block_a = BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN); - let block_b = BlockRef::new(2, AuthorityIndex::new_for_test(0), BlockDigest::MIN); - let block_c = BlockRef::new(3, AuthorityIndex::new_for_test(0), BlockDigest::MIN); - let block_d = BlockRef::new(4, AuthorityIndex::new_for_test(0), BlockDigest::MIN); - - // Lock block A with authority 1 using Live (A at limit for Live) - let guard_a = map - .lock_blocks( - [block_a].into(), - AuthorityIndex::new_for_test(1), - SyncMethod::Live, - ) - .expect("Should lock block A"); - assert_eq!(guard_a.block_refs.len(), 1); - - // Lock block B with authorities 1 & 2 using Periodic (B at limit for Periodic) - let guard_b1 = map - .lock_blocks( - [block_b].into(), - AuthorityIndex::new_for_test(1), - SyncMethod::Periodic, - ) - .expect("Should lock block B"); - let guard_b2 = map - .lock_blocks( - [block_b].into(), - AuthorityIndex::new_for_test(2), - SyncMethod::Periodic, - ) - .expect("Should lock block B again"); - assert_eq!(guard_b1.block_refs.len(), 1); - assert_eq!(guard_b2.block_refs.len(), 1); - - // Lock block C with authority 1 using Periodic (C has 1 lock) - let guard_c = map - .lock_blocks( - [block_c].into(), - AuthorityIndex::new_for_test(1), - SyncMethod::Periodic, - ) - .expect("Should lock block C"); - assert_eq!(guard_c.block_refs.len(), 1); - - // Block D is unlocked - - // WHEN authority 3 requests all 4 blocks with Periodic - let all_blocks = [block_a, block_b, block_c, block_d].into(); - let guard_3 = map - .lock_blocks( - all_blocks, - AuthorityIndex::new_for_test(3), - SyncMethod::Periodic, - ) - .expect("Should get partial lock"); - - // THEN should successfully lock C and D only - // - A: total=1 (at Live limit), authority 3 can still add since using Periodic - // and total < 2 - // - B: total=2 (at Periodic limit), cannot lock - // - C: total=1, can lock (under limit) - // - D: total=0, can lock - assert_eq!( - guard_3.block_refs.len(), - 3, - "Should lock blocks A, C, and D" - ); - assert!( - guard_3.block_refs.contains(&block_a), - "Should contain block A" - ); - assert!( - !guard_3.block_refs.contains(&block_b), - "Should NOT contain block B (at limit)" - ); - assert!( - guard_3.block_refs.contains(&block_c), - "Should contain block C" - ); - assert!( - guard_3.block_refs.contains(&block_d), - "Should contain block D" - ); - } - - #[test] - fn test_inflight_blocks_map_swap_locks_preserves_method() { - // GIVEN - let map = InflightBlocksMap::new(); - let some_block_refs = [ - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ]; - let missing_block_refs = some_block_refs.iter().cloned().collect::>(); - - // WHEN authority 1 locks with Live sync - let authority_1 = AuthorityIndex::new_for_test(1); - let guard_1 = map - .lock_blocks(missing_block_refs.clone(), authority_1, SyncMethod::Live) - .expect("Should lock with Live sync"); - - assert_eq!(guard_1.block_refs.len(), 2); - - // AND we swap to authority 2 - let authority_2 = AuthorityIndex::new_for_test(2); - let guard_2 = map - .swap_locks(guard_1, authority_2) - .expect("Should swap locks"); - - // THEN the new guard should preserve the block refs - assert_eq!(guard_2.block_refs, missing_block_refs); - - // AND authority 3 cannot lock with Live sync (limit of 1 reached) - let authority_3 = AuthorityIndex::new_for_test(3); - let guard_3 = map.lock_blocks(missing_block_refs.clone(), authority_3, SyncMethod::Live); - assert!(guard_3.is_none(), "Should fail - Live limit reached"); - - // BUT authority 3 CAN lock with Periodic sync - let guard_3_periodic = map - .lock_blocks(missing_block_refs, authority_3, SyncMethod::Periodic) - .expect("Should lock with Periodic"); - assert_eq!(guard_3_periodic.block_refs.len(), 2); - } - - #[tokio::test] - async fn test_process_fetched_blocks() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let (commands_sender, _commands_receiver) = - monitored_mpsc::channel("consensus_synchronizer_commands", 1000); - - // Create input test blocks: - // - Authority 0 block at round 60. - // - Authority 1 blocks from round 30 to 93. - let mut expected_blocks = vec![VerifiedBlock::new_for_test(TestBlock::new(60, 0).build())]; - expected_blocks.extend( - (30..=60).map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 1).build())), - ); - assert_eq!( - expected_blocks.len(), - context.parameters.max_blocks_per_sync - ); - - let expected_serialized_blocks = expected_blocks - .iter() - .map(|b| b.serialized().clone()) - .collect::>(); - - let expected_block_refs = expected_blocks - .iter() - .map(|b| b.reference()) - .collect::>(); - - // GIVEN peer to fetch blocks from - let peer_index = AuthorityIndex::new_for_test(2); - - // Create blocks_guard - let inflight_blocks_map = InflightBlocksMap::new(); - let blocks_guard = inflight_blocks_map - .lock_blocks(expected_block_refs.clone(), peer_index, SyncMethod::Live) - .expect("Failed to lock blocks"); - - assert_eq!( - inflight_blocks_map.num_of_locked_blocks(), - expected_block_refs.len() - ); - - // Create a Synchronizer - let verified_cache = Arc::new(SyncMutex::new(LruCache::new( - NonZeroUsize::new(VERIFIED_BLOCKS_CACHE_CAP).unwrap(), - ))); - let result = Synchronizer::< - MockNetworkClient, - NoopBlockVerifier, - MockCoreThreadDispatcher, - >::process_fetched_blocks( - expected_serialized_blocks, - peer_index, - blocks_guard, // The guard is consumed here - core_dispatcher.clone(), - block_verifier, - verified_cache, - commit_vote_monitor, - context.clone(), - commands_sender, - "test", - ) - .await; - - // THEN - assert!(result.is_ok()); - - // Check blocks were sent to core - let added_blocks = core_dispatcher.get_add_blocks().await; - assert_eq!( - added_blocks - .iter() - .map(|b| b.reference()) - .collect::>(), - expected_block_refs, - ); - - // Check blocks were unlocked - assert_eq!(inflight_blocks_map.num_of_locked_blocks(), 0); - } - - #[tokio::test] - async fn test_process_fetched_blocks_duplicates() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let (commands_sender, _commands_receiver) = - monitored_mpsc::channel("consensus_synchronizer_commands", 1000); - - // Create input test blocks: - // - Authority 0 block at round 60. - // - Authority 1 blocks from round 30 to 60. - let mut expected_blocks = vec![VerifiedBlock::new_for_test(TestBlock::new(60, 0).build())]; - expected_blocks.extend( - (30..=60).map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 1).build())), - ); - assert_eq!( - expected_blocks.len(), - context.parameters.max_blocks_per_sync - ); - - let expected_serialized_blocks = expected_blocks - .iter() - .map(|b| b.serialized().clone()) - .collect::>(); - - let expected_block_refs = expected_blocks - .iter() - .map(|b| b.reference()) - .collect::>(); - - // GIVEN peer to fetch blocks from - let peer_index = AuthorityIndex::new_for_test(2); - - // Create blocks_guard - let inflight_blocks_map = InflightBlocksMap::new(); - let blocks_guard = inflight_blocks_map - .lock_blocks(expected_block_refs.clone(), peer_index, SyncMethod::Live) - .expect("Failed to lock blocks"); - - assert_eq!( - inflight_blocks_map.num_of_locked_blocks(), - expected_block_refs.len() - ); - - // Create a shared LruCache that will be reused to verify duplicate prevention - let verified_cache = Arc::new(SyncMutex::new(LruCache::new( - NonZeroUsize::new(VERIFIED_BLOCKS_CACHE_CAP).unwrap(), - ))); - - // WHEN process fetched blocks for the first time - let result = Synchronizer::< - MockNetworkClient, - NoopBlockVerifier, - MockCoreThreadDispatcher, - >::process_fetched_blocks( - expected_serialized_blocks.clone(), - peer_index, - blocks_guard, - core_dispatcher.clone(), - block_verifier.clone(), - verified_cache.clone(), - commit_vote_monitor.clone(), - context.clone(), - commands_sender.clone(), - "test", - ) - .await; - - // THEN - assert!(result.is_ok()); - - // Check blocks were sent to core - let added_blocks = core_dispatcher.get_add_blocks().await; - assert_eq!( - added_blocks - .iter() - .map(|b| b.reference()) - .collect::>(), - expected_block_refs, - ); - - // Check blocks were unlocked - assert_eq!(inflight_blocks_map.num_of_locked_blocks(), 0); - - // PART 2: Verify LruCache prevents duplicate processing - // Try to process the same blocks again (simulating duplicate fetch) - let blocks_guard_second = inflight_blocks_map - .lock_blocks(expected_block_refs.clone(), peer_index, SyncMethod::Live) - .expect("Failed to lock blocks for second call"); - - let result_second = Synchronizer::< - MockNetworkClient, - NoopBlockVerifier, - MockCoreThreadDispatcher, - >::process_fetched_blocks( - expected_serialized_blocks, - peer_index, - blocks_guard_second, - core_dispatcher.clone(), - block_verifier, - verified_cache.clone(), - commit_vote_monitor, - context.clone(), - commands_sender, - "test", - ) - .await; - - assert!(result_second.is_ok()); - - // Verify NO blocks were sent to core on the second call - // because they were already in the LruCache - let added_blocks_second_call = core_dispatcher.get_add_blocks().await; - assert!( - added_blocks_second_call.is_empty(), - "Expected no blocks to be added on second call due to LruCache, but got {} blocks", - added_blocks_second_call.len() - ); - - // Verify the cache contains all the block digests - let cache_size = verified_cache.lock().len(); - assert_eq!( - cache_size, - expected_block_refs.len(), - "Expected {} entries in the LruCache, but got {}", - expected_block_refs.len(), - cache_size - ); - } - - #[tokio::test] - async fn test_successful_fetch_blocks_from_peer() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let network_client = Arc::new(MockNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let handle = Synchronizer::start( - network_client.clone(), - context, - core_dispatcher.clone(), - commit_vote_monitor, - block_verifier, - dag_state, - false, - ); - - // Create some test blocks - let expected_blocks = (0..10) - .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build())) - .collect::>(); - let missing_blocks = expected_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - - // AND stub the fetch_blocks request from peer 1 - let peer = AuthorityIndex::new_for_test(1); - network_client - .stub_fetch_blocks(expected_blocks.clone(), peer, None) - .await; - - // WHEN request missing blocks from peer 1 - assert!(handle.fetch_blocks(missing_blocks, peer).await.is_ok()); - - // Wait a little bit until those have been added in core - sleep(Duration::from_millis(1_000)).await; - - // THEN ensure those ended up in Core - let added_blocks = core_dispatcher.get_add_blocks().await; - assert_eq!(added_blocks, expected_blocks); - } - - #[tokio::test] - async fn saturate_fetch_blocks_from_peer() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let network_client = Arc::new(MockNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - let handle = Synchronizer::start( - network_client.clone(), - context, - core_dispatcher.clone(), - commit_vote_monitor, - block_verifier, - dag_state, - false, - ); - - // Create some test blocks - let expected_blocks = (0..=2 * FETCH_BLOCKS_CONCURRENCY) - .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round as Round, 0).build())) - .collect::>(); - - // Now start sending requests to fetch blocks by trying to saturate peer 1 task - let peer = AuthorityIndex::new_for_test(1); - let mut iter = expected_blocks.iter().peekable(); - while let Some(block) = iter.next() { - // stub the fetch_blocks request from peer 1 and give some high response latency - // so requests can start blocking the peer task. - network_client - .stub_fetch_blocks( - vec![block.clone()], - peer, - Some(Duration::from_millis(5_000)), - ) - .await; - - let mut missing_blocks = BTreeSet::new(); - missing_blocks.insert(block.reference()); - - // WHEN requesting to fetch the blocks, it should not succeed for the last - // request and get an error with "saturated" synchronizer - if iter.peek().is_none() { - match handle.fetch_blocks(missing_blocks, peer).await { - Err(ConsensusError::SynchronizerSaturated(index, _)) => { - assert_eq!(index, peer); - } - _ => panic!("A saturated synchronizer error was expected"), - } - } else { - assert!(handle.fetch_blocks(missing_blocks, peer).await.is_ok()); - } - } - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn synchronizer_periodic_task_fetch_blocks() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let network_client = Arc::new(MockNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - - // Create some test blocks - let expected_blocks = (0..10) - .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build())) - .collect::>(); - let missing_blocks = expected_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - - // AND stub the missing blocks - core_dispatcher - .stub_missing_blocks(missing_blocks.clone()) - .await; - - // AND stub the requests for authority 1 & 2 - // Make the first authority timeout, so the second will be called. "We" are - // authority = 0, so we are skipped anyways. - network_client - .stub_fetch_blocks( - expected_blocks.clone(), - AuthorityIndex::new_for_test(1), - Some(FETCH_REQUEST_TIMEOUT), - ) - .await; - network_client - .stub_fetch_blocks( - expected_blocks.clone(), - AuthorityIndex::new_for_test(2), - None, - ) - .await; - - // WHEN start the synchronizer and wait for a couple of seconds - let _handle = Synchronizer::start( - network_client.clone(), - context, - core_dispatcher.clone(), - commit_vote_monitor, - block_verifier, - dag_state, - false, - ); - - sleep(8 * FETCH_REQUEST_TIMEOUT).await; - - // THEN the missing blocks should now be fetched and added to core - let added_blocks = core_dispatcher.get_add_blocks().await; - assert_eq!(added_blocks, expected_blocks); - - // AND missing blocks should have been consumed by the stub - assert!( - core_dispatcher - .get_missing_blocks() - .await - .unwrap() - .is_empty() - ); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn synchronizer_periodic_task_when_commit_lagging_gets_disabled() { - // GIVEN - let (mut context, _) = Context::new_for_test(4); - context - .protocol_config - .set_consensus_batched_block_sync_for_testing(true); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let network_client = Arc::new(MockNetworkClient::default()); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - - // AND stub some missing blocks. The highest accepted round is 0. Create blocks - // that are above the sync threshold. - let sync_missing_block_round_threshold = context.parameters.commit_sync_batch_size; - let stub_blocks = (sync_missing_block_round_threshold * 2 - ..sync_missing_block_round_threshold * 2 - + context.parameters.max_blocks_per_sync as u32) - .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build())) - .collect::>(); - let missing_blocks = stub_blocks - .iter() - .map(|block| block.reference()) - .collect::>(); - core_dispatcher - .stub_missing_blocks(missing_blocks.clone()) - .await; - // AND stub the requests for authority 1 & 2 - // Make the first authority timeout, so the second will be called. "We" are - // authority = 0, so we are skipped anyways. - let mut expected_blocks = stub_blocks - .iter() - .take(context.parameters.max_blocks_per_sync) - .cloned() - .collect::>(); - network_client - .stub_fetch_blocks( - expected_blocks.clone(), - AuthorityIndex::new_for_test(1), - Some(FETCH_REQUEST_TIMEOUT), - ) - .await; - network_client - .stub_fetch_blocks( - expected_blocks.clone(), - AuthorityIndex::new_for_test(2), - None, - ) - .await; - - // Now create some blocks to simulate a commit lag - let round = context.parameters.commit_sync_batch_size * COMMIT_LAG_MULTIPLIER * 2; - let commit_index: CommitIndex = round - 1; - let blocks = (0..4) - .map(|authority| { - let commit_votes = vec![CommitVote::new(commit_index, CommitDigest::MIN)]; - let block = TestBlock::new(round, authority) - .set_commit_votes(commit_votes) - .build(); - - VerifiedBlock::new_for_test(block) - }) - .collect::>(); - - // Pass them through the commit vote monitor - so now there will be a big commit - // lag to prevent the scheduled synchronizer from running - for block in blocks { - commit_vote_monitor.observe_block(&block); - } - - // Start the synchronizer and wait for a couple of seconds where normally - // the synchronizer should have kicked in. - let _handle = Synchronizer::start( - network_client.clone(), - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor.clone(), - block_verifier, - dag_state.clone(), - false, - ); - - sleep(4 * FETCH_REQUEST_TIMEOUT).await; - - // Since we should be in commit lag mode none of the missed blocks should have - // been fetched - hence nothing should be sent to core for processing. - let added_blocks = core_dispatcher.get_add_blocks().await; - assert_eq!(added_blocks, vec![]); - - println!("Before advancing"); - // AND advance now the local commit index by adding a new commit that matches - // the commit index of quorum - { - let mut d = dag_state.write(); - for index in 1..=commit_index { - let commit = - TrustedCommit::new_for_test(index, CommitDigest::MIN, 0, BlockRef::MIN, vec![]); - - d.add_commit(commit); - } - - println!("Once advanced"); - assert_eq!( - d.last_commit_index(), - commit_vote_monitor.quorum_commit_index() - ); - } - - // Now stub again the missing blocks to fetch the exact same ones. - core_dispatcher - .stub_missing_blocks(missing_blocks.clone()) - .await; - - println!("Final sleep"); - sleep(2 * FETCH_REQUEST_TIMEOUT).await; - - // THEN the missing blocks should now be fetched and added to core - let mut added_blocks = core_dispatcher.get_add_blocks().await; - println!("Final await"); - added_blocks.sort_by_key(|block| block.reference()); - expected_blocks.sort_by_key(|block| block.reference()); - - assert_eq!(added_blocks, expected_blocks); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn synchronizer_fetch_own_last_block() { - // GIVEN - let (context, _) = Context::new_for_test(4); - let context = Arc::new(context.with_parameters(Parameters { - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - })); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let core_dispatcher = Arc::new(MockCoreThreadDispatcher::default()); - let network_client = Arc::new(MockNetworkClient::default()); - let commit_vote_monitor = Arc::new(CommitVoteMonitor::new(context.clone())); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let our_index = AuthorityIndex::new_for_test(0); - - // Create some test blocks - let mut expected_blocks = (8..=10) - .map(|round| VerifiedBlock::new_for_test(TestBlock::new(round, 0).build())) - .collect::>(); - - // Now set different latest blocks for the peers - // For peer 1 we give the block of round 10 (highest) - let block_1 = expected_blocks.pop().unwrap(); - network_client - .stub_fetch_latest_blocks( - vec![block_1.clone()], - AuthorityIndex::new_for_test(1), - vec![our_index], - Some(Duration::from_secs(10)), - ) - .await; - network_client - .stub_fetch_latest_blocks( - vec![block_1], - AuthorityIndex::new_for_test(1), - vec![our_index], - None, - ) - .await; - - // For peer 2 we give the block of round 9 - let block_2 = expected_blocks.pop().unwrap(); - network_client - .stub_fetch_latest_blocks( - vec![block_2.clone()], - AuthorityIndex::new_for_test(2), - vec![our_index], - Some(Duration::from_secs(10)), - ) - .await; - network_client - .stub_fetch_latest_blocks( - vec![block_2], - AuthorityIndex::new_for_test(2), - vec![our_index], - None, - ) - .await; - - // For peer 3 we give a block with lowest round - let block_3 = expected_blocks.pop().unwrap(); - network_client - .stub_fetch_latest_blocks( - vec![block_3.clone()], - AuthorityIndex::new_for_test(3), - vec![our_index], - Some(Duration::from_secs(10)), - ) - .await; - network_client - .stub_fetch_latest_blocks( - vec![block_3], - AuthorityIndex::new_for_test(3), - vec![our_index], - None, - ) - .await; - - // WHEN start the synchronizer and wait for a couple of seconds - let handle = Synchronizer::start( - network_client.clone(), - context.clone(), - core_dispatcher.clone(), - commit_vote_monitor, - block_verifier, - dag_state, - true, - ); - - // Wait at least for the timeout time - sleep(context.parameters.sync_last_known_own_block_timeout * 2).await; - - // Assert that core has been called to set the min propose round - assert_eq!( - core_dispatcher.get_last_own_proposed_round().await, - vec![10] - ); - - // Ensure that all the requests have been called - assert_eq!(network_client.fetch_latest_blocks_pending_calls().await, 0); - - // And we got one retry - assert_eq!( - context - .metrics - .node_metrics - .sync_last_known_own_block_retries - .get(), - 1 - ); - - // Ensure that no panic occurred - if let Err(err) = handle.stop().await { - if err.is_panic() { - std::panic::resume_unwind(err.into_panic()); - } - } - } - #[derive(Default)] - struct SyncMockDispatcher { - missing_blocks: Mutex>>, - added_blocks: Mutex>, - } - - #[async_trait::async_trait] - impl CoreThreadDispatcher for SyncMockDispatcher { - async fn get_missing_blocks( - &self, - ) -> Result>, CoreError> { - Ok(self.missing_blocks.lock().await.clone()) - } - async fn add_blocks( - &self, - blocks: Vec, - ) -> Result, CoreError> { - let mut guard = self.added_blocks.lock().await; - guard.extend(blocks.clone()); - Ok(blocks.iter().map(|b| b.reference()).collect()) - } - - // Stub out the remaining CoreThreadDispatcher methods with defaults: - - async fn check_block_refs( - &self, - block_refs: Vec, - ) -> Result, CoreError> { - // Echo back the requested refs by default - Ok(block_refs.into_iter().collect()) - } - - async fn add_certified_commits( - &self, - _commits: CertifiedCommits, - ) -> Result, CoreError> { - // No additional certified-commit logic in tests - Ok(BTreeSet::new()) - } - - async fn new_block(&self, _round: Round, _force: bool) -> Result<(), CoreError> { - Ok(()) - } - - fn set_quorum_subscribers_exists(&self, _exists: bool) -> Result<(), CoreError> { - Ok(()) - } - - fn set_propagation_delay_and_quorum_rounds( - &self, - _delay: Round, - _received_quorum_rounds: Vec, - _accepted_quorum_rounds: Vec, - ) -> Result<(), CoreError> { - Ok(()) - } - - fn set_last_known_proposed_round(&self, _round: Round) -> Result<(), CoreError> { - Ok(()) - } - - fn highest_received_rounds(&self) -> Vec { - Vec::new() - } - } - - #[tokio::test(flavor = "current_thread")] - async fn known_before_random_peer_fetch() { - { - // 1) Setup 10‐node context and in‐mem DAG - let (ctx, _) = Context::new_for_test(10); - let context = Arc::new(ctx); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let inflight = InflightBlocksMap::new(); - - // 2) One missing block - let missing_vb = VerifiedBlock::new_for_test(TestBlock::new(100, 3).build()); - let missing_ref = missing_vb.reference(); - let missing_blocks = BTreeMap::from([( - missing_ref, - BTreeSet::from([ - AuthorityIndex::new_for_test(2), - AuthorityIndex::new_for_test(3), - AuthorityIndex::new_for_test(4), - ]), - )]); - - // 3) Prepare mocks and stubs - let network_client = Arc::new(MockNetworkClient::default()); - // Stub *all* authorities so none panic: - for i in 1..=9 { - let peer = AuthorityIndex::new_for_test(i); - if i == 1 || i == 4 { - network_client - .stub_fetch_blocks( - vec![missing_vb.clone()], - peer, - Some(2 * FETCH_REQUEST_TIMEOUT), - ) - .await; - continue; - } - network_client - .stub_fetch_blocks(vec![missing_vb.clone()], peer, None) - .await; - } - - // 4) Invoke knowledge-based fetch and random fallback selection - // deterministically - let results = Synchronizer:: - ::fetch_blocks_from_authorities( - context.clone(), - inflight.clone(), - network_client.clone(), - missing_blocks, - dag_state.clone(), - ) - .await; - - // 5) Assert we got exactly two fetches - two from the first two who are aware - // of the missing block (authority 2 and 3) - assert_eq!(results.len(), 2); - - // 6) The knowledge-based‐fetch went to peer 2 and 3 - let (_hot_guard, hot_bytes, hot_peer) = &results[0]; - assert_eq!(*hot_peer, AuthorityIndex::new_for_test(2)); - let (_periodic_guard, _periodic_bytes, periodic_peer) = &results[1]; - assert_eq!(*periodic_peer, AuthorityIndex::new_for_test(3)); - // 7) Verify the returned bytes correspond to that block - let expected = missing_vb.serialized().clone(); - assert_eq!(hot_bytes, &vec![expected]); - } - } - - #[tokio::test(flavor = "current_thread")] - async fn known_before_periodic_peer_fetch_larger_scenario() { - use std::{ - collections::{BTreeMap, BTreeSet}, - sync::Arc, - }; - - use parking_lot::RwLock; - - use crate::{ - block::{Round, TestBlock, VerifiedBlock}, - context::Context, - }; - - // 1) Setup a 10-node context, in-memory DAG, and inflight map - let (ctx, _) = Context::new_for_test(10); - let context = Arc::new(ctx); - let store = Arc::new(MemStore::new()); - let dag_state = Arc::new(RwLock::new(DagState::new(context.clone(), store))); - let inflight = InflightBlocksMap::new(); - let network_client = Arc::new(MockNetworkClient::default()); - - // 2) Create 1000 missing blocks known by authorities 0, 2, and 3 - let mut missing_blocks = BTreeMap::new(); - let mut missing_vbs = Vec::new(); - let known_number_blocks = 10; - for i in 0..1000 { - let vb = VerifiedBlock::new_for_test(TestBlock::new(1000 + i as Round, 0).build()); - let r = vb.reference(); - if i < known_number_blocks { - // First 10 blocks are known by authorities 0, 2 - missing_blocks.insert( - r, - BTreeSet::from([ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(2), - ]), - ); - } else if i >= known_number_blocks && i < 2 * known_number_blocks { - // Second 10 blocks are known by authorities 0, 3 - missing_blocks.insert( - r, - BTreeSet::from([ - AuthorityIndex::new_for_test(0), - AuthorityIndex::new_for_test(3), - ]), - ); - } else { - // The rest are only known by authority 0 - missing_blocks.insert(r, BTreeSet::from([AuthorityIndex::new_for_test(0)])); - } - missing_vbs.push(vb); - } - - // 3) Stub fetches for knowledge-based peers (2 and 3) - let known_peers = [2, 3].map(AuthorityIndex::new_for_test); - let known_vbs_by_peer: Vec<(AuthorityIndex, Vec)> = known_peers - .iter() - .map(|&peer| { - let vbs = missing_vbs - .iter() - .filter(|vb| missing_blocks.get(&vb.reference()).unwrap().contains(&peer)) - .take(context.parameters.max_blocks_per_sync) - .cloned() - .collect::>(); - (peer, vbs) - }) - .collect(); - - for (peer, vbs) in known_vbs_by_peer { - if peer == AuthorityIndex::new_for_test(2) { - // Simulate timeout for peer 2, then fallback to peer 5 - network_client - .stub_fetch_blocks(vbs.clone(), peer, Some(2 * FETCH_REQUEST_TIMEOUT)) - .await; - network_client - .stub_fetch_blocks(vbs.clone(), AuthorityIndex::new_for_test(5), None) - .await; - } else { - network_client - .stub_fetch_blocks(vbs.clone(), peer, None) - .await; - } - } - - // 4) Stub fetches from periodic path peers (1 and 4) - network_client - .stub_fetch_blocks( - missing_vbs[0..context.parameters.max_blocks_per_sync].to_vec(), - AuthorityIndex::new_for_test(1), - None, - ) - .await; - - network_client - .stub_fetch_blocks( - missing_vbs[context.parameters.max_blocks_per_sync - ..2 * context.parameters.max_blocks_per_sync] - .to_vec(), - AuthorityIndex::new_for_test(4), - None, - ) - .await; - - // 5) Execute the fetch logic - let results = Synchronizer::< - MockNetworkClient, - NoopBlockVerifier, - SyncMockDispatcher, - >::fetch_blocks_from_authorities( - context.clone(), - inflight.clone(), - network_client.clone(), - missing_blocks, - dag_state.clone(), - ) - .await; - - // 6) Assert we got 4 fetches: peer 2 (timed out), fallback to 5, then periodic - // from 1 and 4 - assert_eq!(results.len(), 4, "Expected 2 known + 2 random fetches"); - - // 7) First fetch from peer 3 (knowledge-based) - let (_guard3, bytes3, peer3) = &results[0]; - assert_eq!(*peer3, AuthorityIndex::new_for_test(3)); - let expected2 = missing_vbs[known_number_blocks..2 * known_number_blocks] - .iter() - .map(|vb| vb.serialized().clone()) - .collect::>(); - assert_eq!(bytes3, &expected2); - - // 8) Second fetch from peer 1 (periodic) - let (_guard1, bytes1, peer1) = &results[1]; - assert_eq!(*peer1, AuthorityIndex::new_for_test(1)); - let expected1 = missing_vbs[0..context.parameters.max_blocks_per_sync] - .iter() - .map(|vb| vb.serialized().clone()) - .collect::>(); - assert_eq!(bytes1, &expected1); - - // 9) Third fetch from peer 4 (periodic) - let (_guard4, bytes4, peer4) = &results[2]; - assert_eq!(*peer4, AuthorityIndex::new_for_test(4)); - let expected4 = missing_vbs - [context.parameters.max_blocks_per_sync..2 * context.parameters.max_blocks_per_sync] - .iter() - .map(|vb| vb.serialized().clone()) - .collect::>(); - assert_eq!(bytes4, &expected4); - - // 10) Fourth fetch from peer 5 (fallback after peer 2 timeout) - let (_guard5, bytes5, peer5) = &results[3]; - assert_eq!(*peer5, AuthorityIndex::new_for_test(5)); - let expected5 = missing_vbs[0..known_number_blocks] - .iter() - .map(|vb| vb.serialized().clone()) - .collect::>(); - assert_eq!(bytes5, &expected5); - } - - #[tokio::test(flavor = "current_thread")] - async fn test_verify_blocks_deduplication() { - let (context, _keys) = Context::new_for_test(4); - let context = Arc::new(context); - let block_verifier = Arc::new(NoopBlockVerifier {}); - let failing_verifier = Arc::new(FailingBlockVerifier); - let peer1 = AuthorityIndex::new_for_test(1); - let peer2 = AuthorityIndex::new_for_test(2); - - // Create cache with capacity of 5 for eviction testing - let cache = Arc::new(SyncMutex::new(LruCache::new(NonZeroUsize::new(5).unwrap()))); - - // Test 1: Per-peer metric tracking - let block1 = VerifiedBlock::new_for_test(TestBlock::new(10, 0).build()); - let serialized1 = vec![block1.serialized().clone()]; - - // Verify from peer1 (cache miss) - let result = Synchronizer::::verify_blocks( - serialized1.clone(), block_verifier.clone(), cache.clone(), &context, peer1, "live", - ); - assert_eq!(result.unwrap().len(), 1); - - let peer1_hostname = &context.committee.authority(peer1).hostname; - assert_eq!( - context - .metrics - .node_metrics - .synchronizer_skipped_blocks_by_peer - .with_label_values(&[peer1_hostname.as_str(), "live"]) - .get(), - 0 - ); - - // Verify same block from peer2 with different sync method (cache hit) - let result = Synchronizer::::verify_blocks( - serialized1, block_verifier.clone(), cache.clone(), &context, peer2, "periodic", - ); - assert_eq!(result.unwrap().len(), 0, "Should skip cached block"); - - let peer2_hostname = &context.committee.authority(peer2).hostname; - assert_eq!( - context - .metrics - .node_metrics - .synchronizer_skipped_blocks_by_peer - .with_label_values(&[peer2_hostname.as_str(), "periodic"]) - .get(), - 1 - ); - - // Test 2: Invalid blocks not cached - let invalid_block = VerifiedBlock::new_for_test(TestBlock::new(20, 0).build()); - let invalid_serialized = vec![invalid_block.serialized().clone()]; - - assert!(Synchronizer::::verify_blocks( - invalid_serialized.clone(), failing_verifier.clone(), cache.clone(), &context, peer1, "test", - ).is_err()); - assert_eq!(cache.lock().len(), 1, "Invalid block should not be cached"); - - // Verify invalid block fails again (not from cache) - assert!(Synchronizer::::verify_blocks( - invalid_serialized, failing_verifier, cache.clone(), &context, peer1, "test", - ).is_err()); - - // Test 3: Cache eviction - let blocks: Vec<_> = (0..5) - .map(|i| VerifiedBlock::new_for_test(TestBlock::new(30 + i, 0).build())) - .collect(); - - // Fill cache to capacity - for block in &blocks { - Synchronizer::::verify_blocks( - vec![block.serialized().clone()], block_verifier.clone(), cache.clone(), &context, peer1, "test", - ).unwrap(); - } - assert_eq!(cache.lock().len(), 5); - - // Verify first block is evicted when adding new one - let new_block = VerifiedBlock::new_for_test(TestBlock::new(99, 0).build()); - Synchronizer::::verify_blocks( - vec![new_block.serialized().clone()], block_verifier.clone(), cache.clone(), &context, peer1, "test", - ).unwrap(); - - // First block (block1) should be evicted, so re-verifying it should not be a - // cache hit - let block1_serialized = vec![block1.serialized().clone()]; - let result = Synchronizer::::verify_blocks( - block1_serialized, block_verifier.clone(), cache.clone(), &context, peer1, "test", - ); - assert_eq!( - result.unwrap().len(), - 1, - "Evicted block should be re-verified" - ); - - // New block should still be in cache - let new_block_serialized = vec![new_block.serialized().clone()]; - let result = Synchronizer::::verify_blocks( - new_block_serialized, block_verifier, cache, &context, peer1, "test", - ); - assert_eq!(result.unwrap().len(), 0, "New block should be cached"); - } -} diff --git a/consensus/core/src/test_dag.rs b/consensus/core/src/test_dag.rs deleted file mode 100644 index 9d342a5efe5..00000000000 --- a/consensus/core/src/test_dag.rs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; -use rand::{Rng, SeedableRng, rngs::StdRng}; - -use crate::{ - block::{BlockRef, BlockTimestampMs, Round, TestBlock, VerifiedBlock, genesis_blocks}, - context::Context, - dag_state::DagState, - test_dag_builder::DagBuilder, -}; - -// todo: remove this once tests have been refactored to use DagBuilder/DagParser - -/// Build a fully interconnected dag up to the specified round. This function -/// starts building the dag from the specified [`start`] parameter or from -/// genesis if none are specified up to and including the specified round -/// [`stop`] parameter. -pub(crate) fn build_dag( - context: Arc, - dag_state: Arc>, - start: Option>, - stop: Round, -) -> Vec { - let mut ancestors = match start { - Some(start) => { - assert!(!start.is_empty()); - assert_eq!( - start.iter().map(|x| x.round).max(), - start.iter().map(|x| x.round).min() - ); - start - } - None => genesis_blocks(&context) - .iter() - .map(|x| x.reference()) - .collect::>(), - }; - - let num_authorities = context.committee.size(); - let starting_round = ancestors.first().unwrap().round + 1; - for round in starting_round..=stop { - let (references, blocks): (Vec<_>, Vec<_>) = context - .committee - .authorities() - .map(|authority| { - let author_idx = authority.0.value() as u32; - // Test the case where a block from round R+1 has smaller timestamp than a block - // from round R. - let ts = round as BlockTimestampMs / 2 * num_authorities as BlockTimestampMs - + author_idx as BlockTimestampMs; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author_idx) - .set_timestamp_ms(ts) - .set_ancestors(ancestors.clone()) - .build(), - ); - - (block.reference(), block) - }) - .unzip(); - dag_state.write().accept_blocks(blocks); - ancestors = references; - } - - ancestors -} - -// TODO: Add layer_round as input parameter so ancestors can be from any round. -pub(crate) fn build_dag_layer( - // A list of (authority, parents) pairs. For each authority, we add a block - // linking to the specified parents. - connections: Vec<(AuthorityIndex, Vec)>, - dag_state: Arc>, -) -> Vec { - let mut references = Vec::new(); - for (authority, ancestors) in connections { - let round = ancestors.first().unwrap().round + 1; - let author = authority.value() as u32; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_ancestors(ancestors) - .build(), - ); - references.push(block.reference()); - dag_state.write().accept_block(block); - } - references -} - -pub(crate) fn create_random_dag( - seed: u64, - include_leader_percentage: u64, - num_rounds: Round, - context: Arc, -) -> DagBuilder { - assert!( - (0..=100).contains(&include_leader_percentage), - "include_leader_percentage must be in the range 0..100" - ); - - let mut rng = StdRng::seed_from_u64(seed); - let mut dag_builder = DagBuilder::new(context); - - for r in 1..=num_rounds { - let random_num = rng.gen_range(0..100); - let include_leader = random_num <= include_leader_percentage; - dag_builder - .layer(r) - .min_ancestor_links(include_leader, Some(random_num)); - } - - dag_builder -} diff --git a/consensus/core/src/test_dag_builder.rs b/consensus/core/src/test_dag_builder.rs deleted file mode 100644 index cc5c9a5348f..00000000000 --- a/consensus/core/src/test_dag_builder.rs +++ /dev/null @@ -1,858 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::{BTreeMap, HashSet}, - ops::{Bound::Included, RangeInclusive}, - sync::Arc, -}; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; -use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; - -use crate::{ - CommitRef, CommittedSubDag, - block::{ - BlockAPI, BlockDigest, BlockRef, BlockTimestampMs, Round, Slot, TestBlock, VerifiedBlock, - genesis_blocks, - }, - commit::{CertifiedCommit, CommitDigest, DEFAULT_WAVE_LENGTH, TrustedCommit}, - context::Context, - dag_state::DagState, - leader_schedule::{LeaderSchedule, LeaderSwapTable}, - linearizer::{BlockStoreAPI, Linearizer}, -}; - -/// DagBuilder API -/// -/// Usage: -/// -/// DAG Building -/// ``` -/// let context = Arc::new(Context::new_for_test(4).0); -/// let dag_builder = DagBuilder::new(context); -/// dag_builder.layer(1).build(); // Round 1 is fully connected with parents by default. -/// dag_builder.layers(2..10).build(); // Rounds 2 ~ 10 are fully connected with parents by default. -/// dag_builder.layers(11).min_parent_links().build(); // Round 11 is minimally and randomly connected with parents, without weak links. -/// dag_builder.layers(12).no_leader_block(0).build(); // Round 12 misses leader block. Other blocks are fully connected with parents. -/// dag_builder.layers(13).no_leader_link(12, 0).build(); // Round 13 misses votes for leader block. Other blocks are fully connected with parents. -/// dag_builder.layers(14).authorities(vec![3,5]).skip_block().build(); // Round 14 authorities 3 and 5 will not propose any block. -/// dag_builder.layers(15).authorities(vec![3,5]).skip_ancestor_links(vec![1,2]).build(); // Round 15 authorities 3 and 5 will not link to ancestors 1 and 2 -/// dag_builder.layers(16).authorities(vec![3,5]).equivocate(3).build(); // Round 16 authorities 3 and 5 will produce 3 equivocating blocks. -/// ``` -/// -/// Persisting to DagState by Layer -/// ``` -/// let dag_state = Arc::new(RwLock::new(DagState::new( -/// dag_builder.context.clone(), -/// Arc::new(MemStore::new()), -/// ))); -/// let context = Arc::new(Context::new_for_test(4).0); -/// let dag_builder = DagBuilder::new(context); -/// dag_builder -/// .layer(1) -/// .build() -/// .persist_layers(dag_state.clone()); // persist the layer -/// ``` -/// -/// Persisting entire DAG to DagState -/// ``` -/// let dag_state = Arc::new(RwLock::new(DagState::new( -/// dag_builder.context.clone(), -/// Arc::new(MemStore::new()), -/// ))); -/// let context = Arc::new(Context::new_for_test(4).0); -/// let dag_builder = DagBuilder::new(context); -/// dag_builder.layer(1).build(); -/// dag_builder.layers(2..10).build(); -/// dag_builder.persist_all_blocks(dag_state.clone()); // persist entire DAG -/// ``` -/// -/// Printing DAG -/// ``` -/// let context = Arc::new(Context::new_for_test(4).0); -/// let dag_builder = DagBuilder::new(context); -/// dag_builder.layer(1).build(); -/// dag_builder.print(); // pretty print the entire DAG -/// ``` -pub(crate) struct DagBuilder { - pub(crate) context: Arc, - pub(crate) leader_schedule: LeaderSchedule, - // The genesis blocks - pub(crate) genesis: BTreeMap, - // The current set of ancestors that any new layer will attempt to connect to. - pub(crate) last_ancestors: Vec, - // All blocks created by dag builder. Will be used to pretty print or to be - // retrieved for testing/persiting to dag state. - pub(crate) blocks: BTreeMap, - // All the committed sub dags created by the dag builder. - pub(crate) committed_sub_dags: Vec<(CommittedSubDag, TrustedCommit)>, - pub(crate) last_committed_rounds: Vec, - - wave_length: Round, - number_of_leaders: u32, - pipeline: bool, -} - -impl DagBuilder { - pub(crate) fn new(context: Arc) -> Self { - let leader_schedule = LeaderSchedule::new(context.clone(), LeaderSwapTable::default()); - let genesis_blocks = genesis_blocks(&context); - let genesis: BTreeMap = genesis_blocks - .into_iter() - .map(|block| (block.reference(), block)) - .collect(); - let last_ancestors = genesis.keys().cloned().collect(); - Self { - last_committed_rounds: vec![0; context.committee.size()], - context, - leader_schedule, - wave_length: DEFAULT_WAVE_LENGTH, - number_of_leaders: 1, - pipeline: false, - genesis, - last_ancestors, - blocks: BTreeMap::new(), - committed_sub_dags: vec![], - } - } - - pub(crate) fn blocks(&self, rounds: RangeInclusive) -> Vec { - assert!( - !self.blocks.is_empty(), - "No blocks have been created, please make sure that you have called build method" - ); - self.blocks - .iter() - .filter_map(|(block_ref, block)| rounds.contains(&block_ref.round).then_some(block)) - .cloned() - .collect::>() - } - - pub(crate) fn all_blocks(&self) -> Vec { - assert!( - !self.blocks.is_empty(), - "No blocks have been created, please make sure that you have called build method" - ); - self.blocks.values().cloned().collect() - } - - pub(crate) fn get_sub_dag_and_commits( - &mut self, - leader_rounds: RangeInclusive, - ) -> Vec<(CommittedSubDag, TrustedCommit)> { - let (last_leader_round, mut last_commit_ref, mut last_timestamp_ms) = - if let Some((sub_dag, _)) = self.committed_sub_dags.last() { - ( - sub_dag.leader.round, - sub_dag.commit_ref, - sub_dag.timestamp_ms, - ) - } else { - (0, CommitRef::new(0, CommitDigest::MIN), 0) - }; - - struct BlockStorage { - gc_round: Round, - context: Arc, - blocks: BTreeMap, /* the tuple represents the block - * and whether it is committed */ - genesis: BTreeMap, - } - impl BlockStoreAPI for BlockStorage { - fn get_blocks(&self, refs: &[BlockRef]) -> Vec> { - refs.iter() - .map(|block_ref| { - if block_ref.round == 0 { - return self.genesis.get(block_ref).cloned(); - } - self.blocks - .get(block_ref) - .map(|(block, _committed)| block.clone()) - }) - .collect() - } - - fn gc_round(&self) -> Round { - self.gc_round - } - - fn gc_enabled(&self) -> bool { - self.context.protocol_config.gc_depth() > 0 - } - - fn set_committed(&mut self, block_ref: &BlockRef) -> bool { - let Some((_block, committed)) = self.blocks.get_mut(block_ref) else { - panic!("Block {block_ref:?} should be found in store"); - }; - if !*committed { - *committed = true; - return true; - } - false - } - - fn is_committed(&self, block_ref: &BlockRef) -> bool { - self.blocks - .get(block_ref) - .map(|(_, committed)| *committed) - .expect("Block should be found in store") - } - } - let mut storage = BlockStorage { - context: self.context.clone(), - blocks: self - .blocks - .clone() - .into_iter() - .map(|(k, v)| (k, (v, false))) - .collect(), - genesis: self.genesis.clone(), - gc_round: 0, - }; - - // Create any remaining committed sub dags - for leader_block in self - .leader_blocks(last_leader_round + 1..=*leader_rounds.end()) - .into_iter() - .flatten() - { - // set the gc round to the round of the leader block - storage.gc_round = leader_block - .round() - .saturating_sub(1) - .saturating_sub(self.context.protocol_config.gc_depth()); - - let leader_block_ref = leader_block.reference(); - - let to_commit = Linearizer::linearize_sub_dag( - &self.context.clone(), - leader_block.clone(), - self.last_committed_rounds.clone(), - &mut storage, - ); - - last_timestamp_ms = Linearizer::calculate_commit_timestamp( - &self.context.clone(), - &mut storage, - &leader_block, - last_timestamp_ms, - ); - - // Update the last committed rounds - for block in &to_commit { - self.last_committed_rounds[block.author()] = - self.last_committed_rounds[block.author()].max(block.round()); - } - - let commit = TrustedCommit::new_for_test( - last_commit_ref.index + 1, - last_commit_ref.digest, - last_timestamp_ms, - leader_block_ref, - to_commit - .iter() - .map(|block| block.reference()) - .collect::>(), - ); - - last_commit_ref = commit.reference(); - - let sub_dag = CommittedSubDag::new( - leader_block_ref, - to_commit, - last_timestamp_ms, - commit.reference(), - vec![], - ); - - self.committed_sub_dags.push((sub_dag, commit)); - } - - self.committed_sub_dags - .clone() - .into_iter() - .filter(|(sub_dag, _)| leader_rounds.contains(&sub_dag.leader.round)) - .collect() - } - pub(crate) fn leader_blocks( - &self, - rounds: RangeInclusive, - ) -> Vec> { - assert!( - !self.blocks.is_empty(), - "No blocks have been created, please make sure that you have called build method" - ); - rounds - .into_iter() - .map(|round| self.leader_block(round)) - .collect() - } - - pub(crate) fn get_sub_dag_and_certified_commits( - &mut self, - leader_rounds: RangeInclusive, - ) -> Vec<(CommittedSubDag, CertifiedCommit)> { - let commits = self.get_sub_dag_and_commits(leader_rounds); - commits - .into_iter() - .map(|(sub_dag, commit)| { - let certified_commit = - CertifiedCommit::new_certified(commit, sub_dag.blocks.clone()); - (sub_dag, certified_commit) - }) - .collect() - } - - pub(crate) fn leader_block(&self, round: Round) -> Option { - assert!( - !self.blocks.is_empty(), - "No blocks have been created, please make sure that you have called build method" - ); - self.blocks.iter().find_map(|(block_ref, block)| { - (block_ref.round == round - && block_ref.author == self.leader_schedule.elect_leader(round, 0)) - .then_some(block.clone()) - }) - } - - #[expect(unused)] - pub(crate) fn with_wave_length(mut self, wave_length: Round) -> Self { - self.wave_length = wave_length; - self - } - - #[expect(unused)] - pub(crate) fn with_number_of_leaders(mut self, number_of_leaders: u32) -> Self { - self.number_of_leaders = number_of_leaders; - self - } - - #[expect(unused)] - pub(crate) fn with_pipeline(mut self, pipeline: bool) -> Self { - self.pipeline = pipeline; - self - } - - pub(crate) fn layer(&mut self, round: Round) -> LayerBuilder<'_> { - LayerBuilder::new(self, round) - } - - pub(crate) fn layers(&mut self, rounds: RangeInclusive) -> LayerBuilder<'_> { - let mut builder = LayerBuilder::new(self, *rounds.start()); - builder.end_round = Some(*rounds.end()); - builder - } - - pub(crate) fn persist_all_blocks(&self, dag_state: Arc>) { - dag_state - .write() - .accept_blocks(self.blocks.values().cloned().collect()); - } - - pub(crate) fn print(&self) { - let mut dag_str = "DAG {\n".to_string(); - - let mut round = 0; - for block in self.blocks.values() { - if block.round() > round { - round = block.round(); - dag_str.push_str(&format!("Round {round} : \n")); - } - dag_str.push_str(&format!(" Block {block:#?}\n")); - } - dag_str.push_str("}\n"); - - tracing::info!("{dag_str}"); - } - - // TODO: merge into layer builder? - // This method allows the user to specify specific links to ancestors. The - // layer is written to dag state and the blocks are cached in [`DagBuilder`] - // state. - pub(crate) fn layer_with_connections( - &mut self, - connections: Vec<(AuthorityIndex, Vec)>, - round: Round, - ) { - let mut references = Vec::new(); - for (authority, ancestors) in connections { - let author = authority.value() as u32; - let base_ts = round as BlockTimestampMs * 1000; - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, author) - .set_ancestors(ancestors) - .set_timestamp_ms(base_ts + author as u64) - .build(), - ); - references.push(block.reference()); - self.blocks.insert(block.reference(), block.clone()); - } - self.last_ancestors = references; - } - - /// Gets all uncommitted blocks in a slot. - pub(crate) fn get_uncommitted_blocks_at_slot(&self, slot: Slot) -> Vec { - let mut blocks = vec![]; - for (_block_ref, block) in self.blocks.range(( - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MIN)), - Included(BlockRef::new(slot.round, slot.authority, BlockDigest::MAX)), - )) { - blocks.push(block.clone()) - } - blocks - } - - pub(crate) fn genesis_block_refs(&self) -> Vec { - self.genesis.keys().cloned().collect() - } -} - -/// Refer to doc comments for [`DagBuilder`] for usage information. -pub struct LayerBuilder<'a> { - dag_builder: &'a mut DagBuilder, - - start_round: Round, - end_round: Option, - - // Configuration options applied to specified authorities - // TODO: convert configuration options into an enum - specified_authorities: Option>, - // Number of equivocating blocks per specified authority - equivocations: usize, - // Skip block proposal for specified authorities - skip_block: bool, - // Skip specified ancestor links for specified authorities - skip_ancestor_links: Option>, - // Skip leader link for specified authorities - no_leader_link: bool, - - // Skip leader block proposal - no_leader_block: bool, - // Used for leader based configurations - specified_leader_link_offsets: Option>, - specified_leader_block_offsets: Option>, - leader_round: Option, - - // All ancestors will be linked to the current layer - fully_linked_ancestors: bool, - // Only 2f+1 random ancestors will be linked to the current layer using a - // seed, if provided - min_ancestor_links: bool, - min_ancestor_links_random_seed: Option, - // Add random weak links to the current layer using a seed, if provided - random_weak_links: bool, - random_weak_links_random_seed: Option, - - // Ancestors to link to the current layer - ancestors: Vec, - - // The block timestamps for the layer for each specified authority. This will work as base - // timestamp and the round will be added to make sure that timestamps do offset. - timestamps: Vec, - - // Accumulated blocks to write to dag state - blocks: Vec, -} - -#[expect(unused)] -impl<'a> LayerBuilder<'a> { - fn new(dag_builder: &'a mut DagBuilder, start_round: Round) -> Self { - assert!(start_round > 0, "genesis round is created by default"); - let ancestors = dag_builder.last_ancestors.clone(); - Self { - dag_builder, - start_round, - end_round: None, - specified_authorities: None, - equivocations: 0, - skip_block: false, - skip_ancestor_links: None, - no_leader_link: false, - no_leader_block: false, - specified_leader_link_offsets: None, - specified_leader_block_offsets: None, - leader_round: None, - fully_linked_ancestors: true, - min_ancestor_links: false, - min_ancestor_links_random_seed: None, - random_weak_links: false, - random_weak_links_random_seed: None, - ancestors, - timestamps: vec![], - blocks: vec![], - } - } - - // Configuration methods - - // Only link 2f+1 random ancestors to the current layer round using a seed, - // if provided. Also provide a flag to guarantee the leader is included. - // note: configuration is terminal and layer will be built after this call. - pub fn min_ancestor_links(mut self, include_leader: bool, seed: Option) -> Self { - self.min_ancestor_links = true; - self.min_ancestor_links_random_seed = seed; - if include_leader { - self.leader_round = Some(self.ancestors.iter().max_by_key(|b| b.round).unwrap().round); - } - self.fully_linked_ancestors = false; - self.build() - } - - // No links will be created between the specified ancestors and the specified - // authorities at the layer round. - // note: configuration is terminal and layer will be built after this call. - pub fn skip_ancestor_links(mut self, ancestors_to_skip: Vec) -> Self { - // authorities must be specified for this to apply - assert!(self.specified_authorities.is_some()); - self.skip_ancestor_links = Some(ancestors_to_skip); - self.fully_linked_ancestors = false; - self.build() - } - - // Add random weak links to the current layer round using a seed, if provided - pub fn random_weak_links(mut self, seed: Option) -> Self { - self.random_weak_links = true; - self.random_weak_links_random_seed = seed; - self - } - - // Should be called when building a leader round. Will ensure leader block is - // missing. A list of specified leader offsets can be provided to skip those - // leaders. If none are provided all potential leaders for the round will be - // skipped. - pub fn no_leader_block(mut self, specified_leader_offsets: Vec) -> Self { - self.no_leader_block = true; - self.specified_leader_block_offsets = Some(specified_leader_offsets); - self - } - - // Should be called when building a voting round. Will ensure vote is missing. - // A list of specified leader offsets can be provided to skip those leader - // links. If none are provided all potential leaders for the round will be - // skipped. note: configuration is terminal and layer will be built after - // this call. - pub fn no_leader_link( - mut self, - leader_round: Round, - specified_leader_offsets: Vec, - ) -> Self { - self.no_leader_link = true; - self.specified_leader_link_offsets = Some(specified_leader_offsets); - self.leader_round = Some(leader_round); - self.fully_linked_ancestors = false; - self.build() - } - - pub fn authorities(mut self, authorities: Vec) -> Self { - assert!( - self.specified_authorities.is_none(), - "Specified authorities already set" - ); - self.specified_authorities = Some(authorities); - self - } - - // Multiple blocks will be created for the specified authorities at the layer - // round. - pub fn equivocate(mut self, equivocations: usize) -> Self { - // authorities must be specified for this to apply - assert!(self.specified_authorities.is_some()); - self.equivocations = equivocations; - self - } - - // No blocks will be created for the specified authorities at the layer round. - pub fn skip_block(mut self) -> Self { - // authorities must be specified for this to apply - assert!(self.specified_authorities.is_some()); - self.skip_block = true; - self - } - - pub fn with_timestamps(mut self, timestamps: Vec) -> Self { - // authorities must be specified for this to apply - assert!(self.specified_authorities.is_some()); - assert_eq!( - self.specified_authorities.as_ref().unwrap().len(), - timestamps.len(), - "Timestamps should be provided for each specified authority" - ); - self.timestamps = timestamps; - self - } - - // Apply the configurations & build the dag layer(s). - pub fn build(mut self) -> Self { - for round in self.start_round..=self.end_round.unwrap_or(self.start_round) { - tracing::debug!("BUILDING LAYER ROUND {round}..."); - - let authorities = if self.specified_authorities.is_some() { - self.specified_authorities.clone().unwrap() - } else { - self.dag_builder - .context - .committee - .authorities() - .map(|x| x.0) - .collect() - }; - - // TODO: investigate if these configurations can be called in combination - // for the same layer - let mut connections = if self.fully_linked_ancestors { - self.configure_fully_linked_ancestors() - } else if self.min_ancestor_links { - self.configure_min_parent_links() - } else if self.no_leader_link { - self.configure_no_leader_links(authorities.clone(), round) - } else if self.skip_ancestor_links.is_some() { - self.configure_skipped_ancestor_links( - authorities, - self.skip_ancestor_links.clone().unwrap(), - ) - } else { - vec![] - }; - - if self.random_weak_links { - connections.append(&mut self.configure_random_weak_links()); - } - - self.create_blocks(round, connections); - } - - self.dag_builder.last_ancestors = self.ancestors.clone(); - self - } - - pub fn persist_layers(&self, dag_state: Arc>) { - assert!( - !self.blocks.is_empty(), - "Called to persist layers although no blocks have been created. Make sure you have called build before." - ); - dag_state.write().accept_blocks(self.blocks.clone()); - } - - // Layer round is minimally and randomly connected with ancestors. - pub fn configure_min_parent_links(&mut self) -> Vec<(AuthorityIndex, Vec)> { - let quorum_threshold = self.dag_builder.context.committee.quorum_threshold() as usize; - let mut authorities: Vec = self - .dag_builder - .context - .committee - .authorities() - .map(|authority| authority.0) - .collect(); - - let mut rng = match self.min_ancestor_links_random_seed { - Some(s) => StdRng::seed_from_u64(s), - None => StdRng::from_entropy(), - }; - - let mut authorities_to_shuffle = authorities.clone(); - - let mut leaders = vec![]; - if let Some(leader_round) = self.leader_round { - let leader_offsets = (0..self.dag_builder.number_of_leaders).collect::>(); - - for leader_offset in leader_offsets { - leaders.push( - self.dag_builder - .leader_schedule - .elect_leader(leader_round, leader_offset), - ); - } - } - - authorities - .iter() - .map(|authority| { - authorities_to_shuffle.shuffle(&mut rng); - - // TODO: handle quorum threshold properly with stake - let min_ancestors: HashSet = authorities_to_shuffle - .iter() - .take(quorum_threshold) - .cloned() - .collect(); - - ( - *authority, - self.ancestors - .iter() - .filter(|a| { - leaders.contains(&a.author) || min_ancestors.contains(&a.author) - }) - .cloned() - .collect::>(), - ) - }) - .collect() - } - - // TODO: configure layer round randomly connected with weak links. - fn configure_random_weak_links(&mut self) -> Vec<(AuthorityIndex, Vec)> { - unimplemented!("configure_random_weak_links"); - } - - // Layer round misses link to leader, but other blocks are fully connected with - // ancestors. - fn configure_no_leader_links( - &mut self, - authorities: Vec, - round: Round, - ) -> Vec<(AuthorityIndex, Vec)> { - let mut missing_leaders = Vec::new(); - let mut specified_leader_offsets = self - .specified_leader_link_offsets - .clone() - .expect("specified_leader_offsets should be set"); - let leader_round = self.leader_round.expect("leader round should be set"); - - // When no specified leader offsets are available, all leaders are - // expected to be missing. - if specified_leader_offsets.is_empty() { - specified_leader_offsets.extend(0..self.dag_builder.number_of_leaders); - } - - for leader_offset in specified_leader_offsets { - missing_leaders.push( - self.dag_builder - .leader_schedule - .elect_leader(leader_round, leader_offset), - ); - } - - self.configure_skipped_ancestor_links(authorities, missing_leaders) - } - - fn configure_fully_linked_ancestors(&mut self) -> Vec<(AuthorityIndex, Vec)> { - self.dag_builder - .context - .committee - .authorities() - .map(|authority| (authority.0, self.ancestors.clone())) - .collect::>() - } - - fn configure_skipped_ancestor_links( - &mut self, - authorities: Vec, - ancestors_to_skip: Vec, - ) -> Vec<(AuthorityIndex, Vec)> { - let filtered_ancestors = self - .ancestors - .clone() - .into_iter() - .filter(|ancestor| !ancestors_to_skip.contains(&ancestor.author)) - .collect::>(); - authorities - .into_iter() - .map(|authority| (authority, filtered_ancestors.clone())) - .collect::>() - } - - // Creates the blocks for the new layer based on configured connections, also - // sets the ancestors for future layers to be linked to - fn create_blocks(&mut self, round: Round, connections: Vec<(AuthorityIndex, Vec)>) { - let mut references = Vec::new(); - for (authority, ancestors) in connections { - if self.should_skip_block(round, authority) { - continue; - }; - let num_blocks = self.num_blocks_to_create(authority); - - for num_block in 0..num_blocks { - let timestamp = self.block_timestamp(authority, round, num_block); - let block = VerifiedBlock::new_for_test( - TestBlock::new(round, authority.value() as u32) - .set_ancestors(ancestors.clone()) - .set_timestamp_ms(timestamp) - .build(), - ); - references.push(block.reference()); - self.dag_builder - .blocks - .insert(block.reference(), block.clone()); - self.blocks.push(block); - } - } - self.ancestors = references; - } - - fn num_blocks_to_create(&self, authority: AuthorityIndex) -> u32 { - if self.specified_authorities.is_some() - && self - .specified_authorities - .clone() - .unwrap() - .contains(&authority) - { - // Always create 1 block and then the equivocating blocks on top of that. - 1 + self.equivocations as u32 - } else { - 1 - } - } - - fn block_timestamp( - &self, - authority: AuthorityIndex, - round: Round, - num_block: u32, - ) -> BlockTimestampMs { - if let Some(specified_authorities) = &self.specified_authorities { - if !self.timestamps.is_empty() { - if let Some(position) = specified_authorities.iter().position(|&x| x == authority) { - return self.timestamps[position] + (round + num_block) as u64; - } - } - } - let author = authority.value() as u32; - let base_ts = round as BlockTimestampMs * 1000; - base_ts + (author + round + num_block) as u64 - } - - fn should_skip_block(&self, round: Round, authority: AuthorityIndex) -> bool { - // Safe to unwrap as specified authorities has to be set before skip - // is specified. - if self.skip_block - && self - .specified_authorities - .clone() - .unwrap() - .contains(&authority) - { - return true; - } - if self.no_leader_block { - let mut specified_leader_offsets = self - .specified_leader_block_offsets - .clone() - .expect("specified_leader_block_offsets should be set"); - - // When no specified leader offsets are available, all leaders are - // expected to be skipped. - if specified_leader_offsets.is_empty() { - specified_leader_offsets.extend(0..self.dag_builder.number_of_leaders); - } - - for leader_offset in specified_leader_offsets { - let leader = self - .dag_builder - .leader_schedule - .elect_leader(round, leader_offset); - - if leader == authority { - return true; - } - } - } - false - } -} - -// TODO: add unit tests diff --git a/consensus/core/src/test_dag_parser.rs b/consensus/core/src/test_dag_parser.rs deleted file mode 100644 index f00c50a6f75..00000000000 --- a/consensus/core/src/test_dag_parser.rs +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashSet, sync::Arc}; - -use consensus_config::AuthorityIndex; -use nom::{ - IResult, - branch::alt, - bytes::complete::{tag, take_while_m_n, take_while1}, - character::complete::{char, digit1, multispace0, multispace1, space0, space1}, - combinator::{map_res, opt}, - multi::{many0, separated_list0}, - sequence::{delimited, preceded, terminated, tuple}, -}; - -use crate::{ - block::{BlockRef, Round, Slot}, - context::Context, - test_dag_builder::DagBuilder, -}; - -/// DagParser -/// -/// Usage: -/// -/// ``` -/// let dag_str = "DAG { -/// Round 0 : { 4 }, -/// Round 1 : { * }, -/// Round 2 : { * }, -/// Round 3 : { * }, -/// Round 4 : { -/// A -> [-D3], -/// B -> [*], -/// C -> [*], -/// D -> [*], -/// }, -/// Round 5 : { -/// A -> [*], -/// B -> [*], -/// C -> [A4], -/// D -> [A4], -/// }, -/// Round 6 : { * }, -/// Round 7 : { * }, -/// Round 8 : { * }, -/// }"; -/// -/// let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); // parse DAG DSL -/// dag_builder.print(); // print the parsed DAG -/// dag_builder.persist_all_blocks(dag_state.clone()); // persist all blocks to DagState -/// ``` -pub(crate) fn parse_dag(dag_string: &str) -> IResult<&str, DagBuilder> { - let (input, _) = tuple((tag("DAG"), multispace0, char('{')))(dag_string)?; - - let (mut input, num_authors) = parse_genesis(input)?; - - let context = Arc::new(Context::new_for_test(num_authors as usize).0); - let mut dag_builder = DagBuilder::new(context); - - // Parse subsequent rounds - loop { - match parse_round(input, &dag_builder) { - Ok((new_input, (round, connections))) => { - dag_builder.layer_with_connections(connections, round); - input = new_input - } - Err(nom::Err::Error(_)) | Err(nom::Err::Failure(_)) => break, - Err(nom::Err::Incomplete(needed)) => return Err(nom::Err::Incomplete(needed)), - } - } - let (input, _) = tuple((multispace0, char('}')))(input)?; - - Ok((input, dag_builder)) -} - -fn parse_round<'a>( - input: &'a str, - dag_builder: &DagBuilder, -) -> IResult<&'a str, (Round, Vec<(AuthorityIndex, Vec)>)> { - let (input, _) = tuple((multispace0, tag("Round"), space1))(input)?; - let (input, round) = take_while1(|c: char| c.is_ascii_digit())(input)?; - - let (input, connections) = alt(( - |input| parse_fully_connected(input, dag_builder), - |input| parse_specified_connections(input, dag_builder), - ))(input)?; - - Ok((input, (round.parse().unwrap(), connections))) -} - -fn parse_fully_connected<'a>( - input: &'a str, - dag_builder: &DagBuilder, -) -> IResult<&'a str, Vec<(AuthorityIndex, Vec)>> { - let (input, _) = tuple(( - space0, - char(':'), - space0, - char('{'), - space0, - char('*'), - space0, - char('}'), - opt(char(',')), - ))(input)?; - - let ancestors = dag_builder.last_ancestors.clone(); - let connections = dag_builder - .context - .committee - .authorities() - .map(|authority| (authority.0, ancestors.clone())) - .collect::>(); - - Ok((input, connections)) -} - -fn parse_specified_connections<'a>( - input: &'a str, - dag_builder: &DagBuilder, -) -> IResult<&'a str, Vec<(AuthorityIndex, Vec)>> { - let (input, _) = tuple((space0, char(':'), space0, char('{'), multispace0))(input)?; - - // parse specified connections - // case 1: all authorities; [*] - // case 2: specific included authorities; [A0, B0, C0] - // case 3: specific excluded authorities; [-A0] - // case 4: mixed all authorities + specific included/excluded authorities; [*, - // A0] TODO: case 5: byzantine case of multiple blocks per slot; [*]; - // timestamp=1 - let (input, authors_and_connections) = many0(parse_author_and_connections)(input)?; - - let mut output = Vec::new(); - for (author, connections) in authors_and_connections { - let mut block_refs = HashSet::new(); - for connection in connections { - if connection == "*" { - block_refs.extend(dag_builder.last_ancestors.clone()); - } else if connection.starts_with('-') { - let (input, _) = char('-')(connection)?; - let (_, slot) = parse_slot(input)?; - let stored_block_refs = get_blocks(slot, dag_builder); - block_refs.extend(dag_builder.last_ancestors.clone()); - - block_refs.retain(|ancestor| !stored_block_refs.contains(ancestor)); - } else { - let input = connection; - let (_, slot) = parse_slot(input)?; - let stored_block_refs = get_blocks(slot, dag_builder); - - block_refs.extend(stored_block_refs); - } - } - output.push((author, block_refs.into_iter().collect())); - } - - let (input, _) = tuple((multispace0, char('}'), opt(char(','))))(input)?; - - Ok((input, output)) -} - -fn get_blocks(slot: Slot, dag_builder: &DagBuilder) -> Vec { - // note: special case for genesis blocks as they are cached separately - let block_refs = if slot.round == 0 { - dag_builder - .genesis_block_refs() - .into_iter() - .filter(|block| Slot::from(*block) == slot) - .collect::>() - } else { - dag_builder - .get_uncommitted_blocks_at_slot(slot) - .iter() - .map(|block| block.reference()) - .collect::>() - }; - block_refs -} - -fn parse_author_and_connections(input: &str) -> IResult<&str, (AuthorityIndex, Vec<&str>)> { - // parse author - let (input, author) = preceded( - multispace0, - terminated( - take_while1(|c: char| c.is_alphabetic()), - preceded(opt(space0), tag("->")), - ), - )(input)?; - - // parse connections - let (input, connections) = delimited( - preceded(opt(space0), char('[')), - separated_list0(tag(", "), parse_block), - terminated(char(']'), opt(char(','))), - )(input)?; - let (input, _) = opt(multispace1)(input)?; - Ok(( - input, - ( - str_to_authority_index(author).expect("Invalid authority index"), - connections, - ), - )) -} - -fn parse_block(input: &str) -> IResult<&str, &str> { - alt(( - map_res(tag("*"), |s: &str| Ok::<_, nom::error::ErrorKind>(s)), - map_res( - take_while1(|c: char| c.is_alphanumeric() || c == '-'), - |s: &str| Ok::<_, nom::error::ErrorKind>(s), - ), - ))(input) -} - -fn parse_genesis(input: &str) -> IResult<&str, u32> { - let (input, num_authorities) = preceded( - tuple(( - multispace0, - tag("Round"), - space1, - char('0'), - space0, - char(':'), - space0, - char('{'), - space0, - )), - |i| parse_authority_count(i), - )(input)?; - let (input, _) = tuple((space0, char('}'), opt(char(','))))(input)?; - - Ok((input, num_authorities)) -} - -fn parse_authority_count(input: &str) -> IResult<&str, u32> { - let (input, num_str) = digit1(input)?; - Ok((input, num_str.parse().unwrap())) -} - -fn parse_slot(input: &str) -> IResult<&str, Slot> { - let parse_authority = map_res( - take_while_m_n(1, 1, |c: char| c.is_alphabetic() && c.is_uppercase()), - |letter: &str| { - Ok::<_, nom::error::ErrorKind>( - str_to_authority_index(letter).expect("Invalid authority index"), - ) - }, - ); - - let parse_round = map_res(digit1, |digits: &str| digits.parse::()); - - let mut parser = tuple((parse_authority, parse_round)); - - let (input, (authority, round)) = parser(input)?; - Ok((input, Slot::new(round, authority))) -} - -// Helper function to convert a string representation (e.g., 'A' or '[26]') to -// an AuthorityIndex -fn str_to_authority_index(input: &str) -> Option { - if input.starts_with('[') && input.ends_with(']') && input.len() > 2 { - input[1..input.len() - 1] - .parse::() - .ok() - .map(AuthorityIndex::new_for_test) - } else if input.len() == 1 && input.chars().next()?.is_ascii_uppercase() { - // Handle single uppercase ASCII alphabetic character - let alpha_char = input.chars().next().unwrap(); - let index = alpha_char as u32 - 'A' as u32; - Some(AuthorityIndex::new_for_test(index)) - } else { - None - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use super::*; - use crate::block::BlockAPI; - - #[tokio::test] - async fn test_dag_parsing() { - telemetry_subscribers::init_for_testing(); - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { - A -> [*], - B -> [*], - C -> [*], - D -> [*], - }, - Round 4 : { - A -> [A3, B3, C3], - B -> [A3, B3, C3], - C -> [A3, B3, C3], - D -> [*], - }, - Round 5 : { - A -> [*], - B -> [-A4], - C -> [-A4], - D -> [-A4], - }, - Round 6 : { - A -> [A3, B3, C3, A1, B1], - B -> [*, A0], - C -> [-A5], - } - }"; - let result = parse_dag(dag_str); - assert!(result.is_ok()); - - let (_, dag_builder) = result.unwrap(); - assert_eq!(dag_builder.genesis.len(), 4); - assert_eq!(dag_builder.blocks.len(), 23); - - // Check the blocks were correctly parsed in Round 6 - let blocks_a6 = dag_builder - .get_uncommitted_blocks_at_slot(Slot::new(6, AuthorityIndex::new_for_test(0))); - assert_eq!(blocks_a6.len(), 1); - let block_a6 = blocks_a6.first().unwrap(); - assert_eq!(block_a6.round(), 6); - assert_eq!(block_a6.author(), AuthorityIndex::new_for_test(0)); - assert_eq!(block_a6.ancestors().len(), 5); - let expected_block_a6_ancestor_slots = [ - Slot::new(3, AuthorityIndex::new_for_test(0)), - Slot::new(3, AuthorityIndex::new_for_test(1)), - Slot::new(3, AuthorityIndex::new_for_test(2)), - Slot::new(1, AuthorityIndex::new_for_test(0)), - Slot::new(1, AuthorityIndex::new_for_test(1)), - ]; - for ancestor in block_a6.ancestors() { - assert!(expected_block_a6_ancestor_slots.contains(&Slot::from(*ancestor))); - } - - let blocks_b6 = dag_builder - .get_uncommitted_blocks_at_slot(Slot::new(6, AuthorityIndex::new_for_test(1))); - assert_eq!(blocks_b6.len(), 1); - let block_b6 = blocks_b6.first().unwrap(); - assert_eq!(block_b6.round(), 6); - assert_eq!(block_b6.author(), AuthorityIndex::new_for_test(1)); - assert_eq!(block_b6.ancestors().len(), 5); - let expected_block_b6_ancestor_slots = [ - Slot::new(5, AuthorityIndex::new_for_test(0)), - Slot::new(5, AuthorityIndex::new_for_test(1)), - Slot::new(5, AuthorityIndex::new_for_test(2)), - Slot::new(5, AuthorityIndex::new_for_test(3)), - Slot::new(0, AuthorityIndex::new_for_test(0)), - ]; - for ancestor in block_b6.ancestors() { - assert!(expected_block_b6_ancestor_slots.contains(&Slot::from(*ancestor))); - } - - let blocks_c6 = dag_builder - .get_uncommitted_blocks_at_slot(Slot::new(6, AuthorityIndex::new_for_test(2))); - assert_eq!(blocks_c6.len(), 1); - let block_c6 = blocks_c6.first().unwrap(); - assert_eq!(block_c6.round(), 6); - assert_eq!(block_c6.author(), AuthorityIndex::new_for_test(2)); - assert_eq!(block_c6.ancestors().len(), 3); - let expected_block_c6_ancestor_slots = [ - Slot::new(5, AuthorityIndex::new_for_test(1)), - Slot::new(5, AuthorityIndex::new_for_test(2)), - Slot::new(5, AuthorityIndex::new_for_test(3)), - ]; - for ancestor in block_c6.ancestors() { - assert!(expected_block_c6_ancestor_slots.contains(&Slot::from(*ancestor))); - } - } - - #[tokio::test] - async fn test_genesis_round_parsing() { - let dag_str = "Round 0 : { 4 }"; - let result = parse_genesis(dag_str); - assert!(result.is_ok()); - let (_, num_authorities) = result.unwrap(); - - assert_eq!(num_authorities, 4); - } - - #[tokio::test] - async fn test_slot_parsing() { - let dag_str = "A0"; - let result = parse_slot(dag_str); - assert!(result.is_ok()); - let (_, slot) = result.unwrap(); - - assert_eq!(slot.authority, str_to_authority_index("A").unwrap()); - assert_eq!(slot.round, 0); - } - - #[tokio::test] - async fn test_all_round_parsing() { - let dag_str = "Round 1 : { * }"; - let context = Arc::new(Context::new_for_test(4).0); - let dag_builder = DagBuilder::new(context); - let result = parse_round(dag_str, &dag_builder); - assert!(result.is_ok()); - let (_, (round, connections)) = result.unwrap(); - - assert_eq!(round, 1); - for (i, (authority, references)) in connections.into_iter().enumerate() { - assert_eq!(authority, AuthorityIndex::new_for_test(i as u32)); - assert_eq!(references, dag_builder.last_ancestors); - } - } - - #[tokio::test] - async fn test_specific_round_parsing() { - let dag_str = "Round 1 : { - A -> [A0, B0, C0, D0], - B -> [*, A0], - C -> [-A0], - }"; - let context = Arc::new(Context::new_for_test(4).0); - let dag_builder = DagBuilder::new(context); - let result = parse_round(dag_str, &dag_builder); - assert!(result.is_ok()); - let (_, (round, connections)) = result.unwrap(); - - let skipped_slot = Slot::new_for_test(0, 0); // A0 - let mut expected_references = [ - dag_builder.last_ancestors.clone(), - dag_builder.last_ancestors.clone(), - dag_builder - .last_ancestors - .into_iter() - .filter(|ancestor| Slot::from(*ancestor) != skipped_slot) - .collect(), - ]; - - assert_eq!(round, 1); - for (i, (authority, mut references)) in connections.into_iter().enumerate() { - assert_eq!(authority, AuthorityIndex::new_for_test(i as u32)); - references.sort(); - expected_references[i].sort(); - assert_eq!(references, expected_references[i]); - } - } - - #[tokio::test] - async fn test_parse_author_and_connections() { - let expected_authority = str_to_authority_index("A").unwrap(); - - // case 1: all authorities - let dag_str = "A -> [*]"; - let result = parse_author_and_connections(dag_str); - assert!(result.is_ok()); - let (_, (actual_author, actual_connections)) = result.unwrap(); - assert_eq!(actual_author, expected_authority); - assert_eq!(actual_connections, ["*"]); - - // case 2: specific included authorities - let dag_str = "A -> [A0, B0, C0]"; - let result = parse_author_and_connections(dag_str); - assert!(result.is_ok()); - let (_, (actual_author, actual_connections)) = result.unwrap(); - assert_eq!(actual_author, expected_authority); - assert_eq!(actual_connections, ["A0", "B0", "C0"]); - - // case 3: specific excluded authorities - let dag_str = "A -> [-A0, -B0]"; - let result = parse_author_and_connections(dag_str); - assert!(result.is_ok()); - let (_, (actual_author, actual_connections)) = result.unwrap(); - assert_eq!(actual_author, expected_authority); - assert_eq!(actual_connections, ["-A0", "-B0"]); - - // case 4: mixed all authorities + specific included/excluded authorities - let dag_str = "A -> [*, A0, -B0]"; - let result = parse_author_and_connections(dag_str); - assert!(result.is_ok()); - let (_, (actual_author, actual_connections)) = result.unwrap(); - assert_eq!(actual_author, expected_authority); - assert_eq!(actual_connections, ["*", "A0", "-B0"]); - - // TODO: case 5: byzantine case of multiple blocks per slot; [*]; - // timestamp=1 - } - - #[tokio::test] - async fn test_str_to_authority_index() { - assert_eq!( - str_to_authority_index("A"), - Some(AuthorityIndex::new_for_test(0)) - ); - assert_eq!( - str_to_authority_index("Z"), - Some(AuthorityIndex::new_for_test(25)) - ); - assert_eq!( - str_to_authority_index("[26]"), - Some(AuthorityIndex::new_for_test(26)) - ); - assert_eq!( - str_to_authority_index("[100]"), - Some(AuthorityIndex::new_for_test(100)) - ); - assert_eq!(str_to_authority_index("a"), None); - assert_eq!(str_to_authority_index("0"), None); - assert_eq!(str_to_authority_index(" "), None); - assert_eq!(str_to_authority_index("!"), None); - } -} diff --git a/consensus/core/src/tests/base_committer_declarative_tests.rs b/consensus/core/src/tests/base_committer_declarative_tests.rs deleted file mode 100644 index ce5853a57e9..00000000000 --- a/consensus/core/src/tests/base_committer_declarative_tests.rs +++ /dev/null @@ -1,373 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use core::panic; -use std::sync::Arc; - -use parking_lot::RwLock; - -use crate::{ - base_committer::base_committer_builder::BaseCommitterBuilder, block::BlockAPI, - commit::LeaderStatus, context::Context, dag_state::DagState, storage::mem_store::MemStore, - test_dag_parser::parse_dag, -}; - -#[tokio::test] -async fn direct_commit() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context, dag_state.clone()).build(); - - // Round 3 is a leader round - // D3 is an elected leader for wave 1 - // Round 4 is a voting round - // Round 5 is a decision round (acknowledge) - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { - A -> [D3], - B -> [D3], - C -> [D3], - D -> [], - }, - Round 5 : { - A -> [A4, B4, C4], - B -> [A4, B4, C4], - C -> [A4, B4, C4], - D -> [], - }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("a DAG should be valid"); - dag_builder.persist_all_blocks(dag_state); - - let leader_round = committer.leader_round(1); - tracing::info!("Leader round at wave 1: {leader_round}"); - let leader = committer - .elect_leader(leader_round) - .expect("there should be a leader at wave 1"); - let leader_status = committer.try_direct_decide(leader); - if let LeaderStatus::Commit(_) = leader_status { - tracing::info!("Committed: {leader_status}"); - } else { - panic!("Expected a committed leader, got {leader_status}"); - } -} - -#[tokio::test] -async fn direct_skip() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context, dag_state.clone()).build(); - - // Round 3 is a leader round - // D3 is an elected leader for wave 1 - // Round 4 is a voting round - // Round 5 is a decision round (acknowledge) - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { - A -> [D3], - B -> [], - C -> [], - D -> [], - }, - Round 5 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("a DAG should be valid"); - dag_builder.persist_all_blocks(dag_state); - - let leader_round = committer.leader_round(1); - tracing::info!("Leader round at wave 1: {leader_round}"); - let leader = committer - .elect_leader(leader_round) - .expect("there should be a leader at wave 1"); - let leader_status = committer.try_direct_decide(leader); - if let LeaderStatus::Skip(_) = leader_status { - tracing::info!("Skip: {leader_status}"); - } else { - panic!("Expected a skipped leader, got {leader_status}"); - } -} - -#[tokio::test] -async fn direct_undecided() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context, dag_state.clone()).build(); - - // Round 3 is a leader round - // D3 is an elected leader for wave 1 - // Round 4 is a voting round - // Round 5 is a decision round (acknowledge) - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { - A -> [D3], - B -> [D3], - C -> [], - D -> [], - }, - Round 5 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("a DAG should be valid"); - dag_builder.persist_all_blocks(dag_state); - - let leader_round = committer.leader_round(1); - tracing::info!("Leader round at wave 1: {leader_round}"); - let leader = committer - .elect_leader(leader_round) - .expect("there should be a leader at wave 1"); - let leader_status = committer.try_direct_decide(leader); - if let LeaderStatus::Undecided(_) = leader_status { - tracing::info!("Undecided: {leader_status}"); - } else { - panic!("Expected a undecided leader, got {leader_status}"); - } -} - -#[tokio::test] -async fn indirect_commit() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context, dag_state.clone()).build(); - - // Wave 1 - // Round 3 is a leader round - // D3 is an elected leader for wave 1 - // Round 4 is a voting round - // Round 5 is a decision round (acknowledge) - // - // Wave 2 - // Round 6 is a leader round - // C6 is an elected leader for wave 2 - // Round 7 is a voting round - // Round 8 is a decision round (acknowledge) - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { - A -> [D3], - B -> [D3], - C -> [D3], - D -> [], - }, - Round 5 : { - A -> [A4, B4, C4], - B -> [], - C -> [], - D -> [], - }, - Round 6 : { - A -> [], - B -> [], - C -> [A5], - D -> [], - }, - Round 7 : { - A -> [C6], - B -> [C6], - C -> [], - D -> [C6], - }, - Round 8 : { - A -> [A7, B7, D7], - B -> [A7, B7, D7], - C -> [], - D -> [A7, B7, D7], - }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("a DAG should be valid"); - dag_builder.persist_all_blocks(dag_state); - - let leader_round = committer.leader_round(1); - tracing::info!("Leader round wave 1: {leader_round}"); - let leader = committer - .elect_leader(leader_round) - .expect("there should be a leader for wave 1"); - let leader_index = leader.authority; - tracing::info!("Leader index wave 1: {leader_index}"); - - let leader_status_wave1 = committer.try_direct_decide(leader); - if let LeaderStatus::Undecided(direct_undecided) = leader_status_wave1 { - tracing::info!("Direct undecided leader at wave 1: {direct_undecided}"); - } else { - panic!( - "Expected LeaderStatus::Undecided for a leader in wave 1, applying a direct decicion rule, got {leader_status_wave1}" - ); - } - - let leader_round_wave2 = committer.leader_round(2); - tracing::info!("Leader round wave 2: {leader_round_wave2}"); - let leader_wave2 = committer - .elect_leader(leader_round_wave2) - .expect("there should be a leader for wave 2"); - let leader_index_wave2 = leader_wave2.authority; - tracing::info!("Leader index wave 2: {leader_index_wave2}"); - - let leader_status_wave_2 = committer.try_direct_decide(leader_wave2); - if let LeaderStatus::Commit(committed) = leader_status_wave_2.clone() { - tracing::info!("Direct committed leader at wave 2: {committed}"); - } else { - panic!( - "Expected LeaderStatus::Commit for a leader in wave 2, applying a direct decicion rule, got {leader_status_wave_2}" - ); - }; - - let leader_status_wave1_indirect = - committer.try_indirect_decide(leader, [leader_status_wave_2].iter()); - - if let LeaderStatus::Commit(committed) = leader_status_wave1_indirect { - tracing::info!("Indirect committed leader at wave 1: {committed}"); - } else { - panic!( - "Expected LeaderStatus::Commit for a leader in wave 1, applying an indirect decicion rule, got {leader_status_wave1_indirect}" - ); - }; -} - -/// Commit the first leader, indirectly skip the 2nd, and commit the 3rd leader. -#[tokio::test] -async fn indirect_skip() { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context, dag_state.clone()).build(); - - // There are 3 rounds. Every block is connected except - // that only f+1 validators connect to the leader of wave 2 - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { * }, - Round 5 : { * }, - Round 6 : { * }, - Round 7 : { - A -> [*], - B -> [*], - C -> [-C6], - D -> [-C6], - }, - Round 8 : { * }, - Round 9 : { * }, - Round 10 : { * }, - Round 11 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("a DAG should be valid"); - dag_builder.persist_all_blocks(dag_state); - - let leader_round = committer.leader_round(1); - tracing::info!("Leader round wave 1: {leader_round}"); - let leader = committer - .elect_leader(leader_round) - .expect("there should be a leader for wave 1"); - let leader_index = leader.authority; - tracing::info!("Leader index wave 1: {leader_index}"); - - let leader_status_wave1 = committer.try_direct_decide(leader); - if let LeaderStatus::Commit(committed) = leader_status_wave1 { - tracing::info!("Direct undecided leader at wave 1: {committed}"); - } else { - panic!( - "Expected LeaderStatus::Commit for a leader in wave 1, applying a direct decicion rule, got {leader_status_wave1}" - ); - } - - let leader_round_wave_2 = committer.leader_round(2); - tracing::info!("Leader round wave 2: {leader_round_wave_2}"); - let leader_wave2 = committer - .elect_leader(leader_round_wave_2) - .expect("there should be a leader for wave 2"); - let leader_index_wave_2 = leader_wave2.authority; - tracing::info!("Leader index wave 2: {leader_index_wave_2}"); - - let leader_status_wave_2 = committer.try_direct_decide(leader_wave2); - if let LeaderStatus::Undecided(undecided) = leader_status_wave_2.clone() { - tracing::info!("Direct committed leader at wave 2: {undecided}"); - } else { - panic!( - "Expected LeaderStatus::Undecided for a leader in wave 2, applying a direct decicion rule, got {leader_status_wave_2}" - ); - }; - - // Ensure we commit the leaders of wave 1 and 3 and skip the leader of wave 2 - - // 1. Ensure we commit the leader of wave 3. - let leader_round_wave_3 = committer.leader_round(3); - let leader_wave_3 = committer - .elect_leader(leader_round_wave_3) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_3}"); - let leader_status = committer.try_direct_decide(leader_wave_3); - tracing::info!("Leader commit status: {leader_status}"); - - let mut decided_leaders = vec![]; - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_3.authority); - decided_leaders.push(leader_status); - } else { - panic!("Expected a committed leader") - }; - - // 2. Ensure we directly mark leader of wave 2 undecided. - let leader_wave_2 = committer - .elect_leader(leader_round_wave_2) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_2}"); - let leader_status = committer.try_direct_decide(leader_wave_2); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader_wave_2) - } else { - panic!("Expected an undecided leader") - }; - - // 3. Ensure we skip leader of wave 2 indirectly. - tracing::info!("Try indirect commit for leader {leader_wave_2}",); - let leader_status = committer.try_indirect_decide(leader_wave_2, decided_leaders.iter()); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Skip(skipped_slot) = leader_status { - assert_eq!(skipped_slot, leader_wave_2) - } else { - panic!("Expected a skipped leader") - }; -} diff --git a/consensus/core/src/tests/base_committer_tests.rs b/consensus/core/src/tests/base_committer_tests.rs deleted file mode 100644 index e3e44d4853d..00000000000 --- a/consensus/core/src/tests/base_committer_tests.rs +++ /dev/null @@ -1,739 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashSet, sync::Arc}; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; - -use crate::{ - base_committer::base_committer_builder::BaseCommitterBuilder, - block::{BlockAPI, TestBlock, Transaction, VerifiedBlock}, - commit::LeaderStatus, - context::Context, - dag_state::DagState, - storage::mem_store::MemStore, - test_dag::{build_dag, build_dag_layer}, -}; - -/// Commit one leader. -#[tokio::test] -async fn try_direct_commit() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Build fully connected dag with empty blocks. Adding 8 rounds to the dag - // so that we have 2 completed waves and one incomplete wave. - // note: rounds & waves are zero indexed. - let num_rounds_in_dag = 8; - let voting_round_wave_2 = committer.leader_round(2) + 1; - let incomplete_wave_leader_round = 6; - build_dag(context, dag_state, None, voting_round_wave_2); - - // Leader rounds are the first rounds of each wave. In this case rounds 3 & 6. - let mut leader_rounds: Vec = (1..num_rounds_in_dag) - .map(|r| committer.leader_round(r)) - .collect::>() - .into_iter() - .collect(); - - // Iterate from highest leader round first - leader_rounds.sort_by(|a, b| b.cmp(a)); - for round in leader_rounds.into_iter() { - let leader = committer - .elect_leader(round) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader}",); - let leader_status = committer.try_direct_decide(leader); - tracing::info!("Leader commit status: {leader_status}"); - - if round < incomplete_wave_leader_round { - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader.authority) - } else { - panic!("Expected a committed leader at round {round}") - }; - } else { - // The base committer should mark the potential leader in r6 as undecided - // as there is no way to get enough certificates because we did not build - // the dag layer for the decision round of wave 3. - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader) - } else { - panic!("Expected an undecided leader") - }; - } - } -} - -/// Ensure idempotent replies. -#[tokio::test] -async fn idempotence() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Build fully connected dag with empty blocks. Adding 5 rounds to the dag - // aka the decision round of wave 1. - let decision_round_wave_1 = committer.decision_round(1); - build_dag(context, dag_state, None, decision_round_wave_1); - - // Commit one leader. - let leader_round_wave_1 = committer.leader_round(1); - let leader = committer - .elect_leader(leader_round_wave_1) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader}",); - let leader_status = committer.try_direct_decide(leader); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref block) = leader_status { - assert_eq!(block.author(), leader.authority) - } else { - panic!("Expected a committed leader") - }; - - // Commit the same leader again on the same dag state and get the same result - tracing::info!("Try direct commit for leader {leader} again",); - let leader_status = committer.try_direct_decide(leader); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader.authority) - } else { - panic!("Expected a committed leader") - }; -} - -/// Commit one by one each leader as the dag progresses in ideal conditions. -#[tokio::test] -async fn multiple_direct_commit() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - let mut ancestors = None; - for n in 1..=10 { - // note: rounds are zero indexed. - let decision_round = committer.decision_round(n); - ancestors = Some(build_dag( - context.clone(), - dag_state.clone(), - ancestors, - decision_round, - )); - - // Leader round is the first round of each wave. - // note: rounds are zero indexed. - let leader_round = committer.leader_round(n); - let leader = committer - .elect_leader(leader_round) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader}",); - let leader_status = committer.try_direct_decide(leader); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader.authority) - } else { - panic!("Expected a committed leader") - }; - } -} - -/// We directly skip the leader if it has enough blame. -#[tokio::test] -async fn direct_skip() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Add enough blocks to reach the leader round of wave 1. - let leader_round_wave_1 = committer.leader_round(1); - let references_leader_round_wave_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_1, - ); - - // Votes in round 4 will not include the leader of wave 1. - // Filter out that leader. - let leader_wave_1 = committer - .elect_leader(leader_round_wave_1) - .expect("should have elected leader"); - let references_without_leader_wave_1: Vec<_> = references_leader_round_wave_1 - .into_iter() - .filter(|x| x.author != leader_wave_1.authority) - .collect(); - - // Add enough blocks to reach the decision round of wave 1. - let decision_round_wave_1 = committer.decision_round(1); - build_dag( - context, - dag_state, - Some(references_without_leader_wave_1), - decision_round_wave_1, - ); - - // Ensure no blocks are committed. - tracing::info!("Try direct commit for leader {leader_wave_1}",); - let leader_status = committer.try_direct_decide(leader_wave_1); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Skip(skipped_leader) = leader_status { - assert_eq!(skipped_leader, leader_wave_1); - } else { - panic!("Expected to directly skip the leader"); - } -} - -/// Indirect-commit the first leader. -#[tokio::test] -async fn indirect_commit() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Add enough blocks to reach the leader round of wave 1. - let leader_round_wave_1 = committer.leader_round(1); - let references_leader_round_wave_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_1, - ); - - // Filter out that leader. - let leader_wave_1 = committer - .elect_leader(leader_round_wave_1) - .expect("should have elected leader"); - let references_without_leader_wave_1: Vec<_> = references_leader_round_wave_1 - .iter() - .cloned() - .filter(|x| x.author != leader_wave_1.authority) - .collect(); - - // Only 2f+1 validators vote for the leader of wave 1. - let connections_with_leader_wave_1 = context - .committee - .authorities() - .take(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_leader_round_wave_1.clone())) - .collect(); - let references_with_votes_for_leader_wave_1 = - build_dag_layer(connections_with_leader_wave_1, dag_state.clone()); - - // The validators not part of the 2f+1 above do not vote for the leader - // of wave 1 - let connections_without_leader_wave_1 = context - .committee - .authorities() - .skip(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_wave_1.clone())) - .collect(); - let references_without_votes_for_leader_wave_1 = - build_dag_layer(connections_without_leader_wave_1, dag_state.clone()); - - // Only f+1 validators certify the leader of wave 1. - let mut references_decision_round_wave_1 = Vec::new(); - - let connections_with_certs_for_leader_wave_1 = context - .committee - .authorities() - .take(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_with_votes_for_leader_wave_1.clone())) - .collect(); - references_decision_round_wave_1.extend(build_dag_layer( - connections_with_certs_for_leader_wave_1, - dag_state.clone(), - )); - - let references_voting_round_wave_1: Vec<_> = references_without_votes_for_leader_wave_1 - .into_iter() - .chain(references_with_votes_for_leader_wave_1) - .take(context.committee.quorum_threshold() as usize) - .collect(); - - // The validators not part of the f+1 above will not certify the leader of wave - // 1. - let connections_without_votes_for_leader_1 = context - .committee - .authorities() - .skip(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_voting_round_wave_1.clone())) - .collect(); - references_decision_round_wave_1.extend(build_dag_layer( - connections_without_votes_for_leader_1, - dag_state.clone(), - )); - - // Add enough blocks to decide the leader of wave 2 connecting to the references - // manually constructed of the decision round of wave 1. - let decision_round_wave_2 = committer.decision_round(2); - build_dag( - context, - dag_state, - Some(references_decision_round_wave_1), - decision_round_wave_2, - ); - - // Try direct commit leader from wave 2 which should result in Commit - let leader_wave_2 = committer - .elect_leader(committer.leader_round(2)) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_2}"); - let leader_status = committer.try_direct_decide(leader_wave_2); - tracing::info!("Leader commit status: {leader_status}"); - - let mut decided_leaders = vec![]; - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_2.authority); - decided_leaders.push(leader_status); - } else { - panic!("Expected a committed leader") - }; - - // Try direct commit leader from wave 1 which should result in Undecided - tracing::info!("Try direct commit for leader {leader_wave_1}"); - let leader_status = committer.try_direct_decide(leader_wave_1); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader_wave_1) - } else { - panic!("Expected an undecided leader") - }; - - // Quick Summary: - // Leader of wave 2 or C6 has the necessary votes/certs to be directly - // committed. Then, when we get to the leader of wave 1 or D3, we see that - // we cannot direct commit and it is marked as undecided. But this time we - // have a committed anchor so we check if there is a certified link from the - // anchor (c6) to the undecided leader (d3). There is a certified link - // through A5 with votes A4,B4,C4. So we can mark this leader as committed - // indirectly. - - // Ensure we commit the leader of wave 1 indirectly with the committed leader - // of wave 2 as the anchor. - tracing::info!("Try indirect commit for leader {leader_wave_1}",); - let leader_status = committer.try_indirect_decide(leader_wave_1, decided_leaders.iter()); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_1.authority) - } else { - panic!("Expected a committed leader") - }; -} - -/// Commit the first leader, indirectly skip the 2nd, and commit the 3rd leader. -#[tokio::test] -async fn indirect_skip() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Add enough blocks to reach the leader round of wave 2. - let leader_round_wave_2 = committer.leader_round(2); - let references_leader_round_wave_2 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_2, - ); - - // Filter out that leader. - let leader_wave_2 = committer - .elect_leader(leader_round_wave_2) - .expect("should have elected leader"); - let references_without_leader_wave_2: Vec<_> = references_leader_round_wave_2 - .iter() - .cloned() - .filter(|x| x.author != leader_wave_2.authority) - .collect(); - - // Only f+1 validators connect to the leader of wave 2. - let mut references_voting_round_wave_2 = Vec::new(); - - let connections_with_vote_leader_wave_2 = context - .committee - .authorities() - .take(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_leader_round_wave_2.clone())) - .collect(); - - references_voting_round_wave_2.extend(build_dag_layer( - connections_with_vote_leader_wave_2, - dag_state.clone(), - )); - - let connections_without_vote_leader_wave_2 = context - .committee - .authorities() - .skip(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_wave_2.clone())) - .collect(); - - references_voting_round_wave_2.extend(build_dag_layer( - connections_without_vote_leader_wave_2, - dag_state.clone(), - )); - - // Add enough blocks to reach the decision round of wave 3. - let decision_round_wave_3 = committer.decision_round(3); - build_dag( - context, - dag_state, - Some(references_voting_round_wave_2), - decision_round_wave_3, - ); - - // Ensure we commit the leaders of wave 1 and 3 and skip the leader of wave 2 - - // 1. Ensure we commit the leader of wave 3. - let leader_round_wave_3 = committer.leader_round(3); - let leader_wave_3 = committer - .elect_leader(leader_round_wave_3) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_3}"); - let leader_status = committer.try_direct_decide(leader_wave_3); - tracing::info!("Leader commit status: {leader_status}"); - - let mut decided_leaders = vec![]; - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_3.authority); - decided_leaders.push(leader_status); - } else { - panic!("Expected a committed leader") - }; - - // Leader of wave 2 is undecided directly and then skipped indirectly because - // of lack of certified links. - - // 2. Ensure we directly mark leader of wave 2 undecided. - let leader_wave_2 = committer - .elect_leader(leader_round_wave_2) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_2}"); - let leader_status = committer.try_direct_decide(leader_wave_2); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader_wave_2) - } else { - panic!("Expected an undecided leader") - }; - - // 3. Ensure we skip leader of wave 2 indirectly. - tracing::info!("Try indirect commit for leader {leader_wave_2}",); - let leader_status = committer.try_indirect_decide(leader_wave_2, decided_leaders.iter()); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Skip(skipped_slot) = leader_status { - assert_eq!(skipped_slot, leader_wave_2) - } else { - panic!("Expected a skipped leader") - }; - - // Ensure we directly commit the leader of wave 1. - let leader_round_wave_1 = committer.leader_round(1); - let leader_wave_1 = committer - .elect_leader(leader_round_wave_1) - .expect("should have elected leader"); - tracing::info!("Try direct commit for leader {leader_wave_1}"); - let leader_status = committer.try_direct_decide(leader_wave_1); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_1.authority); - } else { - panic!("Expected a committed leader") - }; -} - -/// If there is no leader with enough support nor blame, we commit nothing. -#[tokio::test] -async fn undecided() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Add enough blocks to reach the leader round of wave 1. - let leader_round_wave_1 = committer.leader_round(1); - let references_leader_round_wave_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_1, - ); - - // Filter out that leader. - let leader_wave_1 = committer - .elect_leader(leader_round_wave_1) - .expect("should have elected leader"); - let references_without_leader_wave_1: Vec<_> = references_leader_round_wave_1 - .iter() - .cloned() - .filter(|x| x.author != leader_wave_1.authority) - .collect(); - - // Create a dag layer where only one authority votes for the leader of wave 1. - let mut authorities = context.committee.authorities(); - let connections_leader_wave_1 = vec![( - authorities.next().unwrap().0, - references_leader_round_wave_1, - )]; - - // Also to ensure we have < 2f+1 blames, we take less then that for connections - // (votes) without the leader of wave 1. - let connections_without_leader_wave_1: Vec<_> = authorities - .take((context.committee.quorum_threshold() - 1) as usize) - .map(|authority| (authority.0, references_without_leader_wave_1.clone())) - .collect(); - - let connections_voting_round_wave_1 = connections_leader_wave_1 - .into_iter() - .chain(connections_without_leader_wave_1) - .collect(); - let references_voting_round_wave_1 = - build_dag_layer(connections_voting_round_wave_1, dag_state.clone()); - - // Add enough blocks to reach the decision round of wave 1. - let decision_round_wave_1 = committer.decision_round(1); - build_dag( - context.clone(), - dag_state, - Some(references_voting_round_wave_1), - decision_round_wave_1, - ); - - // Ensure we directly mark leader of wave 1 undecided as there are less than - // 2f+1 blames and 2f+1 certs - tracing::info!("Try direct commit for leader {leader_wave_1}"); - let leader_status = committer.try_direct_decide(leader_wave_1); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader_wave_1) - } else { - panic!("Expected an undecided leader") - }; - - // Ensure we indirectly mark leader of wave 1 undecided as there is no anchor - // to make an indirect decision. - tracing::info!("Try indirect commit for leader {leader_wave_1}"); - let leader_status = committer.try_indirect_decide(leader_wave_1, [].iter()); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Undecided(undecided_slot) = leader_status { - assert_eq!(undecided_slot, leader_wave_1) - } else { - panic!("Expected an undecided leader") - }; -} - -// This test scenario has one authority that is acting in a byzantine manner. It -// will be sending multiple different blocks to different validators for a -// round. The commit rule should handle this and correctly commit the expected -// blocks. -#[tokio::test] -async fn test_byzantine_direct_commit() { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let committer = BaseCommitterBuilder::new(context.clone(), dag_state.clone()).build(); - - // Add enough blocks to reach leader round of wave 4 - let leader_round_wave_4 = committer.leader_round(4); - let references_leader_round_wave_4 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_4, - ); - - // Add blocks to reach voting round of wave 4 - let voting_round_wave_4 = leader_round_wave_4 + 1; - // This includes a "good vote" from validator C which is acting as a byzantine - // validator - let good_references_voting_round_wave_4 = build_dag( - context, - dag_state.clone(), - Some(references_leader_round_wave_4.clone()), - voting_round_wave_4, - ); - - // DagState Update: - // - 'A' got a good vote from 'C' above - // - 'A' will then get a bad vote from 'C' indirectly through the ancenstors of - // the wave 4 decision blocks of B C D - - // Add block layer for wave 4 decision round with no votes for leader A12 - // from a byzantine validator C that sent different blocks to all validators. - - // Filter out leader from wave 4. - let leader_wave_4 = committer - .elect_leader(leader_round_wave_4) - .expect("should have elected leader"); - - // B12 C12 D12 - let references_without_leader_round_wave_4: Vec<_> = references_leader_round_wave_4 - .into_iter() - .filter(|x| x.author != leader_wave_4.authority) - .collect(); - - // Accept these references/blocks as ancestors from decision round blocks in dag - // state - let byzantine_block_c13_1 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![1])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_1.clone()); - - let byzantine_block_c13_2 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![2])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_2.clone()); - - let byzantine_block_c13_3 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4) - .set_transactions(vec![Transaction::new(vec![3])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_3.clone()); - - // Ancestors of decision blocks in round 14 should include multiple byzantine - // non-votes C13 but there are enough good votes to prevent a skip. - // Additionally only one of the non-votes per authority should be counted so - // we should not skip leader A12. - let decision_block_a14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 0) - .set_ancestors(good_references_voting_round_wave_4.clone()) - .build(), - ); - dag_state.write().accept_block(decision_block_a14); - - let good_references_voting_round_wave_4_without_c13 = good_references_voting_round_wave_4 - .into_iter() - .filter(|r| r.author != AuthorityIndex::new_for_test(2)) - .collect::>(); - - let decision_block_b14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 1) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_1.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_b14); - - let decision_block_c14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 2) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_2.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_c14); - - let decision_block_d14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 3) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_3.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_d14); - - // DagState Update: - // - We have A13, B13, D13 & C13 as good votes in the voting round of wave 4 - // - We have 3 byzantine C13 nonvotes that we received as ancestors from - // decision round blocks from B, C, & D. - // - We have B14, C14 & D14 that include this byzantine nonvote and A14 from - // the decision round. But all of these blocks also have good votes from A, B, - // C & D. - // Expect a successful direct commit. - - tracing::info!("Try direct commit for leader {leader_wave_4}"); - let leader_status = committer.try_direct_decide(leader_wave_4); - tracing::info!("Leader commit status: {leader_status}"); - - if let LeaderStatus::Commit(ref committed_block) = leader_status { - assert_eq!(committed_block.author(), leader_wave_4.authority); - } else { - panic!("Expected a committed leader") - }; -} - -// TODO: Add test for indirect commit with a certified link through a byzantine -// validator. - -// TODO: add basic tests for multi leader & pipeline. More tests will be added -// to thoroughly test pipelining and multileader once universal committer lands -// so these tests may not be necessary here. diff --git a/consensus/core/src/tests/pipelined_committer_tests.rs b/consensus/core/src/tests/pipelined_committer_tests.rs deleted file mode 100644 index a58911d9b04..00000000000 --- a/consensus/core/src/tests/pipelined_committer_tests.rs +++ /dev/null @@ -1,793 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; - -use crate::{ - block::{BlockAPI, Slot, TestBlock, Transaction, VerifiedBlock}, - commit::{DEFAULT_WAVE_LENGTH, DecidedLeader}, - context::Context, - dag_state::DagState, - leader_schedule::{LeaderSchedule, LeaderSwapTable}, - storage::mem_store::MemStore, - test_dag::{build_dag, build_dag_layer}, - universal_committer::universal_committer_builder::UniversalCommitterBuilder, -}; - -/// Commit one leader. -#[tokio::test] -async fn direct_commit() { - let (context, dag_state, committer) = basic_test_setup(); - - // note: pipelines, waves & rounds are zero-indexed. - let decision_round_wave_0_pipeline_1 = committer.committers[1].decision_round(0); - build_dag(context, dag_state, None, decision_round_wave_0_pipeline_1); - - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 1); - - let leader_round_wave_0_pipeline_1 = committer.committers[1].leader_round(0); - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!(block.round(), leader_round_wave_0_pipeline_1); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_wave_0_pipeline_1)[0] - ); - } else { - panic!("Expected a committed leader") - }; -} - -/// Ensure idempotent replies. -#[tokio::test] -async fn idempotence() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach decision round of pipeline 1 wave 0 which is round - // 4. note: pipelines, waves & rounds are zero-indexed. - let leader_round_pipeline_1_wave_0 = committer.committers[1].leader_round(0); - let decision_round_pipeline_1_wave_0 = committer.committers[1].decision_round(0); - build_dag(context, dag_state, None, decision_round_pipeline_1_wave_0); - - // Commit one leader. - let last_decided = Slot::new_for_test(0, 0); - let first_sequence = committer.try_decide(last_decided); - assert_eq!(first_sequence.len(), 1); - tracing::info!("Commit sequence: {first_sequence:#?}"); - - if let DecidedLeader::Commit(ref block) = first_sequence[0] { - assert_eq!(block.round(), leader_round_pipeline_1_wave_0); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_pipeline_1_wave_0)[0] - ) - } else { - panic!("Expected a committed leader") - }; - - // Ensure that if try_commit is called again with the same last decided leader - // input the commit decision will be the same. - let first_sequence = committer.try_decide(last_decided); - - assert_eq!(first_sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = first_sequence[0] { - assert_eq!(block.round(), leader_round_pipeline_1_wave_0); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_pipeline_1_wave_0)[0] - ) - } else { - panic!("Expected a committed leader") - }; - - // Ensure we don't commit the same leader again once last decided has been - // updated. - let last_decided = Slot::new(first_sequence[0].round(), first_sequence[0].authority()); - let sequence = committer.try_decide(last_decided); - assert!(sequence.is_empty()); -} - -/// Commit one by one each leader as the dag progresses in ideal conditions. -#[tokio::test] -async fn multiple_direct_commit() { - let (context, dag_state, committer) = basic_test_setup(); - let wave_length = DEFAULT_WAVE_LENGTH; - - let mut last_decided = Slot::new_for_test(0, 0); - let mut ancestors = None; - for n in 1..=10 { - // Build the dag up to the decision round for each pipeline's wave starting - // with wave 1. - // note: pipelines, waves & rounds are zero-indexed. - let pipeline = n % wave_length as usize; - let wave_number = committer.committers[pipeline].wave_number(n as u32); - let decision_round = committer.committers[pipeline].decision_round(wave_number); - let leader_round = committer.committers[pipeline].leader_round(wave_number); - - ancestors = Some(build_dag( - context.clone(), - dag_state.clone(), - ancestors, - decision_round, - )); - - // Because of pipelining we are committing a leader every round. - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!(block.round(), leader_round); - assert_eq!( - block.author(), - *committer.get_leaders(leader_round).first().unwrap() - ); - } else { - panic!("Expected a committed leader") - } - - // Update the last decided leader so only one new leader is committed as - // each new wave is completed. - let last = sequence.into_iter().next_back().unwrap(); - last_decided = Slot::new(last.round(), last.authority()); - } -} - -/// Commit 10 leaders in a row (calling the committer after adding them). -#[tokio::test] -async fn direct_commit_late_call() { - let (context, dag_state, committer) = basic_test_setup(); - let wave_length = DEFAULT_WAVE_LENGTH; - - // note: pipelines, waves & rounds are zero-indexed. - let n = 10; - let pipeline = n % wave_length as usize; - let wave_number = committer.committers[pipeline].wave_number(n as u32); - let decision_round = committer.committers[pipeline].decision_round(wave_number); - - build_dag(context, dag_state, None, decision_round); - - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), n); - for (i, leader_block) in sequence.iter().enumerate() { - // First sequenced leader should be in round 1. - let leader_round = i as u32 + 1; - if let DecidedLeader::Commit(ref block) = leader_block { - assert_eq!(block.round(), leader_round); - assert_eq!(block.author(), committer.get_leaders(leader_round)[0]); - } else { - panic!("Expected a committed leader") - }; - } -} - -/// Do not commit anything if we are still in the first wave. -#[tokio::test] -async fn no_genesis_commit() { - let (context, dag_state, committer) = basic_test_setup(); - - // Pipeline 0 wave 0 will not have a commit because its leader round is the - // genesis round. - // note: pipelines, waves & rounds are zero-indexed. - let decision_round_pipeline_0_wave_0 = committer.committers[0].decision_round(0); - - let mut ancestors = None; - for r in 0..decision_round_pipeline_0_wave_0 { - ancestors = Some(build_dag(context.clone(), dag_state.clone(), ancestors, r)); - - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - assert!(sequence.is_empty()); - } -} - -/// We do not commit anything if we miss the first leader. -#[tokio::test] -async fn direct_skip_no_leader() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach the decision round of the leader of wave 0 for - // pipeline 1 but without the leader block. - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_pipeline_1_wave_0 = committer.committers[1].leader_round(0); - let leader_pipeline_1_wave_0 = committer.get_leaders(leader_round_pipeline_1_wave_0)[0]; - - let genesis: Vec<_> = context - .committee - .authorities() - .map(|index| { - let author_idx = index.0.value() as u32; - let block = TestBlock::new(0, author_idx).build(); - VerifiedBlock::new_for_test(block).reference() - }) - .collect(); - let connections = context - .committee - .authorities() - .filter(|&authority| authority.0 != leader_pipeline_1_wave_0) - .map(|authority| (authority.0, genesis.clone())) - .collect::>(); - let references = build_dag_layer(connections, dag_state.clone()); - - let decision_round_pipeline_1_wave_0 = committer.committers[1].decision_round(0); - build_dag( - context, - dag_state, - Some(references), - decision_round_pipeline_1_wave_0, - ); - - // Ensure no blocks are committed because there are 2f+1 blame (non-votes) for - // the missing leader. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Skip(leader) = sequence[0] { - assert_eq!(leader.authority, leader_pipeline_1_wave_0); - assert_eq!(leader.round, leader_round_pipeline_1_wave_0); - } else { - panic!("Expected to directly skip the leader"); - } -} - -/// We directly skip the leader if it has enough blame. -#[tokio::test] -async fn direct_skip_enough_blame() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach the wave 0 leader for pipeline 1. - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_pipeline_1_wave_0 = committer.committers[1].leader_round(0); - let leader_pipeline_1_wave_0 = committer.get_leaders(leader_round_pipeline_1_wave_0)[0]; - let references_round_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_pipeline_1_wave_0, - ); - - // Filter out that leader. - let references_without_leader_1: Vec<_> = references_round_1 - .iter() - .cloned() - .filter(|x| x.author != leader_pipeline_1_wave_0) - .collect(); - - // 2f+1 validators non votes for that leader. - let connections_without_leader_1 = context - .committee - .authorities() - .take(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_1.clone())) - .collect(); - let references_without_votes_for_leader_1 = - build_dag_layer(connections_without_leader_1, dag_state.clone()); - - // one vote for that leader - let connections_with_leader_1 = context - .committee - .authorities() - .skip(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_round_1.clone())) - .collect(); - let references_with_votes_for_leader_1 = - build_dag_layer(connections_with_leader_1, dag_state.clone()); - - let references: Vec<_> = references_without_votes_for_leader_1 - .into_iter() - .chain(references_with_votes_for_leader_1) - .take(context.committee.quorum_threshold() as usize) - .collect(); - - // Add enough blocks to reach the decision round of the wave 0 leader for - // pipeline 1. - let decision_round_pipeline_1_wave_0 = committer.committers[1].decision_round(0); - build_dag( - context, - dag_state, - Some(references), - decision_round_pipeline_1_wave_0, - ); - - // Ensure the leader is skipped because there are 2f+1 blame (non-votes) for - // the wave 0 leader of pipeline 1. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Skip(leader) = sequence[0] { - assert_eq!(leader.authority, leader_pipeline_1_wave_0); - assert_eq!(leader.round, leader_round_pipeline_1_wave_0); - } else { - panic!("Expected to directly skip the leader"); - } -} - -/// Indirect-commit the first leader. -#[tokio::test] -async fn indirect_commit() { - let (context, dag_state, committer) = basic_test_setup(); - let wave_length = DEFAULT_WAVE_LENGTH; - - // Add enough blocks to reach the wave 0 leader of pipeline 1. - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_pipeline_1_wave_0 = committer.committers[1].leader_round(0); - let references_round_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_pipeline_1_wave_0, - ); - - // Filter out that leader. - let references_without_leader_1: Vec<_> = references_round_1 - .iter() - .cloned() - .filter(|x| { - x.author - != *committer - .get_leaders(leader_round_pipeline_1_wave_0) - .first() - .unwrap() - }) - .collect(); - - // Only 2f+1 validators vote for that leader. - let connections_with_leader_1 = context - .committee - .authorities() - .take(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_round_1.clone())) - .collect(); - let references_with_votes_for_leader_1 = - build_dag_layer(connections_with_leader_1, dag_state.clone()); - - let connections_without_leader_1 = context - .committee - .authorities() - .skip(context.committee.quorum_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_1.clone())) - .collect(); - let references_without_votes_for_leader_1 = - build_dag_layer(connections_without_leader_1, dag_state.clone()); - - // Only f+1 validators certify that leader. - let mut references_round_3 = Vec::new(); - - let connections_with_votes_for_leader_1 = context - .committee - .authorities() - .take(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_with_votes_for_leader_1.clone())) - .collect::>(); - references_round_3.extend(build_dag_layer( - connections_with_votes_for_leader_1, - dag_state.clone(), - )); - - let references: Vec<_> = references_without_votes_for_leader_1 - .into_iter() - .chain(references_with_votes_for_leader_1) - .take(context.committee.quorum_threshold() as usize) - .collect(); - let connections_without_votes_for_leader_1 = context - .committee - .authorities() - .skip(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references.clone())) - .collect::>(); - references_round_3.extend(build_dag_layer( - connections_without_votes_for_leader_1, - dag_state.clone(), - )); - - // Add enough blocks to decide the leader of round 5. The leader of round 2 will - // be skipped (it was the vote for the first leader that we removed) so we - // add enough blocks to indirectly skip it. - let leader_round_5 = 5; - let pipeline_leader_5 = leader_round_5 % wave_length as usize; - let wave_leader_5 = committer.committers[pipeline_leader_5].wave_number(leader_round_5 as u32); - let decision_round_5 = committer.committers[pipeline_leader_5].decision_round(wave_leader_5); - build_dag( - context, - dag_state, - Some(references_round_3), - decision_round_5, - ); - - // Ensure we commit the first leaders. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 5); - - let committed_leader_round = 1; - let leader = committer.get_leaders(committed_leader_round)[0]; - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!(block.round(), committed_leader_round); - assert_eq!(block.author(), leader); - } else { - panic!("Expected a committed leader") - }; - - let skipped_leader_round = 2; - let leader = committer.get_leaders(skipped_leader_round)[0]; - if let DecidedLeader::Skip(ref slot) = sequence[1] { - assert_eq!(slot.round, skipped_leader_round); - assert_eq!(slot.authority, leader); - } else { - panic!("Expected a skipped leader") - }; -} - -/// Commit the first 3 leaders, skip the 4th, and commit the next 3 leaders. -#[tokio::test] -async fn indirect_skip() { - let (context, dag_state, committer) = basic_test_setup(); - let wave_length = DEFAULT_WAVE_LENGTH; - - // Add enough blocks to reach the 4th leader. - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_4 = 4; - let references_round_4 = build_dag(context.clone(), dag_state.clone(), None, leader_round_4); - - // Filter out that leader. - let references_without_leader_4: Vec<_> = references_round_4 - .iter() - .cloned() - .filter(|x| x.author != *committer.get_leaders(leader_round_4).first().unwrap()) - .collect(); - - // Only f+1 validators connect to the 4th leader. - let mut references_round_5 = Vec::new(); - - let connections_with_leader_4 = context - .committee - .authorities() - .take(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_round_4.clone())) - .collect::>(); - references_round_5.extend(build_dag_layer( - connections_with_leader_4, - dag_state.clone(), - )); - - let connections_without_leader_4 = context - .committee - .authorities() - .skip(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_4.clone())) - .collect(); - references_round_5.extend(build_dag_layer( - connections_without_leader_4, - dag_state.clone(), - )); - - // Add enough blocks to reach the decision round of the 7th leader. - let leader_round_7 = 7; - let pipeline_leader_7 = leader_round_7 % wave_length as usize; - let wave_leader_7 = committer.committers[pipeline_leader_7].wave_number(leader_round_7 as u32); - let decision_round_7 = committer.committers[pipeline_leader_7].decision_round(wave_leader_7); - build_dag( - context, - dag_state, - Some(references_round_5), - decision_round_7, - ); - - // Ensure we commit the first 3 leaders, skip the 4th, and commit the last 2 - // leaders. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 7); - - // Ensure we commit the first 3 leaders. - for i in 0..=2 { - // First sequenced leader should be in round 1. - let leader_round = i + 1; - let leader = committer.get_leaders(leader_round)[0]; - if let DecidedLeader::Commit(ref block) = sequence[i as usize] { - assert_eq!(block.author(), leader); - } else { - panic!("Expected a committed leader") - }; - } - - // Ensure we skip the leader of wave 1 (pipeline one) but commit the others. - if let DecidedLeader::Skip(leader) = sequence[3] { - assert_eq!(leader.authority, committer.get_leaders(leader_round_4)[0]); - assert_eq!(leader.round, leader_round_4); - } else { - panic!("Expected a skipped leader") - } - - // Ensure we commit the last 3 leaders. - for i in 4..=6 { - let leader_round = i + 1; - let leader = committer.get_leaders(leader_round)[0]; - if let DecidedLeader::Commit(ref block) = sequence[i as usize] { - assert_eq!(block.author(), leader); - } else { - panic!("Expected a committed leader") - }; - } -} - -/// If there is no leader with enough support nor blame, we commit nothing. -#[tokio::test] -async fn undecided() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach the first leader. - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_1 = 1; - let references_round_1 = build_dag(context.clone(), dag_state.clone(), None, leader_round_1); - - // Filter out that leader. - let references_1_without_leader: Vec<_> = references_round_1 - .iter() - .cloned() - .filter(|x| x.author != *committer.get_leaders(leader_round_1).first().unwrap()) - .collect(); - - // Create a dag layer where only one authority votes for the first leader. - let mut authorities = context.committee.authorities(); - let leader_connection = vec![(authorities.next().unwrap().0, references_round_1)]; - let non_leader_connections: Vec<_> = authorities - .take((context.committee.quorum_threshold() - 1) as usize) - .map(|authority| (authority.0, references_1_without_leader.clone())) - .collect(); - - let connections = leader_connection - .into_iter() - .chain(non_leader_connections) - .collect::>(); - let references_voting_round_1 = build_dag_layer(connections, dag_state.clone()); - - // Add enough blocks to reach the first decision round - let decision_round_1 = committer.committers[1].decision_round(0); - build_dag( - context.clone(), - dag_state, - Some(references_voting_round_1), - decision_round_1, - ); - - // Ensure no blocks are committed. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - assert!(sequence.is_empty()); -} - -// This test scenario has one authority that is acting in a byzantine manner. It -// will be sending multiple different blocks to different validators for a -// round. The commit rule should handle this and correctly commit the expected -// blocks. However when extra dag layers are added and the byzantine node is -// meant to be a leader, its block is skipped as there is not enough votes to -// directly decide it and not any certified links to indirectly commit it. -#[tokio::test] -async fn test_byzantine_validator() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach leader A12 - // note: pipelines, waves & rounds are zero-indexed. - let leader_round_12 = 12; - let references_leader_round_12 = - build_dag(context.clone(), dag_state.clone(), None, leader_round_12); - - // Add blocks to reach voting round for leader A12 - let voting_round_12 = leader_round_12 + 1; - // This includes a "good vote" from validator B which is acting as a byzantine - // validator - let good_references_voting_round_wave_4 = build_dag( - context.clone(), - dag_state.clone(), - Some(references_leader_round_12.clone()), - voting_round_12, - ); - - // DagState Update: - // - A12 got a good vote from 'B' above - // - A12 will then get a bad vote from 'B' indirectly through the ancestors of - // the decision round blocks (B, C, & D) of leader A12 - - // Add block layer for decision round of leader A12 with no votes for leader A12 - // from a byzantine validator B that sent different blocks to all validators. - - // Filter out leader A12 - let leader_12 = committer.get_leaders(leader_round_12)[0]; - let references_without_leader_round_wave_4: Vec<_> = references_leader_round_12 - .into_iter() - .filter(|x| x.author != leader_12) - .collect(); - - // Accept these references/blocks as ancestors from decision round blocks in dag - // state - let byzantine_block_b13_1 = VerifiedBlock::new_for_test( - TestBlock::new(13, 1) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![1])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_b13_1.clone()); - - let byzantine_block_b13_2 = VerifiedBlock::new_for_test( - TestBlock::new(13, 1) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![2])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_b13_2.clone()); - - let byzantine_block_b13_3 = VerifiedBlock::new_for_test( - TestBlock::new(13, 1) - .set_ancestors(references_without_leader_round_wave_4) - .set_transactions(vec![Transaction::new(vec![3])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_b13_3.clone()); - - // Ancestors of decision blocks in round 14 should include multiple byzantine - // non-votes B13 but there are enough good votes to prevent a skip. - // Additionally only one of the non-votes per authority should be counted so - // we should not skip leader A12. - let mut references_round_14 = vec![]; - let decision_block_a14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 0) - .set_ancestors(good_references_voting_round_wave_4.clone()) - .build(), - ); - references_round_14.push(decision_block_a14.reference()); - dag_state.write().accept_block(decision_block_a14); - - let good_references_voting_round_wave_4_without_b13 = good_references_voting_round_wave_4 - .into_iter() - .filter(|r| r.author != AuthorityIndex::new_for_test(1)) - .collect::>(); - - let decision_block_b14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 1) - .set_ancestors( - good_references_voting_round_wave_4_without_b13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_b13_1.reference())) - .collect(), - ) - .build(), - ); - references_round_14.push(decision_block_b14.reference()); - dag_state.write().accept_block(decision_block_b14); - - let decision_block_c14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 2) - .set_ancestors( - good_references_voting_round_wave_4_without_b13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_b13_2.reference())) - .collect(), - ) - .build(), - ); - references_round_14.push(decision_block_c14.reference()); - dag_state.write().accept_block(decision_block_c14); - - let decision_block_d14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 3) - .set_ancestors( - good_references_voting_round_wave_4_without_b13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_b13_3.reference())) - .collect(), - ) - .build(), - ); - references_round_14.push(decision_block_d14.reference()); - dag_state.write().accept_block(decision_block_d14); - - // DagState Update: - // - We have A13, B13, D13 & C13 as good votes in the voting round of leader A12 - // - We have 3 byzantine B13 nonvotes that we received as ancestors from - // decision round blocks from B, C, & D. - // - We have B14, C14 & D14 that include this byzantine nonvote. But all of - // these blocks also have good votes from A, C & D. - - // Expect a successful direct commit of A12 and leaders at rounds 1 ~ 11 as - // pipelining is enabled. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 12); - if let DecidedLeader::Commit(ref block) = sequence[11] { - assert_eq!(block.round(), leader_round_12); - assert_eq!(block.author(), committer.get_leaders(leader_round_12)[0]) - } else { - panic!("Expected a committed leader") - }; - - // Now build an additional dag layer on top of the existing dag so a commit - // decision can be made about leader B13 which is the byzantine validator. - let references_round_15 = build_dag( - context.clone(), - dag_state.clone(), - Some(references_round_14), - 15, - ); - - // Ensure B13 is marked as undecided as there is <2f+1 blame and <2f+1 certs - let last_sequenced = sequence.last().unwrap(); - let last_decided = Slot::new(last_sequenced.round(), last_sequenced.authority()); - let sequence = committer.try_decide(last_decided); - assert!(sequence.is_empty()); - - // Now build an additional 3 dag layers on top of the existing dag so a commit - // decision can be made about leader A16 and then an indirect decision can be - // made about B13 - build_dag(context, dag_state, Some(references_round_15), 18); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 4); - - // Ensure we skip B13 as there is no way to have a certified link to any one - // of the multiple blocks at slot B13. - let skipped_leader_round = 13; - let leader = *committer.get_leaders(skipped_leader_round).first().unwrap(); - if let DecidedLeader::Skip(ref slot) = sequence[0] { - assert_eq!(slot.round, skipped_leader_round); - assert_eq!(slot.authority, leader); - } else { - panic!("Expected a skipped leader") - }; -} - -fn basic_test_setup() -> ( - Arc, - Arc>, - super::UniversalCommitter, -) { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - - // Create committer with pipelining and only 1 leader per leader round - let committer = - UniversalCommitterBuilder::new(context.clone(), leader_schedule, dag_state.clone()) - .with_pipeline(true) - .build(); - - // note: with pipelining and without multi-leader enabled there should be - // three committers. - assert!(committer.committers.len() == 3); - - (context, dag_state, committer) -} diff --git a/consensus/core/src/tests/randomized_tests.rs b/consensus/core/src/tests/randomized_tests.rs deleted file mode 100644 index d2ad9135e7b..00000000000 --- a/consensus/core/src/tests/randomized_tests.rs +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{env, sync::Arc}; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; -use rand::{Rng, SeedableRng, prelude::SliceRandom, rngs::StdRng}; - -use crate::{ - block::{BlockAPI, Slot}, - block_manager::BlockManager, - block_verifier::NoopBlockVerifier, - commit::DecidedLeader, - context::Context, - dag_state::DagState, - leader_schedule::{LeaderSchedule, LeaderSwapTable}, - storage::mem_store::MemStore, - test_dag::create_random_dag, - universal_committer::{ - UniversalCommitter, universal_committer_builder::UniversalCommitterBuilder, - }, -}; - -const NUM_RUNS: u32 = 100; -const NUM_ROUNDS: u32 = 200; - -/// Test builds a randomized dag with the following conditions: -/// - Links to 2f+1 minimum ancestors -/// - Links to leader of previous round. -/// -/// Should result in a direct commit for every round. -#[tokio::test] -async fn test_randomized_dag_all_direct_commit() { - let mut random_test_setup = random_test_setup(); - - for _ in 0..NUM_RUNS { - let seed = random_test_setup.seeded_rng.gen_range(0..10000); - let num_authorities = random_test_setup.seeded_rng.gen_range(4..10); - let authority = authority_setup(num_authorities, 0); - - let include_leader_percentage = 100; - let dag_builder = create_random_dag( - seed, - include_leader_percentage, - NUM_ROUNDS, - authority.context.clone(), - ); - - dag_builder.persist_all_blocks(authority.dag_state.clone()); - - tracing::info!( - "Running test with committee size {num_authorities} & {NUM_ROUNDS} rounds in the DAG..." - ); - - let last_decided = Slot::new_for_test(0, 0); - let sequence = authority.committer.try_decide(last_decided); - tracing::debug!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), (NUM_ROUNDS - 2) as usize); - for (i, leader_block) in sequence.iter().enumerate() { - // First sequenced leader should be in round 1. - let leader_round = i as u32 + 1; - if let DecidedLeader::Commit(ref block) = leader_block { - assert_eq!(block.round(), leader_round); - assert_eq!( - block.author(), - authority.committer.get_leaders(leader_round)[0] - ); - } else { - panic!("Expected a committed leader") - }; - } - } -} - -/// Test builds a randomized dag with the following conditions: -/// - Links to 2f+1 minimum ancestors -/// - Links to leader of previous round 50% of the time. -/// -/// Blocks will randomly be fed through BlockManager and after each accepted -/// block we will try_decide() and if there is a committed sequence we will -/// update last_decided and continue. We do this from the perspective of two -/// different authorities who receive the blocks in different orders and ensure -/// the resulting sequence is the same for both authorities. The resulting -/// sequence will include Commit & Skip decisions and potentially will stop -/// before coming to a decision on all waves as we may have an Undecided leader -/// somewhere early in the sequence. -#[tokio::test] -async fn test_randomized_dag_and_decision_sequence() { - let mut random_test_setup = random_test_setup(); - - for _ in 0..NUM_RUNS { - let seed = random_test_setup.seeded_rng.gen_range(0..10000); - let num_authorities = random_test_setup.seeded_rng.gen_range(4..10); - - // Setup for Authority 1 - let mut authority_1 = authority_setup(num_authorities, 1); - - let include_leader_percentage = 50; - let dag_builder = create_random_dag( - seed, - include_leader_percentage, - NUM_ROUNDS, - authority_1.context.clone(), - ); - - tracing::info!( - "Running test with committee size {num_authorities} & {NUM_ROUNDS} rounds in the DAG..." - ); - - let mut all_blocks = dag_builder.blocks.values().cloned().collect::>(); - all_blocks.shuffle(&mut random_test_setup.seeded_rng); - - let mut sequenced_leaders_1 = vec![]; - let mut last_decided = Slot::new_for_test(0, 0); - let mut i = 0; - while i < all_blocks.len() { - let chunk_size = random_test_setup - .seeded_rng - .gen_range(1..=(all_blocks.len() - i)); - let chunk = &all_blocks[i..i + chunk_size]; - - let _ = authority_1.block_manager.try_accept_blocks(chunk.to_vec()); - let sequence = authority_1.committer.try_decide(last_decided); - - if !sequence.is_empty() { - sequenced_leaders_1.extend(sequence.clone()); - let leader_status = sequence.last().unwrap(); - last_decided = Slot::new(leader_status.round(), leader_status.authority()); - } - - i += chunk_size; - } - - assert!(authority_1.block_manager.is_empty()); - - // Setup for Authority 2 - let mut authority_2 = authority_setup(num_authorities, 2); - - let mut all_blocks = dag_builder.blocks.values().cloned().collect::>(); - all_blocks.shuffle(&mut random_test_setup.seeded_rng); - - let mut sequenced_leaders_2 = vec![]; - let mut last_decided = Slot::new_for_test(0, 0); - let mut i = 0; - while i < all_blocks.len() { - let chunk_size = random_test_setup - .seeded_rng - .gen_range(1..=(all_blocks.len() - i)); - let chunk = &all_blocks[i..i + chunk_size]; - - let _ = authority_2.block_manager.try_accept_blocks(chunk.to_vec()); - let sequence = authority_2.committer.try_decide(last_decided); - - if !sequence.is_empty() { - sequenced_leaders_2.extend(sequence.clone()); - let leader_status = sequence.last().unwrap(); - last_decided = Slot::new(leader_status.round(), leader_status.authority()); - } - - i += chunk_size; - } - - assert!(authority_2.block_manager.is_empty()); - - // Ensure despite the difference in when blocks were received eventually after - // receiving all blocks both authorities should return the same sequence of - // blocks. - assert_eq!(sequenced_leaders_1, sequenced_leaders_2); - } -} - -struct AuthorityTestFixture { - context: Arc, - dag_state: Arc>, - committer: UniversalCommitter, - block_manager: BlockManager, -} - -fn authority_setup(num_authorities: usize, authority_index: u32) -> AuthorityTestFixture { - let context = Arc::new( - Context::new_for_test(num_authorities) - .0 - .with_authority_index(AuthorityIndex::new_for_test(authority_index)), - ); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - - // Create committer with pipelining and only 1 leader per leader round - let committer = - UniversalCommitterBuilder::new(context.clone(), leader_schedule, dag_state.clone()) - .with_pipeline(true) - .build(); - - let block_manager = BlockManager::new( - context.clone(), - dag_state.clone(), - Arc::new(NoopBlockVerifier), - ); - - AuthorityTestFixture { - context, - dag_state, - committer, - block_manager, - } -} - -struct RandomTestFixture { - seeded_rng: StdRng, -} - -fn random_test_setup() -> RandomTestFixture { - telemetry_subscribers::init_for_testing(); - let mut rng = StdRng::from_entropy(); - let seed = match env::var("DAG_TEST_SEED") { - Ok(seed_str) => { - if let Ok(seed) = seed_str.parse::() { - seed - } else { - tracing::warn!("Invalid DAG_TEST_SEED format. Using random seed."); - rng.gen_range(0..10000) - } - } - Err(_) => rng.gen_range(0..10000), - }; - tracing::warn!("Using Random Seed: {seed}"); - - let seeded_rng = StdRng::seed_from_u64(seed); - RandomTestFixture { seeded_rng } -} diff --git a/consensus/core/src/tests/universal_committer_tests.rs b/consensus/core/src/tests/universal_committer_tests.rs deleted file mode 100644 index 8c730a3250d..00000000000 --- a/consensus/core/src/tests/universal_committer_tests.rs +++ /dev/null @@ -1,770 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Arc; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; - -use crate::{ - block::{BlockAPI, Slot, TestBlock, Transaction, VerifiedBlock}, - commit::DecidedLeader, - context::Context, - dag_state::DagState, - leader_schedule::{LeaderSchedule, LeaderSwapTable}, - storage::mem_store::MemStore, - test_dag::{build_dag, build_dag_layer}, - test_dag_builder::DagBuilder, - test_dag_parser::parse_dag, - universal_committer::universal_committer_builder::UniversalCommitterBuilder, -}; - -/// Commit one leader. -#[tokio::test] -async fn direct_commit() { - let mut test_setup = basic_dag_builder_test_setup(); - - // Build fully connected dag with empty blocks adding up to voting round of - // wave 2 to the dag so that we have 2 completed waves and one incomplete wave. - // note: waves & rounds are zero-indexed. - let leader_round_wave_1 = test_setup.committer.committers[0].leader_round(1); - let voting_round_wave_2 = test_setup.committer.committers[0].leader_round(2) + 1; - test_setup - .dag_builder - .layers(1..=voting_round_wave_2) - .build() - .persist_layers(test_setup.dag_state); - - test_setup.dag_builder.print(); - - // Genesis cert will not be included in commit sequence, marking it as last - // decided - let last_decided = Slot::new_for_test(0, 0); - - // The universal committer should mark the potential leaders in leader round 6 - // as undecided because there is no way to get enough certificates for - // leaders of leader round 6 without completing wave 2. - let sequence = test_setup.committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!( - block.author(), - test_setup.committer.get_leaders(leader_round_wave_1)[0] - ) - } else { - panic!("Expected a committed leader") - }; -} - -/// Ensure idempotent replies. -#[tokio::test] -async fn idempotence() { - let (context, dag_state, committer) = basic_test_setup(); - - // note: waves & rounds are zero-indexed. - let leader_round_wave_1 = committer.committers[0].leader_round(1); - let decision_round_wave_1 = committer.committers[0].decision_round(1); - let references_decision_round_wave_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - decision_round_wave_1, - ); - - // Commit one leader. - let last_decided = Slot::new_for_test(0, 0); - let first_sequence = committer.try_decide(last_decided); - assert_eq!(first_sequence.len(), 1); - - if let DecidedLeader::Commit(ref block) = first_sequence[0] { - assert_eq!(first_sequence[0].round(), leader_round_wave_1); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_wave_1)[0] - ) - } else { - panic!("Expected a committed leader") - }; - - // Ensure that if try_commit is called again with the same last decided leader - // input the commit decision will be the same. - let first_sequence = committer.try_decide(last_decided); - - assert_eq!(first_sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = first_sequence[0] { - assert_eq!(first_sequence[0].round(), leader_round_wave_1); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_wave_1)[0] - ) - } else { - panic!("Expected a committed leader") - }; - - // Add more rounds so we have something to commit after the leader of wave 1 - let decision_round_wave_2 = committer.committers[0].decision_round(2); - build_dag( - context, - dag_state, - Some(references_decision_round_wave_1), - decision_round_wave_2, - ); - - // Ensure we don't commit the leader of wave 1 again if we mark it as the - // last decided. - let leader_status_wave_1 = first_sequence.last().unwrap(); - let last_decided = Slot::new( - leader_status_wave_1.round(), - leader_status_wave_1.authority(), - ); - let leader_round_wave_2 = committer.committers[0].leader_round(2); - let second_sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {second_sequence:#?}"); - - assert_eq!(second_sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = second_sequence[0] { - assert_eq!(second_sequence[0].round(), leader_round_wave_2); - assert_eq!( - block.author(), - committer.get_leaders(leader_round_wave_2)[0] - ); - } else { - panic!("Expected a committed leader") - }; -} - -/// Commit one by one each leader as the dag progresses in ideal conditions. -#[tokio::test] -async fn multiple_direct_commit() { - let (context, dag_state, committer) = basic_test_setup(); - - let mut ancestors = None; - let mut last_decided = Slot::new_for_test(0, 0); - for n in 1..=10 { - // Build the dag up to the decision round for each wave starting with wave 1. - // note: waves & rounds are zero-indexed. - let decision_round = committer.committers[0].decision_round(n); - ancestors = Some(build_dag( - context.clone(), - dag_state.clone(), - ancestors, - decision_round, - )); - - // After each wave is complete try commit the leader of that wave. - let leader_round = committer.committers[0].leader_round(n); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!(block.round(), leader_round); - assert_eq!(block.author(), committer.get_leaders(leader_round)[0]); - } else { - panic!("Expected a committed leader") - } - - // Update the last decided leader so only one new leader is committed as - // each new wave is completed. - let leader_status = sequence.last().unwrap(); - last_decided = Slot::new(leader_status.round(), leader_status.authority()); - } -} - -/// Commit 10 leaders in a row (calling the committer after adding them). -#[tokio::test] -async fn direct_commit_late_call() { - let (context, dag_state, committer) = basic_test_setup(); - - // note: waves & rounds are zero-indexed. - let num_waves = 11; - let decision_round_wave_10 = committer.committers[0].decision_round(10); - build_dag(context, dag_state, None, decision_round_wave_10); - - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - // With 11 waves completed, excluding wave 0 with genesis round as its leader - // round, ensure we have 10 leaders committed. - assert_eq!(sequence.len(), num_waves - 1_usize); - for (i, leader_block) in sequence.iter().enumerate() { - let leader_round = committer.committers[0].leader_round(i as u32 + 1); - if let DecidedLeader::Commit(ref block) = leader_block { - assert_eq!(block.round(), leader_round); - assert_eq!(block.author(), committer.get_leaders(leader_round)[0]); - } else { - panic!("Expected a committed leader") - }; - } -} - -/// Do not commit anything if we are still in the first wave. -#[tokio::test] -async fn no_genesis_commit() { - let (context, dag_state, committer) = basic_test_setup(); - - // note: waves & rounds are zero-indexed. - let decision_round_wave_1 = committer.committers[0].decision_round(1); - let mut ancestors = None; - for r in 0..decision_round_wave_1 { - ancestors = Some(build_dag(context.clone(), dag_state.clone(), ancestors, r)); - - let last_committed = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_committed); - tracing::info!("Commit sequence: {sequence:#?}"); - assert!(sequence.is_empty()); - } -} - -/// We directly skip the leader if there are enough non-votes (blames). -#[tokio::test] -async fn direct_skip_no_leader_votes() { - let mut test_setup = basic_dag_builder_test_setup(); - - // Add enough blocks to reach the leader round of wave 1. - // note: waves & rounds are zero-indexed. - let leader_round_wave_1 = test_setup.committer.committers[0].leader_round(1); - test_setup - .dag_builder - .layers(1..=leader_round_wave_1) - .build() - .persist_layers(test_setup.dag_state.clone()); - - // Add enough blocks to reach the decision round of the first leader but without - // votes for the leader of wave 1. - let leader_wave_1 = test_setup.committer.get_leaders(leader_round_wave_1)[0]; - let voting_round_wave_1 = leader_round_wave_1 + 1; - test_setup - .dag_builder - .layer(voting_round_wave_1) - .no_leader_link(leader_round_wave_1, vec![]) - .persist_layers(test_setup.dag_state.clone()); - - let decision_round_wave_1 = test_setup.committer.committers[0].decision_round(1); - test_setup - .dag_builder - .layer(decision_round_wave_1) - .build() - .persist_layers(test_setup.dag_state); - - test_setup.dag_builder.print(); - - // Ensure no blocks are committed because there are 2f+1 blame (non-votes) for - // the leader of wave 1. - let last_decided = Slot::new_for_test(0, 0); - let sequence = test_setup.committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Skip(leader) = sequence[0] { - assert_eq!(leader.authority, leader_wave_1); - assert_eq!(leader.round, leader_round_wave_1); - } else { - panic!("Expected to directly skip the leader"); - } -} - -/// We directly skip the leader if it is missing. -#[tokio::test] -async fn direct_skip_missing_leader_block() { - let mut test_setup = basic_dag_builder_test_setup(); - - // Add enough blocks to reach the decision round of wave 0 - // note: waves & rounds are zero-indexed. - let decision_round_wave_0 = test_setup.committer.committers[0].decision_round(0); - test_setup - .dag_builder - .layers(1..=decision_round_wave_0) - .build(); - - // Create a leader round in the dag without the leader block. - let leader_round_wave_1 = test_setup.committer.committers[0].leader_round(1); - test_setup - .dag_builder - .layer(leader_round_wave_1) - .no_leader_block(vec![]) - .build(); - - // Add enough blocks to reach the decision round of wave 1. - let voting_round_wave_1 = leader_round_wave_1 + 1; - let decision_round_wave_1 = test_setup.committer.committers[0].decision_round(1); - test_setup - .dag_builder - .layers(voting_round_wave_1..=decision_round_wave_1) - .build(); - - test_setup.dag_builder.print(); - test_setup - .dag_builder - .persist_all_blocks(test_setup.dag_state.clone()); - - // Ensure the leader is skipped because the leader is missing. - let last_committed = Slot::new_for_test(0, 0); - let sequence = test_setup.committer.try_decide(last_committed); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 1); - if let DecidedLeader::Skip(leader) = sequence[0] { - assert_eq!( - leader.authority, - test_setup.committer.get_leaders(leader_round_wave_1)[0] - ); - assert_eq!(leader.round, leader_round_wave_1); - } else { - panic!("Expected to directly skip the leader"); - } -} - -/// Indirect-commit the first leader. -#[tokio::test] -async fn indirect_commit() { - telemetry_subscribers::init_for_testing(); - // Dag Notes: - // - Fully connected up to the leader round of wave 1. - // - Only 2f+1 validators vote for the leader of wave 1. - // - Only f+1 validators certify the leader of wave 1. - // - The validators not part of the f+1 above will not certify the leader - // of wave 1. - // - Fully connected blocks to decide the leader of wave 2. - let dag_str = "DAG { - Round 0 : { 4 }, - Round 1 : { * }, - Round 2 : { * }, - Round 3 : { * }, - Round 4 : { - A -> [-D3], - B -> [*], - C -> [*], - D -> [*], - }, - Round 5 : { - A -> [*], - B -> [*], - C -> [A4], - D -> [A4], - }, - Round 6 : { * }, - Round 7 : { * }, - Round 8 : { * }, - }"; - - let (_, dag_builder) = parse_dag(dag_str).expect("Invalid dag"); - let dag_state = Arc::new(RwLock::new(DagState::new( - dag_builder.context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - dag_builder.context.clone(), - LeaderSwapTable::default(), - )); - - dag_builder.print(); - dag_builder.persist_all_blocks(dag_state.clone()); - - // Create committer without pipelining and only 1 leader per leader round - let committer = - UniversalCommitterBuilder::new(dag_builder.context, leader_schedule, dag_state).build(); - // note: without pipelining or multi-leader enabled there should only be one - // committer. - assert!(committer.committers.len() == 1); - - // Ensure we indirectly commit the leader of wave 1 via the directly committed - // leader of wave 2. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 2); - - for (idx, decided_leader) in sequence.iter().enumerate() { - let leader_round = committer.committers[0].leader_round(idx as u32 + 1); - let expected_leader = committer.get_leaders(leader_round)[0]; - if let DecidedLeader::Commit(ref block) = decided_leader { - assert_eq!(block.round(), leader_round); - assert_eq!(block.author(), expected_leader); - } else { - panic!("Expected a committed leader") - }; - } -} - -/// Commit the first leader, skip the 2nd, and commit the 3rd leader. -#[tokio::test] -async fn indirect_skip() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach the leader of wave 2 - // note: waves & rounds are zero-indexed. - let leader_round_wave_2 = committer.committers[0].leader_round(2); - let references_leader_round_wave_2 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_2, - ); - - // Filter out the leader of wave 2. - let leader_wave_2 = committer.get_leaders(leader_round_wave_2)[0]; - let references_without_leader_wave_2: Vec<_> = references_leader_round_wave_2 - .iter() - .cloned() - .filter(|x| x.author != leader_wave_2) - .collect(); - - // Only f+1 validators connect to the leader of wave 2. This is setting up the - // scenario where we have <2f+1 blame & <2f+1 certificates for the leader of - // wave 2 which will mean we mark it as Undecided. Note there are not enough - // votes to form a certified link to the leader of wave 2 as well. - let mut references = Vec::new(); - - let connections_with_leader_wave_2 = context - .committee - .authorities() - .take(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_leader_round_wave_2.clone())) - .collect::>(); - - references.extend(build_dag_layer( - connections_with_leader_wave_2, - dag_state.clone(), - )); - - let connections_without_leader_wave_2 = context - .committee - .authorities() - .skip(context.committee.validity_threshold() as usize) - .map(|authority| (authority.0, references_without_leader_wave_2.clone())) - .collect(); - - references.extend(build_dag_layer( - connections_without_leader_wave_2, - dag_state.clone(), - )); - - // Add enough blocks to reach the decision round of the leader of wave 3. - let decision_round_wave_3 = committer.committers[0].decision_round(3); - build_dag(context, dag_state, Some(references), decision_round_wave_3); - - // Ensure we make a commit decision for the leaders of wave 1 ~ 3 - let last_committed = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_committed); - tracing::info!("Commit sequence: {sequence:#?}"); - assert_eq!(sequence.len(), 3); - - // Ensure we commit the leader of wave 1 directly. - let leader_round_wave_1 = committer.committers[0].leader_round(1); - let leader_wave_1 = committer.get_leaders(leader_round_wave_1)[0]; - if let DecidedLeader::Commit(ref block) = sequence[0] { - assert_eq!(block.round(), leader_round_wave_1); - assert_eq!(block.author(), leader_wave_1); - } else { - panic!("Expected a committed leader") - }; - - // Ensure we skip the leader of wave 2 after it had been marked undecided - // directly. This happens because we do not have enough votes in voting - // round of wave 2 for the certificates of decision round wave 2 to form a - // certified link to the leader of wave 2. - if let DecidedLeader::Skip(leader) = sequence[1] { - assert_eq!(leader.authority, leader_wave_2); - assert_eq!(leader.round, leader_round_wave_2); - } else { - panic!("Expected a skipped leader") - } - - // Ensure we commit the 3rd leader directly. - let leader_round_wave_3 = committer.committers[0].leader_round(3); - let leader_wave_3 = committer.get_leaders(leader_round_wave_3)[0]; - if let DecidedLeader::Commit(ref block) = sequence[2] { - assert_eq!(block.round(), leader_round_wave_3); - assert_eq!(block.author(), leader_wave_3); - } else { - panic!("Expected a committed leader") - } -} - -/// If there is no leader with enough support nor blame, we commit nothing. -#[tokio::test] -async fn undecided() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach the leader of wave 1. - // note: waves & rounds are zero-indexed. - let leader_round_wave_1 = committer.committers[0].leader_round(1); - let references_leader_round_wave_1 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_1, - ); - - // Filter out the leader of wave 1. - let references_without_leader_1: Vec<_> = references_leader_round_wave_1 - .iter() - .cloned() - .filter(|x| x.author != committer.get_leaders(leader_round_wave_1)[0]) - .collect(); - - // Create a dag layer where only one authority votes for the leader of wave 1. - let mut authorities = context.committee.authorities(); - let leader_wave_1_connection = vec![( - authorities.next().unwrap().0, - references_leader_round_wave_1, - )]; - let non_leader_wave_1_connections: Vec<_> = authorities - .take((context.committee.quorum_threshold() - 1) as usize) - .map(|authority| (authority.0, references_without_leader_1.clone())) - .collect(); - - let connections_voting_round_wave_1 = leader_wave_1_connection - .into_iter() - .chain(non_leader_wave_1_connections) - .collect::>(); - let references_voting_round_wave_1 = - build_dag_layer(connections_voting_round_wave_1, dag_state.clone()); - - // Add enough blocks to reach the decision round of the leader of wave 1. - let decision_round_wave_1 = committer.committers[0].decision_round(1); - build_dag( - context.clone(), - dag_state, - Some(references_voting_round_wave_1), - decision_round_wave_1, - ); - - // Ensure outcome of direct & indirect rule is undecided. So not commit - // decisions should be returned. - let last_committed = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_committed); - tracing::info!("Commit sequence: {sequence:#?}"); - assert!(sequence.is_empty()); -} - -// This test scenario has one authority that is acting in a byzantine manner. It -// will be sending multiple different blocks to different validators for a -// round. The commit rule should handle this and correctly commit the expected -// blocks. -#[tokio::test] -async fn test_byzantine_direct_commit() { - let (context, dag_state, committer) = basic_test_setup(); - - // Add enough blocks to reach leader round of wave 4 - // note: waves & rounds are zero-indexed. - let leader_round_wave_4 = committer.committers[0].leader_round(4); - let references_leader_round_wave_4 = build_dag( - context.clone(), - dag_state.clone(), - None, - leader_round_wave_4, - ); - - // Add blocks to reach voting round of wave 4 - let voting_round_wave_4 = committer.committers[0].leader_round(4) + 1; - // This includes a "good vote" from validator C which is acting as a byzantine - // validator - let good_references_voting_round_wave_4 = build_dag( - context, - dag_state.clone(), - Some(references_leader_round_wave_4.clone()), - voting_round_wave_4, - ); - - // DagState Update: - // - 'A' got a good vote from 'C' above - // - 'A' will then get a bad vote from 'C' indirectly through the ancenstors of - // the wave 4 decision blocks of B C D - - // Add block layer for wave 4 decision round with no votes for leader A12 - // from a byzantine validator C that sent different blocks to all validators. - - // Filter out leader from wave 4 { A12 }. - let leader_wave_4 = committer.get_leaders(leader_round_wave_4)[0]; - - // References to blocks from leader round wave 4 { B12 C12 D12 } - let references_without_leader_round_wave_4: Vec<_> = references_leader_round_wave_4 - .into_iter() - .filter(|x| x.author != leader_wave_4) - .collect(); - - // Accept these references/blocks as ancestors from decision round blocks in dag - // state - let byzantine_block_c13_1 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![1])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_1.clone()); - - let byzantine_block_c13_2 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4.clone()) - .set_transactions(vec![Transaction::new(vec![2])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_2.clone()); - - let byzantine_block_c13_3 = VerifiedBlock::new_for_test( - TestBlock::new(13, 2) - .set_ancestors(references_without_leader_round_wave_4) - .set_transactions(vec![Transaction::new(vec![3])]) - .build(), - ); - dag_state - .write() - .accept_block(byzantine_block_c13_3.clone()); - - // Ancestors of decision blocks in round 14 should include multiple byzantine - // non-votes C13 but there are enough good votes to prevent a skip. - // Additionally only one of the non-votes per authority should be counted so - // we should not skip leader A12. - let decision_block_a14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 0) - .set_ancestors(good_references_voting_round_wave_4.clone()) - .build(), - ); - dag_state.write().accept_block(decision_block_a14); - - let good_references_voting_round_wave_4_without_c13 = good_references_voting_round_wave_4 - .into_iter() - .filter(|r| r.author != AuthorityIndex::new_for_test(2)) - .collect::>(); - - let decision_block_b14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 1) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_1.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_b14); - - let decision_block_c14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 2) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_2.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_c14); - - let decision_block_d14 = VerifiedBlock::new_for_test( - TestBlock::new(14, 3) - .set_ancestors( - good_references_voting_round_wave_4_without_c13 - .iter() - .cloned() - .chain(std::iter::once(byzantine_block_c13_3.reference())) - .collect(), - ) - .build(), - ); - dag_state.write().accept_block(decision_block_d14); - - // DagState Update: - // - We have A13, B13, D13 & C13 as good votes in the voting round of wave 4 - // - We have 3 byzantine C13 nonvotes that we received as ancestors from - // decision round blocks from B, C, & D. - // - We have B14, C14 & D14 that include this byzantine nonvote from C13 but - // all of these blocks also have good votes for leader A12 through A, B, D. - - // Expect a successful direct commit of A12 and leaders at rounds 9, 6 & 3. - let last_decided = Slot::new_for_test(0, 0); - let sequence = committer.try_decide(last_decided); - tracing::info!("Commit sequence: {sequence:#?}"); - - assert_eq!(sequence.len(), 4); - if let DecidedLeader::Commit(ref block) = sequence[3] { - assert_eq!( - block.author(), - committer.get_leaders(leader_round_wave_4)[0] - ) - } else { - panic!("Expected a committed leader") - }; -} - -// TODO: Add byzantine variant of tests for indirect/direct -// commit/skip/undecided decisions - -fn basic_test_setup() -> ( - Arc, - Arc>, - super::UniversalCommitter, -) { - telemetry_subscribers::init_for_testing(); - // Committee of 4 with even stake - let context = Arc::new(Context::new_for_test(4).0); - let dag_state = Arc::new(RwLock::new(DagState::new( - context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - context.clone(), - LeaderSwapTable::default(), - )); - - // Create committer without pipelining and only 1 leader per leader round - let committer = - UniversalCommitterBuilder::new(context.clone(), leader_schedule, dag_state.clone()).build(); - - // note: without pipelining or multi-leader enabled there should only be one - // committer. - assert!(committer.committers.len() == 1); - - (context, dag_state, committer) -} - -struct TestSetup { - dag_builder: DagBuilder, - dag_state: Arc>, - committer: super::UniversalCommitter, -} - -// TODO: Make this the basic_test_setup() -fn basic_dag_builder_test_setup() -> TestSetup { - telemetry_subscribers::init_for_testing(); - let context = Arc::new(Context::new_for_test(4).0); - let dag_builder = DagBuilder::new(context); - - let dag_state = Arc::new(RwLock::new(DagState::new( - dag_builder.context.clone(), - Arc::new(MemStore::new()), - ))); - let leader_schedule = Arc::new(LeaderSchedule::new( - dag_builder.context.clone(), - LeaderSwapTable::default(), - )); - - // Create committer without pipelining and only 1 leader per leader round - let committer = UniversalCommitterBuilder::new( - dag_builder.context.clone(), - leader_schedule, - dag_state.clone(), - ) - .build(); - // note: without pipelining or multi-leader enabled there should only be one - // committer. - assert!(committer.committers.len() == 1); - - TestSetup { - dag_builder, - dag_state, - committer, - } -} diff --git a/consensus/core/src/threshold_clock.rs b/consensus/core/src/threshold_clock.rs deleted file mode 100644 index c51e4f1b7bc..00000000000 --- a/consensus/core/src/threshold_clock.rs +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{cmp::Ordering, sync::Arc}; - -use tokio::time::Instant; - -use crate::{ - block::{BlockRef, Round}, - context::Context, - stake_aggregator::{QuorumThreshold, StakeAggregator}, -}; - -pub(crate) struct ThresholdClock { - aggregator: StakeAggregator, - round: Round, - quorum_ts: Instant, - context: Arc, -} - -impl ThresholdClock { - pub(crate) fn new(round: Round, context: Arc) -> Self { - Self { - aggregator: StakeAggregator::new(), - round, - quorum_ts: Instant::now(), - context, - } - } - - /// If quorum (2f+1) is reached, advance to the next round and record - /// latency metrics. Returns true if quorum was reached. - fn try_advance_round(&mut self, new_round: Round) -> bool { - if self.aggregator.reached_threshold(&self.context.committee) { - self.aggregator.clear(); - self.round = new_round; - - let now = Instant::now(); - self.context - .metrics - .node_metrics - .quorum_receive_latency - .observe(now.duration_since(self.quorum_ts).as_secs_f64()); - self.quorum_ts = now; - true - } else { - false - } - } - - /// Add the block reference and advance the round accordingly. - /// - /// Round advancement rules: - /// - block.round < current: ignored (stale block) - /// - block.round > current: jump to block.round, start collecting stake - /// there - /// - block.round == current: continue accumulating stake until quorum - /// (2f+1) reached - /// - /// When quorum is reached, advance to round + 1. - pub(crate) fn add_block(&mut self, block: BlockRef) { - match block.round.cmp(&self.round) { - Ordering::Less => {} - Ordering::Greater => { - // Jump to the new round and start with fresh state - self.aggregator.clear(); - self.aggregator.add(block.author, &self.context.committee); - self.round = block.round; - } - Ordering::Equal => { - self.aggregator.add(block.author, &self.context.committee); - } - } - self.try_advance_round(block.round + 1); - } - - /// Add the block references that have been successfully processed and - /// advance the round accordingly. If the round has indeed advanced then - /// the new round is returned, otherwise None is returned. - #[cfg(test)] - fn add_blocks(&mut self, blocks: Vec) -> Option { - let previous_round = self.round; - for block_ref in blocks { - self.add_block(block_ref); - } - (self.round > previous_round).then_some(self.round) - } - - pub(crate) fn get_round(&self) -> Round { - self.round - } - - pub(crate) fn get_quorum_ts(&self) -> Instant { - self.quorum_ts - } -} - -#[cfg(test)] -mod tests { - use consensus_config::AuthorityIndex; - use iota_common::scoring_metrics::{ScoringMetricsV1, VersionedScoringMetrics}; - use iota_protocol_config::ProtocolConfig; - - use super::*; - use crate::{block::BlockDigest, scoring_metrics_store::MysticetiScoringMetricsStore}; - - #[tokio::test] - async fn test_threshold_clock_add_block() { - let context = Arc::new(Context::new_for_test(4).0); - let mut aggregator = ThresholdClock::new(0, context); - - aggregator.add_block(BlockRef::new( - 0, - AuthorityIndex::new_for_test(0), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 0); - aggregator.add_block(BlockRef::new( - 0, - AuthorityIndex::new_for_test(1), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 0); - aggregator.add_block(BlockRef::new( - 0, - AuthorityIndex::new_for_test(2), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 1); - aggregator.add_block(BlockRef::new( - 1, - AuthorityIndex::new_for_test(0), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 1); - aggregator.add_block(BlockRef::new( - 1, - AuthorityIndex::new_for_test(3), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 1); - aggregator.add_block(BlockRef::new( - 2, - AuthorityIndex::new_for_test(1), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 2); - aggregator.add_block(BlockRef::new( - 1, - AuthorityIndex::new_for_test(1), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 2); - aggregator.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(2), - BlockDigest::default(), - )); - assert_eq!(aggregator.get_round(), 5); - } - - #[tokio::test] - async fn test_threshold_clock_add_blocks() { - let context = Arc::new(Context::new_for_test(4).0); - let mut aggregator = ThresholdClock::new(0, context); - - let block_refs = vec![ - BlockRef::new(0, AuthorityIndex::new_for_test(0), BlockDigest::default()), - BlockRef::new(0, AuthorityIndex::new_for_test(1), BlockDigest::default()), - BlockRef::new(0, AuthorityIndex::new_for_test(2), BlockDigest::default()), - BlockRef::new(1, AuthorityIndex::new_for_test(0), BlockDigest::default()), - BlockRef::new(1, AuthorityIndex::new_for_test(3), BlockDigest::default()), - BlockRef::new(2, AuthorityIndex::new_for_test(1), BlockDigest::default()), - BlockRef::new(1, AuthorityIndex::new_for_test(1), BlockDigest::default()), - BlockRef::new(5, AuthorityIndex::new_for_test(2), BlockDigest::default()), - ]; - - let result = aggregator.add_blocks(block_refs); - assert_eq!(Some(5), result); - } - - /// Test that when jumping to a higher round, the first block's author is - /// tracked, allowing subsequent blocks to form quorum. - #[tokio::test] - async fn test_threshold_clock_round_skip_then_quorum() { - let context = Arc::new(Context::new_for_test(4).0); - let mut clock = ThresholdClock::new(0, context); - - // Jump from round 0 to round 5 - author should be tracked - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(0), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 5); - - // Add more blocks at round 5 to reach quorum (need 3 of 4) - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(1), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 5); - - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(2), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 6); // Quorum reached - } - - /// Test that a super-majority authority (>2/3 stake) immediately advances - /// the round when jumping to a higher round. - #[tokio::test] - async fn test_threshold_clock_super_majority_round_skip() { - use consensus_config::Parameters; - use tempfile::TempDir; - - use crate::metrics::test_metrics; - - // Authority 0 has 5/7 stake (>2/3 quorum threshold) - let (committee, _) = consensus_config::local_committee_and_keys(0, vec![5, 1, 1]); - let committee_size = committee.size(); - let metrics = test_metrics(); - let temp_dir = TempDir::new().unwrap(); - let current_local_metrics_count = - Arc::new(VersionedScoringMetrics::V1(ScoringMetricsV1::new(3))); - let scoring_metrics_store = Arc::new(MysticetiScoringMetricsStore::new( - committee_size, - current_local_metrics_count, - &ProtocolConfig::get_for_max_version_UNSAFE(), - )); - - let context = Arc::new(crate::context::Context::new( - 0, - AuthorityIndex::new_for_test(0), - committee, - Parameters { - db_path: temp_dir.keep(), - ..Default::default() - }, - iota_protocol_config::ProtocolConfig::get_for_max_version_UNSAFE(), - metrics, - scoring_metrics_store, - Arc::new(crate::context::Clock::default()), - )); - - let mut clock = ThresholdClock::new(0, context); - - // Single block from super-majority authority at round 5 reaches quorum - // immediately - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(0), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 6); - - // Stale blocks from round 5 should be ignored - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(1), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 6); - clock.add_block(BlockRef::new( - 5, - AuthorityIndex::new_for_test(2), - BlockDigest::default(), - )); - assert_eq!(clock.get_round(), 6); - } -} diff --git a/consensus/core/src/transaction.rs b/consensus/core/src/transaction.rs deleted file mode 100644 index 1e38f28398d..00000000000 --- a/consensus/core/src/transaction.rs +++ /dev/null @@ -1,869 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::BTreeMap, sync::Arc}; - -use iota_common::debug_fatal; -use iota_metrics::monitored_mpsc::{Receiver, Sender, channel}; -use parking_lot::Mutex; -use tap::tap::TapFallible; -use thiserror::Error; -use tokio::sync::oneshot; -use tracing::{error, warn}; - -use crate::{ - Round, - block::{BlockRef, Transaction}, - context::Context, -}; - -/// The maximum number of transactions pending to the queue to be pulled for -/// block proposal -const MAX_PENDING_TRANSACTIONS: usize = 2_000; - -/// The guard acts as an acknowledgment mechanism for the inclusion of the -/// transactions to a block. When its last transaction is included to a block -/// then `included_in_block_ack` will be signalled. If the guard is dropped -/// without getting acknowledged that means the transactions have not been -/// included to a block and the consensus is shutting down. -pub(crate) struct TransactionsGuard { - // Holds a list of transactions to be included in the block. - // A TransactionsGuard may be partially consumed by `TransactionConsumer`, in which case, this - // holds the remaining transactions. - transactions: Vec, - - included_in_block_ack: oneshot::Sender<(BlockRef, oneshot::Receiver)>, -} - -/// The TransactionConsumer is responsible for fetching the next transactions to -/// be included for the block proposals. The transactions are submitted to a -/// channel which is shared between the TransactionConsumer and the -/// TransactionClient and are pulled every time the `next` method is called. -pub(crate) struct TransactionConsumer { - context: Arc, - tx_receiver: Receiver, - max_transactions_in_block_bytes: u64, - max_num_transactions_in_block: u64, - pending_transactions: Option, - block_status_subscribers: Arc>>>>, -} - -#[derive(Debug, Clone, Eq, PartialEq)] -#[allow(unused)] -pub enum BlockStatus { - /// The block has been sequenced as part of a committed sub dag. That means - /// that any transaction that has been included in the block - /// has been committed as well. - Sequenced(BlockRef), - /// The block has been garbage collected and will never be committed. Any - /// transactions that have been included in the block should also - /// be considered as impossible to be committed as part of this block and - /// might need to be retried - GarbageCollected(BlockRef), -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum LimitReached { - // The maximum number of transactions have been included - MaxNumOfTransactions, - // The maximum number of bytes have been included - MaxBytes, - // All available transactions have been included - AllTransactionsIncluded, -} - -impl TransactionConsumer { - pub(crate) fn new(tx_receiver: Receiver, context: Arc) -> Self { - Self { - tx_receiver, - max_transactions_in_block_bytes: context - .protocol_config - .max_transactions_in_block_bytes(), - max_num_transactions_in_block: context.protocol_config.max_num_transactions_in_block(), - context, - pending_transactions: None, - block_status_subscribers: Arc::new(Mutex::new(BTreeMap::new())), - } - } - - // Attempts to fetch the next transactions that have been submitted for - // sequence. Respects the `max_transactions_in_block_bytes` - // and `max_num_transactions_in_block` parameters specified via protocol config. - // This returns one or more transactions to be included in the block and a - // callback to acknowledge the inclusion of those transactions. Also returns - // a `LimitReached` enum to indicate which limit type has been reached. - pub(crate) fn next(&mut self) -> (Vec, Box, LimitReached) { - let mut transactions = Vec::new(); - let mut acks = Vec::new(); - let mut total_bytes = 0; - let mut limit_reached = LimitReached::AllTransactionsIncluded; - - // Handle one batch of incoming transactions from TransactionGuard. - // The method will return `None` if all the transactions can be included in the - // block. Otherwise none of the transactions will be included in the - // block and the method will return the TransactionGuard. - let mut handle_txs = |t: TransactionsGuard| -> Option { - let transactions_bytes = - t.transactions.iter().map(|t| t.data().len()).sum::() as u64; - let transactions_num = t.transactions.len() as u64; - - if total_bytes + transactions_bytes > self.max_transactions_in_block_bytes { - limit_reached = LimitReached::MaxBytes; - return Some(t); - } - if transactions.len() as u64 + transactions_num > self.max_num_transactions_in_block { - limit_reached = LimitReached::MaxNumOfTransactions; - return Some(t); - } - - total_bytes += transactions_bytes; - - // The transactions can be consumed, register its ack. - acks.push(t.included_in_block_ack); - transactions.extend(t.transactions); - None - }; - - if let Some(t) = self.pending_transactions.take() { - if let Some(pending_transactions) = handle_txs(t) { - debug_fatal!( - "Previously pending transaction(s) should fit into an empty block! Dropping: {:?}", - pending_transactions.transactions - ); - } - } - - // Until we have reached the limit for the pull. - // We may have already reached limit in the first iteration above, in which case - // we stop immediately. - while self.pending_transactions.is_none() { - if let Ok(t) = self.tx_receiver.try_recv() { - self.pending_transactions = handle_txs(t); - } else { - break; - } - } - - let block_status_subscribers = self.block_status_subscribers.clone(); - let gc_enabled = self.context.protocol_config.gc_depth() > 0; - - ( - transactions, - Box::new(move |block_ref: BlockRef| { - let mut block_status_subscribers = block_status_subscribers.lock(); - - for ack in acks { - let (status_tx, status_rx) = oneshot::channel(); - - if gc_enabled { - block_status_subscribers - .entry(block_ref) - .or_default() - .push(status_tx); - } else { - // When gc is not enabled, then report directly the block as sequenced while - // tx is acknowledged for inclusion. As blocks can - // never get garbage collected it is there is actually no meaning to do - // otherwise and also is safer for edge cases. - status_tx.send(BlockStatus::Sequenced(block_ref)).ok(); - } - - let _ = ack.send((block_ref, status_rx)); - } - }), - limit_reached, - ) - } - - /// Notifies all the transaction submitters who are waiting to receive an - /// update on the status of the block. The `committed_blocks` are the - /// blocks that have been committed and the `gc_round` is the round up to - /// which the blocks have been garbage collected. First we'll notify for - /// all the committed blocks, and then for all the blocks that have been - /// garbage collected. - pub(crate) fn notify_own_blocks_status( - &self, - committed_blocks: Vec, - gc_round: Round, - ) { - // Notify for all the committed blocks first - let mut block_status_subscribers = self.block_status_subscribers.lock(); - for block_ref in committed_blocks { - if let Some(subscribers) = block_status_subscribers.remove(&block_ref) { - subscribers.into_iter().for_each(|s| { - let _ = s.send(BlockStatus::Sequenced(block_ref)); - }); - } - } - - // Now notify everyone <= gc_round that their block has been garbage collected - // and clean up the entries - while let Some((block_ref, subscribers)) = block_status_subscribers.pop_first() { - if block_ref.round <= gc_round { - subscribers.into_iter().for_each(|s| { - let _ = s.send(BlockStatus::GarbageCollected(block_ref)); - }); - } else { - block_status_subscribers.insert(block_ref, subscribers); - break; - } - } - } - - #[cfg(test)] - pub(crate) fn subscribe_for_block_status_testing( - &self, - block_ref: BlockRef, - ) -> oneshot::Receiver { - let (tx, rx) = oneshot::channel(); - let mut block_status_subscribers = self.block_status_subscribers.lock(); - block_status_subscribers - .entry(block_ref) - .or_default() - .push(tx); - rx - } - - #[cfg(test)] - fn is_empty(&mut self) -> bool { - if self.pending_transactions.is_some() { - return false; - } - if let Ok(t) = self.tx_receiver.try_recv() { - self.pending_transactions = Some(t); - return false; - } - true - } -} - -#[derive(Clone)] -pub struct TransactionClient { - sender: Sender, - max_transaction_size: u64, - max_transactions_in_block_bytes: u64, - max_transactions_in_block_count: u64, -} - -#[derive(Debug, Error)] -pub enum ClientError { - #[error("Failed to submit transaction, consensus is shutting down: {0}")] - ConsensusShuttingDown(String), - - #[error("Transaction size ({0}B) is over limit ({1}B)")] - OversizedTransaction(u64, u64), - - #[error("Transaction bundle size ({0}B) is over limit ({1}B)")] - OversizedTransactionBundleBytes(u64, u64), - - #[error("Transaction bundle count ({0}) is over limit ({1})")] - OversizedTransactionBundleCount(u64, u64), -} - -impl TransactionClient { - pub(crate) fn new(context: Arc) -> (Self, Receiver) { - let (sender, receiver) = channel("consensus_input", MAX_PENDING_TRANSACTIONS); - - ( - Self { - sender, - max_transaction_size: context.protocol_config.max_transaction_size_bytes(), - max_transactions_in_block_bytes: context - .protocol_config - .max_transactions_in_block_bytes(), - max_transactions_in_block_count: context - .protocol_config - .max_num_transactions_in_block(), - }, - receiver, - ) - } - - /// Submits a list of transactions to be sequenced. The method returns when - /// all the transactions have been successfully included - /// to next proposed blocks. - pub async fn submit( - &self, - transactions: Vec>, - ) -> Result<(BlockRef, oneshot::Receiver), ClientError> { - // TODO: Support returning the block refs for transactions that span multiple - // blocks - let included_in_block = self.submit_no_wait(transactions).await?; - included_in_block - .await - .tap_err(|e| warn!("Transaction acknowledge failed with {:?}", e)) - .map_err(|e| ClientError::ConsensusShuttingDown(e.to_string())) - } - - /// Submits a list of transactions to be sequenced. - /// If any transaction's length exceeds `max_transaction_size`, no - /// transaction will be submitted. That shouldn't be the common case as - /// sizes should be aligned between consensus and client. The method returns - /// a receiver to wait on until the transactions has been included in the - /// next block to get proposed. The consumer should wait on it to - /// consider as inclusion acknowledgement. If the receiver errors then - /// consensus is shutting down and transaction has not been included to - /// any block. If multiple transactions are submitted, the method will - /// attempt to bundle them together in a single block. If the total size of - /// the transactions exceeds `max_transactions_in_block_bytes`, no - /// transaction will be submitted and an error will be returned instead. - /// Similar if transactions exceed `max_transactions_in_block_count` an - /// error will be returned. - pub(crate) async fn submit_no_wait( - &self, - transactions: Vec>, - ) -> Result)>, ClientError> { - let (included_in_block_ack_send, included_in_block_ack_receive) = oneshot::channel(); - - let mut bundle_size = 0; - - if transactions.len() as u64 > self.max_transactions_in_block_count { - return Err(ClientError::OversizedTransactionBundleCount( - transactions.len() as u64, - self.max_transactions_in_block_count, - )); - } - - for transaction in &transactions { - if transaction.len() as u64 > self.max_transaction_size { - return Err(ClientError::OversizedTransaction( - transaction.len() as u64, - self.max_transaction_size, - )); - } - bundle_size += transaction.len() as u64; - - if bundle_size > self.max_transactions_in_block_bytes { - return Err(ClientError::OversizedTransactionBundleBytes( - bundle_size, - self.max_transactions_in_block_bytes, - )); - } - } - - let t = TransactionsGuard { - transactions: transactions.into_iter().map(Transaction::new).collect(), - included_in_block_ack: included_in_block_ack_send, - }; - self.sender - .send(t) - .await - .tap_err(|e| error!("Submit transactions failed with {:?}", e)) - .map_err(|e| ClientError::ConsensusShuttingDown(e.to_string()))?; - Ok(included_in_block_ack_receive) - } -} - -/// `TransactionVerifier` implementation is supplied by IOTA to validate -/// transactions in a block, before acceptance of the block. -pub trait TransactionVerifier: Send + Sync + 'static { - /// Determines if this batch can be voted on - fn verify_batch(&self, batch: &[&[u8]]) -> Result<(), ValidationError>; -} - -#[derive(Debug, Error)] -pub enum ValidationError { - #[error("Invalid transaction: {0}")] - InvalidTransaction(String), -} - -/// `NoopTransactionVerifier` accepts all transactions. -#[cfg(any(test, msim))] -pub struct NoopTransactionVerifier; - -#[cfg(any(test, msim))] -impl TransactionVerifier for NoopTransactionVerifier { - fn verify_batch(&self, _batch: &[&[u8]]) -> Result<(), ValidationError> { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::{sync::Arc, time::Duration}; - - use consensus_config::AuthorityIndex; - use futures::{StreamExt, stream::FuturesUnordered}; - use iota_protocol_config::ProtocolConfig; - use tokio::time::timeout; - - use crate::{ - block::{BlockDigest, BlockRef}, - block_verifier::SignedBlockVerifier, - context::Context, - transaction::{ - BlockStatus, LimitReached, NoopTransactionVerifier, TransactionClient, - TransactionConsumer, - }, - }; - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn basic_submit_and_consume() { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(2_000); // 2KB - config.set_consensus_max_transactions_in_block_bytes_for_testing(2_000); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // submit asynchronously the transactions and keep the waiters - let mut included_in_block_waiters = FuturesUnordered::new(); - for i in 0..3 { - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Shouldn't submit successfully transaction"); - included_in_block_waiters.push(w); - } - - // now pull the transactions from the consumer - let (transactions, ack_transactions, _limit_reached) = consumer.next(); - assert_eq!(transactions.len(), 3); - - for (i, t) in transactions.iter().enumerate() { - let t: String = bcs::from_bytes(t.data()).unwrap(); - assert_eq!(format!("transaction {i}").to_string(), t); - } - - assert!( - timeout(Duration::from_secs(1), included_in_block_waiters.next()) - .await - .is_err(), - "We should expect to timeout as none of the transactions have been acknowledged yet" - ); - - // Now acknowledge the inclusion of transactions - ack_transactions(BlockRef::MIN); - - // Now make sure that all the waiters have returned - while let Some(result) = included_in_block_waiters.next().await { - assert!(result.is_ok()); - } - - // try to pull again transactions, result should be empty - assert!(consumer.is_empty()); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn block_status_update_gc_enabled() { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(2_000); // 2KB - config.set_consensus_max_transactions_in_block_bytes_for_testing(2_000); - config.set_consensus_gc_depth_for_testing(10); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // submit the transactions and include 2 of each on a new block - let mut included_in_block_waiters = FuturesUnordered::new(); - for i in 1..=10 { - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Shouldn't submit successfully transaction"); - included_in_block_waiters.push(w); - - // Every 2 transactions simulate the creation of a new block and acknowledge the - // inclusion of the transactions - if i % 2 == 0 { - let (transactions, ack_transactions, _limit_reached) = consumer.next(); - assert_eq!(transactions.len(), 2); - ack_transactions(BlockRef::new( - i, - AuthorityIndex::new_for_test(0), - BlockDigest::MIN, - )); - } - } - - // Now iterate over all the waiters. Everyone should have been acknowledged. - let mut block_status_waiters = Vec::new(); - while let Some(result) = included_in_block_waiters.next().await { - let (block_ref, block_status_waiter) = - result.expect("Block inclusion waiter shouldn't fail"); - block_status_waiters.push((block_ref, block_status_waiter)); - } - - // Now acknowledge the commit of the blocks 6, 8, 10 and set gc_round = 5, which - // should trigger the garbage collection of blocks 1..=5 - let gc_round = 5; - consumer.notify_own_blocks_status( - vec![ - BlockRef::new(6, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(8, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - BlockRef::new(10, AuthorityIndex::new_for_test(0), BlockDigest::MIN), - ], - gc_round, - ); - - // Now iterate over all the block status waiters. Everyone should have been - // notified. - for (block_ref, waiter) in block_status_waiters { - let block_status = waiter.await.expect("Block status waiter shouldn't fail"); - - if block_ref.round <= gc_round { - assert!(matches!(block_status, BlockStatus::GarbageCollected(_))) - } else { - assert!(matches!(block_status, BlockStatus::Sequenced(_))); - } - } - - // Ensure internal structure is clear - assert!(consumer.block_status_subscribers.lock().is_empty()); - } - - #[tokio::test(flavor = "current_thread", start_paused = true)] - async fn block_status_update_gc_disabled() { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(2_000); // 2KB - config.set_consensus_max_transactions_in_block_bytes_for_testing(2_000); - config.set_consensus_gc_depth_for_testing(0); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // submit the transactions and include 2 of each on a new block - let mut included_in_block_waiters = FuturesUnordered::new(); - for i in 1..=10 { - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Shouldn't submit successfully transaction"); - included_in_block_waiters.push(w); - - // Every 2 transactions simulate the creation of a new block and acknowledge the - // inclusion of the transactions - if i % 2 == 0 { - let (transactions, ack_transactions, _limit_reached) = consumer.next(); - assert_eq!(transactions.len(), 2); - ack_transactions(BlockRef::new( - i, - AuthorityIndex::new_for_test(0), - BlockDigest::MIN, - )); - } - } - - // Now iterate over all the waiters. Everyone should have been acknowledged. - let mut block_status_waiters = Vec::new(); - while let Some(result) = included_in_block_waiters.next().await { - let (block_ref, block_status_waiter) = - result.expect("Block inclusion waiter shouldn't fail"); - block_status_waiters.push((block_ref, block_status_waiter)); - } - - // Now iterate over all the block status waiters. Everyone should have been - // notified and everyone should be considered sequenced. - for (_block_ref, waiter) in block_status_waiters { - let block_status = waiter.await.expect("Block status waiter shouldn't fail"); - assert!(matches!(block_status, BlockStatus::Sequenced(_))); - } - - // Ensure internal structure is clear - assert!(consumer.block_status_subscribers.lock().is_empty()); - } - - #[tokio::test] - async fn submit_over_max_fetch_size_and_consume() { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(100); - config.set_consensus_max_transactions_in_block_bytes_for_testing(100); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - - // submit some transactions - for i in 0..10 { - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let _w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Shouldn't submit successfully transaction"); - } - - // now pull the transactions from the consumer - let mut all_transactions = Vec::new(); - let (transactions, _ack_transactions, _limit_reached) = consumer.next(); - assert_eq!(transactions.len(), 7); - - // ensure their total size is less than `max_bytes_to_fetch` - let total_size: u64 = transactions.iter().map(|t| t.data().len() as u64).sum(); - assert!( - total_size <= context.protocol_config.max_transactions_in_block_bytes(), - "Should have fetched transactions up to {}", - context.protocol_config.max_transactions_in_block_bytes() - ); - all_transactions.extend(transactions); - - // try to pull again transactions, next should be provided - let (transactions, _ack_transactions, _limit_reached) = consumer.next(); - assert_eq!(transactions.len(), 3); - - // ensure their total size is less than `max_bytes_to_fetch` - let total_size: u64 = transactions.iter().map(|t| t.data().len() as u64).sum(); - assert!( - total_size <= context.protocol_config.max_transactions_in_block_bytes(), - "Should have fetched transactions up to {}", - context.protocol_config.max_transactions_in_block_bytes() - ); - all_transactions.extend(transactions); - - // try to pull again transactions, result should be empty - assert!(consumer.is_empty()); - - for (i, t) in all_transactions.iter().enumerate() { - let t: String = bcs::from_bytes(t.data()).unwrap(); - assert_eq!(format!("transaction {i}").to_string(), t); - } - } - - #[tokio::test] - async fn submit_large_batch_and_ack() { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(15); - config.set_consensus_max_transactions_in_block_bytes_for_testing(200); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let mut all_receivers = Vec::new(); - // submit a few transactions individually. - for i in 0..10 { - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Should submit successfully transaction"); - all_receivers.push(w); - } - - // construct an acceptable batch and submit, it should be accepted - { - let transactions: Vec<_> = (10..15) - .map(|i| { - bcs::to_bytes(&format!("transaction {i}")) - .expect("Serialization should not fail.") - }) - .collect(); - let w = client - .submit_no_wait(transactions) - .await - .expect("Should submit successfully transaction"); - all_receivers.push(w); - } - - // submit another individual transaction. - { - let i = 15; - let transaction = - bcs::to_bytes(&format!("transaction {i}")).expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Shouldn't submit successfully transaction"); - all_receivers.push(w); - } - - // construct a over-size-limit batch and submit, it should not be accepted - { - let transactions: Vec<_> = (16..32) - .map(|i| { - bcs::to_bytes(&format!("transaction {i}")) - .expect("Serialization should not fail.") - }) - .collect(); - let result = client.submit_no_wait(transactions).await.unwrap_err(); - assert_eq!( - result.to_string(), - "Transaction bundle size (210B) is over limit (200B)" - ); - } - - // now pull the transactions from the consumer. - // we expect all transactions are fetched in order, not missing any, and not - // exceeding the size limit. - let mut all_acks: Vec> = Vec::new(); - let mut batch_index = 0; - while !consumer.is_empty() { - let (transactions, ack_transactions, _limit_reached) = consumer.next(); - - assert!( - transactions.len() as u64 - <= context.protocol_config.max_num_transactions_in_block(), - "Should have fetched transactions up to {}", - context.protocol_config.max_num_transactions_in_block() - ); - - let total_size: u64 = transactions.iter().map(|t| t.data().len() as u64).sum(); - assert!( - total_size <= context.protocol_config.max_transactions_in_block_bytes(), - "Should have fetched transactions up to {}", - context.protocol_config.max_transactions_in_block_bytes() - ); - - // first batch should contain all transactions from 0..10. The softbundle it is - // to big to fit as well, so it's parked. - if batch_index == 0 { - assert_eq!(transactions.len(), 10); - for (i, transaction) in transactions.iter().enumerate() { - let t: String = bcs::from_bytes(transaction.data()).unwrap(); - assert_eq!(format!("transaction {i}").to_string(), t); - } - // second batch will contain the soft bundle and the additional last - // transaction. - } else if batch_index == 1 { - assert_eq!(transactions.len(), 6); - for (i, transaction) in transactions.iter().enumerate() { - let t: String = bcs::from_bytes(transaction.data()).unwrap(); - assert_eq!(format!("transaction {}", i + 10).to_string(), t); - } - } else { - panic!("Unexpected batch index"); - } - - batch_index += 1; - - all_acks.push(ack_transactions); - } - - // now acknowledge the inclusion of all transactions. - for ack in all_acks { - ack(BlockRef::MIN); - } - - // expect all receivers to be resolved. - for w in all_receivers { - let r = w.await; - assert!(r.is_ok()); - } - } - - #[tokio::test] - async fn test_submit_over_max_block_size_and_validate_block_size() { - // submit transactions individually so we make sure that we have reached the - // block size limit of 10 - { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(100); - config.set_consensus_max_num_transactions_in_block_for_testing(10); - config.set_consensus_max_transactions_in_block_bytes_for_testing(300); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let mut all_receivers = Vec::new(); - - // create enough transactions - let max_num_transactions_in_block = - context.protocol_config.max_num_transactions_in_block(); - for i in 0..2 * max_num_transactions_in_block { - let transaction = bcs::to_bytes(&format!("transaction {i}")) - .expect("Serialization should not fail."); - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Should submit successfully transaction"); - all_receivers.push(w); - } - - // Fetch the next transactions to be included in a block - let (transactions, _ack_transactions, limit) = consumer.next(); - assert_eq!(limit, LimitReached::MaxNumOfTransactions); - assert_eq!(transactions.len() as u64, max_num_transactions_in_block); - - // Now create a block and verify that transactions are within the size limits - let block_verifier = - SignedBlockVerifier::new(context.clone(), Arc::new(NoopTransactionVerifier {})); - - let batch: Vec<_> = transactions.iter().map(|t| t.data()).collect(); - assert!( - block_verifier.check_transactions(&batch).is_ok(), - "Number of transactions limit verification failed" - ); - } - - // submit transactions individually so we make sure that we have reached the - // block size bytes 300 - { - let _guard = ProtocolConfig::apply_overrides_for_testing(|_, mut config| { - config.set_consensus_max_transaction_size_bytes_for_testing(100); - config.set_consensus_max_num_transactions_in_block_for_testing(1_000); - config.set_consensus_max_transactions_in_block_bytes_for_testing(300); - config - }); - - let context = Arc::new(Context::new_for_test(4).0); - let (client, tx_receiver) = TransactionClient::new(context.clone()); - let mut consumer = TransactionConsumer::new(tx_receiver, context.clone()); - let mut all_receivers = Vec::new(); - - let max_transactions_in_block_bytes = - context.protocol_config.max_transactions_in_block_bytes(); - let mut total_size = 0; - loop { - let transaction = bcs::to_bytes(&"transaction".to_string()) - .expect("Serialization should not fail."); - total_size += transaction.len() as u64; - let w = client - .submit_no_wait(vec![transaction]) - .await - .expect("Should submit successfully transaction"); - all_receivers.push(w); - - // create enough transactions to reach the block size limit - if total_size >= 2 * max_transactions_in_block_bytes { - break; - } - } - - // Fetch the next transactions to be included in a block - let (transactions, _ack_transactions, limit) = consumer.next(); - let batch: Vec<_> = transactions.iter().map(|t| t.data()).collect(); - let size = batch.iter().map(|t| t.len() as u64).sum::(); - - assert_eq!(limit, LimitReached::MaxBytes); - assert!( - batch.len() - < context - .protocol_config - .consensus_max_num_transactions_in_block() as usize, - "Should have submitted less than the max number of transactions in a block" - ); - assert!(size <= max_transactions_in_block_bytes); - - // Now create a block and verify that transactions are within the size limits - let block_verifier = - SignedBlockVerifier::new(context.clone(), Arc::new(NoopTransactionVerifier {})); - - assert!( - block_verifier.check_transactions(&batch).is_ok(), - "Total size of transactions limit verification failed" - ); - } - } -} diff --git a/consensus/core/src/universal_committer.rs b/consensus/core/src/universal_committer.rs deleted file mode 100644 index 8c384b493fc..00000000000 --- a/consensus/core/src/universal_committer.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::VecDeque, sync::Arc}; - -use consensus_config::AuthorityIndex; -use parking_lot::RwLock; - -use crate::{ - base_committer::BaseCommitter, - block::{GENESIS_ROUND, Round, Slot}, - commit::{DecidedLeader, Decision}, - context::Context, - dag_state::DagState, -}; - -#[cfg(test)] -#[path = "tests/universal_committer_tests.rs"] -mod universal_committer_tests; - -#[cfg(test)] -#[path = "tests/pipelined_committer_tests.rs"] -mod pipelined_committer_tests; - -/// A universal committer uses a collection of committers to commit a sequence -/// of leaders. It can be configured to use a combination of different commit -/// strategies, including multi-leaders, backup leaders, and pipelines. -pub(crate) struct UniversalCommitter { - /// The per-epoch configuration of this authority. - context: Arc, - /// In memory block store representing the dag state - dag_state: Arc>, - /// The list of committers for multi-leader or pipelining - committers: Vec, -} - -impl UniversalCommitter { - /// Try to decide part of the dag. This function is idempotent and returns - /// an ordered list of decided leaders. - #[tracing::instrument(skip_all, fields(last_decided = %last_decided))] - pub(crate) fn try_decide(&self, last_decided: Slot) -> Vec { - let highest_accepted_round = self.dag_state.read().highest_accepted_round(); - - // Try to decide as many leaders as possible, starting with the highest round. - let mut leaders = VecDeque::new(); - - let last_round = last_decided.round + 1; - - // Keep this code commented for re-use when we re-enable multiple leaders. - // let last_round = match self - // .context - // .protocol_config - // .mysticeti_num_leaders_per_round() - // { - // Some(1) => { - // Ensure that we don't commit any leaders from the same round as last_decided - // until we have full support for multi-leader per round. - // This can happen when we are on a leader schedule boundary and the leader - // elected for the round changes with the new schedule. - // last_decided.round + 1 - // } - // _ => last_decided.round, - // }; - - // try to commit a leader up to the highest_accepted_round - 2. There is no - // reason to try and iterate on higher rounds as in order to make a direct - // decision for a leader at round R we need blocks from round R+2 to figure - // out that enough certificates and support exist to commit a leader. - 'outer: for round in (last_round..=highest_accepted_round.saturating_sub(2)).rev() { - for committer in self.committers.iter().rev() { - // Skip committers that don't have a leader for this round. - let Some(slot) = committer.elect_leader(round) else { - tracing::debug!("No leader for round {round}, skipping"); - continue; - }; - - // now that we reached the last committed leader we can stop the commit rule - if slot == last_decided { - tracing::debug!("Reached last committed {slot}, now exit"); - break 'outer; - } - - tracing::debug!("Trying to decide {slot} with {committer}",); - - // Try to directly decide the leader. - let mut status = committer.try_direct_decide(slot); - tracing::debug!("Outcome of direct rule: {status}"); - - // If we can't directly decide the leader, try to indirectly decide it. - if status.is_decided() { - leaders.push_front((status, Decision::Direct)); - } else { - status = committer.try_indirect_decide(slot, leaders.iter().map(|(x, _)| x)); - tracing::debug!("Outcome of indirect rule: {status}"); - leaders.push_front((status, Decision::Indirect)); - } - } - } - - // The decided sequence is the longest prefix of decided leaders. - let mut decided_leaders = Vec::new(); - for (leader, decision) in leaders { - if leader.round() == GENESIS_ROUND { - continue; - } - let Some(decided_leader) = leader.into_decided_leader() else { - break; - }; - Self::update_metrics(&self.context, &decided_leader, decision); - decided_leaders.push(decided_leader); - } - tracing::debug!("Decided {decided_leaders:?}"); - decided_leaders - } - - /// Return list of leaders for the round. - /// Can return empty vec if round does not have a designated leader. - pub(crate) fn get_leaders(&self, round: Round) -> Vec { - self.committers - .iter() - .filter_map(|committer| committer.elect_leader(round)) - .map(|l| l.authority) - .collect() - } - - /// Update metrics. - pub(crate) fn update_metrics( - context: &Context, - decided_leader: &DecidedLeader, - decision: Decision, - ) { - let decision_str = match decision { - Decision::Direct => "direct", - Decision::Indirect => "indirect", - Decision::Certified => "certified", - }; - let status = match decided_leader { - DecidedLeader::Commit(..) => format!("{decision_str}-commit"), - DecidedLeader::Skip(..) => format!("{decision_str}-skip"), - }; - let leader_host = &context - .committee - .authority(decided_leader.slot().authority) - .hostname; - context - .metrics - .node_metrics - .committed_leaders_total - .with_label_values(&[leader_host, &status]) - .inc(); - } -} - -/// A builder for a universal committer. By default, the builder creates a -/// single base committer, that is, a single leader and no pipeline. -pub(crate) mod universal_committer_builder { - use super::*; - use crate::{ - base_committer::BaseCommitterOptions, commit::DEFAULT_WAVE_LENGTH, - leader_schedule::LeaderSchedule, - }; - - pub(crate) struct UniversalCommitterBuilder { - context: Arc, - leader_schedule: Arc, - dag_state: Arc>, - wave_length: Round, - number_of_leaders: usize, - pipeline: bool, - } - - impl UniversalCommitterBuilder { - pub(crate) fn new( - context: Arc, - leader_schedule: Arc, - dag_state: Arc>, - ) -> Self { - Self { - context, - leader_schedule, - dag_state, - wave_length: DEFAULT_WAVE_LENGTH, - number_of_leaders: 1, - pipeline: false, - } - } - - #[expect(unused)] - pub(crate) fn with_wave_length(mut self, wave_length: Round) -> Self { - self.wave_length = wave_length; - self - } - - pub(crate) fn with_number_of_leaders(mut self, number_of_leaders: usize) -> Self { - self.number_of_leaders = number_of_leaders; - self - } - - pub(crate) fn with_pipeline(mut self, pipeline: bool) -> Self { - self.pipeline = pipeline; - self - } - - pub(crate) fn build(self) -> UniversalCommitter { - let mut committers = Vec::new(); - let pipeline_stages = if self.pipeline { self.wave_length } else { 1 }; - for round_offset in 0..pipeline_stages { - for leader_offset in 0..self.number_of_leaders { - let options = BaseCommitterOptions { - wave_length: self.wave_length, - round_offset, - leader_offset: leader_offset as Round, - }; - let committer = BaseCommitter::new( - self.context.clone(), - self.leader_schedule.clone(), - self.dag_state.clone(), - options, - ); - committers.push(committer); - } - } - - UniversalCommitter { - context: self.context, - dag_state: self.dag_state, - committers, - } - } - } -} diff --git a/consensus/simtests/Cargo.toml b/consensus/simtests/Cargo.toml deleted file mode 100644 index 613f5fa43a9..00000000000 --- a/consensus/simtests/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "consensus-simtests" -version = "0.1.0" -authors = ["IOTA Foundation "] -edition = "2021" -license = "Apache-2.0" -publish = false - -[lints] -workspace = true - -[target.'cfg(msim)'.dependencies] -# external dependencies -anyhow.workspace = true -arc-swap.workspace = true -parking_lot.workspace = true -prometheus.workspace = true -rand.workspace = true -tempfile.workspace = true -tokio.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true -tracing.workspace = true - -# internal dependencies -consensus-config.workspace = true -consensus-core.workspace = true -iota-common.workspace = true -iota-config.workspace = true -iota-macros.workspace = true -iota-metrics.workspace = true -iota-network-stack.workspace = true -iota-protocol-config.workspace = true -iota-simulator.workspace = true -telemetry-subscribers.workspace = true -typed-store.workspace = true diff --git a/consensus/simtests/src/lib.rs b/consensus/simtests/src/lib.rs deleted file mode 100644 index ca4e8251e11..00000000000 --- a/consensus/simtests/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(msim)] -mod node; - -#[cfg(msim)] -#[path = "tests/simtests.rs"] -mod simtests; diff --git a/consensus/simtests/src/node.rs b/consensus/simtests/src/node.rs deleted file mode 100644 index 03f27a3c156..00000000000 --- a/consensus/simtests/src/node.rs +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 -use std::{ - net::{IpAddr, SocketAddr}, - sync::Arc, - time::Duration, -}; - -use anyhow::Result; -use arc_swap::ArcSwapOption; -use consensus_config::{AuthorityIndex, Committee, NetworkKeyPair, Parameters, ProtocolKeyPair}; -use consensus_core::{ - BlockTimestampMs, Clock, CommitConsumer, CommitConsumerMonitor, CommittedSubDag, - ConsensusAuthority, TransactionClient, network::tonic_network::to_socket_addr, - transaction::NoopTransactionVerifier, -}; -use iota_common::scoring_metrics::VersionedScoringMetrics; -use iota_metrics::monitored_mpsc::{UnboundedReceiver, unbounded_channel}; -use iota_protocol_config::{ConsensusNetwork, ProtocolConfig}; -use parking_lot::Mutex; -use prometheus::Registry; -use tempfile::TempDir; -use tracing::{info, trace}; - -#[derive(Clone)] -#[allow(unused)] -pub(crate) struct Config { - pub authority_index: AuthorityIndex, - pub db_dir: Arc, - pub committee: Committee, - pub keypairs: Vec<(NetworkKeyPair, ProtocolKeyPair)>, - pub network_type: ConsensusNetwork, - pub boot_counter: u64, - pub clock_drift: BlockTimestampMs, - pub protocol_config: ProtocolConfig, -} - -pub(crate) struct AuthorityNode { - inner: Mutex>, - config: Config, -} - -impl AuthorityNode { - pub fn new(config: Config) -> Self { - Self { - inner: Default::default(), - config, - } - } - - /// Return the `index` of this Node - pub fn index(&self) -> AuthorityIndex { - self.config.authority_index - } - - /// Start this Node - pub async fn start(&self) -> Result<()> { - info!(index =% self.config.authority_index, "starting in-memory node"); - let config = self.config.clone(); - *self.inner.lock() = Some(AuthorityNodeInner::spawn(config).await); - Ok(()) - } - - pub fn spawn_committed_subdag_consumer(&self) -> Result<()> { - let authority_index = self.config.authority_index; - let inner = self.inner.lock(); - if let Some(inner) = inner.as_ref() { - let mut commit_receiver = inner.take_commit_receiver(); - let commit_consumer_monitor = inner.commit_consumer_monitor(); - let _handle = tokio::spawn(async move { - while let Some(subdag) = commit_receiver.recv().await { - info!(authority =% authority_index, commit_index =% subdag.commit_ref.index, "received committed subdag"); - commit_consumer_monitor.set_highest_handled_commit(subdag.commit_ref.index); - } - }); - } - Ok(()) - } - - pub fn commit_consumer_monitor(&self) -> Arc { - let inner = self.inner.lock(); - if let Some(inner) = inner.as_ref() { - inner.commit_consumer_monitor() - } else { - panic!("Node not initialised"); - } - } - - pub fn transaction_client(&self) -> Arc { - let inner = self.inner.lock(); - if let Some(inner) = inner.as_ref() { - inner.transaction_client() - } else { - panic!("Node not initialised"); - } - } - - /// Stop this Node - pub fn stop(&self) { - info!(index =% self.config.authority_index, "stopping in-memory node"); - *self.inner.lock() = None; - info!(index =% self.config.authority_index, "node stopped"); - } - - /// If this Node is currently running - pub fn is_running(&self) -> bool { - self.inner.lock().as_ref().is_some_and(|c| c.is_alive()) - } -} - -pub(crate) struct AuthorityNodeInner { - handle: Option, - cancel_sender: Option>, - consensus_authority: ConsensusAuthority, - commit_receiver: ArcSwapOption>, - commit_consumer_monitor: Arc, -} - -#[derive(Debug)] -struct NodeHandle { - node_id: iota_simulator::task::NodeId, -} - -/// When dropped, stop and wait for the node running in this node to completely -/// shutdown. -impl Drop for AuthorityNodeInner { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - tracing::info!("shutting down {}", handle.node_id); - iota_simulator::runtime::Handle::try_current().map(|h| h.delete_node(handle.node_id)); - } - } -} - -impl AuthorityNodeInner { - /// Spawn a new Node. - pub async fn spawn(config: Config) -> Self { - let (startup_sender, mut startup_receiver) = tokio::sync::watch::channel(false); - let (cancel_sender, cancel_receiver) = tokio::sync::watch::channel(false); - - let handle = iota_simulator::runtime::Handle::current(); - let builder = handle.create_node(); - - let authority = config.committee.authority(config.authority_index); - let socket_addr = to_socket_addr(&authority.address).unwrap(); - let ip = match socket_addr { - SocketAddr::V4(v4) => IpAddr::V4(*v4.ip()), - _ => panic!("unsupported protocol"), - }; - let init_receiver_swap = Arc::new(ArcSwapOption::empty()); - let int_receiver_swap_clone = init_receiver_swap.clone(); - - let node = builder - .ip(ip) - .name(format!("{}", config.authority_index)) - .init(move || { - info!("Node restarted"); - let config = config.clone(); - let mut cancel_receiver = cancel_receiver.clone(); - let init_receiver_swap_clone = int_receiver_swap_clone.clone(); - let startup_sender_clone = startup_sender.clone(); - - async move { - let (consensus_authority, commit_receiver, commit_consumer_monitor) = - super::node::make_authority(config).await; - - startup_sender_clone.send(true).ok(); - init_receiver_swap_clone.store(Some(Arc::new(( - consensus_authority, - commit_receiver, - commit_consumer_monitor, - )))); - - // run until canceled - loop { - if cancel_receiver.changed().await.is_err() || *cancel_receiver.borrow() { - break; - } - } - trace!("cancellation received; shutting down thread"); - } - }) - .build(); - - startup_receiver.changed().await.unwrap(); - - let Some(init_tuple) = init_receiver_swap.swap(None) else { - panic!("Components should be initialised by now"); - }; - - let Ok((consensus_authority, commit_receiver, commit_consumer_monitor)) = - Arc::try_unwrap(init_tuple) - else { - panic!("commit receiver still in use"); - }; - - Self { - handle: Some(NodeHandle { node_id: node.id() }), - cancel_sender: Some(cancel_sender), - consensus_authority, - commit_receiver: ArcSwapOption::new(Some(Arc::new(commit_receiver))), - commit_consumer_monitor, - } - } - - /// Check to see that the Node is still alive by checking if the receiving - /// side of the `cancel_sender` has been dropped. - pub fn is_alive(&self) -> bool { - if let Some(cancel_sender) = &self.cancel_sender { - // unless the node is deleted, it keeps a reference to its start up function, - // which keeps 1 receiver alive. If the node is actually running, - // the cloned receiver will also be alive, and receiver count will - // be 2. - cancel_sender.receiver_count() > 1 - } else { - false - } - } - - pub fn take_commit_receiver(&self) -> UnboundedReceiver { - if let Some(commit_receiver) = self.commit_receiver.swap(None) { - let Ok(commit_receiver) = Arc::try_unwrap(commit_receiver) else { - panic!("commit receiver still in use"); - }; - - commit_receiver - } else { - panic!("commit receiver already taken"); - } - } - - pub fn commit_consumer_monitor(&self) -> Arc { - self.commit_consumer_monitor.clone() - } - - pub fn transaction_client(&self) -> Arc { - self.consensus_authority.transaction_client() - } -} - -pub(crate) async fn make_authority( - config: Config, -) -> ( - ConsensusAuthority, - UnboundedReceiver, - Arc, -) { - let Config { - authority_index, - db_dir, - committee, - keypairs, - network_type, - boot_counter, - protocol_config, - clock_drift, - } = config; - - let registry = Registry::new(); - - // Cache less blocks to exercise commit sync. - let parameters = Parameters { - db_path: db_dir.path().to_path_buf(), - dag_state_cached_rounds: 5, - commit_sync_parallel_fetches: 2, - commit_sync_batch_size: 3, - sync_last_known_own_block_timeout: Duration::from_millis(2_000), - ..Default::default() - }; - let txn_verifier = NoopTransactionVerifier {}; - - let protocol_keypair = keypairs[authority_index].1.clone(); - let network_keypair = keypairs[authority_index].0.clone(); - - let (commit_sender, commit_receiver) = unbounded_channel("consensus_output"); - - let current_local_metrics_count = Arc::new(VersionedScoringMetrics::new( - committee.size(), - &protocol_config, - )); - let commit_consumer = CommitConsumer::new(commit_sender, 0); - let commit_consumer_monitor = commit_consumer.monitor(); - - let authority = ConsensusAuthority::start( - network_type, - 0, - authority_index, - committee, - parameters, - protocol_config, - protocol_keypair, - network_keypair, - Arc::new(Clock::new_for_test(clock_drift)), - Arc::new(txn_verifier), - commit_consumer, - registry, - current_local_metrics_count, - boot_counter, - ) - .await; - - (authority, commit_receiver, commit_consumer_monitor) -} diff --git a/consensus/simtests/src/tests/simtests.rs b/consensus/simtests/src/tests/simtests.rs deleted file mode 100644 index 048ed1c2948..00000000000 --- a/consensus/simtests/src/tests/simtests.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2025 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 -#[cfg(msim)] -mod test { - use std::{sync::Arc, time::Duration}; - - use consensus_config::{ - Authority, AuthorityIndex, AuthorityKeyPair, Committee, Epoch, NetworkKeyPair, - ProtocolKeyPair, Stake, - }; - use iota_config::local_ip_utils; - use iota_macros::sim_test; - use iota_network_stack::Multiaddr; - use iota_protocol_config::ProtocolConfig; - use iota_simulator::{ - SimConfig, - configs::{bimodal_latency_ms, env_config, uniform_latency_ms}, - }; - use prometheus::Registry; - use rand::{SeedableRng as _, rngs::StdRng}; - use tempfile::TempDir; - use tokio::time::sleep; - use typed_store::DBMetrics; - - use crate::node::{AuthorityNode, Config}; - - fn test_config() -> SimConfig { - env_config( - uniform_latency_ms(10..20), - [ - ( - "regional_high_variance", - bimodal_latency_ms(30..40, 300..800, 0.01), - ), - ( - "global_high_variance", - bimodal_latency_ms(60..80, 500..1500, 0.01), - ), - ], - ) - } - - #[sim_test(config = "test_config()")] - async fn test_committee_start_simple() { - telemetry_subscribers::init_for_testing(); - for median_based_timestamp in vec![true, false] { - tracing::info!("Running with median_based_timestamp = {median_based_timestamp}"); - let db_registry = Registry::new(); - DBMetrics::init(&db_registry); - - const NUM_OF_AUTHORITIES: usize = 10; - let (committee, keypairs) = - local_committee_and_keys(0, [1; NUM_OF_AUTHORITIES].to_vec()); - let mut protocol_config = ProtocolConfig::get_for_max_version_UNSAFE(); - protocol_config.set_consensus_gc_depth_for_testing(3); - protocol_config.set_consensus_median_timestamp_with_checkpoint_enforcement_for_testing( - median_based_timestamp, - ); - - let mut authorities = Vec::with_capacity(committee.size()); - let mut transaction_clients = Vec::with_capacity(committee.size()); - let mut boot_counters = [0; NUM_OF_AUTHORITIES]; - let mut clock_drifts = [0; NUM_OF_AUTHORITIES]; - clock_drifts[0] = 50; - clock_drifts[1] = 100; - clock_drifts[2] = 120; - - for (authority_index, _authority_info) in committee.authorities() { - // Introduce a non-trivial clock drift for the first node (it's time will be - // ahead of the others). This will provide extra reassurance - // around the block timestamp checks. - let config = Config { - authority_index, - db_dir: Arc::new(TempDir::new().unwrap()), - committee: committee.clone(), - keypairs: keypairs.clone(), - network_type: iota_protocol_config::ConsensusNetwork::Tonic, - boot_counter: boot_counters[authority_index], - protocol_config: protocol_config.clone(), - clock_drift: clock_drifts[authority_index.value() as usize], - }; - let node = AuthorityNode::new(config); - - if authority_index != AuthorityIndex::new_for_test(NUM_OF_AUTHORITIES as u32 - 1) { - node.start().await.unwrap(); - node.spawn_committed_subdag_consumer().unwrap(); - - let client = node.transaction_client(); - transaction_clients.push(client); - } - - boot_counters[authority_index] += 1; - authorities.push(node); - } - - let transaction_clients_clone = transaction_clients.clone(); - let _handle = tokio::spawn(async move { - const NUM_TRANSACTIONS: u16 = 1000; - - for i in 0..NUM_TRANSACTIONS { - let txn = vec![i as u8; 16]; - transaction_clients_clone[i as usize % transaction_clients_clone.len()] - .submit(vec![txn]) - .await - .unwrap(); - } - }); - - // wait for authorities - sleep(Duration::from_secs(60)).await; - - // Now start the fourth authority and let it start - authorities[NUM_OF_AUTHORITIES - 1].start().await.unwrap(); - authorities[NUM_OF_AUTHORITIES - 1] - .spawn_committed_subdag_consumer() - .unwrap(); - - // Wait for it to catch up - sleep(Duration::from_secs(230)).await; - let commit_consumer_monitor = - authorities[NUM_OF_AUTHORITIES - 1].commit_consumer_monitor(); - let highest_committed_index = commit_consumer_monitor.highest_handled_commit(); - assert!( - highest_committed_index >= 80, - "Highest handled commit {highest_committed_index} < 80" - ); - } - } - - /// Creates a committee for local testing, and the corresponding key pairs - /// for the authorities. - pub fn local_committee_and_keys( - epoch: Epoch, - authorities_stake: Vec, - ) -> (Committee, Vec<(NetworkKeyPair, ProtocolKeyPair)>) { - let mut authorities = vec![]; - let mut key_pairs = vec![]; - let mut rng = StdRng::from_seed([0; 32]); - for (i, stake) in authorities_stake.into_iter().enumerate() { - let authority_keypair = AuthorityKeyPair::generate(&mut rng); - let protocol_keypair = ProtocolKeyPair::generate(&mut rng); - let network_keypair = NetworkKeyPair::generate(&mut rng); - authorities.push(Authority { - stake, - address: get_available_local_address(), - hostname: format!("test_host_{i}").to_string(), - authority_key: authority_keypair.public(), - protocol_key: protocol_keypair.public(), - network_key: network_keypair.public(), - }); - key_pairs.push((network_keypair, protocol_keypair)); - } - - let committee = Committee::new(epoch, authorities); - (committee, key_pairs) - } - - fn get_available_local_address() -> Multiaddr { - let ip = local_ip_utils::get_new_ip(); - - local_ip_utils::new_udp_address_for_testing(&ip) - } -} diff --git a/crates/iota-aws-orchestrator/src/main.rs b/crates/iota-aws-orchestrator/src/main.rs index 3a0ba1949f3..2882c2d145b 100644 --- a/crates/iota-aws-orchestrator/src/main.rs +++ b/crates/iota-aws-orchestrator/src/main.rs @@ -210,8 +210,7 @@ pub enum Operation { #[arg(long, value_name = "INT", default_value = "400", global = true)] maximum_latency: u16, - /// Switch protocols between mysticeti and starfish every epoch, - /// default: false, aka use starfish in every epoch. + /// Consensus protocol to use (currently only Starfish is supported). #[arg(long, default_value = "starfish", global = true)] consensus_protocol: ConsensusProtocol, @@ -371,8 +370,6 @@ pub enum LatencyTopology { #[derive(ValueEnum, Clone, Debug, Deserialize, Serialize)] pub enum ConsensusProtocol { Starfish, - Mysticeti, - SwapEachEpoch, } fn parse_duration(arg: &str) -> Result { diff --git a/crates/iota-aws-orchestrator/src/protocol/iota.rs b/crates/iota-aws-orchestrator/src/protocol/iota.rs index 563bd7f1539..29b2676d258 100644 --- a/crates/iota-aws-orchestrator/src/protocol/iota.rs +++ b/crates/iota-aws-orchestrator/src/protocol/iota.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use super::{ProtocolCommands, ProtocolMetrics}; use crate::{ - ConsensusProtocol, + benchmark::{BenchmarkParameters, BenchmarkType}, client::Instance, display, @@ -274,17 +274,7 @@ impl ProtocolCommands for IotaProtocol { let max_pipeline_delay = parameters.max_pipeline_delay; let mut setup: Vec = vec![ - match parameters.consensus_protocol { - ConsensusProtocol::Starfish => { - "export CONSENSUS_PROTOCOL=starfish".to_string() - } - ConsensusProtocol::Mysticeti => { - "export CONSENSUS_PROTOCOL=mysticeti".to_string() - } - ConsensusProtocol::SwapEachEpoch => { - "export CONSENSUS_PROTOCOL=swap_each_epoch".to_string() - } - }, + "export CONSENSUS_PROTOCOL=starfish".to_string(), format!("export MAX_PIPELINE_DELAY={max_pipeline_delay}"), // Set protocol config override to disable validator subsidies for // benchmarks diff --git a/crates/iota-config/Cargo.toml b/crates/iota-config/Cargo.toml index 3c6457d6ee9..59060afb5fa 100644 --- a/crates/iota-config/Cargo.toml +++ b/crates/iota-config/Cargo.toml @@ -28,7 +28,6 @@ serde_yaml.workspace = true tracing.workspace = true # internal dependencies -consensus-config.workspace = true iota-genesis-common.workspace = true iota-keys.workspace = true iota-names.workspace = true diff --git a/crates/iota-config/src/node.rs b/crates/iota-config/src/node.rs index 0ff70855353..c9fdd43e150 100644 --- a/crates/iota-config/src/node.rs +++ b/crates/iota-config/src/node.rs @@ -12,7 +12,6 @@ use std::{ }; use anyhow::Result; -use consensus_config::Parameters as ConsensusParameters; use iota_keys::keypair_file::{read_authority_keypair_from_file, read_keypair_from_file}; use iota_names::config::IotaNamesConfig; use iota_types::{ @@ -830,14 +829,6 @@ impl NodeConfig { } } -#[derive(Debug, Clone, Deserialize, Serialize)] -pub enum ConsensusProtocol { - #[serde(rename = "mysticeti")] - Mysticeti, - #[serde(rename = "starfish")] - Starfish, -} - #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct ConsensusConfig { @@ -873,9 +864,6 @@ pub struct ConsensusConfig { /// estimates. pub submit_delay_step_override_millis: Option, - /// Parameters for Mysticeti consensus - pub parameters: Option, - /// Parameters for Starfish consensus #[serde(skip_serializing_if = "Option::is_none")] pub starfish_parameters: Option, diff --git a/crates/iota-core/Cargo.toml b/crates/iota-core/Cargo.toml index ed432856c55..0e75b017d98 100644 --- a/crates/iota-core/Cargo.toml +++ b/crates/iota-core/Cargo.toml @@ -60,8 +60,6 @@ tracing.workspace = true twox-hash = "1.6" # internal dependencies -consensus-config.workspace = true -consensus-core.workspace = true iota-archival.workspace = true iota-authority-aggregation.workspace = true iota-common.workspace = true diff --git a/crates/iota-core/src/authority/scorer.rs b/crates/iota-core/src/authority/scorer.rs index 41eaeba1777..ba0be9b7d27 100644 --- a/crates/iota-core/src/authority/scorer.rs +++ b/crates/iota-core/src/authority/scorer.rs @@ -494,16 +494,15 @@ fn metric_to_score(value: u64, allowance: u64, max: u64, max_score: u64) -> u64 mod tests { use std::sync::atomic::Ordering; - use iota_protocol_config::{ConsensusChoice, ProtocolConfig}; + use iota_protocol_config::ProtocolConfig; use iota_types::messages_consensus::{MisbehaviorsV1, VersionedMisbehaviorReport}; use crate::authority::authority_per_epoch_store::scorer::{ MAX_SCORE, ParametersV1, SCALE_FACTOR, Scorer, calculate_median_report, calculate_scores_v1, }; - fn mock_protocol_config(consensus_choice: ConsensusChoice) -> ProtocolConfig { - let mut config = ProtocolConfig::get_for_max_version_UNSAFE(); - config.set_consensus_choice_for_testing(consensus_choice); + fn mock_protocol_config() -> ProtocolConfig { + let config = ProtocolConfig::get_for_max_version_UNSAFE(); config } @@ -521,7 +520,7 @@ mod tests { fn test_scorer_initialization() { let voting_power = vec![10, 20, 30]; let committee_size = voting_power.len(); - let protocol_config = mock_protocol_config(ConsensusChoice::Mysticeti); + let protocol_config = mock_protocol_config(); let scorer = Scorer::new(voting_power, &protocol_config); @@ -535,7 +534,7 @@ mod tests { fn test_increment_invalid_reports_count() { let voting_power = vec![10, 20, 30]; - let protocol_config = mock_protocol_config(ConsensusChoice::Mysticeti); + let protocol_config = mock_protocol_config(); let scorer = Scorer::new(voting_power, &protocol_config); @@ -587,7 +586,7 @@ mod tests { #[test] fn test_update_scores() { let voting_power = vec![2, 5, 20]; - let protocol_config = mock_protocol_config(ConsensusChoice::Mysticeti); + let protocol_config = mock_protocol_config(); let scorer = Scorer::new(voting_power, &protocol_config); // Before calling update_scores, all scores should be MAX_SCORE diff --git a/crates/iota-core/src/authority_server.rs b/crates/iota-core/src/authority_server.rs index 534d06cf60b..54590a4e883 100644 --- a/crates/iota-core/src/authority_server.rs +++ b/crates/iota-core/src/authority_server.rs @@ -57,7 +57,7 @@ use crate::{ consensus_adapter::{ ConnectionMonitorStatusForTests, ConsensusAdapter, ConsensusAdapterMetrics, }, - mysticeti_adapter::LazyMysticetiClient, + starfish_adapter::LazyStarfishClient, traffic_controller::{ TrafficController, metrics::TrafficControllerMetrics, parse_ip, policies::TrafficTally, }, @@ -119,7 +119,7 @@ impl AuthorityServer { /// Creates a new `AuthorityServer` for testing. pub fn new_for_test(state: Arc) -> Self { let consensus_adapter = Arc::new(ConsensusAdapter::new( - Arc::new(LazyMysticetiClient::new()), + Arc::new(LazyStarfishClient::new()), CheckpointStore::new_for_tests(), state.name, Arc::new(ConnectionMonitorStatusForTests {}), diff --git a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs index 70662de907a..01afa64db88 100644 --- a/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs +++ b/crates/iota-core/src/checkpoints/checkpoint_executor/mod.rs @@ -495,7 +495,7 @@ impl CheckpointExecutor { .await; // Currently this code only runs on validators, where this method call does - // nothing. But in the future, fullnodes may follow the mysticeti dag + // nothing. But in the future, fullnodes may follow the consensus dag // and build their own checkpoints. self.insert_finalized_transactions(&tx_digests, sequence_number, checkpoint.timestamp_ms); diff --git a/crates/iota-core/src/consensus_adapter.rs b/crates/iota-core/src/consensus_adapter.rs index 8b761e7a758..38638350fa9 100644 --- a/crates/iota-core/src/consensus_adapter.rs +++ b/crates/iota-core/src/consensus_adapter.rs @@ -197,23 +197,12 @@ impl ConsensusAdapterMetrics { } } -/// Block status for internal use in the consensus adapter that serves for both -/// Mysticeti and Starfish. +/// Block status for internal use in the consensus adapter. pub enum BlockStatusInternal { Sequenced, GarbageCollected, } -impl From for BlockStatusInternal { - fn from(status: consensus_core::BlockStatus) -> Self { - match status { - consensus_core::BlockStatus::Sequenced(_) => BlockStatusInternal::Sequenced, - consensus_core::BlockStatus::GarbageCollected(_) => { - BlockStatusInternal::GarbageCollected - } - } - } -} impl From for BlockStatusInternal { fn from(status: starfish_core::BlockStatus) -> Self { match status { @@ -1284,7 +1273,7 @@ mod adapter_tests { consensus_adapter::{ ConnectionMonitorStatusForTests, ConsensusAdapter, ConsensusAdapterMetrics, }, - mysticeti_adapter::LazyMysticetiClient, + starfish_adapter::LazyStarfishClient, }; fn test_committee(rng: &mut StdRng, size: usize) -> Committee { @@ -1312,7 +1301,7 @@ mod adapter_tests { // When we define max submit position and delay step let consensus_adapter = ConsensusAdapter::new( - Arc::new(LazyMysticetiClient::new()), + Arc::new(LazyStarfishClient::new()), CheckpointStore::new_for_tests(), *committee.authority_by_index(0).unwrap(), Arc::new(ConnectionMonitorStatusForTests {}), @@ -1342,7 +1331,7 @@ mod adapter_tests { // Without submit position and delay step let consensus_adapter = ConsensusAdapter::new( - Arc::new(LazyMysticetiClient::new()), + Arc::new(LazyStarfishClient::new()), CheckpointStore::new_for_tests(), *committee.authority_by_index(0).unwrap(), Arc::new(ConnectionMonitorStatusForTests {}), diff --git a/crates/iota-core/src/consensus_handler.rs b/crates/iota-core/src/consensus_handler.rs index 1503374b4c6..f1e59304c85 100644 --- a/crates/iota-core/src/consensus_handler.rs +++ b/crates/iota-core/src/consensus_handler.rs @@ -10,11 +10,10 @@ use std::{ }; use arc_swap::ArcSwap; -use consensus_config::Committee as ConsensusCommittee; -use consensus_core::{CommitConsumerMonitor, CommitIndex}; use iota_common::random_util::randomize_cache_capacity_in_tests; use iota_macros::{fail_point, fail_point_if}; use iota_metrics::{monitored_mpsc::UnboundedReceiver, monitored_scope, spawn_monitored_task}; +use starfish_config::Committee as ConsensusCommittee; use iota_types::{ authenticator_state::ActiveJwk, base_types::{AuthorityName, EpochId, ObjectID, SequenceNumber, TransactionDigest}, @@ -208,7 +207,7 @@ impl ConsensusHandler { let round = consensus_output.leader_round(); - // TODO: Is this check necessary? For now mysticeti will not + // TODO: Is this check necessary? For now consensus will not // return more than one leader per round so we are not in danger of // ignoring any commits. assert!( @@ -464,57 +463,6 @@ impl AsyncTransactionScheduler { } } -/// Consensus handler used by Mysticeti. Since Mysticeti repo is not yet -/// integrated, we use a channel to receive the consensus output from Mysticeti. -/// During initialization, the sender is passed into Mysticeti which can send -/// consensus output to the channel. -pub struct MysticetiConsensusHandler { - handle: Option>, -} - -impl MysticetiConsensusHandler { - pub fn new( - last_processed_commit_at_startup: CommitIndex, - mut consensus_handler: ConsensusHandler, - mut receiver: UnboundedReceiver, - commit_consumer_monitor: Arc, - ) -> Self { - let handle = spawn_monitored_task!(async move { - // TODO: pause when execution is overloaded, so consensus can detect the - // backpressure. - while let Some(consensus_output) = receiver.recv().await { - let commit_index = consensus_output.commit_ref.index; - if commit_index <= last_processed_commit_at_startup { - consensus_handler.handle_prior_consensus_output(consensus_output); - } else { - consensus_handler - .handle_consensus_output(consensus_output) - .await; - } - commit_consumer_monitor.set_highest_handled_commit(commit_index); - } - }); - Self { - handle: Some(handle), - } - } - - pub async fn abort(&mut self) { - if let Some(handle) = self.handle.take() { - handle.abort(); - let _ = handle.await; - } - } -} - -impl Drop for MysticetiConsensusHandler { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - handle.abort(); - } - } -} - /// Consensus handler used by Starfish. /// During initialization, the sender is passed into Starfish which can send /// consensus output to the channel. @@ -868,10 +816,6 @@ impl ConsensusCommitInfo { #[cfg(test)] mod tests { - use consensus_core::{ - BlockAPI, CommitDigest, CommitRef, CommittedSubDag, TestBlock, Transaction, VerifiedBlock, - }; - use futures::pin_mut; use iota_protocol_config::{Chain, ConsensusTransactionOrdering}; use iota_types::{ base_types::{AuthorityName, IotaAddress, random_object_ref}, @@ -879,7 +823,6 @@ mod tests { messages_consensus::{ AuthorityCapabilitiesV1, ConsensusTransaction, ConsensusTransactionKind, }, - object::Object, supported_protocol_versions::{ SupportedProtocolVersions, SupportedProtocolVersionsWithHashes, }, @@ -887,141 +830,9 @@ mod tests { CertifiedTransaction, SenderSignedData, TransactionData, TransactionDataAPI, }, }; - use prometheus::Registry; use super::*; - use crate::{ - authority::{ - authority_per_epoch_store::ConsensusStatsAPI, - test_authority_builder::TestAuthorityBuilder, - }, - checkpoints::CheckpointServiceNoop, - consensus_adapter::consensus_tests::{test_certificates, test_gas_objects}, - post_consensus_tx_reorder::PostConsensusTxReorder, - }; - - #[tokio::test(flavor = "current_thread", start_paused = true)] - pub async fn test_consensus_handler() { - // GIVEN - let mut objects = test_gas_objects(); - let shared_object = Object::shared_for_testing(); - objects.push(shared_object.clone()); - - let network_config = - iota_swarm_config::network_config_builder::ConfigBuilder::new_with_temp_dir() - .with_objects(objects.clone()) - .build(); - - let state = TestAuthorityBuilder::new() - .with_network_config(&network_config, 0) - .build() - .await; - - let epoch_store = state.epoch_store_for_testing().clone(); - let new_epoch_start_state = epoch_store.epoch_start_state(); - let consensus_committee = new_epoch_start_state.get_consensus_committee(); - - let metrics = Arc::new(AuthorityMetrics::new(&Registry::new())); - - let backpressure_manager = BackpressureManager::new_for_tests(); - - let mut consensus_handler = ConsensusHandler::new( - epoch_store, - Arc::new(CheckpointServiceNoop {}), - state.transaction_manager().clone(), - state.get_object_cache_reader().clone(), - state.get_transaction_cache_reader().clone(), - Arc::new(ArcSwap::default()), - consensus_committee.clone(), - metrics, - backpressure_manager.subscribe(), - ); - - // AND - // Create test transactions - let transactions = test_certificates(&state, shared_object).await; - let mut blocks = Vec::new(); - - for (i, transaction) in transactions.iter().enumerate() { - let transaction_bytes: Vec = bcs::to_bytes( - &ConsensusTransaction::new_certificate_message(&state.name, transaction.clone()), - ) - .unwrap(); - - // AND create block for each transaction - let block = VerifiedBlock::new_for_test( - TestBlock::new(100 + i as u32, (i % consensus_committee.size()) as u32) - .set_transactions(vec![Transaction::new(transaction_bytes)]) - .build(), - ); - - blocks.push(block); - } - - // AND create the consensus output - let leader_block = blocks[0].clone(); - let committed_sub_dag = CommittedSubDag::new( - leader_block.reference(), - blocks.clone(), - leader_block.timestamp_ms(), - CommitRef::new(10, CommitDigest::MIN), - vec![], - ); - - // Test that the consensus handler respects backpressure. - backpressure_manager.set_backpressure(true); - // Default watermarks are 0,0 which will suppress the backpressure. - backpressure_manager.update_highest_certified_checkpoint(1); - - // AND processing the consensus output once - { - let waiter = consensus_handler.handle_consensus_output(committed_sub_dag.clone()); - pin_mut!(waiter); - - // waiter should not complete within 5 seconds - tokio::time::timeout(std::time::Duration::from_secs(5), &mut waiter) - .await - .unwrap_err(); - - // lift backpressure - backpressure_manager.set_backpressure(false); - - // waiter completes now. - tokio::time::timeout(std::time::Duration::from_secs(100), waiter) - .await - .unwrap(); - } - - // AND capturing the consensus stats - let num_blocks = blocks.len(); - let num_transactions = transactions.len(); - let last_consensus_stats_1 = consensus_handler.last_consensus_stats.clone(); - assert_eq!( - last_consensus_stats_1.index.transaction_index, - num_transactions as u64 - ); - assert_eq!(last_consensus_stats_1.index.sub_dag_index, 10_u64); - assert_eq!(last_consensus_stats_1.index.last_committed_round, 100_u64); - assert_eq!(last_consensus_stats_1.hash, 0); - assert_eq!( - last_consensus_stats_1.stats.get_num_messages(0), - num_blocks as u64 - ); - assert_eq!( - last_consensus_stats_1.stats.get_num_user_transactions(0), - num_transactions as u64 - ); - - // WHEN processing the same output multiple times - // THEN the consensus stats do not update - for _ in 0..2 { - consensus_handler - .handle_consensus_output(committed_sub_dag.clone()) - .await; - let last_consensus_stats_2 = consensus_handler.last_consensus_stats.clone(); - assert_eq!(last_consensus_stats_1, last_consensus_stats_2); - } - } + use crate::post_consensus_tx_reorder::PostConsensusTxReorder; #[test] fn test_order_by_gas_price() { diff --git a/crates/iota-core/src/consensus_manager/mod.rs b/crates/iota-core/src/consensus_manager/mod.rs index 5265fd549b1..ad6fe7a805a 100644 --- a/crates/iota-core/src/consensus_manager/mod.rs +++ b/crates/iota-core/src/consensus_manager/mod.rs @@ -10,11 +10,10 @@ use std::{ use arc_swap::ArcSwapOption; use async_trait::async_trait; -use enum_dispatch::enum_dispatch; use fastcrypto::traits::KeyPair as _; -use iota_config::{ConsensusConfig, NodeConfig, node::ConsensusProtocol}; +use iota_config::{ConsensusConfig, NodeConfig}; use iota_metrics::RegistryService; -use iota_protocol_config::{ConsensusChoice, ProtocolVersion}; +use iota_protocol_config::ProtocolVersion; use iota_types::{committee::EpochId, error::IotaResult, messages_consensus::ConsensusTransaction}; use prometheus::{IntGauge, Registry, register_int_gauge_with_registry}; use tokio::{ @@ -27,13 +26,11 @@ use crate::{ authority::authority_per_epoch_store::AuthorityPerEpochStore, consensus_adapter::{BlockStatusReceiver, ConsensusClient}, consensus_handler::ConsensusHandlerInitializer, - consensus_manager::{mysticeti_manager::MysticetiManager, starfish_manager::StarfishManager}, + consensus_manager::starfish_manager::StarfishManager, consensus_validator::IotaTxValidator, - mysticeti_adapter::LazyMysticetiClient, starfish_adapter::LazyStarfishClient, }; -pub mod mysticeti_manager; pub mod starfish_manager; #[derive(PartialEq)] @@ -43,7 +40,6 @@ pub(crate) enum Running { } #[async_trait] -#[enum_dispatch(ProtocolManager)] pub trait ConsensusManagerTrait { async fn start( &self, @@ -58,61 +54,10 @@ pub trait ConsensusManagerTrait { async fn is_running(&self) -> bool; } -// Wraps the underlying consensus protocol managers to make calling -// the ConsensusManagerTrait easier. -#[enum_dispatch] -enum ProtocolManager { - Mysticeti(MysticetiManager), - Starfish(StarfishManager), -} - -impl ProtocolManager { - /// Creates a new mysticeti manager. - pub fn new_mysticeti( - config: &NodeConfig, - consensus_config: &ConsensusConfig, - registry_service: &RegistryService, - metrics: Arc, - client: Arc, - ) -> Self { - Self::Mysticeti(MysticetiManager::new( - config.protocol_key_pair().copy(), - config.network_key_pair().copy(), - consensus_config.db_path().to_path_buf(), - registry_service.clone(), - metrics, - client, - )) - } - - /// Creates a new starfish manager. - pub fn new_starfish( - config: &NodeConfig, - consensus_config: &ConsensusConfig, - registry_service: &RegistryService, - metrics: Arc, - client: Arc, - ) -> Self { - Self::Starfish(StarfishManager::new( - config.protocol_key_pair().copy(), - config.network_key_pair().copy(), - consensus_config.db_path().to_path_buf(), - registry_service.clone(), - metrics, - client, - )) - } -} - /// Used by IOTA validator to start consensus protocol for each epoch. pub struct ConsensusManager { consensus_config: ConsensusConfig, - mysticeti_manager: ProtocolManager, - starfish_manager: ProtocolManager, - mysticeti_client: Arc, - starfish_client: Arc, - active: parking_lot::Mutex>, - consensus_client: Arc, + starfish_manager: StarfishManager, } impl ConsensusManager { @@ -124,66 +69,25 @@ impl ConsensusManager { consensus_client: Arc, ) -> Self { let metrics = Arc::new(ConsensusManagerMetrics::new(metrics_registry)); - let mysticeti_client = Arc::new(LazyMysticetiClient::new()); - let mysticeti_manager = ProtocolManager::new_mysticeti( - node_config, - consensus_config, - registry_service, - metrics.clone(), - mysticeti_client.clone(), - ); let starfish_client = Arc::new(LazyStarfishClient::new()); - let starfish_manager = ProtocolManager::new_starfish( - node_config, - consensus_config, - registry_service, + consensus_client.set(starfish_client.clone()); + let starfish_manager = StarfishManager::new( + node_config.protocol_key_pair().copy(), + node_config.network_key_pair().copy(), + consensus_config.db_path().to_path_buf(), + registry_service.clone(), metrics, - starfish_client.clone(), + starfish_client, ); Self { consensus_config: consensus_config.clone(), - mysticeti_manager, starfish_manager, - mysticeti_client, - starfish_client, - active: parking_lot::Mutex::new(vec![false; 2]), - consensus_client, } } pub fn get_storage_base_path(&self) -> PathBuf { self.consensus_config.db_path().to_path_buf() } - - // Picks the consensus protocol based on the protocol config and the epoch. - fn pick_protocol(&self, epoch_store: &AuthorityPerEpochStore) -> ConsensusProtocol { - let protocol_config = epoch_store.protocol_config(); - if let Ok(consensus_choice) = std::env::var("CONSENSUS_PROTOCOL") { - match consensus_choice.to_lowercase().as_str() { - "mysticeti" => return ConsensusProtocol::Mysticeti, - "starfish" => return ConsensusProtocol::Starfish, - "swap_each_epoch" => { - let protocol = if epoch_store.epoch().is_multiple_of(2) { - ConsensusProtocol::Starfish - } else { - ConsensusProtocol::Mysticeti - }; - return protocol; - } - _ => { - info!( - "Invalid consensus choice {} in env var. Continue to pick consensus with protocol config", - consensus_choice - ); - } - }; - } - - match protocol_config.consensus_choice() { - ConsensusChoice::Mysticeti => ConsensusProtocol::Mysticeti, - ConsensusChoice::Starfish => ConsensusProtocol::Starfish, - } - } } #[async_trait] @@ -195,31 +99,8 @@ impl ConsensusManagerTrait for ConsensusManager { consensus_handler_initializer: ConsensusHandlerInitializer, tx_validator: IotaTxValidator, ) { - let protocol_manager = { - let mut active = self.active.lock(); - active.iter().enumerate().for_each(|(index, active)| { - assert!( - !*active, - "Cannot start consensus. ConsensusManager protocol {index} is already running" - ); - }); - let protocol = self.pick_protocol(&epoch_store); - info!("Starting consensus protocol {protocol:?} ..."); - match protocol { - ConsensusProtocol::Mysticeti => { - active[0] = true; - self.consensus_client.set(self.mysticeti_client.clone()); - &self.mysticeti_manager - } - ConsensusProtocol::Starfish => { - active[1] = true; - self.consensus_client.set(self.starfish_client.clone()); - &self.starfish_manager - } - } - }; - - protocol_manager + info!("Starting Starfish consensus ..."); + self.starfish_manager .start( node_config, epoch_store, @@ -231,24 +112,11 @@ impl ConsensusManagerTrait for ConsensusManager { async fn shutdown(&self) { info!("Shutting down consensus ..."); - let prev_active = { - let mut active = self.active.lock(); - std::mem::replace(&mut *active, vec![false; 2]) - }; - if prev_active[0] { - self.mysticeti_manager.shutdown().await; - } - - if prev_active[1] { - self.starfish_manager.shutdown().await; - } - - self.consensus_client.clear(); + self.starfish_manager.shutdown().await; } async fn is_running(&self) -> bool { - let active = self.active.lock(); - active.iter().any(|i| *i) + self.starfish_manager.is_running().await } } diff --git a/crates/iota-core/src/consensus_manager/mysticeti_manager.rs b/crates/iota-core/src/consensus_manager/mysticeti_manager.rs deleted file mode 100644 index e79de4927ab..00000000000 --- a/crates/iota-core/src/consensus_manager/mysticeti_manager.rs +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{path::PathBuf, sync::Arc}; - -use arc_swap::ArcSwapOption; -use async_trait::async_trait; -use consensus_config::{Committee, NetworkKeyPair, Parameters, ProtocolKeyPair}; -use consensus_core::{ - Clock, CommitConsumer, CommitConsumerMonitor, CommitIndex, ConsensusAuthority, -}; -use fastcrypto::ed25519; -use iota_config::NodeConfig; -use iota_metrics::{RegistryID, RegistryService, monitored_mpsc::unbounded_channel}; -use iota_protocol_config::ConsensusNetwork; -use iota_types::{ - committee::EpochId, - iota_system_state::epoch_start_iota_system_state::EpochStartSystemStateTrait, -}; -use prometheus::Registry; -use tokio::sync::Mutex; -use tracing::info; - -use crate::{ - authority::authority_per_epoch_store::AuthorityPerEpochStore, - consensus_handler::{ConsensusHandlerInitializer, MysticetiConsensusHandler}, - consensus_manager::{ - ConsensusManagerMetrics, ConsensusManagerTrait, Running, RunningLockGuard, - }, - consensus_validator::IotaTxValidator, - mysticeti_adapter::LazyMysticetiClient, -}; - -#[cfg(test)] -#[path = "../unit_tests/mysticeti_manager_tests.rs"] -pub mod mysticeti_manager_tests; - -pub struct MysticetiManager { - protocol_keypair: ProtocolKeyPair, - network_keypair: NetworkKeyPair, - storage_base_path: PathBuf, - running: Mutex, - metrics: Arc, - registry_service: RegistryService, - authority: ArcSwapOption<(ConsensusAuthority, RegistryID)>, - boot_counter: Mutex, - // Use a shared lazy mysticeti client so we can update the internal mysticeti - // client that gets created for every new epoch. - client: Arc, - consensus_handler: Mutex>, - consumer_monitor: ArcSwapOption, -} - -impl MysticetiManager { - /// NOTE: Mysticeti protocol key uses Ed25519 instead of BLS. - /// But for security, the protocol keypair must be different from the - /// network keypair. - pub fn new( - protocol_keypair: ed25519::Ed25519KeyPair, - network_keypair: ed25519::Ed25519KeyPair, - storage_base_path: PathBuf, - registry_service: RegistryService, - metrics: Arc, - client: Arc, - ) -> Self { - Self { - protocol_keypair: ProtocolKeyPair::new(protocol_keypair), - network_keypair: NetworkKeyPair::new(network_keypair), - storage_base_path, - running: Mutex::new(Running::False), - metrics, - registry_service, - authority: ArcSwapOption::empty(), - client, - consensus_handler: Mutex::new(None), - boot_counter: Mutex::new(0), - consumer_monitor: ArcSwapOption::empty(), - } - } - - fn get_store_path(&self, epoch: EpochId) -> PathBuf { - let mut store_path = self.storage_base_path.clone(); - store_path.push(format!("{epoch}")); - store_path - } - - fn pick_network(&self, epoch_store: &AuthorityPerEpochStore) -> ConsensusNetwork { - if let Ok(type_str) = std::env::var("CONSENSUS_NETWORK") { - match type_str.to_lowercase().as_str() { - "tonic" => return ConsensusNetwork::Tonic, - _ => { - info!( - "Invalid consensus network type {} in env var. Continue to use the value from protocol config.", - type_str - ); - } - } - } - epoch_store.protocol_config().consensus_network() - } -} - -#[async_trait] - -impl ConsensusManagerTrait for MysticetiManager { - /// Starts the Mysticeti consensus manager for the current epoch. - async fn start( - &self, - config: &NodeConfig, - epoch_store: Arc, - consensus_handler_initializer: ConsensusHandlerInitializer, - tx_validator: IotaTxValidator, - ) { - let system_state = epoch_store.epoch_start_state(); - let committee: Committee = system_state.get_consensus_committee(); - let epoch = epoch_store.epoch(); - let protocol_config = epoch_store.protocol_config(); - let network_type = self.pick_network(&epoch_store); - - let Some(_guard) = RunningLockGuard::acquire_start( - &self.metrics, - &self.running, - epoch, - protocol_config.version, - ) - .await - else { - return; - }; - - let consensus_config = config - .consensus_config() - .expect("consensus_config should exist"); - - let parameters = Parameters { - db_path: self.get_store_path(epoch), - ..consensus_config.parameters.clone().unwrap_or_default() - }; - - let own_protocol_key = self.protocol_keypair.public(); - let (own_index, _) = committee - .authorities() - .find(|(_, a)| a.protocol_key == own_protocol_key) - .expect("Own authority should be among the consensus authorities!"); - - let registry = Registry::new_custom(Some("consensus".to_string()), None).unwrap(); - - let (commit_sender, commit_receiver) = unbounded_channel("consensus_output"); - - let consensus_handler = consensus_handler_initializer.new_consensus_handler(); - - let num_prior_commits = protocol_config.consensus_num_requested_prior_commits_at_startup(); - let last_processed_commit = consensus_handler.last_processed_subdag_index() as CommitIndex; - let starting_commit = last_processed_commit.saturating_sub(num_prior_commits); - let consumer = CommitConsumer::new(commit_sender, starting_commit); - let monitor = consumer.monitor(); - - // If there is a previous consumer monitor, it indicates that the consensus - // engine has been restarted, due to an epoch change. However, that on its - // own doesn't tell us much whether it participated on an active epoch or an old - // one. We need to check if it has handled any commits to determine this. - // If indeed any commits did happen, then we assume that node did participate on - // previous run. - let participated_on_previous_run = - if let Some(previous_monitor) = self.consumer_monitor.swap(Some(monitor.clone())) { - previous_monitor.highest_handled_commit() > 0 - } else { - false - }; - - // Increment the boot counter only if the consensus successfully participated in - // the previous run. This is typical during normal epoch changes, where - // the node restarts as expected, and the boot counter is incremented to prevent - // amnesia recovery on the next start. If the node is recovering from a - // restore process and catching up across multiple epochs, it won't handle any - // commits until it reaches the last active epoch. In this scenario, we - // do not increment the boot counter, as we need amnesia recovery to run. - let mut boot_counter = self.boot_counter.lock().await; - if participated_on_previous_run { - *boot_counter += 1; - } else { - info!( - "Node has not participated in previous epoch consensus. Boot counter ({}) will not increment.", - *boot_counter - ); - } - - let authority = ConsensusAuthority::start( - network_type, - epoch_store.epoch_start_config().epoch_start_timestamp_ms(), - own_index, - committee.clone(), - parameters.clone(), - protocol_config.clone(), - self.protocol_keypair.clone(), - self.network_keypair.clone(), - Arc::new(Clock::default()), - Arc::new(tx_validator.clone()), - consumer, - registry.clone(), - epoch_store.scorer.current_local_metrics_count.clone(), - *boot_counter, - ) - .await; - let client = authority.transaction_client(); - - let registry_id = self.registry_service.add(registry.clone()); - - let registered_authority = Arc::new((authority, registry_id)); - self.authority.swap(Some(registered_authority.clone())); - - // Initialize the client to send transactions to this Mysticeti instance. - self.client.set(client); - - // spin up the new mysticeti consensus handler to listen for committed sub dags - let handler = MysticetiConsensusHandler::new( - last_processed_commit, - consensus_handler, - commit_receiver, - monitor, - ); - - let mut consensus_handler = self.consensus_handler.lock().await; - *consensus_handler = Some(handler); - - // Wait until all locally available commits have been processed - info!("replaying commits at startup"); - registered_authority.0.replay_complete().await; - info!("Startup commit replay complete"); - } - - async fn shutdown(&self) { - let Some(_guard) = RunningLockGuard::acquire_shutdown(&self.metrics, &self.running).await - else { - return; - }; - - // Stop consensus submissions. - self.client.clear(); - - // swap with empty to ensure there is no other reference to authority and we can - // safely do Arc unwrap - let r = self.authority.swap(None).unwrap(); - let Ok((authority, registry_id)) = Arc::try_unwrap(r) else { - panic!("Failed to retrieve the mysticeti authority"); - }; - - // shutdown the authority and wait for it - authority.stop().await; - - // drop the old consensus handler to force stop any underlying task running. - let mut consensus_handler = self.consensus_handler.lock().await; - if let Some(mut handler) = consensus_handler.take() { - handler.abort().await; - } - - // unregister the registry id - self.registry_service.remove(registry_id); - } - - async fn is_running(&self) -> bool { - Running::False != *self.running.lock().await - } -} diff --git a/crates/iota-core/src/consensus_manager/starfish_manager.rs b/crates/iota-core/src/consensus_manager/starfish_manager.rs index c5ff7cac766..58416339de5 100644 --- a/crates/iota-core/src/consensus_manager/starfish_manager.rs +++ b/crates/iota-core/src/consensus_manager/starfish_manager.rs @@ -97,7 +97,7 @@ impl ConsensusManagerTrait for StarfishManager { tx_validator: IotaTxValidator, ) { let system_state = epoch_store.epoch_start_state(); - let committee: Committee = system_state.get_starfish_committee(); + let committee: Committee = system_state.get_consensus_committee(); let epoch = epoch_store.epoch(); let protocol_config = epoch_store.protocol_config(); diff --git a/crates/iota-core/src/consensus_types/consensus_output_api.rs b/crates/iota-core/src/consensus_types/consensus_output_api.rs index 74feff6cdb0..cae25373971 100644 --- a/crates/iota-core/src/consensus_types/consensus_output_api.rs +++ b/crates/iota-core/src/consensus_types/consensus_output_api.rs @@ -4,7 +4,6 @@ use std::{collections::BTreeMap, fmt::Display}; -use consensus_core::BlockAPI; use iota_types::{digests::ConsensusCommitDigest, messages_consensus::ConsensusTransaction}; use crate::consensus_types::AuthorityIndex; @@ -133,22 +132,6 @@ macro_rules! impl_consensus_output_api { }; } -// ===== Use the macro for the two concrete types ===== - -// consensus_core::CommittedSubDag: -// - iterate over `self.blocks` -// - per-item accessors: round()/author().value()/transactions() -// - committed_header_refs: map blocks to BlockRef -impl_consensus_output_api! { - type = consensus_core::CommittedSubDag, - commit_digest = consensus_core::CommitDigest, - iterate = |self_, block| self_.blocks.iter(), - round = |block| block.round(), - author = |block| block.author().value(), - txs = |block| block.transactions(), - committed_header_refs = |self_| self_.blocks.iter().map(|b| b.reference()).collect::>() -} - // starfish_core::CommittedSubDag: // - iterate over `self.transactions` (VerifiedTransactions) // - per-item accessors: round()/author().value()/transactions() diff --git a/crates/iota-core/src/consensus_types/mod.rs b/crates/iota-core/src/consensus_types/mod.rs index f636129968b..251dc2ae0dc 100644 --- a/crates/iota-core/src/consensus_types/mod.rs +++ b/crates/iota-core/src/consensus_types/mod.rs @@ -5,5 +5,4 @@ pub(crate) mod consensus_output_api; /// An unique integer ID for a validator used by consensus. -/// In Mysticeti, this is used the same way as the AuthorityIndex type there. pub type AuthorityIndex = u32; diff --git a/crates/iota-core/src/consensus_validator.rs b/crates/iota-core/src/consensus_validator.rs index 3cb1aa0feb9..8770c8b460c 100644 --- a/crates/iota-core/src/consensus_validator.rs +++ b/crates/iota-core/src/consensus_validator.rs @@ -4,7 +4,6 @@ use std::sync::Arc; -use consensus_core; use eyre::WrapErr; use fastcrypto_tbls::dkg_v1; use iota_metrics::monitored_scope; @@ -175,12 +174,6 @@ macro_rules! impl_tx_verifier_for { } }; } -// Use it for both traits: -impl_tx_verifier_for!( - type = IotaTxValidator, - trait = consensus_core::TransactionVerifier, - error = consensus_core::ValidationError, -); impl_tx_verifier_for!( type = IotaTxValidator, trait = starfish_core::TransactionVerifier, @@ -222,7 +215,7 @@ impl IotaTxValidatorMetrics { mod tests { use std::sync::Arc; - use consensus_core::TransactionVerifier as _; + use starfish_core::TransactionVerifier as _; use iota_macros::sim_test; use iota_protocol_config::Chain; use iota_types::{ diff --git a/crates/iota-core/src/epoch/consensus_store_pruner.rs b/crates/iota-core/src/epoch/consensus_store_pruner.rs index f1fa322895c..0d938bc18d5 100644 --- a/crates/iota-core/src/epoch/consensus_store_pruner.rs +++ b/crates/iota-core/src/epoch/consensus_store_pruner.rs @@ -4,7 +4,7 @@ use std::{fs, path::PathBuf, time::Duration}; -use consensus_config::Epoch; +use starfish_config::Epoch; use iota_metrics::spawn_logged_monitored_task; use prometheus::{ IntCounter, IntCounterVec, IntGauge, Registry, register_int_counter_vec_with_registry, diff --git a/crates/iota-core/src/epoch/randomness.rs b/crates/iota-core/src/epoch/randomness.rs index 4dc667a2bf1..aa872cd69ae 100644 --- a/crates/iota-core/src/epoch/randomness.rs +++ b/crates/iota-core/src/epoch/randomness.rs @@ -834,7 +834,7 @@ pub enum DkgStatus { mod tests { use std::num::NonZeroUsize; - use consensus_core::{BlockRef, BlockStatus}; + use starfish_core::{BlockRef, BlockStatus}; use iota_protocol_config::{Chain, ProtocolConfig, ProtocolVersion}; use iota_types::messages_consensus::ConsensusTransactionKind; use tokio::sync::mpsc; @@ -885,7 +885,7 @@ mod tests { tx_consensus.try_send(transactions.to_vec()).unwrap(); true }) - .returning(|_, _| Ok(with_block_status(BlockStatus::Sequenced(BlockRef::MIN)))); + .returning(|_, _| Ok(with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN))))); let state = TestAuthorityBuilder::new() .with_protocol_config(protocol_config.clone()) @@ -1032,8 +1032,8 @@ mod tests { true }) .returning(|_, _| { - Ok(with_block_status(consensus_core::BlockStatus::Sequenced( - BlockRef::MIN, + Ok(with_block_status(starfish_core::BlockStatus::Sequenced( + starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN), ))) }); diff --git a/crates/iota-core/src/lib.rs b/crates/iota-core/src/lib.rs index 1d8069708b9..c479c8978d8 100644 --- a/crates/iota-core/src/lib.rs +++ b/crates/iota-core/src/lib.rs @@ -27,7 +27,6 @@ pub mod jsonrpc_index; pub mod metrics; pub mod mock_consensus; pub mod module_cache_metrics; -pub mod mysticeti_adapter; pub mod overload_monitor; mod par_index_live_object_set; pub(crate) mod post_consensus_tx_reorder; diff --git a/crates/iota-core/src/mock_consensus.rs b/crates/iota-core/src/mock_consensus.rs index 0bc7a44cda9..a9f31730eee 100644 --- a/crates/iota-core/src/mock_consensus.rs +++ b/crates/iota-core/src/mock_consensus.rs @@ -4,13 +4,13 @@ use std::sync::{Arc, Weak}; -use consensus_core::BlockRef; use iota_types::{ error::{IotaError, IotaResult}, messages_consensus::{ConsensusTransaction, ConsensusTransactionKind}, transaction::VerifiedCertificate, }; use prometheus::Registry; +use starfish_core::BlockRef; use tokio::{ sync::{mpsc, oneshot}, task::JoinHandle, @@ -105,8 +105,8 @@ impl MockConsensusClient { self.tx_sender .try_send(transaction.clone()) .map_err(|_| IotaError::from("MockConsensusClient channel overflowed"))?; - Ok(with_block_status(consensus_core::BlockStatus::Sequenced( - BlockRef::MIN, + Ok(with_block_status(starfish_core::BlockStatus::Sequenced( + starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN) ))) } } @@ -132,7 +132,7 @@ impl ConsensusClient for MockConsensusClient { } } -pub(crate) fn with_block_status(status: consensus_core::BlockStatus) -> BlockStatusReceiver { +pub(crate) fn with_block_status(status: starfish_core::BlockStatus) -> BlockStatusReceiver { let (tx, rx) = oneshot::channel(); tx.send(status.into()).ok(); rx diff --git a/crates/iota-core/src/mysticeti_adapter.rs b/crates/iota-core/src/mysticeti_adapter.rs deleted file mode 100644 index c535271e776..00000000000 --- a/crates/iota-core/src/mysticeti_adapter.rs +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use arc_swap::{ArcSwapOption, Guard}; -use consensus_core::{ClientError, TransactionClient}; -use iota_types::{ - error::{IotaError, IotaResult}, - messages_consensus::{ConsensusTransaction, ConsensusTransactionKind}, -}; -use tap::prelude::*; -use tokio::time::{Instant, sleep}; -use tracing::{error, info, warn}; - -use crate::{ - authority::authority_per_epoch_store::AuthorityPerEpochStore, - consensus_adapter, - consensus_adapter::{BlockStatusReceiver, ConsensusClient}, - consensus_handler::SequencedConsensusTransactionKey, -}; - -/// Gets a client to submit transactions to Mysticeti, or waits for one to be -/// available. This hides the complexities of async consensus initialization and -/// submitting to different instances of consensus across epochs. -#[derive(Default, Clone)] -pub struct LazyMysticetiClient { - client: Arc>, -} - -impl LazyMysticetiClient { - pub fn new() -> Self { - Self { - client: Arc::new(ArcSwapOption::empty()), - } - } - - async fn get(&self) -> Guard>> { - let client = self.client.load(); - if client.is_some() { - return client; - } - - // Consensus client is initialized after validators or epoch starts, and cleared - // after an epoch ends. But calls to get() can happen during validator - // startup or epoch change, before consensus finished initializations. - // TODO: maybe listen to updates from consensus manager instead of polling. - let mut count = 0; - let start = Instant::now(); - const RETRY_INTERVAL: Duration = Duration::from_millis(100); - loop { - let client = self.client.load(); - if client.is_some() { - return client; - } else { - sleep(RETRY_INTERVAL).await; - count += 1; - if count % 100 == 0 { - warn!( - "Waiting for consensus to initialize after {:?}", - Instant::now() - start - ); - } - } - } - } - - pub fn set(&self, client: Arc) { - self.client.store(Some(client)); - } - - pub fn clear(&self) { - self.client.store(None); - } -} - -#[async_trait::async_trait] -impl ConsensusClient for LazyMysticetiClient { - async fn submit( - &self, - transactions: &[ConsensusTransaction], - _epoch_store: &Arc, - ) -> IotaResult { - // TODO(mysticeti): confirm comment is still true - // The retrieved TransactionClient can be from the past epoch. Submit would fail - // after Mysticeti shuts down, so there should be no correctness issue. - let client = self.get().await; - let transactions_bytes = transactions - .iter() - .map(|t| bcs::to_bytes(t).expect("Serializing consensus transaction cannot fail")) - .collect::>(); - - let (block_ref, status_waiter) = client - .as_ref() - .expect("Client should always be returned") - .submit(transactions_bytes) - .await - .tap_err(|err| { - // Will be logged by caller as well. - let msg = format!("Transaction submission failed with: {err:?}"); - match err { - ClientError::ConsensusShuttingDown(_) => { - info!("{}", msg); - } - ClientError::OversizedTransaction(_, _) - | ClientError::OversizedTransactionBundleBytes(_, _) - | ClientError::OversizedTransactionBundleCount(_, _) => { - if cfg!(debug_assertions) { - panic!("{}", msg); - } else { - error!("{}", msg); - } - } - }; - }) - .map_err(|err| IotaError::FailedToSubmitToConsensus(err.to_string()))?; - - let is_soft_bundle = transactions.len() > 1; - - if !is_soft_bundle - && matches!( - transactions[0].kind, - ConsensusTransactionKind::EndOfPublish(_) - | ConsensusTransactionKind::CapabilityNotificationV1(_) - | ConsensusTransactionKind::RandomnessDkgMessage(_, _) - | ConsensusTransactionKind::RandomnessDkgConfirmation(_, _) - ) - { - let transaction_key = SequencedConsensusTransactionKey::External(transactions[0].key()); - tracing::info!("Transaction {transaction_key:?} was included in {block_ref}",) - }; - - // Transform the status waiter into a receiver that can be used by the consensus - // adapter. - - let (tx_internal, rx_internal) = - tokio::sync::oneshot::channel::(); - - tokio::spawn(async move { - if let Ok(status) = status_waiter.await { - let _ = tx_internal.send(status.into()); - } - }); - - Ok(rx_internal) - } -} diff --git a/crates/iota-core/src/scoring_decision.rs b/crates/iota-core/src/scoring_decision.rs index f9fafe441bc..c7a7d1b1719 100644 --- a/crates/iota-core/src/scoring_decision.rs +++ b/crates/iota-core/src/scoring_decision.rs @@ -5,7 +5,7 @@ use std::{collections::HashMap, sync::Arc}; use arc_swap::ArcSwap; -use consensus_config::Committee as ConsensusCommittee; +use starfish_config::Committee as ConsensusCommittee; use iota_types::{base_types::AuthorityName, committee::Committee}; use tracing::{debug, instrument}; @@ -92,7 +92,7 @@ mod tests { use std::{collections::HashMap, sync::Arc}; use arc_swap::ArcSwap; - use consensus_config::{Committee as ConsensusCommittee, local_committee_and_keys}; + use starfish_config::{Committee as ConsensusCommittee, local_committee_and_keys}; use iota_types::{committee::Committee, crypto::AuthorityPublicKeyBytes}; use prometheus::Registry; diff --git a/crates/iota-core/src/unit_tests/consensus_tests.rs b/crates/iota-core/src/unit_tests/consensus_tests.rs index 7b7bffecf54..4117a34be2b 100644 --- a/crates/iota-core/src/unit_tests/consensus_tests.rs +++ b/crates/iota-core/src/unit_tests/consensus_tests.rs @@ -4,7 +4,7 @@ use std::{collections::HashSet, time::Duration}; -use consensus_core::{BlockRef, BlockStatus}; +use starfish_core::{BlockRef, BlockStatus}; use fastcrypto::traits::KeyPair; use iota_macros::sim_test; use iota_protocol_config::ProtocolConfig; @@ -243,10 +243,10 @@ async fn submit_transaction_to_consensus_adapter() { // Make a new consensus adapter instance. let block_status_receivers = vec![ - with_block_status(BlockStatus::GarbageCollected(BlockRef::MIN)), - with_block_status(BlockStatus::GarbageCollected(BlockRef::MIN)), - with_block_status(BlockStatus::GarbageCollected(BlockRef::MIN)), - with_block_status(BlockStatus::Sequenced(BlockRef::MIN)), + with_block_status(BlockStatus::GarbageCollected(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN))), + with_block_status(BlockStatus::GarbageCollected(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN))), + with_block_status(BlockStatus::GarbageCollected(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN))), + with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN))), ]; let adapter = make_consensus_adapter_for_test( state.clone(), @@ -295,7 +295,7 @@ async fn submit_multiple_transactions_to_consensus_adapter() { state.clone(), process_via_checkpoint, false, - vec![with_block_status(BlockStatus::Sequenced(BlockRef::MIN))], + vec![with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN)))], ); // Submit the transaction and ensure the adapter reports success to the caller. @@ -332,7 +332,7 @@ async fn submit_checkpoint_signature_to_consensus_adapter() { state.clone(), HashSet::new(), false, - vec![with_block_status(BlockStatus::Sequenced(BlockRef::MIN))], + vec![with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN)))], ); let checkpoint_summary = CheckpointSummary::new( @@ -437,7 +437,7 @@ async fn submit_checkpoint_signature_to_consensus_adapter() { /// are submitted instead of one. #[tokio::test] async fn submit_recovered_end_of_publish_crash_recovery() { - use consensus_core::{BlockRef, BlockStatus}; + use starfish_core::{BlockRef, BlockStatus}; use tokio::sync::Notify; use crate::mock_consensus::with_block_status; @@ -463,7 +463,7 @@ async fn submit_recovered_end_of_publish_crash_recovery() { self.submitted.lock().extend_from_slice(transactions); self.notify.notify_one(); - Ok(with_block_status(BlockStatus::Sequenced(BlockRef::MIN))) + Ok(with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN)))) } } diff --git a/crates/iota-core/src/unit_tests/mysticeti_manager_tests.rs b/crates/iota-core/src/unit_tests/mysticeti_manager_tests.rs deleted file mode 100644 index eb8e99c9f8b..00000000000 --- a/crates/iota-core/src/unit_tests/mysticeti_manager_tests.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// Modifications Copyright (c) 2024 IOTA Stiftung -// SPDX-License-Identifier: Apache-2.0 - -use std::{sync::Arc, time::Duration}; - -use arc_swap::ArcSwap; -use consensus_core::{CommitDigest, CommitRef, CommittedSubDag, TestBlock, VerifiedBlock}; -use fastcrypto::traits::KeyPair; -use futures::FutureExt; -use iota_metrics::{RegistryService, monitored_mpsc::unbounded_channel}; -use iota_swarm_config::network_config_builder::ConfigBuilder; -use iota_types::{ - iota_system_state::epoch_start_iota_system_state::EpochStartSystemStateTrait, - messages_checkpoint::{CertifiedCheckpointSummary, CheckpointContents, CheckpointSummary}, -}; -use prometheus::Registry; -use tokio::{sync::mpsc, time::sleep}; - -use crate::{ - authority::{ - AuthorityMetrics, AuthorityState, backpressure::BackpressureManager, - test_authority_builder::TestAuthorityBuilder, - }, - checkpoints::{CheckpointMetrics, CheckpointService, CheckpointServiceNoop}, - consensus_handler::{ConsensusHandler, ConsensusHandlerInitializer, MysticetiConsensusHandler}, - consensus_manager::{ - ConsensusManagerMetrics, ConsensusManagerTrait, mysticeti_manager::MysticetiManager, - }, - consensus_validator::{IotaTxValidator, IotaTxValidatorMetrics}, - mysticeti_adapter::LazyMysticetiClient, - state_accumulator::StateAccumulator, -}; - -pub fn checkpoint_service_for_testing(state: Arc) -> Arc { - let (output, _result) = mpsc::channel::<(CheckpointContents, CheckpointSummary)>(10); - let epoch_store = state.epoch_store_for_testing(); - let accumulator = Arc::new(StateAccumulator::new_for_tests( - state.get_accumulator_store().clone(), - )); - let (certified_output, _certified_result) = mpsc::channel::(10); - - let checkpoint_service = CheckpointService::build( - state.clone(), - state.get_checkpoint_store().clone(), - epoch_store.clone(), - state.get_transaction_cache_reader().clone(), - Arc::downgrade(&accumulator), - Box::new(output), - Box::new(certified_output), - CheckpointMetrics::new_for_tests(), - 3, - 100_000, - ); - checkpoint_service.spawn().now_or_never().unwrap(); - checkpoint_service -} - -#[tokio::test(flavor = "current_thread", start_paused = true)] -async fn test_mysticeti_manager() { - // GIVEN - let configs = ConfigBuilder::new_with_temp_dir() - .committee_size(4.try_into().unwrap()) - .build(); - - let config = &configs.validator_configs()[0]; - - let consensus_config = config.consensus_config().unwrap(); - let registry_service = RegistryService::new(Registry::new()); - let secret = Arc::pin(config.authority_key_pair().copy()); - let genesis = config.genesis().unwrap(); - - let state = TestAuthorityBuilder::new() - .with_genesis_and_keypair(genesis, &secret) - .build() - .await; - - let metrics = Arc::new(ConsensusManagerMetrics::new(&Registry::new())); - let epoch_store = state.epoch_store_for_testing(); - let client = Arc::new(LazyMysticetiClient::default()); - - let manager = MysticetiManager::new( - config.protocol_key_pair().copy(), - config.network_key_pair().copy(), - consensus_config.db_path().to_path_buf(), - registry_service, - metrics, - client, - ); - - let boot_counter = *manager.boot_counter.lock().await; - assert_eq!(boot_counter, 0); - - for i in 1..=3 { - let consensus_handler_initializer = ConsensusHandlerInitializer::new_for_testing( - state.clone(), - checkpoint_service_for_testing(state.clone()), - ); - - // WHEN start mysticeti - manager - .start( - config, - epoch_store.clone(), - consensus_handler_initializer, - IotaTxValidator::new( - epoch_store.clone(), - Arc::new(CheckpointServiceNoop {}), - state.transaction_manager().clone(), - IotaTxValidatorMetrics::new(&Registry::new()), - ), - ) - .await; - - // THEN - assert!(manager.is_running().await); - let boot_counter = *manager.boot_counter.lock().await; - if i == 1 || i == 2 { - assert_eq!(boot_counter, 0); - } else { - assert_eq!(boot_counter, 1); - } - - // Now try to shut it down - sleep(Duration::from_secs(1)).await; - - // Simulate a commit by bumping the handled commit index so we can ensure that - // boot counter increments only after the first run. Practically we want - // to simulate a case where consensus engine restarts when no commits have - // happened before for first run. - if i > 1 { - let monitor = manager - .consumer_monitor - .load_full() - .expect("A consumer monitor should have been initialised"); - monitor.set_highest_handled_commit(100); - } - - // WHEN - manager.shutdown().await; - - // THEN - assert!(!manager.is_running().await); - } -} - -#[tokio::test(flavor = "current_thread", start_paused = true)] -async fn test_mysticeti_consensus_handler_handles_older_commits() { - // GIVEN - let network_config = ConfigBuilder::new_with_temp_dir() - .committee_size(4.try_into().unwrap()) - .build(); - - let state = TestAuthorityBuilder::new() - .with_network_config(&network_config, 0) - .build() - .await; - - let epoch_store = state.epoch_store_for_testing().clone(); - let new_epoch_start_state = epoch_store.epoch_start_state(); - let consensus_committee = new_epoch_start_state.get_consensus_committee(); - - let metrics = Arc::new(AuthorityMetrics::new(&Registry::new())); - let backpressure_manager = BackpressureManager::new_for_tests(); - - let consensus_handler = ConsensusHandler::new( - epoch_store.clone(), - checkpoint_service_for_testing(state.clone()), - state.transaction_manager().clone(), - state.get_object_cache_reader().clone(), - state.get_transaction_cache_reader().clone(), - Arc::new(ArcSwap::default()), - consensus_committee.clone(), - metrics, - backpressure_manager.subscribe(), - ); - - // Create commits 1-10, where commits 1-7 are "older" (already processed at - // startup) and commits 8-10 are "newer" (should be processed normally) - let all_commits: Vec<_> = (1..=10) - .map(|commit_idx| { - let round = commit_idx as u32; - let leader_authority = (commit_idx % consensus_committee.size() as u64) as u32; - - let leader_block = - VerifiedBlock::new_for_test(TestBlock::new(round, leader_authority).build()); - - let timestamp_ms = round as u64 * 1000; - // Create a simple commit with just the leader block reference - // We don't need full blocks or transactions for this test - CommittedSubDag::new( - leader_block.reference(), - vec![leader_block], - timestamp_ms, - CommitRef::new(commit_idx as u32, CommitDigest::MIN), - vec![], - ) - }) - .collect(); - - // Set last_processed_commit_at_startup to 7 - let last_processed_commit_at_startup = 7; - - let (commit_sender, commit_receiver) = unbounded_channel("consensus_output"); - let commit_consumer = consensus_core::CommitConsumer::new(commit_sender.clone(), 0); - let commit_consumer_monitor = commit_consumer.monitor().clone(); - - // WHEN we create the MysticetiConsensusHandler - let _handler = MysticetiConsensusHandler::new( - last_processed_commit_at_startup, - consensus_handler, - commit_receiver, - commit_consumer_monitor.clone(), - ); - - // Send all commits in order - for commit in all_commits { - commit_sender.send(commit).unwrap(); - } - - // Give time for processing - sleep(Duration::from_millis(100)).await; - - // THEN verify that the highest handled commit is only updated for newer commits - // (8-10) Commits 1-7 should have been processed as prior commits and - // not update the monitor - let highest_handled = commit_consumer_monitor.highest_handled_commit(); - assert_eq!( - highest_handled, 10, - "Expected highest handled commit to be 10, got {}", - highest_handled - ); -} diff --git a/crates/iota-core/src/unit_tests/transaction_tests.rs b/crates/iota-core/src/unit_tests/transaction_tests.rs index 2f7ebe7f034..366372738d7 100644 --- a/crates/iota-core/src/unit_tests/transaction_tests.rs +++ b/crates/iota-core/src/unit_tests/transaction_tests.rs @@ -7,7 +7,7 @@ use std::{ ops::Deref, }; -use consensus_core::{BlockRef, BlockStatus}; +use starfish_core::{BlockRef, BlockStatus}; use fastcrypto::{ed25519::Ed25519KeyPair, traits::KeyPair}; use fastcrypto_zkp::bn254::zk_login::{OIDCProvider, ZkLoginInputs, parse_jwks}; use iota_macros::sim_test; @@ -1702,7 +1702,7 @@ async fn test_handle_soft_bundle_certificates() { authority.clone(), HashSet::new(), true, - vec![with_block_status(BlockStatus::Sequenced(BlockRef::MIN))], + vec![with_block_status(BlockStatus::Sequenced(starfish_core::GenericTransactionRef::BlockRef(BlockRef::MIN)))], ); let server = AuthorityServer::new_for_test_with_consensus_adapter(authority.clone(), adapter); let _metrics = server.metrics.clone(); diff --git a/crates/iota-protocol-config/src/lib.rs b/crates/iota-protocol-config/src/lib.rs index 73345261f85..8979d9dd132 100644 --- a/crates/iota-protocol-config/src/lib.rs +++ b/crates/iota-protocol-config/src/lib.rs @@ -267,9 +267,6 @@ struct FeatureFlags { #[serde(skip_serializing_if = "PerObjectCongestionControlMode::is_none")] per_object_congestion_control_mode: PerObjectCongestionControlMode, - // The consensus protocol to be used for the epoch. - #[serde(skip_serializing_if = "ConsensusChoice::is_mysticeti")] - consensus_choice: ConsensusChoice, // Consensus network to use. #[serde(skip_serializing_if = "ConsensusNetwork::is_tonic")] @@ -518,23 +515,6 @@ impl PerObjectCongestionControlMode { } } -// Configuration options for consensus algorithm. -#[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)] -pub enum ConsensusChoice { - #[default] - Mysticeti, - Starfish, -} - -impl ConsensusChoice { - pub fn is_mysticeti(&self) -> bool { - matches!(self, ConsensusChoice::Mysticeti) - } - pub fn is_starfish(&self) -> bool { - matches!(self, ConsensusChoice::Starfish) - } -} - // Configuration options for consensus network. #[derive(Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)] pub enum ConsensusNetwork { @@ -1390,10 +1370,6 @@ impl ProtocolConfig { self.feature_flags.per_object_congestion_control_mode } - pub fn consensus_choice(&self) -> ConsensusChoice { - self.feature_flags.consensus_choice - } - pub fn consensus_network(&self) -> ConsensusNetwork { self.feature_flags.consensus_network } @@ -2282,9 +2258,7 @@ impl ProtocolConfig { cfg.feature_flags.zklogin_max_epoch_upper_bound_delta = Some(30); } - // Enable Mysticeti on mainnet. - cfg.feature_flags.consensus_choice = ConsensusChoice::Mysticeti; - // Use tonic networking for Mysticeti. + // Use tonic networking for consensus. cfg.feature_flags.consensus_network = ConsensusNetwork::Tonic; cfg.feature_flags.per_object_congestion_control_mode = @@ -2552,7 +2526,6 @@ impl ProtocolConfig { } if chain != Chain::Testnet && chain != Chain::Mainnet { // Switch consensus protocol to Starfish in devnet - cfg.feature_flags.consensus_choice = ConsensusChoice::Starfish; } } 15 => { @@ -2609,7 +2582,6 @@ impl ProtocolConfig { if chain != Chain::Mainnet { // Switch consensus protocol to Starfish in testnet. - cfg.feature_flags.consensus_choice = ConsensusChoice::Starfish; // Enable validator score calculation on testnet cfg.feature_flags.calculate_validator_scores = true; @@ -2714,7 +2686,6 @@ impl ProtocolConfig { } 24 => { // Switch consensus protocol to Starfish in all networks. - cfg.feature_flags.consensus_choice = ConsensusChoice::Starfish; if chain != Chain::Testnet && chain != Chain::Mainnet { // Enable Move-based sponsor account authentication in devnet. @@ -2845,10 +2816,6 @@ impl ProtocolConfig { self.feature_flags.per_object_congestion_control_mode = val; } - pub fn set_consensus_choice_for_testing(&mut self, val: ConsensusChoice) { - self.feature_flags.consensus_choice = val; - } - pub fn set_consensus_network_for_testing(&mut self, val: ConsensusNetwork) { self.feature_flags.consensus_network = val; } diff --git a/crates/iota-swarm-config/src/node_config_builder.rs b/crates/iota-swarm-config/src/node_config_builder.rs index 84394300df6..41dc2caef35 100644 --- a/crates/iota-swarm-config/src/node_config_builder.rs +++ b/crates/iota-swarm-config/src/node_config_builder.rs @@ -163,7 +163,6 @@ impl ValidatorConfigBuilder { max_pending_transactions: None, max_submit_position: self.max_submit_position, submit_delay_step_override_millis: self.submit_delay_step_override_millis, - parameters: Default::default(), starfish_parameters: Default::default(), }; diff --git a/crates/iota-types/Cargo.toml b/crates/iota-types/Cargo.toml index b627f3ceef1..fd3ee2c2ea5 100644 --- a/crates/iota-types/Cargo.toml +++ b/crates/iota-types/Cargo.toml @@ -60,7 +60,6 @@ tonic.workspace = true tracing.workspace = true # internal dependencies -consensus-config.workspace = true iota-enum-compat-util.workspace = true iota-macros.workspace = true iota-network-stack.workspace = true diff --git a/crates/iota-types/src/iota_system_state/epoch_start_iota_system_state.rs b/crates/iota-types/src/iota_system_state/epoch_start_iota_system_state.rs index ec2a1802696..fb04dccfc3d 100644 --- a/crates/iota-types/src/iota_system_state/epoch_start_iota_system_state.rs +++ b/crates/iota-types/src/iota_system_state/epoch_start_iota_system_state.rs @@ -8,11 +8,11 @@ use anemo::{ PeerId, types::{PeerAffinity, PeerInfo}, }; -use consensus_config::{Authority, Committee as ConsensusCommittee}; + use enum_dispatch::enum_dispatch; use iota_protocol_config::ProtocolVersion; use serde::{Deserialize, Serialize}; -use starfish_config::{Authority as StarfishAuthority, Committee as StarfishCommittee}; +use starfish_config::{Authority, Committee as ConsensusCommittee}; use tracing::{error, warn}; use crate::{ @@ -35,7 +35,6 @@ pub trait EpochStartSystemStateTrait { fn get_iota_committee(&self) -> Committee; fn get_iota_committee_with_network_metadata(&self) -> CommitteeWithNetworkMetadata; fn get_consensus_committee(&self) -> ConsensusCommittee; - fn get_starfish_committee(&self) -> StarfishCommittee; fn get_validator_as_p2p_peers(&self, excluding_self: AuthorityName) -> Vec; fn get_authority_names_to_peer_ids(&self) -> HashMap; fn get_authority_names_to_hostnames(&self) -> HashMap; @@ -282,13 +281,6 @@ impl EpochStartSystemStateTrait for EpochStartSystemStateV1 { impl_get_committee!( fn get_consensus_committee -> ConsensusCommittee, authority = Authority, - cfg = consensus_config, - label = "Mysticeti" - ); - - impl_get_committee!( - fn get_starfish_committee -> StarfishCommittee, - authority = StarfishAuthority, cfg = starfish_config, label = "Starfish" ); @@ -413,10 +405,6 @@ impl EpochStartSystemStateTrait for EpochStartSystemStateV2 { self.v1.get_consensus_committee() } - fn get_starfish_committee(&self) -> StarfishCommittee { - self.v1.get_starfish_committee() - } - fn get_validator_as_p2p_peers(&self, excluding_self: AuthorityName) -> Vec { self.v1.get_validator_as_p2p_peers(excluding_self) } @@ -491,7 +479,7 @@ mod test { }; #[test] - fn test_iota_and_mysticeti_committee_are_same() { + fn test_iota_and_consensus_committee_are_same() { // GIVEN let mut committee_validators = vec![]; @@ -559,7 +547,7 @@ mod test { } #[test] - fn test_v2_iota_and_mysticeti_committee_are_same() { + fn test_v2_iota_and_consensus_committee_are_same() { // GIVEN let mut committee_validators = vec![]; let mut non_committee_validators = vec![]; diff --git a/crates/iota-types/src/messages_consensus.rs b/crates/iota-types/src/messages_consensus.rs index 5a9175c0272..fa6ef5f00a7 100644 --- a/crates/iota-types/src/messages_consensus.rs +++ b/crates/iota-types/src/messages_consensus.rs @@ -557,24 +557,6 @@ impl ConsensusTransaction { kind: ConsensusTransactionKind::SignedCapabilityNotificationV1(signed_capabilities), } } - - pub fn new_mysticeti_certificate( - round: u64, - offset: u64, - certificate: CertifiedTransaction, - ) -> Self { - let mut hasher = DefaultHasher::new(); - let tx_digest = certificate.digest(); - tx_digest.hash(&mut hasher); - round.hash(&mut hasher); - offset.hash(&mut hasher); - let tracking_id = hasher.finish().to_le_bytes(); - Self { - tracking_id, - kind: ConsensusTransactionKind::CertifiedTransaction(Box::new(certificate)), - } - } - pub fn new_jwk_fetched(authority: AuthorityName, id: JwkId, jwk: JWK) -> Self { let mut hasher = DefaultHasher::new(); id.hash(&mut hasher); diff --git a/crates/starfish/core/src/commit_consumer.rs b/crates/starfish/core/src/commit_consumer.rs index f3717ab23f6..5167c761ec4 100644 --- a/crates/starfish/core/src/commit_consumer.rs +++ b/crates/starfish/core/src/commit_consumer.rs @@ -13,7 +13,7 @@ pub struct CommitConsumer { // A channel to send the committed sub dags through pub(crate) sender: UnboundedSender, // Index of the last commit that the consumer has processed. This is useful for - // crash/recovery so mysticeti can replay the commits from the next index. + // crash/recovery so consensus can replay the commits from the next index. // First commit in the replayed sequence will have index last_processed_commit_index + 1. // Set 0 to replay from the start (as generated commit sequence starts at index = 1). pub(crate) last_processed_commit_index: CommitIndex, diff --git a/crates/starfish/core/src/lib.rs b/crates/starfish/core/src/lib.rs index 341a8bb86b5..9e9a4aa553e 100644 --- a/crates/starfish/core/src/lib.rs +++ b/crates/starfish/core/src/lib.rs @@ -37,7 +37,7 @@ mod threshold_clock; mod transaction; #[cfg(msim)] pub mod transaction; -mod transaction_ref; +pub(crate) mod transaction_ref; mod transactions_synchronizer; mod universal_committer; @@ -74,3 +74,4 @@ pub use transaction::NoopTransactionVerifier; pub use transaction::{ BlockStatus, ClientError, TransactionClient, TransactionVerifier, ValidationError, }; +pub use transaction_ref::GenericTransactionRef; diff --git a/crates/starfish/core/src/network/tonic_network.rs b/crates/starfish/core/src/network/tonic_network.rs index a0fbfc104ba..4ff2b660fe0 100644 --- a/crates/starfish/core/src/network/tonic_network.rs +++ b/crates/starfish/core/src/network/tonic_network.rs @@ -535,7 +535,7 @@ impl ChannelPool { .keep_alive_timeout(config.keepalive_interval) .http2_keep_alive_interval(config.keepalive_interval) // tcp keepalive is probably unnecessary and is unsupported by msim. - .user_agent("mysticeti") + .user_agent("starfish") .unwrap() .tls_config(client_tls_config) .unwrap(); diff --git a/crates/starfish/core/src/universal_committer.rs b/crates/starfish/core/src/universal_committer.rs index 7a64bf61021..1290a2d5daf 100644 --- a/crates/starfish/core/src/universal_committer.rs +++ b/crates/starfish/core/src/universal_committer.rs @@ -55,7 +55,7 @@ impl UniversalCommitter { // let last_round = match self // .context // .protocol_config - // .mysticeti_num_leaders_per_round() + // .consensus_num_leaders_per_round() // { // Some(1) => { // Ensure that we don't commit any leaders from the same round as last_decided diff --git a/dev-tools/grafana-local/dashboards/consensus-overview.json b/dev-tools/grafana-local/dashboards/consensus-overview.json deleted file mode 100644 index 2ea6598064d..00000000000 --- a/dev-tools/grafana-local/dashboards/consensus-overview.json +++ /dev/null @@ -1,10300 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 100, - "panels": [], - "title": "Overview", - "type": "row" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 5 - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "green", - "value": 15 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 10, - "x": 0, - "y": 1 - }, - "id": 142, - "options": { - "basemap": { - "config": {}, - "name": "Layer 0", - "type": "default" - }, - "controls": { - "mouseWheelZoom": true, - "showAttribution": true, - "showDebug": false, - "showMeasure": false, - "showScale": false, - "showZoom": true - }, - "layers": [ - { - "config": { - "showLegend": true, - "style": { - "color": { - "field": "Value #B", - "fixed": "dark-green" - }, - "opacity": 0.4, - "rotation": { - "fixed": 0, - "max": 360, - "min": -360, - "mode": "mod" - }, - "size": { - "fixed": 5, - "max": 15, - "min": 2 - }, - "symbol": { - "fixed": "img/icons/marker/circle.svg", - "mode": "fixed" - }, - "symbolAlign": { - "horizontal": "center", - "vertical": "center" - }, - "textConfig": { - "fontSize": 12, - "offsetX": 0, - "offsetY": 0, - "textAlign": "center", - "textBaseline": "middle" - } - } - }, - "location": { - "mode": "auto" - }, - "name": "Layer 1", - "tooltip": true, - "type": "markers" - } - ], - "tooltip": { - "mode": "details" - }, - "view": { - "allLayers": true, - "id": "zero", - "lat": 0, - "lon": 0, - "zoom": 1 - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "avg(irate(consensus_committed_messages[2m])) by (authority)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "validator_location{}", - "format": "table", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Validator Location With BPS", - "transformations": [ - { - "filter": { - "id": "byRefId", - "options": "A" - }, - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "latitude", - "longitude", - "name" - ] - } - }, - "topic": "series" - }, - { - "filter": { - "id": "byRefId", - "options": "A" - }, - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "name": "authority" - } - }, - "topic": "series" - }, - { - "id": "joinByField", - "options": { - "byField": "authority", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Value #B": "BPS" - } - } - }, - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "isNull", - "options": {} - }, - "fieldName": "BPS" - } - ], - "match": "any", - "type": "exclude" - } - } - ], - "type": "geomap" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 10, - "y": 1 - }, - "id": 104, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "count(uptime{process=\"validator\"})", - "format": "table", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Active Validators", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 13, - "y": 1 - }, - "id": 54, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(current_epoch)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Current Epoch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 16, - "y": 1 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "value", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg(rate(total_transaction_certificates[2m]) > 1)", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "D" - } - ], - "title": "Current TPS", - "type": "stat" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "${__field.version_prefix}", - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 66.6 - }, - { - "color": "green", - "value": 83.3 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 19, - "y": 1 - }, - "id": 103, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(\n label_replace(\n current_voting_right{host!~\"access.*|backup.*|archive.*|indexer.*\"} / 10000\n * on(host) group_left(version)\n clamp_min(clamp_max(uptime{process=\"validator\"}, 1), 1),\n \"version_prefix\", \"$1\", \"version\", \"(\\\\d+\\\\.\\\\d+\\\\.\\\\d+-rc).*\"\n )\n) by (version_prefix)\n", - "instant": true, - "legendFormat": "{{version_prefix}}", - "range": false, - "refId": "A" - } - ], - "title": "Voting Power per Version", - "type": "gauge" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 10, - "y": 6 - }, - "id": 55, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(highest_synced_checkpoint)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Highest Checkpoint", - "type": "stat" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 14, - "y": 6 - }, - "id": 72, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "last_sent_checkpoint_signature", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Last Sent Checkpoint Signature", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "displayName": "${__field.version_prefix}", - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 11 - }, - { - "color": "#EAB839", - "value": 13 - }, - { - "color": "green", - "value": 14 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 19, - "y": 6 - }, - "id": 174, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "quantile(0.5, avg(irate(consensus_committed_messages[2m])) by (authority)) \n", - "instant": true, - "legendFormat": "{{version_prefix}}", - "range": false, - "refId": "A" - } - ], - "title": "BPS median for all authorities", - "type": "gauge" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of followers/subscriptions per authority ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 154, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (host) (consensus_subscribed_by == 1)\n", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Subscribed by", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of followers/subscriptions per authority ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 172, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (host) (consensus_subscribed_by == 1)\n", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Subscribed by", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "id": 157, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_num_of_bad_nodes", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Number of bad nodes", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Pair of non existing subscritions\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 156, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 47, - "showHeader": true - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(consensus_subscribed_to==0)", - "instant": false, - "legendFormat": "{{host}} - {{authority}}", - "range": true, - "refId": "A" - } - ], - "title": "Not subscribed to", - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 169, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg by(host) (irate(consensus_block_timestamp_drift_ms[5m]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Timestamp Drift", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "orange", - "value": 20 - }, - { - "color": "red", - "value": 40 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "id": 171, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mergeValues": false, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count by (host) (\n rate(consensus_block_timestamp_drift_ms[5m]) > 100\n) > 10", - "instant": false, - "legendFormat": "Number of authorities \"{{host}}\" is delaying blocks of ", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "count by (authority) (\n rate(consensus_block_timestamp_drift_ms[5m]) > 100\n) > 10", - "hide": false, - "instant": false, - "legendFormat": "Number of nodes delaying blocks of \"{{authority}}\"", - "range": true, - "refId": "B" - } - ], - "title": "Nodes with incorrect system clock", - "type": "state-timeline" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of subscriptions per authority (only authorities with problems) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3 - }, - { - "color": "orange", - "value": 13 - }, - { - "color": "red", - "value": 23 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 - }, - "id": 155, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mergeValues": false, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "scalar(quantile(0.5, sum(consensus_subscribed_to) by (host))) - sum(consensus_subscribed_to) by (host) > 1", - "instant": false, - "legendFormat": "{{host}} subscribed to #", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "scalar(quantile(0.5, sum(consensus_subscribed_by) by (host))) - sum(consensus_subscribed_by) by (host) > 1", - "hide": false, - "instant": false, - "legendFormat": "{{host}} subscribed by # ", - "range": true, - "refId": "B" - } - ], - "title": "Subscription issues timeline", - "type": "state-timeline" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Authorities committing less blocks than network median for all authorities. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 5 - }, - { - "color": "yellow", - "value": 10 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 - }, - "id": 173, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "mergeValues": false, - "rowHeight": 0.9, - "showValue": "auto", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": " avg(irate(consensus_committed_messages[2m])) by (authority)\n <\n scalar(\n quantile(0.5, avg(irate(consensus_committed_messages[2m])) by (authority)) / 2\n )\n", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Authorities committing relatively little blocks", - "type": "state-timeline" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 153, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "consensus_reputation_scores{host=\"IOTA 1\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{authority}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Reputation Score seen from IOTA 1", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "text", - "mode": "fixed" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": false, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text", - "wrapText": false - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "orange", - "index": 0 - }, - "to": 86400 - }, - "type": "range" - } - ] - }, - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Version" - }, - "properties": [ - { - "id": "custom.align", - "value": "right" - }, - { - "id": "custom.cellOptions", - "value": { - "applyToRow": true, - "mode": "gradient", - "type": "color-background" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "pattern": ".*-debug$", - "result": { - "color": "red", - "index": 0 - } - }, - "type": "regex" - }, - { - "options": { - "pattern": ".*-rc-.*", - "result": { - "color": "transparent", - "index": 1 - } - }, - "type": "regex" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Voting Right" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Host" - }, - "properties": [ - { - "id": "custom.width", - "value": 200 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Missing Blocks" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "text", - "index": 0 - }, - "to": 10 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "color": "orange", - "index": 1 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 2 - }, - "to": 10000000000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.width", - "value": 122 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Missing Ancestors" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "text", - "index": 0 - }, - "to": 10 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "color": "orange", - "index": 1 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 2 - }, - "to": 100000000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.width", - "value": 145 - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 52 - }, - "id": 49, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 2, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "Synced Checkpoint" - } - ] - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "current_voting_right / 10000", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "current_voting_right", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "uptime{process=\"validator\"}", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "uptime" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(consensus_committed_messages[2m])", - "format": "table", - "hide": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "rate_consensus_committed_messages" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "last_sent_checkpoint_signature", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "last_sent_checkpoint_signature" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "consensus_block_manager_missing_blocks", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "consensus_block_manager_missing_blocks" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "consensus_block_manager_missing_ancestors", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "consensus_block_manager_missing_ancestors" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "highest_synced_checkpoint", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "highest_synced_checkpoint" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "highest_verified_checkpoint", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "highest_verified_checkpoint" - } - ], - "title": "Validators Info", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "byVariable": false, - "include": { - "names": [ - "version", - "Value #uptime", - "Value #last_sent_checkpoint_signature", - "Value #consensus_block_manager_missing_blocks", - "Value #consensus_block_manager_missing_ancestors", - "Value #highest_synced_checkpoint", - "Value #highest_verified_checkpoint", - "Value #current_voting_right", - "host" - ] - } - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Value #A": true - }, - "includeByName": {}, - "indexByName": { - "Value #consensus_block_manager_missing_ancestors": 2, - "Value #consensus_block_manager_missing_blocks": 1, - "Value #current_voting_right": 6, - "Value #highest_synced_checkpoint": 4, - "Value #highest_verified_checkpoint": 5, - "Value #last_sent_checkpoint_signature": 3, - "Value #uptime": 8, - "host": 0, - "version": 7 - }, - "renameByName": { - "Value #A": "Participate", - "Value #B": "Reputation Scores", - "Value #C": "Voting Right", - "Value #D": "Uptime", - "Value #F": "Last Sent Checkpoint", - "Value #G": "Missing Block", - "Value #H": "Missing Ancestors", - "Value #consensus_block_manager_missing_ancestors": "Missing Ancestors", - "Value #consensus_block_manager_missing_blocks": "Missing Blocks", - "Value #current_voting_right": "Voting Right", - "Value #highest_synced_checkpoint": "Synced Checkpoint", - "Value #highest_verified_checkpoint": "Verified Checkpoint", - "Value #last_sent_checkpoint_signature": "Sent Checkpoint Signature", - "Value #uptime": "Uptime", - "host": "Host", - "instance": "", - "version": "Version" - } - } - }, - { - "id": "filterByValue", - "options": { - "filters": [ - { - "config": { - "id": "regex", - "options": { - "value": "wasp*|access*|backup*|archive*|indexer*" - } - }, - "fieldName": "Host" - } - ], - "match": "any", - "type": "exclude" - } - } - ], - "type": "table" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 60 - }, - "id": 67, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "This metric calculates, for each authority, the fraction of 2-minute intervals over the past 24 hours in which the authority committed at least one consensus message.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "green", - "value": 90 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 61 - }, - "id": 96, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": true, - "expr": "sum(\n count_over_time((sum(irate(consensus_committed_messages[2m])) by (authority) > 0 )[24h:2m])\n) by (authority)\n/\n(count_over_time(sum(irate(consensus_committed_messages[2m])) by (authority)[24h:2m]))", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Consensus Reliability (24 hour)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of total number of committed consensus messages", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 61 - }, - "id": 68, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(consensus_committed_messages[2m])) by (authority)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Committed Messages Rate", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 70, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "dark-orange", - "value": 5 - }, - { - "color": "#EAB839", - "value": 10 - }, - { - "color": "green", - "value": 15 - }, - { - "color": "#6ED0E0", - "value": 25 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 19, - "w": 24, - "x": 0, - "y": 69 - }, - "id": 74, - "options": { - "alignValue": "left", - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "mergeValues": false, - "rowHeight": 0.9, - "showValue": "never", - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg(irate(consensus_committed_messages[2m])) by (authority)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "type": "state-timeline" - } - ], - "title": "Committee", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 107, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 125, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope) (\n rate(consensus_scope_processing_time_sum{scope=~\"Core::.*\"}[5m])\n)\n/\nsum by (scope) (\n rate(consensus_scope_processing_time_count{scope=~\"Core::.*\"}[5m])\n)", - "instant": false, - "legendFormat": "{{scope}}", - "range": true, - "refId": "A" - } - ], - "title": "Average CORE Task (scope) Execution Time avaraged over all hosts", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": " Interval between two successive block proposals. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 114, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(consensus_block_proposal_interval_sum[5m])/rate(consensus_block_proposal_interval_count[5m])", - "fullMetaSearch": false, - "includeNullMetadata": false, - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Block Proposal Interval", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Measures time between block timestamp and consensus commit (latency).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 108, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(consensus_block_commit_latency_sum[5m]) / rate(consensus_block_commit_latency_count[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Block commit latency", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Total number of unforced blocks proposed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 112, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_proposed_blocks{force=\"false\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Proposed Blocks (unforced)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Counts blocks accepted into the DAG, labeled by origin (own/others).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 75 - }, - "id": 111, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_accepted_blocks{source=\"others\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{host}}/{{source}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Accepted blocks (others)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Total number of forced blocks created (by host) forcefully via a leader timeout", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 116, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_proposed_blocks{force=\"true\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "{{host}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Proposed Blocks (forced)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Counts blocks accepted into the DAG, labeled by origin (own/others).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 135, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_accepted_blocks{source=\"own\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{host}}/{{source}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Accepted blocks (own)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": " \n\t\n\nNumber of times a leader wait was counted during proposal. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 91 - }, - "id": 113, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_block_proposal_leader_wait_count{host=\"IOTA Foundation 1\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Block Proposal Leader Wait Count", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of forced leader timeouts", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 91 - }, - "id": 109, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_leader_timeout_total{timeout_type=\"true\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Forced Leader Timeouts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Counts how often a new block was created due to a leader timeout", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 99 - }, - "id": 110, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(consensus_leader_timeout_total[5m])", - "refId": "A" - } - ], - "title": "Leader Timeout Total", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Measures how many subdags are included in each consensus commit cycle.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 99 - }, - "id": 115, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "rate(consensus_sub_dags_per_commit_count_sum[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Subdags per commit count", - "type": "timeseries" - } - ], - "title": "Core", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 62 - }, - "id": 95, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of block fetched initiated by live synchronizer on each host, aggregated across all authorities. This helps monitor the overall load each node contributes to the synchronization process by tracking how many blocks it pulls from peers per second. Useful for detecting imbalances or overactive fetchers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Klever" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 63 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (host) (\n rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"live\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Live Fetched Blocks Rate per Host", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of live block fetch requests initiated by each host, aggregated across all authorities. This helps monitor the overall load each node contributes to the synchronization process by tracking how many blocks it pulls from peers per second. Useful for detecting imbalances or overactive requester.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Klever" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 63 - }, - "id": 160, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (host) (\n rate(consensus_synchronizer_requested_blocks_by_authority{type=\"live\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Live Request Rate per Host", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of block fetched authored by each authority, aggregated across all hosts. This helps monitor the overall load each node contributes to the synchronization process by tracking how often its blocks needs to be fetched. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 - }, - "id": 140, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (authority) (\n rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"live\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Live Fetched Blocks Rate per Authority", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of live block requests authored from each authority, aggregated across all hosts. This helps monitor the overall load each node contributes to the synchronization process by tracking how often its blocks is requested. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 71 - }, - "id": 161, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (authority) (\n rate(consensus_synchronizer_requested_blocks_by_authority{type=\"live\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Live Request Rate per Authority", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of currently active periodic fetch scheduler tasks.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": true, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 10 - }, - { - "color": "red", - "value": 100 - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 79 - }, - "id": 118, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_fetch_blocks_scheduler_inflight", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Fetch Scheduler Inflight", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of periodic sync request per block author", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 34, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 79 - }, - "id": 106, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"periodic\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{authority}} blocks requested by {{host}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Periodic Syncing Fetches Overview", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of periodic block fetches initiated by each host, aggregated across all authorities. This helps monitor the overall load each node contributes to the synchronization process by tracking how many blocks it pulls from peers per second. Useful for detecting imbalances or overactive fetchers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 87 - }, - "id": 139, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (host) (\n rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"periodic\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Periodic Fetched Blocks Rate per Host", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of periodic block fetches initiated by each host, aggregated across all authorities. This helps monitor the overall load each node contributes to the synchronization process by tracking how many blocks it pulls from peers per second. Useful for detecting imbalances or overactive fetchers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 87 - }, - "id": 162, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(host) (rate(consensus_synchronizer_requested_blocks_by_authority{type!=\"live\"}[5m]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Periodic Requests Rate per Host", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of periodic block fetches authored from each authority, aggregated across all hosts. This helps monitor the overall load each node contributes to the synchronization process by tracking how often its blocks needs to be fetched. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 95 - }, - "id": 163, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by (authority) (\n rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"periodic\"}[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Periodic Fetched Blocks Rate per Authority", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the rate of periodic block requests authored from each authority, aggregated across all hosts. This helps monitor the overall load each node contributes to the synchronization process by tracking how often its blocks needs to be fetched. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 95 - }, - "id": 141, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(authority) (rate(consensus_synchronizer_fetched_blocks_by_authority{type!=\"live\"}[5m]))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Periodic Requests Rate per Authority", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 103 - }, - "id": 78, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_block_manager_missing_blocks", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Block manager missing blocks", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 103 - }, - "id": 76, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_block_manager_missing_ancestors", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Block manager missing ancestors", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 111 - }, - "id": 92, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_block_manager_suspended_blocks", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Block manager suspended blocks", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "This metric tracks the average number of referencing blocks per missing block. It reflects how many other blocks are waiting on missing data to be processed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 111 - }, - "id": 136, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_block_manager_missing_ancestors\n/\nconsensus_block_manager_missing_blocks", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Ancestor-to-Missing Block Ratio", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of live sync request per block author", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 48, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 119 - }, - "id": 105, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(consensus_synchronizer_fetched_blocks_by_authority{type=\"live\"}[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{authority}} blocks requested by {{host}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Live Syncing", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the top 5 authorities whose blocks most frequently lack required ancestors, based on the 5-minute rate. High rates may indicate improper block propagation or network delays.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byValue", - "options": { - "op": "gte", - "reducer": "allIsZero", - "value": 0 - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": true, - "tooltip": true, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 119 - }, - "id": 137, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "topk(5, rate(consensus_block_manager_missing_ancestors_by_authority[5m]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Top 5 Authorities Causing Missing Ancestors", - "type": "timeseries" - } - ], - "title": "Synchronizer", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 63 - }, - "id": 166, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The number of blocks that the host garbage collected.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 164, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(host) (consensus_block_manager_gced_blocks)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "GCed Blocks (by host)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The number of times a block issued by the authority was garbage collected.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 64 - }, - "id": 167, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(authority) (consensus_block_manager_gced_blocks)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "GCed Blocks (by authority)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The number of blocks unsuspended inside the host. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 72 - }, - "id": 165, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(host) (consensus_block_manager_gc_unsuspended_blocks)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "GCed Unsuspended Blocs (by host)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The number of blocks unsuspended authored by authority summed over all hosts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 72 - }, - "id": 168, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(authority) (consensus_block_manager_gc_unsuspended_blocks)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "GCed Unsuspended Blocs (by authority)", - "type": "timeseries" - } - ], - "title": "Garbage Collection", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 127, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Current number of ongoing fetch tasks. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 65 - }, - "id": 128, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_commit_sync_inflight_fetches", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Inflight fetches", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of number of commits fetched from other authorities.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 65 - }, - "id": 129, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(consensus_commit_sync_fetched_commits[5m]) ", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Fetched commits", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of counts instances where commit sync couldn’t proceed due to a gap in block sequence", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 73 - }, - "id": 130, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(consensus_commit_sync_gap_on_processing[5m]) ", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Commit Sync Gap Retries ", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Average fetch commits from a peer authority. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 73 - }, - "id": 133, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(consensus_commit_sync_fetch_once_latency_sum[5m]) / rate(consensus_commit_sync_fetch_once_latency_count[5m])", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Commit Sync Fetch Once Latency", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Number of pending commit ranges queued for fetch. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 81 - }, - "id": 131, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "consensus_commit_sync_pending_fetches", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Pending fetches", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Rate of fetch failures per authority and error type.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 49, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 81 - }, - "id": 134, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(consensus_commit_sync_fetch_once_errors[5m])", - "instant": false, - "legendFormat": "{{authority}} / {{error}}", - "range": true, - "refId": "A" - } - ], - "title": " Commit Sync Fetch Errors", - "type": "timeseries" - } - ], - "title": "Commit Syncer", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 126, - "panels": [ - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "This plot shows the average time (in milliseconds per second) that all validators collectively spend per consensus processing scope", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 158, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg by (scope) (\n rate(consensus_scope_processing_time_sum[5m])\n)", - "instant": false, - "legendFormat": "{{scope}}", - "range": true, - "refId": "A" - } - ], - "title": "Core Task Execution Time per Scope (m)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "This panel shows the average total time (in seconds per second) spent by each validator (grouped by host) on executing consensus core tasks", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "left", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 159, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "avg by (host) (\n rate(consensus_scope_processing_time_sum[5m])\n)", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Core Task Execution Time by Host (ms)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 74 - }, - "id": 120, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope, host) (\n rate(consensus_scope_processing_time_sum[5m])\n /\n rate(consensus_scope_processing_time_count[5m])\n) ", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Average Task (scope) Execution Time by Host and Task (per call)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 82 - }, - "id": 122, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope, host) (\n rate(consensus_scope_processing_time_sum{scope=~\"Core::.*\"}[5m])\n /\n rate(consensus_scope_processing_time_count{scope=~\"Core::.*\"}[5m])\n) ", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Average CORE Task (scope) Execution Time by Host and Task (per call)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 82 - }, - "id": 121, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope, host) (\n rate(consensus_scope_processing_time_sum{scope=~\"DagState::.*\"}[5m])\n /\n rate(consensus_scope_processing_time_count{scope=~\"DagState::.*\"}[5m])\n) * 1000", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Average DAG_STATE Task (scope) Execution Time by Host and Task (per call)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 90 - }, - "id": 124, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope, host) (\n rate(consensus_scope_processing_time_sum{scope=~\"LeaderSchedule::.*\"}[5m])\n /\n rate(consensus_scope_processing_time_count{scope=~\"LeaderSchedule::.*\"}[5m])\n)", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Average Task Leader_Schedule (scope) Execution Time by Host and Task (per call)", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Displays the average execution time (in seconds) per task call, computed as the rate of total processing time divided by the rate of task invocations over a 5-minute window. Data is grouped by task label and hostname using Prometheus histogram metrics (scope_processing_time_sum / scope_processing_time_count).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 31, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 90 - }, - "id": 123, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (scope, host) (\n rate(consensus_scope_processing_time_sum{scope=~\"CommitObserver::.*\"}[5m])\n /\n rate(consensus_scope_processing_time_count{scope=~\"CommitObserver::.*\"}[5m])\n)", - "instant": false, - "legendFormat": "{{scope}} / {{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Average Task COMMIT_Observer (scope) Execution Time by Host and Task", - "type": "timeseries" - } - ], - "repeat": "Filters", - "repeatDirection": "h", - "title": "Processing time", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 66 - }, - "id": 7, - "panels": [ - { - "datasource": { - "default": false, - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Warning / min", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 67 - }, - "id": 51, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"iota\"} |= \"WARN\" [1m]))", - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"indexer-reader\"} |= \"WARN\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "B" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"indexer-writer\"} |= \"WARN\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "C" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"faucet\"} |= \"WARN\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "D" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "code", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"graphql\"} |= \"WARN\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "E" - } - ], - "title": "Services Warning From Log", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Warning / min", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 67 - }, - "id": 61, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.2.1", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"iota\"} |= `ERROR` != `UserInput` [1m]))", - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"indexer-reader\"} |= `ERROR` != `UserInput` [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "B" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"indexer-writer\"} |= \"ERROR\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "C" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"faucet\"} |= \"ERROR\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "D" - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "editorMode": "builder", - "expr": "sum without(filename, detected_level) (count_over_time({container_name=\"graphql\"} |= \"ERROR\" [1m]))", - "hide": false, - "legendFormat": "{{ host }} - {{ container_name }}", - "queryType": "range", - "refId": "E" - } - ], - "title": "Services Error From Log", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 67 - }, - "id": 64, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(irate(errors_by_route[2m])) by (instance)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Full Node Route Error", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 75 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(total_transaction_certificates[2m])", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "TPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 75 - }, - "id": 28, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "iota_network_peers", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Peering", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 75 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "current_epoch", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Current Epoch", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 83 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "clamp_min(highest_synced_checkpoint - highest_known_checkpoint, 0)", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Checkpoint Sync Lag", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 83 - }, - "id": 60, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "highest_known_checkpoint", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Highest Known Checkpoint", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 83 - }, - "id": 58, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(quorum_driver_total_err_responses[2m])", - "instant": false, - "legendFormat": "{{host}} - {{error}}", - "range": true, - "refId": "A" - } - ], - "title": "Quorum Driver Error", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 91 - }, - "id": 29, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(req_latency_by_route_sum[2m])", - "hide": false, - "legendFormat": "{{host}} - {{ route }}", - "range": true, - "refId": "C" - } - ], - "title": "IOTA Request Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 91 - }, - "id": 31, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(indexer_req_latency_by_route_sum[2m])", - "hide": false, - "legendFormat": "{{host}} - {{job}} - {{route}}", - "range": true, - "refId": "B" - } - ], - "title": "Indexer Request Latency", - "type": "timeseries" - } - ], - "title": "IOTA Node", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 67 - }, - "id": 23, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 68 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(irate(traefik_service_requests_total{instance=~\"lb.*\"}[2m])) by (service)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Requests Per Seconds (Avg 2m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 68 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(irate(traefik_service_request_duration_seconds_sum{instance=~\"lb.*\"}[2m])) by (service)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Response Duration (Avg. 2m)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 68 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "traefik_service_server_up", - "legendFormat": "{{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Services Pool", - "type": "timeseries" - } - ], - "title": "Load Balancer", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 - }, - "id": 15, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 69 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum((sum(irate(node_cpu_seconds_total {mode!=\"idle\"} [1m])) without (cpu)) / count(node_cpu_seconds_total) without (cpu)) without (mode) * 100", - "hide": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "B" - } - ], - "title": "Server CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 69 - }, - "id": 13, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(container_cpu_usage_seconds_total{name=\"iota\"}[5m]) * 100", - "hide": false, - "interval": "", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "IOTA CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 69 - }, - "id": 16, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(container_cpu_usage_seconds_total{name=~\"indexer.*\"}[5m]) * 100", - "hide": false, - "interval": "", - "legendFormat": "{{host}} - {{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Indexer CPU Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 77 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes)))", - "hide": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "B" - } - ], - "title": "Server Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 77 - }, - "id": 14, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "container_memory_rss{name=\"iota\"}", - "hide": false, - "interval": "", - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "IOTA Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 77 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.5.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "container_memory_rss{name=~\"indexer.*\"}", - "hide": false, - "interval": "", - "legendFormat": "{{host}} - {{name}}", - "range": true, - "refId": "A" - } - ], - "title": "Indexer Memory Usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 85 - }, - "id": 42, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by (instance) (\n 100 - ((node_filesystem_avail_bytes{device!~'rootfs'} * 100) / node_filesystem_size_bytes{device!~'rootfs'})\n)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Server Disk Usage", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "iops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 85 - }, - "id": 59, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(irate(node_disk_reads_completed_total[1m]) + irate(node_disk_writes_completed_total[1m])) without (device)", - "instant": false, - "legendFormat": "{{host}}", - "range": true, - "refId": "A" - } - ], - "title": "Total IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 85 - }, - "id": 50, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "scrape_duration_seconds{job!=\"cadvisor\"}", - "legendFormat": "{{host}}: {{job}}", - "range": true, - "refId": "A" - } - ], - "title": "Scraping Delay", - "type": "timeseries" - } - ], - "title": "Performance", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 69 - }, - "id": 44, - "panels": [], - "title": "Inbound / Outbound Network Metrics", - "type": "row" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Summing up bandwidth used by all requests and responses - inbound and outbound. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 177, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum by(host) (irate(consensus_outbound_response_size_sum[2m])+ irate(consensus_outbound_request_size_sum[2m])+irate(consensus_inbound_response_size_sum[2m])+ irate(consensus_inbound_request_size_sum[2m]))", - "legendFormat": "{{host}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus Bandwidth Usage", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 81 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(consensus_inbound_request_latency_count[2m])", - "legendFormat": "{{host}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Inbound Requests Rate", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "p75 - FetchBlocks", - "p90 - FetchBlocks", - "avg - FetchBlocks", - "p50 - FetchBlocks" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 81 - }, - "id": 178, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(consensus_inbound_request_latency_sum[2m]) / irate(consensus_inbound_request_latency_count[2m])", - "hide": false, - "legendFormat": "avg - {{route}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(route, le) (rate(consensus_inbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p50 - {{route}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(route, le) (rate(consensus_inbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p75 - {{route}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(route, le) (rate(consensus_inbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p90 - {{route}}", - "range": true, - "refId": "D" - } - ], - "title": "Inbound Request Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 81 - }, - "id": 175, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(consensus_inbound_response_size_sum[2m]) + irate(consensus_inbound_request_size_sum[2m])", - "legendFormat": "{{host}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Inbound Request+Response Size", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 89 - }, - "id": 45, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(consensus_outbound_request_size_count[2m])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Outbound Requests", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "p75 - FetchBlocks", - "p90 - FetchBlocks", - "avg - FetchBlocks", - "p50 - FetchBlocks" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 89 - }, - "id": 46, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "avg by (route) (rate(consensus_outbound_request_latency_sum[2m]) / rate(consensus_outbound_request_latency_count[2m]))", - "hide": false, - "legendFormat": "avg - {{route}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum by(route, le) (rate(consensus_outbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p50 - {{route}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(route, le) (rate(consensus_outbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p75 - {{route}}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(route, le) (rate(consensus_outbound_request_latency_bucket[5m])))", - "hide": false, - "instant": false, - "legendFormat": "p90 - {{route}}", - "range": true, - "refId": "D" - } - ], - "title": "Outbound Request Latency", - "type": "timeseries" - }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 89 - }, - "id": 176, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "irate(consensus_outbound_response_size_sum[2m]) + irate(consensus_outbound_request_size_sum[2m])", - "legendFormat": "{{host}} {{route}}", - "range": true, - "refId": "A" - } - ], - "title": "Outbound Request+Response Size", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "filters": [ - { - "key": "host", - "operator": "=", - "value": "Klever" - } - ], - "hide": 0, - "name": "Filters", - "skipUrlSync": false, - "type": "adhoc" - } - ] - }, - "time": { - "from": "2025-11-18T06:51:21.412Z", - "to": "2025-11-18T07:19:46.675Z" - }, - "timepicker": {}, - "timezone": "Europe/Berlin", - "title": "IOTA Consensus Overview (provisioned)", - "uid": "eeigyzrajocg0fconsensus3", - "version": 13, - "weekStart": "" -} \ No newline at end of file diff --git a/dev-tools/iota-private-network/README.md b/dev-tools/iota-private-network/README.md index 712895bfb7d..2f8740a4335 100644 --- a/dev-tools/iota-private-network/README.md +++ b/dev-tools/iota-private-network/README.md @@ -91,17 +91,6 @@ To bring up 10 validators and faucet: > - Update IP address assignments in the docker-compose.yaml for the additional validators > - **(Optional)** Adjust the stake distribution in the chosen `genesis-template-.yaml` if you want different validator stakes. -### Optional: Selecting a Consensus Protocol - -You can run the network with an optional consensus protocol flag. There are two options `starfish` and `mysticeti`. -If the flag is not provided, the default protocol is **Starfish**. - -For example, to start with **Mysticeti** consensus protocol (if you prefer the previous consensus protocol): - -```bash -./run.sh -n 10 -p mysticeti -``` - ### Ports - fullnode-1: diff --git a/dev-tools/iota-private-network/docker-compose.yaml b/dev-tools/iota-private-network/docker-compose.yaml index 284392f0593..cc7ae84383d 100644 --- a/dev-tools/iota-private-network/docker-compose.yaml +++ b/dev-tools/iota-private-network/docker-compose.yaml @@ -6,7 +6,6 @@ x-common-validator: &common-validator - RPC_WORKER_THREAD=12 - NEW_CHECKPOINT_WARNING_TIMEOUT_MS=30000 - NEW_CHECKPOINT_PANIC_TIMEOUT_MS=60000 - - CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL:-} command: &common-command [ "/usr/local/bin/iota-node", "--config-path", diff --git a/dev-tools/iota-private-network/experiments/README.md b/dev-tools/iota-private-network/experiments/README.md index 9b11a57073e..70390244ed5 100644 --- a/dev-tools/iota-private-network/experiments/README.md +++ b/dev-tools/iota-private-network/experiments/README.md @@ -48,7 +48,6 @@ docker pull nicolaka/netshoot Supports the following flags: - `-n `: number of validators (default: `4`; any number between `4` and `30` is supported) -- `-p `: consensus protocol (default: `starfish`; another option: `mysticeti`) - `-b `: rebuild Docker images before running (default: `true`) - `-g `: enable geodistributed large network latencies (default: `false`) - `-s `: seed for pseudorandom disruptions (default: `42`) @@ -70,10 +69,10 @@ The script should be run from inside the `iota/dev-tools/iota-private-network/ex # Run default 4-validator Starfish network with small latencies without any additional disruptions ./run-all-benchmark.sh -# Run 10-validator Mysticeti network with large geodistributed latencies for one hour without rebuilding images -./run-all-benchmark.sh -n 10 -p mysticeti -g true -b false +# Run 10-validator network with large geodistributed latencies for one hour without rebuilding images +./run-all-benchmark.sh -n 10 -g true -b false -# Run 30-validator Starfish network with geodistributed latencies, 10% blocked connections, 5% chances for packet loss, 10% for restarts and running for 2 hours +# Run 30-validator network with geodistributed latencies, 10% blocked connections, 5% chances for packet loss, 10% for restarts and running for 2 hours ./run-all-benchmark.sh -n 30 -g true -x 10 -l 5 -r 10 -t 7200 ``` --- @@ -89,7 +88,7 @@ It supports two types of spammer tools, by default the stress test from the iota ./run-all-benchmark.sh -n 4 -S true -T 500 ``` -This will load the default spammer with a TPS of 500 using Starfish (default protocol). +This will load the default spammer with a TPS of 500. ### Required Setup for optional Spammer @@ -108,24 +107,18 @@ Place it at the following relative path from `run-all-benchmark.sh`, or update t The optional spammer allows a special transaction type, called `sizable`, and can be used as follows: ```bash -./run-all-benchmark.sh -n 4 -p mysticeti -S true -T 100 -Z 10KiB +./run-all-benchmark.sh -n 4 -S true -T 100 -Z 10KiB ``` This will launch the spammer from the external repository with the configured transaction rate, TPS=100, and size, 10KiB. -To use Mysticeti instead of Starfish, add `-p mysticeti`: - -```bash -./run-all-benchmark.sh -n 4 -p mysticeti -S true -T 100 -Z 10KiB -``` - ## Main Fuzz Script: `run-all-fuzz.sh` `run-all-fuzz.sh` automates the full workflow: 1. Optionally rebuilds the `iota-node`, `iota-tools`, and `iota-indexer` Docker images. 2. Bootstraps the validator network. -3. Runs the private network with the chosen consensus protocol. +3. Runs the private network. 4. Starts Grafana (available at `http://localhost:3000/dashboards`). 5. Launches `network-fuzz.sh` to apply network latencies and controlled disruptions: - artificial RTTs (topology-dependent), @@ -155,9 +148,6 @@ Supported flags: - `-n `\ Number of validators (default: `4`; supports `4`–`19`). -- `-p `\ - Consensus protocol (default: `starfish`; other option: `mysticeti`). - - `-b `\ Rebuild Docker images before running (default: `true`). @@ -363,7 +353,6 @@ To use the `iota-spammer`: ``` ./run-all-fuzz.sh \ -n 4 \ - -p mysticeti \ -S true \ -C iota-spammer \ -T 100 \ @@ -403,14 +392,13 @@ On exit, `run-all-fuzz.sh`: --- -## Batch Comparison Script: `dual-run.sh` +## Batch Script: `dual-run.sh` -`dual-run.sh` automates **paired runs** of Mysticeti and Starfish under identical network conditions for direct comparison. +`dual-run.sh` runs a sweep of fuzz experiments over a list of disruption parameters. It: - Builds all Docker images once (`iota-node`, `iota-tools`, `iota-indexer`) -- Uses the same fuzz parameters for both protocols - Runs several steps defined by parameter lists: ```bash @@ -419,12 +407,7 @@ It: L_LIST=(10 15 10 10) # % packet loss ``` -Each step executes: - -1. Mysticeti run with `(r, x, l)` -2. short pause -3. Starfish run with identical `(r, x, l)` -4. pause before the next step +Each step executes one run with `(r, x, l)`, then pauses before the next step. Global defaults (inside the script): @@ -444,6 +427,6 @@ Run from `experiments/`: ./dual-run.sh ``` -This produces a sequence of alternating Mysticeti/Starfish experiments using the same disruption settings. +This produces a sequence of fuzz experiments covering the configured disruption settings. --- diff --git a/dev-tools/iota-private-network/experiments/dual-run.sh b/dev-tools/iota-private-network/experiments/dual-run.sh index 5b9c49f07f0..a8237d48e94 100755 --- a/dev-tools/iota-private-network/experiments/dual-run.sh +++ b/dev-tools/iota-private-network/experiments/dual-run.sh @@ -3,7 +3,7 @@ # Copyright (c) 2025 IOTA Stiftung # SPDX-License-Identifier: Apache-2.0 -# dual-run.sh — run sets of experiments (Mysticeti + Starfish per step) +# dual-run.sh — run sets of fuzz experiments across several parameter combinations set -euo pipefail IFS=$'\n\t' @@ -54,7 +54,6 @@ DURATION=3600 # seconds SPAMMER=true SPAMMER_TPS=100 SPAMMER_TYPE="stress" -PAUSE_BETWEEN_PROTOCOLS=60 # seconds PAUSE_BETWEEN_STEPS=180 # seconds # parameter sequences (same length) @@ -64,16 +63,15 @@ L_LIST=(10 15 10 10) # percent nodes with loss run_experiment() { - local PROTO="$1" R="$2" X="$3" L="$4" + local R="$1" X="$2" L="$3" local ts; ts=$(date +%Y%m%d-%H%M%S) - echo "=== ${ts}: starting ${PROTO} (r=${R} x=${X} l=${L}) ===" + echo "=== ${ts}: starting (r=${R} x=${X} l=${L}) ===" sudo -E \ FUZZ_ROUND_SPAN="${FUZZ_ROUND_SPAN}" \ HEAL_EVERY_ROUND="${HEAL_EVERY_ROUND}" \ HEAL_NUM_ROUNDS="${HEAL_NUM_ROUNDS}" \ ./run-all-fuzz.sh \ -n "${NUM_VALIDATORS}" \ - -p "${PROTO}" \ -t "${TOPOLOGY}" \ -b false \ -r "${R}" \ @@ -83,7 +81,7 @@ run_experiment() { -S "${SPAMMER}" \ -T "${SPAMMER_TPS}" \ -C "${SPAMMER_TYPE}" - echo "=== finished ${PROTO} (r=${R} x=${X} l=${L}) ===" + echo "=== finished (r=${R} x=${X} l=${L}) ===" } for i in "${!R_LIST[@]}"; do @@ -92,13 +90,9 @@ for i in "${!R_LIST[@]}"; do L=${L_LIST[$i]} echo echo ">>> Step $((i+1)): r=${R}, x=${X}, l=${L} — $(date +%Y%m%d-%H%M%S)" - run_experiment "mysticeti" "$R" "$X" "$L" - echo "Sleeping ${PAUSE_BETWEEN_PROTOCOLS}s before starfish..." - sleep "${PAUSE_BETWEEN_PROTOCOLS}" - - run_experiment "starfish" "$R" "$X" "$L" + run_experiment "$R" "$X" "$L" echo "Step $((i+1)) complete. Sleeping ${PAUSE_BETWEEN_STEPS}s before next step..." sleep "${PAUSE_BETWEEN_STEPS}" done -echo "All progressive fuzz runs completed." \ No newline at end of file +echo "All progressive fuzz runs completed." diff --git a/dev-tools/iota-private-network/experiments/run-all-benchmark.sh b/dev-tools/iota-private-network/experiments/run-all-benchmark.sh index c630036646e..c780f72da9b 100755 --- a/dev-tools/iota-private-network/experiments/run-all-benchmark.sh +++ b/dev-tools/iota-private-network/experiments/run-all-benchmark.sh @@ -12,7 +12,6 @@ CLEANING=false # =================== CONSTANTS =================== DEFAULT_NUM_VALIDATORS=4 -DEFAULT_PROTOCOL="starfish" DEFAULT_BUILD=true DEFAULT_GEODISTRIBUTED=false DEFAULT_SEED=42 @@ -165,7 +164,7 @@ log() { # --- Usage --- usage() { - echo "Usage: $0 [-n num_validators(4..30)] [-p protocol(mysticeti|starfish)] [-b build_images(true|false)]" + echo "Usage: $0 [-n num_validators(4..30)] [-b build_images(true|false)]" echo " [-g geodistributed(true|false)] [-s seed(number)] [-x percent_block_connection(0..100)] [-l percent_loss_packets(0..100)]" echo " [-t run_duration_seconds] [-d restart_duration_seconds] [-r percent_restart(0..100)]" echo " [-w restart_timeout_seconds] [-M restart_mode(preserve-consensus|full-reset|simple-restart)]" @@ -175,7 +174,6 @@ usage() { # --- Default values --- NUM_VALIDATORS=$DEFAULT_NUM_VALIDATORS -PROTOCOL=$DEFAULT_PROTOCOL BUILD=$DEFAULT_BUILD GEODISTRIBUTED=$DEFAULT_GEODISTRIBUTED SEED=$DEFAULT_SEED @@ -194,10 +192,9 @@ SPAMMER_SIZE_PER_TX=$DEFAULT_SPAMMER_SIZE SPAMMER_TYPE=$DEFAULT_SPAMMER_TYPE # --- Parse command-line arguments --- -while getopts ":n:p:b:g:s:x:l:t:d:r:w:M:E:mS:T:Z:C:h" opt; do +while getopts ":n:b:g:s:x:l:t:d:r:w:M:E:mS:T:Z:C:h" opt; do case "$opt" in n) NUM_VALIDATORS="$OPTARG" ;; - p) PROTOCOL="$OPTARG" ;; b) BUILD="$OPTARG" ;; g) GEODISTRIBUTED="$OPTARG" ;; s) SEED="$OPTARG" ;; @@ -228,7 +225,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # --- Summary --- log "=== SUMMARY ===" log "Number of validators : $NUM_VALIDATORS" -log "Consensus protocol : $PROTOCOL" log "Rebuild images : $BUILD" log "Geodistributed network : $GEODISTRIBUTED" log "Seed : $SEED" @@ -290,7 +286,7 @@ fi (cd .. && ./bootstrap.sh -n "$NUM_VALIDATORS" -e "$EPOCH_DURATION_MS") # --- 3) Bring up docker network --- -(cd .. && ./run.sh -n "$NUM_VALIDATORS" -p "$PROTOCOL") +(cd .. && ./run.sh -n "$NUM_VALIDATORS") log "Sleep 5s to boot validators..." diff --git a/dev-tools/iota-private-network/experiments/run-all-fuzz.sh b/dev-tools/iota-private-network/experiments/run-all-fuzz.sh index f7a3c2d1333..01501205d47 100755 --- a/dev-tools/iota-private-network/experiments/run-all-fuzz.sh +++ b/dev-tools/iota-private-network/experiments/run-all-fuzz.sh @@ -56,7 +56,6 @@ CLEANING=false # =================== CONSTANTS =================== DEFAULT_NUM_VALIDATORS=4 -DEFAULT_PROTOCOL="mysticeti" DEFAULT_BUILD=true DEFAULT_TOPOLOGY="false" # maps to fuzz topology: true|false|ring|star|non-triangle|random|geo-high|geo-low DEFAULT_SEED=42 @@ -220,7 +219,7 @@ log() { # --- Usage --- usage() { - echo "Usage: $0 [-n num_validators(4..30)] [-p protocol(mysticeti|starfish)] [-b build_images(true|false)]" + echo "Usage: $0 [-n num_validators(4..30)] [-b build_images(true|false)]" echo " [-t topology(true|false|ring|star|non-triangle|random|geo-high|geo-low)] [-s seed(number)]" echo " [-x percent_block_connection(0..100)] [-l percent_loss_packets(0..100)]" echo " [-d run_duration_seconds] [-r percent_restart(0..100)] [-m]" @@ -230,7 +229,6 @@ usage() { # --- Default values --- NUM_VALIDATORS=$DEFAULT_NUM_VALIDATORS -PROTOCOL=$DEFAULT_PROTOCOL BUILD=$DEFAULT_BUILD TOPOLOGY=$DEFAULT_TOPOLOGY SEED=$DEFAULT_SEED @@ -251,10 +249,9 @@ HEAL_NUM_ROUNDS=$DEFAULT_HEAL_NUM_ROUNDS # --- Parse command-line arguments --- # NOTE: -t is TOPOLOGY, -d is RUN DURATION -while getopts ":n:p:b:t:s:x:l:d:r:mS:T:Z:C:h" opt; do +while getopts ":n:b:t:s:x:l:d:r:mS:T:Z:C:h" opt; do case "$opt" in n) NUM_VALIDATORS="$OPTARG" ;; - p) PROTOCOL="$OPTARG" ;; b) BUILD="$OPTARG" ;; t) TOPOLOGY="$OPTARG" ;; s) SEED="$OPTARG" ;; @@ -287,7 +284,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # --- Summary --- log "=== SUMMARY ===" log "Number of validators : $NUM_VALIDATORS" -log "Consensus protocol : $PROTOCOL" log "Rebuild images : $BUILD" log "Topology flag : $TOPOLOGY" log "Seed : $SEED" @@ -349,7 +345,7 @@ fi (cd .. && ./bootstrap.sh -n "$NUM_VALIDATORS") # --- 3) Bring up docker network --- -(cd .. && ./run.sh -n "$NUM_VALIDATORS" -p "$PROTOCOL") +(cd .. && ./run.sh -n "$NUM_VALIDATORS") log "Sleep 5s to boot validators..." sleep 5 diff --git a/dev-tools/iota-private-network/fuzzer/README.md b/dev-tools/iota-private-network/fuzzer/README.md index 5c051aee65c..395fdf0e340 100644 --- a/dev-tools/iota-private-network/fuzzer/README.md +++ b/dev-tools/iota-private-network/fuzzer/README.md @@ -17,7 +17,7 @@ The goal of this document is to show how to: ## Environment setup From the repository root for the private network (make sure the validator -network is running, e.g. `./run.sh -n 10 -p mysticeti`): +network is running, e.g. `./run.sh -n 10`): ```bash cd ~/iota/dev-tools/iota-private-network @@ -238,9 +238,9 @@ This will: Long-running stress experiments live under `net_fuzz.experiments`. See `dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/README.md` for -details. Protocol comparison runners (`run_*.py`) also live in +details. Experiment runners (`run_*.py`) also live in `dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/` and automate -running the same scenario across Mysticeti and Starfish. +the full lifecycle (build images, bootstrap network, run scenario, clean up). ## Logging diff --git a/dev-tools/iota-private-network/fuzzer/fuzzer.md b/dev-tools/iota-private-network/fuzzer/fuzzer.md index b5927648702..223ae22bc73 100644 --- a/dev-tools/iota-private-network/fuzzer/fuzzer.md +++ b/dev-tools/iota-private-network/fuzzer/fuzzer.md @@ -180,4 +180,4 @@ Because the topology, timing, and logging are controlled from Python, these runs are suitable as future simtests: we can attach thresholds like “median resync time must be below X seconds” or “block stress must complete without safety violations” and use them as quantitative -regression tests when we evolve Mysticeti, Starfish, or other protocols. +regression tests when we evolve Starfish or other consensus protocols. diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/README.md b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/README.md index 5e612a24dbd..9afea50b678 100644 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/README.md +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/README.md @@ -6,7 +6,7 @@ core library primitives and are designed for multi-minute stress runs. All commands below assume: -- the private network is running (e.g. `./run.sh -n 10 -p mysticeti`) +- the private network is running (e.g. `./run.sh -n 10`) - a virtual environment is active (`source .venv/bin/activate`) - the `PYTHON` environment variable is set to the current interpreter: `export PYTHON=$(python -c 'import sys; print(sys.executable)')` - `net_fuzz` is installed (`pip install -e fuzzer`) @@ -99,17 +99,15 @@ Run: sudo -E "$PYTHON" -m net_fuzz.experiments.sync_stress ``` -## Protocol comparison runners +## Experiment runners -The `run_*_stress.py` scripts orchestrate full end-to-end runs for both -consensus protocols (Mysticeti and Starfish) under the same conditions. Each -runner: +The `run_*_stress.py` scripts orchestrate full end-to-end runs of each +scenario. Each runner: - cleans up any existing network - bootstraps a fresh validator set -- starts the network for a specific protocol +- starts the network - runs the corresponding experiment -- repeats for the second protocol - auto-detects the current validator count from running containers (falls back to 10 if none are running) diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/block_stress.py b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/block_stress.py index ce9c21fb2c4..5e6b394d39b 100644 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/block_stress.py +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/block_stress.py @@ -148,7 +148,7 @@ def verify_topology(validators: list[str], block_latency_ms: int) -> None: def run() -> None: log_path = configure_experiment_logging("block_stress") - # Set fixed seed for reproducibility across different runs (e.g. Mysticeti vs Starfish) + # Set fixed seed for reproducibility across different runs random.seed(42) validators: list[str] = [] collector = None diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_block_stress.py b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_block_stress.py index 67fd41338b1..cff6d9f90c0 100755 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_block_stress.py +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_block_stress.py @@ -13,7 +13,6 @@ # Configuration # --------------------------------------------------------------------- DEFAULT_NUM_VALIDATORS = 10 -PAUSE_BETWEEN_PROTOCOLS = 60 # seconds # --------------------------------------------------------------------- # Paths @@ -36,7 +35,7 @@ def run_command(cmd, cwd=None, check=True, shell=False): def build_images(): """Builds the necessary Docker images.""" print(">>> Building Docker images...") - + images = [ ("iota-node", DOCKER_ROOT / "iota-node"), ("iota-tools", DOCKER_ROOT / "iota-tools"), @@ -71,10 +70,10 @@ def discover_validator_count(default: int = DEFAULT_NUM_VALIDATORS) -> int: print(f"No validator containers detected; using default count {default}") return default -def run_experiment(protocol): - """Runs the block stress test for a specific protocol.""" +def run_experiment(): + """Runs the block stress test.""" print(f"\n{'='*60}") - print(f"=== Starting Block Stress Test for {protocol.upper()} ===") + print("=== Starting Block Stress Test ===") print(f"{'='*60}\n") # 1. Cleanup existing network @@ -84,14 +83,13 @@ def run_experiment(protocol): num_validators = discover_validator_count() # 2. Bootstrap network - print(f">>> Bootstrapping network for {protocol}...") + print(">>> Bootstrapping network...") # bootstrap.sh generates the configuration run_command(["sudo", "./bootstrap.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # 3. Start network - print(f">>> Starting network with {protocol}...") - # run.sh starts the containers and sets the protocol - run_command(["sudo", "./run.sh", "-n", str(num_validators), "-p", protocol], cwd=PRIVNET_DIR) + print(">>> Starting network...") + run_command(["sudo", "./run.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # Wait for network to stabilize print(">>> Waiting for network to stabilize (20s)...") @@ -99,7 +97,7 @@ def run_experiment(protocol): # 4. Run Block Stress Test print(">>> Running Block Stress Test (Python)...") - + # Ensure venv exists (simple check) venv_python = PRIVNET_DIR / ".venv" / "bin" / "python" pip_path = PRIVNET_DIR / ".venv" / "bin" / "pip" @@ -116,7 +114,7 @@ def run_experiment(protocol): # We execute the module net_fuzz.experiments.block_stress env = os.environ.copy() env["PYTHONPATH"] = str(FUZZER_DIR / "src") - + try: run_command( ["sudo", str(venv_python), "-m", "net_fuzz.experiments.block_stress"], @@ -124,36 +122,29 @@ def run_experiment(protocol): check=True ) except SystemExit: - print(f"Test failed for {protocol}") + print("Test failed") # We don't exit here, we might want to continue or cleanup pass except Exception as e: print(f"An error occurred during the test: {e}") - print(f"=== Finished Block Stress Test for {protocol} ===") + print("=== Finished Block Stress Test ===") # 5. Cleanup print(">>> Cleaning up...") run_command(["sudo", "./cleanup.sh"], cwd=PRIVNET_DIR) def main(): - parser = argparse.ArgumentParser(description="Run Block Stress Test on Mysticeti and Starfish") + parser = argparse.ArgumentParser(description="Run Block Stress Test") parser.add_argument("--skip-build", action="store_true", help="Skip building docker images") args = parser.parse_args() if not args.skip_build: build_images() - # Run Mysticeti - run_experiment("mysticeti") - - print(f"Sleeping {PAUSE_BETWEEN_PROTOCOLS}s before next run...") - time.sleep(PAUSE_BETWEEN_PROTOCOLS) - - # Run Starfish - run_experiment("starfish") + run_experiment() - print("\nAll block stress runs completed.") + print("\nBlock stress run completed.") if __name__ == "__main__": main() diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_mirage_stress.py b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_mirage_stress.py index cc4fcfbbb43..04e50d4f859 100755 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_mirage_stress.py +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_mirage_stress.py @@ -13,7 +13,6 @@ # Configuration # --------------------------------------------------------------------- NUM_VALIDATORS = 10 -PAUSE_BETWEEN_PROTOCOLS = 60 # seconds # --------------------------------------------------------------------- # Paths @@ -36,7 +35,7 @@ def run_command(cmd, cwd=None, check=True, shell=False): def build_images(): """Builds the necessary Docker images.""" print(">>> Building Docker images...") - + images = [ ("iota-node", DOCKER_ROOT / "iota-node"), ("iota-tools", DOCKER_ROOT / "iota-tools"), @@ -48,10 +47,10 @@ def build_images(): # Using sudo because the build scripts often require it or docker requires it run_command(["sudo", "./build.sh", "-t", name], cwd=path) -def run_experiment(protocol): - """Runs the mirage stress test for a specific protocol.""" +def run_experiment(): + """Runs the mirage stress test.""" print(f"\n{'='*60}") - print(f"=== Starting Mirage Stress Test for {protocol.upper()} ===") + print("=== Starting Mirage Stress Test ===") print(f"{'='*60}\n") # 1. Cleanup existing network @@ -59,14 +58,13 @@ def run_experiment(protocol): run_command(["sudo", "./cleanup.sh"], cwd=PRIVNET_DIR) # 2. Bootstrap network - print(f">>> Bootstrapping network for {protocol}...") + print(">>> Bootstrapping network...") # bootstrap.sh generates the configuration run_command(["sudo", "./bootstrap.sh", "-n", str(NUM_VALIDATORS)], cwd=PRIVNET_DIR) # 3. Start network - print(f">>> Starting network with {protocol}...") - # run.sh starts the containers and sets the protocol - run_command(["sudo", "./run.sh", "-n", str(NUM_VALIDATORS), "-p", protocol], cwd=PRIVNET_DIR) + print(">>> Starting network...") + run_command(["sudo", "./run.sh", "-n", str(NUM_VALIDATORS)], cwd=PRIVNET_DIR) # Wait for network to stabilize print(">>> Waiting for network to stabilize (20s)...") @@ -74,7 +72,7 @@ def run_experiment(protocol): # 4. Run Mirage Stress Test print(">>> Running Mirage Stress Test (Python)...") - + # Ensure venv exists (simple check) venv_python = PRIVNET_DIR / ".venv" / "bin" / "python" pip_path = PRIVNET_DIR / ".venv" / "bin" / "pip" @@ -90,7 +88,7 @@ def run_experiment(protocol): # We execute the module net_fuzz.experiments.mirage_stress env = os.environ.copy() env["PYTHONPATH"] = str(FUZZER_DIR / "src") - + try: run_command( ["sudo", str(venv_python), "-m", "net_fuzz.experiments.mirage_stress"], @@ -98,36 +96,29 @@ def run_experiment(protocol): check=True ) except SystemExit: - print(f"Test failed for {protocol}") + print("Test failed") # We don't exit here, we might want to continue or cleanup pass except Exception as e: print(f"An error occurred during the test: {e}") - print(f"=== Finished Mirage Stress Test for {protocol} ===") + print("=== Finished Mirage Stress Test ===") # 5. Cleanup print(">>> Cleaning up...") run_command(["sudo", "./cleanup.sh"], cwd=PRIVNET_DIR) def main(): - parser = argparse.ArgumentParser(description="Run Mirage Stress Test on Mysticeti and Starfish") + parser = argparse.ArgumentParser(description="Run Mirage Stress Test") parser.add_argument("--skip-build", action="store_true", help="Skip building docker images") args = parser.parse_args() if not args.skip_build: build_images() - # Run Mysticeti - run_experiment("mysticeti") - - print(f"Sleeping {PAUSE_BETWEEN_PROTOCOLS}s before next run...") - time.sleep(PAUSE_BETWEEN_PROTOCOLS) - - # Run Starfish - run_experiment("starfish") + run_experiment() - print("\nAll mirage stress runs completed.") + print("\nMirage stress run completed.") if __name__ == "__main__": main() diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_non_triangle_stress.py b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_non_triangle_stress.py index 79ffe21071b..7aa0aa77e06 100755 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_non_triangle_stress.py +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_non_triangle_stress.py @@ -13,7 +13,6 @@ # Configuration # --------------------------------------------------------------------- DEFAULT_NUM_VALIDATORS = 10 -PAUSE_BETWEEN_PROTOCOLS = 60 # seconds # --------------------------------------------------------------------- # Paths @@ -36,7 +35,7 @@ def run_command(cmd, cwd=None, check=True, shell=False): def build_images(): """Builds the necessary Docker images.""" print(">>> Building Docker images...") - + images = [ ("iota-node", DOCKER_ROOT / "iota-node"), ("iota-tools", DOCKER_ROOT / "iota-tools"), @@ -71,10 +70,10 @@ def discover_validator_count(default: int = DEFAULT_NUM_VALIDATORS) -> int: print(f"No validator containers detected; using default count {default}") return default -def run_experiment(protocol): - """Runs the non-triangle stress test for a specific protocol.""" +def run_experiment(): + """Runs the non-triangle stress test.""" print(f"\n{'='*60}") - print(f"=== Starting Non-Triangle Stress Test for {protocol.upper()} ===") + print("=== Starting Non-Triangle Stress Test ===") print(f"{'='*60}\n") # 1. Cleanup existing network @@ -84,14 +83,13 @@ def run_experiment(protocol): num_validators = discover_validator_count() # 2. Bootstrap network - print(f">>> Bootstrapping network for {protocol}...") + print(">>> Bootstrapping network...") # bootstrap.sh generates the configuration run_command(["sudo", "./bootstrap.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # 3. Start network - print(f">>> Starting network with {protocol}...") - # run.sh starts the containers and sets the protocol - run_command(["sudo", "./run.sh", "-n", str(num_validators), "-p", protocol], cwd=PRIVNET_DIR) + print(">>> Starting network...") + run_command(["sudo", "./run.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # Wait for network to stabilize print(">>> Waiting for network to stabilize (20s)...") @@ -99,7 +97,7 @@ def run_experiment(protocol): # 4. Run Non-Triangle Stress Test print(">>> Running Non-Triangle Stress Test (Python)...") - + # Ensure venv exists (simple check) venv_python = PRIVNET_DIR / ".venv" / "bin" / "python" pip_path = PRIVNET_DIR / ".venv" / "bin" / "pip" @@ -108,6 +106,7 @@ def run_experiment(protocol): run_command([sys.executable, "-m", "venv", str(PRIVNET_DIR / ".venv")]) run_command([str(pip_path), "install", "--upgrade", "pip"]) + # Ensure the editable install is up-to-date run_command([str(pip_path), "install", "-e", "."], cwd=FUZZER_DIR) # Run the fuzzer script @@ -115,7 +114,7 @@ def run_experiment(protocol): # We execute the module net_fuzz.experiments.non_triangle_stress env = os.environ.copy() env["PYTHONPATH"] = str(FUZZER_DIR / "src") - + try: run_command( ["sudo", str(venv_python), "-m", "net_fuzz.experiments.non_triangle_stress"], @@ -123,36 +122,29 @@ def run_experiment(protocol): check=True ) except SystemExit: - print(f"Test failed for {protocol}") + print("Test failed") # We don't exit here, we might want to continue or cleanup pass except Exception as e: print(f"An error occurred during the test: {e}") - print(f"=== Finished Non-Triangle Stress Test for {protocol} ===") + print("=== Finished Non-Triangle Stress Test ===") # 5. Cleanup print(">>> Cleaning up...") run_command(["sudo", "./cleanup.sh"], cwd=PRIVNET_DIR) def main(): - parser = argparse.ArgumentParser(description="Run Non-Triangle Stress Test on Mysticeti and Starfish") + parser = argparse.ArgumentParser(description="Run Non-Triangle Stress Test") parser.add_argument("--skip-build", action="store_true", help="Skip building docker images") args = parser.parse_args() if not args.skip_build: build_images() - # Run Mysticeti - run_experiment("mysticeti") - - print(f"Sleeping {PAUSE_BETWEEN_PROTOCOLS}s before next run...") - time.sleep(PAUSE_BETWEEN_PROTOCOLS) - - # Run Starfish - run_experiment("starfish") + run_experiment() - print("\nAll non-triangle stress runs completed.") + print("\nNon-triangle stress run completed.") if __name__ == "__main__": main() diff --git a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_sync_stress.py b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_sync_stress.py index d3165562d22..f72baf7e652 100755 --- a/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_sync_stress.py +++ b/dev-tools/iota-private-network/fuzzer/src/net_fuzz/experiments/run_sync_stress.py @@ -13,7 +13,6 @@ # Configuration # --------------------------------------------------------------------- DEFAULT_NUM_VALIDATORS = 10 -PAUSE_BETWEEN_PROTOCOLS = 60 # seconds # --------------------------------------------------------------------- # Paths @@ -36,7 +35,7 @@ def run_command(cmd, cwd=None, check=True, shell=False): def build_images(): """Builds the necessary Docker images.""" print(">>> Building Docker images...") - + images = [ ("iota-node", DOCKER_ROOT / "iota-node"), ("iota-tools", DOCKER_ROOT / "iota-tools"), @@ -71,10 +70,10 @@ def discover_validator_count(default: int = DEFAULT_NUM_VALIDATORS) -> int: print(f"No validator containers detected; using default count {default}") return default -def run_experiment(protocol): - """Runs the sync stress test for a specific protocol.""" +def run_experiment(): + """Runs the sync stress test.""" print(f"\n{'='*60}") - print(f"=== Starting Sync Stress Test for {protocol.upper()} ===") + print("=== Starting Sync Stress Test ===") print(f"{'='*60}\n") # 1. Cleanup existing network @@ -84,14 +83,13 @@ def run_experiment(protocol): num_validators = discover_validator_count() # 2. Bootstrap network - print(f">>> Bootstrapping network for {protocol}...") + print(">>> Bootstrapping network...") # bootstrap.sh generates the configuration run_command(["sudo", "./bootstrap.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # 3. Start network - print(f">>> Starting network with {protocol}...") - # run.sh starts the containers and sets the protocol - run_command(["sudo", "./run.sh", "-n", str(num_validators), "-p", protocol], cwd=PRIVNET_DIR) + print(">>> Starting network...") + run_command(["sudo", "./run.sh", "-n", str(num_validators)], cwd=PRIVNET_DIR) # Wait for network to stabilize print(">>> Waiting for network to stabilize (20s)...") @@ -99,7 +97,7 @@ def run_experiment(protocol): # 4. Run Sync Stress Test print(">>> Running Sync Stress Test (Python)...") - + # Ensure venv exists (simple check) venv_python = PRIVNET_DIR / ".venv" / "bin" / "python" pip_path = PRIVNET_DIR / ".venv" / "bin" / "pip" @@ -115,7 +113,7 @@ def run_experiment(protocol): # We execute the module net_fuzz.experiments.sync_stress env = os.environ.copy() env["PYTHONPATH"] = str(FUZZER_DIR / "src") - + try: run_command( ["sudo", str(venv_python), "-m", "net_fuzz.experiments.sync_stress"], @@ -123,36 +121,29 @@ def run_experiment(protocol): check=True ) except SystemExit: - print(f"Test failed for {protocol}") + print("Test failed") # We don't exit here, we might want to continue or cleanup pass except Exception as e: print(f"An error occurred during the test: {e}") - print(f"=== Finished Sync Stress Test for {protocol} ===") + print("=== Finished Sync Stress Test ===") # 5. Cleanup print(">>> Cleaning up...") run_command(["sudo", "./cleanup.sh"], cwd=PRIVNET_DIR) def main(): - parser = argparse.ArgumentParser(description="Run Sync Stress Test on Mysticeti and Starfish") + parser = argparse.ArgumentParser(description="Run Sync Stress Test") parser.add_argument("--skip-build", action="store_true", help="Skip building docker images") args = parser.parse_args() if not args.skip_build: build_images() - # Run Mysticeti - run_experiment("mysticeti") - - print(f"Sleeping {PAUSE_BETWEEN_PROTOCOLS}s before next run...") - time.sleep(PAUSE_BETWEEN_PROTOCOLS) - - # Run Starfish - run_experiment("starfish") + run_experiment() - print("\nAll sync stress runs completed.") + print("\nSync stress run completed.") if __name__ == "__main__": main() diff --git a/dev-tools/iota-private-network/run.sh b/dev-tools/iota-private-network/run.sh index 23ca94740d5..c5e0a61485d 100755 --- a/dev-tools/iota-private-network/run.sh +++ b/dev-tools/iota-private-network/run.sh @@ -4,50 +4,15 @@ # SPDX-License-Identifier: Apache-2.0 -# Default validator count and consensus +# Default validator count NUM_VALIDATORS=4 -PROTOCOL="starfish" -while getopts "n:p:" opt; do +while getopts "n:" opt; do case "$opt" in n) NUM_VALIDATORS="$OPTARG" ;; - p) PROTOCOL="$OPTARG" ;; *) echo "Usage: $0 [-n num_validators]"; exit 1 ;; esac done shift $((OPTIND -1)) -PRIVNET_DIR="$(realpath "$(dirname "$0")" || echo "$ROOT/dev-tools/iota-private-network")" - -# Set and unset environment variables -set_env_var() { - # Usage: set_env_var KEY VALUE FILE - local key="$1" value="$2" file="$3" - mkdir -p "$(dirname "$file")" - if [ -f "$file" ]; then - if grep -q "^${key}=" "$file"; then - # portable in-place replace without sed -i - tmpfile="$(mktemp)" - awk -v k="$key" -v v="$value" 'BEGIN{changed=0} {if ($0 ~ "^"k"=") {print k"="v; changed=1} else {print}} END{if (!changed) print k"="v}' "$file" > "$tmpfile" && mv "$tmpfile" "$file" - else - echo "${key}=${value}" >> "$file" - fi - else - echo "${key}=${value}" > "$file" - fi -} -unset_env_var() { - # Usage: unset_env_var KEY FILE - local key="$1" file="$2" - [ -f "$file" ] || return 0 - tmpfile="$(mktemp)" - awk -v k="$key" 'BEGIN{removed=0} $0 !~ "^"k"=" {print} $0 ~ "^"k"=" {removed=1} END{}' "$file" > "$tmpfile" && mv "$tmpfile" "$file" -} - -# Manage CONSENSUS_PROTOCOL in .env: set if provided, otherwise remove it -ENV_FILE="$PRIVNET_DIR/.env" -set_env_var CONSENSUS_PROTOCOL "$PROTOCOL" "$ENV_FILE" -echo "Set CONSENSUS_PROTOCOL=$PROTOCOL in $ENV_FILE" - - function start_services() { services="$1" diff --git a/docs/content/about-iota/about-iota.mdx b/docs/content/about-iota/about-iota.mdx index 3d892fe896f..37ad8d4ff77 100644 --- a/docs/content/about-iota/about-iota.mdx +++ b/docs/content/about-iota/about-iota.mdx @@ -80,4 +80,4 @@ Transactions on the IOTA network require [gas](./tokenomics/gas-pricing.mdx) fee ## Consensus on IOTA -IOTA uses a [delegated proof-of-stake (DPoS)](./tokenomics/proof-of-stake.mdx) consensus mechanism to validate on-chain transaction blocks. [Validators](iota-architecture/validator-committee.mdx) must secure a certain amount of IOTA tokens to participate in the network, where they actively run the [Mysticeti](./iota-architecture/consensus.mdx) protocol to finalize transactions. This approach incentivizes honest behavior, enhances security, and ensures an efficient and scalable blockchain while avoiding the high energy demands of Proof-of-Work (PoW) systems, which rely on computationally intensive mining. +IOTA uses a [delegated proof-of-stake (DPoS)](./tokenomics/proof-of-stake.mdx) consensus mechanism to validate on-chain transaction blocks. [Validators](iota-architecture/validator-committee.mdx) must secure a certain amount of IOTA tokens to participate in the network, where they actively run the [Starfish](./iota-architecture/consensus.mdx) consensus protocol to finalize transactions. This approach incentivizes honest behavior, enhances security, and ensures an efficient and scalable blockchain while avoiding the high energy demands of Proof-of-Work (PoW) systems, which rely on computationally intensive mining. diff --git a/docs/content/about-iota/iota-architecture/consensus.mdx b/docs/content/about-iota/iota-architecture/consensus.mdx index c78b77717f1..102c99a65dd 100644 --- a/docs/content/about-iota/iota-architecture/consensus.mdx +++ b/docs/content/about-iota/iota-architecture/consensus.mdx @@ -9,7 +9,7 @@ import Quiz from '@site/src/components/Quiz'; import quizData from '../../../site/static/json/about-iota/iota-architecture/consensus.json'; import ThemedImage from '@theme/ThemedImage'; -The primary goal of consensus in distributed systems is to establish a consistent order of transactions and ensure their availability. In IOTA, consensus is achieved through a combination of Delegated Proof-of-Stake (dPoS) and a Byzantine Fault Tolerant (BFT) protocol. Currently, the Testnet and Mainnet validators employ the Mysticeti BFT protocol, however they will switch to the Starfish BFT protocol, already deployed on the Devnet, once the validators accept the protocol upgrade. +The primary goal of consensus in distributed systems is to establish a consistent order of transactions and ensure their availability. In IOTA, consensus is achieved through a combination of Delegated Proof-of-Stake (dPoS) and a Byzantine Fault Tolerant (BFT) protocol. All IOTA networks — Devnet, Testnet, and Mainnet — run the **Starfish** BFT consensus protocol. IOTA's consensus mechanism ensures: @@ -25,7 +25,7 @@ IOTA's consensus mechanism ensures: IOTA employs Delegated Proof-of-Stake (dPoS) as the overarching consensus mechanism. In this model: - Token holders delegate their voting power to validators. -- Validators participate in block production and execute the Mysticeti protocol to finalize transactions. +- Validators participate in block production and execute the Starfish protocol to finalize transactions. - Staking rewards serve as the main incentive for validators to behave honestly and secure the network. - This structure ensures security and decentralization while maintaining scalability. @@ -44,11 +44,11 @@ Validators that participate in the consensus protocol collectively form the **Co For more details on validator staking, refer to the [Validators Staking](../tokenomics/validators-staking.mdx) page. To understand epoch transitions, see the [Epochs](./epochs.mdx) documentation. -## The Mysticeti Protocol +## The Starfish Protocol -**Mysticeti** is a _BFT_ consensus protocol optimized for **low latency** and **high throughput** using an uncertified _DAG_ structure. It enables efficient and parallel transaction processing while maintaining strong security guarantees. +**Starfish** is a _BFT_ consensus protocol optimized for **low latency** and **high throughput** using an uncertified _DAG_ structure. It is the successor of the earlier Mysticeti protocol and retains its key advantages while strengthening robustness in Byzantine and unstable network environments. -### Key Features of Mysticeti +### Key Features of Starfish - **Parallel Block Proposals** Multiple validators can propose blocks simultaneously, maximizing network bandwidth and improving censorship resistance — a core advantage of DAG-based protocols. @@ -57,32 +57,14 @@ Multiple validators can propose blocks simultaneously, maximizing network bandwi Blocks reach finality in only three rounds of communication, matching the efficiency of pBFT and achieving the theoretical minimum for BFT consensus. - **Optimized Voting** -Unlike traditional leader-based protocols, Mysticeti allows validators to vote and certify blocks in parallel, significantly reducing median and tail latencies. +Validators vote and certify blocks in parallel rather than sequentially behind a single leader, significantly reducing median and tail latencies. - **Fault Tolerance** The protocol remains resilient even if some validators are offline or crashed, without substantially impacting commit latency. -🔗 **Reference:** [Mysticeti Paper](https://arxiv.org/pdf/2310.14821) - ---- - -### Observed Limitations of Mysticeti +### Improvements over Mysticeti -Testing across the IOTA Testnet, Mainnet, and controlled local deployments revealed several scenarios where the Mysticeti performance may degrade due to network behavior or validator misbehavior, such as: - -- Disrupted or blocked connections between validators -- Network latency patterns violating triangle inequality assumptions -- Validators behaving adversarially or delaying messages - -These conditions may increase latency or disrupt transaction propagation, limiting throughput under adverse or hostile environments. - ---- - -## The Starfish Protocol - -**Starfish** is a _BFT_ consensus protocol inspired by Mysticeti and designed specifically to address the limitations above while retaining its advantages. - -### Key Improvements Introduced by Starfish +Starfish was designed to address limitations observed with Mysticeti under disrupted connections, latency patterns violating triangle inequality, and adversarial validator behavior. The main changes are: - **Decoupled Block Headers and Transaction Payloads** Block header metadata is separated from transaction data, enabling efficient and flexible dissemination of headers. @@ -94,10 +76,6 @@ This ensures deterministic and predictable commit performance. - **Linear Communication Cost for Transaction Data** Transactions are encoded into shards using Reed–Solomon erasure coding. Nodes can reconstruct missing data from partial shards, removing redundant broadcasts and enabling scalable throughput. ---- - -These design improvements significantly strengthen Starfish’s robustness in Byzantine and unstable environments. - 🔗 **Reference:** [Starfish Paper](https://eprint.iacr.org/2025/567) - \ No newline at end of file + diff --git a/docs/content/about-iota/iota-architecture/epochs.mdx b/docs/content/about-iota/iota-architecture/epochs.mdx index 54249237eb4..8515b323592 100644 --- a/docs/content/about-iota/iota-architecture/epochs.mdx +++ b/docs/content/about-iota/iota-architecture/epochs.mdx @@ -17,7 +17,7 @@ Epochs are sequentially numbered, and each transaction on the IOTA network inclu ## Consensus & Checkpoints -During each epoch, the Mysticeti consensus protocol operates to finalize transactions. Validators reach agreement by collecting transaction effect certificates, ensuring a consistent transaction order. A key part of this process is the checkpoint, which is a snapshot of finalized transactions at the end of an epoch. The checkpoint includes: +During each epoch, the Starfish consensus protocol operates to finalize transactions. Validators reach agreement by collecting transaction effect certificates, ensuring a consistent transaction order. A key part of this process is the checkpoint, which is a snapshot of finalized transactions at the end of an epoch. The checkpoint includes: - A set of finalized transactions - Validator agreement data diff --git a/docs/content/about-iota/iota-architecture/iota-architecture.mdx b/docs/content/about-iota/iota-architecture/iota-architecture.mdx index 0b49de93ae5..5ab8849404d 100644 --- a/docs/content/about-iota/iota-architecture/iota-architecture.mdx +++ b/docs/content/about-iota/iota-architecture/iota-architecture.mdx @@ -21,7 +21,7 @@ Go to [Life of a Transaction](transaction-lifecycle.mdx). ## Consensus Every transaction on IOTA that touches a shared object must go through a consensus process. This is to ensure that all nodes in the network agree on which transactions should be allowed to modify the ledger state. -IOTA currently uses the [Mysticeti](https://arxiv.org/pdf/2310.14821) consensus algorithm. +IOTA uses the [Starfish](https://eprint.iacr.org/2025/567) consensus algorithm, the successor of [Mysticeti](https://arxiv.org/pdf/2310.14821). Go to [Consensus](consensus.mdx). diff --git a/docs/content/developer/references/iota-compared.mdx b/docs/content/developer/references/iota-compared.mdx index 348140adfa2..74212317049 100644 --- a/docs/content/developer/references/iota-compared.mdx +++ b/docs/content/developer/references/iota-compared.mdx @@ -57,7 +57,7 @@ Unlike most existing blockchain systems (and as the reader may have guessed from ## State-of-the-art consensus -[Mysticeti](https://arxiv.org/pdf/2310.14821) and [Starfish](https://eprint.iacr.org/2025/567) represent the latest variant of decades of work on multi-proposer, high-throughput consensus algorithms that reaches throughput more than 400,000 transactions per second on a WAN, with production cryptography and permanent storage. +[Starfish](https://eprint.iacr.org/2025/567) — the successor of [Mysticeti](https://arxiv.org/pdf/2310.14821) — is the latest variant of decades of work on multi-proposer, high-throughput consensus algorithms that reaches throughput more than 400,000 transactions per second on a WAN, with production cryptography and permanent storage. ## Where IOTA excels diff --git a/docs/content/developer/references/research-papers.mdx b/docs/content/developer/references/research-papers.mdx index 5688d4ff2e1..a0f72d708e9 100644 --- a/docs/content/developer/references/research-papers.mdx +++ b/docs/content/developer/references/research-papers.mdx @@ -9,7 +9,7 @@ This document contains a list of research papers that are relevant to IOTA. - **Link:** https://eprint.iacr.org/2025/567 - **Publication:** not published yet -- **Relevance:** The IOTA consensus developers are currently working on porting Starfish to the production code to improve liveness of the consensus protocol. +- **Relevance:** Starfish is the consensus protocol used by IOTA in production. It succeeds Mysticeti-C and improves liveness in the presence of network disruption and validator misbehavior. - **Summary:** Starfish is a partially synchronous DAG-based BFT protocol that achieves the security properties of certified DAGs, the efficiency of uncertified approaches and linear amortized communication complexity. The key innovation is Encoded Cordial Dissemination, a push-based dissemination strategy that combines Reed-Solomon erasure coding with Data Availability Certificates (DACs). Starfish decouples the transaction data from the block header information in the block structure, enabling push-based dissemination of block headers. Building on the previous uncertified DAG BFT commit rule, Starfish extends it to efficiently verify data availability through committed leader blocks serving as DACs. @@ -17,7 +17,7 @@ This document contains a list of research papers that are relevant to IOTA. - **Link:** https://arxiv.org/pdf/2310.14821 - **Publication:** Network and Distributed System Security Symposium (NDSS), accepted for 2025 -- **Relevance:** The production IOTA code currently employs Mysticeti-C with one leader in each round as a consensus mechanism. +- **Relevance:** Mysticeti-C is the direct predecessor of the Starfish consensus protocol used by IOTA in production. It pioneered the uncertified DAG design that Starfish extends. - **Summary:** Mysticeti is a partially synchronous DAG-based BFT protocol that is built over an uncertified DAG allowing for high resource efficiency. Mysticeti-C achieves a significant latency improvement compared to certified DAG BFTs by avoiding explicit certification of the DAG blocks and deriving certificates from the DAG structure. ## FastPay: High-Performance Byzantine Fault Tolerant Settlement {#fastpay} @@ -36,8 +36,7 @@ This document contains a list of research papers that are relevant to IOTA. - **Link:** https://arxiv.org/pdf/2309.12713 - **Publication:** IEEE International Conference on Distributed Computing Systems (ICDCS), 2024 -- **Relevance:** The IOTA consensus protocol uses a variant of the Mysticeti consensus -algorithm due to its lower latency and leader scoring strategy stemming from the HammerHead's approach to update the leader scheduler in Mysticeti. +- **Relevance:** IOTA's Starfish consensus protocol inherits HammerHead's score-based leader scheduler — initially introduced into the Mysticeti family — for lower latency and improved leader utilization. - **Summary:** We explore the ideas pioneered by Carousel on providing Leader-Utilization and present HammerHead. Unlike Carousel, which is built with a chained and pipelined consensus protocol in mind, HammerHead does not need to worry about chain quality as it is directly provided by the DAG, but needs to make sure that even though validators might commit blocks in different views the safety and liveness is preserved. Our implementation of HammerHead shows a slight performance increase in a faultless diff --git a/docs/site/static/json/about-iota/iota-architecture/consensus.json b/docs/site/static/json/about-iota/iota-architecture/consensus.json index 26b944ba6b0..4960862413a 100644 --- a/docs/site/static/json/about-iota/iota-architecture/consensus.json +++ b/docs/site/static/json/about-iota/iota-architecture/consensus.json @@ -4,7 +4,7 @@ "questionText": "What consensus mechanism does IOTA use to achieve transaction finality?", "answerOptions": [ { "answerText": "Proof-of-Work (PoW)", "isCorrect": false }, - { "answerText": "Delegated Proof-of-Stake (dPoS) combined with the Mysticeti BFT protocol", "isCorrect": true }, + { "answerText": "Delegated Proof-of-Stake (dPoS) combined with the Starfish BFT protocol", "isCorrect": true }, { "answerText": "Pure Proof-of-Stake (PoS)", "isCorrect": false }, { "answerText": "Randomized Leader Election (RLE)", "isCorrect": false } ] @@ -19,7 +19,7 @@ ] }, { - "questionText": "Which of the following is a key feature of the Mysticeti consensus protocol?", + "questionText": "Which of the following is a key feature of the Starfish consensus protocol?", "answerOptions": [ { "answerText": "Uses Proof-of-Work to secure the network", "isCorrect": false }, { "answerText": "Relies on a central coordinator for block approvals", "isCorrect": false }, diff --git a/scripts/compatibility/split-cluster-sync-check.sh b/scripts/compatibility/split-cluster-sync-check.sh index 2aaada07372..0b08df11a55 100755 --- a/scripts/compatibility/split-cluster-sync-check.sh +++ b/scripts/compatibility/split-cluster-sync-check.sh @@ -172,17 +172,7 @@ get_metrics "${CONFIGS[0]}" "$METRICS_DIR/node-0-before-node3.txt" INITIAL_COMMIT_INDEX=$(get_metric_value "$METRICS_DIR/node-0-before-node3.txt" "consensus_last_commit_index") echo "Initial commit index on node-0: $INITIAL_COMMIT_INDEX" -# Detect consensus protocol (Starfish vs Mysticeti) from the release node log. -# The consensus manager logs "Starting consensus protocol Mysticeti ..." or -# "Starting consensus protocol Starfish ..." on every epoch start, making this -# a reliable signal independent of any metric values. -if grep -q "Starting consensus protocol Starfish" "$LOG_DIR/node-0.log" 2>/dev/null; then - CONSENSUS_TYPE="starfish" - echo "Detected consensus protocol: Starfish" -else - CONSENSUS_TYPE="mysticeti" - echo "Detected consensus protocol: Mysticeti" -fi +echo "Consensus protocol: Starfish" echo -e "\n=== Phase 2: Late Start of Candidate Node ===" echo "Starting node-3 (candidate) - should trigger synchronization to catch up..." @@ -208,30 +198,18 @@ fi NODE3_COMMIT_AFTER_JOIN=$(get_metric_value "$METRICS_DIR/node-3-after-join.txt" "consensus_last_commit_index") -if [ "$CONSENSUS_TYPE" = "starfish" ]; then - # Starfish: commit_sync_fetched_commits is labeled by source (commit_sync, fast_commit_sync), so sum across labels - NODE3_COMMIT_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_fetched_commits") - NODE3_HEADER_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_synchronizer_fetched_block_headers_by_peer") - NODE3_TXN_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_transaction_synchronizer_fetched_transactions_by_peer") - NODE3_COMMIT_SYNC_TXN_SIZE=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_total_fetched_transactions_size") -else - # Mysticeti: commit_sync_fetched_commits is labeled by authority, so sum across labels - NODE3_COMMIT_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_fetched_commits") - NODE3_BLOCK_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_synchronizer_fetched_blocks_by_peer") - NODE3_COMMIT_SYNC_BLOCKS=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_fetched_blocks") -fi +# Starfish: commit_sync_fetched_commits is labeled by source (commit_sync, fast_commit_sync), so sum across labels +NODE3_COMMIT_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_fetched_commits") +NODE3_HEADER_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_synchronizer_fetched_block_headers_by_peer") +NODE3_TXN_SYNC=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_transaction_synchronizer_fetched_transactions_by_peer") +NODE3_COMMIT_SYNC_TXN_SIZE=$(sum_metric_values "$METRICS_DIR/node-3-after-join.txt" "consensus_commit_sync_total_fetched_transactions_size") echo "Node-3 metrics after initial sync:" echo " last_commit_index: $NODE3_COMMIT_AFTER_JOIN" echo " commit_sync_fetched_commits (sum): $NODE3_COMMIT_SYNC" -if [ "$CONSENSUS_TYPE" = "starfish" ]; then - echo " synchronizer_fetched_block_headers_by_peer (sum): $NODE3_HEADER_SYNC" - echo " commit_sync_total_fetched_transactions_size: $NODE3_COMMIT_SYNC_TXN_SIZE" - echo " transaction_synchronizer_fetched_transactions_by_peer (sum): $NODE3_TXN_SYNC" -else - echo " synchronizer_fetched_blocks_by_peer (sum): $NODE3_BLOCK_SYNC" - echo " commit_sync_fetched_blocks (sum): $NODE3_COMMIT_SYNC_BLOCKS" -fi +echo " synchronizer_fetched_block_headers_by_peer (sum): $NODE3_HEADER_SYNC" +echo " commit_sync_total_fetched_transactions_size: $NODE3_COMMIT_SYNC_TXN_SIZE" +echo " transaction_synchronizer_fetched_transactions_by_peer (sum): $NODE3_TXN_SYNC" # Check 1: Node-3 caught up past initial commit index if [ "$NODE3_COMMIT_AFTER_JOIN" -le "$INITIAL_COMMIT_INDEX" ]; then @@ -240,45 +218,25 @@ else echo "✓ Node-3 caught up past initial commit index" fi -# Check 2: Synchronizer was active (protocol-specific) -if [ "$CONSENSUS_TYPE" = "starfish" ]; then - # Starfish has a dedicated block header synchronizer - if [ "$NODE3_HEADER_SYNC" -le 0 ]; then - FAILURES+=("FAIL: Header synchronizer was not active (consensus_synchronizer_fetched_block_headers_by_peer = $NODE3_HEADER_SYNC)") - else - echo "✓ Header synchronizer was active (fetched $NODE3_HEADER_SYNC block headers)" - fi +# Check 2: Header synchronizer was active +if [ "$NODE3_HEADER_SYNC" -le 0 ]; then + FAILURES+=("FAIL: Header synchronizer was not active (consensus_synchronizer_fetched_block_headers_by_peer = $NODE3_HEADER_SYNC)") else - # Mysticeti uses a block synchronizer (no separate header sync) - if [ "$NODE3_BLOCK_SYNC" -le 0 ]; then - FAILURES+=("FAIL: Block synchronizer was not active (consensus_synchronizer_fetched_blocks_by_peer = $NODE3_BLOCK_SYNC)") - else - echo "✓ Block synchronizer was active (fetched $NODE3_BLOCK_SYNC blocks)" - fi + echo "✓ Header synchronizer was active (fetched $NODE3_HEADER_SYNC block headers)" fi -# Check 3: Commit syncer was active (applies to both protocols) +# Check 3: Commit syncer was active if [ "$NODE3_COMMIT_SYNC" -le 0 ]; then FAILURES+=("FAIL: Commit syncer was not active (consensus_commit_sync_fetched_commits = $NODE3_COMMIT_SYNC)") else echo "✓ Commit syncer was active (fetched $NODE3_COMMIT_SYNC commits)" fi -# Check 4: Data was fetched via commit syncer (protocol-specific) -if [ "$CONSENSUS_TYPE" = "starfish" ]; then - # Starfish: transactions can come from commit syncer or transaction synchronizer - if [ "$NODE3_COMMIT_SYNC_TXN_SIZE" -le 0 ] && [ "$NODE3_TXN_SYNC" -le 0 ]; then - FAILURES+=("FAIL: No transactions were fetched (commit_sync: $NODE3_COMMIT_SYNC_TXN_SIZE bytes, txn_sync: $NODE3_TXN_SYNC)") - else - echo "✓ Transactions were fetched (commit_sync: $NODE3_COMMIT_SYNC_TXN_SIZE bytes, txn_sync: $NODE3_TXN_SYNC transactions)" - fi +# Check 4: Transactions can come from commit syncer or transaction synchronizer +if [ "$NODE3_COMMIT_SYNC_TXN_SIZE" -le 0 ] && [ "$NODE3_TXN_SYNC" -le 0 ]; then + FAILURES+=("FAIL: No transactions were fetched (commit_sync: $NODE3_COMMIT_SYNC_TXN_SIZE bytes, txn_sync: $NODE3_TXN_SYNC)") else - # Mysticeti: no transaction synchronizer; check blocks fetched via commit sync - if [ "$NODE3_COMMIT_SYNC_BLOCKS" -le 0 ]; then - FAILURES+=("FAIL: No blocks were fetched via commit sync (consensus_commit_sync_fetched_blocks = $NODE3_COMMIT_SYNC_BLOCKS)") - else - echo "✓ Blocks were fetched via commit sync ($NODE3_COMMIT_SYNC_BLOCKS blocks)" - fi + echo "✓ Transactions were fetched (commit_sync: $NODE3_COMMIT_SYNC_TXN_SIZE bytes, txn_sync: $NODE3_TXN_SYNC transactions)" fi # Capture final metrics from all nodes @@ -310,15 +268,10 @@ if [ ${#FAILURES[@]} -eq 0 ]; then echo "✓ All checks passed!" echo "" echo "Successfully verified:" - echo " - Split-cluster with 3 release + 1 candidate node (consensus: $CONSENSUS_TYPE)" + echo " - Split-cluster with 3 release + 1 candidate node (consensus: Starfish)" echo " - Candidate node synchronized after late start (3 minute delay):" - if [ "$CONSENSUS_TYPE" = "starfish" ]; then - echo " • Header Synchronizer: fetched $NODE3_HEADER_SYNC block headers" - echo " • Commit Syncer: fetched $NODE3_COMMIT_SYNC commits ($NODE3_COMMIT_SYNC_TXN_SIZE bytes txn, txn_sync: $NODE3_TXN_SYNC transactions)" - else - echo " • Block Synchronizer: fetched $NODE3_BLOCK_SYNC blocks" - echo " • Commit Syncer: fetched $NODE3_COMMIT_SYNC commits ($NODE3_COMMIT_SYNC_BLOCKS blocks)" - fi + echo " • Header Synchronizer: fetched $NODE3_HEADER_SYNC block headers" + echo " • Commit Syncer: fetched $NODE3_COMMIT_SYNC commits ($NODE3_COMMIT_SYNC_TXN_SIZE bytes txn, txn_sync: $NODE3_TXN_SYNC transactions)" echo " • Caught up from commit $INITIAL_COMMIT_INDEX to $NODE3_COMMIT_AFTER_JOIN" echo " - Synchronization protocols are compatible between release and candidate versions" echo "" diff --git a/scripts/simtest/simtest-run.sh b/scripts/simtest/simtest-run.sh index 1537321ad49..99bd5f5b312 100755 --- a/scripts/simtest/simtest-run.sh +++ b/scripts/simtest/simtest-run.sh @@ -8,12 +8,9 @@ MSIM_WATCHDOG_TIMEOUT_MS=${MSIM_WATCHDOG_TIMEOUT_MS:-180000} # Override the default dir for logs output: SIMTEST_LOGS_DIR="${SIMTEST_LOGS_DIR:-"$HOME/simtest_logs"}" -# Set default consensus implementation: -CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL:-mysticeti} echo "Running simulator tests at commit $(git rev-parse HEAD)" echo "Using MSIM_WATCHDOG_TIMEOUT_MS=${MSIM_WATCHDOG_TIMEOUT_MS} from env var" -echo "Using CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL}" # Function to handle SIGINT signal (Ctrl+C) cleanup() { @@ -26,7 +23,7 @@ cleanup() { trap cleanup SIGINT if [ -z "$NUM_CPUS" ]; then - if [ "$(uname -s)" == "Darwin" ]; + if [ "$(uname -s)" == "Darwin" ]; then NUM_CPUS="$(sysctl -n hw.ncpu)"; # mac else NUM_CPUS=$(cat /proc/cpuinfo | grep processor | wc -l) # ubuntu fi @@ -63,7 +60,7 @@ echo "================================================" echo "Running e2e simtests with $TEST_NUM iterations" echo "================================================" date -echo "Using MSIM_TEST_SEED=${MSIM_TEST_SEED}, MSIM_TEST_NUM=${TEST_NUM}, CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL}, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" +echo "Using MSIM_TEST_SEED=${MSIM_TEST_SEED}, MSIM_TEST_NUM=${TEST_NUM}, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" # This command runs many different tests, so it already uses all CPUs fairly efficiently, and # don't need to be done inside of the for loop below. @@ -71,7 +68,6 @@ echo "Using MSIM_TEST_SEED=${MSIM_TEST_SEED}, MSIM_TEST_NUM=${TEST_NUM}, CONSENS MSIM_TEST_SEED=${MSIM_TEST_SEED} \ MSIM_TEST_NUM=${TEST_NUM} \ MSIM_WATCHDOG_TIMEOUT_MS=${MSIM_WATCHDOG_TIMEOUT_MS} \ -CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL} \ scripts/simtest/cargo-simtest simtest \ --color always \ --test-threads "$NUM_CPUS" \ @@ -93,14 +89,13 @@ date for WORKER_NUMBER in `seq 1 $WORKERS_COUNT`; do SUB_SEED="$WORKER_NUMBER$DATE" LOG_FILE="$LOG_DIR/log-$SUB_SEED" - echo "Iteration $WORKER_NUMBER using MSIM_TEST_SEED=${SUB_SEED}, MSIM_TEST_NUM=1, CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL}, SIM_STRESS_TEST_DURATION_SECS=300, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" + echo "Iteration $WORKER_NUMBER using MSIM_TEST_SEED=${SUB_SEED}, MSIM_TEST_NUM=1, SIM_STRESS_TEST_DURATION_SECS=300, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" # --test-threads 1 is important: parallelism is achieved via the for loop MSIM_TEST_SEED="$SUB_SEED" \ MSIM_TEST_NUM=1 \ MSIM_WATCHDOG_TIMEOUT_MS=${MSIM_WATCHDOG_TIMEOUT_MS} \ SIM_STRESS_TEST_DURATION_SECS=300 \ - CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL} \ scripts/simtest/cargo-simtest simtest \ --color always \ --test-threads 1 \ @@ -121,13 +116,12 @@ date # Check for determinism in stress simtests LOG_FILE="$LOG_DIR/determinism-log" -echo "Using MSIM_TEST_SEED=${MSIM_TEST_SEED}, MSIM_TEST_NUM=1, CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL}, MSIM_TEST_CHECK_DETERMINISM=1, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" +echo "Using MSIM_TEST_SEED=${MSIM_TEST_SEED}, MSIM_TEST_NUM=1, MSIM_TEST_CHECK_DETERMINISM=1, TEST_FILTER=${FINAL_TEST_FILTER}, logging to $LOG_FILE" MSIM_TEST_SEED=${MSIM_TEST_SEED} \ MSIM_TEST_NUM=1 \ MSIM_WATCHDOG_TIMEOUT_MS=${MSIM_WATCHDOG_TIMEOUT_MS} \ MSIM_TEST_CHECK_DETERMINISM=1 \ -CONSENSUS_PROTOCOL=${CONSENSUS_PROTOCOL} \ scripts/simtest/cargo-simtest simtest \ --color always \ --test-threads "$NUM_CPUS" \