From 20c6a4420fa14f3bfe3e98444ab8d701e2dee7b9 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Thu, 26 Feb 2026 12:53:32 -0300 Subject: [PATCH 1/9] feat(ntx-builder): blacklist accounts whose actors crash repeatedly --- CHANGELOG.md | 13 +++ bin/node/src/commands/mod.rs | 8 ++ crates/ntx-builder/src/coordinator.rs | 114 ++++++++++++++++++++- crates/ntx-builder/src/lib.rs | 21 +++- docs/external/src/operator/architecture.md | 4 + docs/internal/src/ntx-builder.md | 6 ++ 6 files changed, 162 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29415370f9..368ac6400f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,19 @@ - Fixed `bundled bootstrap` requiring `--validator.key.hex` or `--validator.key.kms-id` despite a default key being configured ([#1732](https://github.com/0xMiden/node/pull/1732)). - Fixed incorrectly classifying private notes with the network attachment as network notes ([#1378](https://github.com/0xMiden/node/pull/1738)). - Fixed accept header version negotiation rejecting all pre-release versions; pre-release label matching is now lenient, accepting any numeric suffix within the same label (e.g. `alpha.3` accepts `alpha.1`) ([#1755](https://github.com/0xMiden/node/pull/1755)). +- [BREAKING] Removed obsolete `SyncState` RPC endpoint; clients should use `SyncNotes`, `SyncNullifiers`, `SyncAccountVault`, `SyncAccountStorageMaps`, `SyncTransactions`, or `SyncChainMmr` instead ([#1636](https://github.com/0xMiden/miden-node/pull/1636)). +- Added account ID limits for `SyncTransactions`, `SyncAccountVault`, and `SyncAccountStorageMaps` to `GetLimits` responses ([#1636](https://github.com/0xMiden/miden-node/pull/1636)). +- [BREAKING] Added typed `GetAccountError` for `GetAccount` endpoint, splitting `BlockNotAvailable` into `UnknownBlock` and `BlockPruned`. `AccountNotFound` and `AccountNotPublic` now return `InvalidArgument` gRPC status instead of `NotFound`; clients should parse the error details discriminant rather than branching on status codes ([#1646](https://github.com/0xMiden/miden-node/pull/1646)). +- Changed `note_type` field in proto `NoteMetadata` from `uint32` to a `NoteType` enum ([#1594](https://github.com/0xMiden/miden-node/pull/1594)). +- Refactored NTX Builder startup and introduced `NtxBuilderConfig` with configurable parameters ([#1610](https://github.com/0xMiden/miden-node/pull/1610)). +- Refactored NTX Builder actor state into `AccountDeltaTracker` and `NotePool` for clarity, and added tracing instrumentation to event broadcasting ([#1611](https://github.com/0xMiden/miden-node/pull/1611)). +- Add #[track_caller] to tracing/logging helpers ([#1651](https://github.com/0xMiden/miden-node/pull/1651)). +- Added support for generic account loading at genesis ([#1624](https://github.com/0xMiden/miden-node/pull/1624)). +- Improved tracing span fields ([#1650](https://github.com/0xMiden/miden-node/pull/1650)) + - Replaced NTX Builder's in-memory state management with SQLite-backed persistence; account states, notes, and transaction effects are now stored in the database and inflight state is purged on startup ([#1662](https://github.com/0xMiden/miden-node/pull/1662)). +- [BREAKING] Reworked `miden-remote-prover`, removing the `worker`/`proxy` distinction and simplifying to a `worker` with a request queue ([#1688](https://github.com/0xMiden/miden-node/pull/1688)). +- NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/miden-node/pull/1705)). +- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10). ## v0.13.7 (2026-02-25) diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 25c0ddf235..3f4944611d 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -184,6 +184,13 @@ pub struct NtxBuilderConfig { )] pub idle_timeout: Duration, + /// Maximum number of crashes before an account actor is blacklisted. + /// + /// Once an actor for a given account exceeds this crash count, no new actor will be + /// spawned for that account. + #[arg(long = "ntx-builder.max-actor-crashes", default_value_t = 10, value_name = "NUM")] + pub max_actor_crashes: usize, + /// Directory for the ntx-builder's persistent database. /// /// If not set, defaults to the node's data directory. @@ -215,6 +222,7 @@ impl NtxBuilderConfig { .with_tx_prover_url(self.tx_prover_url) .with_script_cache_size(self.script_cache_size) .with_idle_timeout(self.idle_timeout) + .with_max_actor_crashes(self.max_actor_crashes) } } diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index 0188db74e1..6ba3584a5d 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -104,16 +104,29 @@ pub struct Coordinator { /// Database for persistent state. db: Db, + + /// Tracks the number of crashes per account actor. + /// + /// When an actor shuts down due to a DB error, its crash count is incremented. Once + /// the count reaches `max_actor_crashes`, the account is blacklisted and no new actor + /// will be spawned for it. + crash_counts: HashMap, + + /// Maximum number of crashes an account actor is allowed before being blacklisted. + max_actor_crashes: usize, } impl Coordinator { - /// Creates a new coordinator with the specified maximum number of inflight transactions. - pub fn new(max_inflight_transactions: usize, db: Db) -> Self { + /// Creates a new coordinator with the specified maximum number of inflight transactions + /// and the crash threshold for account blacklisting. + pub fn new(max_inflight_transactions: usize, max_actor_crashes: usize, db: Db) -> Self { Self { actor_registry: HashMap::new(), actor_join_set: JoinSet::new(), semaphore: Arc::new(Semaphore::new(max_inflight_transactions)), db, + crash_counts: HashMap::new(), + max_actor_crashes, } } @@ -126,6 +139,18 @@ impl Coordinator { pub fn spawn_actor(&mut self, origin: AccountOrigin, actor_context: &AccountActorContext) { let account_id = origin.id(); + // Skip spawning if the account has been blacklisted due to repeated crashes. + if let Some(&count) = self.crash_counts.get(&account_id) { + if count >= self.max_actor_crashes { + tracing::warn!( + %account_id, + crash_count = count, + "Account blacklisted due to repeated crashes, skipping actor spawn" + ); + return; + } + } + // If an actor already exists for this account ID, something has gone wrong. if let Some(handle) = self.actor_registry.remove(&account_id) { tracing::error!( @@ -187,6 +212,8 @@ impl Coordinator { }, ActorShutdownReason::SemaphoreFailed(err) => Err(err).context("semaphore failed"), ActorShutdownReason::DbError(account_id, err) => { + let count = self.crash_counts.entry(account_id).or_insert(0); + *count += 1; tracing::error!(account_id = %account_id, err = err.as_report(), "Account actor shut down due to DB error"); self.actor_registry.remove(&account_id); Ok(None) @@ -320,16 +347,54 @@ impl Coordinator { #[cfg(test)] mod tests { + use std::num::NonZeroUsize; + use std::sync::Arc; + use std::time::Duration; + use miden_node_proto::domain::mempool::MempoolEvent; + use miden_node_utils::lru_cache::LruCache; + use tokio::sync::{RwLock, mpsc}; + use url::Url; use super::*; + use crate::actor::{AccountActorContext, AccountOrigin}; + use crate::chain_state::ChainState; + use crate::clients::StoreClient; use crate::db::Db; use crate::test_utils::*; /// Creates a coordinator with default settings backed by a temp DB. async fn test_coordinator() -> (Coordinator, tempfile::TempDir) { let (db, dir) = Db::test_setup().await; - (Coordinator::new(4, db), dir) + (Coordinator::new(4, 10, db), dir) + } + + /// Creates a minimal `AccountActorContext` suitable for unit tests. + /// + /// The URLs are fake and actors spawned with this context will fail on their first gRPC call, + /// but this is sufficient for testing coordinator logic (registry, blacklisting, etc.). + fn test_actor_context(db: &Db) -> AccountActorContext { + use miden_protocol::crypto::merkle::mmr::{Forest, MmrPeaks, PartialMmr}; + + let url = Url::parse("http://127.0.0.1:1").unwrap(); + let block_header = mock_block_header(0_u32.into()); + let chain_mmr = PartialMmr::from_peaks(MmrPeaks::new(Forest::new(0), vec![]).unwrap()); + let chain_state = Arc::new(RwLock::new(ChainState::new(block_header, chain_mmr))); + let (request_tx, _request_rx) = mpsc::channel(1); + + AccountActorContext { + block_producer_url: url.clone(), + validator_url: url.clone(), + tx_prover_url: None, + chain_state, + store: StoreClient::new(url), + script_cache: LruCache::new(NonZeroUsize::new(1).unwrap()), + max_notes_per_tx: NonZeroUsize::new(1).unwrap(), + max_note_attempts: 1, + idle_timeout: Duration::from_secs(60), + db: db.clone(), + request_tx, + } } /// Registers a dummy actor handle (no real actor task) in the coordinator's registry. @@ -369,4 +434,47 @@ mod tests { assert_eq!(inactive_targets.len(), 1); assert_eq!(inactive_targets[0], inactive_id); } + + // BLACKLIST TESTS + // ============================================================================================ + + #[tokio::test] + async fn spawn_actor_skips_blacklisted_account() { + let (db, _dir) = Db::test_setup().await; + let max_crashes = 3; + let mut coordinator = Coordinator::new(4, max_crashes, db.clone()); + let actor_context = test_actor_context(&db); + + let account_id = mock_network_account_id(); + + // Simulate the account having reached the crash threshold. + coordinator.crash_counts.insert(account_id, max_crashes); + + coordinator.spawn_actor(AccountOrigin::Store(account_id), &actor_context); + + assert!( + !coordinator.actor_registry.contains_key(&account_id), + "Blacklisted account should not have an actor in the registry" + ); + } + + #[tokio::test] + async fn spawn_actor_allows_below_threshold() { + let (db, _dir) = Db::test_setup().await; + let max_crashes = 3; + let mut coordinator = Coordinator::new(4, max_crashes, db.clone()); + let actor_context = test_actor_context(&db); + + let account_id = mock_network_account_id(); + + // Set crash count below the threshold. + coordinator.crash_counts.insert(account_id, max_crashes - 1); + + coordinator.spawn_actor(AccountOrigin::Store(account_id), &actor_context); + + assert!( + coordinator.actor_registry.contains_key(&account_id), + "Account below crash threshold should have an actor in the registry" + ); + } } diff --git a/crates/ntx-builder/src/lib.rs b/crates/ntx-builder/src/lib.rs index fb63bc5be5..e729f439f4 100644 --- a/crates/ntx-builder/src/lib.rs +++ b/crates/ntx-builder/src/lib.rs @@ -59,6 +59,9 @@ const DEFAULT_SCRIPT_CACHE_SIZE: NonZeroUsize = /// Default duration after which an idle network account actor will deactivate. const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60); +/// Default maximum number of crashes an account actor is allowed before being blacklisted. +const DEFAULT_MAX_ACTOR_CRASHES: usize = 10; + // CONFIGURATION // ================================================================================================= @@ -106,6 +109,13 @@ pub struct NtxBuilderConfig { /// A deactivated account will reactivate if targeted with new notes. pub idle_timeout: Duration, + /// Maximum number of crashes an account actor is allowed before being blacklisted. + /// + /// Once an actor for a given account exceeds this crash count, no new actor will be + /// spawned for that account. This prevents resource exhaustion from repeatedly failing + /// actors. + pub max_actor_crashes: usize, + /// Path to the SQLite database file used for persistent state. pub database_filepath: PathBuf, } @@ -129,6 +139,7 @@ impl NtxBuilderConfig { max_block_count: DEFAULT_MAX_BLOCK_COUNT, account_channel_capacity: DEFAULT_ACCOUNT_CHANNEL_CAPACITY, idle_timeout: DEFAULT_IDLE_TIMEOUT, + max_actor_crashes: DEFAULT_MAX_ACTOR_CRASHES, database_filepath, } } @@ -203,6 +214,13 @@ impl NtxBuilderConfig { self } + /// Sets the maximum number of crashes before an account actor is blacklisted. + #[must_use] + pub fn with_max_actor_crashes(mut self, max: usize) -> Self { + self.max_actor_crashes = max; + self + } + /// Builds and initializes the network transaction builder. /// /// This method connects to the store and block producer services, fetches the current @@ -222,7 +240,8 @@ impl NtxBuilderConfig { db.purge_inflight().await.context("failed to purge inflight state")?; let script_cache = LruCache::new(self.script_cache_size); - let coordinator = Coordinator::new(self.max_concurrent_txs, db.clone()); + let coordinator = + Coordinator::new(self.max_concurrent_txs, self.max_actor_crashes, db.clone()); let store = StoreClient::new(self.store_url.clone()); let block_producer = BlockProducerClient::new(self.block_producer_url.clone()); diff --git a/docs/external/src/operator/architecture.md b/docs/external/src/operator/architecture.md index 6745077526..f5c96f25aa 100644 --- a/docs/external/src/operator/architecture.md +++ b/docs/external/src/operator/architecture.md @@ -58,3 +58,7 @@ Internally, the builder spawns a dedicated actor for each network account that h idle (no notes to consume) for a configurable duration are automatically deactivated to conserve resources, and are re-activated when new notes arrive. The idle timeout can be tuned with the `--ntx-builder.idle-timeout` CLI argument (default: 5 minutes). + +Accounts whose actors crash repeatedly (due to database errors) are automatically blacklisted after a configurable +number of failures, preventing resource exhaustion. The threshold can be set with +`--ntx-builder.max-actor-crashes` (default: 10). diff --git a/docs/internal/src/ntx-builder.md b/docs/internal/src/ntx-builder.md index a662f76584..5b21a028f9 100644 --- a/docs/internal/src/ntx-builder.md +++ b/docs/internal/src/ntx-builder.md @@ -51,5 +51,11 @@ argument (default: 5 minutes). Deactivated actors are re-spawned when new notes targeting their account are detected by the coordinator (via the `send_targeted` path). +If an actor repeatedly crashes (shuts down due to a database error), its crash count is tracked by +the coordinator. Once the count reaches the configurable threshold, the account is **blacklisted** +and no new actor will be spawned for it. This prevents resource exhaustion from a persistently +failing account. The threshold is configurable via the `--ntx-builder.max-actor-crashes` CLI +argument (default: 10). + The block-producer remains blissfully unaware of network transactions. From its perspective a network transaction is simply the same as any other. From c33bbbd127af114ed1f0dd0619fde1381eb366c4 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Thu, 26 Feb 2026 13:58:52 -0300 Subject: [PATCH 2/9] add PR number to changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 368ac6400f..d5c97caa47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,7 @@ - Replaced NTX Builder's in-memory state management with SQLite-backed persistence; account states, notes, and transaction effects are now stored in the database and inflight state is purged on startup ([#1662](https://github.com/0xMiden/miden-node/pull/1662)). - [BREAKING] Reworked `miden-remote-prover`, removing the `worker`/`proxy` distinction and simplifying to a `worker` with a request queue ([#1688](https://github.com/0xMiden/miden-node/pull/1688)). - NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/miden-node/pull/1705)). -- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10). +- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). ## v0.13.7 (2026-02-25) From 312fea25694f3fffea67881f05ad1c10d6e8a826 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 15:33:02 -0300 Subject: [PATCH 3/9] docs: add changelog entry & remove duplicated ones --- CHANGELOG.md | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5c97caa47..365ddaec48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ - NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/node/pull/1705)). - [BREAKING] Modified `TransactionHeader` serialization to allow converting back into the native type after serialization ([#1759](https://github.com/0xMiden/node/issues/1759)). - Removed `chain_tip` requirement from mempool subscription request ([#1771](https://github.com/0xMiden/node/pull/1771)). +- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). + ### Fixes @@ -39,19 +41,6 @@ - Fixed `bundled bootstrap` requiring `--validator.key.hex` or `--validator.key.kms-id` despite a default key being configured ([#1732](https://github.com/0xMiden/node/pull/1732)). - Fixed incorrectly classifying private notes with the network attachment as network notes ([#1378](https://github.com/0xMiden/node/pull/1738)). - Fixed accept header version negotiation rejecting all pre-release versions; pre-release label matching is now lenient, accepting any numeric suffix within the same label (e.g. `alpha.3` accepts `alpha.1`) ([#1755](https://github.com/0xMiden/node/pull/1755)). -- [BREAKING] Removed obsolete `SyncState` RPC endpoint; clients should use `SyncNotes`, `SyncNullifiers`, `SyncAccountVault`, `SyncAccountStorageMaps`, `SyncTransactions`, or `SyncChainMmr` instead ([#1636](https://github.com/0xMiden/miden-node/pull/1636)). -- Added account ID limits for `SyncTransactions`, `SyncAccountVault`, and `SyncAccountStorageMaps` to `GetLimits` responses ([#1636](https://github.com/0xMiden/miden-node/pull/1636)). -- [BREAKING] Added typed `GetAccountError` for `GetAccount` endpoint, splitting `BlockNotAvailable` into `UnknownBlock` and `BlockPruned`. `AccountNotFound` and `AccountNotPublic` now return `InvalidArgument` gRPC status instead of `NotFound`; clients should parse the error details discriminant rather than branching on status codes ([#1646](https://github.com/0xMiden/miden-node/pull/1646)). -- Changed `note_type` field in proto `NoteMetadata` from `uint32` to a `NoteType` enum ([#1594](https://github.com/0xMiden/miden-node/pull/1594)). -- Refactored NTX Builder startup and introduced `NtxBuilderConfig` with configurable parameters ([#1610](https://github.com/0xMiden/miden-node/pull/1610)). -- Refactored NTX Builder actor state into `AccountDeltaTracker` and `NotePool` for clarity, and added tracing instrumentation to event broadcasting ([#1611](https://github.com/0xMiden/miden-node/pull/1611)). -- Add #[track_caller] to tracing/logging helpers ([#1651](https://github.com/0xMiden/miden-node/pull/1651)). -- Added support for generic account loading at genesis ([#1624](https://github.com/0xMiden/miden-node/pull/1624)). -- Improved tracing span fields ([#1650](https://github.com/0xMiden/miden-node/pull/1650)) - - Replaced NTX Builder's in-memory state management with SQLite-backed persistence; account states, notes, and transaction effects are now stored in the database and inflight state is purged on startup ([#1662](https://github.com/0xMiden/miden-node/pull/1662)). -- [BREAKING] Reworked `miden-remote-prover`, removing the `worker`/`proxy` distinction and simplifying to a `worker` with a request queue ([#1688](https://github.com/0xMiden/miden-node/pull/1688)). -- NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/miden-node/pull/1705)). -- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). ## v0.13.7 (2026-02-25) From 47c01236bbbf5c00dfc7ba720fdfb6a4afddfb69 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 15:51:52 -0300 Subject: [PATCH 4/9] review: replace blacklist with deactivated --- CHANGELOG.md | 2 +- bin/node/src/commands/mod.rs | 2 +- crates/ntx-builder/src/coordinator.rs | 18 +++++++++--------- crates/ntx-builder/src/lib.rs | 6 +++--- docs/external/src/operator/architecture.md | 2 +- docs/internal/src/ntx-builder.md | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 365ddaec48..46b716ca4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ - NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/node/pull/1705)). - [BREAKING] Modified `TransactionHeader` serialization to allow converting back into the native type after serialization ([#1759](https://github.com/0xMiden/node/issues/1759)). - Removed `chain_tip` requirement from mempool subscription request ([#1771](https://github.com/0xMiden/node/pull/1771)). -- NTX Builder now blacklists network accounts whose actors crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). +- NTX Builder now deactivates network accounts which crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). ### Fixes diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 3f4944611d..6707f5475b 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -184,7 +184,7 @@ pub struct NtxBuilderConfig { )] pub idle_timeout: Duration, - /// Maximum number of crashes before an account actor is blacklisted. + /// Maximum number of crashes before an account actor is deactivated. /// /// Once an actor for a given account exceeds this crash count, no new actor will be /// spawned for that account. diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index 6ba3584a5d..8e025a61a4 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -108,17 +108,17 @@ pub struct Coordinator { /// Tracks the number of crashes per account actor. /// /// When an actor shuts down due to a DB error, its crash count is incremented. Once - /// the count reaches `max_actor_crashes`, the account is blacklisted and no new actor + /// the count reaches `max_actor_crashes`, the account is deactivated and no new actor /// will be spawned for it. crash_counts: HashMap, - /// Maximum number of crashes an account actor is allowed before being blacklisted. + /// Maximum number of crashes an account actor is allowed before being deactivated. max_actor_crashes: usize, } impl Coordinator { /// Creates a new coordinator with the specified maximum number of inflight transactions - /// and the crash threshold for account blacklisting. + /// and the crash threshold for account deactivation. pub fn new(max_inflight_transactions: usize, max_actor_crashes: usize, db: Db) -> Self { Self { actor_registry: HashMap::new(), @@ -139,13 +139,13 @@ impl Coordinator { pub fn spawn_actor(&mut self, origin: AccountOrigin, actor_context: &AccountActorContext) { let account_id = origin.id(); - // Skip spawning if the account has been blacklisted due to repeated crashes. + // Skip spawning if the account has been deactivated due to repeated crashes. if let Some(&count) = self.crash_counts.get(&account_id) { if count >= self.max_actor_crashes { tracing::warn!( %account_id, crash_count = count, - "Account blacklisted due to repeated crashes, skipping actor spawn" + "Account deactivated due to repeated crashes, skipping actor spawn" ); return; } @@ -372,7 +372,7 @@ mod tests { /// Creates a minimal `AccountActorContext` suitable for unit tests. /// /// The URLs are fake and actors spawned with this context will fail on their first gRPC call, - /// but this is sufficient for testing coordinator logic (registry, blacklisting, etc.). + /// but this is sufficient for testing coordinator logic (registry, deactivation, etc.). fn test_actor_context(db: &Db) -> AccountActorContext { use miden_protocol::crypto::merkle::mmr::{Forest, MmrPeaks, PartialMmr}; @@ -435,11 +435,11 @@ mod tests { assert_eq!(inactive_targets[0], inactive_id); } - // BLACKLIST TESTS + // DEACTIVATED ACCOUNTS // ============================================================================================ #[tokio::test] - async fn spawn_actor_skips_blacklisted_account() { + async fn spawn_actor_skips_deactivated_account() { let (db, _dir) = Db::test_setup().await; let max_crashes = 3; let mut coordinator = Coordinator::new(4, max_crashes, db.clone()); @@ -454,7 +454,7 @@ mod tests { assert!( !coordinator.actor_registry.contains_key(&account_id), - "Blacklisted account should not have an actor in the registry" + "Deactivated account should not have an actor in the registry" ); } diff --git a/crates/ntx-builder/src/lib.rs b/crates/ntx-builder/src/lib.rs index e729f439f4..0ed08a0fd8 100644 --- a/crates/ntx-builder/src/lib.rs +++ b/crates/ntx-builder/src/lib.rs @@ -59,7 +59,7 @@ const DEFAULT_SCRIPT_CACHE_SIZE: NonZeroUsize = /// Default duration after which an idle network account actor will deactivate. const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60); -/// Default maximum number of crashes an account actor is allowed before being blacklisted. +/// Default maximum number of crashes an account actor is allowed before being deactivated. const DEFAULT_MAX_ACTOR_CRASHES: usize = 10; // CONFIGURATION @@ -109,7 +109,7 @@ pub struct NtxBuilderConfig { /// A deactivated account will reactivate if targeted with new notes. pub idle_timeout: Duration, - /// Maximum number of crashes an account actor is allowed before being blacklisted. + /// Maximum number of crashes an account actor is allowed before being deactivated. /// /// Once an actor for a given account exceeds this crash count, no new actor will be /// spawned for that account. This prevents resource exhaustion from repeatedly failing @@ -214,7 +214,7 @@ impl NtxBuilderConfig { self } - /// Sets the maximum number of crashes before an account actor is blacklisted. + /// Sets the maximum number of crashes before an account actor is deactivated. #[must_use] pub fn with_max_actor_crashes(mut self, max: usize) -> Self { self.max_actor_crashes = max; diff --git a/docs/external/src/operator/architecture.md b/docs/external/src/operator/architecture.md index f5c96f25aa..3fe70d0bdb 100644 --- a/docs/external/src/operator/architecture.md +++ b/docs/external/src/operator/architecture.md @@ -59,6 +59,6 @@ idle (no notes to consume) for a configurable duration are automatically deactiv re-activated when new notes arrive. The idle timeout can be tuned with the `--ntx-builder.idle-timeout` CLI argument (default: 5 minutes). -Accounts whose actors crash repeatedly (due to database errors) are automatically blacklisted after a configurable +Accounts whose actors crash repeatedly (due to database errors) are automatically deactivated after a configurable number of failures, preventing resource exhaustion. The threshold can be set with `--ntx-builder.max-actor-crashes` (default: 10). diff --git a/docs/internal/src/ntx-builder.md b/docs/internal/src/ntx-builder.md index 5b21a028f9..c56ab3bd1b 100644 --- a/docs/internal/src/ntx-builder.md +++ b/docs/internal/src/ntx-builder.md @@ -52,7 +52,7 @@ Deactivated actors are re-spawned when new notes targeting their account are det coordinator (via the `send_targeted` path). If an actor repeatedly crashes (shuts down due to a database error), its crash count is tracked by -the coordinator. Once the count reaches the configurable threshold, the account is **blacklisted** +the coordinator. Once the count reaches the configurable threshold, the account is **deactivated** and no new actor will be spawned for it. This prevents resource exhaustion from a persistently failing account. The threshold is configurable via the `--ntx-builder.max-actor-crashes` CLI argument (default: 10). From 2dddcb26682dfb24c491a0de44ee6230419cf53c Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 15:54:29 -0300 Subject: [PATCH 5/9] review: update docs --- bin/node/src/commands/mod.rs | 5 ++--- crates/ntx-builder/src/lib.rs | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 6707f5475b..0c8fb7e1bb 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -184,10 +184,9 @@ pub struct NtxBuilderConfig { )] pub idle_timeout: Duration, - /// Maximum number of crashes before an account actor is deactivated. + /// Maximum number of crashes before an account deactivated. /// - /// Once an actor for a given account exceeds this crash count, no new actor will be - /// spawned for that account. + /// Once this limit is reached, no new transactions will be created for this account. #[arg(long = "ntx-builder.max-actor-crashes", default_value_t = 10, value_name = "NUM")] pub max_actor_crashes: usize, diff --git a/crates/ntx-builder/src/lib.rs b/crates/ntx-builder/src/lib.rs index 0ed08a0fd8..62aa18d66d 100644 --- a/crates/ntx-builder/src/lib.rs +++ b/crates/ntx-builder/src/lib.rs @@ -109,11 +109,9 @@ pub struct NtxBuilderConfig { /// A deactivated account will reactivate if targeted with new notes. pub idle_timeout: Duration, - /// Maximum number of crashes an account actor is allowed before being deactivated. + /// Maximum number of crashes before an account deactivated. /// - /// Once an actor for a given account exceeds this crash count, no new actor will be - /// spawned for that account. This prevents resource exhaustion from repeatedly failing - /// actors. + /// Once this limit is reached, no new transactions will be created for this account. pub max_actor_crashes: usize, /// Path to the SQLite database file used for persistent state. From 26f130a14e6004f64338158a90b01031acc15901 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 15:58:26 -0300 Subject: [PATCH 6/9] review: rename actors with accounts --- CHANGELOG.md | 2 +- bin/node/src/commands/mod.rs | 10 +++++++--- crates/ntx-builder/src/coordinator.rs | 10 +++++----- crates/ntx-builder/src/lib.rs | 12 ++++++------ docs/external/src/operator/architecture.md | 2 +- docs/internal/src/ntx-builder.md | 2 +- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46b716ca4c..418eb7527f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ - NTX Builder actors now deactivate after being idle for a configurable idle timeout (`--ntx-builder.idle-timeout`, default 5 min) and are re-activated when new notes target their account ([#1705](https://github.com/0xMiden/node/pull/1705)). - [BREAKING] Modified `TransactionHeader` serialization to allow converting back into the native type after serialization ([#1759](https://github.com/0xMiden/node/issues/1759)). - Removed `chain_tip` requirement from mempool subscription request ([#1771](https://github.com/0xMiden/node/pull/1771)). -- NTX Builder now deactivates network accounts which crash repeatedly (configurable via `--ntx-builder.max-actor-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). +- NTX Builder now deactivates network accounts which crash repeatedly (configurable via `--ntx-builder.max-account-crashes`, default 10) ([#1712](https://github.com/0xMiden/miden-node/pull/1712)). ### Fixes diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 0c8fb7e1bb..a1b6f8be52 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -187,8 +187,12 @@ pub struct NtxBuilderConfig { /// Maximum number of crashes before an account deactivated. /// /// Once this limit is reached, no new transactions will be created for this account. - #[arg(long = "ntx-builder.max-actor-crashes", default_value_t = 10, value_name = "NUM")] - pub max_actor_crashes: usize, + #[arg( + long = "ntx-builder.max-account-crashes", + default_value_t = 10, + value_name = "NUM" + )] + pub max_account_crashes: usize, /// Directory for the ntx-builder's persistent database. /// @@ -221,7 +225,7 @@ impl NtxBuilderConfig { .with_tx_prover_url(self.tx_prover_url) .with_script_cache_size(self.script_cache_size) .with_idle_timeout(self.idle_timeout) - .with_max_actor_crashes(self.max_actor_crashes) + .with_max_account_crashes(self.max_account_crashes) } } diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index 8e025a61a4..2c4eb8c912 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -108,25 +108,25 @@ pub struct Coordinator { /// Tracks the number of crashes per account actor. /// /// When an actor shuts down due to a DB error, its crash count is incremented. Once - /// the count reaches `max_actor_crashes`, the account is deactivated and no new actor + /// the count reaches `max_account_crashes`, the account is deactivated and no new actor /// will be spawned for it. crash_counts: HashMap, /// Maximum number of crashes an account actor is allowed before being deactivated. - max_actor_crashes: usize, + max_account_crashes: usize, } impl Coordinator { /// Creates a new coordinator with the specified maximum number of inflight transactions /// and the crash threshold for account deactivation. - pub fn new(max_inflight_transactions: usize, max_actor_crashes: usize, db: Db) -> Self { + pub fn new(max_inflight_transactions: usize, max_account_crashes: usize, db: Db) -> Self { Self { actor_registry: HashMap::new(), actor_join_set: JoinSet::new(), semaphore: Arc::new(Semaphore::new(max_inflight_transactions)), db, crash_counts: HashMap::new(), - max_actor_crashes, + max_account_crashes, } } @@ -141,7 +141,7 @@ impl Coordinator { // Skip spawning if the account has been deactivated due to repeated crashes. if let Some(&count) = self.crash_counts.get(&account_id) { - if count >= self.max_actor_crashes { + if count >= self.max_account_crashes { tracing::warn!( %account_id, crash_count = count, diff --git a/crates/ntx-builder/src/lib.rs b/crates/ntx-builder/src/lib.rs index 62aa18d66d..fc3ab0ae9b 100644 --- a/crates/ntx-builder/src/lib.rs +++ b/crates/ntx-builder/src/lib.rs @@ -60,7 +60,7 @@ const DEFAULT_SCRIPT_CACHE_SIZE: NonZeroUsize = const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60); /// Default maximum number of crashes an account actor is allowed before being deactivated. -const DEFAULT_MAX_ACTOR_CRASHES: usize = 10; +const DEFAULT_MAX_ACCOUNT_CRASHES: usize = 10; // CONFIGURATION // ================================================================================================= @@ -112,7 +112,7 @@ pub struct NtxBuilderConfig { /// Maximum number of crashes before an account deactivated. /// /// Once this limit is reached, no new transactions will be created for this account. - pub max_actor_crashes: usize, + pub max_account_crashes: usize, /// Path to the SQLite database file used for persistent state. pub database_filepath: PathBuf, @@ -137,7 +137,7 @@ impl NtxBuilderConfig { max_block_count: DEFAULT_MAX_BLOCK_COUNT, account_channel_capacity: DEFAULT_ACCOUNT_CHANNEL_CAPACITY, idle_timeout: DEFAULT_IDLE_TIMEOUT, - max_actor_crashes: DEFAULT_MAX_ACTOR_CRASHES, + max_account_crashes: DEFAULT_MAX_ACCOUNT_CRASHES, database_filepath, } } @@ -214,8 +214,8 @@ impl NtxBuilderConfig { /// Sets the maximum number of crashes before an account actor is deactivated. #[must_use] - pub fn with_max_actor_crashes(mut self, max: usize) -> Self { - self.max_actor_crashes = max; + pub fn with_max_account_crashes(mut self, max: usize) -> Self { + self.max_account_crashes = max; self } @@ -239,7 +239,7 @@ impl NtxBuilderConfig { let script_cache = LruCache::new(self.script_cache_size); let coordinator = - Coordinator::new(self.max_concurrent_txs, self.max_actor_crashes, db.clone()); + Coordinator::new(self.max_concurrent_txs, self.max_account_crashes, db.clone()); let store = StoreClient::new(self.store_url.clone()); let block_producer = BlockProducerClient::new(self.block_producer_url.clone()); diff --git a/docs/external/src/operator/architecture.md b/docs/external/src/operator/architecture.md index 3fe70d0bdb..694ae66e75 100644 --- a/docs/external/src/operator/architecture.md +++ b/docs/external/src/operator/architecture.md @@ -61,4 +61,4 @@ argument (default: 5 minutes). Accounts whose actors crash repeatedly (due to database errors) are automatically deactivated after a configurable number of failures, preventing resource exhaustion. The threshold can be set with -`--ntx-builder.max-actor-crashes` (default: 10). +`--ntx-builder.max-account-crashes` (default: 10). diff --git a/docs/internal/src/ntx-builder.md b/docs/internal/src/ntx-builder.md index c56ab3bd1b..f15feb5446 100644 --- a/docs/internal/src/ntx-builder.md +++ b/docs/internal/src/ntx-builder.md @@ -54,7 +54,7 @@ coordinator (via the `send_targeted` path). If an actor repeatedly crashes (shuts down due to a database error), its crash count is tracked by the coordinator. Once the count reaches the configurable threshold, the account is **deactivated** and no new actor will be spawned for it. This prevents resource exhaustion from a persistently -failing account. The threshold is configurable via the `--ntx-builder.max-actor-crashes` CLI +failing account. The threshold is configurable via the `--ntx-builder.max-account-crashes` CLI argument (default: 10). The block-producer remains blissfully unaware of network transactions. From its perspective a From d1f2043dd953f98a40a3dd5461a413a07d84fe69 Mon Sep 17 00:00:00 2001 From: Santiago Pittella <87827390+SantiagoPittella@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:59:52 -0300 Subject: [PATCH 7/9] review: improve log format Co-authored-by: Mirko <48352201+Mirko-von-Leipzig@users.noreply.github.com> --- crates/ntx-builder/src/coordinator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index 2c4eb8c912..fb1c1f3c2a 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -143,7 +143,7 @@ impl Coordinator { if let Some(&count) = self.crash_counts.get(&account_id) { if count >= self.max_account_crashes { tracing::warn!( - %account_id, + account.id = %account_id, crash_count = count, "Account deactivated due to repeated crashes, skipping actor spawn" ); From 72934f23807bf921c73961085bb04e00aec06df6 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 16:01:49 -0300 Subject: [PATCH 8/9] review: improve traces --- crates/ntx-builder/src/coordinator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index fb1c1f3c2a..c9dee7eb38 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -214,7 +214,7 @@ impl Coordinator { ActorShutdownReason::DbError(account_id, err) => { let count = self.crash_counts.entry(account_id).or_insert(0); *count += 1; - tracing::error!(account_id = %account_id, err = err.as_report(), "Account actor shut down due to DB error"); + tracing::error!(account.id = %account_id, err = err.as_report(), "Account actor shut down due to DB error"); self.actor_registry.remove(&account_id); Ok(None) }, From d0c58c22132430dcd30b4579a71f267c6021b891 Mon Sep 17 00:00:00 2001 From: SantiagoPittella Date: Wed, 11 Mar 2026 16:09:53 -0300 Subject: [PATCH 9/9] review: move tests helpers as methods --- crates/ntx-builder/src/actor/mod.rs | 37 ++++++++++++++++++ crates/ntx-builder/src/coordinator.rs | 56 ++++++--------------------- 2 files changed, 49 insertions(+), 44 deletions(-) diff --git a/crates/ntx-builder/src/actor/mod.rs b/crates/ntx-builder/src/actor/mod.rs index 46f090f3c8..9d7ae3ef9e 100644 --- a/crates/ntx-builder/src/actor/mod.rs +++ b/crates/ntx-builder/src/actor/mod.rs @@ -98,6 +98,43 @@ pub struct AccountActorContext { pub request_tx: mpsc::Sender, } +#[cfg(test)] +impl AccountActorContext { + /// Creates a minimal `AccountActorContext` suitable for unit tests. + /// + /// The URLs are fake and actors spawned with this context will fail on their first gRPC call, + /// but this is sufficient for testing coordinator logic (registry, deactivation, etc.). + pub fn test(db: &crate::db::Db) -> Self { + use miden_protocol::crypto::merkle::mmr::{Forest, MmrPeaks, PartialMmr}; + use tokio::sync::RwLock; + use url::Url; + + use crate::chain_state::ChainState; + use crate::clients::StoreClient; + use crate::test_utils::mock_block_header; + + let url = Url::parse("http://127.0.0.1:1").unwrap(); + let block_header = mock_block_header(0_u32.into()); + let chain_mmr = PartialMmr::from_peaks(MmrPeaks::new(Forest::new(0), vec![]).unwrap()); + let chain_state = Arc::new(RwLock::new(ChainState::new(block_header, chain_mmr))); + let (request_tx, _request_rx) = mpsc::channel(1); + + Self { + block_producer_url: url.clone(), + validator_url: url.clone(), + tx_prover_url: None, + chain_state, + store: StoreClient::new(url), + script_cache: LruCache::new(NonZeroUsize::new(1).unwrap()), + max_notes_per_tx: NonZeroUsize::new(1).unwrap(), + max_note_attempts: 1, + idle_timeout: Duration::from_secs(60), + db: db.clone(), + request_tx, + } + } +} + // ACCOUNT ORIGIN // ================================================================================================ diff --git a/crates/ntx-builder/src/coordinator.rs b/crates/ntx-builder/src/coordinator.rs index c9dee7eb38..e8b930c7e8 100644 --- a/crates/ntx-builder/src/coordinator.rs +++ b/crates/ntx-builder/src/coordinator.rs @@ -345,58 +345,26 @@ impl Coordinator { } } +#[cfg(test)] +impl Coordinator { + /// Creates a coordinator with default settings backed by a temp DB. + pub async fn test() -> (Self, tempfile::TempDir) { + let (db, dir) = Db::test_setup().await; + (Self::new(4, 10, db), dir) + } +} + #[cfg(test)] mod tests { - use std::num::NonZeroUsize; use std::sync::Arc; - use std::time::Duration; use miden_node_proto::domain::mempool::MempoolEvent; - use miden_node_utils::lru_cache::LruCache; - use tokio::sync::{RwLock, mpsc}; - use url::Url; use super::*; use crate::actor::{AccountActorContext, AccountOrigin}; - use crate::chain_state::ChainState; - use crate::clients::StoreClient; use crate::db::Db; use crate::test_utils::*; - /// Creates a coordinator with default settings backed by a temp DB. - async fn test_coordinator() -> (Coordinator, tempfile::TempDir) { - let (db, dir) = Db::test_setup().await; - (Coordinator::new(4, 10, db), dir) - } - - /// Creates a minimal `AccountActorContext` suitable for unit tests. - /// - /// The URLs are fake and actors spawned with this context will fail on their first gRPC call, - /// but this is sufficient for testing coordinator logic (registry, deactivation, etc.). - fn test_actor_context(db: &Db) -> AccountActorContext { - use miden_protocol::crypto::merkle::mmr::{Forest, MmrPeaks, PartialMmr}; - - let url = Url::parse("http://127.0.0.1:1").unwrap(); - let block_header = mock_block_header(0_u32.into()); - let chain_mmr = PartialMmr::from_peaks(MmrPeaks::new(Forest::new(0), vec![]).unwrap()); - let chain_state = Arc::new(RwLock::new(ChainState::new(block_header, chain_mmr))); - let (request_tx, _request_rx) = mpsc::channel(1); - - AccountActorContext { - block_producer_url: url.clone(), - validator_url: url.clone(), - tx_prover_url: None, - chain_state, - store: StoreClient::new(url), - script_cache: LruCache::new(NonZeroUsize::new(1).unwrap()), - max_notes_per_tx: NonZeroUsize::new(1).unwrap(), - max_note_attempts: 1, - idle_timeout: Duration::from_secs(60), - db: db.clone(), - request_tx, - } - } - /// Registers a dummy actor handle (no real actor task) in the coordinator's registry. fn register_dummy_actor(coordinator: &mut Coordinator, account_id: NetworkAccountId) { let notify = Arc::new(Notify::new()); @@ -411,7 +379,7 @@ mod tests { #[tokio::test] async fn send_targeted_returns_inactive_targets() { - let (mut coordinator, _dir) = test_coordinator().await; + let (mut coordinator, _dir) = Coordinator::test().await; let active_id = mock_network_account_id(); let inactive_id = mock_network_account_id_seeded(42); @@ -443,7 +411,7 @@ mod tests { let (db, _dir) = Db::test_setup().await; let max_crashes = 3; let mut coordinator = Coordinator::new(4, max_crashes, db.clone()); - let actor_context = test_actor_context(&db); + let actor_context = AccountActorContext::test(&db); let account_id = mock_network_account_id(); @@ -463,7 +431,7 @@ mod tests { let (db, _dir) = Db::test_setup().await; let max_crashes = 3; let mut coordinator = Coordinator::new(4, max_crashes, db.clone()); - let actor_context = test_actor_context(&db); + let actor_context = AccountActorContext::test(&db); let account_id = mock_network_account_id();